001 /*
002 * Cumulus4j - Securing your data in the cloud - http://cumulus4j.org
003 * Copyright (C) 2011 NightLabs Consulting GmbH
004 *
005 * This program is free software: you can redistribute it and/or modify
006 * it under the terms of the GNU Affero General Public License as
007 * published by the Free Software Foundation, either version 3 of the
008 * License, or (at your option) any later version.
009 *
010 * This program is distributed in the hope that it will be useful,
011 * but WITHOUT ANY WARRANTY; without even the implied warranty of
012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
013 * GNU Affero General Public License for more details.
014 *
015 * You should have received a copy of the GNU Affero General Public License
016 * along with this program. If not, see <http://www.gnu.org/licenses/>.
017 */
018 package org.cumulus4j.store;
019
020 import java.util.ArrayList;
021 import java.util.Collection;
022 import java.util.Collections;
023 import java.util.HashMap;
024 import java.util.HashSet;
025 import java.util.Map;
026 import java.util.Properties;
027 import java.util.Set;
028 import java.util.WeakHashMap;
029
030 import javax.jdo.FetchPlan;
031 import javax.jdo.PersistenceManager;
032
033 import org.cumulus4j.store.model.ClassMeta;
034 import org.cumulus4j.store.model.ClassMetaDAO;
035 import org.cumulus4j.store.model.DataEntry;
036 import org.cumulus4j.store.model.FieldMeta;
037 import org.cumulus4j.store.model.FieldMetaRole;
038 import org.cumulus4j.store.model.IndexEntryFactoryRegistry;
039 import org.datanucleus.ClassLoaderResolver;
040 import org.datanucleus.NucleusContext;
041 import org.datanucleus.api.jdo.JDOPersistenceManagerFactory;
042 import org.datanucleus.identity.OID;
043 import org.datanucleus.identity.SCOID;
044 import org.datanucleus.metadata.AbstractClassMetaData;
045 import org.datanucleus.metadata.AbstractMemberMetaData;
046 import org.datanucleus.store.AbstractStoreManager;
047 import org.datanucleus.store.ExecutionContext;
048 import org.datanucleus.store.connection.ManagedConnection;
049 import org.datanucleus.store.schema.SchemaAwareStoreManager;
050 import org.slf4j.Logger;
051 import org.slf4j.LoggerFactory;
052
053 /**
054 * Store Manager for Cumulus4J operation.
055 * This StoreManager handles a backend StoreManager for the persistence to the chosen datastore, and optionally
056 * a second backend StoreManager for the persistence of index data to the chosen index datastore.
057 * The user will persist objects of their own classes, and these will be translated into the persistence of
058 * DataEntry, ClassMeta, FieldMeta for the data, as well as various IndexXXX types.
059 *
060 * @author Marco หงุ่ยตระกูล-Schulze - marco at nightlabs dot de
061 */
062 public class Cumulus4jStoreManager extends AbstractStoreManager implements SchemaAwareStoreManager
063 {
064 private static final Logger logger = LoggerFactory.getLogger(Cumulus4jStoreManager.class);
065
066 /** Extension key for marking field as not queryable */
067 public static final String CUMULUS4J_QUERYABLE = "cumulus4j-queryable";
068
069 // private static final SequenceMetaData SEQUENCE_META_DATA_DATA_ENTRY;
070 // static {
071 // SEQUENCE_META_DATA_DATA_ENTRY = new SequenceMetaData(DataEntry.class.getName(), SequenceStrategy.NONTRANSACTIONAL.toString());
072 // SEQUENCE_META_DATA_DATA_ENTRY.setAllocationSize(100);
073 // SEQUENCE_META_DATA_DATA_ENTRY.setDatastoreSequence(DataEntry.class.getName());
074 // }
075
076 private Map<Class<?>, ClassMeta> class2classMeta = Collections.synchronizedMap(new HashMap<Class<?>, ClassMeta>());
077
078 /**
079 * For every class, we keep a set of all known sub-classes (all inheritance-levels down). Note, that the class in
080 * the map-key is contained in the Set (in the map-value).
081 */
082 private Map<Class<?>, Set<Class<?>>> class2subclasses = Collections.synchronizedMap(new HashMap<Class<?>, Set<Class<?>>>());
083
084 private EncryptionHandler encryptionHandler;
085 private EncryptionCoordinateSetManager encryptionCoordinateSetManager;
086
087 private IndexEntryFactoryRegistry indexFactoryRegistry;
088
089 public Cumulus4jStoreManager(ClassLoaderResolver clr, NucleusContext nucleusContext, Map<String, Object> props)
090 {
091 super("cumulus4j", clr, nucleusContext, props);
092
093 logger.info("====================== Cumulus4j ======================");
094 String bundleName = "org.cumulus4j.store";
095 String version = nucleusContext.getPluginManager().getVersionForBundle(bundleName);
096 logger.info("Bundle: " + bundleName + " - Version: " + version);
097 logger.info("=======================================================");
098
099 indexFactoryRegistry = new IndexEntryFactoryRegistry(this);
100 encryptionHandler = new EncryptionHandler();
101 encryptionCoordinateSetManager = new EncryptionCoordinateSetManager();
102 persistenceHandler = new Cumulus4jPersistenceHandler(this);
103 }
104
105 public EncryptionHandler getEncryptionHandler() {
106 return encryptionHandler;
107 }
108
109 public EncryptionCoordinateSetManager getEncryptionCoordinateSetManager() {
110 return encryptionCoordinateSetManager;
111 }
112
113 public IndexEntryFactoryRegistry getIndexFactoryRegistry() {
114 return indexFactoryRegistry;
115 }
116
117 /**
118 * Get the persistent meta-data of a certain class. This persistent meta-data is primarily used for efficient
119 * mapping using long-identifiers instead of fully qualified class names.
120 *
121 * @param ec
122 * @param clazz the {@link Class} for which to query the meta-data.
123 * @return the meta-data. Never returns <code>null</code>.
124 */
125 public ClassMeta getClassMeta(ExecutionContext ec, Class<?> clazz)
126 {
127 ClassMeta result = class2classMeta.get(clazz);
128 if (result != null) {
129 return result;
130 }
131
132 ManagedConnection mconn = this.getConnection(ec);
133 try {
134 PersistenceManagerConnection pmConn = (PersistenceManagerConnection)mconn.getConnection();
135 PersistenceManager pm = pmConn.getDataPM();
136
137 synchronized (this) { // Synchronise in case we have data and index backends
138 // Register the class
139 pm.getFetchPlan().setGroup(FetchPlan.ALL);
140 result = registerClass(ec, pm, clazz);
141
142 // Detach the class in order to cache only detached objects. Make sure fetch-plan detaches all
143 pm.getFetchPlan().setGroup(FetchPlan.ALL);
144 pm.getFetchPlan().setMaxFetchDepth(-1);
145 result = pm.detachCopy(result);
146
147 if (pmConn.indexHasOwnPM()) {
148 // Replicate ClassMeta+FieldMeta to Index datastore
149 PersistenceManager pmIndex = pmConn.getIndexPM();
150 pmIndex.getFetchPlan().setGroup(FetchPlan.ALL);
151 pmIndex.getFetchPlan().setMaxFetchDepth(-1);
152 pmIndex.makePersistent(result);
153 }
154 }
155
156 class2classMeta.put(clazz, result);
157
158 // register in class2subclasses-map
159 Set<Class<?>> currentSubclasses = new HashSet<Class<?>>();
160 Class<?> c = clazz;
161 ClassMeta cm = result;
162 while (cm != null) {
163 currentSubclasses.add(c);
164
165 Set<Class<?>> subclasses;
166 synchronized (class2subclasses) {
167 subclasses = class2subclasses.get(c);
168 if (subclasses == null) {
169 subclasses = Collections.synchronizedSet(new HashSet<Class<?>>());
170 class2subclasses.put(c, subclasses);
171 }
172 }
173
174 subclasses.addAll(currentSubclasses);
175
176 c = c.getSuperclass();
177 cm = cm.getSuperClassMeta();
178 if (cm != null) {
179 if (c == null)
180 throw new IllegalStateException("c == null && cm.className == " + cm.getClassName());
181
182 if (!cm.getClassName().equals(c.getName()))
183 throw new IllegalStateException("cm.className != c.name :: cm.className=" + cm.getClassName() + " c.name=" + c.getName());
184
185 // Store the super-class-meta-data for optimisation reasons (not necessary, but [hopefully] better).
186 class2classMeta.put(c, cm);
187 }
188 }
189 } finally {
190 mconn.release();
191 }
192
193 return result;
194 }
195
196 private ClassMeta registerClass(ExecutionContext ec, PersistenceManager pm, Class<?> clazz)
197 {
198 AbstractClassMetaData dnClassMetaData = getMetaDataManager().getMetaDataForClass(clazz, ec.getClassLoaderResolver());
199 if (dnClassMetaData == null)
200 throw new IllegalArgumentException("The class " + clazz.getName() + " does not have persistence-meta-data! Is it persistence-capable? Is it enhanced?");
201
202 ClassMeta classMeta = new ClassMetaDAO(pm).getClassMeta(clazz, false);
203 boolean classExists = (classMeta != null);
204 if (!classExists) {
205 classMeta = new ClassMeta(clazz);
206 }
207
208 Class<?> superclass = clazz.getSuperclass();
209 if (superclass != null && getMetaDataManager().hasMetaDataForClass(superclass.getName())) {
210 ClassMeta superClassMeta = registerClass(ec, pm, superclass);
211 classMeta.setSuperClassMeta(superClassMeta);
212 }
213
214 Set<String> persistentMemberNames = new HashSet<String>();
215 for (AbstractMemberMetaData memberMetaData : dnClassMetaData.getManagedMembers()) {
216 if (!memberMetaData.isFieldToBePersisted())
217 continue;
218
219 persistentMemberNames.add(memberMetaData.getName());
220 int dnAbsoluteFieldNumber = memberMetaData.getAbsoluteFieldNumber();
221
222 // register primary field-meta
223 FieldMeta primaryFieldMeta = classMeta.getFieldMeta(memberMetaData.getName());
224 if (primaryFieldMeta == null) {
225 // adding field that's so far unknown
226 primaryFieldMeta = new FieldMeta(classMeta, memberMetaData.getName());
227 classMeta.addFieldMeta(primaryFieldMeta);
228 }
229 primaryFieldMeta.setDataNucleusAbsoluteFieldNumber(dnAbsoluteFieldNumber);
230
231 if (memberMetaData.hasCollection()) {
232 // register "collection" field-meta, if appropriate
233 primaryFieldMeta.removeAllSubFieldMetasExcept(FieldMetaRole.collectionElement);
234 FieldMeta subFieldMeta = primaryFieldMeta.getSubFieldMeta(FieldMetaRole.collectionElement);
235 if (subFieldMeta == null) {
236 // adding field that's so far unknown
237 subFieldMeta = new FieldMeta(primaryFieldMeta, FieldMetaRole.collectionElement);
238 primaryFieldMeta.addSubFieldMeta(subFieldMeta);
239 }
240 }
241 else if (memberMetaData.hasArray()) {
242 // register "array" field-meta, if appropriate
243 // TODO shouldn't we handle it exactly as a collection, including reusing 'FieldMetaRole.collectionElement' for this case?
244 primaryFieldMeta.removeAllSubFieldMetasExcept(FieldMetaRole.arrayElement);
245 FieldMeta subFieldMeta = primaryFieldMeta.getSubFieldMeta(FieldMetaRole.arrayElement);
246 if (subFieldMeta == null) {
247 // adding field that's so far unknown
248 subFieldMeta = new FieldMeta(primaryFieldMeta, FieldMetaRole.arrayElement);
249 primaryFieldMeta.addSubFieldMeta(subFieldMeta);
250 }
251 }
252 else if (memberMetaData.hasMap()) {
253 // register "map" field-meta, if appropriate
254 primaryFieldMeta.removeAllSubFieldMetasExcept(FieldMetaRole.mapKey, FieldMetaRole.mapValue);
255
256 FieldMeta subFieldMeta = primaryFieldMeta.getSubFieldMeta(FieldMetaRole.mapKey);
257 if (subFieldMeta == null) {
258 // adding field that's so far unknown
259 subFieldMeta = new FieldMeta(primaryFieldMeta, FieldMetaRole.mapKey);
260 primaryFieldMeta.addSubFieldMeta(subFieldMeta);
261 }
262
263 subFieldMeta = primaryFieldMeta.getSubFieldMeta(FieldMetaRole.mapValue);
264 if (subFieldMeta == null) {
265 // adding field that's so far unknown
266 subFieldMeta = new FieldMeta(primaryFieldMeta, FieldMetaRole.mapValue);
267 primaryFieldMeta.addSubFieldMeta(subFieldMeta);
268 }
269 }
270 else
271 primaryFieldMeta.removeAllSubFieldMetasExcept();
272 }
273
274 for (FieldMeta fieldMeta : new ArrayList<FieldMeta>(classMeta.getFieldMetas())) {
275 if (persistentMemberNames.contains(fieldMeta.getFieldName()))
276 continue;
277
278 // The field is not in the class anymore => remove its persistent reference.
279 classMeta.removeFieldMeta(fieldMeta);
280 }
281
282 if (!classExists) {
283 // Persist the new class and its fields in one call, minimising updates
284 pm.makePersistent(classMeta);
285 }
286 pm.flush(); // Get exceptions as soon as possible by forcing a flush here
287
288 return classMeta;
289 }
290
291 private Map<Object, String> objectID2className = Collections.synchronizedMap(new WeakHashMap<Object, String>());
292
293 /**
294 * Store the association between an objectID and the class-name of the corresponding persistable object in
295 * a {@link WeakHashMap}. This is used for performance optimization of
296 * {@link #getClassNameForObjectID(Object, ClassLoaderResolver, ExecutionContext)}.
297 */
298 public void setClassNameForObjectID(Object id, String className)
299 {
300 objectID2className.put(id, className);
301 }
302
303 @Override
304 public String getClassNameForObjectID(Object id, ClassLoaderResolver clr, ExecutionContext ec)
305 {
306 if (id == null) {
307 return null;
308 }
309
310 String className = objectID2className.get(id);
311 if (className != null) {
312 return className;
313 }
314
315 if (id instanceof SCOID) {
316 // Object is a SCOID
317 className = ((SCOID) id).getSCOClass();
318 }
319 else if (id instanceof OID) {
320 // Object is an OID
321 className = ((OID)id).getPcClass();
322 }
323 else if (getApiAdapter().isSingleFieldIdentity(id)) {
324 // Using SingleFieldIdentity so can assume that object is of the target class
325 className = getApiAdapter().getTargetClassNameForSingleFieldIdentity(id);
326 }
327 else {
328 // Application identity with user PK class, so find all using this PK
329 Collection<AbstractClassMetaData> cmds = getMetaDataManager().getClassMetaDataWithApplicationId(id.getClass().getName());
330 if (cmds != null) {
331 if (cmds.size() == 1) {
332 className = cmds.iterator().next().getFullClassName();
333 }
334 else {
335 ManagedConnection mconn = this.getConnection(ec);
336 try {
337 PersistenceManagerConnection pmConn = (PersistenceManagerConnection)mconn.getConnection();
338 PersistenceManager pmData = pmConn.getDataPM();
339 String objectIDString = id.toString();
340 for (AbstractClassMetaData cmd : cmds) {
341 Class<?> clazz = clr.classForName(cmd.getFullClassName());
342 ClassMeta classMeta = getClassMeta(ec, clazz);
343 DataEntry dataEntry = DataEntry.getDataEntry(pmData, classMeta, objectIDString);
344 if (dataEntry != null) {
345 className = cmd.getFullClassName();
346 }
347 }
348 } finally {
349 mconn.release();
350 }
351 }
352 }
353 }
354
355 if (className != null) {
356 objectID2className.put(id, className);
357 }
358
359 return className;
360 }
361
362 // public long nextDataEntryID(ExecutionContext executionContext)
363 // {
364 // NucleusSequence nucleusSequence = getNucleusSequence(executionContext, SEQUENCE_META_DATA_DATA_ENTRY);
365 // return nucleusSequence.nextValue();
366 // }
367
368 @Override
369 protected String getStrategyForNative(AbstractClassMetaData cmd, int absFieldNumber) {
370 return "increment";
371 // AbstractMemberMetaData mmd = cmd.getMetaDataForManagedMemberAtAbsolutePosition(absFieldNumber);
372 // if (String.class.isAssignableFrom(mmd.getType()) || UUID.class.isAssignableFrom(mmd.getType()))
373 // return "uuid-hex";
374 // else
375 // return "increment";
376 }
377
378 @Override
379 public void createSchema(Set<String> classNames, Properties props) {
380 Cumulus4jConnectionFactory cf =
381 (Cumulus4jConnectionFactory) connectionMgr.lookupConnectionFactory(txConnectionFactoryName);
382 JDOPersistenceManagerFactory pmfData = (JDOPersistenceManagerFactory) cf.getPMFData();
383 JDOPersistenceManagerFactory pmfIndex = (JDOPersistenceManagerFactory) cf.getPMFIndex();
384 if (pmfData.getNucleusContext().getStoreManager() instanceof SchemaAwareStoreManager) {
385 // Create Cumulus4J "Data" (plus "Index" if not separate) schema
386 SchemaAwareStoreManager schemaMgr = (SchemaAwareStoreManager) pmfData.getNucleusContext().getStoreManager();
387 Set<String> cumulus4jClassNames = new HashSet<String>();
388 Collection<Class> pmfClasses = pmfData.getManagedClasses();
389 for (Class cls : pmfClasses) {
390 cumulus4jClassNames.add(cls.getName());
391 }
392 schemaMgr.createSchema(cumulus4jClassNames, new Properties());
393 }
394 if (pmfIndex != null && pmfIndex.getNucleusContext().getStoreManager() instanceof SchemaAwareStoreManager) {
395 // Create Cumulus4J "Index" schema
396 SchemaAwareStoreManager schemaMgr = (SchemaAwareStoreManager) pmfIndex.getNucleusContext().getStoreManager();
397 Set<String> cumulus4jClassNames = new HashSet<String>();
398 Collection<Class> pmfClasses = pmfIndex.getManagedClasses();
399 for (Class cls : pmfClasses) {
400 cumulus4jClassNames.add(cls.getName());
401 }
402 schemaMgr.createSchema(cumulus4jClassNames, new Properties());
403 }
404 }
405
406 @Override
407 public void deleteSchema(Set<String> classNames, Properties props) {
408 Cumulus4jConnectionFactory cf =
409 (Cumulus4jConnectionFactory) connectionMgr.lookupConnectionFactory(txConnectionFactoryName);
410 JDOPersistenceManagerFactory pmfData = (JDOPersistenceManagerFactory) cf.getPMFData();
411 JDOPersistenceManagerFactory pmfIndex = (JDOPersistenceManagerFactory) cf.getPMFIndex();
412 if (pmfData.getNucleusContext().getStoreManager() instanceof SchemaAwareStoreManager) {
413 SchemaAwareStoreManager schemaMgr = (SchemaAwareStoreManager) pmfData.getNucleusContext().getStoreManager();
414 Set<String> cumulus4jClassNames = new HashSet<String>();
415 Collection<Class> pmfClasses = pmfData.getManagedClasses();
416 for (Class cls : pmfClasses) {
417 cumulus4jClassNames.add(cls.getName());
418 }
419 schemaMgr.deleteSchema(cumulus4jClassNames, new Properties());
420 }
421 if (pmfIndex != null && pmfIndex.getNucleusContext().getStoreManager() instanceof SchemaAwareStoreManager) {
422 SchemaAwareStoreManager schemaMgr = (SchemaAwareStoreManager) pmfIndex.getNucleusContext().getStoreManager();
423 Set<String> cumulus4jClassNames = new HashSet<String>();
424 Collection<Class> pmfClasses = pmfIndex.getManagedClasses();
425 for (Class cls : pmfClasses) {
426 cumulus4jClassNames.add(cls.getName());
427 }
428 schemaMgr.deleteSchema(cumulus4jClassNames, new Properties());
429 }
430 }
431
432 @Override
433 public void validateSchema(Set<String> classNames, Properties props) {
434 Cumulus4jConnectionFactory cf =
435 (Cumulus4jConnectionFactory) connectionMgr.lookupConnectionFactory(txConnectionFactoryName);
436 JDOPersistenceManagerFactory pmfData = (JDOPersistenceManagerFactory) cf.getPMFData();
437 JDOPersistenceManagerFactory pmfIndex = (JDOPersistenceManagerFactory) cf.getPMFIndex();
438 if (pmfData.getNucleusContext().getStoreManager() instanceof SchemaAwareStoreManager) {
439 SchemaAwareStoreManager schemaMgr = (SchemaAwareStoreManager) pmfData.getNucleusContext().getStoreManager();
440 Set<String> cumulus4jClassNames = new HashSet<String>();
441 Collection<Class> pmfClasses = pmfData.getManagedClasses();
442 for (Class cls : pmfClasses) {
443 cumulus4jClassNames.add(cls.getName());
444 }
445 schemaMgr.validateSchema(cumulus4jClassNames, new Properties());
446 }
447 if (pmfIndex != null && pmfIndex.getNucleusContext().getStoreManager() instanceof SchemaAwareStoreManager) {
448 SchemaAwareStoreManager schemaMgr = (SchemaAwareStoreManager) pmfIndex.getNucleusContext().getStoreManager();
449 Set<String> cumulus4jClassNames = new HashSet<String>();
450 Collection<Class> pmfClasses = pmfIndex.getManagedClasses();
451 for (Class cls : pmfClasses) {
452 cumulus4jClassNames.add(cls.getName());
453 }
454 schemaMgr.validateSchema(cumulus4jClassNames, new Properties());
455 }
456 }
457 }