diff options
author | Bernard Normier <bernard@zeroc.com> | 2004-04-17 18:47:01 +0000 |
---|---|---|
committer | Bernard Normier <bernard@zeroc.com> | 2004-04-17 18:47:01 +0000 |
commit | b1a91fcf5c95a9bea78d2ae4e8c053c5b6f7228b (patch) | |
tree | 25cead1769d815d026da3270572dd8817d6135b9 /java/src | |
parent | Freeze fixes (diff) | |
download | ice-b1a91fcf5c95a9bea78d2ae4e8c053c5b6f7228b.tar.bz2 ice-b1a91fcf5c95a9bea78d2ae4e8c053c5b6f7228b.tar.xz ice-b1a91fcf5c95a9bea78d2ae4e8c053c5b6f7228b.zip |
Freeze facet update
Diffstat (limited to 'java/src')
-rw-r--r-- | java/src/Freeze/EvictorElement.java | 67 | ||||
-rw-r--r-- | java/src/Freeze/EvictorI.java | 2549 | ||||
-rw-r--r-- | java/src/Freeze/EvictorIteratorI.java | 94 | ||||
-rw-r--r-- | java/src/Freeze/Index.java | 148 | ||||
-rw-r--r-- | java/src/Freeze/ObjectStore.java | 419 | ||||
-rw-r--r-- | java/src/Freeze/Util.java | 14 | ||||
-rw-r--r-- | java/src/IceInternal/ReferenceFactory.java | 2 | ||||
-rw-r--r-- | java/src/IceUtil/Cache.java | 132 | ||||
-rw-r--r-- | java/src/IceUtil/Store.java | 20 |
9 files changed, 1540 insertions, 1905 deletions
diff --git a/java/src/Freeze/EvictorElement.java b/java/src/Freeze/EvictorElement.java new file mode 100644 index 00000000000..c5495dc7264 --- /dev/null +++ b/java/src/Freeze/EvictorElement.java @@ -0,0 +1,67 @@ +// ********************************************************************** +// +// Copyright (c) 2004 +// ZeroC, Inc. +// Billerica, MA, USA +// +// All Rights Reserved. +// +// Ice is free software; you can redistribute it and/or modify it under +// the terms of the GNU General Public License version 2 as published by +// the Free Software Foundation. +// +// ********************************************************************** + +package Freeze; + +class EvictorElement extends Ice.LocalObjectImpl +{ + // + // Clean object; can become modified or destroyed + // + static final byte clean = 0; + + // + // New object; can become clean, dead or destroyed + // + static final byte created = 1; + + // + // Modified object; can become clean or destroyed + // + static final byte modified = 2; + + // + // Being saved. Can become dead or created + // + static final byte destroyed = 3; + + // + // Exists only in the Evictor; for example the object was created + // and later destroyed (without a save in between), or it was + // destroyed on disk but is still in use. Can become created. + // + static final byte dead = 4; + + EvictorElement(Ice.Identity identity, ObjectStore store) + { + this.identity = identity; + this.store = store; + } + + final ObjectStore store; + final Ice.Identity identity; + + // + // Protected by EvictorI + // + java.util.Iterator evictPosition = null; + int usageCount = -1; + boolean stale = false; + + // + // Protected by this + // + ObjectRecord rec = null; + byte status = clean; +} diff --git a/java/src/Freeze/EvictorI.java b/java/src/Freeze/EvictorI.java index db3abf431ad..515f2baa37b 100644 --- a/java/src/Freeze/EvictorI.java +++ b/java/src/Freeze/EvictorI.java @@ -1,6 +1,6 @@ // ********************************************************************** // -// Copyright (c) 2003 +// Copyright (c) 2003-2004 // ZeroC, Inc. // Billerica, MA, USA // @@ -16,32 +16,152 @@ package Freeze; class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable { + + final static String defaultDb = "$default"; + final static String indexPrefix = "$index:"; + public - EvictorI(Ice.Communicator communicator, String envName, String dbName, + EvictorI(Ice.ObjectAdapter adapter, String envName, String filename, + ServantInitializer initializer, Index[] indices, boolean createDb) { - _communicator = communicator; - _dbEnvHolder = SharedDbEnv.get(communicator, envName); + _adapter = adapter; + _communicator = adapter.getCommunicator(); + _initializer = initializer; + + _dbEnvHolder = SharedDbEnv.get(_communicator, envName); _dbEnv = _dbEnvHolder; - _dbName = dbName; - _indices = indices; - - init(envName, createDb); + _filename = filename; + _createDb = createDb; + + init(envName, indices); } public - EvictorI(Ice.Communicator communicator, String envName, - com.sleepycat.db.DbEnv dbEnv, String dbName, + EvictorI(Ice.ObjectAdapter adapter, String envName, + com.sleepycat.db.DbEnv dbEnv, String filename, + ServantInitializer initializer, Index[] indices, boolean createDb) { - _communicator = communicator; - _dbEnvHolder = null; + _adapter = adapter; + _communicator = adapter.getCommunicator(); + _initializer = initializer; + _dbEnv = dbEnv; - _dbName = dbName; - _indices = indices; + _filename = filename; + _createDb = createDb; - init(envName, createDb); + init(envName, indices); + } + + private void + init(String envName, Index[] indices) + { + _trace = _communicator.getProperties().getPropertyAsInt("Freeze.Trace.Evictor"); + _deadlockWarning = _communicator.getProperties().getPropertyAsInt("Freeze.Warn.Deadlocks") != 0; + + _errorPrefix = "Freeze Evictor DbEnv(\"" + envName + "\") Db(\"" + _filename + "\"): "; + + String propertyPrefix = "Freeze.Evictor." + envName + '.' + _filename; + + // + // By default, we save every minute or when the size of the modified + // queue reaches 10. + // + + _saveSizeTrigger = _communicator.getProperties().getPropertyAsIntWithDefault + (propertyPrefix + ".SaveSizeTrigger", 10); + + _savePeriod = _communicator.getProperties().getPropertyAsIntWithDefault + (propertyPrefix + ".SavePeriod", 60 * 1000); + + // + // By default, we save at most 10 * SaveSizeTrigger objects per transaction + // + _maxTxSize = _communicator.getProperties().getPropertyAsIntWithDefault + (propertyPrefix + ".MaxTxSize", 10 * _saveSizeTrigger); + + if(_maxTxSize <= 0) + { + _maxTxSize = 100; + } + + boolean populateEmptyIndices = (_communicator.getProperties().getPropertyAsIntWithDefault + (propertyPrefix + ".PopulateEmptyIndices", 0) != 0); + + // + // Instantiate all Dbs in 2 steps: + // (1) iterate over the indices and create ObjectStore with indices + // (2) open ObjectStores without indices + // + + java.util.List dbs = allDbs(); + // + // Add default db in case it's not there + // + dbs.add(defaultDb); + + + if(indices != null) + { + for(int i = 0; i < indices.length; ++i) + { + String facet = indices[i].facet(); + + if(_storeMap.get(facet) == null) + { + java.util.List storeIndices = new java.util.LinkedList(); + for(int j = i; j < indices.length; ++j) + { + if(indices[j].facet().equals(facet)) + { + storeIndices.add(indices[j]); + } + } + + ObjectStore store = new ObjectStore(facet, _createDb, this, storeIndices, + populateEmptyIndices); + _storeMap.put(facet, store); + } + } + } + + java.util.Iterator p = dbs.iterator(); + while(p.hasNext()) + { + String facet = (String) p.next(); + if(facet.equals(defaultDb)) + { + facet = ""; + } + + if(_storeMap.get(facet) == null) + { + ObjectStore store = new ObjectStore(facet, _createDb, this, new java.util.LinkedList(), + populateEmptyIndices); + + _storeMap.put(facet, store); + } + } + + // + // Start saving thread + // + String threadName; + String programName = _communicator.getProperties().getProperty("Ice.ProgramName"); + if(programName.length() > 0) + { + threadName = programName + "-"; + } + else + { + threadName = ""; + } + threadName += "FreezeEvictorThread(" + envName + '.' + _filename + ")"; + _thread = new Thread(this, threadName); + _thread.start(); } + protected void finalize() @@ -49,6 +169,7 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable if(!_deactivated) { _communicator.getLogger().warning("evictor has not been deactivated"); + deactivate(""); } } @@ -90,131 +211,30 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable return _evictorSize; } - public void - createObject(Ice.Identity ident, Ice.Object servant) + public Ice.ObjectPrx + add(Ice.Object servant, Ice.Identity ident) { - EvictorElement loadedElement = null; - int loadedElementGeneration = 0; - boolean triedToLoadElement = false; + return addFacet(servant, ident, ""); + } + + public Ice.ObjectPrx + addFacet(Ice.Object servant, Ice.Identity ident, String facet) + { // - // Make a copy of ident in case the user later changes it - // (used when inserting into list or map) + // Need to clone in case the given ident changes. // - Ice.Identity identCopy = new Ice.Identity(); - identCopy.name = ident.name; - identCopy.category = ident.category; - - for(;;) + try { - synchronized(this) - { - if(_deactivated) - { - throw new EvictorDeactivatedException(); - } - - EvictorElement element = (EvictorElement)_evictorMap.get(ident); - - if(element == null && triedToLoadElement) - { - if(loadedElementGeneration == _generation) - { - if(loadedElement != null) - { - element = insertElement(null, identCopy, loadedElement); - } - } - else - { - loadedElement = null; - triedToLoadElement = false; - } - } - - boolean replacing = (element != null); - - if(replacing || triedToLoadElement) - { - if(replacing) - { - // - // Destroy all existing facets - // - - java.util.Iterator p = element.facets.entrySet().iterator(); - - while(p.hasNext()) - { - java.util.Map.Entry entry = (java.util.Map.Entry) p.next(); - destroyFacetImpl((Facet) entry.getValue()); - } - } - else - { - // - // Let's insert an empty EvictorElement - // - element = new EvictorElement(); - _evictorMap.put(identCopy, element); - _evictorList.addFirst(identCopy); - element.identity = identCopy; - element.position = _evictorList.iterator(); - element.position.next(); - } - - // - // Add all the new facets (recursively) - // - addFacetImpl(element, servant, new String[0], replacing); - - // - // Evict as many elements as necessary - // - evict(); - break; // for(;;) - } - else - { - loadedElementGeneration = _generation; - } - } - - // - // Try to load element and try again - // - assert(loadedElement == null); - assert(triedToLoadElement == false); - loadedElement = load(ident); - triedToLoadElement = true; + ident = (Ice.Identity) ident.clone(); } - - if(_trace >= 1) + catch(CloneNotSupportedException ex) { - _communicator.getLogger().trace("Freeze.Evictor", - "created \"" + Ice.Util.identityToString(ident) + "\""); + assert false; } - } - - public void - addFacet(Ice.Identity ident, String[] facetPath, Ice.Object servant) - { - if(facetPath.length == 0) - { - throw new EmptyFacetPathException(); - } - - EvictorElement loadedElement = null; - int loadedElementGeneration = 0; - - // - // Make a copy of ident in case the user later changes it - // (used when inserting into list or map) - // - Ice.Identity identCopy = new Ice.Identity(); - identCopy.name = ident.name; - identCopy.category = ident.category; + ObjectStore store = null; + for(;;) { synchronized(this) @@ -223,600 +243,485 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable { throw new EvictorDeactivatedException(); } + + Object o = _storeMap.get(facet); - EvictorElement element = (EvictorElement)_evictorMap.get(ident); - - if(element == null && loadedElement != null) + if(o == null) { - if(loadedElementGeneration == _generation) + if(store != null) { - element = insertElement(null, identCopy, loadedElement); - } - else - { - // - // Discard loadedElement - // - loadedElement = null; + _storeMap.put(facet, store); } } - - if(element != null) + else { - String[] parentPath = new String[facetPath.length - 1]; - System.arraycopy(facetPath, 0, parentPath, 0, facetPath.length - 1); - - Facet facet = (Facet) element.facets.get(new StringArray(parentPath)); - if(facet == null) + if(store != null) { - throw new Ice.FacetNotExistException(); + store.close(); } - - synchronized(facet) - { - if(facet.status == dead || facet.status == destroyed) - { - throw new Ice.FacetNotExistException(); - } - - // - // Throws AlreadyRegisterException if the facet is already registered - // - facet.rec.servant.ice_addFacet(servant, facetPath[facetPath.length - 1]); - } - - // - // We may need to replace (nested) dead or destroyed facets - // - addFacetImpl(element, servant, facetPath, true); - evict(); - break; // for(;;) + store = (ObjectStore) o; } - - loadedElementGeneration = _generation; } - - assert(loadedElement == null); - - // - // Load object and loop - // - loadedElement = load(ident); - if(loadedElement == null) + + if(store == null) { - throw new Ice.ObjectNotExistException(); + assert facet.length() > 0; + store = new ObjectStore(facet, _createDb, this, new java.util.LinkedList(), false); + // loop } - } - - if(_trace >= 1) - { - _communicator.getLogger().trace("Freeze.Evictor", - "added facet " + facetPathToString(facetPath) - + " to \"" + Ice.Util.identityToString(ident) + "\""); - } - } - - public void - destroyObject(Ice.Identity ident) - { - EvictorElement loadedElement = null; - int loadedElementGeneration = 0; - boolean triedToLoadElement = false; - - // - // Make a copy of ident in case the user later changes it - // (used when inserting into list or map) - // - Ice.Identity identCopy = new Ice.Identity(); - identCopy.name = ident.name; - identCopy.category = ident.category; - - for(;;) - { - synchronized(this) + else { - if(_deactivated) - { - throw new EvictorDeactivatedException(); - } - - EvictorElement element = (EvictorElement)_evictorMap.get(ident); - - if(element == null && triedToLoadElement) - { - if(loadedElementGeneration == _generation) - { - if(loadedElement != null) - { - element = insertElement(null, identCopy, loadedElement); - } - } - else - { - loadedElement = null; - triedToLoadElement = false; - } - } - - boolean destroying = (element != null); - - if(destroying || triedToLoadElement) - { - if(destroying) - { - // - // Destroy all existing facets - // - java.util.Iterator p = element.facets.entrySet().iterator(); - while(p.hasNext()) - { - java.util.Map.Entry entry = (java.util.Map.Entry) p.next(); - destroyFacetImpl((Facet) entry.getValue()); - } - } - - // - // Evict as many elements as necessary - // - evict(); - break; // for(;;) - } - else - { - loadedElementGeneration = _generation; - } + break; // for(;;) } - - // - // Try to load element and try again - // - assert(loadedElement == null); - assert(triedToLoadElement == false); - loadedElement = load(ident); - triedToLoadElement = true; - } - - if(_trace >= 1) - { - _communicator.getLogger().trace("Freeze.Evictor", - "destroyed \"" + Ice.Util.identityToString(ident) + "\""); - } - } - - public Ice.Object - removeFacet(Ice.Identity ident, String facetPath[]) - { - if(facetPath.length == 0) - { - throw new EmptyFacetPathException(); } + + assert store != null; + boolean alreadyThere = false; + - Ice.Object result = null; - EvictorElement loadedElement = null; - int loadedElementGeneration = 0; - - // - // Make a copy of ident in case the user later changes it - // (used when inserting into list or map) - // - Ice.Identity identCopy = new Ice.Identity(); - identCopy.name = ident.name; - identCopy.category = ident.category; - for(;;) { + // + // Create a new entry + // + + EvictorElement element = new EvictorElement(ident, store); + element.status = EvictorElement.dead; + element.rec = new ObjectRecord(); + element.rec.stats = new Statistics(); + + Object o = store.cache().add(ident, element); + + if(o != null) + { + element = (EvictorElement) o; + } + synchronized(this) { if(_deactivated) { throw new EvictorDeactivatedException(); } - - EvictorElement element = (EvictorElement)_evictorMap.get(ident); - - if(element == null && loadedElement != null) + + if(element.stale) { - if(loadedElementGeneration == _generation) - { - element = insertElement(null, identCopy, loadedElement); - } - else - { - // - // Discard loadedElement - // - loadedElement = null; - } + // + // Try again + // + continue; } - - if(element != null) + fixEvictPosition(element); + + synchronized(element) { - String[] parentPath = new String[facetPath.length - 1]; - System.arraycopy(facetPath, 0, parentPath, 0, facetPath.length - 1); - - Facet facet = (Facet) element.facets.get(new StringArray(parentPath)); - if(facet == null) - { - throw new Ice.FacetNotExistException(); - } - - synchronized(facet) + switch(element.status) { - if(facet.status == dead || facet.status == destroyed) + case EvictorElement.clean: + case EvictorElement.created: + case EvictorElement.modified: { - throw new Ice.FacetNotExistException(); + alreadyThere = true; + break; + } + case EvictorElement.destroyed: + { + element.status = EvictorElement.modified; + element.rec.servant = servant; + + // + // No need to push it on the modified queue, as a destroyed object + // is either already on the queue or about to be saved. When saved, + // it becomes dead. + // + break; + } + case EvictorElement.dead: + { + element.status = EvictorElement.created; + ObjectRecord rec = element.rec; + + rec.servant = servant; + rec.stats.creationTime = System.currentTimeMillis(); + rec.stats.lastSaveTime = 0; + rec.stats.avgSaveTime = 0; + + addToModifiedQueue(element); + break; + } + default: + { + assert false; + break; } - - // - // Throws NotRegisteredException if the facet is not registered - // - result = facet.rec.servant.ice_removeFacet(facetPath[facetPath.length - 1]); } - removeFacetImpl(element.facets, facetPath); - evict(); - break; // for(;;) } - - loadedElementGeneration = _generation; } - - assert(loadedElement == null); - - // - // Load object and loop - // - loadedElement = load(ident); - if(loadedElement == null) + break; // for(;;) + } + + if(alreadyThere) + { + Ice.AlreadyRegisteredException ex = new Ice.AlreadyRegisteredException(); + ex.kindOfObject = "servant"; + ex.id = Ice.Util.identityToString(ident); + if(facet.length() != 0) { - throw new Ice.ObjectNotExistException(); + ex.id += " -f " + facet; } + throw ex; } - + if(_trace >= 1) { - _communicator.getLogger().trace("Freeze.Evictor", - "removed facet " + facetPathToString(facetPath) - + " from \"" + Ice.Util.identityToString(ident) + "\""); + String objString = "object \"" + Ice.Util.identityToString(ident) + "\""; + if(!facet.equals("")) + { + objString += " with facet \"" + facet + "\""; + } + + _communicator.getLogger().trace( + "Freeze.Evictor", + "added " + objString + " in the database"); } - return result; + + // + // TODO: there is currently no way to create an ObjectPrx + // with a facet! + // + return _adapter.createProxy(ident); } - + public void - removeAllFacets(Ice.Identity ident) + remove(Ice.Identity ident) { - EvictorElement loadedElement = null; - int loadedElementGeneration = 0; + removeFacet(ident, ""); + } + public void + removeFacet(Ice.Identity ident, String facet) + { // - // Make a copy of ident in case the user later changes it - // (used when inserting into list or map) + // Need to clone in case the given ident changes. // - Ice.Identity identCopy = new Ice.Identity(); - identCopy.name = ident.name; - identCopy.category = ident.category; + try + { + ident = (Ice.Identity) ident.clone(); + } + catch(CloneNotSupportedException ex) + { + assert false; + } - for(;;) + ObjectStore store = findStore(facet); + boolean notThere = (store == null); + + if(store != null) { - synchronized(this) + for(;;) { - if(_deactivated) - { - throw new EvictorDeactivatedException(); - } - - EvictorElement element = (EvictorElement)_evictorMap.get(ident); - - if(element == null && loadedElement != null) + // + // Retrieve object + // + + EvictorElement element = (EvictorElement) store.cache().pin(ident); + if(element == null) { - if(loadedElementGeneration == _generation) - { - element = insertElement(null, identCopy, loadedElement); - } - else - { - // - // Discard loadedElement - // - loadedElement = null; - } + notThere = true; } - - if(element != null) + else { - Facet facet = element.mainObject; - synchronized(facet) + synchronized(this) { - if(facet.status == dead || facet.status == destroyed) + if(element.stale) { - throw new Ice.ObjectNotExistException(); + // + // Try again + // + continue; } - facet.rec.servant.ice_removeAllFacets(); - } - - // - // Destroy all facets except main object - // - java.util.Iterator p = element.facets.entrySet().iterator(); - - while(p.hasNext()) - { - java.util.Map.Entry entry = (java.util.Map.Entry) p.next(); - if(entry.getValue() != element.mainObject) + + fixEvictPosition(element); + synchronized(element) { - destroyFacetImpl((Facet) entry.getValue()); + switch(element.status) + { + case EvictorElement.clean: + { + element.status = EvictorElement.destroyed; + element.rec.servant = null; + addToModifiedQueue(element); + break; + } + case EvictorElement.created: + { + element.status = EvictorElement.dead; + element.rec.servant = null; + break; + } + case EvictorElement.modified: + { + element.status = EvictorElement.destroyed; + element.rec.servant = null; + // + // Not necessary to push it on the modified queue, as a modified + // element is either on the queue already or about to be saved + // (at which point it becomes clean) + // + break; + } + case EvictorElement.destroyed: + case EvictorElement.dead: + { + notThere = true; + break; + } + default: + { + assert false; + break; + } + } } } - - evict(); - break; // for(;;) } - - loadedElementGeneration = _generation; + break; // for(;;) } - - assert(loadedElement == null); - - // - // Load object and loop - // - loadedElement = load(ident); - if(loadedElement == null) + } + + if(notThere) + { + Ice.NotRegisteredException ex = new Ice.NotRegisteredException(); + ex.kindOfObject = "servant"; + ex.id = Ice.Util.identityToString(ident); + if(facet.length() != 0) { - throw new Ice.ObjectNotExistException(); + ex.id += " -f " + facet; } + throw ex; } - + if(_trace >= 1) { - _communicator.getLogger().trace("Freeze.Evictor", - "removed all facets from \"" + Ice.Util.identityToString(ident) + "\""); - } - } - - synchronized public void - installServantInitializer(ServantInitializer initializer) - { - if(_deactivated) - { - throw new EvictorDeactivatedException(); + String objString = "object \"" + Ice.Util.identityToString(ident) + "\""; + if(!facet.equals("")) + { + objString += " with facet \"" + facet + "\""; + } + + _communicator.getLogger().trace( + "Freeze.Evictor", + "removed " + objString); } - - _initializer = initializer; } + public EvictorIterator - getIterator(int batchSize, boolean loadServants) + getIterator(String facet, int batchSize) { + ObjectStore store = null; synchronized(this) { if(_deactivated) { throw new EvictorDeactivatedException(); } - saveNowNoSync(); - } - return new EvictorIteratorI(this, batchSize, loadServants); + store = (ObjectStore) _storeMap.get(facet); + if(store != null) + { + saveNowNoSync(); + } + } + return new EvictorIteratorI(store, batchSize); } public boolean hasObject(Ice.Identity ident) { + return hasFacet(ident, ""); + } + + public boolean + hasFacet(Ice.Identity ident, String facet) + { + ObjectStore store = null; + synchronized(this) { if(_deactivated) { throw new EvictorDeactivatedException(); } - - EvictorElement element = (EvictorElement)_evictorMap.get(ident); + + store = (ObjectStore) _storeMap.get(facet); + if(store == null) + { + return false; + } + + EvictorElement element = (EvictorElement) store.cache().getIfPinned(ident); if(element != null) { - synchronized(element.mainObject) + assert !element.stale; + + synchronized(element) { - return (element.mainObject.status != destroyed && element.mainObject.status != dead); + return element.status != EvictorElement.dead && + element.status != EvictorElement.destroyed; } } } - - return dbHasObject(ident); + return store.dbHasObject(ident); } public Ice.Object locate(Ice.Current current, Ice.LocalObjectHolder cookie) { - EvictorElement loadedElement = null; - int loadedElementGeneration = 0; - cookie.value = null; - - // - // Need to copy current.id, as Ice caches and reuses it - // - Ice.Identity ident = new Ice.Identity(); - ident.name = current.id.name; - ident.category = current.id.category; - - for(;;) + Ice.Object result = locateImpl(current, cookie); + + if(result == null) { - EvictorElement element; - boolean objectFound = false; - boolean newObject = false; - + // + // If the object exists in another store, throw FacetNotExistException + // instead of returning null (== ObjectNotExistException) + // + java.util.Map storeMapCopy; synchronized(this) { - // - // If this operation is called on a deactivated servant locator, - // it's a bug in Ice. - // - assert(!_deactivated); - - element = (EvictorElement)_evictorMap.get(ident); - - if(element == null && loadedElement != null) - { - if(loadedElementGeneration == _generation) - { - element = insertElement(null, ident, loadedElement); - newObject = true; - } - else - { - // - // Discard loadedElement - // - loadedElement = null; - } - } + storeMapCopy = new java.util.HashMap(_storeMap); + } - objectFound = (element != null); + java.util.Iterator p = storeMapCopy.entrySet().iterator(); + while(p.hasNext()) + { + java.util.Map.Entry entry = (java.util.Map.Entry) p.next(); - if(objectFound) + // + // Do not check again the current facet + // + if(!current.facet.equals(entry.getKey())) { - // - // Ice object found in evictor map. Push it to the front of - // the evictor list, so that it will be evicted last. - // - if(!newObject) - { - element.position.remove(); - _evictorList.addFirst(ident); - element.position = _evictorList.iterator(); - // - // Position the iterator "on" the element. - // - element.position.next(); - } - - element.usageCount++; - - assert(current.facet != null); - Facet facet = (Facet) element.facets.get(new StringArray(current.facet)); - if(facet != null) - { - cookie.value = facet; - } + ObjectStore store = (ObjectStore) entry.getValue(); + boolean inCache = false; - evict(); - - // - // Later (after releasing the mutex), check that this - // object is not dead or destroyed - // - } - else - { - loadedElementGeneration = _generation; - } - } - - if(objectFound) - { - if(_trace >= 2) - { - _communicator.getLogger().trace("Freeze.Evictor", - "found \"" + Ice.Util.identityToString(ident) + - "\" in the queue"); - } - - if(cookie.value == null) - { - Ice.Object result = null; - synchronized(element.mainObject) + synchronized(this) { - if(element.mainObject.status != destroyed && element.mainObject.status != dead) + EvictorElement element = (EvictorElement) store.cache().getIfPinned(current.id); + if(element != null) { - result = element.mainObject.rec.servant; + inCache = true; + assert !element.stale; + + synchronized(element) + { + if(element.status != EvictorElement.dead && + element.status != EvictorElement.destroyed) + { + throw new Ice.FacetNotExistException(); + } + } } - } - if(_trace >= 2) + if(!inCache) { - _communicator.getLogger().trace("Freeze.Evictor", - " \"" + Ice.Util.identityToString(ident) + - "\" does not have the desired facet " + facetPathToString(current.facet)); - } - synchronized(this) - { - element.usageCount--; - return result; - } - } - else - { - synchronized(element.mainObject) - { - if(element.mainObject.status != destroyed && element.mainObject.status != dead) + if(store.dbHasObject(current.id)) { - return element.mainObject.rec.servant; + throw new Ice.FacetNotExistException(); } } - } - - // - // Object is destroyed or dead: clean-up - // - if(_trace >= 2) - { - _communicator.getLogger().trace("Freeze.Evictor", - "\"" + Ice.Util.identityToString(ident) + - "\" was dead or destroyed"); - } - synchronized(this) - { - element.usageCount--; - return null; - } - } - else - { - // - // Load object now and loop - // - - if(_trace >= 2) - { - _communicator.getLogger().trace( - "Freeze.Evictor", - "couldn't find \"" + Ice.Util.identityToString(ident) + "\" in the queue; " - + "loading \"" + Ice.Util.identityToString(ident) + "\" from the database"); - } - - loadedElement = load(ident); - - if(loadedElement == null) - { - // - // The Ice object with the given identity does not exist, - // client will get an ObjectNotExistException. - // - return null; - } + } } } + return result; + } + + Ice.Object + locateImpl(Ice.Current current, Ice.LocalObjectHolder cookie) + { + cookie.value = null; + + // + // Need to clone as current.id gets reused + // + Ice.Identity ident = null; + try + { + ident = (Ice.Identity) current.id.clone(); + } + catch(CloneNotSupportedException ex) + { + assert false; + } + + ObjectStore store = findStore(current.facet); + if(store == null) + { + return null; + } + + for(;;) + { + EvictorElement element = (EvictorElement) store.cache().pin(ident); + if(element == null) + { + return null; + } + + synchronized(this) + { + assert !_deactivated; + + if(element.stale) + { + // + // try again + // + continue; + } + + synchronized(element) + { + if(element.status == EvictorElement.destroyed || + element.status == EvictorElement.dead) + { + return null; + } + + // + // It's a good one! + // + fixEvictPosition(element); + element.usageCount++; + cookie.value = element; + assert element.rec.servant != null; + return element.rec.servant; + } + } + } } public void finished(Ice.Current current, Ice.Object servant, Ice.LocalObject cookie) { - assert(servant != null); + assert servant != null; if(cookie != null) { - Facet facet= (Facet)cookie; - assert(facet != null); - + EvictorElement element = (EvictorElement) cookie; + boolean enqueue = false; - + if(current.mode != Ice.OperationMode.Nonmutating) { - synchronized(facet) + synchronized(element) { - if(facet.status == clean) + if(element.status == EvictorElement.clean) { // // Assume this operation updated the object // - facet.status = modified; + element.status = EvictorElement.modified; enqueue = true; } } @@ -824,17 +729,21 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable synchronized(this) { - assert(!_deactivated); + // + // Only elements with a usageCount == 0 can become stale and we own + // one count! + // + assert !element.stale; + assert element.usageCount >= 1; // // Decrease the usage count of the evictor queue element. // - assert(facet.element.usageCount >= 1); - facet.element.usageCount--; + element.usageCount--; if(enqueue) { - addToModifiedQueue(facet); + addToModifiedQueue(element); } else { @@ -892,33 +801,28 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable } } - try - { - _db.close(0); - if(_indices != null) - { - for(int i = 0; i < _indices.length; ++i) - { - _indices[i].close(); - } - _indices = null; - } - } - catch(com.sleepycat.db.DbException dx) + java.util.Iterator p = _storeMap.values().iterator(); + while(p.hasNext()) { - DatabaseException ex = new DatabaseException(); - ex.initCause(dx); - ex.message = _errorPrefix + "Db.close: " + dx.getMessage(); - throw ex; + ObjectStore store = (ObjectStore) p.next(); + store.close(); } - _db = null; + if(_dbEnvHolder != null) { _dbEnvHolder.close(); _dbEnvHolder = null; } _dbEnv = null; - _initializer = null; + } + } + + void + initialize(Ice.Identity ident, String facet, Ice.Object servant) + { + if(_initializer != null) + { + _initializer.initialize(_adapter, ident, facet, servant); } } @@ -930,6 +834,8 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable for(;;) { java.util.List allObjects; + java.util.List deadObjects = new java.util.LinkedList(); + int saveNowThreadsSize = 0; synchronized(this) @@ -1000,7 +906,7 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable // for(int i = 0; i < size; i++) { - Facet facet = (Facet) allObjects.get(i); + EvictorElement element = (EvictorElement) allObjects.get(i); boolean tryAgain; @@ -1009,32 +915,31 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable tryAgain = false; Ice.Object servant = null; - synchronized(facet) + synchronized(element) { - byte status = facet.status; + byte status = element.status; switch(status) { - case created: - case modified: + case EvictorElement.created: + case EvictorElement.modified: { - servant = facet.rec.servant; + servant = element.rec.servant; break; } - case destroyed: + case EvictorElement.destroyed: { - if(_trace >= 3) - { - _communicator.getLogger().trace( - "Freeze.Evictor", - "saving/streaming \"" + Ice.Util.identityToString(facet.element.identity) + - "\" " + facetPathToString(facet.path) + ": destroyed -> dead"); - } - - facet.status = dead; - streamedObjectQueue.add(streamFacet(facet, status, streamStart)); + streamedObjectQueue.add(stream(element, streamStart)); + + element.status = EvictorElement.dead; + deadObjects.add(element); break; } + case EvictorElement.dead: + { + deadObjects.add(element); + break; + } default: { // @@ -1053,28 +958,20 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable // synchronized(servant) { - synchronized(facet) + synchronized(element) { - byte status = facet.status; + byte status = element.status; switch(status) { - case created: - case modified: + case EvictorElement.created: + case EvictorElement.modified: { - if(servant == facet.rec.servant) + if(servant == element.rec.servant) { - if(_trace >= 3) - { - _communicator.getLogger().trace( - "Freeze.Evictor", - "saving/streaming \"" + Ice.Util.identityToString(facet.element.identity) + - "\" " + facetPathToString(facet.path) + ": created or modified -> clean"); - } - - facet.status = clean; - streamedObjectQueue.add(streamFacet(facet, status, streamStart)); - + streamedObjectQueue.add(stream(element, streamStart)); + + element.status = EvictorElement.clean; } else { @@ -1082,20 +979,19 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable } break; } - case destroyed: + case EvictorElement.destroyed: { - if(_trace >= 3) - { - _communicator.getLogger().trace( - "Freeze.Evictor", - "saving/streaming \"" + Ice.Util.identityToString(facet.element.identity) + - "\" " + facetPathToString(facet.path) + ": destroyed -> dead"); - } + streamedObjectQueue.add(stream(element, streamStart)); - facet.status = dead; - streamedObjectQueue.add(streamFacet(facet, status, streamStart)); + element.status = EvictorElement.dead; + deadObjects.add(element); break; } + case EvictorElement.dead: + { + deadObjects.add(element); + break; + } default: { // @@ -1154,37 +1050,7 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable for(int i = 0; i < txSize; i++) { StreamedObject obj = (StreamedObject) streamedObjectQueue.get(i); - - switch(obj.status) - { - case created: - case modified: - { - com.sleepycat.db.Dbt dbKey = new com.sleepycat.db.Dbt(obj.key); - com.sleepycat.db.Dbt dbValue = new com.sleepycat.db.Dbt(obj.value); - int flags = (obj.status == created) ? com.sleepycat.db.Db.DB_NOOVERWRITE : 0; - int err = _db.put(tx, dbKey, dbValue, flags); - if(err != 0) - { - throw new DatabaseException(); - } - break; - } - case destroyed: - { - com.sleepycat.db.Dbt dbKey = new com.sleepycat.db.Dbt(obj.key); - int err = _db.del(tx, dbKey, 0); - if(err != 0) - { - throw new DatabaseException(); - } - break; - } - default: - { - assert(false); - } - } + obj.store.save(obj.key, obj.value, obj.status, tx); } com.sleepycat.db.DbTxn toCommit = tx; @@ -1217,7 +1083,7 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable if(_deadlockWarning) { _communicator.getLogger().warning - ("Deadlock in Freeze.EvictorI.run while writing into Db \"" + _dbName + ("Deadlock in Freeze.EvictorI.run while writing into Db \"" + _filename + "\"; retrying ..."); } @@ -1235,37 +1101,45 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable } while(tryAgain); synchronized(this) - { - _generation++; - + { + // + // Release usage count + // for(int i = 0; i < allObjects.size(); i++) { - Facet facet = (Facet) allObjects.get(i); - facet.element.usageCount--; - - if(facet != facet.element.mainObject) + EvictorElement element = (EvictorElement) allObjects.get(i); + element.usageCount--; + } + allObjects.clear(); + + java.util.Iterator p = deadObjects.iterator(); + while(p.hasNext()) + { + EvictorElement element = (EvictorElement) p.next(); + if(element.usageCount == 0) { // - // Remove if dead + // Get rid of unused dead elements // - synchronized(facet) + synchronized(element) { - if(facet.status == dead) + if(element.status == EvictorElement.dead) { - facet.element.facets.remove(new StringArray(facet.path)); - } + evict(element); + } } } } - allObjects.clear(); + + deadObjects.clear(); evict(); - + if(saveNowThreadsSize > 0) { for(int i = 0; i < saveNowThreadsSize; i++) { _saveNowThreads.remove(0); - } + } notifyAll(); } } @@ -1283,7 +1157,8 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable Runtime.getRuntime().halt(1); } } - + + final Ice.Communicator communicator() { @@ -1296,22 +1171,10 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable return _dbEnv; } - final com.sleepycat.db.Db - db() - { - return _db; - } - final String - dbName() - { - return _dbName; - } - - final synchronized int - currentGeneration() + filename() { - return _generation; + return _filename; } final String @@ -1320,6 +1183,12 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable return _errorPrefix; } + final boolean + deadlockWarning() + { + return _deadlockWarning; + } + synchronized void saveNow() { @@ -1331,494 +1200,122 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable saveNowNoSync(); } - boolean - load(com.sleepycat.db.Dbc dbc, com.sleepycat.db.Dbt key, com.sleepycat.db.Dbt value, - java.util.List identities, java.util.List evictorElements) - throws com.sleepycat.db.DbException + private void + saveNowNoSync() { - EvictorElement elt = new EvictorElement(); - int rs = 0; - byte[] root = null; - + Thread myself = Thread.currentThread(); + + _saveNowThreads.add(myself); + notifyAll(); do { - // - // Unmarshal key and data and insert it into elt's facet map - // - EvictorStorageKey esk = unmarshalKey(key.get_data(), _communicator); - - if(root == null) + try { - if(esk.facet.length == 0) - { - // - // Good, we found the object - // - root = marshalRootKey(esk.identity, _communicator); - } - else - { - // - // Otherwise, skip this orphan facet (could be a temporary - // inconsistency on disk) - // - - if(_trace >= 3) - { - _communicator.getLogger().trace - ("Freeze.Evictor", - "Iterator is skipping orphan facet \"" + Ice.Util.identityToString(esk.identity) - + "\" " + facetPathToString(esk.facet)); - } - } + wait(); } - - if(root != null) + catch(InterruptedException ex) { - if(_trace >= 3) - { - _communicator.getLogger().trace - ("Freeze.Evictor", - "Iterator is reading facet \"" + Ice.Util.identityToString(esk.identity) - + "\" " + facetPathToString(esk.facet)); - } - - Facet facet = new Facet(elt); - facet.status = clean; - facet.rec = unmarshalValue(value.get_data(), _communicator); - facet.path = esk.facet; - assert(facet.path != null); - elt.facets.put(new StringArray(esk.facet), facet); - - if(esk.facet.length == 0) - { - identities.add(esk.identity); - elt.mainObject = facet; - } } - rs = dbc.get(key, value, com.sleepycat.db.Db.DB_NEXT); - } - while(rs == 0 && (root == null || startWith(key.get_data(), root))); - - if(root != null) - { - buildFacetMap(elt.facets); - evictorElements.add(elt); - } - return (rs == 0); + } while(_saveNowThreads.contains(myself)); } - - boolean - load(com.sleepycat.db.Dbc dbc, com.sleepycat.db.Dbt key, - com.sleepycat.db.Dbt value, java.util.List identities) - throws com.sleepycat.db.DbException + + private void + evict() { - byte[] root = null; - int rs = 0; - do - { - if(root == null) - { - EvictorStorageKey esk = unmarshalKey(key.get_data(), _communicator); - - if(esk.facet.length == 0) - { - // - // Good, we found a main object - // - root = marshalRootKey(esk.identity, _communicator); - - if(_trace >= 3) - { - _communicator.getLogger().trace - ("Freeze.Evictor", - "Iterator read \"" + Ice.Util.identityToString(esk.identity) - + "\""); - } - identities.add(esk.identity); - } - else - { - // - // Otherwise, skip this orphan facet (could be a temporary - // inconsistency on disk) - // + assert Thread.holdsLock(this); - if(_trace >= 3) - { - _communicator.getLogger().trace - ("Freeze.Evictor", - "Iterator is skipping orphan facet \"" + Ice.Util.identityToString(esk.identity) - + "\" " + facetPathToString(esk.facet)); - } - } - } - rs = dbc.get(key, value, com.sleepycat.db.Db.DB_NEXT); - } - while(rs == 0 && (root == null || startWith(key.get_data(), root))); - return (rs == 0); - } - - void - insert(java.util.List identities, java.util.List evictorElements, int loadedGeneration) - { - assert(identities.size() == evictorElements.size()); - - int size = identities.size(); - - if(size > 0) + java.util.Iterator p = _evictorList.riterator(); + while(p.hasNext() && _currentEvictorSize > _evictorSize) { - synchronized(this) + // + // Get the last unused element from the evictor queue. + // + EvictorElement element = (EvictorElement)p.next(); + if(element.usageCount == 0) { - if(_deactivated) - { - throw new EvictorDeactivatedException(); - } + // + // Fine, servant is not in use (and not in the modifiedQueue) + // + + assert !element.stale; - if(_generation == loadedGeneration) + if(_trace >= 2 || (_trace >= 1 && _evictorList.size() % 50 == 0)) { - for(int i = 0; i < size; ++i) + String objString = "object \"" + Ice.Util.identityToString(element.identity) + "\""; + String facet = element.store.facet(); + if(facet.length() > 0) { - Ice.Identity ident = (Ice.Identity) identities.get(i); - - EvictorElement element = (EvictorElement)_evictorMap.get(ident); - - if(element == null) - { - element = insertElement(null, ident, (EvictorElement) evictorElements.get(i)); - } + objString += " with facet \"" + facet + "\""; } - } + + _communicator.getLogger().trace( + "Freeze.Evictor", + "evicting " + objString + " from the queue; " + "number of elements in the queue: " + + _currentEvictorSize); + } + // - // Otherwise we don't insert them + // Remove last unused element from the evictor queue. // + element.stale = true; + element.store.cache().unpin(element.identity); + p.remove(); + _currentEvictorSize--; } } } - boolean - deadlockWarning() - { - return _deadlockWarning; - } - - static byte[] - marshalRootKey(Ice.Identity v, Ice.Communicator communicator) - { - IceInternal.BasicStream os = new IceInternal.BasicStream(Ice.Util.getInstance(communicator)); - try - { - v.__write(os); - java.nio.ByteBuffer buf = os.prepareWrite(); - byte[] r = new byte[buf.limit()]; - buf.get(r); - return r; - } - finally - { - os.destroy(); - } - } - - static byte[] - marshalKey(EvictorStorageKey v, Ice.Communicator communicator) - { - IceInternal.BasicStream os = new IceInternal.BasicStream(Ice.Util.getInstance(communicator)); - try - { - v.__write(os); - java.nio.ByteBuffer buf = os.prepareWrite(); - byte[] r = new byte[buf.limit()]; - buf.get(r); - return r; - } - finally - { - os.destroy(); - } - } - - static EvictorStorageKey - unmarshalKey(byte[] b, Ice.Communicator communicator) - { - IceInternal.BasicStream is = new IceInternal.BasicStream(Ice.Util.getInstance(communicator)); - try - { - is.resize(b.length, true); - java.nio.ByteBuffer buf = is.prepareRead(); - buf.position(0); - buf.put(b); - buf.position(0); - EvictorStorageKey key = new EvictorStorageKey(); - key.__read(is); - return key; - } - finally - { - is.destroy(); - } - } - - static byte[] - marshalValue(ObjectRecord v, Ice.Communicator communicator) - { - IceInternal.BasicStream os = new IceInternal.BasicStream(Ice.Util.getInstance(communicator)); - os.marshalFacets(false); - try - { - os.startWriteEncaps(); - v.__write(os); - os.writePendingObjects(); - os.endWriteEncaps(); - java.nio.ByteBuffer buf = os.prepareWrite(); - byte[] r = new byte[buf.limit()]; - buf.get(r); - return r; - } - finally - { - os.destroy(); - } - } - - static ObjectRecord - unmarshalValue(byte[] b, Ice.Communicator communicator) - { - IceInternal.BasicStream is = new IceInternal.BasicStream(Ice.Util.getInstance(communicator)); - is.sliceObjects(false); - try - { - is.resize(b.length, true); - java.nio.ByteBuffer buf = is.prepareRead(); - buf.position(0); - buf.put(b); - buf.position(0); - ObjectRecord rec= new ObjectRecord(); - is.startReadEncaps(); - rec.__read(is); - is.readPendingObjects(); - is.endReadEncaps(); - return rec; - } - finally - { - is.destroy(); - } - } - - private void - init(String envName, boolean createDb) + + private void + fixEvictPosition(EvictorElement element) { - _trace = _communicator.getProperties().getPropertyAsInt("Freeze.Trace.Evictor"); - _deadlockWarning = _communicator.getProperties().getPropertyAsInt("Freeze.Warn.Deadlocks") != 0; - - _errorPrefix = "Freeze Evictor DbEnv(\"" + envName + "\") Db(\"" + _dbName + "\"): "; - - String propertyPrefix = "Freeze.Evictor." + envName + '.' + _dbName; - - // - // By default, we save every minute or when the size of the modified - // queue reaches 10. - // - - _saveSizeTrigger = _communicator.getProperties().getPropertyAsIntWithDefault - (propertyPrefix + ".SaveSizeTrigger", 10); - - _savePeriod = _communicator.getProperties().getPropertyAsIntWithDefault - (propertyPrefix + ".SavePeriod", 60 * 1000); + assert Thread.holdsLock(this); - // - // By default, we save at most 10 * SaveSizeTrigger objects per transaction - // - _maxTxSize = _communicator.getProperties().getPropertyAsIntWithDefault - (propertyPrefix + ".MaxTxSize", 10 * _saveSizeTrigger); - - if(_maxTxSize <= 0) + assert !element.stale; + if(element.usageCount < 0) { - _maxTxSize = 100; - } - - boolean populateEmptyIndices = (_communicator.getProperties().getPropertyAsIntWithDefault - (propertyPrefix + ".PopulateEmptyIndices", 0) != 0); - - com.sleepycat.db.DbTxn txn = null; - - try - { - - _db = new com.sleepycat.db.Db(_dbEnv, 0); - - txn = _dbEnv.txn_begin(null, 0); + assert element.evictPosition == null; // - // TODO: FREEZE_DB_MODE + // New object // - int flags = 0; - if(createDb) - { - flags |= com.sleepycat.db.Db.DB_CREATE; - } - _db.open(txn, _dbName, null, com.sleepycat.db.Db.DB_BTREE, flags, 0); - - if(_indices != null) - { - for(int i = 0; i < _indices.length; ++i) - { - _indices[i].associate(this, txn, createDb, populateEmptyIndices); - } - } - - com.sleepycat.db.DbTxn toCommit = txn; - txn = null; - toCommit.commit(0); + element.usageCount = 0; + _currentEvictorSize++; } - catch(java.io.FileNotFoundException dx) - { - NotFoundException ex = new NotFoundException(); - ex.initCause(dx); - ex.message = _errorPrefix + "Db.open: " + dx.getMessage(); - throw ex; - } - catch(com.sleepycat.db.DbException dx) - { - DatabaseException ex = new DatabaseException(); - ex.initCause(dx); - ex.message = _errorPrefix + "Db.open: " + dx.getMessage(); - throw ex; - } - finally + else { - if(txn != null) - { - try - { - txn.abort(); - } - catch(com.sleepycat.db.DbException dx) - { - } - } + assert element.evictPosition != null; + element.evictPosition.remove(); } - + _evictorList.addFirst(element); + element.evictPosition = _evictorList.iterator(); // - // Start saving thread + // Position the iterator "on" the element. // - String threadName; - String programName = _communicator.getProperties().getProperty("Ice.ProgramName"); - if(programName.length() > 0) - { - threadName = programName + "-"; - } - else - { - threadName = ""; - } - threadName += "FreezeEvictorThread(" + envName + '.' + _dbName + ")"; - _thread = new Thread(this, threadName); - _thread.start(); + element.evictPosition.next(); } private void - evict() - { - java.util.Iterator p = _evictorList.riterator(); - while(p.hasNext() && _evictorList.size() > _evictorSize) - { - // - // Get the last unused element from the evictor queue. - // - Ice.Identity ident = (Ice.Identity)p.next(); - EvictorElement element = (EvictorElement)_evictorMap.get(ident); - assert(element != null); - if(element.usageCount == 0) - { - // - // Fine, servant is not in use. - // - assert(ident != null && element != null); - - // - // Remove element from the evictor queue. - // - p.remove(); - _evictorMap.remove(ident); - - if(_trace >= 2 || (_trace >= 1 && _evictorList.size() % 50 == 0)) - { - _communicator.getLogger().trace( - "Freeze.Evictor", - "evicted \"" + Ice.Util.identityToString(ident) + - "\" from the queue; " + "number of elements in the queue: " + - _evictorMap.size()); - } - } - } - } - - private boolean - dbHasObject(Ice.Identity ident) + evict(EvictorElement element) { - EvictorStorageKey esk = new EvictorStorageKey(); - esk.identity = ident; - esk.facet = null; - - byte[] key = marshalKey(esk, _communicator); + assert Thread.holdsLock(this); - com.sleepycat.db.Dbt dbKey = new com.sleepycat.db.Dbt(key); - - // - // Keep 0 length since we're not interested in the data - // - com.sleepycat.db.Dbt dbValue = new com.sleepycat.db.Dbt(); - dbValue.set_flags(com.sleepycat.db.Db.DB_DBT_PARTIAL); + assert !element.stale; - for(;;) - { - try - { - int err = _db.get(null, dbKey, dbValue, 0); - - if(err == 0) - { - return true; - } - else if(err == com.sleepycat.db.Db.DB_NOTFOUND) - { - return false; - } - else - { - throw new DatabaseException(); - } - } - catch(com.sleepycat.db.DbDeadlockException deadlock) - { - if(_deadlockWarning) - { - _communicator.getLogger().warning - ("Deadlock in Freeze.EvictorI.dhHasObject while reading Db \"" + _dbName - + "\"; retrying ..."); - } - - // - // Ignored, try again - // - } - catch(com.sleepycat.db.DbException dx) - { - DatabaseException ex = new DatabaseException(); - ex.initCause(dx); - ex.message = _errorPrefix + "Db.get: " + dx.getMessage(); - throw ex; - } - } + element.evictPosition.remove(); + _currentEvictorSize--; + element.stale = true; + element.store.cache().unpin(element.identity); } + private void - addToModifiedQueue(Facet facet) + addToModifiedQueue(EvictorElement element) { - facet.element.usageCount++; - _modifiedQueue.add(facet); + assert Thread.holdsLock(this); + + element.usageCount++; + _modifiedQueue.add(element); if(_saveSizeTrigger >= 0 && _modifiedQueue.size() >= _saveSizeTrigger) { @@ -1827,650 +1324,165 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable } private StreamedObject - streamFacet(Facet facet, byte status, long streamStart) + stream(EvictorElement element, long streamStart) { - StreamedObject obj = new StreamedObject(); - EvictorStorageKey esk = new EvictorStorageKey(); - esk.identity = facet.element.identity; - esk.facet = facet.path; - obj.key = marshalKey(esk, _communicator); - obj.status = status; - if(status != destroyed) - { - obj.value = writeObjectRecordToValue(streamStart, facet.rec); - } - return obj; - } + assert Thread.holdsLock(element); + assert element.status != EvictorElement.dead; - private void - saveNowNoSync() - { - checkSavingThread(); - - Thread myself = Thread.currentThread(); + StreamedObject obj = new StreamedObject(); - _saveNowThreads.add(myself); - notifyAll(); - do + obj.status = element.status; + obj.store = element.store; + obj.key = ObjectStore.marshalKey(element.identity, _communicator); + + if(element.status != EvictorElement.destroyed) { - try + // + // Update stats first + // + Statistics stats = element.rec.stats; + long diff = streamStart - (stats.creationTime + stats.lastSaveTime); + if(stats.lastSaveTime == 0) { - // - // The timeout is to wake up in the event the saving thread - // dies. - // - wait(15 * 1000); + stats.lastSaveTime = diff; + stats.avgSaveTime = diff; } - catch(InterruptedException ex) + else { + stats.lastSaveTime = streamStart - stats.creationTime; + stats.avgSaveTime = (long)(stats.avgSaveTime * 0.95 + diff * 0.05); } - checkSavingThread(); - } while(_saveNowThreads.contains(myself)); - } - - private void - checkSavingThread() - { - if(!_thread.isAlive()) - { - DatabaseException ex = new DatabaseException(); - ex.message = _errorPrefix + "saving thread is dead"; - throw ex; + obj.value = ObjectStore.marshalValue(element.rec, _communicator); } + return obj; } - - private byte[] - writeObjectRecordToValue(long streamStart, ObjectRecord rec) - { - // - // Update stats first - // - Statistics stats = rec.stats; - long diff = streamStart - (stats.creationTime + stats.lastSaveTime); - if(stats.lastSaveTime == 0) - { - stats.lastSaveTime = diff; - stats.avgSaveTime = diff; - } - else - { - stats.lastSaveTime = streamStart - stats.creationTime; - stats.avgSaveTime = (long)(stats.avgSaveTime * 0.95 + diff * 0.05); - } - return marshalValue(rec, _communicator); - } - - private EvictorElement - load(Ice.Identity ident) + + private synchronized ObjectStore + findStore(String facet) { - // - // This method attempts to restore an object and all of its facets from the database. It works by - // iterating over the database keys that match the "root" key. The root key is the encoded portion - // of the EvictorStorageKey struct that the object and its facets all have in common, namely the - // identity. - // - byte[] root = marshalRootKey(ident, _communicator); - - com.sleepycat.db.Dbt dbKey; - com.sleepycat.db.Dbt dbValue = new com.sleepycat.db.Dbt(); - - EvictorElement result = null; - for(;;) + if(_deactivated) { - result = new EvictorElement(); - - com.sleepycat.db.Dbc dbc = null; - - try - { - dbc = _db.cursor(null, 0); - - // - // Get first pair - // - dbKey = new com.sleepycat.db.Dbt(root); - int rs = dbc.get(dbKey, dbValue, com.sleepycat.db.Db.DB_SET_RANGE); - - while(rs == 0 && startWith(dbKey.get_data(), root)) - { - // - // Unmarshal key and data and insert it into result's facet map - // - EvictorStorageKey esk = unmarshalKey(dbKey.get_data(), _communicator); - - Facet facet = new Facet(result); - facet.status = clean; - facet.rec = unmarshalValue(dbValue.get_data(), _communicator); - facet.path = esk.facet; - result.facets.put(new StringArray(esk.facet), facet); - if(esk.facet.length == 0) - { - result.mainObject = facet; - } - - // - // Next facet - // - rs = dbc.get(dbKey, dbValue, com.sleepycat.db.Db.DB_NEXT); - } - - break; // for (;;) - } - catch(com.sleepycat.db.DbDeadlockException deadlock) - { - if(_deadlockWarning) - { - _communicator.getLogger().warning - ("Deadlock in Freeze.EvictorI.load while iterating over Db \"" + _dbName - + "\"; retrying ..."); - } - - // - // Ignored, try again - // - } - catch(com.sleepycat.db.DbException dx) - { - DatabaseException ex = new DatabaseException(); - ex.initCause(dx); - ex.message = _errorPrefix + "Db.get: " + dx.getMessage(); - throw ex; - } - finally - { - if(dbc != null) - { - try - { - dbc.close(); - dbc = null; - } - catch(com.sleepycat.db.DbException dx) - { - } - } - } - } - - if(result.facets.size() == 0) - { - if(_trace >= 2) - { - _communicator.getLogger().trace( - "Freeze.Evictor", - "could not find \"" + Ice.Util.identityToString(ident) + - "\" in the database"); - } - return null; + throw new EvictorDeactivatedException(); } - - // - // Let's fix-up the facets tree in result - // - buildFacetMap(result.facets); - return result; + return (ObjectStore) _storeMap.get(facet); } - private void - buildFacetMap(java.util.Map facets) - { - java.util.Iterator p = facets.entrySet().iterator(); - - while(p.hasNext()) - { - java.util.Map.Entry entry = (java.util.Map.Entry) p.next(); - - String[] path = ((StringArray) entry.getKey()).array; - - if(path.length > 0) - { - String[] parent = new String[path.length - 1]; - System.arraycopy(path, 0, parent, 0, path.length - 1); - Facet parentFacet = (Facet) facets.get(new StringArray(parent)); - if(parentFacet != null) - { - Facet childFacet = (Facet) entry.getValue(); - parentFacet.rec.servant.ice_addFacet(childFacet.rec.servant, path[path.length - 1]); - } - // - // otherwise skip disconnected facet (could be a temporary inconsistency on disk) - // - - } - } - } - - private EvictorElement - insertElement(Ice.ObjectAdapter adapter, Ice.Identity ident, EvictorElement element) - { - if(_initializer != null) - { - _initializer.initialize(adapter, ident, element.mainObject.rec.servant); - } - - _evictorMap.put(ident, element); - _evictorList.addFirst(ident); - - element.position = _evictorList.iterator(); - // - // Position the iterator "on" the element. - // - element.position.next(); - - element.identity = ident; - return element; - } - private void - addFacetImpl(EvictorElement element, Ice.Object servant, String[] facetPath, boolean replacing) + private java.util.List + allDbs() { - java.util.Map facets = element.facets; + java.util.List result = new java.util.LinkedList(); - boolean insertIt = true; + com.sleepycat.db.Db db = null; + com.sleepycat.db.Dbc dbc = null; - StringArray facetPathArray = new StringArray(facetPath); - - if(replacing) + try { - Facet facet = (Facet) facets.get(facetPathArray); - if(facet != null) + db = new com.sleepycat.db.Db(_dbEnv, 0); + db.open(null, _filename, null, com.sleepycat.db.Db.DB_UNKNOWN, com.sleepycat.db.Db.DB_RDONLY, 0); + + dbc = db.cursor(null, 0); + + com.sleepycat.db.Dbt key = new com.sleepycat.db.Dbt(); + key.set_flags(com.sleepycat.db.Db.DB_DBT_MALLOC); + + com.sleepycat.db.Dbt value = new com.sleepycat.db.Dbt(); + value.set_flags(com.sleepycat.db.Db.DB_DBT_MALLOC); + + boolean more = true; + while(more) { - synchronized(facet) + more = (dbc.get(key, value, com.sleepycat.db.Db.DB_NEXT) == 0); + if(more) { - switch(facet.status) + // + // Assumes Berkeley-DB encodes the db names in UTF-8! + // + String dbName = new String(key.get_data(), 0, key.get_size(), "UTF8"); + + if(!dbName.startsWith(indexPrefix)) { - case clean: - { - if(_trace >= 3) - { - _communicator.getLogger().trace( - "Freeze.Evictor", - "addFacetImpl \"" + Ice.Util.identityToString(element.identity) + - "\" " + facetPathToString(facetPath) + ": clean -> modified"); - } - - facet.status = modified; - addToModifiedQueue(facet); - break; - } - case created: - case modified: - { - if(_trace >= 3) - { - _communicator.getLogger().trace( - "Freeze.Evictor", - "addFacetImpl \"" + Ice.Util.identityToString(element.identity) + - "\" " + facetPathToString(facetPath) + ": created or modified (unchanged)"); - } - - // - // Nothing to do. - // No need to push it on the modified queue as a created resp - // modified facet is either already on the queue or about - // to be saved. When saved, it becomes clean. - // - break; - } - case destroyed: - { - if(_trace >= 3) - { - _communicator.getLogger().trace( - "Freeze.Evictor", - "addFacetImpl \"" + Ice.Util.identityToString(element.identity) + - "\" " + facetPathToString(facetPath) + ": destroyed -> modified"); - } - - facet.status = modified; - // - // No need to push it on the modified queue, as a destroyed facet - // is either already on the queue or about to be saved. When saved, - // it becomes dead. - // - break; - } - case dead: - { - if(_trace >= 3) - { - _communicator.getLogger().trace( - "Freeze.Evictor", - "addFacetImpl \"" + Ice.Util.identityToString(element.identity) + - "\" " + facetPathToString(facetPath) + ": dead -> created"); - } - - facet.status = created; - addToModifiedQueue(facet); - break; - } - default: - { - assert(false); - break; - } + result.add(dbName); } - facet.rec.servant = servant; - insertIt = false; } } + + dbc.close(); + dbc = null; + db.close(0); + db = null; } - - if(insertIt) + catch(java.io.UnsupportedEncodingException ix) { - if(_trace >= 3) - { - _communicator.getLogger().trace( - "Freeze.Evictor", - "addFacetImpl \"" + Ice.Util.identityToString(element.identity) + - "\" " + facetPathToString(facetPath) + ": new facet (created)"); - } - - Facet facet = new Facet(element); - facet.status = created; - facet.path = facetPath; - - facet.rec = new ObjectRecord(); - ObjectRecord rec = facet.rec; - rec.servant = servant; - rec.stats = new Statistics(); - rec.stats.creationTime = System.currentTimeMillis(); - rec.stats.lastSaveTime = 0; - rec.stats.avgSaveTime = 0; - - facets.put(facetPathArray, facet); - if(facetPath.length == 0) - { - element.mainObject = facet; - } - addToModifiedQueue(facet); + DatabaseException ex = new DatabaseException(); + ex.initCause(ix); + ex.message = _errorPrefix + "cannot decode database names"; + throw ex; } - - if(servant != null) + catch(java.io.FileNotFoundException ix) { // - // Add servant's facets + // New file // - String[] facetList = servant.ice_facets(null); - for(int i = 0; i < facetList.length; i++) - { - String[] newFacetPath = new String[facetPath.length + 1]; - System.arraycopy(facetPath, 0, newFacetPath, 0, facetPath.length); - String currentName = facetList[i]; - newFacetPath[newFacetPath.length - 1] = currentName; - addFacetImpl(element, servant.ice_findFacet(currentName), newFacetPath, replacing); - } - } - } - - private void - removeFacetImpl(java.util.Map facets, String[] facetPath) - { - Facet facet = (Facet) facets.get(new StringArray(facetPath)); - Ice.Object servant = null; - - if(facet != null) - { - servant = destroyFacetImpl(facet); } - // - // else should we raise an exception? - // - - if(servant != null) + catch(com.sleepycat.db.DbException dx) { - // - // Remove servant's facets - // - String[] facetList = servant.ice_facets(null); - for(int i = 0; i < facetList.length; i++) - { - String[] newFacetPath = new String[facetPath.length + 1]; - System.arraycopy(facetPath, 0, newFacetPath, 0, facetPath.length); - String currentName = facetList[i]; - newFacetPath[newFacetPath.length - 1] = currentName; - removeFacetImpl(facets, newFacetPath); - } + DatabaseException ex = new DatabaseException(); + ex.initCause(dx); + ex.message = _errorPrefix + "Db.open: " + dx.getMessage(); + throw ex; } - } - - private Ice.Object - destroyFacetImpl(Facet facet) - { - synchronized(facet) + finally { - switch(facet.status) + if(dbc != null) { - case clean: - { - if(_trace >= 3) - { - _communicator.getLogger().trace( - "Freeze.Evictor", - "destroyFacetImpl \"" + Ice.Util.identityToString(facet.element.identity) + - "\" " + facetPathToString(facet.path) + ": clean -> destroyed"); - } - - facet.status = destroyed; - addToModifiedQueue(facet); - break; - } - case created: - { - if(_trace >= 3) - { - _communicator.getLogger().trace( - "Freeze.Evictor", - "destroyFacetImpl \"" + Ice.Util.identityToString(facet.element.identity) + - "\" " + facetPathToString(facet.path) + ": created -> dead"); - } - - facet.status = dead; - break; - } - case modified: - { - if(_trace >= 3) - { - _communicator.getLogger().trace( - "Freeze.Evictor", - "destroyFacetImpl \"" + Ice.Util.identityToString(facet.element.identity) + - "\" " + facetPathToString(facet.path) + ": modified -> destroyed"); - } - - - facet.status = destroyed; - // - // Not necessary to push it on the modified queue, as a modified - // element is either on the queue already or about to be saved - // (at which point it becomes clean) - // - break; - } - case destroyed: - case dead: + try { - if(_trace >= 3) - { - _communicator.getLogger().trace( - "Freeze.Evictor", - "destroyFacetImpl \"" + Ice.Util.identityToString(facet.element.identity) + - "\" " + facetPathToString(facet.path) + ": was already dead or destroyed"); - } - - // - // Nothing to do! - // - break; + dbc.close(); } - default: + catch(com.sleepycat.db.DbException dx) { - assert(false); - break; + // Ignored } } - return facet.rec.servant; - } - } - - - private boolean - startWith(byte[] key, byte[] root) - { - if(key.length >= root.length) - { - for(int i = 0; i < root.length; i++) + if(db != null) { - if(key[i] != root[i]) + try { - return false; + db.close(0); } - } - return true; - } - else - { - return false; - } - } - - class StreamedObject - { - byte[] key; - byte[] value; - byte status; - } - - class EvictorElement - { - java.util.Iterator position; - int usageCount = 0; - java.util.Map facets = new java.util.HashMap(); - Ice.Identity identity; - Facet mainObject; - } - - class Facet extends Ice.LocalObjectImpl - { - Facet(EvictorElement evictorElement) - { - element = evictorElement; - } - byte status; - ObjectRecord rec; - EvictorElement element; - String[] path; - } - - // - // Wrapper to use a String[] as key of a HashMap. - // - class StringArray - { - StringArray(String[] a) - { - assert(a != null); - array = a; - } - - public boolean equals(java.lang.Object o) - { - if(o instanceof StringArray) - { - StringArray rhs = (StringArray) o; - if(rhs.array.length == array.length) + catch(com.sleepycat.db.DbException dx) { - for(int i = 0; i < array.length; i++) - { - if(!array[i].equals(rhs.array[i])) - { - return false; - } - } - return true; + // Ignored } } - return false; - } - - public int hashCode() - { - int result = 0; - for(int i = 0; i < array.length; i++) - { - result ^= array[i].hashCode(); - } - return result; } - - String[] array; - } - - static String - facetPathToString(String[] facetPath) - { - String result = ""; - if(facetPath.length == 0) - { - result = "(main object)"; - } - else - { - for(int i = 0; i < facetPath.length - 1; ++i) - { - result += facetPath[i] + '/'; - } - result += facetPath[facetPath.length - 1]; - } return result; } + + static class StreamedObject + { + byte[] key = null; + byte[] value = null; + byte status = EvictorElement.dead; + ObjectStore store = null; + } // - // Clean object; can become modified or destroyed - // - private static final byte clean = 0; - - // - // New objects; can become clean, dead or destroyed - // - private static final byte created = 1; - - // - // Modified object; can become clean or destroyed - // - private static final byte modified = 2; - - // - // Being saved. Can become dead or created - // - private static final byte destroyed = 3; - - // - // Exists only in the Evictor; for example the object was created - // and later destroyed (without a save in between), or it was - // destroyed on disk but is still in use. Can become created. + // Map of string (facet) to ObjectStore // - private static final byte dead = 4; - + private final java.util.Map _storeMap = new java.util.HashMap(); + // - // Map of Ice.Identity to EvictorElement + // List of EvictorElement with stable iterators // - private java.util.Map _evictorMap = new java.util.HashMap(); + private final Freeze.LinkedList _evictorList = new Freeze.LinkedList(); private int _evictorSize = 10; - - // - // The C++ Evictor uses std::list<EvictorMap::iterator> which allows - // holding of iterators across list changes. Unfortunately, Java - // iterators are invalidated as soon as the underlying collection - // is changed, so it's not possible to use the same technique. - // - // This is a list of Ice.Identity. - // - private Freeze.LinkedList _evictorList = new Freeze.LinkedList(); + private int _currentEvictorSize = 0; // // The _modifiedQueue contains a queue of all modified facets @@ -2481,41 +1493,32 @@ class EvictorI extends Ice.LocalObjectImpl implements Evictor, Runnable private boolean _deactivated = false; - private Ice.Communicator _communicator; - private SharedDbEnv _dbEnvHolder; + private final Ice.ObjectAdapter _adapter; + private final Ice.Communicator _communicator; + + private final ServantInitializer _initializer; + + private SharedDbEnv _dbEnvHolder; private com.sleepycat.db.DbEnv _dbEnv; - private com.sleepycat.db.Db _db; - private String _dbName; - private Index[] _indices; - private ServantInitializer _initializer; + + private final String _filename; + private final boolean _createDb; + private int _trace = 0; - private boolean _deadlockWarning; - + // // Threads that have requested a "saveNow" and are waiting for // its completion // - private java.util.List _saveNowThreads = new java.util.ArrayList(); + private final java.util.List _saveNowThreads = new java.util.ArrayList(); private int _saveSizeTrigger; private int _maxTxSize; - private long _savePeriod; private Thread _thread; + private String _errorPrefix; - // - // _generation is incremented after committing changes - // to disk, when releasing the usage count of the element - // that contains the created/modified/destroyed facets. - // Like the usage count, it is protected by the Evictor mutex. - // - // It is used to detect updates when loading an element and its - // facets without holding the Evictor mutex. If the generation - // is the same before the loading and later when the Evictor - // mutex is locked again, and the map still does not contain - // this element, then the loaded value is current. - // - private int _generation = 0; + private boolean _deadlockWarning; } diff --git a/java/src/Freeze/EvictorIteratorI.java b/java/src/Freeze/EvictorIteratorI.java index 22ed67d245a..81aeaa2ff41 100644 --- a/java/src/Freeze/EvictorIteratorI.java +++ b/java/src/Freeze/EvictorIteratorI.java @@ -43,13 +43,13 @@ class EvictorIteratorI extends Ice.LocalObjectImpl implements EvictorIterator } } - EvictorIteratorI(EvictorI evictor, int batchSize, boolean loadServants) + EvictorIteratorI(ObjectStore store, int batchSize) { - _evictor = evictor; + _store = store; + _more = (store != null); _batchSize = batchSize; - _loadServants = loadServants; - assert(batchSize > 0); + assert batchSize > 0; // // We should use DB_DBT_REALLOC, but it's buggy in 4.1.25 @@ -57,17 +57,10 @@ class EvictorIteratorI extends Ice.LocalObjectImpl implements EvictorIterator // _key.set_flags(com.sleepycat.db.Db.DB_DBT_MALLOC); - if(_loadServants) - { - _value.set_flags(com.sleepycat.db.Db.DB_DBT_MALLOC); - } - else - { - // - // dlen is 0, so we should not retrieve any value - // - _value.set_flags(com.sleepycat.db.Db.DB_DBT_PARTIAL); - } + // + // dlen is 0, so we should not retrieve any value + // + _value.set_flags(com.sleepycat.db.Db.DB_DBT_PARTIAL); } private java.util.Iterator @@ -80,7 +73,7 @@ class EvictorIteratorI extends Ice.LocalObjectImpl implements EvictorIterator java.util.List evictorElements = null; - Ice.Communicator communicator = _evictor.communicator(); + Ice.Communicator communicator = _store.communicator(); byte[] firstKey = null; if(_key.get_size() > 0) @@ -88,9 +81,7 @@ class EvictorIteratorI extends Ice.LocalObjectImpl implements EvictorIterator firstKey = new byte[_key.get_size()]; System.arraycopy(_key.get_data(), 0, firstKey, 0, firstKey.length); } - - int loadedGeneration = 0; - + try { for(;;) @@ -98,11 +89,7 @@ class EvictorIteratorI extends Ice.LocalObjectImpl implements EvictorIterator com.sleepycat.db.Dbc dbc = null; _batch = new java.util.ArrayList(); - if(_loadServants) - { - evictorElements = new java.util.ArrayList(); - } - + try { // @@ -117,28 +104,34 @@ class EvictorIteratorI extends Ice.LocalObjectImpl implements EvictorIterator // flags = com.sleepycat.db.Db.DB_SET_RANGE; } + + dbc = _store.db().cursor(null, 0); - if(_loadServants) + boolean done = false; + do { - loadedGeneration = _evictor.currentGeneration(); - } - dbc = _evictor.db().cursor(null, 0); - _more = (dbc.get(_key, _value, flags) == 0); + _more = (dbc.get(_key, _value, flags) == 0); - while(_batch.size() < _batchSize && _more) - { - // - // Even when count is 0, we read one more record (unless we reach the end) - // - if(_loadServants) + if(_more) { - _more = _evictor.load(dbc, _key, _value, _batch, evictorElements); - } - else - { - _more = _evictor.load(dbc, _key, _value, _batch); + flags = com.sleepycat.db.Db.DB_NEXT; + + if(_batch.size() < _batchSize) + { + Ice.Identity ident = ObjectStore.unmarshalKey(_key.get_data(), communicator); + _batch.add(ident); + } + else + { + // + // Keep the last element in _key + // + done = true; + } } } + while(!done && _more); + break; // for (;;) } catch(com.sleepycat.db.DbDeadlockException dx) @@ -154,10 +147,11 @@ class EvictorIteratorI extends Ice.LocalObjectImpl implements EvictorIterator _key.set_size(0); } - if(_evictor.deadlockWarning()) + if(_store.evictor().deadlockWarning()) { communicator.getLogger().warning - ("Deadlock in Freeze.EvictorIteratorI.load while iterating over Db \"" + _evictor.dbName() + ("Deadlock in Freeze.EvictorIteratorI.load while iterating over Db \"" + + _store.evictor().filename() + "/" + _store.dbName() + "\"; retrying ..."); } @@ -187,7 +181,7 @@ class EvictorIteratorI extends Ice.LocalObjectImpl implements EvictorIterator { DatabaseException ex = new DatabaseException(); ex.initCause(dx); - ex.message = _evictor.errorPrefix() + "Db.cursor: " + dx.getMessage(); + ex.message = _store.evictor().errorPrefix() + "Db.cursor: " + dx.getMessage(); throw ex; } @@ -197,22 +191,16 @@ class EvictorIteratorI extends Ice.LocalObjectImpl implements EvictorIterator } else { - if(_loadServants) - { - _evictor.insert(_batch, evictorElements, loadedGeneration); - } - return _batch.listIterator(); } } - private EvictorI _evictor; - private int _batchSize; - private boolean _loadServants; + private final ObjectStore _store; + private final int _batchSize; private java.util.Iterator _batchIterator; - private com.sleepycat.db.Dbt _key = new com.sleepycat.db.Dbt(); - private com.sleepycat.db.Dbt _value = new com.sleepycat.db.Dbt(); + private final com.sleepycat.db.Dbt _key = new com.sleepycat.db.Dbt(); + private final com.sleepycat.db.Dbt _value = new com.sleepycat.db.Dbt(); private java.util.List _batch = null; private boolean _more = true; } diff --git a/java/src/Freeze/Index.java b/java/src/Freeze/Index.java index fafcf713e35..d024cdcf233 100644 --- a/java/src/Freeze/Index.java +++ b/java/src/Freeze/Index.java @@ -27,26 +27,25 @@ public abstract class Index implements com.sleepycat.db.DbSecondaryKeyCreate com.sleepycat.db.Dbt value, com.sleepycat.db.Dbt result) { - Ice.Communicator communicator = _evictor.communicator(); + Ice.Communicator communicator = _store.communicator(); - EvictorStorageKey esk = EvictorI.unmarshalKey(key.get_data(), communicator); - if(esk.facet.length == 0) - { - ObjectRecord rec = EvictorI.unmarshalValue(value.get_data(), communicator); + Ice.Identity ident = ObjectStore.unmarshalKey(key.get_data(), communicator); + ObjectRecord rec = ObjectStore.unmarshalValue(value.get_data(), communicator); - byte[] secondaryKey = marshalKey(rec.servant); - if(secondaryKey != null) - { - result.set_data(secondaryKey); - result.set_size(secondaryKey.length); - return 0; - } + byte[] secondaryKey = marshalKey(rec.servant); + if(secondaryKey != null) + { + result.set_data(secondaryKey); + result.set_size(secondaryKey.length); + return 0; + } + else + { + // + // Don't want to index this one + // + return com.sleepycat.db.Db.DB_DONOTINDEX; } - - // - // Don't want to index this one - // - return com.sleepycat.db.Db.DB_DONOTINDEX; } // @@ -61,9 +60,22 @@ public abstract class Index implements com.sleepycat.db.DbSecondaryKeyCreate return secondaryKeyCreate(secondary, key, value, result); } - protected Index(String name) + public String + name() + { + return _name; + } + + public String + facet() + { + return _facet; + } + + protected Index(String name, String facet) { _name = name; + _facet = facet; } protected abstract byte[] @@ -83,8 +95,8 @@ public abstract class Index implements com.sleepycat.db.DbSecondaryKeyCreate // value.set_flags(com.sleepycat.db.Db.DB_DBT_PARTIAL); - Ice.Communicator communicator = _evictor.communicator(); - _evictor.saveNow(); + Ice.Communicator communicator = _store.communicator(); + _store.evictor().saveNow(); java.util.List identities; @@ -101,31 +113,32 @@ public abstract class Index implements com.sleepycat.db.DbSecondaryKeyCreate // Move to the first record // dbc = _db.cursor(null, 0); - boolean more = (dbc.pget(key, pkey, value, com.sleepycat.db.Db.DB_SET) == 0); - - while((firstN <= 0 || identities.size() < firstN) && more) - { - EvictorStorageKey esk = EvictorI.unmarshalKey(pkey.get_data(), communicator); + int flags = com.sleepycat.db.Db.DB_SET; - if(esk.facet.length == 0) + boolean found; + + do + { + found = (dbc.pget(key, pkey, value, flags) == 0); + + if(found) { - identities.add(esk.identity); + Ice.Identity ident = ObjectStore.unmarshalKey(pkey.get_data(), communicator); + identities.add(ident); + flags = com.sleepycat.db.Db.DB_NEXT_DUP; } - // - // Else skip "orphan" facet (could be just a temporary inconsistency - // on disk) - // - - more = (dbc.pget(key, pkey, value, com.sleepycat.db.Db.DB_NEXT_DUP) == 0); } - break; // for (;;) + while((firstN <= 0 || identities.size() < firstN) && found); + + break; // for(;;) } catch(com.sleepycat.db.DbDeadlockException dx) { - if(_evictor.deadlockWarning()) + if(_store.evictor().deadlockWarning()) { - communicator().getLogger().warning - ("Deadlock in Freeze.Index.untypedFindFirst while iterating over Db \"" + _evictor.dbName() + communicator.getLogger().warning + ("Deadlock in Freeze.Index.untypedFindFirst while iterating over Db \"" + + _store.evictor().filename() + "/" + _dbName + "\"; retrying ..."); } @@ -155,7 +168,7 @@ public abstract class Index implements com.sleepycat.db.DbSecondaryKeyCreate { DatabaseException ex = new DatabaseException(); ex.initCause(dx); - ex.message = _evictor.errorPrefix() + "Db.cursor: " + dx.getMessage(); + ex.message = _store.evictor().errorPrefix() + "Db.cursor: " + dx.getMessage(); throw ex; } @@ -185,7 +198,7 @@ public abstract class Index implements com.sleepycat.db.DbSecondaryKeyCreate // dlen is 0, so we should not retrieve any value // value.set_flags(com.sleepycat.db.Db.DB_DBT_PARTIAL); - _evictor.saveNow(); + _store.evictor().saveNow(); try { @@ -208,10 +221,11 @@ public abstract class Index implements com.sleepycat.db.DbSecondaryKeyCreate } catch(com.sleepycat.db.DbDeadlockException dx) { - if(_evictor.deadlockWarning()) + if(_store.evictor().deadlockWarning()) { - communicator().getLogger().warning - ("Deadlock in Freeze.Index.untypedCount while iterating over Db \"" + _evictor.dbName() + _store.communicator().getLogger().warning + ("Deadlock in Freeze.Index.untypedCount while iterating over Db \"" + + _store.evictor().filename() + "/" + _dbName + "\"; retrying ..."); } @@ -241,20 +255,25 @@ public abstract class Index implements com.sleepycat.db.DbSecondaryKeyCreate { DatabaseException ex = new DatabaseException(); ex.initCause(dx); - ex.message = _evictor.errorPrefix() + "Db.cursor: " + dx.getMessage(); + ex.message = _store.evictor().errorPrefix() + "Db.cursor: " + dx.getMessage(); throw ex; } } + protected final Ice.Communicator + communicator() + { + return _store.communicator(); + } void - associate(EvictorI evictor, com.sleepycat.db.DbTxn txn, boolean createDb, boolean populateIndex) + associate(ObjectStore store, com.sleepycat.db.DbTxn txn, boolean createDb, boolean populateIndex) throws com.sleepycat.db.DbException, java.io.FileNotFoundException { assert(txn != null); - _evictor = evictor; + _store = store; - _db= new com.sleepycat.db.Db(evictor.dbEnv(), 0); + _db= new com.sleepycat.db.Db(_store.evictor().dbEnv(), 0); _db.set_flags(com.sleepycat.db.Db.DB_DUP | com.sleepycat.db.Db.DB_DUPSORT); int flags = 0; @@ -262,14 +281,17 @@ public abstract class Index implements com.sleepycat.db.DbSecondaryKeyCreate { flags = com.sleepycat.db.Db.DB_CREATE; } - _db.open(txn, evictor.dbName() + "." + _name, null, com.sleepycat.db.Db.DB_BTREE, flags, 0); + + _dbName = EvictorI.indexPrefix + store.dbName() + "." + _name; + + _db.open(txn, _store.evictor().filename(), _dbName, com.sleepycat.db.Db.DB_BTREE, flags, 0); flags = 0; if(populateIndex) { flags = com.sleepycat.db.Db.DB_CREATE; } - evictor.db().associate(txn, _db, this, flags); + _store.db().associate(txn, _db, this, flags); } void @@ -285,33 +307,17 @@ public abstract class Index implements com.sleepycat.db.DbSecondaryKeyCreate { DatabaseException ex = new DatabaseException(); ex.initCause(dx); - ex.message = _evictor.errorPrefix() + "Db.close: " + dx.getMessage(); + ex.message = _store.evictor().errorPrefix() + "Db.close: " + dx.getMessage(); throw ex; } _db = null; } } + + private final String _name; + private final String _facet; + private String _dbName; - - final com.sleepycat.db.Db - db() - { - return _db; - } - - final EvictorI - evictor() - { - return _evictor; - } - - final protected Ice.Communicator - communicator() - { - return _evictor.communicator(); - } - - private String _name; - private com.sleepycat.db.Db _db; - private EvictorI _evictor; + private com.sleepycat.db.Db _db = null; + private ObjectStore _store = null; } diff --git a/java/src/Freeze/ObjectStore.java b/java/src/Freeze/ObjectStore.java new file mode 100644 index 00000000000..541e2503874 --- /dev/null +++ b/java/src/Freeze/ObjectStore.java @@ -0,0 +1,419 @@ +// ********************************************************************** +// +// Copyright (c) 2004 +// ZeroC, Inc. +// Billerica, MA, USA +// +// All Rights Reserved. +// +// Ice is free software; you can redistribute it and/or modify it under +// the terms of the GNU General Public License version 2 as published by +// the Free Software Foundation. +// +// ********************************************************************** + +package Freeze; + +class ObjectStore implements IceUtil.Store +{ + + ObjectStore(String facet, boolean createDb, EvictorI evictor, + java.util.List indices, boolean populateEmptyIndices) + { + _cache = new IceUtil.Cache(this); + + _facet = facet; + _evictor = evictor; + _indices = indices; + _communicator = evictor.communicator(); + + if(facet.equals("")) + { + _dbName = EvictorI.defaultDb; + } + else + { + _dbName = facet; + } + + com.sleepycat.db.DbTxn txn = null; + com.sleepycat.db.DbEnv dbEnv = evictor.dbEnv(); + + try + { + _db = new com.sleepycat.db.Db(dbEnv, 0); + + txn = dbEnv.txn_begin(null, 0); + + // + // TODO: FREEZE_DB_MODE + // + int flags = 0; + if(createDb) + { + flags |= com.sleepycat.db.Db.DB_CREATE; + } + _db.open(txn, evictor.filename(), _dbName, com.sleepycat.db.Db.DB_BTREE, flags, 0); + + + java.util.Iterator p = _indices.iterator(); + while(p.hasNext()) + { + Index index = (Index) p.next(); + index.associate(this, txn, createDb, populateEmptyIndices); + } + + com.sleepycat.db.DbTxn toCommit = txn; + txn = null; + toCommit.commit(0); + } + catch(java.io.FileNotFoundException dx) + { + NotFoundException ex = new NotFoundException(); + ex.initCause(dx); + ex.message = _evictor.errorPrefix() + "Db.open: " + dx.getMessage(); + throw ex; + } + catch(com.sleepycat.db.DbException dx) + { + DatabaseException ex = new DatabaseException(); + ex.initCause(dx); + ex.message = _evictor.errorPrefix() + "Db.open: " + dx.getMessage(); + throw ex; + } + finally + { + if(txn != null) + { + try + { + txn.abort(); + } + catch(com.sleepycat.db.DbException dx) + { + } + } + } + } + + protected void + finalize() + { + if(_db != null) + { + close(); + } + } + + void + close() + { + try + { + _db.close(0); + + java.util.Iterator p = _indices.iterator(); + while(p.hasNext()) + { + Index index = (Index) p.next(); + index.close(); + } + _indices.clear(); + } + catch(com.sleepycat.db.DbException dx) + { + DatabaseException ex = new DatabaseException(); + ex.initCause(dx); + ex.message = _evictor.errorPrefix() + "Db.close: " + dx.getMessage(); + throw ex; + } + _db = null; + } + + boolean + dbHasObject(Ice.Identity ident) + { + byte[] key = marshalKey(ident, _communicator); + com.sleepycat.db.Dbt dbKey = new com.sleepycat.db.Dbt(key); + + // + // Keep 0 length since we're not interested in the data + // + com.sleepycat.db.Dbt dbValue = new com.sleepycat.db.Dbt(); + dbValue.set_flags(com.sleepycat.db.Db.DB_DBT_PARTIAL); + + for(;;) + { + try + { + int err = _db.get(null, dbKey, dbValue, 0); + + if(err == 0) + { + return true; + } + else if(err == com.sleepycat.db.Db.DB_NOTFOUND) + { + return false; + } + else + { + throw new DatabaseException(); + } + } + catch(com.sleepycat.db.DbDeadlockException deadlock) + { + if(_evictor.deadlockWarning()) + { + _communicator.getLogger().warning + ("Deadlock in Freeze.ObjectStore.dhHasObject while reading Db \"" + + _evictor.filename() + "/" + _dbName + + "\"; retrying ..."); + } + + // + // Ignored, try again + // + } + catch(com.sleepycat.db.DbException dx) + { + DatabaseException ex = new DatabaseException(); + ex.initCause(dx); + ex.message = _evictor.errorPrefix() + "Db.get: " + dx.getMessage(); + throw ex; + } + } + } + + void + save(byte[] key, byte[] value, byte status, com.sleepycat.db.DbTxn tx) + throws com.sleepycat.db.DbException + { + switch(status) + { + case EvictorElement.created: + case EvictorElement.modified: + { + com.sleepycat.db.Dbt dbKey = new com.sleepycat.db.Dbt(key); + com.sleepycat.db.Dbt dbValue = new com.sleepycat.db.Dbt(value); + int flags = (status == EvictorElement.created) ? com.sleepycat.db.Db.DB_NOOVERWRITE : 0; + int err = _db.put(tx, dbKey, dbValue, flags); + if(err != 0) + { + throw new DatabaseException(); + } + break; + } + case EvictorElement.destroyed: + { + com.sleepycat.db.Dbt dbKey = new com.sleepycat.db.Dbt(key); + int err = _db.del(tx, dbKey, 0); + if(err != 0) + { + throw new DatabaseException(); + } + break; + } + default: + { + assert false; + } + } + } + + static byte[] + marshalKey(Ice.Identity v, Ice.Communicator communicator) + { + IceInternal.BasicStream os = new IceInternal.BasicStream(Ice.Util.getInstance(communicator)); + try + { + v.__write(os); + java.nio.ByteBuffer buf = os.prepareWrite(); + byte[] r = new byte[buf.limit()]; + buf.get(r); + return r; + } + finally + { + os.destroy(); + } + } + + static Ice.Identity + unmarshalKey(byte[] b, Ice.Communicator communicator) + { + IceInternal.BasicStream is = new IceInternal.BasicStream(Ice.Util.getInstance(communicator)); + try + { + is.resize(b.length, true); + java.nio.ByteBuffer buf = is.prepareRead(); + buf.position(0); + buf.put(b); + buf.position(0); + Ice.Identity key = new Ice.Identity(); + key.__read(is); + return key; + } + finally + { + is.destroy(); + } + } + + static byte[] + marshalValue(ObjectRecord v, Ice.Communicator communicator) + { + IceInternal.BasicStream os = new IceInternal.BasicStream(Ice.Util.getInstance(communicator)); + try + { + os.startWriteEncaps(); + v.__write(os); + os.writePendingObjects(); + os.endWriteEncaps(); + java.nio.ByteBuffer buf = os.prepareWrite(); + byte[] r = new byte[buf.limit()]; + buf.get(r); + return r; + } + finally + { + os.destroy(); + } + } + + static ObjectRecord + unmarshalValue(byte[] b, Ice.Communicator communicator) + { + IceInternal.BasicStream is = new IceInternal.BasicStream(Ice.Util.getInstance(communicator)); + is.sliceObjects(false); + try + { + is.resize(b.length, true); + java.nio.ByteBuffer buf = is.prepareRead(); + buf.position(0); + buf.put(b); + buf.position(0); + ObjectRecord rec= new ObjectRecord(); + is.startReadEncaps(); + rec.__read(is); + is.readPendingObjects(); + is.endReadEncaps(); + return rec; + } + finally + { + is.destroy(); + } + } + + + final IceUtil.Cache + cache() + { + return _cache; + } + + final com.sleepycat.db.Db + db() + { + return _db; + } + + final Ice.Communicator + communicator() + { + return _communicator; + } + + final EvictorI + evictor() + { + return _evictor; + } + + final String + facet() + { + return _facet; + } + + final String + dbName() + { + return _dbName; + } + + public Object + load(Object identObj) + { + Ice.Identity ident = (Ice.Identity) identObj; + + byte[] key = marshalKey(ident, _communicator); + + com.sleepycat.db.Dbt dbKey = new com.sleepycat.db.Dbt(key); + com.sleepycat.db.Dbt dbValue = new com.sleepycat.db.Dbt(); + + for(;;) + { + try + { + int rs = _db.get(null, dbKey, dbValue, 0); + + if(rs == com.sleepycat.db.Db.DB_NOTFOUND) + { + return null; + } + else if (rs != 0) + { + assert false; + throw new DatabaseException(); + } + break; + } + catch(com.sleepycat.db.DbDeadlockException deadlock) + { + if(_evictor.deadlockWarning()) + { + _communicator.getLogger().warning + ("Deadlock in Freeze.ObjectStore.load while reading Db \"" + + _evictor.filename() + "/" + _dbName + + "\"; retrying ..."); + } + + // + // Ignored, try again + // + } + catch(com.sleepycat.db.DbException dx) + { + DatabaseException ex = new DatabaseException(); + ex.initCause(dx); + ex.message = _evictor.errorPrefix() + "Db.get: " + dx.getMessage(); + throw ex; + } + } + + EvictorElement result = new EvictorElement(ident, this); + result.rec = unmarshalValue(dbValue.get_data(), _communicator); + + _evictor.initialize(ident, _facet, result.rec.servant); + return result; + } + + private final IceUtil.Cache _cache; + + private com.sleepycat.db.Db _db; + private final String _facet; + private final String _dbName; + private final EvictorI _evictor; + private final java.util.List _indices; + private final Ice.Communicator _communicator; + +} + + + + + + + diff --git a/java/src/Freeze/Util.java b/java/src/Freeze/Util.java index da69cc758e1..55e5a8d0d7a 100644 --- a/java/src/Freeze/Util.java +++ b/java/src/Freeze/Util.java @@ -18,18 +18,18 @@ public class Util { public static Evictor - createEvictor(Ice.Communicator communicator, String envName, String dbName, - Index[] indices, boolean createDb) + createEvictor(Ice.ObjectAdapter adapter, String envName, String filename, + ServantInitializer initializer, Index[] indices, boolean createDb) { - return new EvictorI(communicator, envName, dbName, indices, createDb); + return new EvictorI(adapter, envName, filename, initializer, indices, createDb); } public static Evictor - createEvictor(Ice.Communicator communicator, String envName, - com.sleepycat.db.DbEnv dbEnv, String dbName, - Index[] indices, boolean createDb) + createEvictor(Ice.ObjectAdapter adapter, String envName, + com.sleepycat.db.DbEnv dbEnv, String filename, + ServantInitializer initializer, Index[] indices, boolean createDb) { - return new EvictorI(communicator, envName, dbEnv, dbName, indices, createDb); + return new EvictorI(adapter, envName, dbEnv, filename, initializer, indices, createDb); } public static Connection diff --git a/java/src/IceInternal/ReferenceFactory.java b/java/src/IceInternal/ReferenceFactory.java index 9eadfb9a4f8..26472d92ba0 100644 --- a/java/src/IceInternal/ReferenceFactory.java +++ b/java/src/IceInternal/ReferenceFactory.java @@ -440,7 +440,7 @@ public final class ReferenceFactory // For compatibility with the old FacetPath. // String[] facetPath = s.readStringSeq(); - String facet = null; + String facet = ""; if(facetPath.length > 0) // TODO: Throw an exception if facetPath has more than one element? { facet = facetPath[0]; diff --git a/java/src/IceUtil/Cache.java b/java/src/IceUtil/Cache.java new file mode 100644 index 00000000000..feb7a789b53 --- /dev/null +++ b/java/src/IceUtil/Cache.java @@ -0,0 +1,132 @@ +// ********************************************************************** +// +// Copyright (c) 2004 +// ZeroC, Inc. +// Billerica, MA, USA +// +// All Rights Reserved. +// +// Ice is free software; you can redistribute it and/or modify it under +// the terms of the GNU General Public License version 2 as published by +// the Free Software Foundation. +// +// ********************************************************************** + +package IceUtil; + +// +// An abstraction to efficiently populate a Cache, without holding +// a lock while loading from a database. +// +// TODO: implement efficiently! +// + +public class Cache +{ + public Cache(Store store) + { + _store = store; + } + + public Object + getIfPinned(Object key) + { + synchronized(_map) + { + return _map.get(key); + } + } + + public Object + pin(Object key) + { + synchronized(_map) + { + Object o = _map.get(key); + if(o == null) + { + o = _store.load(key); + if(o != null) + { + _map.put(key, o); + } + } + return o; + } + } + + public Object + unpin(Object key) + { + synchronized(_map) + { + return _map.remove(key); + } + } + + public void + clear() + { + synchronized(_map) + { + _map.clear(); + } + } + + public int + size() + { + synchronized(_map) + { + return _map.size(); + } + } + + public Object + add(Object key, Object value) + { + assert value != null; + + synchronized(_map) + { + Object existingVal = _map.put(key, value); + if(existingVal != null) + { + _map.put(key, existingVal); + return existingVal; + } + + // + // Let's check if it's in the store + // + existingVal = _store.load(key); + if(existingVal != null) + { + _map.put(key, existingVal); + return existingVal; + } + else + { + return null; + } + } + } + + public Object + pin(Object key, Object value) + { + synchronized(_map) + { + Object existingVal = _map.put(key, value); + if(existingVal != null) + { + _map.put(key, existingVal); + } + return existingVal; + } + } + + private final java.util.Map _map = new java.util.HashMap(); + private final Store _store; + +} diff --git a/java/src/IceUtil/Store.java b/java/src/IceUtil/Store.java new file mode 100644 index 00000000000..6e5e46c3555 --- /dev/null +++ b/java/src/IceUtil/Store.java @@ -0,0 +1,20 @@ +// ********************************************************************** +// +// Copyright (c) 2004 +// ZeroC, Inc. +// Billerica, MA, USA +// +// All Rights Reserved. +// +// Ice is free software; you can redistribute it and/or modify it under +// the terms of the GNU General Public License version 2 as published by +// the Free Software Foundation. +// +// ********************************************************************** + +package IceUtil; + +public interface Store +{ + Object load(Object key); +} |