Evictor improvements and much more testing

10 files changed, 831 insertions, 132 deletions

diff --git a/cpp/CHANGES b/cpp/CHANGES
index b205f022655..f85d954fb10 100644
--- a/cpp/CHANGES
+++ b/cpp/CHANGES
@@ -1,6 +1,45 @@
 Changes since version 1.3.0
 ---------------------------
 
+- Freeze Evictor changes to adapt to the facet redesign, and 
+  improvements:
+  
+  * The Freeze Evictor now has add, addFacet, remove and removeFacet 
+    operations with the same signature and behavior as the 
+    Ice ObjectAdapter.
+
+  * The createObject and destroyObject operations were kept for 
+    compatibility with Ice 1.2/1.3, but are now deprecated.
+
+  * getIterator now takes a facet parameter, and no longer takes
+    a loadServants parameter.
+
+  * Indices can now be defined for facets (no just the "" facet)
+ 
+  * The ServantInitializer initialize operation now takes a facet
+    parameter. 
+
+  * The ServantInitializer is now provided to the Evictor in the
+    createEvictor call. In this way, the Evictor can consider the
+    servant initializer immutable and avoid locking.
+
+  * Servant initialization is now performed without holding any
+    Evictor lock. As a result a servant initializer can perform
+    any remote call, except one that comes back to the object
+    being initialized (this would result in a deadlock).
+
+  * The Evictor is now associated with a single object adapter;
+    this object adapter is provided in the createEvictor call.
+
+  * The Evictor is now "deactivate-safe": if you deactivate the
+    evictor while other threads are running other Evictor 
+    operations, deactivate() will wait until these operations
+    have completed.
+
+  * The new keep and release operations let you lock and unlock 
+    an object in the Evictor cache.
+
+
 - Removed a bogus assert that could happen under certain race
   conditions when a connection timeout occurs.
 
diff --git a/cpp/src/Freeze/EvictorI.cpp b/cpp/src/Freeze/EvictorI.cpp
index 422f323e9c5..d734b66665a 100644
--- a/cpp/src/Freeze/EvictorI.cpp
+++ b/cpp/src/Freeze/EvictorI.cpp
@@ -78,6 +78,110 @@ checkIdentity(const Identity& ident)
 }
 
 //
+// DeactivateController
+//
+
+Freeze::DeactivateController::Guard::Guard(DeactivateController& controller) :
+    _controller(controller)
+{
+    Lock sync(controller);
+    if(controller._deactivated || _controller._deactivating)
+    {
+	throw EvictorDeactivatedException(__FILE__, __LINE__);
+    }
+    controller._guardCount++;
+}
+
+Freeze::DeactivateController::Guard::~Guard()
+{
+    Lock sync(_controller);
+    _controller._guardCount--;
+    if(_controller._deactivating && _controller._guardCount == 0)
+    {
+	//
+	// Notify all the threads -- although we only want to
+	// reach the thread doing the deactivation.
+	//
+	_controller.notifyAll();
+    }
+}
+
+Freeze::DeactivateController::DeactivateController(EvictorI* evictor) :
+    _evictor(evictor),
+    _deactivating(false),
+    _deactivated(false),
+    _guardCount(0)
+{
+}
+
+bool
+Freeze::DeactivateController::deactivated() const
+{
+    Lock sync(*this);
+    return _deactivated || _deactivating;
+}
+
+bool
+Freeze::DeactivateController::deactivate()
+{
+    Lock sync(*this);
+
+    if(_deactivated)
+    {
+	return false;
+    }
+    
+    if(_deactivating)
+    {
+	//
+	// Wait for deactivated
+	//
+	while(!_deactivated)
+	{
+	    wait();
+	}
+	return false;
+    }
+    else
+    {
+	_deactivating = true;
+	while(_guardCount > 0)
+	{
+	    if(_evictor->trace() >= 1)
+	    {
+		Trace out(_evictor->communicator()->getLogger(), "Freeze.Evictor");
+		out << "*** Waiting for " << _guardCount << " threads to complete before starting deactivation.";
+	    }
+
+	    wait();
+	}
+
+	if(_evictor->trace() >= 1)
+	{
+	    Trace out(_evictor->communicator()->getLogger(), "Freeze.Evictor");
+	    out << "Starting deactivation.";
+	}
+	return true;
+    }
+}
+
+void
+Freeze::DeactivateController::deactivationComplete()
+{
+    if(_evictor->trace() >= 1)
+    {
+	Trace out(_evictor->communicator()->getLogger(), "Freeze.Evictor");
+	out << "Deactivation complete.";
+    }
+
+    Lock sync(*this);
+    _deactivated = true;
+    _deactivating = false;
+    notifyAll();
+}
+
+
+//
 // EvictorI
 //
 
@@ -89,7 +193,8 @@ Freeze::EvictorI::EvictorI(const ObjectAdapterPtr& adapter,
 			   bool createDb) :
     _evictorSize(10),
     _currentEvictorSize(0),
-    _deactivated(false),
+    _deactivateController(this),
+    _savingThreadDone(false),
     _adapter(adapter),
     _communicator(adapter->getCommunicator()),
     _initializer(initializer),
@@ -114,7 +219,8 @@ Freeze::EvictorI::EvictorI(const ObjectAdapterPtr& adapter,
     
     _evictorSize(10),
     _currentEvictorSize(0),
-    _deactivated(false),
+    _deactivateController(this),
+    _savingThreadDone(false),
     _adapter(adapter),
     _communicator(adapter->getCommunicator()),
     _initializer(initializer),
@@ -228,7 +334,7 @@ Freeze::EvictorI::init(const string& envName, const vector<IndexPtr>& indices)
 
 Freeze::EvictorI::~EvictorI()
 {
-    if(!_deactivated)
+    if(!_deactivateController.deactivated())
     {
 	Warning out(_communicator->getLogger());
 	out << "evictor has not been deactivated";
@@ -251,12 +357,9 @@ Freeze::EvictorI::~EvictorI()
 void
 Freeze::EvictorI::setSize(Int evictorSize)
 {
-    Lock sync(*this);
+    DeactivateController::Guard deactivateGuard(_deactivateController);
 
-    if(_deactivated)
-    {
-	throw EvictorDeactivatedException(__FILE__, __LINE__);
-    }
+    Lock sync(*this);
 
     //
     // Ignore requests to set the evictor size to values smaller than zero.
@@ -280,13 +383,8 @@ Freeze::EvictorI::setSize(Int evictorSize)
 Int
 Freeze::EvictorI::getSize()
 {
+    DeactivateController::Guard deactivateGuard(_deactivateController);
     Lock sync(*this);
-
-    if(_deactivated)
-    {
-	throw EvictorDeactivatedException(__FILE__, __LINE__);
-    }
-
     return static_cast<Int>(_evictorSize);
 }
 
@@ -301,19 +399,15 @@ Ice::ObjectPrx
 Freeze::EvictorI::addFacet(const ObjectPtr& servant, const Identity& ident, const string& facet)
 {
     checkIdentity(ident);
-
+    DeactivateController::Guard deactivateGuard(_deactivateController);
+   
     ObjectStore* store = 0;
     
     for(;;)
     {
 	{
 	    Lock sync(*this);
-	    
-	    if(_deactivated)
-	    {
-		throw EvictorDeactivatedException(__FILE__, __LINE__);
-	    }
-	    
+	     
 	    StoreMap::iterator p = _storeMap.find(facet);
 	    if(p == _storeMap.end())
 	    {
@@ -363,11 +457,6 @@ Freeze::EvictorI::addFacet(const ObjectPtr& servant, const Identity& ident, cons
 	{
 	    Lock sync(*this);
 
-	    if(_deactivated)
-	    {
-		throw EvictorDeactivatedException(__FILE__, __LINE__);
-	    }
-
 	    if(element->stale)
 	    {
 		//
@@ -460,7 +549,8 @@ void
 Freeze::EvictorI::createObject(const Identity& ident, const ObjectPtr& servant)
 {
     checkIdentity(ident);
-
+    DeactivateController::Guard deactivateGuard(_deactivateController);
+  
     ObjectStore* store = findStore("");
     assert(store != 0);
   
@@ -482,11 +572,6 @@ Freeze::EvictorI::createObject(const Identity& ident, const ObjectPtr& servant)
 	{
 	    Lock sync(*this);
 
-	    if(_deactivated)
-	    {
-		throw EvictorDeactivatedException(__FILE__, __LINE__);
-	    }
-
 	    if(element->stale)
 	    {
 		//
@@ -565,7 +650,8 @@ void
 Freeze::EvictorI::removeFacet(const Identity& ident, const string& facet)
 {
     checkIdentity(ident);
-
+    DeactivateController::Guard deactivateGuard(_deactivateController);
+   
     ObjectStore* store = findStore(facet);
     bool notThere = (store == 0);
 
@@ -708,6 +794,7 @@ void
 Freeze::EvictorI::keepFacet(const Identity& ident, const string& facet)
 {
     checkIdentity(ident);
+    DeactivateController::Guard deactivateGuard(_deactivateController);
 
     bool notThere = false;
 
@@ -728,10 +815,6 @@ Freeze::EvictorI::keepFacet(const Identity& ident, const string& facet)
 	    }
 	    
 	    Lock sync(*this);
-	    if(_deactivated)
-	    {
-		throw EvictorDeactivatedException(__FILE__, __LINE__);
-	    }
 	    
 	    if(element->stale)
 	    {
@@ -802,15 +885,11 @@ void
 Freeze::EvictorI::releaseFacet(const Identity& ident, const string& facet)
 {
     checkIdentity(ident);
+    DeactivateController::Guard deactivateGuard(_deactivateController);
 
     {
 	Lock sync(*this);
 	
-	if(_deactivated)
-	{
-	    throw EvictorDeactivatedException(__FILE__, __LINE__);
-	}
-	
 	StoreMap::iterator p = _storeMap.find(facet);
 	if(p != _storeMap.end())
 	{
@@ -856,13 +935,11 @@ Freeze::EvictorI::releaseFacet(const Identity& ident, const string& facet)
 EvictorIteratorPtr
 Freeze::EvictorI::getIterator(const string& facet, Int batchSize)
 {
+    DeactivateController::Guard deactivateGuard(_deactivateController);
+
     ObjectStore* store = 0;
     {
 	Lock sync(*this);
-	if(_deactivated)
-	{
-	    throw EvictorDeactivatedException(__FILE__, __LINE__);
-	}
 	
 	StoreMap::iterator p = _storeMap.find(facet);
 	if(p != _storeMap.end())
@@ -884,16 +961,13 @@ bool
 Freeze::EvictorI::hasFacet(const Identity& ident, const string& facet)
 {
     checkIdentity(ident);
+    DeactivateController::Guard deactivateGuard(_deactivateController);
+
     ObjectStore* store = 0;
 
     {
 	Lock sync(*this);
-	
-	if(_deactivated)
-	{
-	    throw EvictorDeactivatedException(__FILE__, __LINE__);
-	}
-	
+       
 	StoreMap::iterator p = _storeMap.find(facet);
 	if(p == _storeMap.end())
 	{
@@ -977,6 +1051,11 @@ Freeze::EvictorI::locate(const Current& current, LocalObjectPtr& cookie)
 ObjectPtr
 Freeze::EvictorI::locateImpl(const Current& current, LocalObjectPtr& cookie)
 {
+    //
+    // If only Ice calls locate/finished/deactivate, then it cannot be deactivated.
+    //
+    DeactivateController::Guard deactivateGuard(_deactivateController);
+
     cookie = 0;
 
     ObjectStore* store = findStore(current.facet);
@@ -994,7 +1073,6 @@ Freeze::EvictorI::locateImpl(const Current& current, LocalObjectPtr& cookie)
 	}
 	
 	Lock sync(*this);
-	assert(!_deactivated);
 
 	if(element->stale)
 	{
@@ -1027,6 +1105,11 @@ Freeze::EvictorI::finished(const Current& current, const ObjectPtr& servant, con
 {
     assert(servant);
 
+    //
+    // If only Ice calls locate/finished/deactivate, then it cannot be deactivated.
+    //
+    DeactivateController::Guard deactivateGuard(_deactivateController);
+
     if(cookie != 0)
     {
 	EvictorElementPtr element = EvictorElementPtr::dynamicCast(cookie);
@@ -1049,7 +1132,7 @@ Freeze::EvictorI::finished(const Current& current, const ObjectPtr& servant, con
 	}
 	
 	Lock sync(*this);
-	
+
 	//
 	// Only elements with a usageCount == 0 can become stale and we own 
 	// one count!
@@ -1079,42 +1162,41 @@ Freeze::EvictorI::finished(const Current& current, const ObjectPtr& servant, con
 void
 Freeze::EvictorI::deactivate(const string&)
 {
-    Lock sync(*this);
-    
-    //
-    // TODO: wait until all outstanding requests have completed
-    //
-
-    if(!_deactivated)
-    { 	
-	if(_trace >= 1)
+    if(_deactivateController.deactivate())
+    {
+	try
 	{
-	    Trace out(_communicator->getLogger(), "Freeze.Evictor");
-	    out << "deactivating, saving unsaved Ice objects to the database";
-	}
-
-	saveNowNoSync();
-	
-	//
-	// Set the evictor size to zero, meaning that we will evict
-	// everything possible.
-	//
-	_evictorSize = 0;
-	evict();
-	
-	_deactivated = true;
-	notifyAll();
-	sync.release();
-	getThreadControl().join();
+	    Lock sync(*this);
 
-	for(StoreMap::iterator p = _storeMap.begin(); p != _storeMap.end(); ++p)
+	    saveNowNoSync();
+	    
+	    //
+	    // Set the evictor size to zero, meaning that we will evict
+	    // everything possible.
+	    //
+	    _evictorSize = 0;
+	    evict();
+	    
+	    _savingThreadDone = true;
+	    notifyAll();
+	    sync.release();
+	    getThreadControl().join();
+	    
+	    for(StoreMap::iterator p = _storeMap.begin(); p != _storeMap.end(); ++p)
+	    {
+		(*p).second->close();
+	    }
+	    
+	    _dbEnv = 0;
+	    _dbEnvHolder = 0;
+	    _initializer = 0;
+	}
+	catch(...)
 	{
-	    (*p).second->close();
+	    _deactivateController.deactivationComplete();
+	    throw;
 	}
-
-	_dbEnv = 0;
-	_dbEnvHolder = 0;
-	_initializer = 0;
+	_deactivateController.deactivationComplete();
     }
 }
 
@@ -1143,7 +1225,8 @@ Freeze::EvictorI::run()
 	    
 	    {
 		Lock sync(*this);
-		while(!_deactivated &&
+
+		while(!_savingThreadDone &&
 		      (_saveNowThreads.size() == 0) &&
 		      (_saveSizeTrigger < 0 || static_cast<Int>(_modifiedQueue.size()) < _saveSizeTrigger))
 		{
@@ -1162,14 +1245,10 @@ Freeze::EvictorI::run()
 		
 		saveNowThreadsSize = _saveNowThreads.size();
 		
-		if(_deactivated)
+		if(_savingThreadDone)
 		{
 		    assert(_modifiedQueue.size() == 0);
-		    if(saveNowThreadsSize > 0)
-		    {
-			_saveNowThreads.clear();
-			notifyAll();
-		    }
+		    assert(saveNowThreadsSize == 0);
 		    break; // for(;;)
 		}
 		
@@ -1501,12 +1580,6 @@ void
 Freeze::EvictorI::saveNow()
 {
     Lock sync(*this);
-
-    if(_deactivated)
-    {
-	throw EvictorDeactivatedException(__FILE__, __LINE__);
-    }
-
     saveNowNoSync();
 }
 
@@ -1670,11 +1743,7 @@ Freeze::ObjectStore*
 Freeze::EvictorI::findStore(const string& facet) const
 {
     Lock sync(*this);
-    if(_deactivated)
-    {
-	throw EvictorDeactivatedException(__FILE__, __LINE__);
-    }
-
+  
     StoreMap::const_iterator p = _storeMap.find(facet);
     if(p == _storeMap.end())
     {
diff --git a/cpp/src/Freeze/EvictorI.h b/cpp/src/Freeze/EvictorI.h
index 8cbbae5315e..b5d61be1ae2 100644
--- a/cpp/src/Freeze/EvictorI.h
+++ b/cpp/src/Freeze/EvictorI.h
@@ -30,6 +30,57 @@
 namespace Freeze
 {
 
+class EvictorI;
+
+//
+// Helper class to prevent deactivation while the Evictor is in use,
+// and to queue deactivate() calls.
+//
+class DeactivateController : private IceUtil::Monitor<IceUtil::Mutex>
+{
+public:
+
+    //
+    // Prevents deactivation; the constructor raises 
+    // EvictorDeactivatedException if _deactivated or _deactivating is true.
+    //
+    class Guard
+    {
+    public:
+	Guard(DeactivateController&);
+	~Guard();
+
+    private:
+	DeactivateController& _controller;
+    };
+
+    DeactivateController(EvictorI*);    
+    
+    //
+    // Used mostly in asserts
+    //
+    bool deactivated() const;
+
+    //
+    // Returns true if this thread is supposed to do the deactivation and
+    // call deactivationComplete() once done.
+    //
+    bool deactivate();
+
+    void deactivationComplete();
+
+private:
+    
+    friend class Guard;
+
+    EvictorI* _evictor;
+    bool _deactivating;
+    bool _deactivated;
+    int _guardCount;
+};
+
+
+
 class EvictorI : public Evictor,  public IceUtil::Monitor<IceUtil::Mutex>, public IceUtil::Thread
 {
 public:
@@ -77,11 +128,14 @@ public:
     //
     void saveNow();
 
+    DeactivateController& deactivateController();
     const Ice::CommunicatorPtr& communicator() const;
     DbEnv* dbEnv() const;
     const std::string& filename() const;
 
     bool deadlockWarning() const;
+    Ice::Int trace() const;
+
 
     void initialize(const Ice::Identity&, const std::string&, const Ice::ObjectPtr&);
 
@@ -135,7 +189,8 @@ private:
     //
     std::deque<EvictorElementPtr> _modifiedQueue;
 
-    bool _deactivated;
+    DeactivateController _deactivateController;
+    bool _savingThreadDone;
 
     Ice::ObjectAdapterPtr _adapter;
     Ice::CommunicatorPtr _communicator;
@@ -163,6 +218,13 @@ private:
     bool _deadlockWarning;
 };
 
+
+inline DeactivateController&
+EvictorI::deactivateController()
+{
+    return _deactivateController;
+}
+
 inline const Ice::CommunicatorPtr&
 EvictorI::communicator() const
 {
@@ -181,6 +243,12 @@ EvictorI::deadlockWarning() const
     return _deadlockWarning;
 }
 
+inline Ice::Int
+EvictorI::trace() const
+{
+    return _trace;
+}
+
 }
 
 #endif
diff --git a/cpp/src/Freeze/EvictorIteratorI.cpp b/cpp/src/Freeze/EvictorIteratorI.cpp
index c5062d8cb50..b4a6386ac6d 100644
--- a/cpp/src/Freeze/EvictorIteratorI.cpp
+++ b/cpp/src/Freeze/EvictorIteratorI.cpp
@@ -64,6 +64,9 @@ Freeze::EvictorIteratorI::next()
 vector<Identity>::const_iterator
 Freeze::EvictorIteratorI::nextBatch()
 {
+    DeactivateController::Guard 
+	deactivateGuard(_store->evictor()->deactivateController());
+
     _batch.clear();
 
     if(!_more)
diff --git a/cpp/src/Freeze/IndexI.cpp b/cpp/src/Freeze/IndexI.cpp
index 7fb7c3c728f..48ed36c874f 100644
--- a/cpp/src/Freeze/IndexI.cpp
+++ b/cpp/src/Freeze/IndexI.cpp
@@ -41,6 +41,9 @@ Freeze::IndexI::IndexI(Index& index) :
 vector<Identity>
 Freeze::IndexI::untypedFindFirst(const Key& bytes, Int firstN) const
 {
+    DeactivateController::Guard 
+	deactivateGuard(_store->evictor()->deactivateController());
+
     Dbt dbKey;
     initializeInDbt(bytes, dbKey);
 
@@ -174,7 +177,9 @@ Freeze::IndexI::untypedFind(const Key& bytes) const
 Int
 Freeze::IndexI::untypedCount(const Key& bytes) const
 {
- 
+    DeactivateController::Guard 
+	deactivateGuard(_store->evictor()->deactivateController());
+
     Dbt dbKey;
     initializeInDbt(bytes, dbKey);
     
diff --git a/cpp/test/Freeze/evictor/Client.cpp b/cpp/test/Freeze/evictor/Client.cpp
index 8c87ed1f057..0d76f6bcbcc 100644
--- a/cpp/test/Freeze/evictor/Client.cpp
+++ b/cpp/test/Freeze/evictor/Client.cpp
@@ -69,6 +69,205 @@ private:
     vector<Test::ServantPrx>& _servants;
 };
 
+class ReadForeverThread : public Thread
+{
+public:
+
+    ReadForeverThread(vector<Test::ServantPrx>& servants) :
+	_servants(servants)
+    {
+    }
+    
+    virtual void
+    run()
+    {
+	for(;;)
+	{
+	    try
+	    {
+		for(int i = 0; i < static_cast<int>(_servants.size()); ++i)
+		{
+		    test(_servants[i]->slowGetValue() == i);
+		}
+	    }
+	    catch(const Ice::ConnectionRefusedException& cr)
+	    {
+		//
+		// Expected
+		//
+		return;
+	    }
+	    catch(const Ice::LocalException& e)
+	    {
+		cerr << "Caught unexpected : " << e << endl;
+		test(false);
+		return;
+	    }
+	    catch(...)
+	    {
+		test(false);
+		return;
+	    }
+	}
+    }
+
+private:
+    vector<Test::ServantPrx>& _servants;
+};
+
+class AddForeverThread : public Thread
+{
+public:
+
+    AddForeverThread(const Test::RemoteEvictorPrx& evictor, int id) :
+	_evictor(evictor),
+	_id(id)
+    {
+    }
+    
+    virtual void
+    run()
+    {
+	int index = _id * 1000;
+
+	for(;;)
+	{
+	    try
+	    {
+		_evictor->createServant(index++, 0);
+	    }
+	    catch(const Test::EvictorDeactivatedException&)
+	    {
+		//
+		// Expected
+		//
+		return;
+	    }
+	    catch(const Ice::ObjectNotExistException&)
+	    {
+		//
+		// Expected
+		//
+		return;
+	    }
+	    catch(const Ice::LocalException& e)
+	    {
+		cerr << "Caught unexpected : " << e << endl;
+		test(false);
+		return;
+	    }
+	    catch(...)
+	    {
+		test(false);
+		return;
+	    }
+	}
+    }
+
+private:
+    Test::RemoteEvictorPrx _evictor;
+    int _id;
+};
+
+
+
+class CreateDestroyThread : public Thread
+{
+public:
+
+    CreateDestroyThread(const Test::RemoteEvictorPrx& evictor, int id, int size) :
+	_evictor(evictor),
+	_id(id),
+	_size(size)
+    {
+    }
+    
+    virtual void
+    run()
+    {
+	try
+	{
+	    int loops = 50;
+	    while(loops-- > 0)
+	    {
+		for(int i = 0; i < _size; i++)
+		{
+		    if(i == _id)
+		    {
+			//
+			// Create when odd, destroy when even.
+			//
+			
+			if(loops % 2 == 0)
+			{
+			    Test::ServantPrx servant = _evictor->getServant(i);
+			    servant->destroy();
+			    
+			    //
+			    // Twice
+			    //
+			    try
+			    {
+				servant->destroy();
+				test(false);
+			    }
+			    catch(const Ice::ObjectNotExistException&)
+			    {
+				// Expected
+			    }
+			}
+			else
+			{
+			    Test::ServantPrx servant = _evictor->createServant(i, i);
+			    
+			    //
+			    // Twice
+			    //
+			    try
+			    {
+				servant = _evictor->createServant(i, 0);
+				test(false);
+			    }
+			    catch(const Test::AlreadyRegisteredException&)
+			    {
+				// Expected
+			    }
+			}
+		    }
+		    else
+		    {
+			//
+			// Just read/write the value
+			//
+			Test::ServantPrx servant = _evictor->getServant(i);
+			try
+			{
+			    int val = servant->getValue();
+			    test(val == i || val == -i);
+			    servant->setValue(-val);
+			}
+			catch(const Ice::ObjectNotExistException&)
+			{
+			    // Expected from time to time
+			}
+		    }
+		}
+	    }
+	}
+	catch(...)
+	{
+	    //
+	    // Unexpected!
+	    //
+	    test(false);
+	}
+    }
+private:
+    Test::RemoteEvictorPrx _evictor;
+    int _id;
+    int _size;
+};
+
 
 
 int
@@ -259,21 +458,114 @@ run(int argc, char* argv[], const Ice::CommunicatorPtr& communicator)
 	    // Expected
 	}
     }
+          
+    //
+    // Recreate servants, set transient value
+    //  
+    servants.clear();
+    for(i = 0; i < size; i++)
+    {
+	servants.push_back(evictor->createServant(i, i));
+	servants[i]->setTransientValue(i);
+    }
     
     //
-    // Allocate space for size+1 servants.
+    // Create and destroy 100 servants to make sure we save and evict
     //
-    servants.clear();
+    for(i = 0; i < 100; i++)
+    {
+	Test::ServantPrx servant = evictor->createServant(size + i, size + i);
+	servant->destroy();
+    }
     
     //
-    // Recreate servants.
+    // Check the transient value
     //
     for(i = 0; i < size; i++)
     {
-	servants.push_back(evictor->createServant(i, i));
+	test(servants[i]->getTransientValue() == -1);
+    }
+    
+    //
+    // Now with keep
+    //
+    for(i = 0; i < size; i++)
+    {
+	servants[i]->keepInCache();
+	servants[i]->keepInCache();
+	servants[i]->setTransientValue(i);
+    }
+
+    //
+    // Create and destroy 100 servants to make sure we save and evict
+    //
+    for(i = 0; i < 100; i++)
+    {
+	Test::ServantPrx servant = evictor->createServant(size + i, size + i);
+	servant->destroy();
     }
     
     //
+    // Check the transient value
+    //
+    for(i = 0; i < size; i++)
+    {
+	test(servants[i]->getTransientValue() == i);
+    }
+
+    //
+    // Again, after one release
+    //
+    for(i = 0; i < size; i++)
+    {
+	servants[i]->release();
+    }
+    for(i = 0; i < 100; i++)
+    {
+	Test::ServantPrx servant = evictor->createServant(size + i, size + i);
+	servant->destroy();
+    }
+    for(i = 0; i < size; i++)
+    {
+	test(servants[i]->getTransientValue() == i);
+    }
+
+    //
+    // Again, after a second release
+    //
+    for(i = 0; i < size; i++)
+    {
+	servants[i]->release();
+    }
+    for(i = 0; i < 100; i++)
+    {
+	Test::ServantPrx servant = evictor->createServant(size + i, size + i);
+	servant->destroy();
+    }
+    for(i = 0; i < size; i++)
+    {
+	test(servants[i]->getTransientValue() == -1);
+    }
+
+
+    //
+    // Release one more time
+    //
+    for(i = 0; i < size; i++)
+    {
+	try
+	{
+	    servants[i]->release();
+	    test(false);
+	}
+	catch(const Test::NotRegisteredException&)
+	{
+	    // Expected
+	}
+    }
+
+
+    //
     // Deactivate and recreate evictor, to ensure that servants
     // are restored properly after database close and reopen.
     //
@@ -295,18 +587,20 @@ run(int argc, char* argv[], const Ice::CommunicatorPtr& communicator)
     evictor->setSize(size / 2);
     servants[0]->destroy();
 
-    const int threadCount = size * 2;
-
-    ThreadPtr threads[threadCount];
-    for(i = 0; i < threadCount; i++)
     {
-	threads[i] = new ReadThread(servants);
-	threads[i]->start();
-    }
-
-    for(i = 0; i < threadCount; i++)
-    {
-	threads[i]->getThreadControl().join();
+	const int threadCount = size * 2;
+	
+	ThreadPtr threads[threadCount];
+	for(i = 0; i < threadCount; i++)
+	{
+	    threads[i] = new ReadThread(servants);
+	    threads[i]->start();
+	}
+	
+	for(i = 0; i < threadCount; i++)
+	{
+	    threads[i]->getThreadControl().join();
+	}
     }
     
     //
@@ -316,20 +610,109 @@ run(int argc, char* argv[], const Ice::CommunicatorPtr& communicator)
     evictor->destroyAllServants("facet1");
     evictor->destroyAllServants("facet2");
 
+    //
+    // CreateDestroy threads
+    //
+    {
+	const int threadCount = size;;
+	
+	ThreadPtr threads[threadCount];
+	for(i = 0; i < threadCount; i++)
+	{
+	    threads[i] = new CreateDestroyThread(evictor, i, size);
+	    threads[i]->start();
+	}
+	
+	for(i = 0; i < threadCount; i++)
+	{
+	    threads[i]->getThreadControl().join();
+	}
+
+	//
+	// Verify all destroyed
+	// 
+	for(i = 0; i < size; i++)   
+	{
+	    try
+	    {
+		servants[i]->getValue();
+		test(false);
+	    }
+	    catch(const Ice::ObjectNotExistException&)
+	    {
+		// Expected
+	    }
+	}
+    }
+
+    //
+    // Recreate servants.
+    //  
+    servants.clear();
     for(i = 0; i < size; i++)
     {
-	try
+	servants.push_back(evictor->createServant(i, i));
+    }
+
+    //
+    // Deactivate in the middle of remote AMD operations
+    // (really testing Ice here)
+    //
+    {
+	const int threadCount = size;
+	
+	ThreadPtr threads[threadCount];
+	for(i = 0; i < threadCount; i++)
 	{
-	    servants[i]->getValue();
-	    test(false);
+	    threads[i] = new ReadForeverThread(servants);
+	    threads[i]->start();
 	}
-	catch(const Ice::ObjectNotExistException&)
+
+	ThreadControl::sleep(Time::milliSeconds(500));
+	evictor->deactivate();
+
+	for(i = 0; i < threadCount; i++)
 	{
-	    // Expected
+	    threads[i]->getThreadControl().join();
 	}
     }
 
+    //
+    // Resurrect
+    //
+    evictor = factory->createEvictor("Test");
+    evictor->destroyAllServants("");
+
+    //
+    // Deactivate in the middle of adds
+    //
+    {
+	const int threadCount = size;
+	
+	ThreadPtr threads[threadCount];
+	for(i = 0; i < threadCount; i++)
+	{
+	    threads[i] = new AddForeverThread(evictor, i);
+	    threads[i]->start();
+	}
+
+	ThreadControl::sleep(Time::milliSeconds(500));
+	evictor->deactivate();
+	
+	for(i = 0; i < threadCount; i++)
+	{
+	    threads[i]->getThreadControl().join();
+	}
+    }
+    
+    
+    //
+    // Clean up.
+    //
+    evictor = factory->createEvictor("Test");
+    evictor->destroyAllServants("");
     evictor->deactivate();
+
     cout << "ok" << endl;
 
     factory->shutdown();
diff --git a/cpp/test/Freeze/evictor/Test.ice b/cpp/test/Freeze/evictor/Test.ice
index 3ac2ad2b8f5..296fd4cb8bc 100644
--- a/cpp/test/Freeze/evictor/Test.ice
+++ b/cpp/test/Freeze/evictor/Test.ice
@@ -26,9 +26,15 @@ exception NotRegisteredException
 {
 };
 
+exception EvictorDeactivatedException
+{
+};
+
 class Servant
 {
     nonmutating int getValue();
+    ["amd"] nonmutating int slowGetValue();
+
     void setValue(int value);
 
     ["ami", "amd"] void setValueAsync(int value);
@@ -37,6 +43,11 @@ class Servant
     nonmutating void addFacet(string name, string data) throws AlreadyRegisteredException;
     nonmutating void removeFacet(string name) throws NotRegisteredException;
 
+    nonmutating int getTransientValue();
+    void setTransientValue(int value);
+    void keepInCache();
+    void release() throws NotRegisteredException;
+
     void destroy();
 
     int value;
@@ -53,7 +64,10 @@ class Facet extends Servant
 interface RemoteEvictor
 {
     void setSize(int size);
-    Servant* createServant(int id, int value);
+
+    Servant* createServant(int id, int value) 
+	throws AlreadyRegisteredException, EvictorDeactivatedException;
+
     Servant* getServant(int id);
 
     void deactivate();
diff --git a/cpp/test/Freeze/evictor/TestI.cpp b/cpp/test/Freeze/evictor/TestI.cpp
index b6f811c1da9..219a1690c73 100644
--- a/cpp/test/Freeze/evictor/TestI.cpp
+++ b/cpp/test/Freeze/evictor/TestI.cpp
@@ -18,12 +18,39 @@
 
 using namespace std;
 using namespace Ice;
+using namespace IceUtil;
 
-Test::ServantI::ServantI()
+
+class DelayedResponse : public Thread
+{
+public:
+
+    DelayedResponse(const Test::AMD_Servant_slowGetValuePtr& cb, int val) :
+	_cb(cb),
+	_val(val)
+    {
+    }
+    
+    virtual void
+    run()
+    {
+	ThreadControl::sleep(Time::milliSeconds(500));
+	_cb->ice_response(_val);
+    }
+
+private:
+    Test::AMD_Servant_slowGetValuePtr _cb;
+    int _val;
+};
+
+
+Test::ServantI::ServantI() :
+    _transientValue(-1)
 {
 }
 
 Test::ServantI::ServantI(const RemoteEvictorIPtr& remoteEvictor, const Freeze::EvictorPtr& evictor, Ice::Int val) :
+    _transientValue(-1),
     _remoteEvictor(remoteEvictor),
     _evictor(evictor)
 {
@@ -44,6 +71,24 @@ Test::ServantI::getValue(const Current&) const
     return value;
 }
 
+Int
+Test::ServantI::slowGetValue(const Current&) const
+{
+    IceUtil::ThreadControl::sleep(IceUtil::Time::seconds(1));
+    Lock sync(*this);
+    return value;
+}
+
+void
+Test::ServantI::slowGetValue_async(const AMD_Servant_slowGetValuePtr& cb,
+				   const Current&) const
+{
+    IceUtil::ThreadControl::sleep(IceUtil::Time::seconds(1));
+    Lock sync(*this);
+    (new DelayedResponse(cb, value))->start().detach();
+}
+
+
 void
 Test::ServantI::setValue(Int val, const Current&)
 {
@@ -100,10 +145,51 @@ Test::ServantI::removeFacet(const string& name, const Current& current) const
     }
 }
 
+
+Ice::Int
+Test::ServantI::getTransientValue(const Current& current) const
+{
+    Lock sync(*this);
+    return _transientValue;
+}
+
+void
+Test::ServantI::setTransientValue(Ice::Int val, const Current& current)
+{
+    Lock sync(*this);
+    _transientValue = val;
+}
+
+void
+Test::ServantI::keepInCache(const Current& current)
+{
+    _evictor->keep(current.id);
+}
+
+void
+Test::ServantI::release(const Current& current)
+{
+    try
+    {
+	_evictor->release(current.id);
+    }
+    catch(const Ice::NotRegisteredException&)
+    {
+	throw NotRegisteredException();
+    }
+}
+
 void
 Test::ServantI::destroy(const Current& current)
 {
-    _evictor->remove(current.id);
+    try
+    {
+	_evictor->remove(current.id);
+    }
+    catch(const Ice::NotRegisteredException& e)
+    {
+	throw Ice::ObjectNotExistException(__FILE__, __LINE__);
+    }
 }
 
 
@@ -190,8 +276,23 @@ Test::RemoteEvictorI::createServant(Int id, Int value, const Current&)
     ostr << id;
     ident.name = ostr.str();
     ServantPtr servant = new ServantI(this, _evictor, value);
-    _evictor->add(servant, ident);
-    return ServantPrx::uncheckedCast(_evictorAdapter->createProxy(ident));
+    try
+    {
+	return ServantPrx::uncheckedCast(_evictor->add(servant, ident));
+    }
+    catch(const Ice::AlreadyRegisteredException&)
+    {
+	throw AlreadyRegisteredException();
+    }
+    catch(const Ice::ObjectAdapterDeactivatedException&)
+    {
+	throw EvictorDeactivatedException();
+    }
+    catch(const Freeze::EvictorDeactivatedException& ex)
+    {
+	throw EvictorDeactivatedException();
+    }
+
 }
 
 Test::ServantPrx
@@ -214,6 +315,7 @@ Test::RemoteEvictorI::deactivate(const Current& current)
     _adapter->remove(stringToIdentity(_category));
 }
 
+
 void
 Test::RemoteEvictorI::destroyAllServants(const string& facetName, const Current&)
 {
diff --git a/cpp/test/Freeze/evictor/TestI.h b/cpp/test/Freeze/evictor/TestI.h
index 25f1687fb60..8c291ed9f96 100644
--- a/cpp/test/Freeze/evictor/TestI.h
+++ b/cpp/test/Freeze/evictor/TestI.h
@@ -39,6 +39,15 @@ public:
 
     virtual ::Ice::Int getValue(const Ice::Current& = Ice::Current()) const;
 
+    //
+    // Used only if you remove ["amd"] from Test.ice
+    // 
+    virtual ::Ice::Int slowGetValue(const Ice::Current& = Ice::Current()) const;
+
+    
+    virtual void slowGetValue_async(const AMD_Servant_slowGetValuePtr&,
+				    const Ice::Current& = Ice::Current()) const;
+
     virtual void setValue(::Ice::Int, const Ice::Current& = Ice::Current());
 
     virtual void setValueAsync_async(const AMD_Servant_setValueAsyncPtr&, Ice::Int,
@@ -50,10 +59,17 @@ public:
 
     virtual void removeFacet(const std::string&, const Ice::Current& = Ice::Current()) const;
 
+
+    virtual Ice::Int getTransientValue(const Ice::Current& = Ice::Current()) const;
+    virtual void setTransientValue(Ice::Int, const Ice::Current& = Ice::Current());
+    virtual void keepInCache(const Ice::Current& = Ice::Current());
+    virtual void release(const Ice::Current& = Ice::Current());
+
     virtual void destroy(const Ice::Current& = Ice::Current());
 
 protected:
 
+    Ice::Int _transientValue;
     RemoteEvictorIPtr _remoteEvictor;
     Freeze::EvictorPtr _evictor;
     AMD_Servant_setValueAsyncPtr _setValueAsyncCB;
diff --git a/cpp/test/Freeze/evictor/config b/cpp/test/Freeze/evictor/config
index 87439a6188d..a83dc289c11 100644
--- a/cpp/test/Freeze/evictor/config
+++ b/cpp/test/Freeze/evictor/config
@@ -1,7 +1,7 @@
 Freeze.Evictor.db.Test.SaveSizeTrigger=6
 Freeze.Evictor.db.Test.SavePeriod=2
 
-#Freeze.Trace.Evictor=3
+#Freeze.Trace.Evictor=1
 #Freeze.Trace.DbEnv=3
 
 Ice.ThreadPool.Server.SizeMax=20