Replication fixes, more work on the replication tests

16 files changed, 228 insertions, 100 deletions

diff --git a/cpp/src/IceGrid/Database.cpp b/cpp/src/IceGrid/Database.cpp
index 82d16f884b3..954efeb75fa 100644
--- a/cpp/src/IceGrid/Database.cpp
+++ b/cpp/src/IceGrid/Database.cpp
@@ -193,12 +193,6 @@ Database::~Database()
 {
 }
 
-void
-Database::destroy()
-{
-    _nodeCache.destroy(); // Break cyclic reference count.
-}
-
 std::string
 Database::getInstanceName() const
 {
diff --git a/cpp/src/IceGrid/Database.h b/cpp/src/IceGrid/Database.h
index 5e23eea25bf..5fb64b72996 100644
--- a/cpp/src/IceGrid/Database.h
+++ b/cpp/src/IceGrid/Database.h
@@ -54,8 +54,6 @@ public:
 	     const RegistryInfo&, bool);
     virtual ~Database();
     
-    void destroy();
-
     std::string getInstanceName() const;
     bool isMaster() const { return _master; }
     const TraceLevelsPtr& getTraceLevels() const { return _traceLevels; }
diff --git a/cpp/src/IceGrid/NodeCache.cpp b/cpp/src/IceGrid/NodeCache.cpp
index 645a7f60475..21619cc5be7 100644
--- a/cpp/src/IceGrid/NodeCache.cpp
+++ b/cpp/src/IceGrid/NodeCache.cpp
@@ -192,16 +192,6 @@ NodeCache::NodeCache(const Ice::CommunicatorPtr& communicator, ReplicaCache& rep
 {
 }
 
-void
-NodeCache::destroy()
-{
-    map<string, NodeEntryPtr> entries = _entries; // Copying the map is necessary as setSession might remove the entry.
-    for(map<string, NodeEntryPtr>::const_iterator p = entries.begin(); p != entries.end(); ++p)
-    {
-	p->second->setSession(0); // Break cyclic reference count.
-    }    
-}
-
 NodeEntryPtr
 NodeCache::get(const string& name, bool create) const
 {
diff --git a/cpp/src/IceGrid/NodeCache.h b/cpp/src/IceGrid/NodeCache.h
index cfedcea35b6..6060ec912c5 100644
--- a/cpp/src/IceGrid/NodeCache.h
+++ b/cpp/src/IceGrid/NodeCache.h
@@ -79,8 +79,6 @@ public:
 
     NodeCache(const Ice::CommunicatorPtr&, ReplicaCache&, bool);
 
-    void destroy();
-
     NodeEntryPtr get(const std::string&, bool = false) const;
 
     const Ice::CommunicatorPtr& getCommunicator() const { return _communicator; }
diff --git a/cpp/src/IceGrid/NodeI.cpp b/cpp/src/IceGrid/NodeI.cpp
index 0c3d1bce136..463296597ad 100644
--- a/cpp/src/IceGrid/NodeI.cpp
+++ b/cpp/src/IceGrid/NodeI.cpp
@@ -182,6 +182,7 @@ NodeI::NodeI(const Ice::ObjectAdapterPtr& adapter,
 	     const NodePrx& proxy,
 	     const string& name,
 	     const UserAccountMapperPrx& mapper) :
+    _communicator(adapter->getCommunicator()),
     _adapter(adapter),
     _sessions(sessions),
     _activator(activator),
@@ -192,14 +193,14 @@ NodeI::NodeI(const Ice::ObjectAdapterPtr& adapter,
     _waitTime(0),
     _userAccountMapper(mapper),
     _serial(1),
-    _platform("IceGrid.Node", adapter->getCommunicator(), _traceLevels)
+    _platform("IceGrid.Node", _communicator, _traceLevels)
 {
     _dataDir = _platform.getDataDir();
     _serversDir = _dataDir + "/servers";
     _tmpDir = _dataDir + "/tmp";
 
-    Ice::PropertiesPtr properties = getCommunicator()->getProperties();
-    const_cast<string&>(_instanceName) = getCommunicator()->getDefaultLocator()->ice_getIdentity().category;
+    Ice::PropertiesPtr properties = _communicator->getProperties();
+    const_cast<string&>(_instanceName) = _communicator->getDefaultLocator()->ice_getIdentity().category;
     const_cast<Ice::Int&>(_waitTime) = properties->getPropertyAsIntWithDefault("IceGrid.Node.WaitTime", 60);
 }
 
@@ -395,7 +396,7 @@ NodeI::patch(const string& application,
 	    FileServerPrx icepatch;
 	    if(!appDistrib.icepatch.empty())
 	    {
-		icepatch = FileServerPrx::checkedCast(getCommunicator()->stringToProxy(appDistrib.icepatch));
+		icepatch = FileServerPrx::checkedCast(_communicator->stringToProxy(appDistrib.icepatch));
 		if(!icepatch)
 		{
 		    throw "proxy `" + appDistrib.icepatch + "' is not a file server.";
@@ -414,7 +415,7 @@ NodeI::patch(const string& application,
 		    continue;
 		}
 
-		icepatch = FileServerPrx::checkedCast(getCommunicator()->stringToProxy(dist.icepatch));
+		icepatch = FileServerPrx::checkedCast(_communicator->stringToProxy(dist.icepatch));
 		if(!icepatch)
 		{
 		    throw "proxy `" + dist.icepatch + "' is not a file server.";
@@ -512,7 +513,7 @@ NodeI::shutdown(const Ice::Current&) const
 Ice::CommunicatorPtr
 NodeI::getCommunicator() const
 {
-    return _adapter->getCommunicator();
+    return _communicator;
 }
 
 Ice::ObjectAdapterPtr
diff --git a/cpp/src/IceGrid/NodeI.h b/cpp/src/IceGrid/NodeI.h
index 13b2adfa42d..322cb69b76d 100644
--- a/cpp/src/IceGrid/NodeI.h
+++ b/cpp/src/IceGrid/NodeI.h
@@ -81,6 +81,7 @@ private:
     std::set<ServerIPtr> getApplicationServers(const std::string&) const;
     Ice::Identity createServerIdentity(const std::string&) const;
 
+    const Ice::CommunicatorPtr _communicator;
     const Ice::ObjectAdapterPtr _adapter;
     NodeSessionManager& _sessions;
     const ActivatorPtr _activator;
diff --git a/cpp/src/IceGrid/NodeSessionManager.cpp b/cpp/src/IceGrid/NodeSessionManager.cpp
index c6742056400..bc22006450d 100644
--- a/cpp/src/IceGrid/NodeSessionManager.cpp
+++ b/cpp/src/IceGrid/NodeSessionManager.cpp
@@ -18,12 +18,12 @@ using namespace IceGrid;
 
 NodeSessionKeepAliveThread::NodeSessionKeepAliveThread(const InternalRegistryPrx& registry,
 						       const NodeIPtr& node,
-						       const IceGrid::QueryPrx& query) : 
+						       const vector<QueryPrx>& queryObjects) : 
     SessionKeepAliveThread<NodeSessionPrx>(registry),
     _node(node),
-    _query(query)
+    _queryObjects(queryObjects)
 {
-    assert(registry && node && query);
+    assert(registry && node && !_queryObjects.empty());
     string name = registry->ice_getIdentity().name;
     const string prefix("InternalRegistry-");
     string::size_type pos = name.find(prefix);
@@ -48,6 +48,7 @@ NodeSessionKeepAliveThread::createSession(const InternalRegistryPrx& registry, I
 	    out << "trying to establish session with replica `" << _name << "'";
 	}
 
+	set<InternalRegistryPrx> used;
 	if(!registry->ice_getEndpoints().empty())
 	{
 	    try
@@ -57,25 +58,35 @@ NodeSessionKeepAliveThread::createSession(const InternalRegistryPrx& registry, I
 	    catch(const Ice::LocalException& ex)
 	    {
 		exception.reset(ex.ice_clone());
+		used.insert(registry);
 		setRegistry(InternalRegistryPrx::uncheckedCast(registry->ice_endpoints(Ice::EndpointSeq())));
 	    }
 	}
 
 	if(!session)
 	{
-	    try
+	    for(vector<QueryPrx>::const_iterator p = _queryObjects.begin(); p != _queryObjects.end(); ++p)
 	    {
-		Ice::ObjectPrx obj = _query->findObjectById(registry->ice_getIdentity());
-		InternalRegistryPrx newRegistry = InternalRegistryPrx::uncheckedCast(obj);
-		if(newRegistry && newRegistry != registry)
+		InternalRegistryPrx newRegistry;
+		try
 		{
-		    session = createSessionImpl(newRegistry, timeout);
-		    setRegistry(newRegistry);
+		    Ice::ObjectPrx obj = (*p)->findObjectById(registry->ice_getIdentity());
+		    newRegistry = InternalRegistryPrx::uncheckedCast(obj);
+		    if(newRegistry && used.find(newRegistry) == used.end())
+		    {
+			session = createSessionImpl(newRegistry, timeout);
+			setRegistry(newRegistry);
+			break;
+		    }
+		}
+		catch(const Ice::LocalException& ex)
+		{
+		    exception.reset(ex.ice_clone());
+		    if(newRegistry)
+		    {
+			used.insert(newRegistry);
+		    }
 		}
-	    }
-	    catch(const Ice::LocalException& ex)
-	    {
-		exception.reset(ex.ice_clone());
 	    }
 	}
     }
@@ -197,8 +208,22 @@ NodeSessionManager::create(const NodeIPtr& node)
     assert(communicator->getDefaultLocator());
     Ice::Identity id = communicator->getDefaultLocator()->ice_getIdentity();
 
+    //
+    // Initialize the IceGrid::Query objects. The IceGrid::Query
+    // interface is used to lookup the registry proxy in case it
+    // becomes unavailable. Since replicas might not always have
+    // an up to date registry proxy, we need to query all the
+    // replicas.
+    //
+    Ice::EndpointSeq endpoints = communicator->getDefaultLocator()->ice_getEndpoints();
     id.name = "Query";
-    _query = QueryPrx::uncheckedCast(communicator->stringToProxy(communicator->identityToString(id)));
+    QueryPrx query = QueryPrx::uncheckedCast(communicator->stringToProxy(communicator->identityToString(id)));
+    for(Ice::EndpointSeq::const_iterator p = endpoints.begin(); p != endpoints.end(); ++p)
+    {
+	Ice::EndpointSeq singleEndpoint;
+	singleEndpoint.push_back(*p);
+	_queryObjects.push_back(QueryPrx::uncheckedCast(query->ice_endpoints(singleEndpoint)));
+    }
 
     id.name = "InternalRegistry-Master";
     _master = InternalRegistryPrx::uncheckedCast(communicator->stringToProxy(communicator->identityToString(id)));
@@ -281,7 +306,7 @@ NodeSessionManager::replicaAdded(const InternalRegistryPrx& replica)
 	return p->second;
     }
 
-    NodeSessionKeepAliveThreadPtr thread = new NodeSessionKeepAliveThread(replica, _node, _query);
+    NodeSessionKeepAliveThreadPtr thread = new NodeSessionKeepAliveThread(replica, _node, _queryObjects);
     _sessions.insert(make_pair(replica->ice_getIdentity(), thread));
     thread->start();
     return thread;
@@ -334,14 +359,26 @@ NodeSessionManager::syncReplicas(const InternalRegistryPrxSeq& replicas)
 	}
 	else
 	{
-	    thread = new NodeSessionKeepAliveThread(*p, _node, _query);
+	    thread = new NodeSessionKeepAliveThread(*p, _node, _queryObjects);
 	    thread->start();
 	    thread->tryCreateSession(*p);
 	}
 	_sessions.insert(make_pair((*p)->ice_getIdentity(), thread));
     }
 
-    NodeSessionMap::const_iterator q;
+    NodeSessionMap::iterator q = sessions.begin();
+    while(q != sessions.end())
+    {
+	if(q->second->getSession()) // Don't destroy sessions which are still alive!
+	{
+	    _sessions.insert(make_pair(q->first, q->second));
+	    sessions.erase(q++);
+	}
+	else
+	{
+	    ++q;
+	}
+    }
     for(q = sessions.begin(); q != sessions.end(); ++q)
     {
 	q->second->terminate();
@@ -385,10 +422,22 @@ NodeSessionManager::createdSession(const NodeSessionPrx& session)
 	    else
 	    {
 		replicas.clear();
-		Ice::ObjectProxySeq proxies = _query->findAllObjectsByType(InternalRegistry::ice_staticId());
-		for(Ice::ObjectProxySeq::const_iterator p = proxies.begin(); p != proxies.end(); ++p)
+		set<Ice::ObjectPrx> proxies;
+		for(vector<QueryPrx>::const_iterator p = _queryObjects.begin(); p != _queryObjects.end(); ++p)
+		{
+		    try
+		    {
+			Ice::ObjectProxySeq prxs = (*p)->findAllObjectsByType(InternalRegistry::ice_staticId());
+			proxies.insert(prxs.begin(), prxs.end());
+		    }
+		    catch(const Ice::LocalException& ex)
+		    {
+			// IGNORE
+		    }
+		}
+		for(set<Ice::ObjectPrx>::const_iterator q = proxies.begin(); q != proxies.end(); ++q)
 		{
-		    replicas.push_back(InternalRegistryPrx::uncheckedCast(*p));
+		    replicas.push_back(InternalRegistryPrx::uncheckedCast(*q));
 		}
 	    }
 	}
diff --git a/cpp/src/IceGrid/NodeSessionManager.h b/cpp/src/IceGrid/NodeSessionManager.h
index 2988c869573..84fa666e638 100644
--- a/cpp/src/IceGrid/NodeSessionManager.h
+++ b/cpp/src/IceGrid/NodeSessionManager.h
@@ -28,7 +28,7 @@ class NodeSessionKeepAliveThread : public SessionKeepAliveThread<NodeSessionPrx>
 {
 public:
 
-    NodeSessionKeepAliveThread(const InternalRegistryPrx&, const NodeIPtr&, const IceGrid::QueryPrx&);
+    NodeSessionKeepAliveThread(const InternalRegistryPrx&, const NodeIPtr&, const std::vector<QueryPrx>&);
 
     virtual NodeSessionPrx createSession(const InternalRegistryPrx&, IceUtil::Time&);
     virtual void destroySession(const NodeSessionPrx&);
@@ -40,7 +40,7 @@ protected:
 
     const NodeIPtr _node;
     const std::string _name;
-    const IceGrid::QueryPrx _query;
+    const std::vector<QueryPrx> _queryObjects;
 };
 typedef IceUtil::Handle<NodeSessionKeepAliveThread> NodeSessionKeepAliveThreadPtr;
 
@@ -69,7 +69,7 @@ private:
     public:
 
 	Thread(NodeSessionManager& manager) : 
-	    NodeSessionKeepAliveThread(manager._master, manager._node, manager._query),
+	    NodeSessionKeepAliveThread(manager._master, manager._node, manager._queryObjects),
 	    _manager(manager)
         {
 	}
@@ -93,7 +93,7 @@ private:
 
     const NodeIPtr _node;
     ThreadPtr _thread;
-    QueryPrx _query;
+    std::vector<QueryPrx> _queryObjects;
     InternalRegistryPrx _master;
     unsigned long _serial;
     bool _destroyed;
diff --git a/cpp/src/IceGrid/ReapThread.cpp b/cpp/src/IceGrid/ReapThread.cpp
index 22be3410b1e..65234e93807 100644
--- a/cpp/src/IceGrid/ReapThread.cpp
+++ b/cpp/src/IceGrid/ReapThread.cpp
@@ -90,6 +90,7 @@ ReapThread::terminate()
 	Lock sync(*this);
 	if(_terminated)
 	{
+	    assert(_sessions.empty());
 	    return;
 	}
 	_terminated = true;
@@ -97,7 +98,7 @@ ReapThread::terminate()
 	reap.swap(_sessions);
     }
 
-    for(list<ReapableItem>::const_iterator p = reap.begin(); p != reap.end(); ++p)
+    for(list<ReapableItem>::iterator p = reap.begin(); p != reap.end(); ++p)
     {
 	p->item->destroy(true);
     }
diff --git a/cpp/src/IceGrid/RegistryI.cpp b/cpp/src/IceGrid/RegistryI.cpp
index f4790311f6b..f3654aa3718 100644
--- a/cpp/src/IceGrid/RegistryI.cpp
+++ b/cpp/src/IceGrid/RegistryI.cpp
@@ -301,7 +301,7 @@ RegistryI::start(bool nowarn)
     InternalRegistryPrx internalRegistry = setupInternalRegistry(registryAdapter);
     if(_master)
     {
-	NodePrxSeq nodes = registerReplicas(internalRegistry, replicas);
+	nodes = registerReplicas(internalRegistry, replicas, nodes);
 	registerNodes(internalRegistry, nodes);
     }
     else
@@ -390,7 +390,10 @@ RegistryI::setupLocator(const Ice::ObjectAdapterPtr& clientAdapter,
     Identity locatorId;
     locatorId.category = _instanceName;
     locatorId.name = "Locator";
-    clientAdapter->add(new LocatorI(_communicator, _database, locatorRegistry), locatorId);
+    LocatorPtr locator = new LocatorI(_communicator, _database, locatorRegistry);
+    clientAdapter->add(locator, locatorId);
+    locatorId.name = "Locator-" + _replicaName;
+    clientAdapter->add(locator, locatorId);
     
     obj = registryAdapter->addWithUUID(new LocatorI(_communicator, _database, locatorRegistry));
     return LocatorPrx::uncheckedCast(obj);
@@ -613,7 +616,9 @@ RegistryI::stop()
 	_iceStorm = 0;
     }
 
-    _database->destroy();
+    _wellKnownObjects = 0;
+    _clientSessionFactory = 0;
+    _adminSessionFactory = 0;
     _database = 0;
 }
 
@@ -1112,31 +1117,55 @@ RegistryI::getSSLInfo(const ConnectionPtr& connection, string& userDN)
 }
 
 NodePrxSeq
-RegistryI::registerReplicas(const InternalRegistryPrx& internalRegistry, const InternalRegistryPrxSeq& replicas)
+RegistryI::registerReplicas(const InternalRegistryPrx& internalRegistry, 
+			    const InternalRegistryPrxSeq& replicas,
+			    const NodePrxSeq& dbNodes)
 {
     set<NodePrx> nodes;
+    nodes.insert(dbNodes.begin(), dbNodes.end());
+
     for(InternalRegistryPrxSeq::const_iterator r = replicas.begin(); r != replicas.end(); ++r)
     {
 	if((*r)->ice_getIdentity() != internalRegistry->ice_getIdentity())
 	{
+	    string replicaName;
+	    if(_traceLevels && _traceLevels->replica > 1)
+	    {
+		replicaName = (*r)->ice_getIdentity().name;
+		const string prefix("InternalRegistry-");
+		string::size_type pos = replicaName.find(prefix);
+		if(pos != string::npos)
+		{
+		    replicaName = replicaName.substr(prefix.size());
+		}
+
+		Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
+		out << "creating replica `" << replicaName << "' session";
+	    }
+
 	    try
 	    {
 		(*r)->registerWithReplica(internalRegistry);
 		NodePrxSeq nds = (*r)->getNodes();
 		nodes.insert(nds.begin(), nds.end());
+
+		if(_traceLevels && _traceLevels->replica > 1)
+		{
+		    Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
+		    out << "replica `" << replicaName << "' session created";
+		}
 	    }
-	    catch(const Ice::LocalException&)
+	    catch(const Ice::LocalException& ex)
 	    {
+		if(_traceLevels && _traceLevels->replica > 1)
+		{
+		    Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
+		    out << "replica `" << replicaName << "' session creation failed:\n" << ex;
+		}
 	    }
 	}
     }
     
-    if(nodes.empty())
-    {
-	NodePrxSeq nds = internalRegistry->getNodes();
-	nodes.insert(nds.begin(), nds.end());
-    }
-
 #ifdef _RWSTD_NO_MEMBER_TEMPLATES
     NodePrxSeq result;
     for(set<NodePrx>::iterator p = nodes.begin(); p != nodes.end(); ++p)
@@ -1154,12 +1183,38 @@ RegistryI::registerNodes(const InternalRegistryPrx& internalRegistry, const Node
 {
     for(NodePrxSeq::const_iterator p = nodes.begin(); p != nodes.end(); ++p)
     {
+	string nodeName;
+	if(_traceLevels && _traceLevels->node > 1)
+	{
+	    nodeName = (*p)->ice_getIdentity().name;
+	    const string prefix("Node-");
+	    string::size_type pos = nodeName.find(prefix);
+	    if(pos != string::npos)
+	    {
+		nodeName = nodeName.substr(prefix.size());
+	    }
+	    
+	    Ice::Trace out(_traceLevels->logger, _traceLevels->nodeCat);
+	    out << "creating node `" << nodeName << "' session";
+	}
+	
 	try
 	{
 	    NodePrx::uncheckedCast(*p)->registerWithReplica(internalRegistry);
+	    
+	    if(_traceLevels && _traceLevels->node > 1)
+	    {
+		Ice::Trace out(_traceLevels->logger, _traceLevels->nodeCat);
+		out << "node `" << nodeName << "' session created";
+	    }
 	}
-	catch(const Ice::LocalException&)
+	catch(const Ice::LocalException& ex)
 	{
+	    if(_traceLevels && _traceLevels->node > 1)
+	    {
+		Ice::Trace out(_traceLevels->logger, _traceLevels->nodeCat);
+		out << "node `" << nodeName << "' session creation failed:\n" << ex;
+	    }
 	}
     }
 }
diff --git a/cpp/src/IceGrid/RegistryI.h b/cpp/src/IceGrid/RegistryI.h
index b1b0e9463f0..9dd9580ae99 100644
--- a/cpp/src/IceGrid/RegistryI.h
+++ b/cpp/src/IceGrid/RegistryI.h
@@ -90,7 +90,7 @@ private:
     Glacier2::SSLPermissionsVerifierPrx getSSLPermissionsVerifier(const Ice::LocatorPrx&, const std::string&, bool);
     Glacier2::SSLInfo getSSLInfo(const Ice::ConnectionPtr&, std::string&);
 
-    NodePrxSeq registerReplicas(const InternalRegistryPrx&, const InternalRegistryPrxSeq&);
+    NodePrxSeq registerReplicas(const InternalRegistryPrx&, const InternalRegistryPrxSeq&, const NodePrxSeq&);
     void registerNodes(const InternalRegistryPrx&, const NodePrxSeq&);
     
     const Ice::CommunicatorPtr _communicator;
diff --git a/cpp/src/IceGrid/ReplicaSessionI.cpp b/cpp/src/IceGrid/ReplicaSessionI.cpp
index dc432499f85..78197e3ea1c 100644
--- a/cpp/src/IceGrid/ReplicaSessionI.cpp
+++ b/cpp/src/IceGrid/ReplicaSessionI.cpp
@@ -110,6 +110,13 @@ ReplicaSessionI::registerWellKnownObjects(const ObjectInfoSeq& objects, const Ic
 	_replicaWellKnownObjects = objects;
 	serial = _database->addOrUpdateObjectsInDatabase(objects);
     }
+
+    //
+    // We wait for the replica to receive the database replication
+    // updates. This is to ensure that the replica well-known objects
+    // are correctly setup when the replica starts accepting requests
+    // from clients (if the replica is being started).
+    //
     _database->getObserverTopic(ObjectObserverTopicName)->waitForSyncedSubscribers(serial, _name);
 }
 
@@ -162,11 +169,9 @@ ReplicaSessionI::destroy(const Ice::Current& current)
 
     if(!shutdown)
     {
-	cerr << "updating well known objects " << _name << endl;
 	_wellKnownObjects->updateReplicatedWellKnownObjects(); // No need to update these if we're shutting down.
     }
 
-    cerr << "removing replica " << _name << endl;
     _database->removeReplica(_name, this, shutdown);
 
     if(current.adapter)
diff --git a/cpp/src/IceGrid/ReplicaSessionManager.cpp b/cpp/src/IceGrid/ReplicaSessionManager.cpp
index 8413261f8af..28c59984c0a 100644
--- a/cpp/src/IceGrid/ReplicaSessionManager.cpp
+++ b/cpp/src/IceGrid/ReplicaSessionManager.cpp
@@ -250,13 +250,15 @@ ReplicaSessionManager::create(const string& name,
 			      const InternalRegistryPrx& internalRegistry)
 {
     Ice::CommunicatorPtr comm = database->getCommunicator();
-    string instName = comm->getDefaultLocator()->ice_getIdentity().category;
-
     {
 	Lock sync(*this);
+
+	Ice::Identity id;
+	id.category = comm->getDefaultLocator()->ice_getIdentity().category;
+	id.name = "InternalRegistry-Master";
+
 	
-	_master = InternalRegistryPrx::uncheckedCast(comm->stringToProxy(instName + "/InternalRegistry-Master"));
-	
+	_master = InternalRegistryPrx::uncheckedCast(comm->stringToProxy(comm->identityToString(id)));
 	_name = name;
 	_info = info;
 	_internalRegistry = internalRegistry;
@@ -272,7 +274,8 @@ ReplicaSessionManager::create(const string& name,
 	// replicas.
 	//
 	Ice::EndpointSeq endpoints = comm->getDefaultLocator()->ice_getEndpoints();
-	QueryPrx query = QueryPrx::uncheckedCast(comm->stringToProxy(instName + "/Query"));
+	id.name = "Query";
+	QueryPrx query = QueryPrx::uncheckedCast(comm->stringToProxy(comm->identityToString(id)));
 	for(Ice::EndpointSeq::const_iterator p = endpoints.begin(); p != endpoints.end(); ++p)
 	{
 	    Ice::EndpointSeq singleEndpoint;
@@ -334,12 +337,23 @@ ReplicaSessionManager::destroy()
 
     _thread->terminate();
     _thread->getThreadControl().join();
+
+    _database = 0;
+    _wellKnownObjects = 0;
 }
 
 void
 ReplicaSessionManager::registerAllWellKnownObjects()
 {
     //
+    // Always register first the well-known objects with the
+    // database. Then, if there's a session, we register them with the
+    // session and this will eventually override the ones with just
+    // registered with the ones from the master.
+    //
+    _wellKnownObjects->registerAll();
+
+    //
     // If there's an active session, register the well-known objects
     // with the session.
     //
@@ -452,17 +466,28 @@ ReplicaSessionManager::createSession(const InternalRegistryPrx& registry, IceUti
 	
     if(session)
     {
-	if(_traceLevels && _traceLevels->replica > 0)
-	{
-	    Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
-	    out << "established session with master replica";
-	}
+	//
+	// Register all the well-known objects with the replica session.
+	//
+	_wellKnownObjects->registerAll(session);
     }
     else
     {
-	if(_traceLevels && _traceLevels->replica > 1)
+	//
+	// Re-register all the well known objects with the local database.
+	//
+	_wellKnownObjects->registerAll();
+    }
+
+    if(_traceLevels && _traceLevels->replica > 0)
+    {
+	Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
+	if(session)
+	{
+	    out << "established session with master replica";
+	}
+	else
 	{
-	    Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
 	    out << "failed to establish session with master replica:\n";
 	    if(exception.get())
 	    {
@@ -497,11 +522,6 @@ ReplicaSessionManager::createSessionImpl(const InternalRegistryPrx& registry, Ic
 	DatabaseObserverPtr servant = new MasterDatabaseObserverI(_thread, _database, session);
 	_observer = DatabaseObserverPrx::uncheckedCast(_database->getInternalAdapter()->addWithUUID(servant));	
 	session->setDatabaseObserver(_observer);
-
-	//
-	// Register all the well-known objects with the replica session.
-	//
-	_wellKnownObjects->registerAll(session);
 	return session;
     }
     catch(const Ice::LocalException&)
@@ -517,11 +537,6 @@ ReplicaSessionManager::createSessionImpl(const InternalRegistryPrx& registry, Ic
 	    }
 	    _observer = 0;
 	}
-
-	//
-	// Re-register all the well known objects with the local database.
-	//
-	_wellKnownObjects->registerAll();
 	throw;
     }
 }
diff --git a/cpp/src/IceGrid/ServerI.cpp b/cpp/src/IceGrid/ServerI.cpp
index 312a37649ea..c64bdf1e3ba 100644
--- a/cpp/src/IceGrid/ServerI.cpp
+++ b/cpp/src/IceGrid/ServerI.cpp
@@ -948,8 +948,9 @@ ServerI::load(const AMD_Node_loadServerPtr& amdCB, const ServerInfo& info, bool
 	// - the application uuid, the application revision and the
 	//   session id didn't change.
 	//
-	// - the load command if from a replica and the given
-	//   descriptor is from another application or out-of-date.
+	// - the load command if from a slave and the given descriptor
+	//   is from another application or doesn't have the same
+	//   version.
 	//
 	// - the descriptor and the session id didn't change.
 	//
@@ -964,10 +965,10 @@ ServerI::load(const AMD_Node_loadServerPtr& amdCB, const ServerInfo& info, bool
 		ex.reason = "server descriptor from replica is from another application (`" + info.uuid + "')";
 		throw ex;
 	    }
-	    else if(_info.revision > info.revision)
+	    else if(_info.revision != info.revision)
 	    {
 		ostringstream os;
-		os << "server descriptor from replica is too old:\n";
+		os << "server descriptor from replica has different version:\n";
 		os << "current revision: " << _info.revision << "\n";
  		os << "replica revision: " << info.revision;
 		throw DeploymentException(os.str());
@@ -1794,6 +1795,10 @@ ServerI::updateImpl(const ServerInfo& info)
 	{
 	    t->second->destroy();
 	}
+	catch(const Ice::ObjectAdapterDeactivatedException&)
+	{
+	    // IGNORE
+	}
 	catch(const Ice::LocalException& ex)
 	{
 	    Ice::Error out(_node->getTraceLevels()->logger);
@@ -2291,6 +2296,7 @@ ServerI::setStateNoSync(InternalServerState st, const std::string& reason)
 	}
 	catch(const Ice::ObjectAdapterDeactivatedException&)
 	{
+	    // IGNORE
 	}
 	_info = ServerInfo();
     }
@@ -2408,14 +2414,27 @@ ServerI::addAdapter(const AdapterDescriptor& desc, const CommunicatorDescriptorP
     Ice::Identity id;
     id.category = _this->ice_getIdentity().category + "Adapter";
     id.name = _id + "-" + desc.id;
-    AdapterPrx proxy = AdapterPrx::uncheckedCast(_node->getAdapter()->createProxy(id));
-    ServerAdapterIPtr servant = ServerAdapterIPtr::dynamicCast(_node->getAdapter()->find(id));
-    if(!servant)
+    try
+    {
+	AdapterPrx proxy = AdapterPrx::uncheckedCast(_node->getAdapter()->createProxy(id));
+	ServerAdapterIPtr servant = ServerAdapterIPtr::dynamicCast(_node->getAdapter()->find(id));
+	if(!servant)
+	{
+	    servant = new ServerAdapterI(_node, this, _id, proxy, desc.id, _waitTime);
+	    _node->getAdapter()->add(servant, id);
+	}
+ 	_adapters.insert(make_pair(desc.id, servant));
+    }
+    catch(const Ice::ObjectAdapterDeactivatedException&)
     {
-	servant = new ServerAdapterI(_node, this, _id, proxy, desc.id, _waitTime);
-	_node->getAdapter()->add(servant, id);
+	// IGNORE
     }
-    _adapters.insert(make_pair(desc.id, servant));
+    catch(const Ice::LocalException& ex)
+    {
+	Ice::Error out(_node->getTraceLevels()->logger);
+	out << "couldn't add adapter `" << desc.id << "':\n" << ex;
+    }
+
     return desc.id;
 }
 
diff --git a/cpp/src/IceGrid/SessionManager.h b/cpp/src/IceGrid/SessionManager.h
index 0dcb3b006e2..02558d63848 100644
--- a/cpp/src/IceGrid/SessionManager.h
+++ b/cpp/src/IceGrid/SessionManager.h
@@ -143,7 +143,7 @@ public:
     }
 
     virtual bool
-    tryCreateSession(InternalRegistryPrx registry)
+    tryCreateSession(const InternalRegistryPrx& registry)
     {
 	{
 	    Lock sync(*this);
@@ -170,7 +170,7 @@ public:
 	    {
 		wait();
 	    }
-	}	
+	}
 	return true;
     }
 
diff --git a/cpp/src/IceGrid/SessionServantLocatorI.cpp b/cpp/src/IceGrid/SessionServantLocatorI.cpp
index b35941d7cda..3eec8bab349 100644
--- a/cpp/src/IceGrid/SessionServantLocatorI.cpp
+++ b/cpp/src/IceGrid/SessionServantLocatorI.cpp
@@ -41,6 +41,8 @@ SessionServantLocatorI::finished(const Ice::Current&, const Ice::ObjectPtr&, con
 void
 SessionServantLocatorI::deactivate(const std::string&)
 {
+    Lock sync(*this);
+    _servants.clear();
 }
 
 Ice::ObjectPrx