Replication fixes, more work on the replication tests

25 files changed, 691 insertions, 128 deletions

diff --git a/cpp/config/IceGridAdmin.py b/cpp/config/IceGridAdmin.py
index a946fcc081d..0e72e2df35f 100644
--- a/cpp/config/IceGridAdmin.py
+++ b/cpp/config/IceGridAdmin.py
@@ -42,6 +42,8 @@ nodeOptions = r' --Ice.Warn.Connections=0' + \
 registryOptions = r' --Ice.Warn.Connections=0' + \
                   r' --IceGrid.Registry.PermissionsVerifier=IceGrid/NullPermissionsVerifier' + \
                   r' --IceGrid.Registry.AdminPermissionsVerifier=IceGrid/NullPermissionsVerifier' + \
+                  r' --IceGrid.Registry.SSLPermissionsVerifier=IceGrid/NullSSLPermissionsVerifier' + \
+                  r' --IceGrid.Registry.AdminSSLPermissionsVerifier=IceGrid/NullSSLPermissionsVerifier' + \
                   r' --IceGrid.Registry.Server.Endpoints=default' + \
                   r' --IceGrid.Registry.Internal.Endpoints=default' + \
                   r' --IceGrid.Registry.SessionManager.Endpoints=default' + \
@@ -205,7 +207,6 @@ def iceGridTest(name, application, additionalOptions = "", applicationOptions =
         print "ok"
 
     print "starting client...",
-
     command = client + TestUtil.clientOptions + " " + clientOptions
 
     if TestUtil.debug:
diff --git a/cpp/demo/IceGrid/simple/config.replica1 b/cpp/demo/IceGrid/simple/config.replica1
index 65fe6f3a7ac..a7acf3e3357 100644
--- a/cpp/demo/IceGrid/simple/config.replica1
+++ b/cpp/demo/IceGrid/simple/config.replica1
@@ -17,5 +17,5 @@ IceGrid.Registry.AdminPermissionsVerifier=DemoIceGrid/NullPermissionsVerifier
 #
 # Trace properties.
 #
-IceGrid.Registry.Trace.Node=1
+IceGrid.Registry.Trace.Node=2
 IceGrid.Registry.Trace.Replica=2
diff --git a/cpp/demo/IceGrid/simple/config.replica2 b/cpp/demo/IceGrid/simple/config.replica2
index e5f33208543..afbfaa4f4eb 100644
--- a/cpp/demo/IceGrid/simple/config.replica2
+++ b/cpp/demo/IceGrid/simple/config.replica2
@@ -17,5 +17,5 @@ IceGrid.Registry.AdminPermissionsVerifier=DemoIceGrid/NullPermissionsVerifier
 #
 # Trace properties.
 #
-IceGrid.Registry.Trace.Node=1
+IceGrid.Registry.Trace.Node=2
 IceGrid.Registry.Trace.Replica=2
diff --git a/cpp/src/IceGrid/Database.cpp b/cpp/src/IceGrid/Database.cpp
index 82d16f884b3..954efeb75fa 100644
--- a/cpp/src/IceGrid/Database.cpp
+++ b/cpp/src/IceGrid/Database.cpp
@@ -193,12 +193,6 @@ Database::~Database()
 {
 }
 
-void
-Database::destroy()
-{
-    _nodeCache.destroy(); // Break cyclic reference count.
-}
-
 std::string
 Database::getInstanceName() const
 {
diff --git a/cpp/src/IceGrid/Database.h b/cpp/src/IceGrid/Database.h
index 5e23eea25bf..5fb64b72996 100644
--- a/cpp/src/IceGrid/Database.h
+++ b/cpp/src/IceGrid/Database.h
@@ -54,8 +54,6 @@ public:
 	     const RegistryInfo&, bool);
     virtual ~Database();
     
-    void destroy();
-
     std::string getInstanceName() const;
     bool isMaster() const { return _master; }
     const TraceLevelsPtr& getTraceLevels() const { return _traceLevels; }
diff --git a/cpp/src/IceGrid/NodeCache.cpp b/cpp/src/IceGrid/NodeCache.cpp
index 645a7f60475..21619cc5be7 100644
--- a/cpp/src/IceGrid/NodeCache.cpp
+++ b/cpp/src/IceGrid/NodeCache.cpp
@@ -192,16 +192,6 @@ NodeCache::NodeCache(const Ice::CommunicatorPtr& communicator, ReplicaCache& rep
 {
 }
 
-void
-NodeCache::destroy()
-{
-    map<string, NodeEntryPtr> entries = _entries; // Copying the map is necessary as setSession might remove the entry.
-    for(map<string, NodeEntryPtr>::const_iterator p = entries.begin(); p != entries.end(); ++p)
-    {
-	p->second->setSession(0); // Break cyclic reference count.
-    }    
-}
-
 NodeEntryPtr
 NodeCache::get(const string& name, bool create) const
 {
diff --git a/cpp/src/IceGrid/NodeCache.h b/cpp/src/IceGrid/NodeCache.h
index cfedcea35b6..6060ec912c5 100644
--- a/cpp/src/IceGrid/NodeCache.h
+++ b/cpp/src/IceGrid/NodeCache.h
@@ -79,8 +79,6 @@ public:
 
     NodeCache(const Ice::CommunicatorPtr&, ReplicaCache&, bool);
 
-    void destroy();
-
     NodeEntryPtr get(const std::string&, bool = false) const;
 
     const Ice::CommunicatorPtr& getCommunicator() const { return _communicator; }
diff --git a/cpp/src/IceGrid/NodeI.cpp b/cpp/src/IceGrid/NodeI.cpp
index 0c3d1bce136..463296597ad 100644
--- a/cpp/src/IceGrid/NodeI.cpp
+++ b/cpp/src/IceGrid/NodeI.cpp
@@ -182,6 +182,7 @@ NodeI::NodeI(const Ice::ObjectAdapterPtr& adapter,
 	     const NodePrx& proxy,
 	     const string& name,
 	     const UserAccountMapperPrx& mapper) :
+    _communicator(adapter->getCommunicator()),
     _adapter(adapter),
     _sessions(sessions),
     _activator(activator),
@@ -192,14 +193,14 @@ NodeI::NodeI(const Ice::ObjectAdapterPtr& adapter,
     _waitTime(0),
     _userAccountMapper(mapper),
     _serial(1),
-    _platform("IceGrid.Node", adapter->getCommunicator(), _traceLevels)
+    _platform("IceGrid.Node", _communicator, _traceLevels)
 {
     _dataDir = _platform.getDataDir();
     _serversDir = _dataDir + "/servers";
     _tmpDir = _dataDir + "/tmp";
 
-    Ice::PropertiesPtr properties = getCommunicator()->getProperties();
-    const_cast<string&>(_instanceName) = getCommunicator()->getDefaultLocator()->ice_getIdentity().category;
+    Ice::PropertiesPtr properties = _communicator->getProperties();
+    const_cast<string&>(_instanceName) = _communicator->getDefaultLocator()->ice_getIdentity().category;
     const_cast<Ice::Int&>(_waitTime) = properties->getPropertyAsIntWithDefault("IceGrid.Node.WaitTime", 60);
 }
 
@@ -395,7 +396,7 @@ NodeI::patch(const string& application,
 	    FileServerPrx icepatch;
 	    if(!appDistrib.icepatch.empty())
 	    {
-		icepatch = FileServerPrx::checkedCast(getCommunicator()->stringToProxy(appDistrib.icepatch));
+		icepatch = FileServerPrx::checkedCast(_communicator->stringToProxy(appDistrib.icepatch));
 		if(!icepatch)
 		{
 		    throw "proxy `" + appDistrib.icepatch + "' is not a file server.";
@@ -414,7 +415,7 @@ NodeI::patch(const string& application,
 		    continue;
 		}
 
-		icepatch = FileServerPrx::checkedCast(getCommunicator()->stringToProxy(dist.icepatch));
+		icepatch = FileServerPrx::checkedCast(_communicator->stringToProxy(dist.icepatch));
 		if(!icepatch)
 		{
 		    throw "proxy `" + dist.icepatch + "' is not a file server.";
@@ -512,7 +513,7 @@ NodeI::shutdown(const Ice::Current&) const
 Ice::CommunicatorPtr
 NodeI::getCommunicator() const
 {
-    return _adapter->getCommunicator();
+    return _communicator;
 }
 
 Ice::ObjectAdapterPtr
diff --git a/cpp/src/IceGrid/NodeI.h b/cpp/src/IceGrid/NodeI.h
index 13b2adfa42d..322cb69b76d 100644
--- a/cpp/src/IceGrid/NodeI.h
+++ b/cpp/src/IceGrid/NodeI.h
@@ -81,6 +81,7 @@ private:
     std::set<ServerIPtr> getApplicationServers(const std::string&) const;
     Ice::Identity createServerIdentity(const std::string&) const;
 
+    const Ice::CommunicatorPtr _communicator;
     const Ice::ObjectAdapterPtr _adapter;
     NodeSessionManager& _sessions;
     const ActivatorPtr _activator;
diff --git a/cpp/src/IceGrid/NodeSessionManager.cpp b/cpp/src/IceGrid/NodeSessionManager.cpp
index c6742056400..bc22006450d 100644
--- a/cpp/src/IceGrid/NodeSessionManager.cpp
+++ b/cpp/src/IceGrid/NodeSessionManager.cpp
@@ -18,12 +18,12 @@ using namespace IceGrid;
 
 NodeSessionKeepAliveThread::NodeSessionKeepAliveThread(const InternalRegistryPrx& registry,
 						       const NodeIPtr& node,
-						       const IceGrid::QueryPrx& query) : 
+						       const vector<QueryPrx>& queryObjects) : 
     SessionKeepAliveThread<NodeSessionPrx>(registry),
     _node(node),
-    _query(query)
+    _queryObjects(queryObjects)
 {
-    assert(registry && node && query);
+    assert(registry && node && !_queryObjects.empty());
     string name = registry->ice_getIdentity().name;
     const string prefix("InternalRegistry-");
     string::size_type pos = name.find(prefix);
@@ -48,6 +48,7 @@ NodeSessionKeepAliveThread::createSession(const InternalRegistryPrx& registry, I
 	    out << "trying to establish session with replica `" << _name << "'";
 	}
 
+	set<InternalRegistryPrx> used;
 	if(!registry->ice_getEndpoints().empty())
 	{
 	    try
@@ -57,25 +58,35 @@ NodeSessionKeepAliveThread::createSession(const InternalRegistryPrx& registry, I
 	    catch(const Ice::LocalException& ex)
 	    {
 		exception.reset(ex.ice_clone());
+		used.insert(registry);
 		setRegistry(InternalRegistryPrx::uncheckedCast(registry->ice_endpoints(Ice::EndpointSeq())));
 	    }
 	}
 
 	if(!session)
 	{
-	    try
+	    for(vector<QueryPrx>::const_iterator p = _queryObjects.begin(); p != _queryObjects.end(); ++p)
 	    {
-		Ice::ObjectPrx obj = _query->findObjectById(registry->ice_getIdentity());
-		InternalRegistryPrx newRegistry = InternalRegistryPrx::uncheckedCast(obj);
-		if(newRegistry && newRegistry != registry)
+		InternalRegistryPrx newRegistry;
+		try
 		{
-		    session = createSessionImpl(newRegistry, timeout);
-		    setRegistry(newRegistry);
+		    Ice::ObjectPrx obj = (*p)->findObjectById(registry->ice_getIdentity());
+		    newRegistry = InternalRegistryPrx::uncheckedCast(obj);
+		    if(newRegistry && used.find(newRegistry) == used.end())
+		    {
+			session = createSessionImpl(newRegistry, timeout);
+			setRegistry(newRegistry);
+			break;
+		    }
+		}
+		catch(const Ice::LocalException& ex)
+		{
+		    exception.reset(ex.ice_clone());
+		    if(newRegistry)
+		    {
+			used.insert(newRegistry);
+		    }
 		}
-	    }
-	    catch(const Ice::LocalException& ex)
-	    {
-		exception.reset(ex.ice_clone());
 	    }
 	}
     }
@@ -197,8 +208,22 @@ NodeSessionManager::create(const NodeIPtr& node)
     assert(communicator->getDefaultLocator());
     Ice::Identity id = communicator->getDefaultLocator()->ice_getIdentity();
 
+    //
+    // Initialize the IceGrid::Query objects. The IceGrid::Query
+    // interface is used to lookup the registry proxy in case it
+    // becomes unavailable. Since replicas might not always have
+    // an up to date registry proxy, we need to query all the
+    // replicas.
+    //
+    Ice::EndpointSeq endpoints = communicator->getDefaultLocator()->ice_getEndpoints();
     id.name = "Query";
-    _query = QueryPrx::uncheckedCast(communicator->stringToProxy(communicator->identityToString(id)));
+    QueryPrx query = QueryPrx::uncheckedCast(communicator->stringToProxy(communicator->identityToString(id)));
+    for(Ice::EndpointSeq::const_iterator p = endpoints.begin(); p != endpoints.end(); ++p)
+    {
+	Ice::EndpointSeq singleEndpoint;
+	singleEndpoint.push_back(*p);
+	_queryObjects.push_back(QueryPrx::uncheckedCast(query->ice_endpoints(singleEndpoint)));
+    }
 
     id.name = "InternalRegistry-Master";
     _master = InternalRegistryPrx::uncheckedCast(communicator->stringToProxy(communicator->identityToString(id)));
@@ -281,7 +306,7 @@ NodeSessionManager::replicaAdded(const InternalRegistryPrx& replica)
 	return p->second;
     }
 
-    NodeSessionKeepAliveThreadPtr thread = new NodeSessionKeepAliveThread(replica, _node, _query);
+    NodeSessionKeepAliveThreadPtr thread = new NodeSessionKeepAliveThread(replica, _node, _queryObjects);
     _sessions.insert(make_pair(replica->ice_getIdentity(), thread));
     thread->start();
     return thread;
@@ -334,14 +359,26 @@ NodeSessionManager::syncReplicas(const InternalRegistryPrxSeq& replicas)
 	}
 	else
 	{
-	    thread = new NodeSessionKeepAliveThread(*p, _node, _query);
+	    thread = new NodeSessionKeepAliveThread(*p, _node, _queryObjects);
 	    thread->start();
 	    thread->tryCreateSession(*p);
 	}
 	_sessions.insert(make_pair((*p)->ice_getIdentity(), thread));
     }
 
-    NodeSessionMap::const_iterator q;
+    NodeSessionMap::iterator q = sessions.begin();
+    while(q != sessions.end())
+    {
+	if(q->second->getSession()) // Don't destroy sessions which are still alive!
+	{
+	    _sessions.insert(make_pair(q->first, q->second));
+	    sessions.erase(q++);
+	}
+	else
+	{
+	    ++q;
+	}
+    }
     for(q = sessions.begin(); q != sessions.end(); ++q)
     {
 	q->second->terminate();
@@ -385,10 +422,22 @@ NodeSessionManager::createdSession(const NodeSessionPrx& session)
 	    else
 	    {
 		replicas.clear();
-		Ice::ObjectProxySeq proxies = _query->findAllObjectsByType(InternalRegistry::ice_staticId());
-		for(Ice::ObjectProxySeq::const_iterator p = proxies.begin(); p != proxies.end(); ++p)
+		set<Ice::ObjectPrx> proxies;
+		for(vector<QueryPrx>::const_iterator p = _queryObjects.begin(); p != _queryObjects.end(); ++p)
+		{
+		    try
+		    {
+			Ice::ObjectProxySeq prxs = (*p)->findAllObjectsByType(InternalRegistry::ice_staticId());
+			proxies.insert(prxs.begin(), prxs.end());
+		    }
+		    catch(const Ice::LocalException& ex)
+		    {
+			// IGNORE
+		    }
+		}
+		for(set<Ice::ObjectPrx>::const_iterator q = proxies.begin(); q != proxies.end(); ++q)
 		{
-		    replicas.push_back(InternalRegistryPrx::uncheckedCast(*p));
+		    replicas.push_back(InternalRegistryPrx::uncheckedCast(*q));
 		}
 	    }
 	}
diff --git a/cpp/src/IceGrid/NodeSessionManager.h b/cpp/src/IceGrid/NodeSessionManager.h
index 2988c869573..84fa666e638 100644
--- a/cpp/src/IceGrid/NodeSessionManager.h
+++ b/cpp/src/IceGrid/NodeSessionManager.h
@@ -28,7 +28,7 @@ class NodeSessionKeepAliveThread : public SessionKeepAliveThread<NodeSessionPrx>
 {
 public:
 
-    NodeSessionKeepAliveThread(const InternalRegistryPrx&, const NodeIPtr&, const IceGrid::QueryPrx&);
+    NodeSessionKeepAliveThread(const InternalRegistryPrx&, const NodeIPtr&, const std::vector<QueryPrx>&);
 
     virtual NodeSessionPrx createSession(const InternalRegistryPrx&, IceUtil::Time&);
     virtual void destroySession(const NodeSessionPrx&);
@@ -40,7 +40,7 @@ protected:
 
     const NodeIPtr _node;
     const std::string _name;
-    const IceGrid::QueryPrx _query;
+    const std::vector<QueryPrx> _queryObjects;
 };
 typedef IceUtil::Handle<NodeSessionKeepAliveThread> NodeSessionKeepAliveThreadPtr;
 
@@ -69,7 +69,7 @@ private:
     public:
 
 	Thread(NodeSessionManager& manager) : 
-	    NodeSessionKeepAliveThread(manager._master, manager._node, manager._query),
+	    NodeSessionKeepAliveThread(manager._master, manager._node, manager._queryObjects),
 	    _manager(manager)
         {
 	}
@@ -93,7 +93,7 @@ private:
 
     const NodeIPtr _node;
     ThreadPtr _thread;
-    QueryPrx _query;
+    std::vector<QueryPrx> _queryObjects;
     InternalRegistryPrx _master;
     unsigned long _serial;
     bool _destroyed;
diff --git a/cpp/src/IceGrid/ReapThread.cpp b/cpp/src/IceGrid/ReapThread.cpp
index 22be3410b1e..65234e93807 100644
--- a/cpp/src/IceGrid/ReapThread.cpp
+++ b/cpp/src/IceGrid/ReapThread.cpp
@@ -90,6 +90,7 @@ ReapThread::terminate()
 	Lock sync(*this);
 	if(_terminated)
 	{
+	    assert(_sessions.empty());
 	    return;
 	}
 	_terminated = true;
@@ -97,7 +98,7 @@ ReapThread::terminate()
 	reap.swap(_sessions);
     }
 
-    for(list<ReapableItem>::const_iterator p = reap.begin(); p != reap.end(); ++p)
+    for(list<ReapableItem>::iterator p = reap.begin(); p != reap.end(); ++p)
     {
 	p->item->destroy(true);
     }
diff --git a/cpp/src/IceGrid/RegistryI.cpp b/cpp/src/IceGrid/RegistryI.cpp
index f4790311f6b..f3654aa3718 100644
--- a/cpp/src/IceGrid/RegistryI.cpp
+++ b/cpp/src/IceGrid/RegistryI.cpp
@@ -301,7 +301,7 @@ RegistryI::start(bool nowarn)
     InternalRegistryPrx internalRegistry = setupInternalRegistry(registryAdapter);
     if(_master)
     {
-	NodePrxSeq nodes = registerReplicas(internalRegistry, replicas);
+	nodes = registerReplicas(internalRegistry, replicas, nodes);
 	registerNodes(internalRegistry, nodes);
     }
     else
@@ -390,7 +390,10 @@ RegistryI::setupLocator(const Ice::ObjectAdapterPtr& clientAdapter,
     Identity locatorId;
     locatorId.category = _instanceName;
     locatorId.name = "Locator";
-    clientAdapter->add(new LocatorI(_communicator, _database, locatorRegistry), locatorId);
+    LocatorPtr locator = new LocatorI(_communicator, _database, locatorRegistry);
+    clientAdapter->add(locator, locatorId);
+    locatorId.name = "Locator-" + _replicaName;
+    clientAdapter->add(locator, locatorId);
     
     obj = registryAdapter->addWithUUID(new LocatorI(_communicator, _database, locatorRegistry));
     return LocatorPrx::uncheckedCast(obj);
@@ -613,7 +616,9 @@ RegistryI::stop()
 	_iceStorm = 0;
     }
 
-    _database->destroy();
+    _wellKnownObjects = 0;
+    _clientSessionFactory = 0;
+    _adminSessionFactory = 0;
     _database = 0;
 }
 
@@ -1112,31 +1117,55 @@ RegistryI::getSSLInfo(const ConnectionPtr& connection, string& userDN)
 }
 
 NodePrxSeq
-RegistryI::registerReplicas(const InternalRegistryPrx& internalRegistry, const InternalRegistryPrxSeq& replicas)
+RegistryI::registerReplicas(const InternalRegistryPrx& internalRegistry, 
+			    const InternalRegistryPrxSeq& replicas,
+			    const NodePrxSeq& dbNodes)
 {
     set<NodePrx> nodes;
+    nodes.insert(dbNodes.begin(), dbNodes.end());
+
     for(InternalRegistryPrxSeq::const_iterator r = replicas.begin(); r != replicas.end(); ++r)
     {
 	if((*r)->ice_getIdentity() != internalRegistry->ice_getIdentity())
 	{
+	    string replicaName;
+	    if(_traceLevels && _traceLevels->replica > 1)
+	    {
+		replicaName = (*r)->ice_getIdentity().name;
+		const string prefix("InternalRegistry-");
+		string::size_type pos = replicaName.find(prefix);
+		if(pos != string::npos)
+		{
+		    replicaName = replicaName.substr(prefix.size());
+		}
+
+		Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
+		out << "creating replica `" << replicaName << "' session";
+	    }
+
 	    try
 	    {
 		(*r)->registerWithReplica(internalRegistry);
 		NodePrxSeq nds = (*r)->getNodes();
 		nodes.insert(nds.begin(), nds.end());
+
+		if(_traceLevels && _traceLevels->replica > 1)
+		{
+		    Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
+		    out << "replica `" << replicaName << "' session created";
+		}
 	    }
-	    catch(const Ice::LocalException&)
+	    catch(const Ice::LocalException& ex)
 	    {
+		if(_traceLevels && _traceLevels->replica > 1)
+		{
+		    Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
+		    out << "replica `" << replicaName << "' session creation failed:\n" << ex;
+		}
 	    }
 	}
     }
     
-    if(nodes.empty())
-    {
-	NodePrxSeq nds = internalRegistry->getNodes();
-	nodes.insert(nds.begin(), nds.end());
-    }
-
 #ifdef _RWSTD_NO_MEMBER_TEMPLATES
     NodePrxSeq result;
     for(set<NodePrx>::iterator p = nodes.begin(); p != nodes.end(); ++p)
@@ -1154,12 +1183,38 @@ RegistryI::registerNodes(const InternalRegistryPrx& internalRegistry, const Node
 {
     for(NodePrxSeq::const_iterator p = nodes.begin(); p != nodes.end(); ++p)
     {
+	string nodeName;
+	if(_traceLevels && _traceLevels->node > 1)
+	{
+	    nodeName = (*p)->ice_getIdentity().name;
+	    const string prefix("Node-");
+	    string::size_type pos = nodeName.find(prefix);
+	    if(pos != string::npos)
+	    {
+		nodeName = nodeName.substr(prefix.size());
+	    }
+	    
+	    Ice::Trace out(_traceLevels->logger, _traceLevels->nodeCat);
+	    out << "creating node `" << nodeName << "' session";
+	}
+	
 	try
 	{
 	    NodePrx::uncheckedCast(*p)->registerWithReplica(internalRegistry);
+	    
+	    if(_traceLevels && _traceLevels->node > 1)
+	    {
+		Ice::Trace out(_traceLevels->logger, _traceLevels->nodeCat);
+		out << "node `" << nodeName << "' session created";
+	    }
 	}
-	catch(const Ice::LocalException&)
+	catch(const Ice::LocalException& ex)
 	{
+	    if(_traceLevels && _traceLevels->node > 1)
+	    {
+		Ice::Trace out(_traceLevels->logger, _traceLevels->nodeCat);
+		out << "node `" << nodeName << "' session creation failed:\n" << ex;
+	    }
 	}
     }
 }
diff --git a/cpp/src/IceGrid/RegistryI.h b/cpp/src/IceGrid/RegistryI.h
index b1b0e9463f0..9dd9580ae99 100644
--- a/cpp/src/IceGrid/RegistryI.h
+++ b/cpp/src/IceGrid/RegistryI.h
@@ -90,7 +90,7 @@ private:
     Glacier2::SSLPermissionsVerifierPrx getSSLPermissionsVerifier(const Ice::LocatorPrx&, const std::string&, bool);
     Glacier2::SSLInfo getSSLInfo(const Ice::ConnectionPtr&, std::string&);
 
-    NodePrxSeq registerReplicas(const InternalRegistryPrx&, const InternalRegistryPrxSeq&);
+    NodePrxSeq registerReplicas(const InternalRegistryPrx&, const InternalRegistryPrxSeq&, const NodePrxSeq&);
     void registerNodes(const InternalRegistryPrx&, const NodePrxSeq&);
     
     const Ice::CommunicatorPtr _communicator;
diff --git a/cpp/src/IceGrid/ReplicaSessionI.cpp b/cpp/src/IceGrid/ReplicaSessionI.cpp
index dc432499f85..78197e3ea1c 100644
--- a/cpp/src/IceGrid/ReplicaSessionI.cpp
+++ b/cpp/src/IceGrid/ReplicaSessionI.cpp
@@ -110,6 +110,13 @@ ReplicaSessionI::registerWellKnownObjects(const ObjectInfoSeq& objects, const Ic
 	_replicaWellKnownObjects = objects;
 	serial = _database->addOrUpdateObjectsInDatabase(objects);
     }
+
+    //
+    // We wait for the replica to receive the database replication
+    // updates. This is to ensure that the replica well-known objects
+    // are correctly setup when the replica starts accepting requests
+    // from clients (if the replica is being started).
+    //
     _database->getObserverTopic(ObjectObserverTopicName)->waitForSyncedSubscribers(serial, _name);
 }
 
@@ -162,11 +169,9 @@ ReplicaSessionI::destroy(const Ice::Current& current)
 
     if(!shutdown)
     {
-	cerr << "updating well known objects " << _name << endl;
 	_wellKnownObjects->updateReplicatedWellKnownObjects(); // No need to update these if we're shutting down.
     }
 
-    cerr << "removing replica " << _name << endl;
     _database->removeReplica(_name, this, shutdown);
 
     if(current.adapter)
diff --git a/cpp/src/IceGrid/ReplicaSessionManager.cpp b/cpp/src/IceGrid/ReplicaSessionManager.cpp
index 8413261f8af..28c59984c0a 100644
--- a/cpp/src/IceGrid/ReplicaSessionManager.cpp
+++ b/cpp/src/IceGrid/ReplicaSessionManager.cpp
@@ -250,13 +250,15 @@ ReplicaSessionManager::create(const string& name,
 			      const InternalRegistryPrx& internalRegistry)
 {
     Ice::CommunicatorPtr comm = database->getCommunicator();
-    string instName = comm->getDefaultLocator()->ice_getIdentity().category;
-
     {
 	Lock sync(*this);
+
+	Ice::Identity id;
+	id.category = comm->getDefaultLocator()->ice_getIdentity().category;
+	id.name = "InternalRegistry-Master";
+
 	
-	_master = InternalRegistryPrx::uncheckedCast(comm->stringToProxy(instName + "/InternalRegistry-Master"));
-	
+	_master = InternalRegistryPrx::uncheckedCast(comm->stringToProxy(comm->identityToString(id)));
 	_name = name;
 	_info = info;
 	_internalRegistry = internalRegistry;
@@ -272,7 +274,8 @@ ReplicaSessionManager::create(const string& name,
 	// replicas.
 	//
 	Ice::EndpointSeq endpoints = comm->getDefaultLocator()->ice_getEndpoints();
-	QueryPrx query = QueryPrx::uncheckedCast(comm->stringToProxy(instName + "/Query"));
+	id.name = "Query";
+	QueryPrx query = QueryPrx::uncheckedCast(comm->stringToProxy(comm->identityToString(id)));
 	for(Ice::EndpointSeq::const_iterator p = endpoints.begin(); p != endpoints.end(); ++p)
 	{
 	    Ice::EndpointSeq singleEndpoint;
@@ -334,12 +337,23 @@ ReplicaSessionManager::destroy()
 
     _thread->terminate();
     _thread->getThreadControl().join();
+
+    _database = 0;
+    _wellKnownObjects = 0;
 }
 
 void
 ReplicaSessionManager::registerAllWellKnownObjects()
 {
     //
+    // Always register first the well-known objects with the
+    // database. Then, if there's a session, we register them with the
+    // session and this will eventually override the ones with just
+    // registered with the ones from the master.
+    //
+    _wellKnownObjects->registerAll();
+
+    //
     // If there's an active session, register the well-known objects
     // with the session.
     //
@@ -452,17 +466,28 @@ ReplicaSessionManager::createSession(const InternalRegistryPrx& registry, IceUti
 	
     if(session)
     {
-	if(_traceLevels && _traceLevels->replica > 0)
-	{
-	    Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
-	    out << "established session with master replica";
-	}
+	//
+	// Register all the well-known objects with the replica session.
+	//
+	_wellKnownObjects->registerAll(session);
     }
     else
     {
-	if(_traceLevels && _traceLevels->replica > 1)
+	//
+	// Re-register all the well known objects with the local database.
+	//
+	_wellKnownObjects->registerAll();
+    }
+
+    if(_traceLevels && _traceLevels->replica > 0)
+    {
+	Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
+	if(session)
+	{
+	    out << "established session with master replica";
+	}
+	else
 	{
-	    Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
 	    out << "failed to establish session with master replica:\n";
 	    if(exception.get())
 	    {
@@ -497,11 +522,6 @@ ReplicaSessionManager::createSessionImpl(const InternalRegistryPrx& registry, Ic
 	DatabaseObserverPtr servant = new MasterDatabaseObserverI(_thread, _database, session);
 	_observer = DatabaseObserverPrx::uncheckedCast(_database->getInternalAdapter()->addWithUUID(servant));	
 	session->setDatabaseObserver(_observer);
-
-	//
-	// Register all the well-known objects with the replica session.
-	//
-	_wellKnownObjects->registerAll(session);
 	return session;
     }
     catch(const Ice::LocalException&)
@@ -517,11 +537,6 @@ ReplicaSessionManager::createSessionImpl(const InternalRegistryPrx& registry, Ic
 	    }
 	    _observer = 0;
 	}
-
-	//
-	// Re-register all the well known objects with the local database.
-	//
-	_wellKnownObjects->registerAll();
 	throw;
     }
 }
diff --git a/cpp/src/IceGrid/ServerI.cpp b/cpp/src/IceGrid/ServerI.cpp
index 312a37649ea..c64bdf1e3ba 100644
--- a/cpp/src/IceGrid/ServerI.cpp
+++ b/cpp/src/IceGrid/ServerI.cpp
@@ -948,8 +948,9 @@ ServerI::load(const AMD_Node_loadServerPtr& amdCB, const ServerInfo& info, bool
 	// - the application uuid, the application revision and the
 	//   session id didn't change.
 	//
-	// - the load command if from a replica and the given
-	//   descriptor is from another application or out-of-date.
+	// - the load command if from a slave and the given descriptor
+	//   is from another application or doesn't have the same
+	//   version.
 	//
 	// - the descriptor and the session id didn't change.
 	//
@@ -964,10 +965,10 @@ ServerI::load(const AMD_Node_loadServerPtr& amdCB, const ServerInfo& info, bool
 		ex.reason = "server descriptor from replica is from another application (`" + info.uuid + "')";
 		throw ex;
 	    }
-	    else if(_info.revision > info.revision)
+	    else if(_info.revision != info.revision)
 	    {
 		ostringstream os;
-		os << "server descriptor from replica is too old:\n";
+		os << "server descriptor from replica has different version:\n";
 		os << "current revision: " << _info.revision << "\n";
  		os << "replica revision: " << info.revision;
 		throw DeploymentException(os.str());
@@ -1794,6 +1795,10 @@ ServerI::updateImpl(const ServerInfo& info)
 	{
 	    t->second->destroy();
 	}
+	catch(const Ice::ObjectAdapterDeactivatedException&)
+	{
+	    // IGNORE
+	}
 	catch(const Ice::LocalException& ex)
 	{
 	    Ice::Error out(_node->getTraceLevels()->logger);
@@ -2291,6 +2296,7 @@ ServerI::setStateNoSync(InternalServerState st, const std::string& reason)
 	}
 	catch(const Ice::ObjectAdapterDeactivatedException&)
 	{
+	    // IGNORE
 	}
 	_info = ServerInfo();
     }
@@ -2408,14 +2414,27 @@ ServerI::addAdapter(const AdapterDescriptor& desc, const CommunicatorDescriptorP
     Ice::Identity id;
     id.category = _this->ice_getIdentity().category + "Adapter";
     id.name = _id + "-" + desc.id;
-    AdapterPrx proxy = AdapterPrx::uncheckedCast(_node->getAdapter()->createProxy(id));
-    ServerAdapterIPtr servant = ServerAdapterIPtr::dynamicCast(_node->getAdapter()->find(id));
-    if(!servant)
+    try
+    {
+	AdapterPrx proxy = AdapterPrx::uncheckedCast(_node->getAdapter()->createProxy(id));
+	ServerAdapterIPtr servant = ServerAdapterIPtr::dynamicCast(_node->getAdapter()->find(id));
+	if(!servant)
+	{
+	    servant = new ServerAdapterI(_node, this, _id, proxy, desc.id, _waitTime);
+	    _node->getAdapter()->add(servant, id);
+	}
+ 	_adapters.insert(make_pair(desc.id, servant));
+    }
+    catch(const Ice::ObjectAdapterDeactivatedException&)
     {
-	servant = new ServerAdapterI(_node, this, _id, proxy, desc.id, _waitTime);
-	_node->getAdapter()->add(servant, id);
+	// IGNORE
     }
-    _adapters.insert(make_pair(desc.id, servant));
+    catch(const Ice::LocalException& ex)
+    {
+	Ice::Error out(_node->getTraceLevels()->logger);
+	out << "couldn't add adapter `" << desc.id << "':\n" << ex;
+    }
+
     return desc.id;
 }
 
diff --git a/cpp/src/IceGrid/SessionManager.h b/cpp/src/IceGrid/SessionManager.h
index 0dcb3b006e2..02558d63848 100644
--- a/cpp/src/IceGrid/SessionManager.h
+++ b/cpp/src/IceGrid/SessionManager.h
@@ -143,7 +143,7 @@ public:
     }
 
     virtual bool
-    tryCreateSession(InternalRegistryPrx registry)
+    tryCreateSession(const InternalRegistryPrx& registry)
     {
 	{
 	    Lock sync(*this);
@@ -170,7 +170,7 @@ public:
 	    {
 		wait();
 	    }
-	}	
+	}
 	return true;
     }
 
diff --git a/cpp/src/IceGrid/SessionServantLocatorI.cpp b/cpp/src/IceGrid/SessionServantLocatorI.cpp
index b35941d7cda..3eec8bab349 100644
--- a/cpp/src/IceGrid/SessionServantLocatorI.cpp
+++ b/cpp/src/IceGrid/SessionServantLocatorI.cpp
@@ -41,6 +41,8 @@ SessionServantLocatorI::finished(const Ice::Current&, const Ice::ObjectPtr&, con
 void
 SessionServantLocatorI::deactivate(const std::string&)
 {
+    Lock sync(*this);
+    _servants.clear();
 }
 
 Ice::ObjectPrx
diff --git a/cpp/test/IceGrid/replication/AllTests.cpp b/cpp/test/IceGrid/replication/AllTests.cpp
index 759a9170276..3c4120a72f5 100644
--- a/cpp/test/IceGrid/replication/AllTests.cpp
+++ b/cpp/test/IceGrid/replication/AllTests.cpp
@@ -14,6 +14,7 @@
 #include <IceGrid/Query.h>
 #include <IceGrid/Registry.h>
 #include <IceGrid/Admin.h>
+#include <IceGrid/UserAccountMapper.h>
 #include <TestCommon.h>
 #include <Test.h>
 
@@ -73,17 +74,59 @@ typedef IceUtil::Handle<SessionKeepAliveThread> SessionKeepAliveThreadPtr;
 void
 waitForRegistryState(const IceGrid::AdminPrx& admin, const std::string& registry, bool up)
 {
+//     int nRetry = 0;
+//     while(nRetry < 15)
+//     {
+// 	try
+// 	{
+// 	    if(admin->pingRegistry(registry) && up) // Wait for the registry to be removed.
+// 	    {
+// 		return;
+// 	    }
+// 	}
+// 	catch(const RegistryNotExistException&)
+// 	{
+// 	    if(!up)
+// 	    {
+// 		return;
+// 	    }
+// 	}
+	
+// 	IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
+// 	++nRetry;
+//     }
+//     if(admin->pingRegistry(registry) != up)
+//     {
+// 	cerr << "registry state change timed out:" << endl;
+// 	cerr << "registry: " << registry << endl;
+// 	cerr << "state: " << up << endl;
+//     }
+
+    int nRetry = 0;
+    while(nRetry < 15)
+    {
+	if(admin->getServerState(registry) == (up ? Active : Inactive))
+	{
+	    return;
+	} 
+    }
+    test(false);
+}
+
+void
+waitForNodeState(const IceGrid::AdminPrx& admin, const std::string& node, bool up)
+{
     int nRetry = 0;
     while(nRetry < 15)
     {
 	try
 	{
-	    if(admin->pingRegistry(registry) && up) // Wait for the registry to be removed.
+	    if(admin->pingNode(node) && up) // Wait for the node to be removed.
 	    {
 		return;
 	    }
 	}
-	catch(const RegistryNotExistException&)
+	catch(const NodeNotExistException&)
 	{
 	    if(!up)
 	    {
@@ -94,10 +137,10 @@ waitForRegistryState(const IceGrid::AdminPrx& admin, const std::string& registry
 	IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
 	++nRetry;
     }
-    if(admin->pingRegistry(registry) != up)
+    if(admin->pingNode(node) != up)
     {
-	cerr << "registry state change timed out:" << endl;
-	cerr << "registry: " << registry << endl;
+	cerr << "node state change timed out:" << endl;
+	cerr << "node: " << node << endl;
 	cerr << "state: " << up << endl;
     }
 }
@@ -231,11 +274,14 @@ allTests(const Ice::CommunicatorPtr& comm)
     instantiateServer(admin, "IceGridRegistry", params);
 
     Ice::LocatorPrx masterLocator = 
-	Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("TestIceGrid/Locator:default -p 12050"));
+	Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("TestIceGrid/Locator-Master:default -p 12050"));
     Ice::LocatorPrx slave1Locator = 
-	Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("TestIceGrid/Locator:default -p 12051"));
+	Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("TestIceGrid/Locator-Slave1:default -p 12051"));
     Ice::LocatorPrx slave2Locator = 
-	Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("TestIceGrid/Locator:default -p 12052"));
+	Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("TestIceGrid/Locator-Slave2:default -p 12052"));
+
+    Ice::LocatorPrx replicatedLocator = 
+	Ice::LocatorPrx::uncheckedCast(comm->stringToProxy("TestIceGrid/Locator:default -p 12050:default -p 12051"));
 
     AdminPrx masterAdmin, slave1Admin, slave2Admin;
 
@@ -305,6 +351,13 @@ allTests(const Ice::CommunicatorPtr& comm)
 	waitForRegistryState(admin, "Slave2", false);
 
 	info = masterAdmin->getObjectInfo(comm->stringToIdentity("TestIceGrid/Locator"));
+	// We eventually need to wait here for the update of the replicated objects to propagate to the replica.
+ 	int nRetry = 0;
+	while(slave1Admin->getObjectInfo(comm->stringToIdentity("TestIceGrid/Locator")) != info && nRetry < 15)
+	{
+	    IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
+	    ++nRetry;
+	}
 	test(slave1Admin->getObjectInfo(comm->stringToIdentity("TestIceGrid/Locator")) == info);
 	test(info.type == Ice::Locator::ice_staticId());
 	endpoints = info.proxy->ice_getEndpoints();
@@ -313,22 +366,129 @@ allTests(const Ice::CommunicatorPtr& comm)
 	test(endpoints[1]->toString() == slave1Locator->ice_getEndpoints()[0]->toString());
 
 	info = masterAdmin->getObjectInfo(comm->stringToIdentity("TestIceGrid/Query"));
+	nRetry = 0;
+	while(slave1Admin->getObjectInfo(comm->stringToIdentity("TestIceGrid/Query")) != info && nRetry < 15)
+	{
+	    IceUtil::ThreadControl::sleep(IceUtil::Time::milliSeconds(500));
+	    ++nRetry;
+	}
 	test(slave1Admin->getObjectInfo(comm->stringToIdentity("TestIceGrid/Query")) == info);
 	test(info.type == IceGrid::Query::ice_staticId());
 	endpoints = info.proxy->ice_getEndpoints();
 	test(endpoints.size() == 2);
 	test(endpoints[0]->toString() == masterLocator->ice_getEndpoints()[0]->toString());
 	test(endpoints[1]->toString() == slave1Locator->ice_getEndpoints()[0]->toString());
-    }
-    cout << "ok" << endl;
 
-    cout << "testing replicated query interface... " << flush;
-    {
+	QueryPrx query;
+	query = QueryPrx::uncheckedCast(comm->stringToProxy("TestIceGrid/Query:" + endpoints[0]->toString()));
+	Ice::ObjectProxySeq objs = query->findAllObjectsByType("::IceGrid::Registry");
+	test(objs.size() == 2);
+	query = QueryPrx::uncheckedCast(comm->stringToProxy("TestIceGrid/Query:" + endpoints[1]->toString()));
+	test(objs == query->findAllObjectsByType("::IceGrid::Registry"));
     }
     cout << "ok" << endl;
 
     cout << "testing well-known IceGrid objects... " << flush;
     {
+	//
+	// Test Registry well-known object (we have already tested
+	// admin session creation for the creation of the admin
+	// session above!)
+	//
+	RegistryPrx masterRegistry = RegistryPrx::checkedCast(
+	    comm->stringToProxy("TestIceGrid/Registry")->ice_locator(replicatedLocator));
+	RegistryPrx slave1Registry = RegistryPrx::checkedCast(
+	    comm->stringToProxy("TestIceGrid/Registry-Slave1")->ice_locator(replicatedLocator));
+
+	SessionPrx session = masterRegistry->createSession("dummy", "dummy");
+	session->destroy();
+	if(comm->getProperties()->getProperty("Ice.Default.Protocol") == "ssl")
+	{ 
+	    session = masterRegistry->createSessionFromSecureConnection();
+	    session->destroy();
+	}
+	else
+	{
+	    try
+	    {
+		masterRegistry->createSessionFromSecureConnection();
+	    }
+	    catch(const PermissionDeniedException&)
+	    {
+	    }
+	}
+
+	try
+	{
+	    slave1Registry->createSession("dummy", "");
+	}
+	catch(const PermissionDeniedException&)
+	{
+	}
+	try
+	{
+	    slave1Registry->createSessionFromSecureConnection();
+	}
+	catch(const PermissionDeniedException&)
+	{
+	}
+
+	//
+	// Test registry user-account mapper.
+	//
+	UserAccountMapperPrx masterMapper = UserAccountMapperPrx::checkedCast(
+	    comm->stringToProxy("TestIceGrid/RegistryUserAccountMapper")->ice_locator(replicatedLocator));
+	UserAccountMapperPrx slave1Mapper = UserAccountMapperPrx::checkedCast(
+	    comm->stringToProxy("TestIceGrid/RegistryUserAccountMapper-Slave1")->ice_locator(replicatedLocator));
+
+	test(masterMapper->getUserAccount("Dummy User Account1") == "dummy1");
+	test(masterMapper->getUserAccount("Dummy User Account2") == "dummy2");
+	test(slave1Mapper->getUserAccount("Dummy User Account1") == "dummy1");
+	test(slave1Mapper->getUserAccount("Dummy User Account2") == "dummy2");
+	try
+	{
+	    masterMapper->getUserAccount("unknown");
+	    test(false);
+	}
+	catch(UserAccountNotFoundException&)
+	{
+	}
+	try
+	{
+	    slave1Mapper->getUserAccount("unknown");
+	    test(false);
+	}
+	catch(UserAccountNotFoundException&)
+	{
+	}
+	
+	//
+	// Test SessionManager, SSLSessionManager,
+	// AdminSessionManager, AdminSSLSessionManager
+	//
+	comm->stringToProxy("TestIceGrid/SessionManager")->ice_locator(replicatedLocator)->ice_ping();
+	comm->stringToProxy("TestIceGrid/SSLSessionManager")->ice_locator(replicatedLocator)->ice_ping();
+	try
+	{
+	    comm->stringToProxy("TestIceGrid/SessionManager-Slave1")->ice_locator(replicatedLocator)->ice_ping();
+	    test(false);
+	}
+	catch(const Ice::NotRegisteredException&)
+	{
+	}
+	try
+	{
+	    comm->stringToProxy("TestIceGrid/SSLSessionManager-Slave1")->ice_locator(replicatedLocator)->ice_ping();
+	    test(false);
+	}
+	catch(const Ice::NotRegisteredException&)
+	{
+	}
+
+	comm->stringToProxy("TestIceGrid/AdminSessionManager")->ice_locator(replicatedLocator)->ice_ping();
+	comm->stringToProxy("TestIceGrid/AdminSSLSessionManager")->ice_locator(replicatedLocator)->ice_ping();
+	comm->stringToProxy("TestIceGrid/AdminSessionManager-Slave1")->ice_locator(replicatedLocator)->ice_ping();
+	comm->stringToProxy("TestIceGrid/AdminSSLSessionManager-Slave1")->ice_locator(replicatedLocator)->ice_ping();
     }
     cout << "ok" << endl;
 
@@ -597,11 +757,9 @@ allTests(const Ice::CommunicatorPtr& comm)
     }
     cout << "ok" << endl;
 
-    removeServer(admin, "Slave2");
-    slave1Admin->shutdown();
-    removeServer(admin, "Slave1");
-    masterAdmin->shutdown();
-    removeServer(admin, "Master");
+    params.clear();
+    params["id"] = "Node1";
+    instantiateServer(admin, "IceGridNode", params);
 
     //
     // Test node session establishment.
@@ -610,9 +768,273 @@ allTests(const Ice::CommunicatorPtr& comm)
     // - shutdown slave1, start slave1 -> node should re-connect
     // - shutdown master
     // - shutdown slave2, start slave2 -> node should re-connect
-    // - start slave3 -> node can't connect to it
     // - shutdown slave1
-    // - start master -> node connects to master, slave3
+    // - start master -> node connects to master
     // - start slave1 -> node connects to slave1
     //
+    cout << "testing node session establishment... " << flush;
+    {
+	admin->startServer("Node1");
+	
+	waitForNodeState(masterAdmin, "Node1", true);
+	waitForNodeState(slave1Admin, "Node1", true);
+
+	admin->startServer("Slave2");
+	slave2Admin = createAdminSession(slave2Locator, "Slave2");
+
+	waitForNodeState(slave2Admin, "Node1", true); // Node should connect.
+
+	slave1Admin->shutdown();
+	waitForRegistryState(admin, "Slave1", false);
+	admin->startServer("Slave1");
+	slave1Admin = createAdminSession(slave1Locator, "Slave1");
+
+	try
+	{
+	    test(slave1Admin->pingNode("Node1")); // Node should be re-connected.
+	}
+	catch(const NodeNotExistException&)
+	{
+	    test(false);
+	}
+
+	masterAdmin->shutdown();
+	waitForRegistryState(admin, "Master", false);
+
+	slave2Admin->shutdown();
+	waitForRegistryState(admin, "Slave2", false);
+	admin->startServer("Slave2");
+	slave2Admin = createAdminSession(slave2Locator, "Slave2");
+
+	try
+	{
+	    test(slave2Admin->pingNode("Node1")); // Node should be re-connected even if the master is down.
+	}
+	catch(const NodeNotExistException&)
+	{
+	    test(false);
+	}
+
+	slave1Admin->shutdown();
+	waitForRegistryState(admin, "Slave1", false);
+
+	admin->startServer("Master");
+	masterAdmin = createAdminSession(masterLocator, "");
+
+	try
+	{
+	    test(masterAdmin->pingNode("Node1")); // Node should be re-connected.
+	}
+	catch(const NodeNotExistException&)
+	{
+	    test(false);
+	}
+
+	admin->startServer("Slave1");
+	slave1Admin = createAdminSession(slave1Locator, "Slave1");
+
+	try
+	{
+	    test(slave1Admin->pingNode("Node1")); // Node should be re-connected.
+	}
+	catch(const NodeNotExistException&)
+	{
+	    test(false);
+	}
+
+	try
+	{
+	    test(masterAdmin->pingNode("Node1"));
+	}
+	catch(const NodeNotExistException&)
+	{
+	    test(false);
+	}
+
+	try
+	{
+	    test(slave2Admin->pingNode("Node1"));
+	}
+	catch(const NodeNotExistException&)
+	{
+	    test(false);
+	}
+
+	slave2Admin->shutdown();
+	waitForRegistryState(admin, "Slave2", false);
+	admin->startServer("Slave2");
+	slave2Admin = createAdminSession(slave2Locator, "Slave2");
+	try
+	{
+	    test(slave2Admin->pingNode("Node1"));
+	}
+	catch(const NodeNotExistException&)
+	{
+	    test(false);
+	}
+    }
+    cout << "ok" << endl;
+
+    //
+    // Testing updates with out-of-date replicas.
+    //
+    cout << "testing out-of-date replicas... " << flush;
+    {
+	ApplicationDescriptor app;
+	app.name = "TestApp";
+	app.description = "added application";
+
+	ServerDescriptorPtr server = new ServerDescriptor();
+	server->id = "Server";
+	server->exe = comm->getProperties()->getProperty("TestDir") + "/server";
+	server->pwd = ".";
+	server->activation = "on-demand";
+	AdapterDescriptor adapter;
+	adapter.name = "TestAdapter";
+	adapter.id = "TestAdapter.Server";
+	adapter.registerProcess = true;
+	PropertyDescriptor property;
+	property.name = "TestAdapter.Endpoints";
+	property.value = "default";
+	server->propertySet.properties.push_back(property);
+	property.name = "Identity";
+	property.value = "test";
+	server->propertySet.properties.push_back(property);
+	ObjectDescriptor object;
+	object.id = comm->stringToIdentity("test");
+	object.type = "::Test::TestIntf";
+	adapter.objects.push_back(object);
+	server->adapters.push_back(adapter);
+	app.nodes["Node1"].servers.push_back(server);
+
+	masterAdmin->addApplication(app);
+
+	comm->stringToProxy("test")->ice_locator(masterLocator)->ice_locatorCacheTimeout(0)->ice_ping();
+	comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
+	comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
+	masterAdmin->stopServer("Server");
+
+	//
+	// Shutdown Slave2 and update application.
+	//
+	slave2Admin->shutdown();
+	waitForRegistryState(admin, "Slave2", false);
+
+	ApplicationUpdateDescriptor update;
+	update.name = "TestApp";
+	NodeUpdateDescriptor node;
+	node.name = "Node1";
+	node.servers.push_back(server);
+	update.nodes.push_back(node);
+	property.name = "Dummy";
+	property.value = "val";
+	server->propertySet.properties.push_back(property);
+	masterAdmin->updateApplication(update);
+
+	comm->stringToProxy("test")->ice_locator(masterLocator)->ice_locatorCacheTimeout(0)->ice_ping();
+	comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
+
+	masterAdmin->shutdown();
+	waitForRegistryState(admin, "Master", false);
+
+	admin->startServer("Slave2");
+	slave2Admin = createAdminSession(slave2Locator, "Slave2");
+	try
+	{
+	    slave2Admin->startServer("Server");
+	    test(false);
+	}
+	catch(const DeploymentException&)
+	{
+	}
+	try
+	{
+	    comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
+	    test(false);
+	}
+	catch(const Ice::NoEndpointException&)
+	{
+	}
+
+	admin->startServer("Master");
+	masterAdmin = createAdminSession(masterLocator, "");
+
+ 	slave2Admin->shutdown();
+ 	waitForRegistryState(admin, "Slave2", false);
+	admin->startServer("Slave2");
+	slave2Admin = createAdminSession(slave2Locator, "Slave2");
+
+	comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
+
+	//
+	// Shutdown Node1 and update the application, then, shutdown
+	// the master.
+	//
+ 	slave1Admin->shutdownNode("Node1");
+
+	property.name = "Dummy2";
+	property.value = "val";
+	server->propertySet.properties.push_back(property);
+	masterAdmin->updateApplication(update);
+
+	masterAdmin->shutdown();
+	waitForRegistryState(admin, "Master", false);
+
+	//
+	// Restart Node1 and Slave2, Slave2 still has the old version
+	// of the server so it should be able to load it. Slave1 has 
+	// a more recent version, so it can't load it.
+	//
+
+	admin->startServer("Node1");
+	
+	waitForNodeState(slave1Admin, "Node1", true);
+	waitForNodeState(slave2Admin, "Node1", true);
+
+	comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
+	slave2Admin->stopServer("Server");
+
+	try
+	{
+	    comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();	
+	}
+	catch(const Ice::NoEndpointException&)
+	{
+	}
+
+	comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
+	slave2Admin->stopServer("Server");
+
+	//
+	// Start the master. This will re-load the server on the node
+	// and update the out-of-date replicas.
+	//
+	admin->startServer("Master");
+	masterAdmin = createAdminSession(masterLocator, "");
+
+	waitForNodeState(masterAdmin, "Node1", true);
+
+	comm->stringToProxy("test")->ice_locator(masterLocator)->ice_locatorCacheTimeout(0)->ice_ping();
+	comm->stringToProxy("test")->ice_locator(slave1Locator)->ice_locatorCacheTimeout(0)->ice_ping();
+	comm->stringToProxy("test")->ice_locator(slave2Locator)->ice_locatorCacheTimeout(0)->ice_ping();
+
+	slave2Admin->stopServer("Server");
+
+	masterAdmin->removeApplication("TestApp");
+    }    
+    cout << "ok" << endl;
+
+    cout << "testing master upgrade... " << flush;
+    {
+    }
+    cout << "ok" << endl;
+    
+    slave1Admin->shutdownNode("Node1");
+    removeServer(admin, "Node1");
+
+    removeServer(admin, "Slave2");
+    slave1Admin->shutdown();
+    removeServer(admin, "Slave1");
+    masterAdmin->shutdown();
+    removeServer(admin, "Master");
 }
diff --git a/cpp/test/IceGrid/replication/Client.cpp b/cpp/test/IceGrid/replication/Client.cpp
index ad32326fc12..791fdb080c7 100644
--- a/cpp/test/IceGrid/replication/Client.cpp
+++ b/cpp/test/IceGrid/replication/Client.cpp
@@ -30,6 +30,7 @@ main(int argc, char* argv[])
     try
     {
 	communicator = Ice::initialize(argc, argv);
+	communicator->getProperties()->parseCommandLineOptions("", Ice::argsToStringSeq(argc, argv));
 	status = run(argc, argv, communicator);
     }
     catch(const Ice::Exception& ex)
diff --git a/cpp/test/IceGrid/replication/application.xml b/cpp/test/IceGrid/replication/application.xml
index 512ef3f3f1d..027af3e54f1 100644
--- a/cpp/test/IceGrid/replication/application.xml
+++ b/cpp/test/IceGrid/replication/application.xml
@@ -13,7 +13,8 @@
         <property name="IceGrid.Node.Data" value="${node.datadir}/servers/${server}/dbs/data"/>
         <property name="IceGrid.Node.PropertiesOverride"
                   value="${properties-override} Ice.ServerIdleTime=0 Ice.PrintProcessId=0 Ice.PrintAdapterReady=0"/>
-        <property name="Ice.Default.Locator" value="TestIceGrid/Locator:default -p 12050"/>
+        <property name="Ice.Default.Locator" value="TestIceGrid/Locator:default -p 12050:default -p 12051:default -p 12052"/>
+        <property name="IceGrid.Node.Trace.Replica" value="0"/> 
       </server>
     </server-template>
 
@@ -22,7 +23,10 @@
       <parameter name="port"/>
       <server id="${id}" exe="${ice.dir}/bin/icegridregistry" activation="manual">
         <option>--nowarn</option>
-        <dbenv name="data"/>
+        <dbenv name="data">
+          <!-- Try to make the test run a bit faster... -->
+          <dbproperty name="set_flags" value="DB_TXN_NOSYNC"/>
+        </dbenv>
         <property name="IceGrid.InstanceName" value="TestIceGrid"/>
         <property name="IceGrid.Registry.Client.Endpoints" value="default -h 127.0.0.1 -p ${port}"/>
         <property name="IceGrid.Registry.Server.Endpoints" value="default"/>
@@ -30,11 +34,16 @@
         <property name="IceGrid.Registry.SessionManager.Endpoints" value="default"/>
         <property name="IceGrid.Registry.ReplicaName" value="${server}"/>
         <property name="IceGrid.Registry.Data" value="${node.datadir}/servers/${server}/dbs/data"/>
+        <property name="IceGrid.Registry.PermissionsVerifier" value="TestIceGrid/NullPermissionsVerifier"/>
+        <property name="IceGrid.Registry.SSLPermissionsVerifier" value="TestIceGrid/NullSSLPermissionsVerifier"/>
         <property name="IceGrid.Registry.AdminPermissionsVerifier" value="TestIceGrid/NullPermissionsVerifier"/>
         <property name="IceGrid.Registry.SessionTimeout" value="0"/>
 	<property name="IceGrid.Registry.DynamicRegistration" value="1"/>
-        <property name="Ice.Default.Locator" value="TestIceGrid/Locator:default -p 12050"/>
-        <property name="IceGrid.Registry.Trace.Replica" value="3"/> 
+        <property name="Ice.Default.Locator" value="TestIceGrid/Locator:default -p 12050:default -p 12051:default -p 12052"/>
+        <property name="IceGrid.Registry.Trace.Replica" value="0"/> 
+        <property name="IceGrid.Registry.Trace.Node" value="0"/> 
+        <property name="IceGrid.Registry.Trace.Locator" value="0"/> 
+        <property name="IceGrid.Registry.UserAccounts" value="${test.dir}/useraccounts.txt"/>
       </server>
     </server-template>
 
diff --git a/cpp/test/IceGrid/replication/run.py b/cpp/test/IceGrid/replication/run.py
index 915581cc4e8..f8888c79309 100755
--- a/cpp/test/IceGrid/replication/run.py
+++ b/cpp/test/IceGrid/replication/run.py
@@ -26,6 +26,6 @@ testdir = os.path.join(toplevel, "test", name)
 
 TestUtil.addLdPath(testdir)
 
-IceGridAdmin.iceGridTest(name, "application.xml", "", \
+IceGridAdmin.iceGridTest(name, "application.xml", "--IceDir=\"" + toplevel + "\" --TestDir=\"" + testdir + "\"", \
                          ' \\"properties-override=' + TestUtil.clientServerOptions.replace("--", "") + '\\"')
 sys.exit(0)
diff --git a/cpp/test/IceGrid/replication/useraccounts.txt b/cpp/test/IceGrid/replication/useraccounts.txt
new file mode 100644
index 00000000000..de136797a01
--- /dev/null
+++ b/cpp/test/IceGrid/replication/useraccounts.txt
@@ -0,0 +1,2 @@
+dummy1 Dummy User Account1
+dummy2 Dummy User Account2
\ No newline at end of file
diff --git a/cpp/test/IceGrid/session/run.py b/cpp/test/IceGrid/session/run.py
index 170f938ef33..328a337a411 100755
--- a/cpp/test/IceGrid/session/run.py
+++ b/cpp/test/IceGrid/session/run.py
@@ -42,7 +42,7 @@ print "ok"
 IceGridAdmin.registryOptions += \
                              r' --IceGrid.Registry.DynamicRegistration' + \
                              r' --IceGrid.Registry.PermissionsVerifier="ClientPermissionsVerifier"' + \
-                             r' --IceGrid.Registry.AdminPermissionsVerifier="AdminPermissionsVerifier:tcp -p 12002"' + \
+                             r' --IceGrid.Registry.AdminPermissionsVerifier="AdminPermissionsVerifier:tcp -p 12002"'+ \
                              r' --IceGrid.Registry.SSLPermissionsVerifier="SSLPermissionsVerifier"'
 
 IceGridAdmin.iceGridTest(name, "application.xml", \