Code cleanup and fixed startup scalability issue of the registry (it no

    longer contacts all the nodes).

21 files changed, 465 insertions, 321 deletions

diff --git a/cpp/src/IceGrid/AdapterCache.cpp b/cpp/src/IceGrid/AdapterCache.cpp
index 087049e83bc..b210c858aae 100644
--- a/cpp/src/IceGrid/AdapterCache.cpp
+++ b/cpp/src/IceGrid/AdapterCache.cpp
@@ -50,14 +50,7 @@ public:
     pair<float, ServerAdapterEntryPtr>
     operator()(const ServerAdapterEntryPtr& value)
     {
-	try
-	{
-	    return make_pair(value->getLeastLoadedNodeLoad(_loadSample), value);
-	}
-	catch(const Ice::Exception&)
-	{
-	    return make_pair(1.0f, value);
-	}
+	return make_pair(value->getLeastLoadedNodeLoad(_loadSample), value);
     }
 
     LoadSample _loadSample;
@@ -75,7 +68,7 @@ struct TransformToReplica : public unary_function<const pair<string, ServerAdapt
 }
 
 ServerAdapterEntryPtr
-AdapterCache::addServerAdapter(const AdapterDescriptor& desc, const ServerEntryPtr& server)
+AdapterCache::addServerAdapter(const AdapterDescriptor& desc, const ServerEntryPtr& server, const string& app)
 {
     Lock sync(*this);
     assert(!getImpl(desc.id));
@@ -84,7 +77,7 @@ AdapterCache::addServerAdapter(const AdapterDescriptor& desc, const ServerEntryP
     int priority = 0;
     is >> priority;
 
-    ServerAdapterEntryPtr entry = new ServerAdapterEntry(*this, desc.id, desc.replicaGroupId, priority, server);
+    ServerAdapterEntryPtr entry = new ServerAdapterEntry(*this, desc.id, app, desc.replicaGroupId, priority, server);
     addImpl(desc.id, entry);
 
     if(!desc.replicaGroupId.empty())
@@ -190,9 +183,10 @@ AdapterCache::removeImpl(const string& id)
     Cache<string, AdapterEntry>::removeImpl(id);
 }
 
-AdapterEntry::AdapterEntry(AdapterCache& cache, const string& id) :
+AdapterEntry::AdapterEntry(AdapterCache& cache, const string& id, const string& application) :
     _cache(cache),
-    _id(id)
+    _id(id),
+    _application(application)
 {
 }
 
@@ -208,12 +202,19 @@ AdapterEntry::getId() const
     return _id;
 }
 
+string
+AdapterEntry::getApplication() const
+{
+    return _application;
+}
+
 ServerAdapterEntry::ServerAdapterEntry(AdapterCache& cache,
 				       const string& id,
+				       const string& application,
 				       const string& replicaGroupId, 
 				       int priority,
 				       const ServerEntryPtr& server) : 
-    AdapterEntry(cache, id),
+    AdapterEntry(cache, id, application),
     _replicaGroupId(replicaGroupId),
     _priority(priority),
     _server(server)
@@ -239,13 +240,27 @@ ServerAdapterEntry::getProxies(int& nReplicas, bool& replicaGroup)
 float
 ServerAdapterEntry::getLeastLoadedNodeLoad(LoadSample loadSample) const
 {
-    return getServer()->getLoad(loadSample);
-}
-
-string
-ServerAdapterEntry::getApplication() const
-{
-    return getServer()->getApplication();
+    try
+    {
+	return _server->getLoad(loadSample);
+    }
+    catch(const ServerNotExistException&)
+    {
+	// This might happen if the application is updated concurrently.
+    }
+    catch(const NodeNotExistException&)
+    {
+	// This might happen if the application is updated concurrently.
+    }
+    catch(const NodeUnreachableException&)
+    {
+    }
+    catch(const Ice::Exception& ex)
+    {
+	Ice::Error error(_cache.getTraceLevels()->logger);
+	error << "unexpected exception while getting node load:\n" << ex;
+    }
+    return 999.9f;
 }
 
 AdapterInfoSeq
@@ -271,11 +286,10 @@ ServerAdapterEntry::getProxy(const string& replicaGroupId, bool upToDate) const
 {
     if(replicaGroupId.empty())
     {
-	return getServer()->getAdapter(_id, upToDate);
+	return _server->getAdapter(_id, upToDate);
     }
     else
     {
-	Lock sync(*this);
 	if(_replicaGroupId != replicaGroupId)
 	{
 	    throw Ice::InvalidReplicaGroupIdException();
@@ -290,20 +304,11 @@ ServerAdapterEntry::getPriority() const
     return _priority;
 }
 
-ServerEntryPtr
-ServerAdapterEntry::getServer() const
-{
-    Lock sync(*this);
-    assert(_server);
-    return _server;
-}
-
 ReplicaGroupEntry::ReplicaGroupEntry(AdapterCache& cache,
 				     const string& id,
 				     const string& application,
 				     const LoadBalancingPolicyPtr& policy) : 
-    AdapterEntry(cache, id),
-    _application(application),
+    AdapterEntry(cache, id, application),
     _lastReplica(0)
 {
     update(policy);
@@ -469,23 +474,22 @@ ReplicaGroupEntry::getLeastLoadedNodeLoad(LoadSample loadSample) const
 	replicas = _replicas;
     }
 
-    //
-    // This must be done outside the synchronization block since
-    // min_element() will call and lock each server entry.
-    //
-    RandomNumberGenerator rng;
-    random_shuffle(replicas.begin(), replicas.end(), rng);
-    vector<pair<float, ServerAdapterEntryPtr> > rl;
-    transform(replicas.begin(), replicas.end(), back_inserter(rl), TransformToReplicaLoad(loadSample));
-    AdapterEntryPtr adpt = min_element(rl.begin(), rl.end(), ReplicaLoadComp())->second;
-    return adpt->getLeastLoadedNodeLoad(loadSample);
-}
-
-string
-ReplicaGroupEntry::getApplication() const
-{
-    Lock sync(*this);
-    return _application;
+    if(replicas.empty())
+    {
+	return 999.9f;
+    }
+    else if(replicas.size() == 1)
+    {
+	return replicas.back()->getLeastLoadedNodeLoad(loadSample);
+    }
+    else
+    {
+	RandomNumberGenerator rng;
+	random_shuffle(replicas.begin(), replicas.end(), rng);
+	vector<pair<float, ServerAdapterEntryPtr> > rl;
+	transform(replicas.begin(), replicas.end(), back_inserter(rl), TransformToReplicaLoad(loadSample));
+	return min_element(rl.begin(), rl.end(), ReplicaLoadComp())->first;
+    }
 }
 
 AdapterInfoSeq
diff --git a/cpp/src/IceGrid/AdapterCache.h b/cpp/src/IceGrid/AdapterCache.h
index cec41f9214f..4fbd397517d 100644
--- a/cpp/src/IceGrid/AdapterCache.h
+++ b/cpp/src/IceGrid/AdapterCache.h
@@ -28,25 +28,26 @@ typedef std::vector<ServerEntryPtr> ServerEntrySeq;
 class AdapterEntry;
 typedef IceUtil::Handle<AdapterEntry> AdapterEntryPtr;
 
-class AdapterEntry : virtual public IceUtil::Shared, public IceUtil::Mutex
+class AdapterEntry : virtual public IceUtil::Shared
 {
 public:
     
-    AdapterEntry(AdapterCache&, const std::string&);
+    AdapterEntry(AdapterCache&, const std::string&, const std::string&);
 
     virtual std::vector<std::pair<std::string, AdapterPrx> > getProxies(int&, bool&) = 0;
     virtual float getLeastLoadedNodeLoad(LoadSample) const = 0;
-    virtual std::string getApplication() const = 0;
     virtual AdapterInfoSeq getAdapterInfo() const = 0;
 
     virtual bool canRemove();
 
     std::string getId() const;
+    std::string getApplication() const;
     
 protected:
 
     AdapterCache& _cache;
     const std::string _id;
+    const std::string _application;
 };
 typedef IceUtil::Handle<AdapterEntry> AdapterEntryPtr;
 
@@ -54,11 +55,11 @@ class ServerAdapterEntry : public AdapterEntry
 {
 public:
 
-    ServerAdapterEntry(AdapterCache&, const std::string&, const std::string&, int, const ServerEntryPtr&);
+    ServerAdapterEntry(AdapterCache&, const std::string&, const std::string&, const std::string&, int, 
+		       const ServerEntryPtr&);
 
     virtual std::vector<std::pair<std::string, AdapterPrx> > getProxies(int&, bool&);
     virtual float getLeastLoadedNodeLoad(LoadSample) const;
-    virtual std::string getApplication() const;
     virtual AdapterInfoSeq getAdapterInfo() const;
     virtual const std::string& getReplicaGroupId() const { return _replicaGroupId; }
 
@@ -66,8 +67,6 @@ public:
     int getPriority() const;
     
 private:
-    
-    ServerEntryPtr getServer() const;
 
     const std::string _replicaGroupId;
     const int _priority;
@@ -75,7 +74,7 @@ private:
 };
 typedef IceUtil::Handle<ServerAdapterEntry> ServerAdapterEntryPtr;
 
-class ReplicaGroupEntry : public AdapterEntry
+class ReplicaGroupEntry : public AdapterEntry, public IceUtil::Mutex
 {
 public:
 
@@ -83,7 +82,6 @@ public:
 
     virtual std::vector<std::pair<std::string, AdapterPrx> > getProxies(int&, bool&);
     virtual float getLeastLoadedNodeLoad(LoadSample) const;
-    virtual std::string getApplication() const;
     virtual AdapterInfoSeq getAdapterInfo() const;
 
     void addReplica(const std::string&, const ServerAdapterEntryPtr&);
@@ -93,8 +91,6 @@ public:
 
 private:
 
-    const std::string _application;
-
     LoadBalancingPolicyPtr _loadBalancing;
     int _loadBalancingNReplicas;
     LoadSample _loadSample;
@@ -107,7 +103,7 @@ class AdapterCache : public CacheByString<AdapterEntry>
 {
 public:
 
-    ServerAdapterEntryPtr addServerAdapter(const AdapterDescriptor&, const ServerEntryPtr&);
+    ServerAdapterEntryPtr addServerAdapter(const AdapterDescriptor&, const ServerEntryPtr&, const std::string&);
     ReplicaGroupEntryPtr addReplicaGroup(const ReplicaGroupDescriptor&, const std::string&);
 
     AdapterEntryPtr get(const std::string&) const;
diff --git a/cpp/src/IceGrid/AdminSessionI.cpp b/cpp/src/IceGrid/AdminSessionI.cpp
index 5fe3f014d5f..db49207138d 100644
--- a/cpp/src/IceGrid/AdminSessionI.cpp
+++ b/cpp/src/IceGrid/AdminSessionI.cpp
@@ -219,44 +219,14 @@ AdminSessionI::openNodeStdErr(const std::string& name, const Ice::Current& curre
 FileIteratorPrx 
 AdminSessionI::openRegistryStdOut(const std::string& name, const Ice::Current& current)
 {
-    FileReaderPrx reader;
-    if(name == _replicaName)
-    {
-	Ice::Identity internalRegistryId;
-	internalRegistryId.category = _database->getInstanceName();
-	internalRegistryId.name = "InternalRegistry-" + _replicaName;
-
-	Ice::CommunicatorPtr communicator = current.adapter->getCommunicator();
-	string proxyStr = communicator->identityToString(internalRegistryId);
-	reader = FileReaderPrx::uncheckedCast(communicator->stringToProxy(proxyStr));
-    }
-    else 
-    {
-	reader = _database->getReplica(name);
-    }
-
+    FileReaderPrx reader = name == _replicaName ? _database->getInternalRegistry() : _database->getReplica(name);
     return addFileIterator(reader, "stdout", current);
 }
 
 FileIteratorPrx
 AdminSessionI::openRegistryStdErr(const std::string& name, const Ice::Current& current)
 {
-    FileReaderPrx reader;
-    if(name == _replicaName)
-    {
-	Ice::Identity internalRegistryId;
-	internalRegistryId.category = _database->getInstanceName();
-	internalRegistryId.name = "InternalRegistry-" + _replicaName;
-
-	Ice::CommunicatorPtr communicator = current.adapter->getCommunicator();
-	string proxyStr = communicator->identityToString(internalRegistryId);
-	reader = FileReaderPrx::uncheckedCast(communicator->stringToProxy(proxyStr));
-    }
-    else 
-    {
-	reader = _database->getReplica(name);
-    }
-
+    FileReaderPrx reader = name == _replicaName ? _database->getInternalRegistry() : _database->getReplica(name);
     return addFileIterator(reader, "stderr", current);
 }
 
diff --git a/cpp/src/IceGrid/Database.cpp b/cpp/src/IceGrid/Database.cpp
index 9b33ce7917a..d9126730814 100644
--- a/cpp/src/IceGrid/Database.cpp
+++ b/cpp/src/IceGrid/Database.cpp
@@ -382,7 +382,7 @@ Database::addApplication(const ApplicationInfo& info, AdminSessionI* session)
     {
 	try
 	{
-	    for_each(entries.begin(), entries.end(), IceUtil::voidMemFun(&ServerEntry::sync));
+	    for_each(entries.begin(), entries.end(), IceUtil::voidMemFun(&ServerEntry::load));
 	}
 	catch(const DeploymentException& ex)
 	{
@@ -595,7 +595,7 @@ Database::removeApplication(const string& name, AdminSessionI* session)
     {
 	try
 	{
-	    for_each(entries.begin(), entries.end(), IceUtil::voidMemFun(&ServerEntry::sync));
+	    for_each(entries.begin(), entries.end(), IceUtil::voidMemFun(&ServerEntry::load));
 	}
 	catch(const DeploymentException&)
 	{
@@ -638,6 +638,12 @@ Database::addNode(const string& name, const NodeSessionIPtr& session)
     addInternalObject(info, true);
 }
 
+void
+Database::setNodeProxy(const string& name, const NodePrx& node)
+{
+    _nodeCache.get(name)->setProxy(node);
+}
+
 NodePrx 
 Database::getNode(const string& name) const
 {
@@ -730,7 +736,7 @@ Database::waitForApplicationReplication(const AMD_NodeSession_waitForApplication
 }
 
 void
-Database::removeReplica(const string& name, const ReplicaSessionIPtr& session, bool shutdown)
+Database::removeReplica(const string& name, bool shutdown)
 {
     _registryObserverTopic->registryDown(name);
     _replicaCache.remove(name, shutdown);
@@ -742,6 +748,30 @@ Database::getAllReplicas(const string& expression)
     return _replicaCache.getAll(expression);
 }
 
+void
+Database::setInternalRegistry(const InternalRegistryPrx& proxy)
+{
+    _replicaCache.setInternalRegistry(proxy);
+}
+
+InternalRegistryPrx
+Database::getInternalRegistry() const
+{
+    return _replicaCache.getInternalRegistry();
+}
+
+void
+Database::loadServer(const std::string& id)
+{
+    _serverCache.get(id)->load();
+}
+
+void
+Database::unloadServer(const std::string& id)
+{
+    _serverCache.get(id)->unload();
+}
+
 ServerInfo
 Database::getServerInfo(const std::string& id, bool resolve)
 {
@@ -1659,12 +1689,12 @@ Database::finishApplicationUpdate(ServerEntrySeq& entries,
     if(_master)
     {
 	//
-	// Synchronize the servers on the nodes. If a server couldn't be
+	// Load the servers on the nodes. If a server couldn't be
 	// deployed we unload the application and throw.
 	//
 	try
 	{
-	    for_each(entries.begin(), entries.end(), IceUtil::voidMemFun(&ServerEntry::sync));
+	    for_each(entries.begin(), entries.end(), IceUtil::voidMemFun(&ServerEntry::load));
 	}
 	catch(const DeploymentException& ex)
 	{
@@ -1681,7 +1711,7 @@ Database::finishApplicationUpdate(ServerEntrySeq& entries,
 
 	    try
 	    {
-		for_each(entries.begin(), entries.end(), IceUtil::voidMemFun(&ServerEntry::sync));
+		for_each(entries.begin(), entries.end(), IceUtil::voidMemFun(&ServerEntry::load));
 	    }
 	    catch(const DeploymentException& ex)
 	    {
diff --git a/cpp/src/IceGrid/Database.h b/cpp/src/IceGrid/Database.h
index 5fb64b72996..08825a5d98a 100644
--- a/cpp/src/IceGrid/Database.h
+++ b/cpp/src/IceGrid/Database.h
@@ -81,6 +81,7 @@ public:
     Ice::StringSeq getAllApplications(const std::string& = std::string());
 
     void addNode(const std::string&, const NodeSessionIPtr&);
+    void setNodeProxy(const std::string&, const NodePrx&);
     NodePrx getNode(const std::string&) const;
     NodeInfo getNodeInfo(const std::string&) const;
     void removeNode(const std::string&, const NodeSessionIPtr&, bool);
@@ -92,11 +93,15 @@ public:
     void replicaReceivedUpdate(const std::string&, TopicName, int, const std::string&);
     void waitForApplicationReplication(const AMD_NodeSession_waitForApplicationReplicationPtr&, const std::string&, 
 				       int);
-    void removeReplica(const std::string&, const ReplicaSessionIPtr&, bool);
+    void removeReplica(const std::string&, bool);
     Ice::StringSeq getAllReplicas(const std::string& = std::string());
+    void setInternalRegistry(const InternalRegistryPrx&);
+    InternalRegistryPrx getInternalRegistry() const;
 
     ServerInfo getServerInfo(const std::string&, bool = false);
     ServerPrx getServer(const std::string&, bool = true);
+    void loadServer(const std::string&);
+    void unloadServer(const std::string&);
     ServerPrx getServerWithTimeouts(const std::string&, int&, int&, std::string&, bool = true);
     Ice::StringSeq getAllServers(const std::string& = std::string());
     Ice::StringSeq getAllNodeServers(const std::string&);
diff --git a/cpp/src/IceGrid/Internal.ice b/cpp/src/IceGrid/Internal.ice
index 90764b0b8d9..bb915f43924 100644
--- a/cpp/src/IceGrid/Internal.ice
+++ b/cpp/src/IceGrid/Internal.ice
@@ -229,10 +229,12 @@ interface Node extends FileReader
 
     /**
      *
-     * Establish a session to the given replica.
+     * Establish a session to the given replica, this method only
+     * returns once the registration was attempted (unlike
+     * replicaAdded below).
      * 
      **/
-    void registerWithReplica(InternalRegistry* replica);
+    ["ami"] void registerWithReplica(InternalRegistry* replica);
 
     /**
      *
diff --git a/cpp/src/IceGrid/NodeCache.cpp b/cpp/src/IceGrid/NodeCache.cpp
index 21619cc5be7..a43e3335798 100644
--- a/cpp/src/IceGrid/NodeCache.cpp
+++ b/cpp/src/IceGrid/NodeCache.cpp
@@ -66,17 +66,6 @@ public:
 	{
 	    ex.ice_throw();
 	}
-	catch(const NodeNotExistException& ex)
-	{
-	    if(_traceLevels && _traceLevels->server > 1)
-	    {
-		Ice::Trace out(_traceLevels->logger, _traceLevels->serverCat);
-		out << "couldn't load `" << _id << "' on node `" << _node << "':\n" << ex;
-	    }
-	    ostringstream os;
-	    os << ex;
-	    _server->exception(NodeUnreachableException(_node, os.str()));
-	}
 	catch(const DeploymentException& ex)
 	{
 	    if(_traceLevels && _traceLevels->server > 1)
@@ -139,17 +128,6 @@ public:
 	{
 	    ex.ice_throw();
 	}
-	catch(const NodeNotExistException& ex)
-	{
-	    if(_traceLevels && _traceLevels->server > 1)
-	    {
-		Ice::Trace out(_traceLevels->logger, _traceLevels->serverCat);
-		out << "couldn't unload `" << _id << "' on node `" << _node << "':\n" << ex;
-	    }
-	    ostringstream os;
-	    os << ex;
- 	    _server->exception(NodeUnreachableException(_node, os.str()));
-	}
 	catch(const DeploymentException& ex)
 	{
 	    if(_traceLevels && _traceLevels->server > 1)
@@ -183,6 +161,30 @@ private:
     const string _node;
 };
 
+class RegisterCB : public AMI_Node_registerWithReplica
+{
+public:
+
+    RegisterCB(const NodeEntryPtr& node) : _node(node)
+    {
+    }
+
+    void
+    ice_response()
+    {
+	_node->finishedRegistration();
+    }
+
+    void
+    ice_exception(const Ice::Exception& ex)
+    {
+	_node->finishedRegistration(ex);
+    }
+
+private:
+    const NodeEntryPtr _node;
+};
+
 };
 
 NodeCache::NodeCache(const Ice::CommunicatorPtr& communicator, ReplicaCache& replicaCache, bool master) :
@@ -215,7 +217,8 @@ NodeCache::get(const string& name, bool create) const
 NodeEntry::NodeEntry(NodeCache& cache, const std::string& name) : 
     _cache(cache),
     _ref(0),
-    _name(name)
+    _name(name),
+    _registering(false)
 {
 }
 
@@ -256,19 +259,21 @@ NodeEntry::setSession(const NodeSessionIPtr& session)
 {
     Lock sync(*this);
 
-    if(session)
+    if(session && _session)
     {
-	// If the current session has just been destroyed, wait for the setSession(0) call.
-	assert(session != _session);
-	while(_session && _session->isDestroyed()) 
+	if(_session->isDestroyed())
 	{
-	    wait();
+	    // If the current session has just been destroyed, wait for the setSession(0) call.
+	    assert(session != _session);
+	    while(_session) 
+	    {
+		wait();
+	    }
+	}
+	else
+	{
+	    throw NodeActiveException();
 	}
-    }
-    
-    if(session && _session)
-    {
-	throw NodeActiveException();
     }
     else if(!session && !_session)
     {
@@ -287,7 +292,20 @@ NodeEntry::setSession(const NodeSessionIPtr& session)
     {
 	_cache.getReplicaCache().nodeAdded(session->getNode());
     }
-    
+
+    //
+    // Clear the saved proxy, the node has established a session
+    // so we won't need anymore to try to register it with this
+    // registry.
+    //
+    _proxy = 0;
+	
+    if(_registering)
+    {
+	_registering = false;
+	notifyAll();
+    }
+
     if(session)
     {
 	if(_cache.getTraceLevels() && _cache.getTraceLevels()->node > 0)
@@ -310,10 +328,7 @@ NodePrx
 NodeEntry::getProxy() const
 {
     Lock sync(*this);
-    if(!_session)
-    {
-	throw NodeUnreachableException(_name, "the node is not active");
-    }
+    checkSession();
     return _session->getNode();
 }
 
@@ -321,10 +336,7 @@ NodeInfo
 NodeEntry::getInfo() const
 {
     Lock sync(*this);
-    if(!_session)
-    {
-	throw NodeUnreachableException(_name, "the node is not active");
-    }
+    checkSession();
     return _session->getInfo();
 }
 
@@ -344,19 +356,23 @@ LoadInfo
 NodeEntry::getLoadInfoAndLoadFactor(const string& application, float& loadFactor) const
 {
     Lock sync(*this);
-    if(!_session)
+    checkSession();
+
+    map<string, NodeDescriptor>::const_iterator p = _descriptors.find(application);
+    if(p == _descriptors.end())
     {
-	throw NodeUnreachableException(_name, "the node is not active");
+	throw NodeNotExistException(); // The node doesn't exist in the given application.
     }
-    map<string, NodeDescriptor>::const_iterator p = _descriptors.find(application);
+    
+    //
+    // TODO: Cache the load factors? Parsing the load factor for each
+    // call could be costly.
+    //
     loadFactor = -1.0f;
-    if(p != _descriptors.end())
+    if(!p->second.loadFactor.empty())
     {
-	if(!p->second.loadFactor.empty())
-	{
-	    istringstream is(p->second.loadFactor);
-	    is >> loadFactor;
-	}
+	istringstream is(p->second.loadFactor);
+	is >> loadFactor;
     }
     if(loadFactor < 0.0f)
     {
@@ -380,6 +396,7 @@ NodeEntry::getLoadInfoAndLoadFactor(const string& application, float& loadFactor
 	    loadFactor = 1.0f;
 	}
     }
+
     return _session->getLoadInfo();
 }
 
@@ -400,10 +417,7 @@ NodeEntry::loadServer(const ServerEntryPtr& entry, const ServerInfo& server, con
 	ServerDescriptorPtr desc;
 	{
 	    Lock sync(*this);
-	    if(!_session)
-	    {
-		throw NodeUnreachableException(_name, "the node is not active");
-	    }
+	    checkSession();
 	    node = _session->getNode();
 	    timeout = _session->getTimeout();
 	    try
@@ -472,10 +486,8 @@ ServerInfo
 NodeEntry::getServerInfo(const ServerInfo& server, const SessionIPtr& session)
 {
     Lock sync(*this);
-    if(!_session)
-    {
-	throw NodeUnreachableException(_name, "the node is not active");
-    }
+    checkSession();
+
     ServerInfo info = server;
     info.descriptor = getServerDescriptor(server, session);
     assert(info.descriptor);
@@ -530,7 +542,8 @@ NodeEntry::__decRef()
     bool doRemove = false;
     bool doDelete = false;
     {
-	Lock sync(*this);
+	Lock sync(*this); // We use a recursive mutex so it's fine to
+			  // create Ptr with the mutex locked.
 	assert(_ref > 0);
 	--_ref;
 	
@@ -553,3 +566,108 @@ NodeEntry::__decRef()
 	delete this;
     }
 }
+
+void
+NodeEntry::checkSession() const
+{ 
+    if(_session && !_session->isDestroyed())
+    {
+	return;
+    }
+    else if(!_proxy && !_registering)
+    {
+	throw NodeUnreachableException(_name, "the node is not active");
+    }
+    else if(_proxy)
+    {
+	//
+	// If the node proxy is set, we attempt to get the node to
+	// register with this registry.
+	//	
+	assert(!_registering);
+
+	if(_cache.getTraceLevels() && _cache.getTraceLevels()->node > 0)
+	{
+	    Ice::Trace out(_cache.getTraceLevels()->logger, _cache.getTraceLevels()->nodeCat);
+	    out << "creating node `" << _name << "' session";
+	}
+
+	NodeEntry* self = const_cast<NodeEntry*>(this);
+	//
+	// NOTE: setting _registering to true must be done before the
+	// call otherwise if the callback is call immediately we'll
+	// hang in the while loop.
+	//
+	_registering = true; 
+	_proxy->registerWithReplica_async(new RegisterCB(self), _cache.getReplicaCache().getInternalRegistry());
+	_proxy = 0; // Registration with the proxy is only attempted once.
+    }
+
+    while(_registering)
+    {
+	wait();
+    }
+    
+    if(!_session)
+    {
+	throw NodeUnreachableException(_name, "the node is not active");
+    }
+}
+
+void
+NodeEntry::setProxy(const NodePrx& node)
+{
+    Lock sync(*this);
+    
+    //
+    // If the node has already established a session with the
+    // registry, no need to remember its proxy, we don't need to get
+    // it to register with this registry since it's already
+    // registered.
+    //
+    if(!_session)
+    {
+	_proxy = node;
+    }
+}
+
+void
+NodeEntry::finishedRegistration()
+{
+    Lock sync(*this);
+    if(_cache.getTraceLevels() && _cache.getTraceLevels()->node > 0)
+    {
+	Ice::Trace out(_cache.getTraceLevels()->logger, _cache.getTraceLevels()->nodeCat);
+	if(_session)
+	{
+	    out << "node `" << _name << "' session created";
+	}
+	else
+	{
+	    out << "node `" << _name << "' session creation failed";
+	}
+    }    
+
+    if(_registering)
+    {
+	_registering = false;
+	notifyAll();
+    }
+}
+
+void
+NodeEntry::finishedRegistration(const Ice::Exception& ex)
+{
+    Lock sync(*this);
+    if(_cache.getTraceLevels() && _cache.getTraceLevels()->node > 0)
+    {
+	Ice::Trace out(_cache.getTraceLevels()->logger, _cache.getTraceLevels()->nodeCat);
+	out << "node `" << _name << "' session creation failed:\n" << ex;
+    }    
+
+    if(_registering)
+    {
+	_registering = false;
+	notifyAll();
+    }
+}
diff --git a/cpp/src/IceGrid/NodeCache.h b/cpp/src/IceGrid/NodeCache.h
index 6060ec912c5..e2228f275f2 100644
--- a/cpp/src/IceGrid/NodeCache.h
+++ b/cpp/src/IceGrid/NodeCache.h
@@ -31,7 +31,7 @@ typedef IceUtil::Handle<ServerEntry> ServerEntryPtr;
 
 class ReplicaCache;
 
-class NodeEntry : public IceUtil::Monitor<IceUtil::Mutex>
+class NodeEntry : public IceUtil::Monitor<IceUtil::RecMutex>
 {
 public:
     
@@ -44,6 +44,7 @@ public:
     void addServer(const ServerEntryPtr&);
     void removeServer(const ServerEntryPtr&);
     void setSession(const NodeSessionIPtr&);
+    void setSavedProxy(const NodePrx&);
 
     NodePrx getProxy() const;
     NodeInfo getInfo() const;
@@ -59,6 +60,11 @@ public:
     void __incRef();
     void __decRef();
 
+    void checkSession() const;
+    void setProxy(const NodePrx&);
+    void finishedRegistration();
+    void finishedRegistration(const Ice::Exception&);
+
 private:
     
     ServerDescriptorPtr getServerDescriptor(const ServerInfo&, const SessionIPtr&);
@@ -70,6 +76,9 @@ private:
     NodeSessionIPtr _session;
     std::map<std::string, ServerEntryPtr> _servers;
     std::map<std::string, NodeDescriptor> _descriptors;
+
+    mutable bool _registering;
+    mutable NodePrx _proxy;
 };
 typedef IceUtil::Handle<NodeEntry> NodeEntryPtr;
 
diff --git a/cpp/src/IceGrid/NodeI.cpp b/cpp/src/IceGrid/NodeI.cpp
index d787583ed3e..aae67c97964 100644
--- a/cpp/src/IceGrid/NodeI.cpp
+++ b/cpp/src/IceGrid/NodeI.cpp
@@ -481,7 +481,7 @@ NodeI::registerWithReplica(const InternalRegistryPrx& replica, const Ice::Curren
 void
 NodeI::replicaAdded(const InternalRegistryPrx& replica, const Ice::Current&)
 {
-    _sessions.replicaAdded(replica);
+    _sessions.replicaAdded(replica, false);
 }
 
 void
diff --git a/cpp/src/IceGrid/NodeSessionI.cpp b/cpp/src/IceGrid/NodeSessionI.cpp
index 3f58722c742..8df3407c322 100644
--- a/cpp/src/IceGrid/NodeSessionI.cpp
+++ b/cpp/src/IceGrid/NodeSessionI.cpp
@@ -94,7 +94,7 @@ NodeSessionI::loadServers(const Ice::Current& current) const
     {
 	try
 	{
-	    _database->getServer(*p);
+	    _database->loadServer(*p);
 	}
 	catch(const Ice::UserException&)
 	{
@@ -130,6 +130,19 @@ NodeSessionI::destroy(const Ice::Current& current)
 	_destroy = true;
     }
 
+    Ice::StringSeq servers = _database->getAllNodeServers(_name);
+    for(Ice::StringSeq::const_iterator p = servers.begin(); p != servers.end(); ++p)
+    {
+	try
+	{
+	    _database->unloadServer(*p);
+	}
+	catch(const Ice::UserException&)
+	{
+	    // Ignore.
+	}
+    }
+
     _database->removeNode(_name, this, !current.adapter);
 
     if(current.adapter)
diff --git a/cpp/src/IceGrid/NodeSessionManager.cpp b/cpp/src/IceGrid/NodeSessionManager.cpp
index 1bef61986af..69485d3cdee 100644
--- a/cpp/src/IceGrid/NodeSessionManager.cpp
+++ b/cpp/src/IceGrid/NodeSessionManager.cpp
@@ -253,13 +253,13 @@ NodeSessionManager::create(const InternalRegistryPrx& replica)
     if(replica->ice_getIdentity() == _master->ice_getIdentity())
     {
 	thread = _thread;
+	thread->setRegistry(replica);
+	thread->tryCreateSession();
     }
     else
     {
-	thread = replicaAdded(replica);
+	replicaAdded(replica, true);
     }
-    thread->setRegistry(replica);
-    thread->tryCreateSession();
 }
 
 bool
@@ -301,27 +301,30 @@ NodeSessionManager::destroy()
     }
 }
 
-NodeSessionKeepAliveThreadPtr
-NodeSessionManager::replicaAdded(const InternalRegistryPrx& replica)
+void
+NodeSessionManager::replicaAdded(const InternalRegistryPrx& replica, bool waitTryCreateSession)
 {
     Lock sync(*this);
     if(_destroyed)
     {
-	return 0;
+	return;
     }
 
     ++_serial;
     NodeSessionMap::const_iterator p = _sessions.find(replica->ice_getIdentity());
+    NodeSessionKeepAliveThreadPtr thread;
     if(p != _sessions.end())
     {
-	return p->second;
+	thread = p->second;
+	thread->setRegistry(replica);
     }
-
-    NodeSessionKeepAliveThreadPtr thread = new NodeSessionKeepAliveThread(replica, _node, _queryObjects);
-    _sessions.insert(make_pair(replica->ice_getIdentity(), thread));
-    thread->start();
-    thread->tryCreateSession(false);
-    return thread;
+    else
+    {
+	thread = new NodeSessionKeepAliveThread(replica, _node, _queryObjects);
+	_sessions.insert(make_pair(replica->ice_getIdentity(), thread));
+	thread->start();
+    }
+    thread->tryCreateSession(waitTryCreateSession);
 }
 
 void
diff --git a/cpp/src/IceGrid/NodeSessionManager.h b/cpp/src/IceGrid/NodeSessionManager.h
index ee22a262521..6c546b7da8c 100644
--- a/cpp/src/IceGrid/NodeSessionManager.h
+++ b/cpp/src/IceGrid/NodeSessionManager.h
@@ -58,7 +58,7 @@ public:
     void terminate();
     void destroy();
 
-    NodeSessionKeepAliveThreadPtr replicaAdded(const InternalRegistryPrx&);
+    void replicaAdded(const InternalRegistryPrx&, bool);
     void replicaRemoved(const InternalRegistryPrx&);
 
     NodeSessionPrx getMasterNodeSession() const { return _thread->getSession(); }
diff --git a/cpp/src/IceGrid/Parser.cpp b/cpp/src/IceGrid/Parser.cpp
index 213d826e186..09e585227a7 100644
--- a/cpp/src/IceGrid/Parser.cpp
+++ b/cpp/src/IceGrid/Parser.cpp
@@ -1297,8 +1297,8 @@ Parser::dumpFile(const string& reader, const string& filename, const list<string
     
     IceUtil::Options opts;
     opts.addOpt("f", "follow");
-    opts.addOpt("h", "head", IceUtil::Options::NeedArg); //, "20"); // TODO: Fix
-    opts.addOpt("t", "tail", IceUtil::Options::NeedArg); //, "20"); // TODO: Fix
+    opts.addOpt("h", "head", IceUtil::Options::NeedArg);
+    opts.addOpt("t", "tail", IceUtil::Options::NeedArg);
 
     vector<string> args;
     try
diff --git a/cpp/src/IceGrid/RegistryI.cpp b/cpp/src/IceGrid/RegistryI.cpp
index 629fcc77302..1e4bf5bf07d 100644
--- a/cpp/src/IceGrid/RegistryI.cpp
+++ b/cpp/src/IceGrid/RegistryI.cpp
@@ -303,7 +303,6 @@ RegistryI::start(bool nowarn)
     for(p = proxies.begin(); p != proxies.end(); ++p)
     {
 	nodes.push_back(NodePrx::uncheckedCast(*p));
-	_database->removeInternalObject((*p)->ice_getIdentity());
     }
 
     InternalRegistryPrxSeq replicas;
@@ -311,7 +310,6 @@ RegistryI::start(bool nowarn)
     for(p = proxies.begin(); p != proxies.end(); ++p)
     {
 	replicas.push_back(InternalRegistryPrx::uncheckedCast(*p));
-	_database->removeObject((*p)->ice_getIdentity());
     }
 
     //
@@ -457,7 +455,10 @@ RegistryI::setupInternalRegistry(const Ice::ObjectAdapterPtr& registryAdapter)
     ObjectPtr internalRegistry = new InternalRegistryI(this, _database, _reaper, _wellKnownObjects, _session);
     Ice::ObjectPrx proxy = registryAdapter->add(internalRegistry, internalRegistryId);
     _wellKnownObjects->add(proxy, InternalRegistry::ice_staticId());
-    return InternalRegistryPrx::uncheckedCast(proxy);
+
+    InternalRegistryPrx registry = InternalRegistryPrx::uncheckedCast(proxy);
+    _database->setInternalRegistry(registry);
+    return registry;
 }
 
 void
@@ -1182,6 +1183,18 @@ RegistryI::registerReplicas(const InternalRegistryPrx& internalRegistry,
 	    }
 	    catch(const Ice::LocalException& ex)
 	    {
+		//
+		// Clear the proxy from the database if we can't
+		// contact the replica.
+		//
+		try
+		{
+		    _database->removeObject((*r)->ice_getIdentity());
+		}
+		catch(const ObjectNotRegisteredException&)
+		{
+		}
+
 		if(_traceLevels && _traceLevels->replica > 1)
 		{
 		    Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
@@ -1206,39 +1219,27 @@ RegistryI::registerReplicas(const InternalRegistryPrx& internalRegistry,
 void
 RegistryI::registerNodes(const InternalRegistryPrx& internalRegistry, const NodePrxSeq& nodes)
 {
+    const string prefix("Node-");
     for(NodePrxSeq::const_iterator p = nodes.begin(); p != nodes.end(); ++p)
     {
-	string nodeName;
-	if(_traceLevels && _traceLevels->node > 1)
+	assert((*p)->ice_getIdentity().name.find(prefix) != string::npos);
+	try
 	{
-	    nodeName = (*p)->ice_getIdentity().name;
-	    const string prefix("Node-");
-	    string::size_type pos = nodeName.find(prefix);
-	    if(pos != string::npos)
-	    {
-		nodeName = nodeName.substr(prefix.size());
-	    }
-	    
-	    Ice::Trace out(_traceLevels->logger, _traceLevels->nodeCat);
-	    out << "creating node `" << nodeName << "' session";
+	    _database->setNodeProxy((*p)->ice_getIdentity().name.substr(prefix.size()), *p);
 	}
-	
-	try
+	catch(const NodeNotExistException&)
 	{
-	    NodePrx::uncheckedCast(*p)->registerWithReplica(internalRegistry);
-	    
-	    if(_traceLevels && _traceLevels->node > 1)
+	    //
+	    // Ignore, if nothing's deployed on the node we won't need
+	    // to contact it for locator requests so we don't need to
+	    // keep its proxy.
+	    //
+	    try
 	    {
-		Ice::Trace out(_traceLevels->logger, _traceLevels->nodeCat);
-		out << "node `" << nodeName << "' session created";
+		_database->removeInternalObject((*p)->ice_getIdentity());
 	    }
-	}
-	catch(const Ice::LocalException& ex)
-	{
-	    if(_traceLevels && _traceLevels->node > 1)
+	    catch(const ObjectNotRegisteredException&)
 	    {
-		Ice::Trace out(_traceLevels->logger, _traceLevels->nodeCat);
-		out << "node `" << nodeName << "' session creation failed:\n" << ex;
 	    }
 	}
     }
diff --git a/cpp/src/IceGrid/ReplicaCache.cpp b/cpp/src/IceGrid/ReplicaCache.cpp
index 5717f3e66f0..c04f680deba 100644
--- a/cpp/src/IceGrid/ReplicaCache.cpp
+++ b/cpp/src/IceGrid/ReplicaCache.cpp
@@ -37,42 +37,32 @@ ReplicaCache::ReplicaCache(const Ice::CommunicatorPtr& communicator, const IceSt
 ReplicaEntryPtr
 ReplicaCache::add(const string& name, const ReplicaSessionIPtr& session)
 {
-    ReplicaEntryPtr entry;
+    Lock sync(*this);
+    
+    while(true)
     {
-	Lock sync(*this);
-
-	while(true)
+	ReplicaEntryPtr entry = getImpl(name);
+	if(entry)
 	{
-	    ReplicaEntryPtr entry = getImpl(name);
-	    if(entry)
+	    if(entry->getSession()->isDestroyed())
 	    {
-		if(entry->getSession()->isDestroyed())
-		{
-		    wait();
-		    continue;
-		}
-		else
-		{
-		    throw ReplicaActiveException();
-		}
+		wait();
+		continue;
+	    }
+	    else
+	    {
+		throw ReplicaActiveException();
 	    }
-	    break;
-	}
-    
-	if(_traceLevels && _traceLevels->replica > 0)
-	{
-	    Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
-	    out << "replica `" << name << "' up";
 	}
+	break;
+    }
     
-	entry = addImpl(name, new ReplicaEntry(name, session));
+    if(_traceLevels && _traceLevels->replica > 0)
+    {
+	Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
+	out << "replica `" << name << "' up";
     }
-
-    //
-    // Note: it's safe to do this outside the synchronization because
-    // remove() can't be called until this method returns (and until
-    // the replica session is fully created).
-    //
+    
     try
     {
 	_nodes->replicaAdded(session->getInternalRegistry());
@@ -91,25 +81,23 @@ ReplicaCache::add(const string& name, const ReplicaSessionIPtr& session)
 	}
     }
 
-    return entry;
+    return addImpl(name, new ReplicaEntry(name, session));
 }
 
 ReplicaEntryPtr
 ReplicaCache::remove(const string& name, bool shutdown)
 {
-    ReplicaEntryPtr entry;
-    {
-	Lock sync(*this);
-	entry = getImpl(name);
-	assert(entry);
-	removeImpl(name);
-	notifyAll();
-	
-	if(_traceLevels && _traceLevels->replica > 0)
-	{
-	    Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
-	    out << "replica `" << name << "' down";
-	}
+    Lock sync(*this);
+
+    ReplicaEntryPtr entry = getImpl(name);
+    assert(entry);
+    removeImpl(name);
+    notifyAll();
+    
+    if(_traceLevels && _traceLevels->replica > 0)
+    {
+	Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
+	out << "replica `" << name << "' down";
     }
 
     if(!shutdown)
@@ -221,6 +209,24 @@ ReplicaCache::getEndpoints(const string& name, const Ice::ObjectPrx& proxy) cons
     return _communicator->stringToProxy("dummy")->ice_endpoints(endpoints);
 }
 
+void
+ReplicaCache::setInternalRegistry(const InternalRegistryPrx& proxy)
+{
+    // 
+    // Setup this replica internal registry proxy.
+    //
+    _self = proxy;
+}
+
+InternalRegistryPrx
+ReplicaCache::getInternalRegistry() const
+{
+    // 
+    // This replica internal registry proxy.
+    //
+    return _self;
+}
+
 ReplicaEntry::ReplicaEntry(const std::string& name, const ReplicaSessionIPtr& session) : 
     _name(name),
     _session(session)
diff --git a/cpp/src/IceGrid/ReplicaCache.h b/cpp/src/IceGrid/ReplicaCache.h
index bcb0c982d3a..2a91e36a83e 100644
--- a/cpp/src/IceGrid/ReplicaCache.h
+++ b/cpp/src/IceGrid/ReplicaCache.h
@@ -58,11 +58,15 @@ public:
 
     Ice::ObjectPrx getEndpoints(const std::string&, const Ice::ObjectPrx&) const;
 
+    void setInternalRegistry(const InternalRegistryPrx&);
+    InternalRegistryPrx getInternalRegistry() const;
+
 private:
 
     const Ice::CommunicatorPtr _communicator;
     const IceStorm::TopicPrx _topic;
     const NodePrx _nodes;
+    InternalRegistryPrx _self; // This replica internal registry proxy.
 };
 
 };
diff --git a/cpp/src/IceGrid/ReplicaSessionI.cpp b/cpp/src/IceGrid/ReplicaSessionI.cpp
index 78197e3ea1c..f558e3339d2 100644
--- a/cpp/src/IceGrid/ReplicaSessionI.cpp
+++ b/cpp/src/IceGrid/ReplicaSessionI.cpp
@@ -172,7 +172,7 @@ ReplicaSessionI::destroy(const Ice::Current& current)
 	_wellKnownObjects->updateReplicatedWellKnownObjects(); // No need to update these if we're shutting down.
     }
 
-    _database->removeReplica(_name, this, shutdown);
+    _database->removeReplica(_name, shutdown);
 
     if(current.adapter)
     {
diff --git a/cpp/src/IceGrid/ServerCache.cpp b/cpp/src/IceGrid/ServerCache.cpp
index 88c0cbac8ad..124359630d2 100644
--- a/cpp/src/IceGrid/ServerCache.cpp
+++ b/cpp/src/IceGrid/ServerCache.cpp
@@ -26,19 +26,20 @@ namespace IceGrid
 
     struct AddCommunicator : std::unary_function<CommunicatorDescriptorPtr&, void>
     {
-	AddCommunicator(ServerCache& serverCache, const ServerEntryPtr& entry) :
-	    _serverCache(serverCache), _entry(entry)
+	AddCommunicator(ServerCache& serverCache, const ServerEntryPtr& entry, const string& application) :
+	    _serverCache(serverCache), _entry(entry), _application(application)
 	{
 	}
 	
 	void
 	operator()(const CommunicatorDescriptorPtr& desc)
 	{
-	    _serverCache.addCommunicator(desc, _entry);
+	    _serverCache.addCommunicator(desc, _entry, _application);
 	}
 	
 	ServerCache& _serverCache;
 	const ServerEntryPtr _entry;
+	const string _application;
     };
 
     struct RemoveCommunicator : std::unary_function<CommunicatorDescriptorPtr&, void>
@@ -86,7 +87,7 @@ ServerCache::add(const ServerInfo& info)
     entry->update(info);
     _nodeCache.get(info.node, true)->addServer(entry);
 
-    forEachCommunicator(AddCommunicator(*this, entry))(info.descriptor);
+    forEachCommunicator(AddCommunicator(*this, entry, info.application))(info.descriptor);
 
     if(_traceLevels && _traceLevels->server > 0)
     {
@@ -153,13 +154,14 @@ ServerCache::clear(const string& id)
 }
 
 void
-ServerCache::addCommunicator(const CommunicatorDescriptorPtr& comm, const ServerEntryPtr& server)
+ServerCache::addCommunicator(const CommunicatorDescriptorPtr& comm, 
+			     const ServerEntryPtr& server,
+			     const string& application)
 {
-    const string application = server->getApplication();
     for(AdapterDescriptorSeq::const_iterator q = comm->adapters.begin() ; q != comm->adapters.end(); ++q)
     {
 	assert(!q->id.empty());
-	_adapterCache.addServerAdapter(*q, server);
+	_adapterCache.addServerAdapter(*q, server, application);
 
 	ObjectDescriptorSeq::const_iterator r;
 	for(r = q->objects.begin(); r != q->objects.end(); ++r)
@@ -208,7 +210,7 @@ ServerEntry::ServerEntry(ServerCache& cache, const string& id) :
 }
 
 void
-ServerEntry::sync()
+ServerEntry::load()
 {
     try
     {
@@ -221,6 +223,18 @@ ServerEntry::sync()
 }
 
 void
+ServerEntry::unload()
+{
+    Lock sync(*this);
+    if(_loaded.get())
+    {
+	_load = _loaded;
+    }
+    _proxy = 0;
+    _adapters.clear();
+}
+
+void
 ServerEntry::update(const ServerInfo& info)
 {
     Lock sync(*this);
@@ -324,28 +338,15 @@ ServerEntry::getId() const
 ServerPrx
 ServerEntry::getProxy(int& activationTimeout, int& deactivationTimeout, string& node, bool upToDate)
 {
-    ServerPrx proxy;
     {
 	Lock sync(*this);
 	if(_loaded.get() || _proxy && !upToDate) // Synced or if not up to date is fine
 	{
 	    assert(_loaded.get() || _load.get());
-	    proxy = _proxy;
 	    activationTimeout = _activationTimeout;
 	    deactivationTimeout = _deactivationTimeout;
 	    node = _loaded.get() ? _loaded->node : _load->node;
-	}
-    }
-
-    if(proxy)
-    {
-	try
-	{
-	    proxy->ice_ping();
-	    return proxy;
-	}
-	catch(const Ice::LocalException&)
-	{
+	    return _proxy;
 	}
     }
 
@@ -378,8 +379,6 @@ ServerEntry::getProxy(int& activationTimeout, int& deactivationTimeout, string&
 AdapterPrx
 ServerEntry::getAdapter(const string& id, bool upToDate)
 {
-    AdapterPrx proxy;
-
     {
 	Lock sync(*this);
 	if(_loaded.get() || _proxy && !upToDate) // Synced or if not up to date is fine
@@ -387,8 +386,8 @@ ServerEntry::getAdapter(const string& id, bool upToDate)
 	    AdapterPrxDict::const_iterator p = _adapters.find(id);
 	    if(p != _adapters.end())
 	    {
-		proxy = p->second;
-		assert(proxy);
+		assert(p->second);
+		return p->second;
 	    }
 	    else
 	    {
@@ -397,18 +396,6 @@ ServerEntry::getAdapter(const string& id, bool upToDate)
 	}
     }
 
-    if(proxy)
-    {
-	try
-	{
-	    proxy->ice_ping();
- 	    return proxy;
-	}
-	catch(const Ice::LocalException&)
-	{
-	}
-    }
-
     while(true)
     {    
 	syncImpl(true);
@@ -439,28 +426,6 @@ ServerEntry::getAdapter(const string& id, bool upToDate)
     }
 }
 
-NodeEntryPtr
-ServerEntry::getNode() const
-{
-    Lock sync(*this);
-    if(!_loaded.get() && !_load.get())
-    {
-	throw ServerNotExistException();
-    }
-    return _loaded.get() ? _cache.getNodeCache().get(_loaded->node) : _cache.getNodeCache().get(_load->node);
-}
-
-string
-ServerEntry::getApplication() const
-{
-    Lock sync(*this);
-    if(!_loaded.get() && !_load.get())
-    {
-	throw ServerNotExistException();
-    }
-    return _loaded.get() ? _loaded->application : _load->application;
-}
-
 float
 ServerEntry::getLoad(LoadSample sample) const
 {
diff --git a/cpp/src/IceGrid/ServerCache.h b/cpp/src/IceGrid/ServerCache.h
index 4b39130d77b..d056e43c852 100644
--- a/cpp/src/IceGrid/ServerCache.h
+++ b/cpp/src/IceGrid/ServerCache.h
@@ -36,7 +36,9 @@ public:
     
     ServerEntry(ServerCache&, const std::string&);
 
-    void sync();
+    void load();
+    void unload();
+
     void update(const ServerInfo&);
     void destroy();
 
@@ -45,8 +47,6 @@ public:
 
     ServerPrx getProxy(int&, int&, std::string&, bool);
     AdapterPrx getAdapter(const std::string&, bool);
-    NodeEntryPtr getNode() const;
-    std::string getApplication() const;
     float getLoad(LoadSample) const;
 
     bool canRemove();
@@ -103,7 +103,7 @@ public:
 
 private:
     
-    void addCommunicator(const CommunicatorDescriptorPtr&, const ServerEntryPtr&);
+    void addCommunicator(const CommunicatorDescriptorPtr&, const ServerEntryPtr&, const std::string&);
     void removeCommunicator(const CommunicatorDescriptorPtr&, const ServerEntryPtr&);
 
     friend struct AddCommunicator;
diff --git a/cpp/test/IceGrid/replicaGroup/AllTests.cpp b/cpp/test/IceGrid/replicaGroup/AllTests.cpp
index 814a67bbac0..f20c1cb7ac6 100644
--- a/cpp/test/IceGrid/replicaGroup/AllTests.cpp
+++ b/cpp/test/IceGrid/replicaGroup/AllTests.cpp
@@ -522,7 +522,15 @@ allTests(const Ice::CommunicatorPtr& comm)
 	params["id"] = "Server2";
 	instantiateServer(admin, "Server", "localnode", params);
 	TestIntfPrx obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Random"));
-	test(obj->getReplicaIdAndShutdown() == "Server2.ReplicatedAdapter");
+	test(obj->getReplicaId() == "Server2.ReplicatedAdapter");
+
+	//
+	// Also make sure that findObjectByTypeOnLeastLoadedNode still work.
+	//
+	QueryPrx query = IceGrid::QueryPrx::checkedCast(comm->stringToProxy("IceGrid/Query"));
+	obj = TestIntfPrx::uncheckedCast(query->findObjectByTypeOnLeastLoadedNode("::Test::TestIntf", LoadSample1));
+	test(obj->getReplicaId() == "Server2.ReplicatedAdapter");
+	
 	removeServer(admin, "Server1");
 	removeServer(admin, "Server2");
 
@@ -532,7 +540,7 @@ allTests(const Ice::CommunicatorPtr& comm)
 	params["id"] = "Server2";
 	instantiateServer(admin, "Server", "localnode", params);
 	obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("RoundRobin"));
-	test(obj->getReplicaIdAndShutdown() == "Server2.ReplicatedAdapter");
+	test(obj->getReplicaId() == "Server2.ReplicatedAdapter");
 	removeServer(admin, "Server1");
 	removeServer(admin, "Server2");
 
@@ -542,7 +550,7 @@ allTests(const Ice::CommunicatorPtr& comm)
 	params["id"] = "Server2";
 	instantiateServer(admin, "Server", "localnode", params);
 	obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Adaptive"));
-	test(obj->getReplicaIdAndShutdown() == "Server2.ReplicatedAdapter");
+	test(obj->getReplicaId() == "Server2.ReplicatedAdapter");
 	removeServer(admin, "Server1");
 	removeServer(admin, "Server2");
 
@@ -552,7 +560,7 @@ allTests(const Ice::CommunicatorPtr& comm)
 	params["id"] = "Server1";
 	instantiateServer(admin, "Server", "inactivenode", params);
 	obj = TestIntfPrx::uncheckedCast(comm->stringToProxy("Random"));
-	test(svcReplicaIds.find(obj->getReplicaIdAndShutdown()) != svcReplicaIds.end());
+	test(svcReplicaIds.find(obj->getReplicaId()) != svcReplicaIds.end());
 	removeServer(admin, "IceBox1");
 	removeServer(admin, "Server1");
     };
diff --git a/cpp/test/IceGrid/replication/AllTests.cpp b/cpp/test/IceGrid/replication/AllTests.cpp
index 1785cafafef..fc40d2fe626 100644
--- a/cpp/test/IceGrid/replication/AllTests.cpp
+++ b/cpp/test/IceGrid/replication/AllTests.cpp
@@ -752,6 +752,16 @@ allTests(const Ice::CommunicatorPtr& comm)
     instantiateServer(admin, "IceGridNode", params);
 
     //
+    // Add an application which is using Node1. Otherwise, when a
+    // registry restarts it would throw aways the proxy of the nodes
+    // because the node isn't used by any application.
+    //
+    ApplicationDescriptor app;
+    app.name = "DummyApp";
+    app.nodes["Node1"].description = "dummy node";
+    masterAdmin->addApplication(app);
+
+    //
     // Test node session establishment.
     //
     // - start master, start slave1, start node, start slave2