More IceGrid fixes to reduce startup delays if some slaves are unavailable

4 files changed, 103 insertions, 50 deletions

diff --git a/cpp/src/IceGrid/NodeSessionManager.cpp b/cpp/src/IceGrid/NodeSessionManager.cpp
index 0863039e950..ccfc8026f87 100644
--- a/cpp/src/IceGrid/NodeSessionManager.cpp
+++ b/cpp/src/IceGrid/NodeSessionManager.cpp
@@ -65,8 +65,15 @@ NodeSessionKeepAliveThread::createSession(InternalRegistryPrx& registry, IceUtil
 
         if(!session)
         {
-            for(vector<QueryPrx>::const_iterator p = _queryObjects.begin(); p != _queryObjects.end(); ++p)
+            vector<Ice::AsyncResultPtr> results;
+            for(vector<QueryPrx>::const_iterator q = _queryObjects.begin(); q != _queryObjects.end(); ++q)
             {
+                results.push_back((*q)->begin_findObjectById(registry->ice_getIdentity()));
+            }
+            
+            for(vector<Ice::AsyncResultPtr>::const_iterator p = results.begin(); p != results.end(); ++p)
+            {
+                QueryPrx query = QueryPrx::uncheckedCast((*p)->getProxy());
                 if(isDestroyed())
                 {
                     break;
@@ -75,7 +82,7 @@ NodeSessionKeepAliveThread::createSession(InternalRegistryPrx& registry, IceUtil
                 InternalRegistryPrx newRegistry;
                 try
                 {
-                    Ice::ObjectPrx obj = (*p)->findObjectById(registry->ice_getIdentity());
+                    Ice::ObjectPrx obj = query->end_findObjectById(*p);
                     newRegistry = InternalRegistryPrx::uncheckedCast(obj);
                     if(newRegistry && used.find(newRegistry) == used.end())
                     {
@@ -528,9 +535,16 @@ NodeSessionManager::createdSession(const NodeSessionPrx& session)
     }
     else
     {
+        vector<Ice::AsyncResultPtr> results;
+        for(vector<QueryPrx>::const_iterator q = _queryObjects.begin(); q != _queryObjects.end(); ++q)
+        {
+            results.push_back((*q)->begin_findAllObjectsByType(InternalRegistry::ice_staticId()));
+        }
+
         map<Ice::Identity, Ice::ObjectPrx> proxies;
-        for(vector<QueryPrx>::const_iterator p = _queryObjects.begin(); p != _queryObjects.end(); ++p)
+        for(vector<Ice::AsyncResultPtr>::const_iterator p = results.begin(); p != results.end(); ++p)
         {
+            QueryPrx query = QueryPrx::uncheckedCast((*p)->getProxy());
             if(isDestroyed())
             {
                 return;
@@ -538,7 +552,7 @@ NodeSessionManager::createdSession(const NodeSessionPrx& session)
 
             try
             {
-                Ice::ObjectProxySeq prxs = (*p)->findAllObjectsByType(InternalRegistry::ice_staticId());
+                Ice::ObjectProxySeq prxs = query->end_findAllObjectsByType(*p);
                 for(Ice::ObjectProxySeq::const_iterator q = prxs.begin(); q != prxs.end(); ++q)
                 {
                     //
@@ -594,13 +608,19 @@ NodeSessionManager::createdSession(const NodeSessionPrx& session)
     // the replica sessions are created before the node adapter is
     // activated.
     //
+    IceUtil::Time before = IceUtil::Time::now();
     for(vector<NodeSessionKeepAliveThreadPtr>::const_iterator p = sessions.begin(); p != sessions.end(); ++p)
     {
         if(isDestroyed())
         {
             return;
         }
-        (*p)->tryCreateSession(true, IceUtil::Time::seconds(5));
+        IceUtil::Time timeout = IceUtil::Time::seconds(5) - (IceUtil::Time::now() - before);
+        if(timeout <= IceUtil::Time())
+        {
+            break;
+        }
+        (*p)->tryCreateSession(true, timeout);
     }
 }
 
diff --git a/cpp/src/IceGrid/RegistryI.cpp b/cpp/src/IceGrid/RegistryI.cpp
index c4dd057da1d..9716346703a 100644
--- a/cpp/src/IceGrid/RegistryI.cpp
+++ b/cpp/src/IceGrid/RegistryI.cpp
@@ -1245,58 +1245,66 @@ RegistryI::registerReplicas(const InternalRegistryPrx& internalRegistry,
 {
     set<NodePrx> nodes;
     nodes.insert(dbNodes.begin(), dbNodes.end());
-
+    vector<Ice::AsyncResultPtr> results;
     for(InternalRegistryPrxSeq::const_iterator r = replicas.begin(); r != replicas.end(); ++r)
     {
         if((*r)->ice_getIdentity() != internalRegistry->ice_getIdentity())
         {
-            string replicaName;
-            if(_traceLevels && _traceLevels->replica > 1)
+            results.push_back((*r)->begin_registerWithReplica(internalRegistry));
+        }
+    }
+
+    for(vector<Ice::AsyncResultPtr>::const_iterator p = results.begin(); p != results.end(); ++p)
+    {
+        InternalRegistryPrx replica = InternalRegistryPrx::uncheckedCast((*p)->getProxy());
+
+        string replicaName;
+        if(_traceLevels && _traceLevels->replica > 1)
+        {
+            replicaName = replica->ice_getIdentity().name;
+            const string prefix("InternalRegistry-");
+            string::size_type pos = replicaName.find(prefix);
+            if(pos != string::npos)
             {
-                replicaName = (*r)->ice_getIdentity().name;
-                const string prefix("InternalRegistry-");
-                string::size_type pos = replicaName.find(prefix);
-                if(pos != string::npos)
-                {
-                    replicaName = replicaName.substr(prefix.size());
-                }
+                replicaName = replicaName.substr(prefix.size());
+            }
+            
+            Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
+            out << "creating replica `" << replicaName << "' session";
+        }
+
+        
+        try
+        {
+            replica->end_registerWithReplica(*p);
+            
+            NodePrxSeq nds = replica->getNodes();
+            nodes.insert(nds.begin(), nds.end());
 
+            if(_traceLevels && _traceLevels->replica > 1)
+            {
                 Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
-                out << "creating replica `" << replicaName << "' session";
+                out << "replica `" << replicaName << "' session created";
             }
-
+        }
+        catch(const Ice::LocalException& ex)
+        {
+            //
+            // Clear the proxy from the database if we can't
+            // contact the replica.
+            //
             try
             {
-                (*r)->registerWithReplica(internalRegistry);
-
-                NodePrxSeq nds = (*r)->getNodes();
-                nodes.insert(nds.begin(), nds.end());
-
-                if(_traceLevels && _traceLevels->replica > 1)
-                {
-                    Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
-                    out << "replica `" << replicaName << "' session created";
-                }
+                _database->removeObject(replica->ice_getIdentity());
             }
-            catch(const Ice::LocalException& ex)
+            catch(const ObjectNotRegisteredException&)
             {
-                //
-                // Clear the proxy from the database if we can't
-                // contact the replica.
-                //
-                try
-                {
-                    _database->removeObject((*r)->ice_getIdentity());
-                }
-                catch(const ObjectNotRegisteredException&)
-                {
-                }
-
-                if(_traceLevels && _traceLevels->replica > 1)
-                {
-                    Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
-                    out << "replica `" << replicaName << "' session creation failed:\n" << ex;
-                }
+            }
+            
+            if(_traceLevels && _traceLevels->replica > 1)
+            {
+                Ice::Trace out(_traceLevels->logger, _traceLevels->replicaCat);
+                out << "replica `" << replicaName << "' session creation failed:\n" << ex;
             }
         }
     }
diff --git a/cpp/src/IceGrid/ReplicaSessionManager.cpp b/cpp/src/IceGrid/ReplicaSessionManager.cpp
index 6dd555ab4d0..ca7ff6e6950 100644
--- a/cpp/src/IceGrid/ReplicaSessionManager.cpp
+++ b/cpp/src/IceGrid/ReplicaSessionManager.cpp
@@ -333,11 +333,18 @@ NodePrxSeq
 ReplicaSessionManager::getNodes(const NodePrxSeq& nodes) const
 {
     assert(_thread && _thread->getRegistry());
-    try
+    if(_thread->getSession())
     {
-        return _thread->getRegistry()->getNodes();
+        try
+        {
+            return _thread->getRegistry()->getNodes();
+        }
+        catch(const Ice::LocalException&)
+        {
+            return nodes;
+        }
     }
-    catch(const Ice::LocalException&)
+    else 
     {
         return nodes;
     }
@@ -448,12 +455,24 @@ ReplicaSessionManager::createSession(InternalRegistryPrx& registry, IceUtil::Tim
 
         if(!session)
         {
-            for(vector<QueryPrx>::const_iterator p = _queryObjects.begin(); p != _queryObjects.end(); ++p)
+            vector<Ice::AsyncResultPtr> results;
+            for(vector<QueryPrx>::const_iterator q = _queryObjects.begin(); q != _queryObjects.end(); ++q)
+            {
+                results.push_back((*q)->begin_findObjectById(registry->ice_getIdentity()));
+            }
+            
+            for(vector<Ice::AsyncResultPtr>::const_iterator p = results.begin(); p != results.end(); ++p)
             {
+                QueryPrx query = QueryPrx::uncheckedCast((*p)->getProxy());
+                if(isDestroyed())
+                {
+                    break;
+                }
+
                 InternalRegistryPrx newRegistry;
                 try
                 {
-                    Ice::ObjectPrx obj = (*p)->findObjectById(registry->ice_getIdentity());
+                    Ice::ObjectPrx obj = query->end_findObjectById(*p);
                     newRegistry = InternalRegistryPrx::uncheckedCast(obj);
                     if(newRegistry && used.find(newRegistry) == used.end())
                     {
diff --git a/cpp/src/IceGrid/ReplicaSessionManager.h b/cpp/src/IceGrid/ReplicaSessionManager.h
index f4e20d821c7..09402cef5a2 100644
--- a/cpp/src/IceGrid/ReplicaSessionManager.h
+++ b/cpp/src/IceGrid/ReplicaSessionManager.h
@@ -84,6 +84,12 @@ private:
 
     friend class Thread;
 
+    bool isDestroyed() 
+    {
+        Lock sync(*this);
+        return !_thread;
+    }
+    
     ReplicaSessionPrx createSession(InternalRegistryPrx&, IceUtil::Time&);
     ReplicaSessionPrx createSessionImpl(const InternalRegistryPrx&, IceUtil::Time&);
     void destroySession(const ReplicaSessionPrx&);