Updated fault tolerance tests to correctly destroy the communicator created

    by the client. This has exposed a bug in the Ice run time. With the way
    the test now works, the client receives a ConnectionRefusedException
    whereas, previously, the adapter was deactivated on the server side.
    Depending on how endpoints are shuffled in Reference.cpp, this
    occasionally causes a test failure. It appears that we need to change
    the run time to
- recognize when an endpoint has gone dead and react correctly
- avoid re-trying endpoints that have previously failed
As is, the test gets slower and slower towards the end because it keeps
    trying endpoints of servers that were killed in previous iterations.

7 files changed, 92 insertions, 6 deletions

diff --git a/cpp/test/Ice/faultTolerance/AllTests.cpp b/cpp/test/Ice/faultTolerance/AllTests.cpp
index efbd59dbc17..493a3e9923c 100644
--- a/cpp/test/Ice/faultTolerance/AllTests.cpp
+++ b/cpp/test/Ice/faultTolerance/AllTests.cpp
@@ -213,10 +213,17 @@ allTests(const Ice::CommunicatorPtr& communicator, const vector<int>& ports)
 
 	if(j == 0)
 	{
+	    ostringstream str;
+	    str << (ports[i] + 1);
+	    string cleanerPort = str.str();
+	    Ice::ObjectPrx objPrx = communicator->stringToProxy("Cleaner:default -t 60000 -p " + cleanerPort);
+	    CleanerPrx cleaner = CleanerPrx::checkedCast(objPrx);
+
 	    if(!ami)
 	    {
 		cout << "shutting down server #" << i << "... " << flush;
 		obj->shutdown();
+		cleaner->cleanup();
 		cout << "ok" << endl;
 	    }
 	    else
@@ -224,6 +231,7 @@ allTests(const Ice::CommunicatorPtr& communicator, const vector<int>& ports)
 		cout << "shutting down server #" << i << " with AMI... " << flush;
 		AMI_Test_shutdownIPtr cb = new AMI_Test_shutdownI;
 		obj->shutdown_async(cb);
+		cleaner->cleanup();
 		test(cb->check());
 		cout << "ok" << endl;
 	    }
@@ -328,6 +336,10 @@ allTests(const Ice::CommunicatorPtr& communicator, const vector<int>& ports)
 	obj->ice_ping();
 	test(false);
     }
+    catch(const Ice::TimeoutException&)
+    {
+        test(false);
+    }
     catch(const Ice::LocalException&)
     {
 	cout << "ok" << endl;
diff --git a/cpp/test/Ice/faultTolerance/Client.cpp b/cpp/test/Ice/faultTolerance/Client.cpp
index 054fa0b9c46..96a11f9d0c8 100644
--- a/cpp/test/Ice/faultTolerance/Client.cpp
+++ b/cpp/test/Ice/faultTolerance/Client.cpp
@@ -22,6 +22,12 @@ usage(const char* n)
 int
 run(int argc, char* argv[], const Ice::CommunicatorPtr& communicator)
 {
+    if(argc < 3)
+    {
+        usage(argv[0]);
+	return 1;
+    }
+
     vector<int> ports;
     for(int i = 1; i < argc; ++i)
     {
diff --git a/cpp/test/Ice/faultTolerance/Server.cpp b/cpp/test/Ice/faultTolerance/Server.cpp
index 94ffbdb6448..3793ad15671 100644
--- a/cpp/test/Ice/faultTolerance/Server.cpp
+++ b/cpp/test/Ice/faultTolerance/Server.cpp
@@ -21,6 +21,12 @@ usage(const char* n)
 int
 run(int argc, char* argv[], const Ice::CommunicatorPtr& communicator)
 {
+    if(argc != 2)
+    {
+        usage(argv[0]);
+	return EXIT_FAILURE;
+    }
+
     int port = 0;
     for(int i = 1; i < argc; ++i)
     {
@@ -48,14 +54,41 @@ run(int argc, char* argv[], const Ice::CommunicatorPtr& communicator)
 	return EXIT_FAILURE;
     }
 
+    CleanerPtr cleaner = new CleanerI(communicator);
+
     ostringstream endpts;
     endpts << "default  -p " << port;
     communicator->getProperties()->setProperty("TestAdapter.Endpoints", endpts.str());
     Ice::ObjectAdapterPtr adapter = communicator->createObjectAdapter("TestAdapter");
-    Ice::ObjectPtr object = new TestI(adapter);
+    Ice::ObjectPtr object = new TestI(adapter, cleaner);
     adapter->add(object, Ice::stringToIdentity("test"));
+
+    int dummyArgc = 0;
+    char* dummyArgv[] = { 0 };
+
+    Ice::CommunicatorPtr cleanupCommunicator = Ice::initialize(dummyArgc, dummyArgv);
+
+    ostringstream str;
+    str << (port + 1);
+    string cleanupPort = str.str();
+    cleanupCommunicator->getProperties()->setProperty("CleanupAdapter.Endpoints", "default -p " + cleanupPort);
+
+    Ice::ObjectAdapterPtr cleanupAdapter = cleanupCommunicator->createObjectAdapter("CleanupAdapter");
+    cleanupAdapter->add(cleaner, Ice::stringToIdentity("Cleaner"));
+
+    string adapterReady = cleanupCommunicator->getProperties()->getProperty("Ice.PrintAdapterReady");
+    cleanupCommunicator->getProperties()->setProperty("Ice.PrintAdapterReady", "0");
+
+    cleanupAdapter->activate();
+    cleanupCommunicator->getProperties()->setProperty("Ice.PrintAdapterReady", adapterReady);
+
     adapter->activate();
     communicator->waitForShutdown();
+
+    cleaner->cleanup();
+
+    cleanupCommunicator->destroy();
+
     return EXIT_SUCCESS;
 }
 
diff --git a/cpp/test/Ice/faultTolerance/Test.ice b/cpp/test/Ice/faultTolerance/Test.ice
index aa61a3d687f..38f995a0104 100644
--- a/cpp/test/Ice/faultTolerance/Test.ice
+++ b/cpp/test/Ice/faultTolerance/Test.ice
@@ -19,4 +19,9 @@
     idempotent int pid();
 };
 
+interface Cleaner
+{
+    void cleanup();
+};
+
 #endif
diff --git a/cpp/test/Ice/faultTolerance/TestI.cpp b/cpp/test/Ice/faultTolerance/TestI.cpp
index 1f439c7ed7e..60c6e2dc250 100644
--- a/cpp/test/Ice/faultTolerance/TestI.cpp
+++ b/cpp/test/Ice/faultTolerance/TestI.cpp
@@ -10,8 +10,9 @@
 #include <Ice/Ice.h>
 #include <TestI.h>
 
-TestI::TestI(const Ice::ObjectAdapterPtr& adapter) :
-    _adapter(adapter)
+TestI::TestI(const Ice::ObjectAdapterPtr& adapter, const CleanerPtr& cleaner) :
+    _adapter(adapter),
+    _cleaner(cleaner)
 {
 }
 
@@ -48,3 +49,18 @@ TestI::pid(const Ice::Current&)
     return getpid();
 #endif
 }
+
+CleanerI::CleanerI(const Ice::CommunicatorPtr& communicator)
+{
+    _communicator = communicator;
+}
+
+void
+CleanerI::cleanup(const Ice::Current&)
+{
+    if(_communicator)
+    {
+        _communicator->destroy();
+	_communicator = 0;
+    }
+}
diff --git a/cpp/test/Ice/faultTolerance/TestI.h b/cpp/test/Ice/faultTolerance/TestI.h
index b9cd2fee481..3458cece6d7 100644
--- a/cpp/test/Ice/faultTolerance/TestI.h
+++ b/cpp/test/Ice/faultTolerance/TestI.h
@@ -16,7 +16,7 @@ class TestI : public Test
 {
 public:
 
-    TestI(const Ice::ObjectAdapterPtr&);
+    TestI(const Ice::ObjectAdapterPtr&, const CleanerPtr& cleaner);
 
     virtual void shutdown(const Ice::Current&);
     virtual void abort(const Ice::Current&);
@@ -27,6 +27,20 @@ public:
 private:
 
     Ice::ObjectAdapterPtr _adapter;
+    CleanerPtr _cleaner;
+};
+
+class CleanerI : public Cleaner
+{
+public:
+
+    CleanerI(const Ice::CommunicatorPtr&);
+
+    virtual void cleanup(const Ice::Current&);
+
+private:
+
+    Ice::CommunicatorPtr _communicator;
 };
 
 #endif
diff --git a/cpp/test/Ice/faultTolerance/run.py b/cpp/test/Ice/faultTolerance/run.py
index d3882612077..b415944f184 100755
--- a/cpp/test/Ice/faultTolerance/run.py
+++ b/cpp/test/Ice/faultTolerance/run.py
@@ -32,14 +32,14 @@ base = 12340
 serverPipes = { }
 for i in range(0, num):
     print "starting server #%d..." % (i + 1),
-    serverPipes[i] = os.popen(server + TestUtil.serverOptions + " %d" % (base + i) + " 2>&1")
+    serverPipes[i] = os.popen(server + TestUtil.serverOptions + " %d" % (base + 2 * i) + " 2>&1")
     TestUtil.getServerPid(serverPipes[i])
     TestUtil.getAdapterReady(serverPipes[i])
     print "ok"
 
 ports = ""
 for i in range(0, num):
-    ports = "%s %d" % (ports, base + i)
+    ports = "%s %d" % (ports, base + 2 * i)
 print "starting client...",
 clientPipe = os.popen(client + TestUtil.clientOptions + " " + ports + " 2>&1")
 print "ok"