ObjectNotExistException are now always retried.

11 files changed, 214 insertions, 75 deletions

diff --git a/cpp/src/Ice/LocatorInfo.cpp b/cpp/src/Ice/LocatorInfo.cpp
index 50e16b9ef32..91531d24648 100644
--- a/cpp/src/Ice/LocatorInfo.cpp
+++ b/cpp/src/Ice/LocatorInfo.cpp
@@ -17,7 +17,6 @@
 #include <Ice/Reference.h>
 #include <Ice/Functional.h>
 #include <Ice/IdentityUtil.h>
-
 #include <iterator>
 
 using namespace std;
@@ -185,7 +184,6 @@ void
 IceInternal::LocatorTable::addProxy(const Identity& id, const ObjectPrx& proxy)
 {
     IceUtil::Mutex::Lock sync(*this);
-    
     _objectMap.insert(make_pair(id, proxy));
 }
 
@@ -201,9 +199,7 @@ IceInternal::LocatorTable::removeProxy(const Identity& id)
     }
 
     ObjectPrx proxy = p->second;
-
     _objectMap.erase(p);
-    
     return proxy;
 }
 
@@ -294,18 +290,20 @@ IceInternal::LocatorInfo::getEndpoints(const IndirectReferencePtr& ref, bool& ca
 	}
 	else
 	{
+	    bool objectCached = true;
 	    if(!_table->getProxy(ref->getIdentity(), object))
 	    {
-		cached = false;
-		
+		objectCached = false;
 		object = _locator->findObjectById(ref->getIdentity());
 	    }
 
+	    bool endpointsCached = true;
 	    if(object)
 	    {
 		DirectReferencePtr odr = DirectReferencePtr::dynamicCast(object->__reference());
 		if(odr)
 		{
+		    endpointsCached = false;
 		    endpoints = odr->getEndpoints();
 		}
 		else
@@ -314,15 +312,17 @@ IceInternal::LocatorInfo::getEndpoints(const IndirectReferencePtr& ref, bool& ca
 		    assert(oir);
 		    if(!oir->getAdapterId().empty())
 		    {
-			endpoints = getEndpoints(oir, cached);
+			endpoints = getEndpoints(oir, endpointsCached);
 		    }
 		}
 	    }
 
-	    if(!cached && !endpoints.empty())
+	    if(!objectCached && !endpoints.empty())
 	    {
 		_table->addProxy(ref->getIdentity(), object);
 	    }
+
+	    cached = objectCached || endpointsCached;
 	}
     }
     catch(const AdapterNotFoundException&)
diff --git a/cpp/src/Ice/Outgoing.cpp b/cpp/src/Ice/Outgoing.cpp
index da5b5e3185e..4c444b54ed5 100644
--- a/cpp/src/Ice/Outgoing.cpp
+++ b/cpp/src/Ice/Outgoing.cpp
@@ -191,7 +191,11 @@ IceInternal::Outgoing::invoke()
 		// guarantees that all outstanding requests can safely
 		// be repeated.
 		//
-		if(dynamic_cast<CloseConnectionException*>(_exception.get()))
+		// An ObjectNotExistException can always be retried as
+		// well without violating "at-most-once".
+		//
+		if(dynamic_cast<CloseConnectionException*>(_exception.get()) ||
+		   dynamic_cast<ObjectNotExistException*>(_exception.get()))
 		{
 		    _exception->ice_throw();
 		}
diff --git a/cpp/src/Ice/OutgoingAsync.cpp b/cpp/src/Ice/OutgoingAsync.cpp
index 558efb747ad..044b0dbca0f 100644
--- a/cpp/src/Ice/OutgoingAsync.cpp
+++ b/cpp/src/Ice/OutgoingAsync.cpp
@@ -204,12 +204,6 @@ IceInternal::OutgoingAsync::__finished(const LocalException& exc)
 
     if(_reference)
     {
-	IndirectReferencePtr ir = IndirectReferencePtr::dynamicCast(_reference);
-	if(ir && ir->getLocatorInfo())
-	{
-	    ir->getLocatorInfo()->clearObjectCache(ir);
-	}
-	
 	bool doRetry = false;
 	
 	//
@@ -221,14 +215,18 @@ IceInternal::OutgoingAsync::__finished(const LocalException& exc)
 	// can also retry if the operation mode is Nonmutating or
 	// Idempotent.
 	//
-	if(_mode == Nonmutating || _mode == Idempotent || dynamic_cast<const CloseConnectionException*>(&exc))
+	// An ObjectNotExistException can always be retried as
+	// well without violating "at-most-once".
+	//
+	if(_mode == Nonmutating || _mode == Idempotent || dynamic_cast<const CloseConnectionException*>(&exc) ||
+	   dynamic_cast<const ObjectNotExistException*>(&exc))
 	{
 	    try
 	    {
 		ProxyFactoryPtr proxyFactory = _reference->getInstance()->proxyFactory();
 		if(proxyFactory)
 		{
-		    proxyFactory->checkRetryAfterException(exc, _cnt);
+		    proxyFactory->checkRetryAfterException(exc, _reference, _cnt);
 		}
 		else
 		{
@@ -373,16 +371,10 @@ IceInternal::OutgoingAsync::__send()
 	    }
 	    catch(const LocalException& ex)
 	    {
-		IndirectReferencePtr ir = IndirectReferencePtr::dynamicCast(_reference);
-		if(ir && ir->getLocatorInfo())
-		{
-		    ir->getLocatorInfo()->clearObjectCache(ir);
-		}
-		
 		ProxyFactoryPtr proxyFactory = _reference->getInstance()->proxyFactory();
 		if(proxyFactory)
 		{
-		    proxyFactory->checkRetryAfterException(ex, _cnt);
+		    proxyFactory->checkRetryAfterException(ex, _reference, _cnt);
 		}
 		else
 		{
diff --git a/cpp/src/Ice/Proxy.cpp b/cpp/src/Ice/Proxy.cpp
index 761841d3c5f..b2dc3e67db9 100644
--- a/cpp/src/Ice/Proxy.cpp
+++ b/cpp/src/Ice/Proxy.cpp
@@ -688,12 +688,6 @@ IceProxy::Ice::Object::__handleException(const LocalException& ex, int& cnt)
 	_delegate = 0;
     }
 
-    IndirectReferencePtr ir = IndirectReferencePtr::dynamicCast(_reference);
-    if(ir && ir->getLocatorInfo())
-    {
-	ir->getLocatorInfo()->clearObjectCache(ir);
-    }
-
     if(ice_isOneway() || ice_isBatchOneway())
     {
 	//
@@ -712,7 +706,7 @@ IceProxy::Ice::Object::__handleException(const LocalException& ex, int& cnt)
     ProxyFactoryPtr proxyFactory = _reference->getInstance()->proxyFactory();
     if(proxyFactory)
     {
-	proxyFactory->checkRetryAfterException(ex, cnt);
+	proxyFactory->checkRetryAfterException(ex, _reference, cnt);
     }
     else
     {
@@ -734,12 +728,6 @@ IceProxy::Ice::Object::__rethrowException(const LocalException& ex)
 	_delegate = 0;
     }
 
-    IndirectReferencePtr ir = IndirectReferencePtr::dynamicCast(_reference);
-    if(ir && ir->getLocatorInfo())
-    {
-	ir->getLocatorInfo()->clearObjectCache(ir);
-    }
-
     ex.ice_throw();
 }
 
diff --git a/cpp/src/Ice/ProxyFactory.cpp b/cpp/src/Ice/ProxyFactory.cpp
index 7a06429de1b..31f08404e25 100644
--- a/cpp/src/Ice/ProxyFactory.cpp
+++ b/cpp/src/Ice/ProxyFactory.cpp
@@ -13,6 +13,7 @@
 #include <Ice/Instance.h>
 #include <Ice/Proxy.h>
 #include <Ice/ReferenceFactory.h>
+#include <Ice/LocatorInfo.h>
 #include <Ice/BasicStream.h>
 #include <Ice/Properties.h>
 #include <Ice/LoggerUtil.h>
@@ -87,13 +88,23 @@ IceInternal::ProxyFactory::referenceToProxy(const ReferencePtr& ref) const
 }
 
 void
-IceInternal::ProxyFactory::checkRetryAfterException(const LocalException& ex, int& cnt) const
+IceInternal::ProxyFactory::checkRetryAfterException(const LocalException& ex, const ReferencePtr& ref, int& cnt) const
 {
     //
-    // We don't retry *NotExistException, which are all derived from
-    // RequestFailedException.
+    // We retry ObjectNotExistException if the reference is
+    // indirect. Otherwise, we don't retry other *NotExistException,
+    // which are all derived from RequestFailedException.
     //
-    if(dynamic_cast<const RequestFailedException*>(&ex))
+    if(dynamic_cast<const ObjectNotExistException*>(&ex))
+    {
+	IndirectReferencePtr ir = IndirectReferencePtr::dynamicCast(ref);
+	if(!ir || !ir->getLocatorInfo())
+	{
+	    ex.ice_throw();
+	}
+	ir->getLocatorInfo()->clearObjectCache(ir);
+    }
+    else if(dynamic_cast<const RequestFailedException*>(&ex))
     {
 	ex.ice_throw();
     }
diff --git a/cpp/src/Ice/ProxyFactory.h b/cpp/src/Ice/ProxyFactory.h
index ffc35b2b205..c45ed72350c 100644
--- a/cpp/src/Ice/ProxyFactory.h
+++ b/cpp/src/Ice/ProxyFactory.h
@@ -35,7 +35,7 @@ public:
 
     Ice::ObjectPrx referenceToProxy(const ReferencePtr&) const;
 
-    void checkRetryAfterException(const Ice::LocalException&, int&) const;
+    void checkRetryAfterException(const Ice::LocalException&, const ReferencePtr&, int&) const;
 
 private:
 
diff --git a/cpp/test/Ice/location/AllTests.cpp b/cpp/test/Ice/location/AllTests.cpp
index 73bf5e07b4c..8737ae4fba2 100644
--- a/cpp/test/Ice/location/AllTests.cpp
+++ b/cpp/test/Ice/location/AllTests.cpp
@@ -24,7 +24,8 @@ allTests(const Ice::CommunicatorPtr& communicator, const string& ref)
     Ice::ObjectPrx base = communicator->stringToProxy("test @ TestAdapter");
     Ice::ObjectPrx base2 = communicator->stringToProxy("test @ TestAdapter");
     Ice::ObjectPrx base3 = communicator->stringToProxy("test");
-    Ice::ObjectPrx base4 = communicator->stringToProxy("ServerManager");
+    Ice::ObjectPrx base4 = communicator->stringToProxy("ServerManager"); 
+    Ice::ObjectPrx base5 = communicator->stringToProxy("test2");
     cout << "ok" << endl;
 
     cout << "starting server... " << flush;
@@ -43,22 +44,13 @@ allTests(const Ice::CommunicatorPtr& communicator, const string& ref)
     test(obj3);
     ServerManagerPrx obj4 = ServerManagerPrx::checkedCast(base4);
     test(obj4);
+    TestIntfPrx obj5 = TestIntfPrx::checkedCast(base5);
+    test(obj5);
     cout << "ok" << endl;
  
-    cout << "testing object reference from server... " << flush;
-    HelloPrx hello = obj->getHello();
-    hello->sayHello();
-    cout << "ok" << endl;
-
-    cout << "shutdown server... " << flush;
+    cout << "testing id@AdapterId indirect proxy... " << flush;
     obj->shutdown();
-    cout << "ok" << endl;
-
-    cout << "restarting server... " << flush;
     manager->startServer();
-    cout << "ok" << endl;
-
-    cout << "testing whether server is still reachable... " << flush;
     try
     {
 	obj2 = TestIntfPrx::checkedCast(base2);
@@ -70,8 +62,93 @@ allTests(const Ice::CommunicatorPtr& communicator, const string& ref)
     }
     cout << "ok" << endl;    
     
-    cout << "testing object reference from server... " << flush;
-    hello->sayHello();
+    cout << "testing identity indirect proxy... " << flush;
+    obj->shutdown();
+    manager->startServer();
+    try
+    {
+	obj3 = TestIntfPrx::checkedCast(base3);
+	obj3->ice_ping();
+    }
+    catch(const Ice::LocalException&)
+    {
+	test(false);
+    }
+    try
+    {
+	obj2 = TestIntfPrx::checkedCast(base2);
+	obj2->ice_ping();
+    }
+    catch(const Ice::LocalException&)
+    {
+	test(false);
+    }
+    obj->shutdown();
+    manager->startServer();
+    try
+    {
+	obj2 = TestIntfPrx::checkedCast(base2);
+	obj2->ice_ping();
+    }
+    catch(const Ice::LocalException&)
+    {
+	test(false);
+    }
+    try
+    {
+	obj3 = TestIntfPrx::checkedCast(base3);
+	obj3->ice_ping();
+    }
+    catch(const Ice::LocalException&)
+    {
+	test(false);
+    }
+    obj->shutdown();
+    manager->startServer();
+
+    try
+    {
+	obj2 = TestIntfPrx::checkedCast(base2);
+	obj2->ice_ping();
+    }
+    catch(const Ice::LocalException&)
+    {
+	test(false);
+    }
+    obj->shutdown();
+    manager->startServer();
+    try
+    {
+	obj3 = TestIntfPrx::checkedCast(base3);
+	obj3->ice_ping();
+    }
+    catch(const Ice::LocalException&)
+    {
+	test(false);
+    }
+    obj->shutdown();
+    manager->startServer();
+    try
+    {
+	obj2 = TestIntfPrx::checkedCast(base2);
+	obj2->ice_ping();
+    }
+    catch(const Ice::LocalException&)
+    {
+	test(false);
+    }
+    obj->shutdown();
+    manager->startServer();
+
+    try
+    {
+	obj5 = TestIntfPrx::checkedCast(base5);
+	obj5->ice_ping();
+    }
+    catch(const Ice::LocalException&)
+    {
+	test(false);
+    }
     cout << "ok" << endl;
 
     cout << "testing reference with unknown identity... " << flush;
@@ -102,6 +179,27 @@ allTests(const Ice::CommunicatorPtr& communicator, const string& ref)
     }
     cout << "ok" << endl;
 
+    cout << "testing object reference from server... " << flush;
+    HelloPrx hello = obj->getHello();
+    hello->sayHello();
+    cout << "ok" << endl;
+
+    cout << "testing object reference from server after shutdown... " << flush;
+    obj->shutdown();
+    manager->startServer();
+    hello->sayHello();
+    cout << "ok" << endl;
+
+    cout << "testing object migration... " << flush;
+    hello = HelloPrx::checkedCast(communicator->stringToProxy("hello"));
+    obj->migrateHello();
+    hello->sayHello();
+    obj->migrateHello();
+    hello->sayHello();
+    obj->migrateHello();
+    hello->sayHello();
+    cout << "ok" << endl;
+
     cout << "shutdown server... " << flush;
     obj->shutdown();
     cout << "ok" << endl;
@@ -114,8 +212,24 @@ allTests(const Ice::CommunicatorPtr& communicator, const string& ref)
     }
     catch(const Ice::LocalException&)
     {
-	cout << "ok" << endl;
     }
+    try
+    {
+	obj3->ice_ping();
+	test(false);
+    }
+    catch(const Ice::LocalException&)
+    {
+    }
+    try
+    {
+	obj5->ice_ping();
+	test(false);
+    }
+    catch(const Ice::LocalException&)
+    {
+    }
+    cout << "ok" << endl;
 
     cout << "shutdown server manager... " << flush;
     manager->shutdown();
diff --git a/cpp/test/Ice/location/Server.cpp b/cpp/test/Ice/location/Server.cpp
index 5c11e8a6cf8..f605493b73e 100644
--- a/cpp/test/Ice/location/Server.cpp
+++ b/cpp/test/Ice/location/Server.cpp
@@ -27,9 +27,6 @@ run(int argc, char* argv[], const Ice::CommunicatorPtr& communicator)
 
     Ice::ObjectAdapterPtr adapter = communicator->createObjectAdapter("ServerManager");
 
-    Ice::ObjectPtr object = new ServerManagerI(adapter);
-    adapter->add(object, Ice::stringToIdentity("ServerManager"));
-
     //
     // We also register a sample server locator which implements the
     // locator interface, this locator is used by the clients and the
@@ -37,7 +34,8 @@ run(int argc, char* argv[], const Ice::CommunicatorPtr& communicator)
     //
     ServerLocatorRegistryPtr registry = new ServerLocatorRegistry();
     registry->addObject(adapter->createProxy(Ice::stringToIdentity("ServerManager")));
-    registry->addObject(communicator->stringToProxy("test@TestAdapter"));
+    Ice::ObjectPtr object = new ServerManagerI(adapter, registry);
+    adapter->add(object, Ice::stringToIdentity("ServerManager"));
 
     Ice::LocatorRegistryPrx registryPrx = 
 	Ice::LocatorRegistryPrx::uncheckedCast(adapter->add(registry, Ice::stringToIdentity("registry")));
diff --git a/cpp/test/Ice/location/Test.ice b/cpp/test/Ice/location/Test.ice
index c71e4590897..630a3067b09 100644
--- a/cpp/test/Ice/location/Test.ice
+++ b/cpp/test/Ice/location/Test.ice
@@ -29,6 +29,8 @@ interface TestIntf
     void shutdown();
 
     Hello* getHello();
+
+    void migrateHello();
 };
 
 };
diff --git a/cpp/test/Ice/location/TestI.cpp b/cpp/test/Ice/location/TestI.cpp
index db785ef3dae..64a2b146a92 100644
--- a/cpp/test/Ice/location/TestI.cpp
+++ b/cpp/test/Ice/location/TestI.cpp
@@ -13,8 +13,8 @@
 
 using namespace Test;
 
-ServerManagerI::ServerManagerI(const Ice::ObjectAdapterPtr& adapter) :
-    _adapter(adapter)
+ServerManagerI::ServerManagerI(const Ice::ObjectAdapterPtr& adapter, const ServerLocatorRegistryPtr& registry) :
+    _adapter(adapter), _registry(registry)
 {
 }
 
@@ -38,12 +38,20 @@ ServerManagerI::startServer(const Ice::Current&)
     serverCommunicator->getProperties()->setProperty("TestAdapter.AdapterId", "TestAdapter");
     Ice::ObjectAdapterPtr adapter = serverCommunicator->createObjectAdapter("TestAdapter");
 
+    serverCommunicator->getProperties()->setProperty("TestAdapter2.Endpoints", "default");
+    serverCommunicator->getProperties()->setProperty("TestAdapter2.AdapterId", "TestAdapter2");
+    Ice::ObjectAdapterPtr adapter2 = serverCommunicator->createObjectAdapter("TestAdapter2");
+
     Ice::ObjectPrx locator = serverCommunicator->stringToProxy("locator:default -p 12345");
     adapter->setLocator(Ice::LocatorPrx::uncheckedCast(locator));
+    adapter2->setLocator(Ice::LocatorPrx::uncheckedCast(locator));
+
+    Ice::ObjectPtr object = new TestI(adapter, adapter2, _registry);
+    _registry->addObject(adapter->add(object, Ice::stringToIdentity("test")));
+    _registry->addObject(adapter->add(object, Ice::stringToIdentity("test2")));
 
-    Ice::ObjectPtr object = new TestI(adapter);
-    Ice::ObjectPrx proxy = adapter->add(object, Ice::stringToIdentity("test"));
     adapter->activate();
+    adapter2->activate();
 }
 
 void
@@ -57,23 +65,38 @@ ServerManagerI::shutdown(const Ice::Current&)
 }
 
 
-TestI::TestI(const Ice::ObjectAdapterPtr& adapter) :
-    _adapter(adapter)
+TestI::TestI(const Ice::ObjectAdapterPtr& adapter, 
+	     const Ice::ObjectAdapterPtr& adapter2, 
+	     const ServerLocatorRegistryPtr& registry) :
+    _adapter1(adapter), _adapter2(adapter2), _registry(registry)
 {
-    Ice::ObjectPtr servant = new HelloI();
-    _adapter->add(servant, Ice::stringToIdentity("hello")); 
+    _registry->addObject(_adapter1->add(new HelloI(), Ice::stringToIdentity("hello")));
 }
 
 void
 TestI::shutdown(const Ice::Current&)
 {
-    _adapter->getCommunicator()->shutdown();
+    _adapter1->getCommunicator()->shutdown();
 }
 
 HelloPrx
 TestI::getHello(const Ice::Current&)
 {
-    return HelloPrx::uncheckedCast(_adapter->createProxy(Ice::stringToIdentity("hello")));
+    return HelloPrx::uncheckedCast(_adapter1->createProxy(Ice::stringToIdentity("hello")));
+}
+
+void
+TestI::migrateHello(const Ice::Current&)
+{
+    const Ice::Identity id = Ice::stringToIdentity("hello");
+    try
+    {
+	_registry->addObject(_adapter2->add(_adapter1->remove(id), id));
+    }
+    catch(Ice::NotRegisteredException&)
+    {
+	_registry->addObject(_adapter1->add(_adapter2->remove(id), id));
+    }
 }
 
 void
diff --git a/cpp/test/Ice/location/TestI.h b/cpp/test/Ice/location/TestI.h
index 7a5ac5c779f..c046d78e15c 100644
--- a/cpp/test/Ice/location/TestI.h
+++ b/cpp/test/Ice/location/TestI.h
@@ -12,11 +12,13 @@
 
 #include <Test.h>
 #include <vector>
+#include <ServerLocator.h>
 
 class ServerManagerI : public Test::ServerManager
 {
 public:
-    ServerManagerI(const Ice::ObjectAdapterPtr&);
+
+    ServerManagerI(const Ice::ObjectAdapterPtr&, const ServerLocatorRegistryPtr&);
     
     virtual void startServer(const Ice::Current&);
     virtual void shutdown(const Ice::Current&);
@@ -25,6 +27,7 @@ private:
 
     Ice::ObjectAdapterPtr _adapter;
     std::vector<Ice::CommunicatorPtr> _communicators;
+    ServerLocatorRegistryPtr _registry;
 };
 
 class HelloI : public Test::Hello
@@ -38,14 +41,18 @@ class TestI : public Test::TestIntf
 {
 public:
 
-    TestI(const Ice::ObjectAdapterPtr&);
+    TestI(const Ice::ObjectAdapterPtr&, const Ice::ObjectAdapterPtr&, const ServerLocatorRegistryPtr&);
 
     virtual void shutdown(const Ice::Current&);
     virtual ::Test::HelloPrx getHello(const Ice::Current&);
+    virtual void migrateHello(const Ice::Current&);
 
 private:
 
-    Ice::ObjectAdapterPtr _adapter;
+    Ice::ObjectAdapterPtr _adapter1;
+    Ice::ObjectAdapterPtr _adapter2;
+    ServerLocatorRegistryPtr _registry;
+
 };
 
 #endif