Added "always" activation mode.

17 files changed, 383 insertions, 192 deletions

diff --git a/cpp/src/IceGrid/AdminI.cpp b/cpp/src/IceGrid/AdminI.cpp
index 13f10e87769..8ac5c45f1a8 100644
--- a/cpp/src/IceGrid/AdminI.cpp
+++ b/cpp/src/IceGrid/AdminI.cpp
@@ -711,6 +711,7 @@ AdminI::getNodeLoad(const string& name, const Current&) const
 	os << ex;
 	throw NodeUnreachableException(name, os.str());
     }    
+    return LoadInfo(); // Keep the compiler happy.
 }
 
 void
diff --git a/cpp/src/IceGrid/Database.cpp b/cpp/src/IceGrid/Database.cpp
index f88f4b139b2..9ba16639f8f 100644
--- a/cpp/src/IceGrid/Database.cpp
+++ b/cpp/src/IceGrid/Database.cpp
@@ -163,6 +163,12 @@ Database::~Database()
 {
 }
 
+void
+Database::destroy()
+{
+    _nodeCache.destroy(); // Break cyclic reference count.
+}
+
 std::string
 Database::getInstanceName() const
 {
diff --git a/cpp/src/IceGrid/Database.h b/cpp/src/IceGrid/Database.h
index 3c778b976b7..9cf856d2c90 100644
--- a/cpp/src/IceGrid/Database.h
+++ b/cpp/src/IceGrid/Database.h
@@ -46,6 +46,8 @@ public:
     
     Database(const Ice::ObjectAdapterPtr&, const std::string&, const std::string&, int, const TraceLevelsPtr&);
     virtual ~Database();
+    
+    void destroy();
 
     std::string getInstanceName() const;
 
diff --git a/cpp/src/IceGrid/DescriptorBuilder.cpp b/cpp/src/IceGrid/DescriptorBuilder.cpp
index 18c959b7a8f..0608a9c5399 100644
--- a/cpp/src/IceGrid/DescriptorBuilder.cpp
+++ b/cpp/src/IceGrid/DescriptorBuilder.cpp
@@ -102,7 +102,8 @@ XmlAttributesHelper::asBool(const string& name) const
     IceXML::Attributes::const_iterator p = _attributes.find(name);
     if(p == _attributes.end())
     {
-	throw "missing attribute '" + name + "'";	
+	throw "missing attribute '" + name + "'";
+	return true; // Keep the compiler happy.
     }
     else if(p->second == "true")
     {
@@ -115,6 +116,7 @@ XmlAttributesHelper::asBool(const string& name) const
     else
     {
 	throw "invalid attribute `" + name + "': value is not 'false' or 'true'";
+	return true; // Keep the compiler happy.
     }
 }
 
@@ -138,6 +140,7 @@ XmlAttributesHelper::asBool(const string& name, bool def) const
     else
     {
 	throw "invalid attribute `" + name + "': value is not 'false' or 'true'";
+	return true; // Keep the compiler happy.
     }
 }
 
diff --git a/cpp/src/IceGrid/DescriptorHelper.cpp b/cpp/src/IceGrid/DescriptorHelper.cpp
index a85d3fbe3ec..f4c6393c583 100644
--- a/cpp/src/IceGrid/DescriptorHelper.cpp
+++ b/cpp/src/IceGrid/DescriptorHelper.cpp
@@ -526,6 +526,7 @@ Resolver::getVariable(const string& name, bool checkParams, bool& param) const
     }    
 
     throw "undefined variable `" + name + "'";
+    return ""; // To keep the compiler happy.
 }
 
 map<string, string>
@@ -938,7 +939,8 @@ ServerHelper::instantiateImpl(const ServerDescriptorPtr& instance, const Resolve
     instance->pwd = resolve(_desc->pwd, "working directory path");
     instance->activation = resolve(_desc->activation, "activation");
     instance->applicationDistrib = _desc->applicationDistrib;
-    if(!instance->activation.empty() && instance->activation != "manual" && instance->activation != "on-demand")
+    if(!instance->activation.empty() && 
+       instance->activation != "manual" && instance->activation != "on-demand" && instance->activation != "always")
     {
 	resolve.exception("unknown activation `" + instance->activation + "'");
     }
@@ -975,6 +977,12 @@ ServerHelper::instantiate(const Resolver& resolver) const
 }
 
 void
+ServerHelper::print(Output& out) const
+{
+    print(out, "", "");
+}
+
+void
 ServerHelper::print(Output& out, const string& application, const string& node) const
 {
     out << "server `" + _desc->id + "'";
@@ -1099,6 +1107,12 @@ IceBoxHelper::instantiateImpl(const IceBoxDescriptorPtr& instance, const Resolve
 }
 
 void
+IceBoxHelper::print(Output& out) const
+{
+    print(out, "", "");
+}
+
+void
 IceBoxHelper::print(Output& out, const string& application, const string& node) const
 {
     out << "icebox `" + _desc->id + "'";
@@ -1252,7 +1266,7 @@ ServiceInstanceHelper::print(Output& out) const
     else
     {
 	assert(!_template.empty());
-	out << nl << "service instance";
+	out << "service instance";
 	out << sb;
 	out << nl << "template = `" << _template << "'";
 	out << nl << "parameters";
diff --git a/cpp/src/IceGrid/DescriptorHelper.h b/cpp/src/IceGrid/DescriptorHelper.h
index 85043c40de7..610a20b7989 100644
--- a/cpp/src/IceGrid/DescriptorHelper.h
+++ b/cpp/src/IceGrid/DescriptorHelper.h
@@ -67,7 +67,7 @@ public:
 
     virtual void getIds(std::multiset<std::string>&, std::multiset<Ice::Identity>&) const;
     
-    virtual void print(IceUtil::Output&) const;
+    void print(IceUtil::Output&) const;
 
 protected:
 
@@ -120,7 +120,8 @@ public:
     ServerDescriptorPtr getDescriptor() const;
     virtual ServerDescriptorPtr instantiate(const Resolver&) const;
 
-    void print(IceUtil::Output&, const std::string& = std::string(), const std::string& = std::string()) const;
+    void print(IceUtil::Output&) const;
+    void print(IceUtil::Output&, const std::string&, const std::string&) const;
 
 protected:
 
@@ -149,7 +150,8 @@ public:
 
     virtual void getIds(std::multiset<std::string>&, std::multiset<Ice::Identity>&) const;
 
-    void print(IceUtil::Output&, const std::string& = std::string(), const std::string& = std::string()) const;
+    void print(IceUtil::Output&) const;
+    void print(IceUtil::Output&, const std::string&, const std::string&) const;
 
 protected:
 
diff --git a/cpp/src/IceGrid/IceGridNode.cpp b/cpp/src/IceGrid/IceGridNode.cpp
index 3535cf3ab4c..ceaf2ed94e5 100644
--- a/cpp/src/IceGrid/IceGridNode.cpp
+++ b/cpp/src/IceGrid/IceGridNode.cpp
@@ -518,6 +518,7 @@ NodeService::stop()
     }
     catch(...)
     {
+	assert(false);
     }
 
     //
@@ -531,6 +532,7 @@ NodeService::stop()
     }
     catch(...)
     {
+	assert(false);
     }
 
     //
@@ -553,6 +555,7 @@ NodeService::stop()
     }
     catch(...)
     {
+	assert(false);
     }
 
     _activator = 0;
diff --git a/cpp/src/IceGrid/NodeCache.cpp b/cpp/src/IceGrid/NodeCache.cpp
index 584efdc9e19..c3022bf0c9a 100644
--- a/cpp/src/IceGrid/NodeCache.cpp
+++ b/cpp/src/IceGrid/NodeCache.cpp
@@ -159,6 +159,15 @@ NodeCache::NodeCache(int sessionTimeout) : _sessionTimeout(sessionTimeout)
 {
 }
 
+void
+NodeCache::destroy()
+{
+    for(map<string, NodeEntryPtr>::const_iterator p = _entries.begin(); p != _entries.end(); ++p)
+    {
+	p->second->setSession(0); // Break cyclic reference count.
+    }    
+}
+
 NodeEntryPtr
 NodeCache::get(const string& name, bool create) const
 {
diff --git a/cpp/src/IceGrid/NodeCache.h b/cpp/src/IceGrid/NodeCache.h
index d9ce180440b..72cfab6ef31 100644
--- a/cpp/src/IceGrid/NodeCache.h
+++ b/cpp/src/IceGrid/NodeCache.h
@@ -68,6 +68,8 @@ public:
 
     NodeCache(int);
 
+    void destroy();
+
     NodeEntryPtr get(const std::string&, bool = false) const;
 
     int getSessionTimeout() { return _sessionTimeout; }
diff --git a/cpp/src/IceGrid/NodeI.cpp b/cpp/src/IceGrid/NodeI.cpp
index 6a548afddcf..4484eea004d 100644
--- a/cpp/src/IceGrid/NodeI.cpp
+++ b/cpp/src/IceGrid/NodeI.cpp
@@ -797,7 +797,7 @@ NodeI::initObserver(const Ice::StringSeq& servers)
 	{
 	    try
 	    {
-		server->addDynamicInfo(serverInfos, adapterInfos);
+		server->getDynamicInfo(serverInfos, adapterInfos);
 	    }
 	    catch(const Ice::ObjectNotExistException&)
 	    {
diff --git a/cpp/src/IceGrid/Parser.cpp b/cpp/src/IceGrid/Parser.cpp
index 4a48c07a0f9..4630a64d351 100644
--- a/cpp/src/IceGrid/Parser.cpp
+++ b/cpp/src/IceGrid/Parser.cpp
@@ -512,16 +512,7 @@ Parser::describeServiceTemplate(const list<string>& args)
 		out << nl << "parameters = `" << toString(q->second.parameters) << "'";
 	    }
 	    out << nl;
-	    ServiceDescriptorPtr service = ServiceDescriptorPtr::dynamicCast(q->second.descriptor);
-	    IceBoxDescriptorPtr iceBox = IceBoxDescriptorPtr::dynamicCast(service);
-	    if(iceBox)
-	    {
-		IceBoxHelper(iceBox).print(out);
-	    }
-	    else
-	    {
-		ServiceHelper(service).print(out);
-	    }
+	    ServiceHelper(ServiceDescriptorPtr::dynamicCast(q->second.descriptor)).print(out);
 	    out << eb;
 	    out << nl;
 	}
diff --git a/cpp/src/IceGrid/RegistryI.cpp b/cpp/src/IceGrid/RegistryI.cpp
index 0871e62844b..6c2e55efe18 100644
--- a/cpp/src/IceGrid/RegistryI.cpp
+++ b/cpp/src/IceGrid/RegistryI.cpp
@@ -401,14 +401,20 @@ RegistryI::stop()
 {
     _reaper->terminate();
     _reaper->getThreadControl().join();
+    _reaper = 0;
 
     if(_adminReaper)
     {
 	_adminReaper->terminate();
 	_adminReaper->getThreadControl().join();
+	_adminReaper = 0;
     }
 
     _iceStorm->stop();
+    _iceStorm = 0;
+
+    _database->destroy();
+    _database = 0;
 }
 
 NodeSessionPrx
diff --git a/cpp/src/IceGrid/ServerAdapterI.cpp b/cpp/src/IceGrid/ServerAdapterI.cpp
index f016b95aa0e..e884d9cf0cc 100644
--- a/cpp/src/IceGrid/ServerAdapterI.cpp
+++ b/cpp/src/IceGrid/ServerAdapterI.cpp
@@ -69,7 +69,7 @@ ServerAdapterI::activate_async(const AMD_Adapter_activatePtr& cb, const Ice::Cur
     //
     try
     {
-	_server->start_async(0);
+	_server->start(ServerI::OnDemand);
 	return;
     }
     catch(const ServerStartException&)
@@ -189,7 +189,7 @@ ServerAdapterI::clear()
 }
 
 void 
-ServerAdapterI::activationFailed(bool destroyed)
+ServerAdapterI::activationFailed(bool timeout)
 {
 
     //
@@ -198,7 +198,7 @@ ServerAdapterI::activationFailed(bool destroyed)
     if(_node->getTraceLevels()->adapter > 1)
     {
 	Ice::Trace out(_node->getTraceLevels()->logger, _node->getTraceLevels()->adapterCat);
-	if(!destroyed)
+	if(timeout)
 	{
 	    out << "server `" + _serverId + "' adapter `" << _id << "' activation timed out";
 	}
diff --git a/cpp/src/IceGrid/ServerCache.h b/cpp/src/IceGrid/ServerCache.h
index a04c2f97807..80716aadfba 100644
--- a/cpp/src/IceGrid/ServerCache.h
+++ b/cpp/src/IceGrid/ServerCache.h
@@ -32,7 +32,7 @@ class ServerEntry : public IceUtil::Shared, public IceUtil::Monitor<IceUtil::Mut
 public:
     
     ServerEntry(Cache<std::string, ServerEntry>&, const std::string&);
-    
+
     void sync();
     void update(const ServerInfo&);
     void destroy();
diff --git a/cpp/src/IceGrid/ServerI.cpp b/cpp/src/IceGrid/ServerI.cpp
index f70f76636f1..20c50f79d23 100644
--- a/cpp/src/IceGrid/ServerI.cpp
+++ b/cpp/src/IceGrid/ServerI.cpp
@@ -278,7 +278,8 @@ public:
     {
     }
 
-    static bool isStarted(ServerI::InternalServerState state)
+    static bool 
+    isStarted(ServerI::InternalServerState state)
     {
 	return state == ServerI::ActivationTimeout || state == ServerI::Active;
     }
@@ -351,7 +352,8 @@ public:
     {
     }
 
-    static bool isStopped(ServerI::InternalServerState state)
+    static bool 
+    isStopped(ServerI::InternalServerState state)
     {
 	return state == ServerI::Inactive || state == ServerI::Patching || state == ServerI::Loading;
     }
@@ -403,6 +405,36 @@ private:
     vector<AMD_Server_stopPtr> _stopCB;
 };
 
+class DelayedStart : public WaitItem
+{
+public:
+ 
+    DelayedStart(const ServerIPtr& server) : 
+	_server(server)
+    {
+    }
+    
+    virtual void expired(bool destroyed)
+    {
+	if(!destroyed)
+	{
+	    try
+	    {
+		_server->start(ServerI::Always);
+	    }
+	    catch(const ServerStartException& ex)
+	    {
+		cerr << ex.reason << endl;
+		// TODO: Log?
+	    }
+	}
+    }
+
+private:
+
+    const ServerIPtr _server;
+};
+
 struct EnvironmentEval : std::unary_function<string, string>
 {
     string
@@ -513,66 +545,7 @@ ServerI::~ServerI()
 void
 ServerI::start_async(const AMD_Server_startPtr& amdCB, const Ice::Current&)
 {
-    ServerCommandPtr command;
-    {
-	Lock sync(*this);
-	checkDestroyed();
-	if(_state == Destroying)
-	{
-	    throw ServerStartException(_id, "The server is being destroyed.");
-	}
-	
-	//
-	// The server is disabled because it failed and if the time of
-	// the failure is now past the configured duration or if the
-	// server is manualy started, we re-enable the server.
-	//
-	if(_activation == Disabled &&
-	   _failureTime != IceUtil::Time() &&
-	   (amdCB || 
-	    (_disableOnFailure > 0 &&
-	     (_failureTime + IceUtil::Time::seconds(_disableOnFailure) < IceUtil::Time::now()))))
-	{
-	    _failureTime = IceUtil::Time();
-	    _activation = _previousActivation;
-	}
-
-	//
-	// If the amd callback is set, it's a remote start call to
-	// manually activate the server. Otherwise it's a call to
-	// activate the server on demand (called from ServerAdapterI).
-	//
-	if(_activation == Disabled)
-	{
-	    throw ServerStartException(_id, "The server is disabled.");
-	}
-	else if(_activation == Manual && !amdCB)
-	{
-	    throw ServerStartException(_id, "The server activation doesn't allow this activation mode.");
-	}
-	else if(_state == ActivationTimeout)
-	{
-	    throw ServerStartException(_id, "The server activation timed out.");
-	}
-	else if(_state == Active)
-	{
-	    throw ServerStartException(_id, "The server is already active.");
-	}
-
-	if(!_start)
-	{
-	    _start = new StartCommand(this, _node->getWaitQueue(), _activationTimeout);
-	}
-	if(amdCB)
-	{
-	    _start->addCallback(amdCB);
-	}
-	command = nextCommand();
-    }
-    if(command)
-    {
-	command->execute();
-    }
+    start(Manual, amdCB);
 }
 
 void
@@ -615,7 +588,7 @@ ServerI::sendSignal(const string& signal, const Ice::Current& current)
 void
 ServerI::writeMessage(const string& message, Ice::Int fd, const Ice::Current& current)
 {
-    IceUtil::Monitor< ::IceUtil::Mutex>::Lock sync(*this);
+    Lock sync(*this);
     if(_process != 0)
     {
 	try
@@ -644,17 +617,32 @@ ServerI::getPid(const Ice::Current&) const
 void 
 ServerI::setEnabled(bool enabled, const ::Ice::Current&)
 {
+    bool activate = false;
     {
 	Lock sync(*this);
-	if(enabled && _activation < Disabled || !enabled && _activation == Disabled)
+	if(enabled && _activation < Disabled || 
+	   !enabled && _activation == Disabled && _failureTime == IceUtil::Time())
 	{
 	    return;
 	}
 
 	_failureTime = IceUtil::Time();
-	_activation = enabled ? (_desc->activation  == "on-demand" ? OnDemand : Manual) : Disabled;
+	_activation = enabled ? toServerActivation(_desc->activation) : Disabled;
+	activate = _state == Inactive && _activation == Always;
     }
-    
+
+    if(activate)
+    {
+	try
+	{
+	    start(Always);
+	}
+	catch(const ServerStartException&)
+	{
+	    // TODO: Log?
+	}
+    }
+
     NodeObserverPrx observer = _node->getObserver();
     if(observer)
     {
@@ -728,10 +716,110 @@ ServerI::getDistribution() const
 }
 
 void
+ServerI::getDynamicInfo(ServerDynamicInfoSeq& serverInfos, AdapterDynamicInfoSeq& adapterInfos) const
+{
+    //
+    // Add server info if it's not inactive.
+    //
+    ServerAdapterDict adapters;
+    {
+	Lock sync(*this);
+	if(_state == ServerI::Inactive && _activation != Disabled)
+	{
+	    return;
+	}
+	adapters = _adapters;
+	serverInfos.push_back(getDynamicInfo());
+    }
+
+    //
+    // Add adapters info.
+    //
+    for(ServerAdapterDict::const_iterator p = adapters.begin(); p != adapters.end(); ++p)
+    {
+	try
+	{
+	    AdapterDynamicInfo adapter;
+	    adapter.id = p->first;
+	    adapter.proxy = p->second->getDirectProxy();
+	    adapterInfos.push_back(adapter);
+	}
+	catch(const AdapterNotActiveException&)
+	{
+	}
+	catch(const Ice::ObjectNotExistException&)
+	{
+	}
+    }
+}
+
+void
+ServerI::start(ServerActivation activation, const AMD_Server_startPtr& amdCB)
+{
+    ServerCommandPtr command;
+    {
+	Lock sync(*this);
+	checkDestroyed();
+
+	//
+	// Re-enable the server if disabled because of a failure and if
+	// activated manually.
+	//
+	enableAfterFailure(activation == Manual);
+
+	//
+	// Check the current activation mode and the requested activation.
+	//
+	if(_activation == Disabled)
+	{
+	    throw ServerStartException(_id, "The server is disabled.");
+	}
+	else if(_activation == Manual && activation != Manual)
+	{
+	    throw ServerStartException(_id, "The server activation doesn't allow this activation mode.");
+	}
+	else if(_activation != Always && activation == Always)
+	{
+	    assert(!amdCB);
+	    return; // Nothing to do.
+	}
+
+	//
+	// Check the current state.
+	//
+	if(_state == ActivationTimeout)
+	{
+	    throw ServerStartException(_id, "The server activation timed out.");
+	}
+	else if(_state == Active)
+	{
+	    throw ServerStartException(_id, "The server is already active.");
+	}
+	else if(_state == Destroying)
+	{
+	    throw ServerStartException(_id, "The server is being destroyed.");
+	}
+
+	if(!_start)
+	{
+	    _start = new StartCommand(this, _node->getWaitQueue(), _activationTimeout);
+	}
+	if(amdCB)
+	{
+	    _start->addCallback(amdCB);
+	}
+	command = nextCommand();
+    }
+    if(command)
+    {
+	command->execute();
+    }
+}
+
+void
 ServerI::load(const AMD_Node_loadServerPtr& amdCB, const string& application, const ServerDescriptorPtr& desc)
 {
     stop_async(0);
-
     ServerCommandPtr command;
     {
 	Lock sync(*this);
@@ -770,6 +858,28 @@ ServerI::load(const AMD_Node_loadServerPtr& amdCB, const string& application, co
     }
 }
 
+void
+ServerI::destroy(const AMD_Node_destroyServerPtr& amdCB)
+{
+    ServerCommandPtr command;
+    {
+	Lock sync(*this);
+	if(!_destroy)
+	{
+	    _destroy = new DestroyCommand(this, _state != Inactive && _state != Loading && _state != Patching);
+	}
+	if(amdCB)
+	{
+	    _destroy->addCallback(amdCB);
+	}
+	command = nextCommand();
+    }
+    if(command)
+    {
+	command->execute();
+    }
+}
+
 bool
 ServerI::startPatch(bool shutdown)
 {
@@ -821,28 +931,6 @@ ServerI::finishPatch()
 }
 
 void
-ServerI::destroy(const AMD_Node_destroyServerPtr& amdCB)
-{
-    ServerCommandPtr command;
-    {
-	Lock sync(*this);
-	if(!_destroy)
-	{
-	    _destroy = new DestroyCommand(this, _state != Inactive && _state != Loading && _state != Patching);
-	}
-	if(amdCB)
-	{
-	    _destroy->addCallback(amdCB);
-	}
-	command = nextCommand();
-    }
-    if(command)
-    {
-	command->execute();
-    }
-}
-
-void
 ServerI::adapterActivated(const string& id)
 {
     ServerCommandPtr command;
@@ -870,6 +958,39 @@ ServerI::checkDestroyed()
 }
 
 void
+ServerI::disableOnFailure()
+{
+    if(_disableOnFailure != 0 && _activation != Disabled)
+    {
+	_previousActivation = _activation;
+	_activation = Disabled;
+	_failureTime = IceUtil::Time::now();
+    }
+}
+
+void
+ServerI::enableAfterFailure(bool force)
+{
+    if(_disableOnFailure == 0 || _failureTime == IceUtil::Time())
+    {
+	return;
+    }
+
+    if(force || 
+       _disableOnFailure > 0 && (_failureTime + IceUtil::Time::seconds(_disableOnFailure) < IceUtil::Time::now()))
+    {
+	_activation = _previousActivation;
+	_failureTime = IceUtil::Time();
+    }
+
+    if(_timer)
+    {
+	_node->getWaitQueue()->remove(_timer);
+	_timer = 0;
+    }
+}
+
+void
 ServerI::adapterDeactivated(const string& id)
 {
     Lock sync(*this);
@@ -917,7 +1038,7 @@ ServerI::activationFailed(bool destroyed)
     {
 	try
 	{
-	    p->second->activationFailed(destroyed);
+	    p->second->activationFailed(!destroyed);
 	}
 	catch(const Ice::ObjectNotExistException&)
 	{
@@ -930,44 +1051,6 @@ ServerI::activationFailed(bool destroyed)
 }
 
 void
-ServerI::addDynamicInfo(ServerDynamicInfoSeq& serverInfos, AdapterDynamicInfoSeq& adapterInfos) const
-{
-    //
-    // Add server info if it's not inactive.
-    //
-    ServerAdapterDict adapters;
-    {
-	Lock sync(*this);
-	if(_state == ServerI::Inactive && _activation != Disabled)
-	{
-	    return;
-	}
-	adapters = _adapters;
-	serverInfos.push_back(getDynamicInfo());
-    }
-
-    //
-    // Add adapters info.
-    //
-    for(ServerAdapterDict::const_iterator p = adapters.begin(); p != adapters.end(); ++p)
-    {
-	try
-	{
-	    AdapterDynamicInfo adapter;
-	    adapter.id = p->first;
-	    adapter.proxy = p->second->getDirectProxy();
-	    adapterInfos.push_back(adapter);
-	}
-	catch(const AdapterNotActiveException&)
-	{
-	}
-	catch(const Ice::ObjectNotExistException&)
-	{
-	}
-    }
-}
-
-void
 ServerI::activate()
 {
     ServerDescriptorPtr desc;
@@ -1012,7 +1095,13 @@ ServerI::activate()
 	ServerCommandPtr command;
 	{
 	    Lock sync(*this);
-	    assert(_state == Activating);
+	    if(_state != Activating)
+	    {
+		//
+		// It's possible that the server has already terminated.
+		//
+		return;
+	    }
 	    _pid = pid;
 	    setStateNoSync(ServerI::WaitForActivation);
  	    checkActivation();
@@ -1038,17 +1127,29 @@ ServerI::activate()
 	os << ex;
 	failure = os.str();
     }
+
     for(ServerAdapterDict::iterator r = adpts.begin(); r != adpts.end(); ++r)
     {
 	try
 	{
-	    r->second->activationFailed(true);
+	    r->second->activationFailed(false);
 	}
 	catch(const Ice::ObjectNotExistException&)
 	{
 	}
-    }    
-    setState(ServerI::Inactive, failure);
+    }
+    
+    ServerCommandPtr command;
+    {
+	Lock sync(*this);
+	disableOnFailure();
+	setStateNoSync(ServerI::Inactive, failure);
+	command = nextCommand();
+    }
+    if(command)
+    {
+	command->execute();
+    }
 }
 
 void
@@ -1198,25 +1299,20 @@ ServerI::terminated(const string& msg, int status)
 	_process = 0;
 	_pid = 0;
 
-	if(_disableOnFailure != 0 && _activation != Disabled)
-	{
-	    bool failed = false;
+	bool failed = false;
 #ifndef _WIN32
-	    failed = WIFEXITED(status) && WEXITSTATUS(status) != 0;
-	    if(WIFSIGNALED(status))
-	    {
-		int s = WTERMSIG(status);
-		failed = s == SIGABRT || s == SIGILL || s == SIGBUS || s == SIGFPE || s == SIGSEGV;
-	    }
+	failed = WIFEXITED(status) && WEXITSTATUS(status) != 0;
+	if(WIFSIGNALED(status))
+	{
+	    int s = WTERMSIG(status);
+	    failed = s == SIGABRT || s == SIGILL || s == SIGBUS || s == SIGFPE || s == SIGSEGV;
+	}
 #else
-	    failed = status != 0;
+	failed = status != 0;
 #endif
-	    if(failed)
-	    {
-		_previousActivation = _activation;
-		_activation = Disabled;
-		_failureTime = IceUtil::Time::now();
-	    }
+	if(failed)
+	{
+	    disableOnFailure();
 	}
 	
 	if(_state != ServerI::Destroying)
@@ -1317,9 +1413,16 @@ ServerI::updateImpl()
 	}
     }
 
-    if(_activation < Disabled)
+    if(_activation != Disabled || _failureTime != IceUtil::Time())
+    {
+	_activation = toServerActivation(_desc->activation);
+	_failureTime = IceUtil::Time();
+    }
+
+    if(_timer)
     {
-	_activation = _desc->activation  == "on-demand" ? OnDemand : Manual;
+	_node->getWaitQueue()->remove(_timer);
+	_timer = 0;
     }
 	
     istringstream at(_desc->activationTimeout);
@@ -1700,6 +1803,10 @@ ServerI::setStateNoSync(InternalServerState st, const std::string& reason)
 
     if(_state == Destroyed && !_load)
     {
+	//
+	// If the server is destroyed and there's no load command, we
+	// remove the servant from the ASM.
+	//
 	try
 	{
 	    _node->getAdapter()->remove(_this->ice_getIdentity());
@@ -1708,6 +1815,24 @@ ServerI::setStateNoSync(InternalServerState st, const std::string& reason)
 	{
 	}
     }
+    else if(_state == Inactive && _activation == Disabled && 
+	    _disableOnFailure > 0 && _failureTime != IceUtil::Time() && _previousActivation == Always)
+    {
+	//
+	// If the server was disabled because it failed, we schedule a
+	// callback to re-enable it.
+	//
+	_timer = new DelayedStart(this);
+	_node->getWaitQueue()->add(_timer, IceUtil::Time::seconds(_disableOnFailure));
+    }
+    else if(_state == Inactive && _activation == Always && previous != Activating && previous != ActivationTimeout)
+    {
+	if(!_start)
+	{
+	    _start = new StartCommand(this, _node->getWaitQueue(), _activationTimeout);
+	}
+    }
+
 
     if(toServerState(previous) != toServerState(_state))
     {
@@ -1840,7 +1965,7 @@ ServerI::updateConfigFile(const string& serverDir, const CommunicatorDescriptorP
 		ServiceDescriptorPtr s = ServiceDescriptorPtr::dynamicCast(p->descriptor);
 		const string path = serverDir + "/config/config_" + s->name;
 		props.push_back(createProperty("IceBox.Service." + s->name, 
-					       s->entry + " --Ice.Config=\"" + path + "\""));
+					       s->entry + " --Ice.Config=" + path));
 		servicesStr += s->name + " ";
 	    }
 	    props.push_back(createProperty("IceBox.LoadOrder", servicesStr));
@@ -1990,6 +2115,29 @@ ServerI::toServerState(InternalServerState st) const
     }
 }
 
+ServerI::ServerActivation
+ServerI::toServerActivation(const string& activation) const
+{
+    if(activation == "on-demand")
+    {
+	return OnDemand;
+    }
+    else if(activation == "always")
+    {
+	return Always;
+    }
+    else if(activation == "manual" || activation.empty())
+    {
+	return Manual;
+    }
+    else
+    {
+	Ice::Warning out(_node->getTraceLevels()->logger);
+	out << "unknown activation mode `" << activation << "' for server `" << _id << "'";
+	return Manual;
+    }
+}
+
 ServerDynamicInfo
 ServerI::getDynamicInfo() const
 {
diff --git a/cpp/src/IceGrid/ServerI.h b/cpp/src/IceGrid/ServerI.h
index 5d50ad740c5..db0e53bd557 100644
--- a/cpp/src/IceGrid/ServerI.h
+++ b/cpp/src/IceGrid/ServerI.h
@@ -57,6 +57,7 @@ public:
 
     enum ServerActivation
     {
+	Always,
 	OnDemand,
 	Manual,
 	Disabled
@@ -82,18 +83,19 @@ public:
     ServerActivation getActivationMode() const;
     const std::string& getId() const;
     DistributionDescriptor getDistribution() const;
+    void getDynamicInfo(ServerDynamicInfoSeq&, AdapterDynamicInfoSeq&) const;
 
+    void start(ServerActivation, const AMD_Server_startPtr& = AMD_Server_startPtr());
     void load(const AMD_Node_loadServerPtr&, const std::string&, const ServerDescriptorPtr&);
+    void destroy(const AMD_Node_destroyServerPtr&);
     bool startPatch(bool);
     bool waitForPatch();
     void finishPatch();
-    void destroy(const AMD_Node_destroyServerPtr&);
 
     void adapterActivated(const std::string&);
     void adapterDeactivated(const std::string&);
     void activationFailed(bool);
     void deactivationFailed();
-    void addDynamicInfo(ServerDynamicInfoSeq&, AdapterDynamicInfoSeq&) const;
 
     void activate();
     void kill();
@@ -107,6 +109,8 @@ private:
     void updateImpl();
     void checkActivation();
     void checkDestroyed();
+    void disableOnFailure();
+    void enableAfterFailure(bool);
 
     void setState(InternalServerState, const std::string& = std::string());
     ServerCommandPtr nextCommand();
@@ -117,6 +121,7 @@ private:
     void updateDbEnv(const std::string&, const DbEnvDescriptor&);
     PropertyDescriptor createProperty(const std::string&, const std::string& = std::string());
     ServerState toServerState(InternalServerState) const;
+    ServerActivation toServerActivation(const std::string&) const;
     ServerDynamicInfo getDynamicInfo() const;
 
     const NodeIPtr _node;
@@ -140,6 +145,7 @@ private:
     std::set<std::string> _activeAdapters;
     IceUtil::Time _failureTime;
     ServerActivation _previousActivation;
+    WaitItemPtr _timer;
 
     DestroyCommandPtr _destroy;
     StopCommandPtr _stop;
diff --git a/cpp/src/IceGrid/WaitQueue.cpp b/cpp/src/IceGrid/WaitQueue.cpp
index 6d7b058e6a4..7e8c9969155 100644
--- a/cpp/src/IceGrid/WaitQueue.cpp
+++ b/cpp/src/IceGrid/WaitQueue.cpp
@@ -58,7 +58,7 @@ WaitQueue::run()
 	    //
 	    // Notify expired items.
 	    //
-	    while(!_waitQueue.empty())
+	    while(!_waitQueue.empty() && !_destroyed)
 	    {
 		WaitItemPtr item = _waitQueue.front();		    
 		if(item->getExpirationTime() <= IceUtil::Time::now())
@@ -76,7 +76,7 @@ WaitQueue::run()
 		    // Wait until the next item expire or a notification. Note: in any case we 
 		    // get out of this loop to get a chance to execute the work queue.
 		    //
-		    timedWait(item->getExpirationTime() - IceUtil::Time::now());
+		    timedWait(item->getExpirationTime() - IceUtil::Time::now());	    		    
 		}
 	    }
 	}
@@ -97,6 +97,11 @@ WaitQueue::run()
 		}
 	    }
 	}
+
+	if(_destroyed)
+	{
+	    break;
+	}
     }
     
     if(!_waitQueue.empty())
@@ -134,26 +139,19 @@ WaitQueue::add(const WaitItemPtr& item, const IceUtil::Time& wait)
     //
     bool notifyThread = _waitQueue.empty();
 
-    if(wait == IceUtil::Time::seconds(0))
-    {
-	item->setExpirationTime(IceUtil::Time::now());
-    }
-    else
+    IceUtil::Time expire = IceUtil::Time::now() + wait;
+    item->setExpirationTime(expire);
+    
+    list<WaitItemPtr>::iterator p = _waitQueue.begin();
+    while(p != _waitQueue.end())
     {
-	IceUtil::Time expire = IceUtil::Time::now() + wait;
-	item->setExpirationTime(expire);
-	
-	list<WaitItemPtr>::iterator p = _waitQueue.begin();
-	while(p != _waitQueue.end())
+	if((*p)->getExpirationTime() >= expire)
 	{
-	    if((*p)->getExpirationTime() >= expire)
-	    {
-		break;
-	    }
-	    ++p;
+	    break;
 	}
-	_waitQueue.insert(p, item);
+	++p;
     }
+    _waitQueue.insert(p, item);
 
     if(notifyThread)
     {