// ********************************************************************** // // Copyright (c) 2003-2006 ZeroC, Inc. All rights reserved. // // This copy of Ice is licensed to you under the terms described in the // ICE_LICENSE file included in this distribution. // // ********************************************************************** #include #include #include #include #include #include #include #include #include #include #include using namespace std; using namespace IcePatch2; using namespace IceGrid; class LogPatcherFeedback : public PatcherFeedback { public: LogPatcherFeedback(const TraceLevelsPtr& traceLevels, const string& dest) : _traceLevels(traceLevels), _startedPatch(false), _lastProgress(0), _dest(dest) { } void setPatchingPath(const string& path) { _path = path; _startedPatch = false; _lastProgress = 0; } virtual bool noFileSummary(const string& reason) { if(_traceLevels->patch > 0) { Ice::Trace out(_traceLevels->logger, _traceLevels->patchCat); out << _dest << ": can't load summary file (will perform a thorough patch):\n" << reason; } return true; } virtual bool checksumStart() { if(_traceLevels->patch > 0) { Ice::Trace out(_traceLevels->logger, _traceLevels->patchCat); out << _dest << ": started checksum calculation"; } return true; } virtual bool checksumProgress(const string& path) { if(_traceLevels->patch > 2) { Ice::Trace out(_traceLevels->logger, _traceLevels->patchCat); out << _dest << ": calculating checksum for " << getBasename(path); } return true; } virtual bool checksumEnd() { if(_traceLevels->patch > 0) { Ice::Trace out(_traceLevels->logger, _traceLevels->patchCat); out << _dest << ": finished checksum calculation"; } return true; } virtual bool fileListStart() { if(_traceLevels->patch > 0) { Ice::Trace out(_traceLevels->logger, _traceLevels->patchCat); out << _dest << ": getting list of file to patch"; } return true; } virtual bool fileListProgress(Ice::Int percent) { return true; } virtual bool fileListEnd() { if(_traceLevels->patch > 0) { Ice::Trace out(_traceLevels->logger, _traceLevels->patchCat); out << _dest << ": getting list of file to patch completed"; } return true; } virtual bool patchStart(const string& path, Ice::Long size, Ice::Long totalProgress, Ice::Long totalSize) { if(_traceLevels->patch > 1 && totalSize > (1024 * 1024)) { int progress = static_cast(static_cast(totalProgress) / totalSize * 100.0); progress /= 5; progress *= 5; if(progress != _lastProgress) { _lastProgress = progress; Ice::Trace out(_traceLevels->logger, _traceLevels->patchCat); out << _dest << ": downloaded " << progress << "% (" << totalProgress << '/' << totalSize << ')'; if(!_path.empty()) { out << " of " << _path; } } } else if(_traceLevels->patch > 0) { if(!_startedPatch) { Ice::Trace out(_traceLevels->logger, _traceLevels->patchCat); int roundedSize = static_cast(static_cast(totalSize) / 1024); if(roundedSize == 0 && totalSize > 0) { roundedSize = 1; } out << _dest << ": downloading " << (_path.empty() ? string("") : (_path + " ")) << roundedSize << "KB "; _startedPatch = true; } } return true; } virtual bool patchProgress(Ice::Long progress, Ice::Long size, Ice::Long totalProgress, Ice::Long totalSize) { return true; } virtual bool patchEnd() { return true; } private: const TraceLevelsPtr _traceLevels; bool _startedPatch; int _lastProgress; string _path; string _dest; }; NodeI::NodeI(const Ice::ObjectAdapterPtr& adapter, NodeSessionManager& sessions, const ActivatorPtr& activator, const WaitQueuePtr& waitQueue, const TraceLevelsPtr& traceLevels, const NodePrx& proxy, const string& name, const UserAccountMapperPrx& mapper) : _communicator(adapter->getCommunicator()), _adapter(adapter), _sessions(sessions), _activator(activator), _waitQueue(waitQueue), _traceLevels(traceLevels), _name(name), _proxy(proxy), _redirectErrToOut(false), _waitTime(0), _userAccountMapper(mapper), _platform("IceGrid.Node", _communicator, _traceLevels), _fileCache(new FileCache()), _serial(1) { Ice::PropertiesPtr properties = _communicator->getProperties(); const_cast(_dataDir) = _platform.getDataDir(); const_cast(_serversDir) = _dataDir + "/servers"; const_cast(_tmpDir) = _dataDir + "/tmp"; const_cast(_instanceName) = _communicator->getDefaultLocator()->ice_getIdentity().category; const_cast(_waitTime) = properties->getPropertyAsIntWithDefault("IceGrid.Node.WaitTime", 60); const_cast(_outputDir) = properties->getProperty("IceGrid.Node.Output"); const_cast(_redirectErrToOut) = properties->getPropertyAsInt("IceGrid.Node.RedirectErrToOut") > 0; } NodeI::~NodeI() { } void NodeI::loadServer_async(const AMD_Node_loadServerPtr& amdCB, const ServerInfo& info, bool fromMaster, const Ice::Current& current) { Lock sync(*this); ++_serial; Ice::Identity id = createServerIdentity(info.descriptor->id); // // Check if we already have a servant for this server. If that's // the case, the server is already loaded and we just need to // update it. // while(true) { bool added = false; ServerIPtr server = ServerIPtr::dynamicCast(_adapter->find(id)); if(!server) { ServerPrx proxy = ServerPrx::uncheckedCast(_adapter->createProxy(id)); server = new ServerI(this, proxy, _serversDir, info.descriptor->id, _waitTime); _adapter->add(server, id); added = true; } try { server->load(amdCB, info, fromMaster); } catch(const Ice::ObjectNotExistException&) { assert(!added); continue; } catch(const Ice::Exception&) { if(added) { _adapter->remove(id); } throw; } break; } } void NodeI::destroyServer_async(const AMD_Node_destroyServerPtr& amdCB, const string& serverId, const string& uuid, int revision, const Ice::Current& current) { Lock sync(*this); ++_serial; ServerIPtr server = ServerIPtr::dynamicCast(_adapter->find(createServerIdentity(serverId))); if(!server) { server = new ServerI(this, 0, _serversDir, serverId, _waitTime); } // // Destroy the server object if it's loaded. // try { server->destroy(amdCB, uuid, revision); } catch(const Ice::ObjectNotExistException&) { } } void NodeI::patch(const string& application, const string& server, const DistributionDescriptor& appDistrib, bool shutdown, const Ice::Current&) { { Lock sync(*this); while(_patchInProgress.find(application) != _patchInProgress.end()) { wait(); } _patchInProgress.insert(application); } set servers; if(!appDistrib.icepatch.empty()) { // // Get all the application servers (even the ones which // don't have a distribution since they depend on the // application distribution). // servers = getApplicationServers(application); } else if(server.empty()) { // // Get all the application servers which have a distribution. // servers = getApplicationServers(application); set::iterator s = servers.begin(); while(s != servers.end()) { if((*s)->getDistribution().icepatch.empty()) { servers.erase(s++); } else { ++s; } } } else { // // Get the given server. // Ice::Identity id = createServerIdentity(server); ServerIPtr svr = ServerIPtr::dynamicCast(_adapter->find(id)); if(svr) { servers.insert(svr); } } try { set::iterator s = servers.begin(); vector running; while(s != servers.end()) { try { if(!(*s)->startPatch(shutdown)) { running.push_back((*s)->getId()); servers.erase(s++); } else { ++s; } } catch(const Ice::ObjectNotExistException&) { servers.erase(s++); } } if((servers.empty() || !appDistrib.icepatch.empty()) && !running.empty()) { PatchException ex; string reason = "patch on node `" + _name + "' failed:\n"; if(running.size() == 1) { reason += "server `" + toString(running) + "' is active"; } else { reason += "servers `" + toString(running, ", ") + "' are active"; } ex.reasons.push_back(reason); throw ex; } for(s = servers.begin(); s != servers.end(); ++s) { (*s)->waitForPatch(); } try { // // Patch the application. // FileServerPrx icepatch; if(!appDistrib.icepatch.empty()) { icepatch = FileServerPrx::checkedCast(_communicator->stringToProxy(appDistrib.icepatch)); if(!icepatch) { throw "proxy `" + appDistrib.icepatch + "' is not a file server."; } patch(icepatch, "distrib/" + application, appDistrib.directories); } // // Patch the server(s). // for(s = servers.begin(); s != servers.end(); ++s) { DistributionDescriptor dist = (*s)->getDistribution(); if(dist.icepatch.empty() || (!server.empty() && (*s)->getId() != server)) { continue; } icepatch = FileServerPrx::checkedCast(_communicator->stringToProxy(dist.icepatch)); if(!icepatch) { throw "proxy `" + dist.icepatch + "' is not a file server."; } patch(icepatch, "servers/" + (*s)->getId() + "/distrib", dist.directories); if(!server.empty()) { break; } } } catch(const Ice::LocalException& e) { ostringstream os; os << "patch on node `" + _name + "' failed:\n" << e; PatchException ex; ex.reasons.push_back(os.str()); throw ex; } catch(const string& e) { PatchException ex; ex.reasons.push_back("patch on node `" + _name + "' failed:\n" + e); throw ex; } for(s = servers.begin(); s != servers.end(); ++s) { (*s)->finishPatch(); } { Lock sync(*this); _patchInProgress.erase(application); notifyAll(); } } catch(...) { for(set::const_iterator s = servers.begin(); s != servers.end(); ++s) { (*s)->finishPatch(); } { Lock sync(*this); _patchInProgress.erase(application); notifyAll(); } throw; } } void NodeI::registerWithReplica(const InternalRegistryPrx& replica, const Ice::Current&) { _sessions.create(replica); } void NodeI::replicaAdded(const InternalRegistryPrx& replica, const Ice::Current&) { _sessions.replicaAdded(replica); } void NodeI::replicaRemoved(const InternalRegistryPrx& replica, const Ice::Current&) { _sessions.replicaRemoved(replica); } std::string NodeI::getName(const Ice::Current&) const { return _name; } std::string NodeI::getHostname(const Ice::Current&) const { return _platform.getHostname(); } LoadInfo NodeI::getLoad(const Ice::Current&) const { return _platform.getLoadInfo(); } void NodeI::shutdown(const Ice::Current&) const { _activator->shutdown(); } Ice::StringSeq NodeI::readLines(const string& filename, Ice::Long pos, int count, Ice::Long& newPos, const Ice::Current&) const { string file; if(filename == "stderr") { file = _communicator->getProperties()->getProperty("Ice.StdErr"); if(file.empty()) { throw FileNotAvailableException("Ice.StdErr configuration property is not set"); } } else if(filename == "stdout") { file = _communicator->getProperties()->getProperty("Ice.StdOut"); if(file.empty()) { throw FileNotAvailableException("Ice.StdOut configuration property is not set"); } } else { throw FileNotAvailableException("unknown file"); } return _fileCache->read(file, pos, count, newPos); } void NodeI::destroy() { IceUtil::Mutex::Lock sync(_serversLock); _serversByApplication.clear(); } Ice::CommunicatorPtr NodeI::getCommunicator() const { return _communicator; } Ice::ObjectAdapterPtr NodeI::getAdapter() const { return _adapter; } ActivatorPtr NodeI::getActivator() const { return _activator; } WaitQueuePtr NodeI::getWaitQueue() const { return _waitQueue; } TraceLevelsPtr NodeI::getTraceLevels() const { return _traceLevels; } UserAccountMapperPrx NodeI::getUserAccountMapper() const { return _userAccountMapper; } PlatformInfo& NodeI::getPlatformInfo() const { return _platform; } FileCachePtr NodeI::getFileCache() const { return _fileCache; } string NodeI::getOutputDir() const { return _outputDir; } bool NodeI::getRedirectErrToOut() const { return _redirectErrToOut; } NodeSessionPrx NodeI::registerWithRegistry(const InternalRegistryPrx& registry) { return registry->registerNode(_name, _proxy, _platform.getNodeInfo()); } void NodeI::checkConsistency(const NodeSessionPrx& session) { // // We use a serial number to keep track of the concurrent changes // on the node. When a server is loaded/destroyed the serial is // incremented. This allows to ensure that the list of servers // returned by the registry is consistent with the servers // currently deployed on the node: if the serial didn't change // after getting the list of servers from the registry, we have // the accurate list of servers that should be deployed on the // node. // unsigned long serial = 0; Ice::StringSeq servers; while(true) { { Lock sync(*this); if(serial == _serial) { _serial = 1; // We can reset the serial number. checkConsistencyNoSync(servers); break; } serial = _serial; } assert(session); try { servers = session->getServers(); } catch(const Ice::LocalException&) { return; // The connection with the session was lost. } sort(servers.begin(), servers.end()); } } void NodeI::addObserver(const NodeSessionPrx& session, const NodeObserverPrx& observer) { IceUtil::Mutex::Lock sync(_observerMutex); assert(_observers.find(session) == _observers.end()); _observers.insert(make_pair(session, observer)); ServerDynamicInfoSeq serverInfos; AdapterDynamicInfoSeq adapterInfos; for(map::const_iterator p = _serversDynamicInfo.begin(); p != _serversDynamicInfo.end(); ++p) { assert(p->second.state != Destroyed && (p->second.state != Inactive || !p->second.enabled)); serverInfos.push_back(p->second); } for(map::const_iterator q = _adaptersDynamicInfo.begin(); q != _adaptersDynamicInfo.end(); ++q) { assert(q->second.proxy); adapterInfos.push_back(q->second); } try { NodeDynamicInfo info; info.info = _platform.getNodeInfo(); info.servers = serverInfos; info.adapters = adapterInfos; observer->nodeUp(info); } catch(const Ice::LocalException& ex) { Ice::Warning out(_traceLevels->logger); out << "unexpected observer exception:\n" << ex; } } void NodeI::removeObserver(const NodeSessionPrx& session) { IceUtil::Mutex::Lock sync(_observerMutex); assert(_observers.find(session) != _observers.end()); _observers.erase(session); } void NodeI::observerUpdateServer(const ServerDynamicInfo& info) { IceUtil::Mutex::Lock sync(_observerMutex); if(info.state == Destroyed || info.state == Inactive && info.enabled) { _serversDynamicInfo.erase(info.id); } else { _serversDynamicInfo[info.id] = info; } // // Send the update and make sure we don't send the update twice to // the same observer (it's possible for the observer to be // registered twice if a replica is removed and added right away // after). // set sent; for(map::const_iterator p = _observers.begin(); p != _observers.end(); ++p) { if(sent.find(p->second) == sent.end()) { try { p->second->updateServer(_name, info); sent.insert(p->second); } catch(const Ice::LocalException&) { // IGNORE } } } } void NodeI::observerUpdateAdapter(const AdapterDynamicInfo& info) { IceUtil::Mutex::Lock sync(_observerMutex); if(info.proxy) { _adaptersDynamicInfo[info.id] = info; } else { _adaptersDynamicInfo.erase(info.id); } // // Send the update and make sure we don't send the update twice to // the same observer (it's possible for the observer to be // registered twice if a replica is removed and added right away // after). // set sent; for(map::const_iterator p = _observers.begin(); p != _observers.end(); ++p) { if(sent.find(p->second) == sent.end()) { try { p->second->updateAdapter(_name, info); } catch(const Ice::LocalException&) { // IGNORE } } } } void NodeI::addServer(const ServerIPtr& server, const string& application, bool dependsOnApplicationDistrib) { IceUtil::Mutex::Lock sync(_serversLock); if(dependsOnApplicationDistrib) { map >::iterator p = _serversByApplication.find(application); if(p == _serversByApplication.end()) { map >::value_type v(application, set()); p = _serversByApplication.insert(p, v); } p->second.insert(server); } } void NodeI::removeServer(const ServerIPtr& server, const std::string& application, bool dependsOnApplicationDistrib) { IceUtil::Mutex::Lock sync(_serversLock); if(dependsOnApplicationDistrib) { map >::iterator p = _serversByApplication.find(application); if(p != _serversByApplication.end()) { p->second.erase(server); if(p->second.empty()) { _serversByApplication.erase(p); } } } } void NodeI::checkConsistencyNoSync(const Ice::StringSeq& servers) { // // Check if the servers directory doesn't contain more servers // than the registry really knows. // Ice::StringSeq contents; try { contents = readDirectory(_serversDir); } catch(const string& msg) { Ice::Error out(_traceLevels->logger); out << "couldn't read directory `" << _serversDir << "':" << msg; return; } vector remove; set_difference(contents.begin(), contents.end(), servers.begin(), servers.end(), back_inserter(remove)); // // Remove the extra servers if possible. // { vector::iterator p = remove.begin(); while(p != remove.end()) { ServerIPtr server = ServerIPtr::dynamicCast(_adapter->find(createServerIdentity(*p))); if(server) { // // If the server is loaded, we invoke on it to destroy it. // try { server->destroy(0, "", 0); p = remove.erase(p); continue; } catch(const Ice::LocalException& ex) { Ice::Error out(_traceLevels->logger); out << "server `" << *p << "' destroy failed:" << ex; } catch(const string&) { assert(false); } } try { if(canRemoveServerDirectory(*p)) { // // If the server directory can be removed and we // either remove it or back it up before to remove it. // removeRecursive(_serversDir + "/" + *p); p = remove.erase(p); continue; } } catch(const string& msg) { Ice::Warning out(_traceLevels->logger); out << "removing server directory `" << _serversDir << "/" << *p << "' failed:" << msg; } ++p; } } if(remove.empty()) { return; } // // If there's server that couldn't be removed we move them to the // temporary backup directory. First, we rotate the temporary // backup directories. // try { contents.clear(); contents = readDirectory(_tmpDir); } catch(const string& msg) { Ice::Error out(_traceLevels->logger); out << "couldn't read directory `" << _tmpDir << "':" << msg; return; } if(contents.size() < 10) { ostringstream os; os << "servers-" << contents.size(); contents.push_back(os.str()); sort(contents.begin(), contents.end(), greater()); } else if(contents.size() == 10) { sort(contents.begin(), contents.end(), greater()); try { removeRecursive(_tmpDir + "/" + *contents.begin()); } catch(const string& msg) { Ice::Warning out(_traceLevels->logger); out << msg; } } try { Ice::StringSeq::const_iterator p; for(p = contents.begin(); p != (contents.end() - 1); ++p) { rename(_tmpDir + "/" + *(p + 1), _tmpDir + "/" + *p); } createDirectoryRecursive(_tmpDir + "/servers-0"); for(p = remove.begin(); p != remove.end(); ++p) { rename(_serversDir + "/" + *p, _tmpDir + "/servers-0/" + *p); } } catch(const string& msg) { Ice::Warning out(_traceLevels->logger); out << "rotation failed: " << msg; } } NodeSessionPrx NodeI::getMasterNodeSession() const { return _sessions.getMasterNodeSession(); } bool NodeI::canRemoveServerDirectory(const string& name) { // // Check if there's files which we didn't create. // Ice::StringSeq contents = readDirectory(_serversDir + "/" + name); remove(contents.begin(), contents.end(), "dbs"); remove(contents.begin(), contents.end(), "config"); remove(contents.begin(), contents.end(), "distrib"); if(!contents.empty()) { return false; } contents = readDirectory(_serversDir + "/" + name + "/config"); Ice::StringSeq::const_iterator p; for(p = contents.begin() ; p != contents.end(); ++p) { if(p->find("config") != 0) { return false; } } contents = readDirectory(_serversDir + "/" + name + "/dbs"); for(p = contents.begin() ; p != contents.end(); ++p) { Ice::StringSeq files = readDirectory(_serversDir + "/" + name + "/dbs/" + *p); remove(files.begin(), files.end(), "DB_CONFIG"); if(!files.empty()) { return false; } } return true; } void NodeI::patch(const FileServerPrx& icepatch, const string& dest, const vector& directories) { PatcherFeedbackPtr feedback = new LogPatcherFeedback(_traceLevels, dest); IcePatch2::createDirectory(_dataDir + "/" + dest); PatcherPtr patcher = new Patcher(icepatch, feedback, _dataDir + "/" + dest, false, 100, 1); bool aborted = !patcher->prepare(); if(!aborted) { if(directories.empty()) { aborted = !patcher->patch(""); } else { for(vector::const_iterator p = directories.begin(); p != directories.end(); ++p) { dynamic_cast(feedback.get())->setPatchingPath(*p); if(!patcher->patch(*p)) { aborted = true; break; } } } } if(!aborted) { patcher->finish(); } // // Update the files owner/group // } set NodeI::getApplicationServers(const string& application) const { IceUtil::Mutex::Lock sync(_serversLock); set servers; map >::const_iterator p = _serversByApplication.find(application); if(p != _serversByApplication.end()) { servers = p->second; } return servers; } Ice::Identity NodeI::createServerIdentity(const string& name) const { Ice::Identity id; id.category = _instanceName + "-Server"; id.name = name; return id; }