diff options
author | Benoit Foucher <benoit@zeroc.com> | 2005-10-20 13:24:19 +0000 |
---|---|---|
committer | Benoit Foucher <benoit@zeroc.com> | 2005-10-20 13:24:19 +0000 |
commit | 280c0231d0a4630a3327dca70d467613e6e97aea (patch) | |
tree | e408728be00bcac1b06a6582e437602f8526c5af /cpp/src | |
parent | - Update required version of Mono to 1.1.8 in INSTALL.MONO file (diff) | |
download | ice-280c0231d0a4630a3327dca70d467613e6e97aea.tar.bz2 ice-280c0231d0a4630a3327dca70d467613e6e97aea.tar.xz ice-280c0231d0a4630a3327dca70d467613e6e97aea.zip |
Observer bug fixes.
Diffstat (limited to 'cpp/src')
-rw-r--r-- | cpp/src/Ice/PropertyNames.cpp | 3 | ||||
-rw-r--r-- | cpp/src/Ice/PropertyNames.h | 2 | ||||
-rw-r--r-- | cpp/src/IceGrid/IceGridNode.cpp | 16 | ||||
-rw-r--r-- | cpp/src/IceGrid/Internal.ice | 7 | ||||
-rw-r--r-- | cpp/src/IceGrid/NodeI.cpp | 8 | ||||
-rw-r--r-- | cpp/src/IceGrid/NodeI.h | 2 | ||||
-rw-r--r-- | cpp/src/IceGrid/NodeSessionI.cpp | 7 | ||||
-rw-r--r-- | cpp/src/IceGrid/ObserverSessionI.cpp | 52 | ||||
-rw-r--r-- | cpp/src/IceGrid/ObserverSessionI.h | 16 | ||||
-rw-r--r-- | cpp/src/IceGrid/RegistryI.cpp | 24 | ||||
-rw-r--r-- | cpp/src/IceGrid/RegistryI.h | 2 | ||||
-rw-r--r-- | cpp/src/IceGrid/SessionManagerI.cpp | 15 | ||||
-rw-r--r-- | cpp/src/IceGrid/SessionManagerI.h | 4 | ||||
-rw-r--r-- | cpp/src/IceGrid/Topics.cpp | 30 | ||||
-rw-r--r-- | cpp/src/IceGrid/TraceLevels.cpp | 3 | ||||
-rw-r--r-- | cpp/src/IceGrid/TraceLevels.h | 3 |
16 files changed, 150 insertions, 44 deletions
diff --git a/cpp/src/Ice/PropertyNames.cpp b/cpp/src/Ice/PropertyNames.cpp index 824b9a1ea73..a216ee11b7b 100644 --- a/cpp/src/Ice/PropertyNames.cpp +++ b/cpp/src/Ice/PropertyNames.cpp @@ -7,7 +7,7 @@ // // ********************************************************************** -// Generated by makeprops.py from file `../../../config/PropertyNames.def', Thu Oct 20 08:53:58 2005 +// Generated by makeprops.py from file `../config/PropertyNames.def', Thu Oct 20 16:22:26 2005 // IMPORTANT: Do not edit this file -- any edits made here will be lost! @@ -160,6 +160,7 @@ const char* IceInternal::PropertyNames::IceGridProps[] = "IceGrid.Registry.Trace.Node", "IceGrid.Registry.Trace.Object", "IceGrid.Registry.Trace.Server", + "IceGrid.Registry.Trace.Session", 0 }; diff --git a/cpp/src/Ice/PropertyNames.h b/cpp/src/Ice/PropertyNames.h index 410bf35eef4..6d2132afd9b 100644 --- a/cpp/src/Ice/PropertyNames.h +++ b/cpp/src/Ice/PropertyNames.h @@ -7,7 +7,7 @@ // // ********************************************************************** -// Generated by makeprops.py from file `../../../config/PropertyNames.def', Thu Oct 20 08:53:58 2005 +// Generated by makeprops.py from file `../config/PropertyNames.def', Thu Oct 20 16:22:26 2005 // IMPORTANT: Do not edit this file -- any edits made here will be lost! diff --git a/cpp/src/IceGrid/IceGridNode.cpp b/cpp/src/IceGrid/IceGridNode.cpp index fe694d478bc..72f16891863 100644 --- a/cpp/src/IceGrid/IceGridNode.cpp +++ b/cpp/src/IceGrid/IceGridNode.cpp @@ -59,8 +59,11 @@ public: Lock sync(*this); while(!_shutdown) { - _node->keepAlive(); - + int timeout = _node->keepAlive(); + if(timeout > 0) + { + _timeout = IceUtil::Time::seconds(timeout); + } if(!_shutdown) { timedWait(_timeout); @@ -79,7 +82,7 @@ public: private: const NodeIPtr _node; - const IceUtil::Time _timeout; + IceUtil::Time _timeout; bool _shutdown; }; typedef IceUtil::Handle<KeepAliveThread> KeepAliveThreadPtr; @@ -444,10 +447,11 @@ NodeService::start(int argc, char* argv[]) adapter->add(_node, nodeProxy->ice_getIdentity()); // - // Register this node with the node registry. + // Start the keep alive thread. By default we start the thread + // with a 5s timeout, then we'll use the registry node session + // timeout / 2. // - int timeout = properties->getPropertyAsIntWithDefault("IceGrid.Node.KeepAliveTimeout", 5); // 5 seconds - _keepAliveThread = new KeepAliveThread(_node, timeout); + _keepAliveThread = new KeepAliveThread(_node, 5); _keepAliveThread->start(); // diff --git a/cpp/src/IceGrid/Internal.ice b/cpp/src/IceGrid/Internal.ice index f4785861cf1..64d8b124b3b 100644 --- a/cpp/src/IceGrid/Internal.ice +++ b/cpp/src/IceGrid/Internal.ice @@ -318,6 +318,13 @@ interface Registry /** * + * Return the node session timeout. + * + **/ + nonmutating int getTimeout(); + + /** + * * Shutdown the registry. * **/ diff --git a/cpp/src/IceGrid/NodeI.cpp b/cpp/src/IceGrid/NodeI.cpp index b08158e2fd0..0fbaea01cc5 100644 --- a/cpp/src/IceGrid/NodeI.cpp +++ b/cpp/src/IceGrid/NodeI.cpp @@ -456,7 +456,7 @@ NodeI::setSession(const NodeSessionPrx& session, const NodeObserverPrx& observer _observer = observer; } -void +int NodeI::keepAlive() { NodeSessionPrx session = getSession(); @@ -477,12 +477,13 @@ NodeI::keepAlive() { Ice::PropertiesPtr properties = getCommunicator()->getProperties(); const string instanceNameProperty = "IceGrid.InstanceName"; - string instanceName = properties->getPropertyWithDefault(instanceNameProperty, "IceGrid"); - Ice::ObjectPrx obj = getCommunicator()->stringToProxy(instanceName + "/Registry@IceGrid.Registry.Internal"); + string instName = properties->getPropertyWithDefault(instanceNameProperty, "IceGrid"); + Ice::ObjectPrx obj = getCommunicator()->stringToProxy(instName + "/Registry@IceGrid.Registry.Internal"); RegistryPrx registry = RegistryPrx::uncheckedCast(obj); NodeObserverPrx observer; setSession(registry->registerNode(_name, _proxy, _platform.getNodeInfo(), observer), observer); checkConsistency(); + return registry->getTimeout() / 2; } catch(const NodeActiveException&) { @@ -495,6 +496,7 @@ NodeI::keepAlive() _traceLevels->logger->warning(os.str()); } } + return 0; } void diff --git a/cpp/src/IceGrid/NodeI.h b/cpp/src/IceGrid/NodeI.h index 96ceccce07a..06f491bde5c 100644 --- a/cpp/src/IceGrid/NodeI.h +++ b/cpp/src/IceGrid/NodeI.h @@ -54,7 +54,7 @@ public: NodeSessionPrx getSession() const; void setSession(const NodeSessionPrx&, const NodeObserverPrx&); - void keepAlive(); + int keepAlive(); void stop(); private: diff --git a/cpp/src/IceGrid/NodeSessionI.cpp b/cpp/src/IceGrid/NodeSessionI.cpp index 5a127f981fe..cbd06ca581e 100644 --- a/cpp/src/IceGrid/NodeSessionI.cpp +++ b/cpp/src/IceGrid/NodeSessionI.cpp @@ -48,16 +48,15 @@ NodeSessionI::keepAlive(const LoadInfo& load, const Ice::Current& current) throw Ice::ObjectNotExistException(__FILE__, __LINE__); } + _timestamp = IceUtil::Time::now(); + _load = load; + if(_traceLevels->node > 2) { Ice::Trace out(_traceLevels->logger, _traceLevels->nodeCat); out << "node `" << _name << "' keep alive "; out << "(load = " << _load.avg1 << ", " << _load.avg5 << ", " << _load.avg15 << ")"; } - - _timestamp = IceUtil::Time::now(); - _load = load; - } Ice::StringSeq diff --git a/cpp/src/IceGrid/ObserverSessionI.cpp b/cpp/src/IceGrid/ObserverSessionI.cpp index df59d636a8d..14594c85206 100644 --- a/cpp/src/IceGrid/ObserverSessionI.cpp +++ b/cpp/src/IceGrid/ObserverSessionI.cpp @@ -17,14 +17,32 @@ using namespace IceGrid; ObserverSessionI::ObserverSessionI(const string& userId, const DatabasePtr& database, RegistryObserverTopic& registryObserverTopic, - NodeObserverTopic& nodeObserverTopic) : + NodeObserverTopic& nodeObserverTopic, + int timeout) : _userId(userId), + _timeout(timeout), + _traceLevels(database->getTraceLevels()), _updating(false), _destroyed(false), _database(database), - _registryObserverTopic(registryObserverTopic), + _registryObserverTopic(registryObserverTopic), _nodeObserverTopic(nodeObserverTopic) { + if(_traceLevels && _traceLevels->observer > 0) + { + Ice::Trace out(_traceLevels->logger, _traceLevels->observerCat); + out << "observer session `" << _userId << "' created"; + } +} + +ObserverSessionI::~ObserverSessionI() +{ +} + +int +ObserverSessionI::getTimeout(const Ice::Current&) const +{ + return _timeout; } void @@ -40,8 +58,8 @@ ObserverSessionI::setObservers(const RegistryObserverPrx& registryObserver, throw ex; } - _registryObserver = registryObserver; - _nodeObserver = nodeObserver; + _registryObserver = RegistryObserverPrx::uncheckedCast(registryObserver->ice_timeout(_timeout * 1000)); + _nodeObserver = NodeObserverPrx::uncheckedCast(nodeObserver->ice_timeout(_timeout * 1000)); // // Subscribe to the topics. @@ -196,18 +214,29 @@ ObserverSessionI::destroy(const Ice::Current& current) _updating = false; } + _destroyed = true; + // // Unsubscribe from the topics. // _registryObserverTopic.unsubscribe(_registryObserver); _nodeObserverTopic.unsubscribe(_nodeObserver); + + if(_traceLevels && _traceLevels->observer > 0) + { + Ice::Trace out(_traceLevels->logger, _traceLevels->observerCat); + out << "observer session `" << _userId << "' destroyed"; + } + + current.adapter->remove(current.id); } LocalObserverSessionI::LocalObserverSessionI(const string& userId, const DatabasePtr& database, RegistryObserverTopic& registryObserverTopic, - NodeObserverTopic& nodeObserverTopic) : - ObserverSessionI(userId, database, registryObserverTopic, nodeObserverTopic), + NodeObserverTopic& nodeObserverTopic, + int timeout) : + ObserverSessionI(userId, database, registryObserverTopic, nodeObserverTopic, timeout), _timestamp(IceUtil::Time::now()) { } @@ -224,6 +253,12 @@ LocalObserverSessionI::keepAlive(const Ice::Current& current) } _timestamp = IceUtil::Time::now(); + + if(_traceLevels->observer > 1) + { + Ice::Trace out(_traceLevels->logger, _traceLevels->observerCat); + out << "session `" << _userId << "' keep alive"; + } } IceUtil::Time @@ -236,8 +271,9 @@ LocalObserverSessionI::timestamp() const Glacier2ObserverSessionI::Glacier2ObserverSessionI(const string& userId, const DatabasePtr& database, RegistryObserverTopic& registryObserverTopic, - NodeObserverTopic& nodeObserverTopic) : - ObserverSessionI(userId, database, registryObserverTopic, nodeObserverTopic) + NodeObserverTopic& nodeObserverTopic, + int timeout) : + ObserverSessionI(userId, database, registryObserverTopic, nodeObserverTopic, timeout) { } diff --git a/cpp/src/IceGrid/ObserverSessionI.h b/cpp/src/IceGrid/ObserverSessionI.h index 5509e74f3d1..3feacbb9864 100644 --- a/cpp/src/IceGrid/ObserverSessionI.h +++ b/cpp/src/IceGrid/ObserverSessionI.h @@ -22,11 +22,17 @@ namespace IceGrid class Database; typedef IceUtil::Handle<Database> DatabasePtr; +class TraceLevels; +typedef IceUtil::Handle<TraceLevels> TraceLevelsPtr; + class ObserverSessionI : public Session, public SessionI, public IceUtil::Mutex { public: - ObserverSessionI(const std::string&, const DatabasePtr&, RegistryObserverTopic&, NodeObserverTopic&); + ObserverSessionI(const std::string&, const DatabasePtr&, RegistryObserverTopic&, NodeObserverTopic&, int); + virtual ~ObserverSessionI(); + + virtual int getTimeout(const Ice::Current&) const; virtual void setObservers(const RegistryObserverPrx&, const NodeObserverPrx&, const Ice::Current&); virtual void setObserversByIdentity(const Ice::Identity&, const Ice::Identity&, const Ice::Current&); @@ -43,6 +49,8 @@ public: protected: const std::string _userId; + const int _timeout; + const TraceLevelsPtr _traceLevels; bool _updating; bool _destroyed; @@ -60,10 +68,10 @@ class LocalObserverSessionI : public ObserverSessionI { public: - LocalObserverSessionI(const std::string&, const DatabasePtr&, RegistryObserverTopic&, NodeObserverTopic&); + LocalObserverSessionI(const std::string&, const DatabasePtr&, RegistryObserverTopic&, NodeObserverTopic&, int); virtual void keepAlive(const Ice::Current&); - + virtual IceUtil::Time timestamp() const; private: @@ -75,7 +83,7 @@ class Glacier2ObserverSessionI : public ObserverSessionI { public: - Glacier2ObserverSessionI(const std::string&, const DatabasePtr&, RegistryObserverTopic&, NodeObserverTopic&); + Glacier2ObserverSessionI(const std::string&, const DatabasePtr&, RegistryObserverTopic&, NodeObserverTopic&, int); virtual void keepAlive(const Ice::Current&); diff --git a/cpp/src/IceGrid/RegistryI.cpp b/cpp/src/IceGrid/RegistryI.cpp index 13cedb583c0..d37efab7549 100644 --- a/cpp/src/IceGrid/RegistryI.cpp +++ b/cpp/src/IceGrid/RegistryI.cpp @@ -234,6 +234,13 @@ RegistryI::start(bool nowarn) _reaper = new ReapThread(_nodeSessionTimeout); _reaper->start(); + int adminSessionTimeout = properties->getPropertyAsIntWithDefault("IceGrid.Registry.AdminSessionTimeout", 10); + if(adminSessionTimeout != _nodeSessionTimeout) + { + _adminReaper = new ReapThread(adminSessionTimeout); + _adminReaper->start(); + } + // // Create the internal registries (node, server, adapter, object). // @@ -252,7 +259,7 @@ RegistryI::start(bool nowarn) // bool dynamicReg = properties->getPropertyAsInt("IceGrid.Registry.DynamicRegistration") > 0; ObjectPtr locatorRegistry = new LocatorRegistryI(_database, dynamicReg); - ObjectPrx obj = serverAdapter->add(locatorRegistry, stringToIdentity(instanceName + "/" + IceUtil::generateUUID())); + ObjectPrx obj = serverAdapter->add(locatorRegistry, stringToIdentity(instanceName + "/"+ IceUtil::generateUUID())); LocatorRegistryPrx locatorRegistryPrx = LocatorRegistryPrx::uncheckedCast(obj->ice_collocationOptimization(false)); ObjectPtr locator = new LocatorI(_communicator, _database, locatorRegistryPrx); Identity locatorId = stringToIdentity(instanceName + "/Locator"); @@ -360,7 +367,8 @@ RegistryI::start(bool nowarn) // // Create the session manager. // - ObjectPtr sessionManager = new SessionManagerI(*regTopic, *nodeTopic, _database, _reaper); + ReapThreadPtr reaper = _adminReaper ? _adminReaper : _reaper; + ObjectPtr sessionManager = new SessionManagerI(*regTopic, *nodeTopic, _database, reaper, adminSessionTimeout); Identity sessionManagerId = stringToIdentity(instanceName + "/SessionManager"); adminAdapter->add(sessionManager, sessionManagerId); ObjectPrx sessionManagerPrx = adminAdapter->createDirectProxy(sessionManagerId); @@ -394,6 +402,12 @@ RegistryI::stop() _reaper->terminate(); _reaper->getThreadControl().join(); + if(_adminReaper) + { + _adminReaper->terminate(); + _adminReaper->getThreadControl().join(); + } + _iceStorm->stop(); } @@ -409,6 +423,12 @@ RegistryI::registerNode(const std::string& name, const NodePrx& node, const Node return proxy; } +int +RegistryI::getTimeout(const Ice::Current& current) const +{ + return _nodeSessionTimeout; +} + void RegistryI::shutdown(const Ice::Current& current) { diff --git a/cpp/src/IceGrid/RegistryI.h b/cpp/src/IceGrid/RegistryI.h index 3f5b7d568aa..d52a13a1dd5 100644 --- a/cpp/src/IceGrid/RegistryI.h +++ b/cpp/src/IceGrid/RegistryI.h @@ -34,6 +34,7 @@ public: virtual NodeSessionPrx registerNode(const std::string&, const NodePrx&, const NodeInfo&, NodeObserverPrx&, const Ice::Current&); + virtual int getTimeout(const Ice::Current&) const; virtual void shutdown(const Ice::Current& current); virtual IceStorm::TopicManagerPrx getTopicManager(); @@ -43,6 +44,7 @@ private: Ice::CommunicatorPtr _communicator; DatabasePtr _database; ReapThreadPtr _reaper; + ReapThreadPtr _adminReaper; int _nodeSessionTimeout; IceStorm::ServicePtr _iceStorm; diff --git a/cpp/src/IceGrid/SessionManagerI.cpp b/cpp/src/IceGrid/SessionManagerI.cpp index 201471aa015..1b57c03489a 100644 --- a/cpp/src/IceGrid/SessionManagerI.cpp +++ b/cpp/src/IceGrid/SessionManagerI.cpp @@ -20,15 +20,21 @@ using namespace IceGrid; SessionManagerI::SessionManagerI(RegistryObserverTopic& regTopic, NodeObserverTopic& nodeTopic, const DatabasePtr& database, - const ReapThreadPtr& reaper) : - _registryObserverTopic(regTopic), _nodeObserverTopic(nodeTopic), _database(database), _reaper(reaper) + const ReapThreadPtr& reaper, + int sessionTimeout) : + _registryObserverTopic(regTopic), + _nodeObserverTopic(nodeTopic), + _database(database), + _reaper(reaper), + _sessionTimeout(sessionTimeout) { } Glacier2::SessionPrx SessionManagerI::create(const string& userId, const Ice::Current& current) { - SessionIPtr session = new Glacier2ObserverSessionI(userId, _database, _registryObserverTopic, _nodeObserverTopic); + SessionIPtr session = + new Glacier2ObserverSessionI(userId, _database, _registryObserverTopic, _nodeObserverTopic, _sessionTimeout); Glacier2::SessionPrx proxy = Glacier2::SessionPrx::uncheckedCast(current.adapter->addWithUUID(session)); _reaper->add(proxy, session); return proxy; @@ -37,7 +43,8 @@ SessionManagerI::create(const string& userId, const Ice::Current& current) SessionPrx SessionManagerI::createLocalSession(const string& userId, const Ice::Current& current) { - SessionIPtr session = new LocalObserverSessionI(userId, _database, _registryObserverTopic, _nodeObserverTopic); + SessionIPtr session = + new LocalObserverSessionI(userId, _database, _registryObserverTopic, _nodeObserverTopic, _sessionTimeout); SessionPrx proxy = SessionPrx::uncheckedCast(current.adapter->addWithUUID(session)); _reaper->add(proxy, session); return proxy; diff --git a/cpp/src/IceGrid/SessionManagerI.h b/cpp/src/IceGrid/SessionManagerI.h index 962c7813275..9ca8ce03269 100644 --- a/cpp/src/IceGrid/SessionManagerI.h +++ b/cpp/src/IceGrid/SessionManagerI.h @@ -31,7 +31,7 @@ class SessionManagerI : virtual public SessionManager { public: - SessionManagerI(RegistryObserverTopic&, NodeObserverTopic&, const DatabasePtr&, const ReapThreadPtr&); + SessionManagerI(RegistryObserverTopic&, NodeObserverTopic&, const DatabasePtr&, const ReapThreadPtr&, int); virtual Glacier2::SessionPrx create(const std::string&, const Ice::Current&); virtual SessionPrx createLocalSession(const std::string&, const Ice::Current&); @@ -42,7 +42,7 @@ private: NodeObserverTopic& _nodeObserverTopic; const DatabasePtr _database; const ReapThreadPtr _reaper; - + int _sessionTimeout; }; } diff --git a/cpp/src/IceGrid/Topics.cpp b/cpp/src/IceGrid/Topics.cpp index fed78928fdb..a601ca311ee 100644 --- a/cpp/src/IceGrid/Topics.cpp +++ b/cpp/src/IceGrid/Topics.cpp @@ -181,14 +181,22 @@ NodeObserverTopic::subscribe(const NodeObserverPrx& observer, int serial) { if(serial == -1) { - Lock sync(*this); NodeDynamicInfoSeq nodes; - nodes.reserve(_nodes.size()); - for(map<string, NodeDynamicInfo>::const_iterator p = _nodes.begin(); p != _nodes.end(); ++p) + int serial; { - nodes.push_back(p->second); + Lock sync(*this); + nodes.reserve(_nodes.size()); + for(map<string, NodeDynamicInfo>::const_iterator p = _nodes.begin(); p != _nodes.end(); ++p) + { + nodes.push_back(p->second); + } + serial = _serial; } - observer->init_async(new NodeInitCB(this, observer, _serial), nodes); + // + // TODO: Race conditions are possible here, we should + // check the serial and eventually retry if it changed. + // + observer->init_async(new NodeInitCB(this, observer, serial), nodes); return; } @@ -309,9 +317,15 @@ RegistryObserverTopic::subscribe(const RegistryObserverPrx& observer, int serial { if(serial == -1) { - Lock sync(*this); - assert(_serial != -1); - observer->init_async(new RegistryInitCB(this, observer, _serial), _serial, _applications); + ApplicationDescriptorSeq applications; + int serial; + { + Lock sync(*this); + assert(_serial != -1); + serial = _serial; + applications = _applications; + } + observer->init_async(new RegistryInitCB(this, observer, serial), serial, applications); return; } diff --git a/cpp/src/IceGrid/TraceLevels.cpp b/cpp/src/IceGrid/TraceLevels.cpp index fa0f2aa90a0..4c814d7f1cb 100644 --- a/cpp/src/IceGrid/TraceLevels.cpp +++ b/cpp/src/IceGrid/TraceLevels.cpp @@ -28,6 +28,8 @@ TraceLevels::TraceLevels(const Ice::PropertiesPtr& properties, const Ice::Logger activatorCat("Activator"), patch(0), patchCat("Patch"), + observer(0), + observerCat("Observer"), logger(theLogger) { string keyBase = isNode ? "IceGrid.Node.Trace." : "IceGrid.Registry.Trace."; @@ -38,6 +40,7 @@ TraceLevels::TraceLevels(const Ice::PropertiesPtr& properties, const Ice::Logger const_cast<int&>(object) = properties->getPropertyAsInt(keyBase + objectCat); const_cast<int&>(activator) = properties->getPropertyAsInt(keyBase + activatorCat); const_cast<int&>(patch) = properties->getPropertyAsInt(keyBase + patchCat); + const_cast<int&>(observer) = properties->getPropertyAsInt(keyBase + observerCat); } TraceLevels::~TraceLevels() diff --git a/cpp/src/IceGrid/TraceLevels.h b/cpp/src/IceGrid/TraceLevels.h index 4a5b42a482f..fb198d74fe4 100644 --- a/cpp/src/IceGrid/TraceLevels.h +++ b/cpp/src/IceGrid/TraceLevels.h @@ -45,6 +45,9 @@ public: const int patch; const char* patchCat; + const int observer; + const char* observerCat; + const Ice::LoggerPtr logger; }; |