New memoryCache implementation

New mapFind helper function

12 files changed, 200 insertions, 14 deletions

diff --git a/project2/common/Jamfile.jam b/project2/common/Jamfile.jam
index 9f8f012..5bb0507 100644
--- a/project2/common/Jamfile.jam
+++ b/project2/common/Jamfile.jam
@@ -18,7 +18,7 @@ lib p2common :
 	sourceObject.cpp task.cpp variables.cpp variableConvert.cpp view.cpp xmlObjectLoader.cpp exceptions.cpp cache.cpp
 	sessionContainer.cpp sessionClearTask.cpp session.cpp sessionSetTask.cpp commonObjects.cpp taskHost.cpp checkHost.cpp
 	rowView.cpp rowSet.cpp rowProcessor.cpp config.cpp fileStrmVarWriter.cpp noOutputExecute.cpp columns.cpp scopeObject.cpp
-	transform.cpp definedColumns.cpp structExceptHandling.cpp validDateCheck.cpp
+	transform.cpp definedColumns.cpp structExceptHandling.cpp validDateCheck.cpp memoryCache.cpp
 	variables-modconfig.cpp
 	variables-modlocalparam.cpp
 	variables-modlookup.cpp
diff --git a/project2/common/commonObjects.cpp b/project2/common/commonObjects.cpp
index 2ffa353..8ad1a97 100644
--- a/project2/common/commonObjects.cpp
+++ b/project2/common/commonObjects.cpp
@@ -12,7 +12,7 @@ CommonObjects::~CommonObjects()
 RowSetPtr
 CommonObjects::getSource(const std::string & name) const
 {
-	return safeMapFind<DataSourceNotFound>(rowSets, name)->second;
+	return safeMapLookup<DataSourceNotFound>(rowSets, name);
 }
 
 CommonObjects::DataSources::const_iterator
diff --git a/project2/common/config.cpp b/project2/common/config.cpp
index 6f5b243..cb316e5 100644
--- a/project2/common/config.cpp
+++ b/project2/common/config.cpp
@@ -51,13 +51,13 @@ Configuration::getCurrentConfig() const
 {
 	load();
 	Glib::ustring platformName(resolveCurrentConfig());
-	return safeMapFind<NoSuchPlatform>(platforms, platformName)->second;
+	return safeMapLookup<NoSuchPlatform>(platforms, platformName);
 }
 
 const Glib::ustring &
 Configuration::Platform::getValue(const Glib::ustring & key) const
 {
-	return safeMapFind<NoSuchConfigurationValue>(values, key)->second;
+	return safeMapLookup<NoSuchConfigurationValue>(values, key);
 }
 
 Configuration::Platform::Platform(const xmlpp::Element * e)
diff --git a/project2/common/iHaveParameters.cpp b/project2/common/iHaveParameters.cpp
index 18996e5..cded7e1 100644
--- a/project2/common/iHaveParameters.cpp
+++ b/project2/common/iHaveParameters.cpp
@@ -23,7 +23,7 @@ IHaveParameters::~IHaveParameters()
 VariableType
 IHaveParameters::getParameter(const Glib::ustring & name) const
 {
-	return safeMapFind<ParamNotFound>(parameters, name)->second;
+	return safeMapLookup<ParamNotFound>(parameters, name);
 }
 
 void
diff --git a/project2/common/memoryCache.cpp b/project2/common/memoryCache.cpp
new file mode 100644
index 0000000..34d2cf9
--- /dev/null
+++ b/project2/common/memoryCache.cpp
@@ -0,0 +1,175 @@
+#include <pch.hpp>
+#include "logger.h"
+#include "cache.h"
+#include <boost/foreach.hpp>
+#include <boost/program_options.hpp>
+#include "rowSet.h"
+#include "presenter.h"
+#include "safeMapFind.h"
+#include <boost/multi_index_container.hpp>
+#include <boost/multi_index/member.hpp>
+#include <boost/multi_index/ordered_index.hpp>
+
+class MemoryCache : public Cache {
+	public:
+		typedef std::vector<VariableType> Key;
+		class CachedRowSet : public RowSet, public RowSetPresenter {
+			public:
+				class MemoryCacheRow : public RowState {
+					public:
+						typedef std::map<Glib::ustring, const VariableType> AttrMap;
+						MemoryCacheRow(const Columns * c) :
+							columns(c) {
+						}
+						const Columns & getColumns() const {
+							return *columns;
+						}
+						RowAttribute resolveAttr(const Glib::ustring & attrName) const {
+							return boost::bind(&safeMapLookup<AttributeDoesNotExist, AttrMap>, attrs, attrName);
+						}
+					private:
+						friend class CachedRowSet;
+						const Columns * columns;
+						AttrMap attrs;
+				};
+				typedef boost::shared_ptr<MemoryCacheRow> MemoryCacheRowPtr;
+				typedef std::list<MemoryCacheRowPtr> DataCache;
+
+				CachedRowSet(const std::vector<VariableType> & k) :
+					RowSet(NULL),
+					key(k),
+					createdAt(time(NULL))
+				{
+				}
+
+				void execute(const Glib::ustring&, const RowProcessor * rp) const {
+					BOOST_FOREACH(const DataCache::value_type & mcr, dataCache) {
+						mcr->process(rp, false);
+					}
+				}
+
+				void addAttribute(const Glib::ustring & name, const VariableType & value) const {
+					cur->attrs.insert(MemoryCacheRow::AttrMap::value_type(name, value));
+				}
+
+				void addNamedValue(const Glib::ustring & name, const VariableType & value) const {
+					if (cur->fields.size() <= col) {
+						cur->fields.resize(col + 1);
+						columns.insert(new Column(col, name));
+					}
+					cur->fields[col++] = value;
+				}
+
+				void addNewRow(const Glib::ustring&) const {
+					col = 0;
+					cur = MemoryCacheRowPtr(new MemoryCacheRow(&columns));
+				}
+
+				void finishRow() const {
+					dataCache.push_back(cur);
+					cur.reset();
+				}
+
+				const Key key;
+				const time_t createdAt;
+
+			private:
+				mutable unsigned int col;
+				mutable Columns columns;
+				mutable DataCache dataCache;
+				mutable MemoryCacheRowPtr cur;
+
+		};
+		typedef boost::intrusive_ptr<CachedRowSet> CachedRowSetPtr;
+		typedef boost::intrusive_ptr<const CachedRowSet> CachedRowSetCPtr;
+
+		struct IndexByKey { };
+		struct IndexByTime { };
+		typedef boost::multi_index::multi_index_container<
+			CachedRowSetCPtr,
+			boost::multi_index::indexed_by<
+				boost::multi_index::ordered_unique<
+				boost::multi_index::tag<IndexByKey>, BOOST_MULTI_INDEX_MEMBER(CachedRowSet, const Key, key)>,
+			boost::multi_index::ordered_non_unique<
+				boost::multi_index::tag<IndexByTime>, BOOST_MULTI_INDEX_MEMBER(CachedRowSet, const time_t, createdAt)>
+				> > CacheStore;
+
+		MemoryCache(const xmlpp::Element * p) :
+			Cache(p)
+		{
+		}
+
+		RowSetCPtr getCachedRowSet(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps) const {
+			Key key(makeKey(n, f, ps));
+			CacheStore::index<IndexByKey>::type::const_iterator i = Store.get<IndexByKey>().find(key);
+			if (i == Store.get<IndexByKey>().end()) {
+				return NULL;
+			}
+			if ((*i)->createdAt < (time(NULL) - CacheLife)) {
+				Store.erase(i);
+				return NULL;
+			}
+			return *i;
+		}
+
+		RowSetPresenterPtr openFor(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps) {
+			return (cur = new CachedRowSet(makeKey(n, f, ps)));
+		}
+
+		void save(const Glib::ustring & , const Glib::ustring & , const IHaveParameters * ) {
+			if (cur) {
+				Store.insert(cur);
+				cur.reset();
+			}
+		}
+
+		void discard(const Glib::ustring & , const Glib::ustring & , const IHaveParameters * ) {
+			cur.reset();
+		}
+
+	private:
+		Key makeKey(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps) const {
+			Key key;
+			key.push_back(n);
+			key.push_back(f);
+			applyKeys(boost::bind(&Key::push_back, &key, _2), ps);
+			return key;
+		}
+
+		CachedRowSetPtr cur;
+
+		friend class CustomMemoryCacheLoader;
+		static time_t CacheLife;
+		static CacheStore Store;
+};
+time_t MemoryCache::CacheLife;
+MemoryCache::CacheStore MemoryCache::Store;
+
+namespace po = boost::program_options;
+class CustomMemoryCacheLoader : public ElementLoaderImpl<MemoryCache> {
+	public:
+		CustomMemoryCacheLoader() :
+			opts("Memory Cache options")
+		{
+			opts.add_options()
+				("cache.memory.life", po::value(&MemoryCache::CacheLife)->default_value(3600),
+				 "The age of cache entries after which they are removed (seconds)")
+				;
+		}
+
+		po::options_description * options() {
+			return &opts;
+		}
+
+		void onPeriodic() {
+			typedef MemoryCache::CacheStore::index<MemoryCache::IndexByTime>::type::iterator iter;
+			iter x = MemoryCache::Store.get<MemoryCache::IndexByTime>().begin();
+			iter y = MemoryCache::Store.get<MemoryCache::IndexByTime>().upper_bound(time(NULL) - MemoryCache::CacheLife);
+			MemoryCache::Store.get<MemoryCache::IndexByTime>().erase(x, y);
+		}
+
+	private:
+		po::options_description opts;
+};
+DECLARE_CUSTOM_LOADER("memorycache", CustomMemoryCacheLoader);
+
diff --git a/project2/common/safeMapFind.h b/project2/common/safeMapFind.h
index b27caf3..b461074 100644
--- a/project2/common/safeMapFind.h
+++ b/project2/common/safeMapFind.h
@@ -23,5 +23,16 @@ defaultMapFind(const Map & map, const typename Map::key_type & key, const typena
 	return i->second;
 }
 
+template <class Ex, class Map>
+const typename Map::mapped_type &
+safeMapLookup(const Map & map, const typename Map::key_type & key)
+{
+	typename Map::const_iterator i = map.find(key);
+	if (i == map.end()) {
+		throw Ex(key);
+	}
+	return i->second;
+}
+
 #endif
 
diff --git a/project2/common/session.cpp b/project2/common/session.cpp
index 9736c4d..fd8f569 100644
--- a/project2/common/session.cpp
+++ b/project2/common/session.cpp
@@ -32,7 +32,7 @@ Session::Empty() const
 VariableType
 Session::GetValue(const Glib::ustring & name) const
 {
-	return safeMapFind<VariableNotFound>(vars, name)->second;
+	return safeMapLookup<VariableNotFound>(vars, name);
 }
 
 void
diff --git a/project2/common/variables-modlookup.cpp b/project2/common/variables-modlookup.cpp
index 836bfea..408c98b 100644
--- a/project2/common/variables-modlookup.cpp
+++ b/project2/common/variables-modlookup.cpp
@@ -49,7 +49,7 @@ class VariableLookup : public VariableImplDyn, public RowProcessor {
 			BOOST_FOREACH(const Parameters::value_type & p, parameters) {
 				k.push_back(p.second);
 			}
-			return safeMapFind<NotFound>(map, k)->second;
+			return safeMapLookup<NotFound>(map, k);
 		}
 	private:
 		void fillCache() const
diff --git a/project2/console/consoleAppEngine.cpp b/project2/console/consoleAppEngine.cpp
index cf0548d..a34e078 100644
--- a/project2/console/consoleAppEngine.cpp
+++ b/project2/console/consoleAppEngine.cpp
@@ -80,7 +80,7 @@ ConsoleApplicationEngine::addAppData(const Presenter *) const
 Glib::ustring
 ConsoleApplicationEngine::resolveCurrentConfig() const
 {
-	return safeMapFind<UnknownPlatformAlias>(conplat, _env->getPlatform())->second;
+	return safeMapLookup<UnknownPlatformAlias>(conplat, _env->getPlatform());
 }
 
 void
diff --git a/project2/json/couchSession.cpp b/project2/json/couchSession.cpp
index 35650f2..fbe38f8 100644
--- a/project2/json/couchSession.cpp
+++ b/project2/json/couchSession.cpp
@@ -22,7 +22,7 @@ class CouchSessionContainer : public SessionContainer {
 		virtual SessionPtr getSession(const UUID & uuid) const {
 			try {
 				json::Object obj = getSessionFromServer(uuid);
-				if (boost::get<json::Number>(*safeMapFind<Session::VariableNotFound>(obj, ExpiryKey)->second) > time(NULL)) {
+				if (boost::get<json::Number>(*safeMapLookup<Session::VariableNotFound>(obj, ExpiryKey)) > time(NULL)) {
 					SessionPtr s = new Session(uuid);
 					BOOST_FOREACH(const json::Object::value_type & v, obj) {
 						s->SetValue(v.first, boost::apply_visitor(JsonToProject2(), *v.second));
@@ -162,10 +162,10 @@ class CustomCouchSessionLoader : public SessionContainerLoaderImpl<CouchSessionC
 					c->setopt(CURLOPT_URL, (b + "_temp_view").c_str());
 					c->performRead(boost::bind(CouchSessionContainer::append, &msg, _1, _2));
 					json::Object o = json::parseObject(msg);
-					BOOST_FOREACH(const json::Array::value_type & v, boost::get<json::Array>(*safeMapFind<ParamNotFound>(o, "rows")->second)) {
+					BOOST_FOREACH(const json::Array::value_type & v, boost::get<json::Array>(*safeMapLookup<ParamNotFound>(o, "rows"))) {
 						json::Object rec = boost::get<json::Object>(*v);
-						UUID u = boost::get<json::String>(*safeMapFind<ParamNotFound>(rec, "id")->second).raw();
-						Glib::ustring & rev = boost::get<json::String>(*safeMapFind<ParamNotFound>(rec, "value")->second);
+						UUID u = boost::get<json::String>(*safeMapLookup<ParamNotFound>(rec, "id")).raw();
+						Glib::ustring & rev = boost::get<json::String>(*safeMapLookup<ParamNotFound>(rec, "value"));
 						deleteSession(u, rev);
 					}
 					return;
diff --git a/project2/xml/xmlDocumentCache.cpp b/project2/xml/xmlDocumentCache.cpp
index 9941336..696b2e5 100644
--- a/project2/xml/xmlDocumentCache.cpp
+++ b/project2/xml/xmlDocumentCache.cpp
@@ -83,7 +83,7 @@ XmlDocumentCache::getDocument(const Glib::ustring & url, const char * encoding)
 		cbf.perform();
 		queued.clear();
 	}
-	return safeMapFind<DownloadFailed>(documents, url)->second().get();
+	return safeMapLookup<DownloadFailed>(documents, url)().get();
 }
 
 void
diff --git a/project2/xml/xpathRows.cpp b/project2/xml/xpathRows.cpp
index 7dc9c2b..e65104d 100644
--- a/project2/xml/xpathRows.cpp
+++ b/project2/xml/xpathRows.cpp
@@ -64,7 +64,7 @@ XPathRows::newCurl() const
 void
 XPathRows::execute(const Glib::ustring & filter, const RowProcessor * rp) const
 {
-	FilterViewPtr fv = safeMapFind<FilterNotFound>(fvs, filter)->second;
+	FilterViewPtr fv = safeMapLookup<FilterNotFound>(fvs, filter);
 
 	typedef boost::shared_ptr<xmlXPathObject> xmlXPathObjectSPtr;
 	typedef boost::shared_ptr<xmlXPathContext> xmlXPathContextSPtr;