Split xmlRawRows into 2 variants (preloaded and load from URL)

Push key builder down into the cache base
Add an xmlMemCache impl. that stores XML documents in memory
Pass cache spec params to close also

10 files changed, 243 insertions, 69 deletions

diff --git a/project2/Jamfile.jam b/project2/Jamfile.jam
index e9013ef..baccf01 100644
--- a/project2/Jamfile.jam
+++ b/project2/Jamfile.jam
@@ -85,7 +85,7 @@ lib p2common :
 
 lib p2xml :
 	rawView.cpp xmlPresenter.cpp transformHtml.cpp transformText.cpp xmlRows.cpp
-	xmlRawRows.cpp xslRows.cpp xslRowsCache.cpp xslPreFetch.cpp xmlCache.cpp sessionXml.cpp
+	xmlRawRows.cpp xslRows.cpp xslRowsCache.cpp xslPreFetch.cpp xmlMemCache.cpp xmlCache.cpp sessionXml.cpp
 	:
 	<include>../libmisc
 	<library>libxmlpp
diff --git a/project2/cache.cpp b/project2/cache.cpp
index bdf012b..32dc3c5 100644
--- a/project2/cache.cpp
+++ b/project2/cache.cpp
@@ -2,6 +2,7 @@
 #include "rowSet.h"
 #include "rowProcessor.h"
 #include "logger.h"
+#include <boost/foreach.hpp>
 
 Cache::Cache(const xmlpp::Element * p) :
 	IHaveParameters(p),
@@ -26,3 +27,16 @@ bool Cache::checkAndExecute(const Glib::ustring & n, const Glib::ustring & f, co
 	return false;
 }
 
+void
+Cache::applyKeys(const boost::function2<void, const std::string &, const VariableType &> & f, const IHaveParameters * ps) const
+{
+	BOOST_FOREACH(const IHaveParameters::Parameters::value_type & p, allParameters()) {
+		f(p.first, p.second->value());
+	}
+	if (inherit) {
+		BOOST_FOREACH(const IHaveParameters::Parameters::value_type & p, ps->allParameters()) {
+			f(p.first, p.second->value());
+		}
+	}
+}
+
diff --git a/project2/cache.h b/project2/cache.h
index 9678356..899a01e 100644
--- a/project2/cache.h
+++ b/project2/cache.h
@@ -16,10 +16,11 @@ class Cache : public IHaveParameters, public SourceObject {
 
 		bool checkAndExecute(const Glib::ustring &, const Glib::ustring &, const RowProcessor *);
 		virtual PresenterPtr openFor(const Glib::ustring &, const Glib::ustring &, const IHaveParameters *) = 0;
-		virtual void close() = 0;
+		virtual void close(const Glib::ustring &, const Glib::ustring &, const IHaveParameters *) = 0;
 
 	protected:
 		virtual RowSetCPtr getCachedRowSet(const Glib::ustring &, const Glib::ustring &, const IHaveParameters *) const = 0;
+		void applyKeys(const boost::function2<void, const std::string &, const VariableType &> & f, const IHaveParameters *) const;
 		const bool inherit;
 };
 typedef boost::intrusive_ptr<Cache> CachePtr;
diff --git a/project2/rowProcessor.cpp b/project2/rowProcessor.cpp
index 17f83f4..efd188c 100644
--- a/project2/rowProcessor.cpp
+++ b/project2/rowProcessor.cpp
@@ -36,7 +36,7 @@ RowProcessor::execute() const
 	source->execute(filter, this);
 	tc.clear();
 	BOOST_FOREACH(const CachePtr & c, caches) {
-		c->close();
+		c->close(source->name, filter, this);
 	}
 }
 
diff --git a/project2/xmlCache.cpp b/project2/xmlCache.cpp
index 22937d6..86b098a 100644
--- a/project2/xmlCache.cpp
+++ b/project2/xmlCache.cpp
@@ -9,7 +9,6 @@
 #include <boost/program_options.hpp>
 #include <boost/filesystem/path.hpp>
 #include <boost/filesystem/convenience.hpp>
-#include <libxml++/nodes/textnode.h>
 #include <libxml++/document.h>
 
 class XmlCache : public Cache {
@@ -37,15 +36,15 @@ class XmlCache : public Cache {
 			return NULL;
 		}
 
-		PresenterPtr openFor(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps)
+		PresenterPtr openFor(const Glib::ustring & n, const Glib::ustring &, const IHaveParameters *)
 		{
-			target = getCacheFile(n, f, ps);
 			writeTo = new XmlPresenter(n, Glib::ustring(), Glib::ustring());
 			return writeTo;
 		}
 
-		void close()
+		void close(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps)
 		{
+			boost::filesystem::path target = getCacheFile(n, f, ps);
 			try {
 				boost::filesystem::create_directories(target.parent_path());
 				const xmlpp::Document * d = *writeTo;
@@ -61,24 +60,18 @@ class XmlCache : public Cache {
 		boost::filesystem::path getCacheFile(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps) const
 		{
 			boost::filesystem::path cache = Store / n.raw() / f.raw();
-			appendPath(cache, this);
-			if (inherit) {
-				appendPath(cache, ps);
-			}
+			applyKeys(boost::bind(&appendPath, &cache, _1, _2), ps);
 			cache /= FileName;
 			return cache;
 		}
-		static void appendPath(boost::filesystem::path & cache, const IHaveParameters * ps)
+
+		static void appendPath(boost::filesystem::path * cache, const std::string & n, const VariableType & v)
 		{
-			BOOST_FOREACH(const IHaveParameters::Parameters::value_type & p, ps->allParameters()) {
-				std::string v = p.second->value();
-				cache /= p.first.raw();
-				cache /= v;
-			}
+			*cache /= n;
+			*cache /= v.operator const std::string &();
 		}
 
 		XmlPresenterPtr writeTo;
-		boost::filesystem::path target;
 
 		friend class CustomXmlCacheLoader;
 		static boost::filesystem::path Store;
diff --git a/project2/xmlMemCache.cpp b/project2/xmlMemCache.cpp
new file mode 100644
index 0000000..af54eb5
--- /dev/null
+++ b/project2/xmlMemCache.cpp
@@ -0,0 +1,120 @@
+#include "cache.h"
+#include "logger.h"
+#include "xmlObjectLoader.h"
+#include "iHaveParameters.h"
+#include "xmlRawRows.h"
+#include "xmlPresenter.h"
+#include <sys/stat.h>
+#include <boost/foreach.hpp>
+#include <boost/program_options.hpp>
+#include <libxml++/document.h>
+#include <boost/multi_index_container.hpp>
+#include <boost/multi_index/member.hpp>
+#include <boost/multi_index/ordered_index.hpp>
+
+class XmlMemCache : public Cache {
+	public:
+		class CacheEntry : public IntrusivePtrBase {
+			public:
+				std::vector<std::string> key;
+				time_t createdAt;
+				boost::shared_ptr<const xmlpp::Document> doc;
+		};
+		typedef boost::intrusive_ptr<CacheEntry> CacheEntryPtr;
+		typedef boost::intrusive_ptr<const CacheEntry> CacheEntryCPtr;
+
+		struct IndexByKey { };
+		struct IndexByTime { };
+		typedef boost::multi_index::multi_index_container<
+			boost::intrusive_ptr<const CacheEntry>,
+			boost::multi_index::indexed_by<
+				boost::multi_index::ordered_unique<
+				boost::multi_index::tag<IndexByKey>, BOOST_MULTI_INDEX_MEMBER(CacheEntry, const std::vector<std::string>, key)>,
+			boost::multi_index::ordered_non_unique<
+				boost::multi_index::tag<IndexByTime>, BOOST_MULTI_INDEX_MEMBER(CacheEntry, const time_t, createdAt)>
+				> > CacheStore;
+
+		XmlMemCache(const xmlpp::Element * p) :
+			Cache(p)
+		{
+		}
+
+		void loadComplete(const CommonObjects*)
+		{
+		}
+
+		RowSetCPtr getCachedRowSet(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps) const
+		{
+			std::vector<std::string> key;
+			key.push_back(n);
+			key.push_back(f);
+			applyKeys(boost::bind(&std::vector<std::string>::push_back, &key, _2), ps);
+			CacheStore::index<IndexByKey>::type::const_iterator i = Store.get<IndexByKey>().find(key);
+			if (i == Store.get<IndexByKey>().end()) {
+				return NULL;
+			}
+			if ((*i)->createdAt < (time(NULL) - CacheLife)) {
+				Store.erase(i);
+				return NULL;
+			}
+			return new XmlMemRawRows((*i)->doc);
+		}
+
+		PresenterPtr openFor(const Glib::ustring & n, const Glib::ustring &, const IHaveParameters *)
+		{
+			writeTo = new XmlPresenter(n, Glib::ustring(), Glib::ustring());
+			return writeTo;
+		}
+
+		void close(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps)
+		{
+			CacheEntryPtr ce = new CacheEntry();
+			ce->key.push_back(n);
+			ce->key.push_back(f);
+			applyKeys(boost::bind(&std::vector<std::string>::push_back, &ce->key, _2), ps);
+			time(&ce->createdAt);
+			ce->doc = *(const boost::shared_ptr<xmlpp::Document>*)(*writeTo);
+			Store.insert(ce);
+		}
+
+	private:
+		XmlPresenterPtr writeTo;
+
+		friend class CustomXmlMemCacheLoader;
+		static time_t CacheLife;
+		static CacheStore Store;
+};
+
+time_t XmlMemCache::CacheLife;
+XmlMemCache::CacheStore XmlMemCache::Store;
+
+namespace po = boost::program_options;
+class CustomXmlMemCacheLoader : public ElementLoaderImpl<XmlMemCache> {
+	public:
+		CustomXmlMemCacheLoader() :
+			opts("XML Memory Cache options")
+		{
+			opts.add_options()
+				("cache.xmlmem.life", po::value(&XmlMemCache::CacheLife)->default_value(3600),
+				 "The age of cache entries after which they are removed (seconds)")
+				;
+		}
+
+		po::options_description * options()
+		{
+			return &opts;
+		}
+
+		void onIdle()
+		{
+			typedef XmlMemCache::CacheStore::index<XmlMemCache::IndexByTime>::type::iterator iter;
+			iter x = XmlMemCache::Store.get<XmlMemCache::IndexByTime>().begin();
+			iter y = XmlMemCache::Store.get<XmlMemCache::IndexByTime>().upper_bound(time(NULL) - XmlMemCache::CacheLife);
+			XmlMemCache::Store.get<XmlMemCache::IndexByTime>().erase(x, y);
+		}
+
+	private:
+		po::options_description opts;
+};
+DECLARE_CUSTOM_LOADER("xmlmemcache", CustomXmlMemCacheLoader);
+
diff --git a/project2/xmlPresenter.cpp b/project2/xmlPresenter.cpp
index b9e433c..adaed38 100644
--- a/project2/xmlPresenter.cpp
+++ b/project2/xmlPresenter.cpp
@@ -40,6 +40,11 @@ XmlPresenter::createDoc(const Glib::ustring & responseRootNodeName, const Glib::
 	}
 }
 
+XmlPresenter::operator const boost::shared_ptr<xmlpp::Document> * () const
+{
+	return &responseDoc;
+}
+
 XmlPresenter::operator const xmlpp::Document * () const
 {
 	return responseDoc.get();
diff --git a/project2/xmlPresenter.h b/project2/xmlPresenter.h
index 9b32c31..ae40470 100644
--- a/project2/xmlPresenter.h
+++ b/project2/xmlPresenter.h
@@ -8,7 +8,7 @@ namespace xmlpp {
 	class Document;
 }
 
-class XmlPresenter : public ContentPresenter, public SourceOf<xmlpp::Document>, public SourceOf<xmlDoc> {
+class XmlPresenter : public ContentPresenter, public SourceOf<xmlpp::Document>, public SourceOf<xmlDoc>, public SourceOf<boost::shared_ptr<xmlpp::Document> > {
 	public:
 		typedef boost::shared_ptr<xmlpp::Document> XmlDocumentPtr;
 		XmlPresenter(const Glib::ustring & responseRootNodeName, const Glib::ustring & responseStyle, const Glib::ustring & contentType);
@@ -24,6 +24,7 @@ class XmlPresenter : public ContentPresenter, public SourceOf<xmlpp::Document>,
 
 		operator const xmlpp::Document * () const;
 		operator const xmlDoc * () const;
+		operator const boost::shared_ptr<xmlpp::Document> * () const;
 
 	private:
 		void createDoc(const Glib::ustring & responseRootNodeName, const Glib::ustring & responseStyle) const;
diff --git a/project2/xmlRawRows.cpp b/project2/xmlRawRows.cpp
index 76a87e2..68a4e8c 100644
--- a/project2/xmlRawRows.cpp
+++ b/project2/xmlRawRows.cpp
@@ -5,60 +5,60 @@
 
 DECLARE_LOADER("xmlrawrows", XmlRawRows);
 
-const Columns &
-XmlRawRows::XmlRowState::getColumns() const
-{
-	return cols;
-}
+class XmlRowState : public RowState {
+	public:
+		const Columns & getColumns() const
+		{
+			return cols;
+		}
 
-void
-XmlRawRows::XmlRowState::foreachAttr(const AttrAction & action) const
-{
-	BOOST_FOREACH(const xmlpp::Attribute * a, e->get_attributes()) {
-		action(a->get_name(), a->get_value());
-	}
-	RowState::foreachAttr(action);
-}
+		void foreachAttr(const AttrAction & action) const
+		{
+			BOOST_FOREACH(const xmlpp::Attribute * a, e->get_attributes()) {
+				action(a->get_name(), a->get_value());
+			}
+			RowState::foreachAttr(action);
+		}
 
-RowState::RowAttribute
-XmlRawRows::XmlRowState::resolveAttr(const Glib::ustring & n) const
-{
-	if (e->get_attribute(n)) {
-		return boost::bind(&XmlRowState::getAttr, this, n);
-	}
-	return RowState::resolveAttr(n);
-}
+		RowState::RowAttribute resolveAttr(const Glib::ustring & n) const
+		{
+			if (e->get_attribute(n)) {
+				return boost::bind(&XmlRowState::getAttr, this, n);
+			}
+			return RowState::resolveAttr(n);
+		}
 
-VariableType
-XmlRawRows::XmlRowState::getAttr(const Glib::ustring & n) const
-{
-	if (const xmlpp::Attribute * a = e->get_attribute(n)) {
-		return a->get_value();
-	}
-	throw AttributeDoesNotExist(n);
-}
+		VariableType getAttr(const Glib::ustring & n) const
+		{
+			if (const xmlpp::Attribute * a = e->get_attribute(n)) {
+				return a->get_value();
+			}
+			throw AttributeDoesNotExist(n);
+		}
 
-XmlRawRows::XmlRawRows(const xmlpp::Element * p) :
-	RowSet(p),
-	path(p, "path")
+		Columns cols;
+		const xmlpp::Element * e;
+};
+
+
+XmlRawRowsBase::XmlRawRowsBase(const xmlpp::Element * p) :
+	RowSet(p)
 {
 }
 
-XmlRawRows::XmlRawRows(const Glib::ustring & p) :
-	RowSet(NULL),
-	path(p)
+XmlRawRowsBase::XmlRawRowsBase() :
+	RowSet(NULL)
 {
 }
 
-void XmlRawRows::loadComplete(const CommonObjects*)
+void XmlRawRowsBase::loadComplete(const CommonObjects*)
 {
 }
 
-void XmlRawRows::execute(const Glib::ustring&, const RowProcessor * rp) const
+void XmlRawRowsBase::execute(const xmlpp::Document * doc, const RowProcessor * rp) const
 {
-	xmlpp::DomParser x(path());
 	XmlRowState rs;
-	BOOST_FOREACH(const xmlpp::Node * n, x.get_document()->get_root_node()->get_children()) {
+	BOOST_FOREACH(const xmlpp::Node * n, doc->get_root_node()->get_children()) {
 		if ((rs.e = dynamic_cast<const xmlpp::Element *>(n))) {
 			if (rs.cols.empty()) {
 				unsigned int col = 0;
@@ -84,3 +84,30 @@ void XmlRawRows::execute(const Glib::ustring&, const RowProcessor * rp) const
 	}
 }
 
+XmlRawRows::XmlRawRows(const xmlpp::Element * p) :
+	XmlRawRowsBase(p),
+	path(p, "path")
+{
+}
+
+XmlRawRows::XmlRawRows(const Glib::ustring & p) :
+	path(p)
+{
+}
+
+void XmlRawRows::execute(const Glib::ustring&, const RowProcessor * rp) const
+{
+	xmlpp::DomParser x(path());
+	XmlRawRowsBase::execute(x.get_document(), rp);
+}
+
+XmlMemRawRows::XmlMemRawRows(boost::shared_ptr<const xmlpp::Document> d) :
+	doc(d)
+{
+}
+
+void XmlMemRawRows::execute(const Glib::ustring&, const RowProcessor * rp) const
+{
+	XmlRawRowsBase::execute(doc.get(), rp);
+}
+
diff --git a/project2/xmlRawRows.h b/project2/xmlRawRows.h
index 7a4fd0b..a4e7899 100644
--- a/project2/xmlRawRows.h
+++ b/project2/xmlRawRows.h
@@ -1,25 +1,38 @@
 #include "rowSet.h"
 
-class XmlRawRows : public RowSet {
+namespace xmlpp {
+	class Document;
+}
+
+class XmlRawRowsBase : public RowSet {
 	public:
-		class XmlRowState : public RowState {
-			public:
-				const Columns& getColumns() const;
-				void foreachAttr(const AttrAction & action) const;
-				RowState::RowAttribute resolveAttr(const Glib::ustring & n) const;
-				VariableType getAttr(const Glib::ustring & n) const;
+		XmlRawRowsBase(const xmlpp::Element * p);
+		XmlRawRowsBase();
 
-				Columns cols;
-				const xmlpp::Element * e;
-		};
+		void loadComplete(const CommonObjects*);
+	
+	protected:
+		void execute(const xmlpp::Document *, const RowProcessor * rp) const;
+};
 
+class XmlRawRows : public XmlRawRowsBase {
+	public:
 		XmlRawRows(const xmlpp::Element * p);
 		XmlRawRows(const Glib::ustring & p);
 
-		void loadComplete(const CommonObjects*);
 		void execute(const Glib::ustring&, const RowProcessor * rp) const;
 
 	private:
 		const Variable path;
 };
 
+class XmlMemRawRows : public XmlRawRowsBase {
+	public:
+		XmlMemRawRows(boost::shared_ptr<const xmlpp::Document> d);
+
+		void execute(const Glib::ustring&, const RowProcessor * rp) const;
+
+	private:
+		boost::shared_ptr<const xmlpp::Document> doc;
+};
+