Add a rowSet for XML files

Add basic rowSet caching
Add rowSet caching support for storage in XML files

18 files changed, 403 insertions, 20 deletions

diff --git a/project2/Jamfile.jam b/project2/Jamfile.jam
index aed9e3d..38c4399 100644
--- a/project2/Jamfile.jam
+++ b/project2/Jamfile.jam
@@ -55,11 +55,11 @@ lib p2uuid :
 
 lib p2common :
 	appEngine.cpp dataSource.cpp environment.cpp fileStarGlibIoChannel.cpp iHaveParameters.cpp library.cpp iHaveSubTasks.cpp
-	iterate.cpp paramChecker.cpp presenter.cpp rawView.cpp logger.cpp if.cpp xmlScriptParser.cpp viewHost.cpp
-	sourceObject.cpp task.cpp variables.cpp variableConvert.cpp view.cpp xmlObjectLoader.cpp exceptions.cpp
-	sessionClearTask.cpp session.cpp sessionSetTask.cpp commonObjects.cpp xmlPresenter.cpp taskHost.cpp checkHost.cpp
+	iterate.cpp paramChecker.cpp presenter.cpp logger.cpp if.cpp xmlScriptParser.cpp viewHost.cpp
+	sourceObject.cpp task.cpp variables.cpp variableConvert.cpp view.cpp xmlObjectLoader.cpp exceptions.cpp cache.cpp
+	sessionContainer.cpp sessionClearTask.cpp session.cpp sessionSetTask.cpp commonObjects.cpp taskHost.cpp checkHost.cpp
 	rowView.cpp rowSet.cpp rowProcessor.cpp config.cpp fileStrmVarWriter.cpp noOutputExecute.cpp columns.cpp scopeObject.cpp
-	transform.cpp transformHtml.cpp transformText.cpp definedColumns.cpp structExceptHandling.cpp validDateCheck.cpp
+	transform.cpp definedColumns.cpp structExceptHandling.cpp validDateCheck.cpp
 	variables-modconfig.cpp
 	variables-modlocalparam.cpp
 	variables-modlookup.cpp
@@ -76,7 +76,6 @@ lib p2common :
 	<library>boost_filesystem
 	<library>boost_date_time
 	<library>boost_program_options
-	<library>libxslt
 	: :
 	<include>.
 	<include>../libmisc
@@ -85,13 +84,15 @@ lib p2common :
 	;
 
 lib p2xml :
-	xmlRows.cpp xslRows.cpp xslRowsCache.cpp xslPreFetch.cpp
+	rawView.cpp xmlPresenter.cpp transformHtml.cpp transformText.cpp xmlRows.cpp
+	xmlRawRows.cpp xslRows.cpp xslRowsCache.cpp xslPreFetch.cpp xmlCache.cpp
 	:
 	<include>../libmisc
 	<library>libxmlpp
 	<library>p2common
 	<library>p2url
 	<library>libxslt
+	<library>boost_filesystem
 	;
 
 lib p2processes :
@@ -124,7 +125,7 @@ lib p2regex :
 	;
 
 lib p2xmlSession :
-	sessionContainer.cpp sessionXml.cpp
+	sessionXml.cpp
 	:
 	<include>../libmisc
 	<library>libxmlpp
@@ -185,6 +186,7 @@ lib p2mail :
 	<library>libxslt
 	<library>esmtp
 	<library>p2common
+	<library>p2xml
 	;
 
 lib p2web :
@@ -198,7 +200,7 @@ lib p2web :
 	<library>p2uuid
 	<library>boost_program_options
 	<library>boost_filesystem
-	<library>p2xmlSession
+	<library>p2xml
 	: :
 	<library>p2parts
 	<library>p2xmlSession
diff --git a/project2/cache.cpp b/project2/cache.cpp
new file mode 100644
index 0000000..0db108b
--- /dev/null
+++ b/project2/cache.cpp
@@ -0,0 +1,26 @@
+#include "cache.h"
+#include "rowSet.h"
+#include "rowProcessor.h"
+#include "logger.h"
+
+Cache::Cache(const xmlpp::Element * p) :
+	SourceObject(p)
+{
+}
+
+bool Cache::checkAndExecute(const Glib::ustring & n, const Glib::ustring & f, const RowProcessor * rp)
+{
+	RowSetCPtr cached = getCachedRowSet(n, f, rp);
+	if (cached) {
+		try {
+			Logger()->messagef(LOG_ERR, "Executing from cache");
+			cached->execute(f, rp);
+			return true;
+		}
+		catch (...) {
+			Logger()->messagef(LOG_WARNING, "Cache failed");
+		}
+	}
+	return false;
+}
+
diff --git a/project2/cache.h b/project2/cache.h
new file mode 100644
index 0000000..358d933
--- /dev/null
+++ b/project2/cache.h
@@ -0,0 +1,27 @@
+#ifndef CACHE_H
+#define CACHE_H
+
+#include "sourceObject.h"
+#include "presenter.h"
+
+class RowProcessor;
+class IHaveParameters;
+class RowSet;
+class RowState;
+typedef boost::intrusive_ptr<const RowSet> RowSetCPtr;
+
+class Cache : public SourceObject {
+	public:
+		Cache(const xmlpp::Element * p);
+
+		bool checkAndExecute(const Glib::ustring &, const Glib::ustring &, const RowProcessor *);
+		virtual PresenterPtr openFor(const Glib::ustring &, const Glib::ustring &, const IHaveParameters *) = 0;
+		virtual void close() = 0;
+
+	protected:
+		virtual RowSetCPtr getCachedRowSet(const Glib::ustring &, const Glib::ustring &, const IHaveParameters *) const = 0;
+};
+typedef boost::intrusive_ptr<Cache> CachePtr;
+
+#endif
+
diff --git a/project2/environment.cpp b/project2/environment.cpp
index 12591bf..0c563cc 100644
--- a/project2/environment.cpp
+++ b/project2/environment.cpp
@@ -75,6 +75,7 @@ Environment::init()
 	LoaderBase::onAllComponents(boost::bind(
 				injectSettingsHelper, argc, argv, boost::bind(&ComponentLoader::options, _1), (po::positional_options_description*)NULL));
 
+	Logger()->clear();
 	if (clLevel >= 0) {
 		Logger()->addLogger(new ConsoleLogDriver(stderr, clLevel, false));
 	}
@@ -86,7 +87,6 @@ Environment::init()
 Environment::~Environment()
 {
 	currentEnv = NULL;
-	Logger()->clear();
 }
 
 const Environment *
diff --git a/project2/fsRows.cpp b/project2/fsRows.cpp
index b308f38..cb73b04 100644
--- a/project2/fsRows.cpp
+++ b/project2/fsRows.cpp
@@ -244,6 +244,20 @@ FsRows::SearchState::resolveAttr(const Glib::ustring & a) const
 	return RowState::resolveAttr(a);
 }
 
+void
+FsRows::SearchState::foreachAttr(const AttrAction & action) const
+{
+	action(field_relPath, fileRelPath());
+	action(field_size, fileSize());
+	action(field_modDate, fileModDate());
+	action(field_user, fileUser());
+	action(field_group, fileGroup());
+	action(field_mode, fileMode());
+	action(field_perms, filePerms());
+	action(field_type, fileType());
+	RowState::foreachAttr(action);
+}
+
 VariableType
 FsRows::SearchState::fileRelPath() const
 {
diff --git a/project2/fsRows.h b/project2/fsRows.h
index 8cd1a9f..3356b37 100644
--- a/project2/fsRows.h
+++ b/project2/fsRows.h
@@ -38,6 +38,7 @@ class FsRows : public RowSet {
 				SearchState(const boost::filesystem::path & r);
 
 				virtual RowAttribute resolveAttr(const Glib::ustring & attrName) const;
+				virtual void foreachAttr(const AttrAction & action) const;
 				virtual const Columns & getColumns() const;
 
 				VariableType fileRelPath() const;
diff --git a/project2/iHaveParameters.cpp b/project2/iHaveParameters.cpp
index d47fc42..e254a4f 100644
--- a/project2/iHaveParameters.cpp
+++ b/project2/iHaveParameters.cpp
@@ -61,3 +61,9 @@ IHaveParameters::getScopedParameter(const Glib::ustring & name)
 	throw ParamNotFound(name);
 }
 
+const IHaveParameters::Parameters &
+IHaveParameters::allParameters() const
+{
+	return parameters;
+}
+
diff --git a/project2/iHaveParameters.h b/project2/iHaveParameters.h
index c6dcd45..51378ee 100644
--- a/project2/iHaveParameters.h
+++ b/project2/iHaveParameters.h
@@ -16,14 +16,16 @@ class IHaveParameters {
 				const Glib::ustring name;
 				const Variable value;
 		};
+		typedef boost::intrusive_ptr<Parameter> ParameterPtr;
+		typedef std::map<Glib::ustring, ParameterPtr> Parameters;
+
 		IHaveParameters(const xmlpp::Element * p);
 		virtual ~IHaveParameters() = 0;
 
+		const Parameters & allParameters() const;
 		VariableType getParameter(const Glib::ustring &) const;
 		static VariableType getScopedParameter(const Glib::ustring &);
 
-		typedef boost::intrusive_ptr<Parameter> ParameterPtr;
-		typedef std::map<Glib::ustring, ParameterPtr> Parameters;
 	protected:
 		Parameters parameters;
 
diff --git a/project2/iterate.cpp b/project2/iterate.cpp
index dc59757..119e839 100644
--- a/project2/iterate.cpp
+++ b/project2/iterate.cpp
@@ -36,9 +36,7 @@ Iterate::rowReady(const RowState *) const
 void
 Iterate::execute() const
 {
-	IHaveParameters::push(this);
-	ScopeObject _ihp(boost::bind(&IHaveParameters::pop, this));
-	source->execute(filter, this);
+	RowProcessor::execute();
 }
 
 void
diff --git a/project2/rowProcessor.cpp b/project2/rowProcessor.cpp
index caff826..17f83f4 100644
--- a/project2/rowProcessor.cpp
+++ b/project2/rowProcessor.cpp
@@ -1,6 +1,7 @@
 #include "rowProcessor.h"
+#include "logger.h"
 #include "commonObjects.h"
-#include <cstdlib>
+#include "scopeObject.h"
 #include <boost/foreach.hpp>
 
 RowProcessor::RowProcessor(const xmlpp::Element * p) :
@@ -8,6 +9,9 @@ RowProcessor::RowProcessor(const xmlpp::Element * p) :
 	recordSource(p->get_attribute_value("source")),
 	filter(p->get_attribute_value("filter"))
 {
+	LoaderBase loader(true);
+	loader.supportedStorers.insert(Storer::into(&caches));
+	loader.collectAll(p, true, IgnoreUnsupported);
 }
 
 void
@@ -16,3 +20,35 @@ RowProcessor::loadComplete(const CommonObjects * co)
 	source = co->getSource(recordSource);
 }
 
+void
+RowProcessor::execute() const
+{
+	IHaveParameters::push(this);
+	ScopeObject _ihp(boost::bind(&IHaveParameters::pop, this));
+	BOOST_FOREACH(const CachePtr & c, caches) {
+		if (c->checkAndExecute(source->name, filter, this)) {
+			return;
+		}
+	}
+	BOOST_FOREACH(const CachePtr & c, caches) {
+		tc.insert(c->openFor(source->name, filter, this));
+	}
+	source->execute(filter, this);
+	tc.clear();
+	BOOST_FOREACH(const CachePtr & c, caches) {
+		c->close();
+	}
+}
+
+void
+RowProcessor::rowReadyInternal(const RowState * rs) const
+{
+	BOOST_FOREACH(const TargetCaches::value_type & c, tc) {
+		c->pushSub(filter.empty() ? "row" : filter);
+		rs->foreachColumn(boost::bind(&Presenter::addField, c, _2, _3));
+		rs->foreachAttr(boost::bind(&Presenter::addAttr, c, _1, _2));
+		c->popSub();
+	}
+	rowReady(rs);
+}
+
diff --git a/project2/rowProcessor.h b/project2/rowProcessor.h
index 38142e5..c25dc73 100644
--- a/project2/rowProcessor.h
+++ b/project2/rowProcessor.h
@@ -5,6 +5,10 @@
 #include "sourceObject.h"
 #include "iHaveParameters.h"
 #include "rowSet.h"
+#include "cache.h"
+#include "xmlStorage.h"
+
+class Presenter;
 
 /// Base class for Project2 components that work with row sets
 class RowProcessor : public IHaveParameters {
@@ -17,9 +21,16 @@ class RowProcessor : public IHaveParameters {
 
 	protected:
 		boost::intrusive_ptr<RowSet> source;
+		void execute() const;
+
 	private:
 		friend class RowState;
+		void rowReadyInternal(const RowState *) const;
 		virtual void rowReady(const RowState *) const = 0;
+		typedef ANONORDEREDSTORAGEOF(Cache) Caches;
+		Caches caches;
+		typedef std::set<PresenterPtr> TargetCaches;
+		mutable TargetCaches tc;
 };
 
 #endif
diff --git a/project2/rowSet.cpp b/project2/rowSet.cpp
index acc9246..6902fd3 100644
--- a/project2/rowSet.cpp
+++ b/project2/rowSet.cpp
@@ -33,7 +33,7 @@ RowState::process(const RowProcessor * rp, bool r)
 	rowNum += 1;
 	stack.push_back(this);
 	ScopeObject s(boost::bind(&RowState::RowValuesStack::pop_back, &stack));
-	rp->rowReady(this);
+	rp->rowReadyInternal(this);
 	if (r) {
 		reset();
 	}
@@ -91,3 +91,9 @@ RowState::foreachColumn(const ColumnAction & action) const
 	}
 }
 
+void
+RowState::foreachAttr(const AttrAction &) const
+{
+	// rowNum is magic, so it doesn't count :)
+}
+
diff --git a/project2/rowSet.h b/project2/rowSet.h
index a9919f9..a181059 100644
--- a/project2/rowSet.h
+++ b/project2/rowSet.h
@@ -36,6 +36,7 @@ class RowState {
 
 		typedef boost::function0<VariableType> RowAttribute;
 		typedef boost::function3<void, unsigned int, const Glib::ustring &, const VariableType &> ColumnAction;
+		typedef boost::function2<void, const Glib::ustring &, const VariableType &> AttrAction;
 		SimpleMessageException(AttributeDoesNotExist);
 
 		VariableType getRowNum() const;
@@ -45,6 +46,7 @@ class RowState {
 		virtual VariableType getCurrentValue(const Glib::ustring & id) const;
 		virtual RowAttribute resolveAttr(const Glib::ustring & attrName) const;
 		void foreachColumn(const ColumnAction & action) const;
+		virtual void foreachAttr(const AttrAction & action) const;
 		virtual const Columns & getColumns() const = 0;
 
 		typedef std::vector<VariableType> FieldValues;
diff --git a/project2/rowView.cpp b/project2/rowView.cpp
index 51472c5..092192f 100644
--- a/project2/rowView.cpp
+++ b/project2/rowView.cpp
@@ -59,10 +59,8 @@ RowView::execute(const Presenter * p) const
 {
 	presenter = p;
 	presenter->pushSub(rootName);
-	ScopeObject _ihp(boost::bind(&IHaveParameters::pop, this));
-	IHaveParameters::push(this);
 	ScopeObject pres(boost::bind(&Presenter::popSub, p));
-	source->execute(filter, this);
+	RowProcessor::execute();
 }
 
 void
diff --git a/project2/sourceObject.cpp b/project2/sourceObject.cpp
index b2bcf12..a5736ba 100644
--- a/project2/sourceObject.cpp
+++ b/project2/sourceObject.cpp
@@ -3,7 +3,7 @@
 unsigned int SourceObject::loadOrder = 1;
 
 SourceObject::SourceObject(const xmlpp::Element * p) :
-	name(p->get_attribute_value("name")),
+	name(p ? p->get_attribute_value("name") : "anon"),
 	order(loadOrder++)
 {
 }
diff --git a/project2/xmlCache.cpp b/project2/xmlCache.cpp
new file mode 100644
index 0000000..0a166e8
--- /dev/null
+++ b/project2/xmlCache.cpp
@@ -0,0 +1,143 @@
+#include "cache.h"
+#include "logger.h"
+#include "xmlObjectLoader.h"
+#include "iHaveParameters.h"
+#include "xmlRawRows.h"
+#include "xmlPresenter.h"
+#include <sys/stat.h>
+#include <boost/foreach.hpp>
+#include <boost/program_options.hpp>
+#include <boost/filesystem/path.hpp>
+#include <boost/filesystem/convenience.hpp>
+#include <libxml++/nodes/textnode.h>
+#include <libxml++/document.h>
+
+class XmlCache : public Cache {
+	public:
+		XmlCache(const xmlpp::Element * p) :
+			Cache(p)
+		{
+		}
+
+		void loadComplete(const CommonObjects*)
+		{
+		}
+
+		RowSetCPtr getCachedRowSet(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps) const
+		{
+			boost::filesystem::path cache = getCacheFile(n, f, ps);
+			struct stat st;
+			if (stat(cache.string().c_str(), &st) == 0) {
+				if (st.st_mtime < time(NULL) - CacheLife) {
+					boost::filesystem::remove(cache);
+					return NULL;
+				}
+				return new XmlRawRows(cache.string());
+			}
+			return NULL;
+		}
+
+		PresenterPtr openFor(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps)
+		{
+			target = getCacheFile(n, f, ps);
+			writeTo = new XmlPresenter(n, Glib::ustring(), Glib::ustring());
+			return writeTo;
+		}
+
+		void close()
+		{
+			try {
+				boost::filesystem::create_directories(target.parent_path());
+				const xmlpp::Document * d = *writeTo;
+				const_cast<xmlpp::Document *>(d)->write_to_file(target.string());
+				writeTo.reset();
+			}
+			catch (...) {
+				Logger()->messagef(LOG_WARNING, "Failed to save cache (%s)", target.string().c_str());
+			}
+		}
+
+	private:
+		boost::filesystem::path getCacheFile(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps) const
+		{
+			boost::filesystem::path cache = Store / n.raw() / f.raw();
+			BOOST_FOREACH(const IHaveParameters::Parameters::value_type & p, ps->allParameters()) {
+				std::string v = p.second->value();
+				cache /= p.first.raw();
+				cache /= v;
+			}
+			cache /= FileName;
+			return cache;
+		}
+
+		XmlPresenterPtr writeTo;
+		boost::filesystem::path target;
+
+		friend class CustomXmlCacheLoader;
+		static boost::filesystem::path Store;
+		static std::string FileName;
+		static time_t CacheLife;
+};
+
+boost::filesystem::path XmlCache::Store;
+std::string XmlCache::FileName;
+time_t XmlCache::CacheLife;
+
+namespace po = boost::program_options;
+class CustomXmlCacheLoader : public ElementLoaderImpl<XmlCache> {
+	public:
+		CustomXmlCacheLoader() :
+			opts("XML Cache options")
+		{
+			opts.add_options()
+				("cache.xml.store", po::value(&XmlCache::Store)->default_value("/tmp/project2.cache"),
+				 "The root folder of the cache storage area")
+				("cache.xml.filename", po::value(&XmlCache::FileName)->default_value("cache.xml"),
+				 "The filename to store the data in")
+				("cache.xml.life", po::value(&XmlCache::CacheLife)->default_value(3600),
+				 "The age of cache entries after which they are removed (seconds)")
+				;
+		}
+
+		po::options_description * options()
+		{
+			return &opts;
+		}
+
+		void onIdle()
+		{
+			emptyDir(XmlCache::Store);
+		}
+
+		bool emptyDir(const boost::filesystem::path & dir)
+		{
+			bool files = false;
+			boost::filesystem::directory_iterator end;
+			for (boost::filesystem::directory_iterator itr(dir); itr != end; ++itr) {
+				struct stat st;
+				stat(itr->path().string().c_str(), &st);
+				if (S_ISDIR(st.st_mode)) {
+					if (emptyDir(*itr)) {
+						files = true;
+					}
+					else {
+						boost::filesystem::remove(*itr);
+					}
+				}
+				else {
+					if (st.st_mtime > time(NULL) - XmlCache::CacheLife) {
+						files = true;
+					}
+					else {
+						boost::filesystem::remove(*itr);
+					}
+				}
+			}
+			return files;
+		}
+
+	private:
+		po::options_description opts;
+};
+DECLARE_CUSTOM_LOADER("xmlcache", CustomXmlCacheLoader);
+
diff --git a/project2/xmlRawRows.cpp b/project2/xmlRawRows.cpp
new file mode 100644
index 0000000..76a87e2
--- /dev/null
+++ b/project2/xmlRawRows.cpp
@@ -0,0 +1,86 @@
+#include "xmlRawRows.h"
+#include <boost/foreach.hpp>
+#include <libxml++/parsers/domparser.h>
+#include <libxml++/nodes/textnode.h>
+
+DECLARE_LOADER("xmlrawrows", XmlRawRows);
+
+const Columns &
+XmlRawRows::XmlRowState::getColumns() const
+{
+	return cols;
+}
+
+void
+XmlRawRows::XmlRowState::foreachAttr(const AttrAction & action) const
+{
+	BOOST_FOREACH(const xmlpp::Attribute * a, e->get_attributes()) {
+		action(a->get_name(), a->get_value());
+	}
+	RowState::foreachAttr(action);
+}
+
+RowState::RowAttribute
+XmlRawRows::XmlRowState::resolveAttr(const Glib::ustring & n) const
+{
+	if (e->get_attribute(n)) {
+		return boost::bind(&XmlRowState::getAttr, this, n);
+	}
+	return RowState::resolveAttr(n);
+}
+
+VariableType
+XmlRawRows::XmlRowState::getAttr(const Glib::ustring & n) const
+{
+	if (const xmlpp::Attribute * a = e->get_attribute(n)) {
+		return a->get_value();
+	}
+	throw AttributeDoesNotExist(n);
+}
+
+XmlRawRows::XmlRawRows(const xmlpp::Element * p) :
+	RowSet(p),
+	path(p, "path")
+{
+}
+
+XmlRawRows::XmlRawRows(const Glib::ustring & p) :
+	RowSet(NULL),
+	path(p)
+{
+}
+
+void XmlRawRows::loadComplete(const CommonObjects*)
+{
+}
+
+void XmlRawRows::execute(const Glib::ustring&, const RowProcessor * rp) const
+{
+	xmlpp::DomParser x(path());
+	XmlRowState rs;
+	BOOST_FOREACH(const xmlpp::Node * n, x.get_document()->get_root_node()->get_children()) {
+		if ((rs.e = dynamic_cast<const xmlpp::Element *>(n))) {
+			if (rs.cols.empty()) {
+				unsigned int col = 0;
+				BOOST_FOREACH(const xmlpp::Node * in, rs.e->get_children()) {
+					if (const xmlpp::Element * ie = dynamic_cast<const xmlpp::Element *>(in)) {
+						rs.cols.insert(new Column(col++, ie->get_name()));
+					}
+				}
+				rs.fields.resize(col);
+			}
+			unsigned int col = 0;
+			BOOST_FOREACH(const xmlpp::Node * in, rs.e->get_children()) {
+				if (const xmlpp::Element * ie = dynamic_cast<const xmlpp::Element *>(in)) {
+					const xmlpp::TextNode * t = ie->get_child_text();
+					if (t) {
+						rs.fields[col] = t->get_content();
+					}
+					col++;
+				}
+			}
+			rs.process(rp);
+		}
+	}
+}
+
diff --git a/project2/xmlRawRows.h b/project2/xmlRawRows.h
new file mode 100644
index 0000000..7a4fd0b
--- /dev/null
+++ b/project2/xmlRawRows.h
@@ -0,0 +1,25 @@
+#include "rowSet.h"
+
+class XmlRawRows : public RowSet {
+	public:
+		class XmlRowState : public RowState {
+			public:
+				const Columns& getColumns() const;
+				void foreachAttr(const AttrAction & action) const;
+				RowState::RowAttribute resolveAttr(const Glib::ustring & n) const;
+				VariableType getAttr(const Glib::ustring & n) const;
+
+				Columns cols;
+				const xmlpp::Element * e;
+		};
+
+		XmlRawRows(const xmlpp::Element * p);
+		XmlRawRows(const Glib::ustring & p);
+
+		void loadComplete(const CommonObjects*);
+		void execute(const Glib::ustring&, const RowProcessor * rp) const;
+
+	private:
+		const Variable path;
+};
+