diff options
| -rw-r--r-- | project2/Jamfile.jam | 18 | ||||
| -rw-r--r-- | project2/cache.cpp | 26 | ||||
| -rw-r--r-- | project2/cache.h | 27 | ||||
| -rw-r--r-- | project2/environment.cpp | 2 | ||||
| -rw-r--r-- | project2/fsRows.cpp | 14 | ||||
| -rw-r--r-- | project2/fsRows.h | 1 | ||||
| -rw-r--r-- | project2/iHaveParameters.cpp | 6 | ||||
| -rw-r--r-- | project2/iHaveParameters.h | 6 | ||||
| -rw-r--r-- | project2/iterate.cpp | 4 | ||||
| -rw-r--r-- | project2/rowProcessor.cpp | 38 | ||||
| -rw-r--r-- | project2/rowProcessor.h | 11 | ||||
| -rw-r--r-- | project2/rowSet.cpp | 8 | ||||
| -rw-r--r-- | project2/rowSet.h | 2 | ||||
| -rw-r--r-- | project2/rowView.cpp | 4 | ||||
| -rw-r--r-- | project2/sourceObject.cpp | 2 | ||||
| -rw-r--r-- | project2/xmlCache.cpp | 143 | ||||
| -rw-r--r-- | project2/xmlRawRows.cpp | 86 | ||||
| -rw-r--r-- | project2/xmlRawRows.h | 25 | 
18 files changed, 403 insertions, 20 deletions
| diff --git a/project2/Jamfile.jam b/project2/Jamfile.jam index aed9e3d..38c4399 100644 --- a/project2/Jamfile.jam +++ b/project2/Jamfile.jam @@ -55,11 +55,11 @@ lib p2uuid :  lib p2common :  	appEngine.cpp dataSource.cpp environment.cpp fileStarGlibIoChannel.cpp iHaveParameters.cpp library.cpp iHaveSubTasks.cpp -	iterate.cpp paramChecker.cpp presenter.cpp rawView.cpp logger.cpp if.cpp xmlScriptParser.cpp viewHost.cpp -	sourceObject.cpp task.cpp variables.cpp variableConvert.cpp view.cpp xmlObjectLoader.cpp exceptions.cpp -	sessionClearTask.cpp session.cpp sessionSetTask.cpp commonObjects.cpp xmlPresenter.cpp taskHost.cpp checkHost.cpp +	iterate.cpp paramChecker.cpp presenter.cpp logger.cpp if.cpp xmlScriptParser.cpp viewHost.cpp +	sourceObject.cpp task.cpp variables.cpp variableConvert.cpp view.cpp xmlObjectLoader.cpp exceptions.cpp cache.cpp +	sessionContainer.cpp sessionClearTask.cpp session.cpp sessionSetTask.cpp commonObjects.cpp taskHost.cpp checkHost.cpp  	rowView.cpp rowSet.cpp rowProcessor.cpp config.cpp fileStrmVarWriter.cpp noOutputExecute.cpp columns.cpp scopeObject.cpp -	transform.cpp transformHtml.cpp transformText.cpp definedColumns.cpp structExceptHandling.cpp validDateCheck.cpp +	transform.cpp definedColumns.cpp structExceptHandling.cpp validDateCheck.cpp  	variables-modconfig.cpp  	variables-modlocalparam.cpp  	variables-modlookup.cpp @@ -76,7 +76,6 @@ lib p2common :  	<library>boost_filesystem  	<library>boost_date_time  	<library>boost_program_options -	<library>libxslt  	: :  	<include>.  	<include>../libmisc @@ -85,13 +84,15 @@ lib p2common :  	;  lib p2xml : -	xmlRows.cpp xslRows.cpp xslRowsCache.cpp xslPreFetch.cpp +	rawView.cpp xmlPresenter.cpp transformHtml.cpp transformText.cpp xmlRows.cpp +	xmlRawRows.cpp xslRows.cpp xslRowsCache.cpp xslPreFetch.cpp xmlCache.cpp  	:  	<include>../libmisc  	<library>libxmlpp  	<library>p2common  	<library>p2url  	<library>libxslt +	<library>boost_filesystem  	;  lib p2processes : @@ -124,7 +125,7 @@ lib p2regex :  	;  lib p2xmlSession : -	sessionContainer.cpp sessionXml.cpp +	sessionXml.cpp  	:  	<include>../libmisc  	<library>libxmlpp @@ -185,6 +186,7 @@ lib p2mail :  	<library>libxslt  	<library>esmtp  	<library>p2common +	<library>p2xml  	;  lib p2web : @@ -198,7 +200,7 @@ lib p2web :  	<library>p2uuid  	<library>boost_program_options  	<library>boost_filesystem -	<library>p2xmlSession +	<library>p2xml  	: :  	<library>p2parts  	<library>p2xmlSession diff --git a/project2/cache.cpp b/project2/cache.cpp new file mode 100644 index 0000000..0db108b --- /dev/null +++ b/project2/cache.cpp @@ -0,0 +1,26 @@ +#include "cache.h" +#include "rowSet.h" +#include "rowProcessor.h" +#include "logger.h" + +Cache::Cache(const xmlpp::Element * p) : +	SourceObject(p) +{ +} + +bool Cache::checkAndExecute(const Glib::ustring & n, const Glib::ustring & f, const RowProcessor * rp) +{ +	RowSetCPtr cached = getCachedRowSet(n, f, rp); +	if (cached) { +		try { +			Logger()->messagef(LOG_ERR, "Executing from cache"); +			cached->execute(f, rp); +			return true; +		} +		catch (...) { +			Logger()->messagef(LOG_WARNING, "Cache failed"); +		} +	} +	return false; +} + diff --git a/project2/cache.h b/project2/cache.h new file mode 100644 index 0000000..358d933 --- /dev/null +++ b/project2/cache.h @@ -0,0 +1,27 @@ +#ifndef CACHE_H +#define CACHE_H + +#include "sourceObject.h" +#include "presenter.h" + +class RowProcessor; +class IHaveParameters; +class RowSet; +class RowState; +typedef boost::intrusive_ptr<const RowSet> RowSetCPtr; + +class Cache : public SourceObject { +	public: +		Cache(const xmlpp::Element * p); + +		bool checkAndExecute(const Glib::ustring &, const Glib::ustring &, const RowProcessor *); +		virtual PresenterPtr openFor(const Glib::ustring &, const Glib::ustring &, const IHaveParameters *) = 0; +		virtual void close() = 0; + +	protected: +		virtual RowSetCPtr getCachedRowSet(const Glib::ustring &, const Glib::ustring &, const IHaveParameters *) const = 0; +}; +typedef boost::intrusive_ptr<Cache> CachePtr; + +#endif + diff --git a/project2/environment.cpp b/project2/environment.cpp index 12591bf..0c563cc 100644 --- a/project2/environment.cpp +++ b/project2/environment.cpp @@ -75,6 +75,7 @@ Environment::init()  	LoaderBase::onAllComponents(boost::bind(  				injectSettingsHelper, argc, argv, boost::bind(&ComponentLoader::options, _1), (po::positional_options_description*)NULL)); +	Logger()->clear();  	if (clLevel >= 0) {  		Logger()->addLogger(new ConsoleLogDriver(stderr, clLevel, false));  	} @@ -86,7 +87,6 @@ Environment::init()  Environment::~Environment()  {  	currentEnv = NULL; -	Logger()->clear();  }  const Environment * diff --git a/project2/fsRows.cpp b/project2/fsRows.cpp index b308f38..cb73b04 100644 --- a/project2/fsRows.cpp +++ b/project2/fsRows.cpp @@ -244,6 +244,20 @@ FsRows::SearchState::resolveAttr(const Glib::ustring & a) const  	return RowState::resolveAttr(a);  } +void +FsRows::SearchState::foreachAttr(const AttrAction & action) const +{ +	action(field_relPath, fileRelPath()); +	action(field_size, fileSize()); +	action(field_modDate, fileModDate()); +	action(field_user, fileUser()); +	action(field_group, fileGroup()); +	action(field_mode, fileMode()); +	action(field_perms, filePerms()); +	action(field_type, fileType()); +	RowState::foreachAttr(action); +} +  VariableType  FsRows::SearchState::fileRelPath() const  { diff --git a/project2/fsRows.h b/project2/fsRows.h index 8cd1a9f..3356b37 100644 --- a/project2/fsRows.h +++ b/project2/fsRows.h @@ -38,6 +38,7 @@ class FsRows : public RowSet {  				SearchState(const boost::filesystem::path & r);  				virtual RowAttribute resolveAttr(const Glib::ustring & attrName) const; +				virtual void foreachAttr(const AttrAction & action) const;  				virtual const Columns & getColumns() const;  				VariableType fileRelPath() const; diff --git a/project2/iHaveParameters.cpp b/project2/iHaveParameters.cpp index d47fc42..e254a4f 100644 --- a/project2/iHaveParameters.cpp +++ b/project2/iHaveParameters.cpp @@ -61,3 +61,9 @@ IHaveParameters::getScopedParameter(const Glib::ustring & name)  	throw ParamNotFound(name);  } +const IHaveParameters::Parameters & +IHaveParameters::allParameters() const +{ +	return parameters; +} + diff --git a/project2/iHaveParameters.h b/project2/iHaveParameters.h index c6dcd45..51378ee 100644 --- a/project2/iHaveParameters.h +++ b/project2/iHaveParameters.h @@ -16,14 +16,16 @@ class IHaveParameters {  				const Glib::ustring name;  				const Variable value;  		}; +		typedef boost::intrusive_ptr<Parameter> ParameterPtr; +		typedef std::map<Glib::ustring, ParameterPtr> Parameters; +  		IHaveParameters(const xmlpp::Element * p);  		virtual ~IHaveParameters() = 0; +		const Parameters & allParameters() const;  		VariableType getParameter(const Glib::ustring &) const;  		static VariableType getScopedParameter(const Glib::ustring &); -		typedef boost::intrusive_ptr<Parameter> ParameterPtr; -		typedef std::map<Glib::ustring, ParameterPtr> Parameters;  	protected:  		Parameters parameters; diff --git a/project2/iterate.cpp b/project2/iterate.cpp index dc59757..119e839 100644 --- a/project2/iterate.cpp +++ b/project2/iterate.cpp @@ -36,9 +36,7 @@ Iterate::rowReady(const RowState *) const  void  Iterate::execute() const  { -	IHaveParameters::push(this); -	ScopeObject _ihp(boost::bind(&IHaveParameters::pop, this)); -	source->execute(filter, this); +	RowProcessor::execute();  }  void diff --git a/project2/rowProcessor.cpp b/project2/rowProcessor.cpp index caff826..17f83f4 100644 --- a/project2/rowProcessor.cpp +++ b/project2/rowProcessor.cpp @@ -1,6 +1,7 @@  #include "rowProcessor.h" +#include "logger.h"  #include "commonObjects.h" -#include <cstdlib> +#include "scopeObject.h"  #include <boost/foreach.hpp>  RowProcessor::RowProcessor(const xmlpp::Element * p) : @@ -8,6 +9,9 @@ RowProcessor::RowProcessor(const xmlpp::Element * p) :  	recordSource(p->get_attribute_value("source")),  	filter(p->get_attribute_value("filter"))  { +	LoaderBase loader(true); +	loader.supportedStorers.insert(Storer::into(&caches)); +	loader.collectAll(p, true, IgnoreUnsupported);  }  void @@ -16,3 +20,35 @@ RowProcessor::loadComplete(const CommonObjects * co)  	source = co->getSource(recordSource);  } +void +RowProcessor::execute() const +{ +	IHaveParameters::push(this); +	ScopeObject _ihp(boost::bind(&IHaveParameters::pop, this)); +	BOOST_FOREACH(const CachePtr & c, caches) { +		if (c->checkAndExecute(source->name, filter, this)) { +			return; +		} +	} +	BOOST_FOREACH(const CachePtr & c, caches) { +		tc.insert(c->openFor(source->name, filter, this)); +	} +	source->execute(filter, this); +	tc.clear(); +	BOOST_FOREACH(const CachePtr & c, caches) { +		c->close(); +	} +} + +void +RowProcessor::rowReadyInternal(const RowState * rs) const +{ +	BOOST_FOREACH(const TargetCaches::value_type & c, tc) { +		c->pushSub(filter.empty() ? "row" : filter); +		rs->foreachColumn(boost::bind(&Presenter::addField, c, _2, _3)); +		rs->foreachAttr(boost::bind(&Presenter::addAttr, c, _1, _2)); +		c->popSub(); +	} +	rowReady(rs); +} + diff --git a/project2/rowProcessor.h b/project2/rowProcessor.h index 38142e5..c25dc73 100644 --- a/project2/rowProcessor.h +++ b/project2/rowProcessor.h @@ -5,6 +5,10 @@  #include "sourceObject.h"  #include "iHaveParameters.h"  #include "rowSet.h" +#include "cache.h" +#include "xmlStorage.h" + +class Presenter;  /// Base class for Project2 components that work with row sets  class RowProcessor : public IHaveParameters { @@ -17,9 +21,16 @@ class RowProcessor : public IHaveParameters {  	protected:  		boost::intrusive_ptr<RowSet> source; +		void execute() const; +  	private:  		friend class RowState; +		void rowReadyInternal(const RowState *) const;  		virtual void rowReady(const RowState *) const = 0; +		typedef ANONORDEREDSTORAGEOF(Cache) Caches; +		Caches caches; +		typedef std::set<PresenterPtr> TargetCaches; +		mutable TargetCaches tc;  };  #endif diff --git a/project2/rowSet.cpp b/project2/rowSet.cpp index acc9246..6902fd3 100644 --- a/project2/rowSet.cpp +++ b/project2/rowSet.cpp @@ -33,7 +33,7 @@ RowState::process(const RowProcessor * rp, bool r)  	rowNum += 1;  	stack.push_back(this);  	ScopeObject s(boost::bind(&RowState::RowValuesStack::pop_back, &stack)); -	rp->rowReady(this); +	rp->rowReadyInternal(this);  	if (r) {  		reset();  	} @@ -91,3 +91,9 @@ RowState::foreachColumn(const ColumnAction & action) const  	}  } +void +RowState::foreachAttr(const AttrAction &) const +{ +	// rowNum is magic, so it doesn't count :) +} + diff --git a/project2/rowSet.h b/project2/rowSet.h index a9919f9..a181059 100644 --- a/project2/rowSet.h +++ b/project2/rowSet.h @@ -36,6 +36,7 @@ class RowState {  		typedef boost::function0<VariableType> RowAttribute;  		typedef boost::function3<void, unsigned int, const Glib::ustring &, const VariableType &> ColumnAction; +		typedef boost::function2<void, const Glib::ustring &, const VariableType &> AttrAction;  		SimpleMessageException(AttributeDoesNotExist);  		VariableType getRowNum() const; @@ -45,6 +46,7 @@ class RowState {  		virtual VariableType getCurrentValue(const Glib::ustring & id) const;  		virtual RowAttribute resolveAttr(const Glib::ustring & attrName) const;  		void foreachColumn(const ColumnAction & action) const; +		virtual void foreachAttr(const AttrAction & action) const;  		virtual const Columns & getColumns() const = 0;  		typedef std::vector<VariableType> FieldValues; diff --git a/project2/rowView.cpp b/project2/rowView.cpp index 51472c5..092192f 100644 --- a/project2/rowView.cpp +++ b/project2/rowView.cpp @@ -59,10 +59,8 @@ RowView::execute(const Presenter * p) const  {  	presenter = p;  	presenter->pushSub(rootName); -	ScopeObject _ihp(boost::bind(&IHaveParameters::pop, this)); -	IHaveParameters::push(this);  	ScopeObject pres(boost::bind(&Presenter::popSub, p)); -	source->execute(filter, this); +	RowProcessor::execute();  }  void diff --git a/project2/sourceObject.cpp b/project2/sourceObject.cpp index b2bcf12..a5736ba 100644 --- a/project2/sourceObject.cpp +++ b/project2/sourceObject.cpp @@ -3,7 +3,7 @@  unsigned int SourceObject::loadOrder = 1;  SourceObject::SourceObject(const xmlpp::Element * p) : -	name(p->get_attribute_value("name")), +	name(p ? p->get_attribute_value("name") : "anon"),  	order(loadOrder++)  {  } diff --git a/project2/xmlCache.cpp b/project2/xmlCache.cpp new file mode 100644 index 0000000..0a166e8 --- /dev/null +++ b/project2/xmlCache.cpp @@ -0,0 +1,143 @@ +#include "cache.h" +#include "logger.h" +#include "xmlObjectLoader.h" +#include "iHaveParameters.h" +#include "xmlRawRows.h" +#include "xmlPresenter.h" +#include <sys/stat.h> +#include <boost/foreach.hpp> +#include <boost/program_options.hpp> +#include <boost/filesystem/path.hpp> +#include <boost/filesystem/convenience.hpp> +#include <libxml++/nodes/textnode.h> +#include <libxml++/document.h> + +class XmlCache : public Cache { +	public: +		XmlCache(const xmlpp::Element * p) : +			Cache(p) +		{ +		} + +		void loadComplete(const CommonObjects*) +		{ +		} + +		RowSetCPtr getCachedRowSet(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps) const +		{ +			boost::filesystem::path cache = getCacheFile(n, f, ps); +			struct stat st; +			if (stat(cache.string().c_str(), &st) == 0) { +				if (st.st_mtime < time(NULL) - CacheLife) { +					boost::filesystem::remove(cache); +					return NULL; +				} +				return new XmlRawRows(cache.string()); +			} +			return NULL; +		} + +		PresenterPtr openFor(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps) +		{ +			target = getCacheFile(n, f, ps); +			writeTo = new XmlPresenter(n, Glib::ustring(), Glib::ustring()); +			return writeTo; +		} + +		void close() +		{ +			try { +				boost::filesystem::create_directories(target.parent_path()); +				const xmlpp::Document * d = *writeTo; +				const_cast<xmlpp::Document *>(d)->write_to_file(target.string()); +				writeTo.reset(); +			} +			catch (...) { +				Logger()->messagef(LOG_WARNING, "Failed to save cache (%s)", target.string().c_str()); +			} +		} + +	private: +		boost::filesystem::path getCacheFile(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps) const +		{ +			boost::filesystem::path cache = Store / n.raw() / f.raw(); +			BOOST_FOREACH(const IHaveParameters::Parameters::value_type & p, ps->allParameters()) { +				std::string v = p.second->value(); +				cache /= p.first.raw(); +				cache /= v; +			} +			cache /= FileName; +			return cache; +		} + +		XmlPresenterPtr writeTo; +		boost::filesystem::path target; + +		friend class CustomXmlCacheLoader; +		static boost::filesystem::path Store; +		static std::string FileName; +		static time_t CacheLife; +}; + +boost::filesystem::path XmlCache::Store; +std::string XmlCache::FileName; +time_t XmlCache::CacheLife; + +namespace po = boost::program_options; +class CustomXmlCacheLoader : public ElementLoaderImpl<XmlCache> { +	public: +		CustomXmlCacheLoader() : +			opts("XML Cache options") +		{ +			opts.add_options() +				("cache.xml.store", po::value(&XmlCache::Store)->default_value("/tmp/project2.cache"), +				 "The root folder of the cache storage area") +				("cache.xml.filename", po::value(&XmlCache::FileName)->default_value("cache.xml"), +				 "The filename to store the data in") +				("cache.xml.life", po::value(&XmlCache::CacheLife)->default_value(3600), +				 "The age of cache entries after which they are removed (seconds)") +				; +		} + +		po::options_description * options() +		{ +			return &opts; +		} + +		void onIdle() +		{ +			emptyDir(XmlCache::Store); +		} + +		bool emptyDir(const boost::filesystem::path & dir) +		{ +			bool files = false; +			boost::filesystem::directory_iterator end; +			for (boost::filesystem::directory_iterator itr(dir); itr != end; ++itr) { +				struct stat st; +				stat(itr->path().string().c_str(), &st); +				if (S_ISDIR(st.st_mode)) { +					if (emptyDir(*itr)) { +						files = true; +					} +					else { +						boost::filesystem::remove(*itr); +					} +				} +				else { +					if (st.st_mtime > time(NULL) - XmlCache::CacheLife) { +						files = true; +					} +					else { +						boost::filesystem::remove(*itr); +					} +				} +			} +			return files; +		} + +	private: +		po::options_description opts; +}; +DECLARE_CUSTOM_LOADER("xmlcache", CustomXmlCacheLoader); + diff --git a/project2/xmlRawRows.cpp b/project2/xmlRawRows.cpp new file mode 100644 index 0000000..76a87e2 --- /dev/null +++ b/project2/xmlRawRows.cpp @@ -0,0 +1,86 @@ +#include "xmlRawRows.h" +#include <boost/foreach.hpp> +#include <libxml++/parsers/domparser.h> +#include <libxml++/nodes/textnode.h> + +DECLARE_LOADER("xmlrawrows", XmlRawRows); + +const Columns & +XmlRawRows::XmlRowState::getColumns() const +{ +	return cols; +} + +void +XmlRawRows::XmlRowState::foreachAttr(const AttrAction & action) const +{ +	BOOST_FOREACH(const xmlpp::Attribute * a, e->get_attributes()) { +		action(a->get_name(), a->get_value()); +	} +	RowState::foreachAttr(action); +} + +RowState::RowAttribute +XmlRawRows::XmlRowState::resolveAttr(const Glib::ustring & n) const +{ +	if (e->get_attribute(n)) { +		return boost::bind(&XmlRowState::getAttr, this, n); +	} +	return RowState::resolveAttr(n); +} + +VariableType +XmlRawRows::XmlRowState::getAttr(const Glib::ustring & n) const +{ +	if (const xmlpp::Attribute * a = e->get_attribute(n)) { +		return a->get_value(); +	} +	throw AttributeDoesNotExist(n); +} + +XmlRawRows::XmlRawRows(const xmlpp::Element * p) : +	RowSet(p), +	path(p, "path") +{ +} + +XmlRawRows::XmlRawRows(const Glib::ustring & p) : +	RowSet(NULL), +	path(p) +{ +} + +void XmlRawRows::loadComplete(const CommonObjects*) +{ +} + +void XmlRawRows::execute(const Glib::ustring&, const RowProcessor * rp) const +{ +	xmlpp::DomParser x(path()); +	XmlRowState rs; +	BOOST_FOREACH(const xmlpp::Node * n, x.get_document()->get_root_node()->get_children()) { +		if ((rs.e = dynamic_cast<const xmlpp::Element *>(n))) { +			if (rs.cols.empty()) { +				unsigned int col = 0; +				BOOST_FOREACH(const xmlpp::Node * in, rs.e->get_children()) { +					if (const xmlpp::Element * ie = dynamic_cast<const xmlpp::Element *>(in)) { +						rs.cols.insert(new Column(col++, ie->get_name())); +					} +				} +				rs.fields.resize(col); +			} +			unsigned int col = 0; +			BOOST_FOREACH(const xmlpp::Node * in, rs.e->get_children()) { +				if (const xmlpp::Element * ie = dynamic_cast<const xmlpp::Element *>(in)) { +					const xmlpp::TextNode * t = ie->get_child_text(); +					if (t) { +						rs.fields[col] = t->get_content(); +					} +					col++; +				} +			} +			rs.process(rp); +		} +	} +} + diff --git a/project2/xmlRawRows.h b/project2/xmlRawRows.h new file mode 100644 index 0000000..7a4fd0b --- /dev/null +++ b/project2/xmlRawRows.h @@ -0,0 +1,25 @@ +#include "rowSet.h" + +class XmlRawRows : public RowSet { +	public: +		class XmlRowState : public RowState { +			public: +				const Columns& getColumns() const; +				void foreachAttr(const AttrAction & action) const; +				RowState::RowAttribute resolveAttr(const Glib::ustring & n) const; +				VariableType getAttr(const Glib::ustring & n) const; + +				Columns cols; +				const xmlpp::Element * e; +		}; + +		XmlRawRows(const xmlpp::Element * p); +		XmlRawRows(const Glib::ustring & p); + +		void loadComplete(const CommonObjects*); +		void execute(const Glib::ustring&, const RowProcessor * rp) const; + +	private: +		const Variable path; +}; + | 
