From 91827112c5c2c1e3a8ebb6bbaa9072cd30dcec51 Mon Sep 17 00:00:00 2001 From: randomdan Date: Mon, 22 Aug 2011 00:30:29 +0000 Subject: Add a rowSet for XML files Add basic rowSet caching Add rowSet caching support for storage in XML files --- project2/Jamfile.jam | 18 +++--- project2/cache.cpp | 26 ++++++++ project2/cache.h | 27 ++++++++ project2/environment.cpp | 2 +- project2/fsRows.cpp | 14 +++++ project2/fsRows.h | 1 + project2/iHaveParameters.cpp | 6 ++ project2/iHaveParameters.h | 6 +- project2/iterate.cpp | 4 +- project2/rowProcessor.cpp | 38 +++++++++++- project2/rowProcessor.h | 11 ++++ project2/rowSet.cpp | 8 ++- project2/rowSet.h | 2 + project2/rowView.cpp | 4 +- project2/sourceObject.cpp | 2 +- project2/xmlCache.cpp | 143 +++++++++++++++++++++++++++++++++++++++++++ project2/xmlRawRows.cpp | 86 ++++++++++++++++++++++++++ project2/xmlRawRows.h | 25 ++++++++ 18 files changed, 403 insertions(+), 20 deletions(-) create mode 100644 project2/cache.cpp create mode 100644 project2/cache.h create mode 100644 project2/xmlCache.cpp create mode 100644 project2/xmlRawRows.cpp create mode 100644 project2/xmlRawRows.h diff --git a/project2/Jamfile.jam b/project2/Jamfile.jam index aed9e3d..38c4399 100644 --- a/project2/Jamfile.jam +++ b/project2/Jamfile.jam @@ -55,11 +55,11 @@ lib p2uuid : lib p2common : appEngine.cpp dataSource.cpp environment.cpp fileStarGlibIoChannel.cpp iHaveParameters.cpp library.cpp iHaveSubTasks.cpp - iterate.cpp paramChecker.cpp presenter.cpp rawView.cpp logger.cpp if.cpp xmlScriptParser.cpp viewHost.cpp - sourceObject.cpp task.cpp variables.cpp variableConvert.cpp view.cpp xmlObjectLoader.cpp exceptions.cpp - sessionClearTask.cpp session.cpp sessionSetTask.cpp commonObjects.cpp xmlPresenter.cpp taskHost.cpp checkHost.cpp + iterate.cpp paramChecker.cpp presenter.cpp logger.cpp if.cpp xmlScriptParser.cpp viewHost.cpp + sourceObject.cpp task.cpp variables.cpp variableConvert.cpp view.cpp xmlObjectLoader.cpp exceptions.cpp cache.cpp + sessionContainer.cpp sessionClearTask.cpp session.cpp sessionSetTask.cpp commonObjects.cpp taskHost.cpp checkHost.cpp rowView.cpp rowSet.cpp rowProcessor.cpp config.cpp fileStrmVarWriter.cpp noOutputExecute.cpp columns.cpp scopeObject.cpp - transform.cpp transformHtml.cpp transformText.cpp definedColumns.cpp structExceptHandling.cpp validDateCheck.cpp + transform.cpp definedColumns.cpp structExceptHandling.cpp validDateCheck.cpp variables-modconfig.cpp variables-modlocalparam.cpp variables-modlookup.cpp @@ -76,7 +76,6 @@ lib p2common : boost_filesystem boost_date_time boost_program_options - libxslt : : . ../libmisc @@ -85,13 +84,15 @@ lib p2common : ; lib p2xml : - xmlRows.cpp xslRows.cpp xslRowsCache.cpp xslPreFetch.cpp + rawView.cpp xmlPresenter.cpp transformHtml.cpp transformText.cpp xmlRows.cpp + xmlRawRows.cpp xslRows.cpp xslRowsCache.cpp xslPreFetch.cpp xmlCache.cpp : ../libmisc libxmlpp p2common p2url libxslt + boost_filesystem ; lib p2processes : @@ -124,7 +125,7 @@ lib p2regex : ; lib p2xmlSession : - sessionContainer.cpp sessionXml.cpp + sessionXml.cpp : ../libmisc libxmlpp @@ -185,6 +186,7 @@ lib p2mail : libxslt esmtp p2common + p2xml ; lib p2web : @@ -198,7 +200,7 @@ lib p2web : p2uuid boost_program_options boost_filesystem - p2xmlSession + p2xml : : p2parts p2xmlSession diff --git a/project2/cache.cpp b/project2/cache.cpp new file mode 100644 index 0000000..0db108b --- /dev/null +++ b/project2/cache.cpp @@ -0,0 +1,26 @@ +#include "cache.h" +#include "rowSet.h" +#include "rowProcessor.h" +#include "logger.h" + +Cache::Cache(const xmlpp::Element * p) : + SourceObject(p) +{ +} + +bool Cache::checkAndExecute(const Glib::ustring & n, const Glib::ustring & f, const RowProcessor * rp) +{ + RowSetCPtr cached = getCachedRowSet(n, f, rp); + if (cached) { + try { + Logger()->messagef(LOG_ERR, "Executing from cache"); + cached->execute(f, rp); + return true; + } + catch (...) { + Logger()->messagef(LOG_WARNING, "Cache failed"); + } + } + return false; +} + diff --git a/project2/cache.h b/project2/cache.h new file mode 100644 index 0000000..358d933 --- /dev/null +++ b/project2/cache.h @@ -0,0 +1,27 @@ +#ifndef CACHE_H +#define CACHE_H + +#include "sourceObject.h" +#include "presenter.h" + +class RowProcessor; +class IHaveParameters; +class RowSet; +class RowState; +typedef boost::intrusive_ptr RowSetCPtr; + +class Cache : public SourceObject { + public: + Cache(const xmlpp::Element * p); + + bool checkAndExecute(const Glib::ustring &, const Glib::ustring &, const RowProcessor *); + virtual PresenterPtr openFor(const Glib::ustring &, const Glib::ustring &, const IHaveParameters *) = 0; + virtual void close() = 0; + + protected: + virtual RowSetCPtr getCachedRowSet(const Glib::ustring &, const Glib::ustring &, const IHaveParameters *) const = 0; +}; +typedef boost::intrusive_ptr CachePtr; + +#endif + diff --git a/project2/environment.cpp b/project2/environment.cpp index 12591bf..0c563cc 100644 --- a/project2/environment.cpp +++ b/project2/environment.cpp @@ -75,6 +75,7 @@ Environment::init() LoaderBase::onAllComponents(boost::bind( injectSettingsHelper, argc, argv, boost::bind(&ComponentLoader::options, _1), (po::positional_options_description*)NULL)); + Logger()->clear(); if (clLevel >= 0) { Logger()->addLogger(new ConsoleLogDriver(stderr, clLevel, false)); } @@ -86,7 +87,6 @@ Environment::init() Environment::~Environment() { currentEnv = NULL; - Logger()->clear(); } const Environment * diff --git a/project2/fsRows.cpp b/project2/fsRows.cpp index b308f38..cb73b04 100644 --- a/project2/fsRows.cpp +++ b/project2/fsRows.cpp @@ -244,6 +244,20 @@ FsRows::SearchState::resolveAttr(const Glib::ustring & a) const return RowState::resolveAttr(a); } +void +FsRows::SearchState::foreachAttr(const AttrAction & action) const +{ + action(field_relPath, fileRelPath()); + action(field_size, fileSize()); + action(field_modDate, fileModDate()); + action(field_user, fileUser()); + action(field_group, fileGroup()); + action(field_mode, fileMode()); + action(field_perms, filePerms()); + action(field_type, fileType()); + RowState::foreachAttr(action); +} + VariableType FsRows::SearchState::fileRelPath() const { diff --git a/project2/fsRows.h b/project2/fsRows.h index 8cd1a9f..3356b37 100644 --- a/project2/fsRows.h +++ b/project2/fsRows.h @@ -38,6 +38,7 @@ class FsRows : public RowSet { SearchState(const boost::filesystem::path & r); virtual RowAttribute resolveAttr(const Glib::ustring & attrName) const; + virtual void foreachAttr(const AttrAction & action) const; virtual const Columns & getColumns() const; VariableType fileRelPath() const; diff --git a/project2/iHaveParameters.cpp b/project2/iHaveParameters.cpp index d47fc42..e254a4f 100644 --- a/project2/iHaveParameters.cpp +++ b/project2/iHaveParameters.cpp @@ -61,3 +61,9 @@ IHaveParameters::getScopedParameter(const Glib::ustring & name) throw ParamNotFound(name); } +const IHaveParameters::Parameters & +IHaveParameters::allParameters() const +{ + return parameters; +} + diff --git a/project2/iHaveParameters.h b/project2/iHaveParameters.h index c6dcd45..51378ee 100644 --- a/project2/iHaveParameters.h +++ b/project2/iHaveParameters.h @@ -16,14 +16,16 @@ class IHaveParameters { const Glib::ustring name; const Variable value; }; + typedef boost::intrusive_ptr ParameterPtr; + typedef std::map Parameters; + IHaveParameters(const xmlpp::Element * p); virtual ~IHaveParameters() = 0; + const Parameters & allParameters() const; VariableType getParameter(const Glib::ustring &) const; static VariableType getScopedParameter(const Glib::ustring &); - typedef boost::intrusive_ptr ParameterPtr; - typedef std::map Parameters; protected: Parameters parameters; diff --git a/project2/iterate.cpp b/project2/iterate.cpp index dc59757..119e839 100644 --- a/project2/iterate.cpp +++ b/project2/iterate.cpp @@ -36,9 +36,7 @@ Iterate::rowReady(const RowState *) const void Iterate::execute() const { - IHaveParameters::push(this); - ScopeObject _ihp(boost::bind(&IHaveParameters::pop, this)); - source->execute(filter, this); + RowProcessor::execute(); } void diff --git a/project2/rowProcessor.cpp b/project2/rowProcessor.cpp index caff826..17f83f4 100644 --- a/project2/rowProcessor.cpp +++ b/project2/rowProcessor.cpp @@ -1,6 +1,7 @@ #include "rowProcessor.h" +#include "logger.h" #include "commonObjects.h" -#include +#include "scopeObject.h" #include RowProcessor::RowProcessor(const xmlpp::Element * p) : @@ -8,6 +9,9 @@ RowProcessor::RowProcessor(const xmlpp::Element * p) : recordSource(p->get_attribute_value("source")), filter(p->get_attribute_value("filter")) { + LoaderBase loader(true); + loader.supportedStorers.insert(Storer::into(&caches)); + loader.collectAll(p, true, IgnoreUnsupported); } void @@ -16,3 +20,35 @@ RowProcessor::loadComplete(const CommonObjects * co) source = co->getSource(recordSource); } +void +RowProcessor::execute() const +{ + IHaveParameters::push(this); + ScopeObject _ihp(boost::bind(&IHaveParameters::pop, this)); + BOOST_FOREACH(const CachePtr & c, caches) { + if (c->checkAndExecute(source->name, filter, this)) { + return; + } + } + BOOST_FOREACH(const CachePtr & c, caches) { + tc.insert(c->openFor(source->name, filter, this)); + } + source->execute(filter, this); + tc.clear(); + BOOST_FOREACH(const CachePtr & c, caches) { + c->close(); + } +} + +void +RowProcessor::rowReadyInternal(const RowState * rs) const +{ + BOOST_FOREACH(const TargetCaches::value_type & c, tc) { + c->pushSub(filter.empty() ? "row" : filter); + rs->foreachColumn(boost::bind(&Presenter::addField, c, _2, _3)); + rs->foreachAttr(boost::bind(&Presenter::addAttr, c, _1, _2)); + c->popSub(); + } + rowReady(rs); +} + diff --git a/project2/rowProcessor.h b/project2/rowProcessor.h index 38142e5..c25dc73 100644 --- a/project2/rowProcessor.h +++ b/project2/rowProcessor.h @@ -5,6 +5,10 @@ #include "sourceObject.h" #include "iHaveParameters.h" #include "rowSet.h" +#include "cache.h" +#include "xmlStorage.h" + +class Presenter; /// Base class for Project2 components that work with row sets class RowProcessor : public IHaveParameters { @@ -17,9 +21,16 @@ class RowProcessor : public IHaveParameters { protected: boost::intrusive_ptr source; + void execute() const; + private: friend class RowState; + void rowReadyInternal(const RowState *) const; virtual void rowReady(const RowState *) const = 0; + typedef ANONORDEREDSTORAGEOF(Cache) Caches; + Caches caches; + typedef std::set TargetCaches; + mutable TargetCaches tc; }; #endif diff --git a/project2/rowSet.cpp b/project2/rowSet.cpp index acc9246..6902fd3 100644 --- a/project2/rowSet.cpp +++ b/project2/rowSet.cpp @@ -33,7 +33,7 @@ RowState::process(const RowProcessor * rp, bool r) rowNum += 1; stack.push_back(this); ScopeObject s(boost::bind(&RowState::RowValuesStack::pop_back, &stack)); - rp->rowReady(this); + rp->rowReadyInternal(this); if (r) { reset(); } @@ -91,3 +91,9 @@ RowState::foreachColumn(const ColumnAction & action) const } } +void +RowState::foreachAttr(const AttrAction &) const +{ + // rowNum is magic, so it doesn't count :) +} + diff --git a/project2/rowSet.h b/project2/rowSet.h index a9919f9..a181059 100644 --- a/project2/rowSet.h +++ b/project2/rowSet.h @@ -36,6 +36,7 @@ class RowState { typedef boost::function0 RowAttribute; typedef boost::function3 ColumnAction; + typedef boost::function2 AttrAction; SimpleMessageException(AttributeDoesNotExist); VariableType getRowNum() const; @@ -45,6 +46,7 @@ class RowState { virtual VariableType getCurrentValue(const Glib::ustring & id) const; virtual RowAttribute resolveAttr(const Glib::ustring & attrName) const; void foreachColumn(const ColumnAction & action) const; + virtual void foreachAttr(const AttrAction & action) const; virtual const Columns & getColumns() const = 0; typedef std::vector FieldValues; diff --git a/project2/rowView.cpp b/project2/rowView.cpp index 51472c5..092192f 100644 --- a/project2/rowView.cpp +++ b/project2/rowView.cpp @@ -59,10 +59,8 @@ RowView::execute(const Presenter * p) const { presenter = p; presenter->pushSub(rootName); - ScopeObject _ihp(boost::bind(&IHaveParameters::pop, this)); - IHaveParameters::push(this); ScopeObject pres(boost::bind(&Presenter::popSub, p)); - source->execute(filter, this); + RowProcessor::execute(); } void diff --git a/project2/sourceObject.cpp b/project2/sourceObject.cpp index b2bcf12..a5736ba 100644 --- a/project2/sourceObject.cpp +++ b/project2/sourceObject.cpp @@ -3,7 +3,7 @@ unsigned int SourceObject::loadOrder = 1; SourceObject::SourceObject(const xmlpp::Element * p) : - name(p->get_attribute_value("name")), + name(p ? p->get_attribute_value("name") : "anon"), order(loadOrder++) { } diff --git a/project2/xmlCache.cpp b/project2/xmlCache.cpp new file mode 100644 index 0000000..0a166e8 --- /dev/null +++ b/project2/xmlCache.cpp @@ -0,0 +1,143 @@ +#include "cache.h" +#include "logger.h" +#include "xmlObjectLoader.h" +#include "iHaveParameters.h" +#include "xmlRawRows.h" +#include "xmlPresenter.h" +#include +#include +#include +#include +#include +#include +#include + +class XmlCache : public Cache { + public: + XmlCache(const xmlpp::Element * p) : + Cache(p) + { + } + + void loadComplete(const CommonObjects*) + { + } + + RowSetCPtr getCachedRowSet(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps) const + { + boost::filesystem::path cache = getCacheFile(n, f, ps); + struct stat st; + if (stat(cache.string().c_str(), &st) == 0) { + if (st.st_mtime < time(NULL) - CacheLife) { + boost::filesystem::remove(cache); + return NULL; + } + return new XmlRawRows(cache.string()); + } + return NULL; + } + + PresenterPtr openFor(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps) + { + target = getCacheFile(n, f, ps); + writeTo = new XmlPresenter(n, Glib::ustring(), Glib::ustring()); + return writeTo; + } + + void close() + { + try { + boost::filesystem::create_directories(target.parent_path()); + const xmlpp::Document * d = *writeTo; + const_cast(d)->write_to_file(target.string()); + writeTo.reset(); + } + catch (...) { + Logger()->messagef(LOG_WARNING, "Failed to save cache (%s)", target.string().c_str()); + } + } + + private: + boost::filesystem::path getCacheFile(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps) const + { + boost::filesystem::path cache = Store / n.raw() / f.raw(); + BOOST_FOREACH(const IHaveParameters::Parameters::value_type & p, ps->allParameters()) { + std::string v = p.second->value(); + cache /= p.first.raw(); + cache /= v; + } + cache /= FileName; + return cache; + } + + XmlPresenterPtr writeTo; + boost::filesystem::path target; + + friend class CustomXmlCacheLoader; + static boost::filesystem::path Store; + static std::string FileName; + static time_t CacheLife; +}; + +boost::filesystem::path XmlCache::Store; +std::string XmlCache::FileName; +time_t XmlCache::CacheLife; + +namespace po = boost::program_options; +class CustomXmlCacheLoader : public ElementLoaderImpl { + public: + CustomXmlCacheLoader() : + opts("XML Cache options") + { + opts.add_options() + ("cache.xml.store", po::value(&XmlCache::Store)->default_value("/tmp/project2.cache"), + "The root folder of the cache storage area") + ("cache.xml.filename", po::value(&XmlCache::FileName)->default_value("cache.xml"), + "The filename to store the data in") + ("cache.xml.life", po::value(&XmlCache::CacheLife)->default_value(3600), + "The age of cache entries after which they are removed (seconds)") + ; + } + + po::options_description * options() + { + return &opts; + } + + void onIdle() + { + emptyDir(XmlCache::Store); + } + + bool emptyDir(const boost::filesystem::path & dir) + { + bool files = false; + boost::filesystem::directory_iterator end; + for (boost::filesystem::directory_iterator itr(dir); itr != end; ++itr) { + struct stat st; + stat(itr->path().string().c_str(), &st); + if (S_ISDIR(st.st_mode)) { + if (emptyDir(*itr)) { + files = true; + } + else { + boost::filesystem::remove(*itr); + } + } + else { + if (st.st_mtime > time(NULL) - XmlCache::CacheLife) { + files = true; + } + else { + boost::filesystem::remove(*itr); + } + } + } + return files; + } + + private: + po::options_description opts; +}; +DECLARE_CUSTOM_LOADER("xmlcache", CustomXmlCacheLoader); + diff --git a/project2/xmlRawRows.cpp b/project2/xmlRawRows.cpp new file mode 100644 index 0000000..76a87e2 --- /dev/null +++ b/project2/xmlRawRows.cpp @@ -0,0 +1,86 @@ +#include "xmlRawRows.h" +#include +#include +#include + +DECLARE_LOADER("xmlrawrows", XmlRawRows); + +const Columns & +XmlRawRows::XmlRowState::getColumns() const +{ + return cols; +} + +void +XmlRawRows::XmlRowState::foreachAttr(const AttrAction & action) const +{ + BOOST_FOREACH(const xmlpp::Attribute * a, e->get_attributes()) { + action(a->get_name(), a->get_value()); + } + RowState::foreachAttr(action); +} + +RowState::RowAttribute +XmlRawRows::XmlRowState::resolveAttr(const Glib::ustring & n) const +{ + if (e->get_attribute(n)) { + return boost::bind(&XmlRowState::getAttr, this, n); + } + return RowState::resolveAttr(n); +} + +VariableType +XmlRawRows::XmlRowState::getAttr(const Glib::ustring & n) const +{ + if (const xmlpp::Attribute * a = e->get_attribute(n)) { + return a->get_value(); + } + throw AttributeDoesNotExist(n); +} + +XmlRawRows::XmlRawRows(const xmlpp::Element * p) : + RowSet(p), + path(p, "path") +{ +} + +XmlRawRows::XmlRawRows(const Glib::ustring & p) : + RowSet(NULL), + path(p) +{ +} + +void XmlRawRows::loadComplete(const CommonObjects*) +{ +} + +void XmlRawRows::execute(const Glib::ustring&, const RowProcessor * rp) const +{ + xmlpp::DomParser x(path()); + XmlRowState rs; + BOOST_FOREACH(const xmlpp::Node * n, x.get_document()->get_root_node()->get_children()) { + if ((rs.e = dynamic_cast(n))) { + if (rs.cols.empty()) { + unsigned int col = 0; + BOOST_FOREACH(const xmlpp::Node * in, rs.e->get_children()) { + if (const xmlpp::Element * ie = dynamic_cast(in)) { + rs.cols.insert(new Column(col++, ie->get_name())); + } + } + rs.fields.resize(col); + } + unsigned int col = 0; + BOOST_FOREACH(const xmlpp::Node * in, rs.e->get_children()) { + if (const xmlpp::Element * ie = dynamic_cast(in)) { + const xmlpp::TextNode * t = ie->get_child_text(); + if (t) { + rs.fields[col] = t->get_content(); + } + col++; + } + } + rs.process(rp); + } + } +} + diff --git a/project2/xmlRawRows.h b/project2/xmlRawRows.h new file mode 100644 index 0000000..7a4fd0b --- /dev/null +++ b/project2/xmlRawRows.h @@ -0,0 +1,25 @@ +#include "rowSet.h" + +class XmlRawRows : public RowSet { + public: + class XmlRowState : public RowState { + public: + const Columns& getColumns() const; + void foreachAttr(const AttrAction & action) const; + RowState::RowAttribute resolveAttr(const Glib::ustring & n) const; + VariableType getAttr(const Glib::ustring & n) const; + + Columns cols; + const xmlpp::Element * e; + }; + + XmlRawRows(const xmlpp::Element * p); + XmlRawRows(const Glib::ustring & p); + + void loadComplete(const CommonObjects*); + void execute(const Glib::ustring&, const RowProcessor * rp) const; + + private: + const Variable path; +}; + -- cgit v1.2.3