From 9517a97e7c1821a30e93c63ab11fbe023a794f04 Mon Sep 17 00:00:00 2001 From: randomdan Date: Tue, 25 Oct 2011 19:41:26 +0000 Subject: Finally sort the stupid names problem in the XML lib and remove xmlMemCache for flawed from day one --- project2/xml/Jamfile.jam | 2 +- project2/xml/xmlDocumentCache.cpp | 107 +++++++++++++++++++++++ project2/xml/xmlDocumentCache.h | 38 +++++++++ project2/xml/xmlDocumentPrefetch.cpp | 57 +++++++++++++ project2/xml/xmlDocumentPrefetch.h | 30 +++++++ project2/xml/xmlMemCache.cpp | 121 -------------------------- project2/xml/xpathRows.cpp | 160 +++++++++++++++++++++++++++++++++++ project2/xml/xpathRows.h | 64 ++++++++++++++ project2/xml/xslPreFetch.cpp | 57 ------------- project2/xml/xslPreFetch.h | 30 ------- project2/xml/xslRows.cpp | 160 ----------------------------------- project2/xml/xslRows.h | 64 -------------- project2/xml/xslRowsCache.cpp | 107 ----------------------- project2/xml/xslRowsCache.h | 38 --------- 14 files changed, 457 insertions(+), 578 deletions(-) create mode 100644 project2/xml/xmlDocumentCache.cpp create mode 100644 project2/xml/xmlDocumentCache.h create mode 100644 project2/xml/xmlDocumentPrefetch.cpp create mode 100644 project2/xml/xmlDocumentPrefetch.h delete mode 100644 project2/xml/xmlMemCache.cpp create mode 100644 project2/xml/xpathRows.cpp create mode 100644 project2/xml/xpathRows.h delete mode 100644 project2/xml/xslPreFetch.cpp delete mode 100644 project2/xml/xslPreFetch.h delete mode 100644 project2/xml/xslRows.cpp delete mode 100644 project2/xml/xslRows.h delete mode 100644 project2/xml/xslRowsCache.cpp delete mode 100644 project2/xml/xslRowsCache.h diff --git a/project2/xml/Jamfile.jam b/project2/xml/Jamfile.jam index 8a4a5cc..6078290 100644 --- a/project2/xml/Jamfile.jam +++ b/project2/xml/Jamfile.jam @@ -15,7 +15,7 @@ cpp-pch pch : pch.hpp : lib p2xml : pch rawView.cpp xmlPresenter.cpp transformXml.cpp transformHtml.cpp transformText.cpp xmlRows.cpp - xmlRawRows.cpp xslRows.cpp xslRowsCache.cpp xslPreFetch.cpp xmlMemCache.cpp xmlCache.cpp sessionXml.cpp + xmlRawRows.cpp xpathRows.cpp xmlDocumentCache.cpp xmlDocumentPrefetch.cpp xmlCache.cpp sessionXml.cpp : ../libmisc libxmlpp diff --git a/project2/xml/xmlDocumentCache.cpp b/project2/xml/xmlDocumentCache.cpp new file mode 100644 index 0000000..9941336 --- /dev/null +++ b/project2/xml/xmlDocumentCache.cpp @@ -0,0 +1,107 @@ +#include +#include +#include "xmlDocumentCache.h" +#include +#include +#include "exceptions.h" +#include "curlHelper.h" +#include "safeMapFind.h" + +XmlDocumentCache::Documents XmlDocumentCache::documents; +XmlDocumentCache::Queued XmlDocumentCache::queued; +CurlBulkFetcher XmlDocumentCache::cbf; + +SimpleMessageException(XmlParseError); +SimpleMessageException(DownloadFailed); + +template +static XmlDocumentCache::DocumentPtr helperThrow(const std::string & msg) { + throw Exception(msg); +} +static XmlDocumentCache::DocumentPtr helperReturnDocument(XmlDocumentCache::DocumentPtr dp) { + return dp; +} + +class XmlDocumentCachePopulator : public CurlCompleteCallback { + public: + XmlDocumentCachePopulator(CurlPtr ch, const Glib::ustring & u, bool h, bool w, const char * e) : + CurlCompleteCallback(ch), + handler(boost::bind(&XmlDocumentCachePopulator::append, this, _1, _2)), + url(u), + html(h), + warnings(w), + encoding(e ? strdup(e) : NULL) + { + ch->setReadHandler(handler); + } + ~XmlDocumentCachePopulator() + { + free(encoding); + } + void call(CurlBulkFetcher *) + { + int flags = 0; + flags |= warnings ? 0 : XML_PARSE_NOWARNING | XML_PARSE_NOERROR; + xmlDocPtr doc = html ? + htmlReadMemory(buf.c_str(), buf.length(), url.c_str(), encoding, flags) : + xmlReadMemory(buf.c_str(), buf.length(), url.c_str(), encoding, flags); + if (!doc) { + Logger()->messagef(LOG_DEBUG, "Download of '%s' succeeded, but parsing failed with error '%s'", url.c_str(), xmlGetLastError()->message); + XmlDocumentCache::documents.insert(XmlDocumentCache::Documents::value_type(url, + boost::bind(helperThrow, std::string(xmlGetLastError()->message)))); + } + XmlDocumentCache::documents.insert(XmlDocumentCache::Documents::value_type(url, + boost::bind(helperReturnDocument, XmlDocumentCache::DocumentPtr(doc, xmlFreeDoc)))); + Logger()->messagef(LOG_DEBUG, "Download of '%s' completed, stored", url.c_str()); + } + void error(CurlBulkFetcher *, const char * error) + { + Logger()->messagef(LOG_DEBUG, "Download of '%s' failed with error '%s'", url.c_str(), error); + XmlDocumentCache::documents.insert(XmlDocumentCache::Documents::value_type(url, + boost::bind(helperThrow, std::string(error)))); + } + size_t append(const char * c, size_t b) + { + buf.append(c, b); + return b; + } + + Curl::ReadHandler handler; + std::string buf; + const Glib::ustring url; + const bool html; + const bool warnings; + char * encoding; +}; + +xmlDocPtr +XmlDocumentCache::getDocument(const Glib::ustring & url, const char * encoding) const +{ + Documents::const_iterator i = documents.find(url); + if (i == documents.end()) { + queue(url, encoding); + cbf.perform(); + queued.clear(); + } + return safeMapFind(documents, url)->second().get(); +} + +void +XmlDocumentCache::queue(const Glib::ustring & url, const char * encoding) const +{ + if (queued.find(url) == queued.end()) { + cbf.curls.insert(new XmlDocumentCachePopulator(newCurl(), url, asHtml(), withWarnings(), encoding)); + queued.insert(url); + } +} + +class XmlDocumentCacheClearer : public ComponentLoader { + public: + void onIteration() + { + Logger()->messagef(LOG_DEBUG, "%s: Clearing XML document cache", __PRETTY_FUNCTION__); + XmlDocumentCache::documents.clear(); + } +}; +DECLARE_CUSTOM_COMPONENT_LOADER("XmlDocumentCacheClearer", XmlDocumentCacheClearer, XmlDocumentCacheClearer, ComponentLoader); + diff --git a/project2/xml/xmlDocumentCache.h b/project2/xml/xmlDocumentCache.h new file mode 100644 index 0000000..190c989 --- /dev/null +++ b/project2/xml/xmlDocumentCache.h @@ -0,0 +1,38 @@ +#ifndef XMLDOCUMENTROWSCACHE_H +#define XMLDOCUMENTROWSCACHE_H + +#include +#include +#include +#include +#include +#include + +class XmlDocumentCache { + public: + typedef std::set Queued; + typedef boost::shared_ptr DocumentPtr; + typedef boost::function0 ReturnDocument; + typedef std::map Documents; + + protected: + static Queued queued; + static Documents documents; + + void queue(const Glib::ustring & url, const char * encoding) const; + + virtual CurlPtr newCurl() const = 0; + virtual bool asHtml() const = 0; + virtual bool withWarnings() const = 0; + + protected: + xmlDocPtr getDocument(const Glib::ustring & url, const char * encoding) const; + + private: + static CurlBulkFetcher cbf; + + friend class XmlDocumentCachePopulator; + friend class XmlDocumentCacheClearer; +}; + +#endif diff --git a/project2/xml/xmlDocumentPrefetch.cpp b/project2/xml/xmlDocumentPrefetch.cpp new file mode 100644 index 0000000..f2ff4d0 --- /dev/null +++ b/project2/xml/xmlDocumentPrefetch.cpp @@ -0,0 +1,57 @@ +#include +#include "xmlDocumentPrefetch.h" +#include "xmlObjectLoader.h" + +DECLARE_LOADER("xmldocumentprefetch", XmlDocumentPrefetch); + +XmlDocumentPrefetch::XmlDocumentPrefetch(const xmlpp::Element * p) : + SourceObject(p), + View(p), + Task(p), + VariableCurlHelper(p), + html(p->get_attribute_value("html") == "true"), + warnings(p->get_attribute_value("warnings") != "false"), + encoding(p, "encoding", false) +{ +} + +XmlDocumentPrefetch::~XmlDocumentPrefetch() +{ +} + +void +XmlDocumentPrefetch::execute(const Presenter*) const +{ + execute(); +} + +void +XmlDocumentPrefetch::execute() const +{ + queue(url(), encoding()); +} + +void +XmlDocumentPrefetch::loadComplete(const CommonObjects *) +{ +} + + +CurlPtr +XmlDocumentPrefetch::newCurl() const +{ + return VariableCurlHelper::newCurl(); +} + +bool +XmlDocumentPrefetch::asHtml() const +{ + return html; +} + +bool +XmlDocumentPrefetch::withWarnings() const +{ + return warnings; +} + diff --git a/project2/xml/xmlDocumentPrefetch.h b/project2/xml/xmlDocumentPrefetch.h new file mode 100644 index 0000000..e6ccbec --- /dev/null +++ b/project2/xml/xmlDocumentPrefetch.h @@ -0,0 +1,30 @@ +#ifndef XMLDOCUMENTPREFETCH_H +#define XMLDOCUMENTPREFETCH_H + +#include "xmlDocumentCache.h" +#include "curlHelper.h" +#include "view.h" +#include "task.h" +#include + +/// Project2 component to queue up CURL objects to be downloaded +class XmlDocumentPrefetch : public View, public Task, XmlDocumentCache, VariableCurlHelper { + public: + XmlDocumentPrefetch(const xmlpp::Element * p); + ~XmlDocumentPrefetch(); + + void execute(const Presenter*) const; + void execute() const; + void loadComplete(const CommonObjects *); + + const bool html; + const bool warnings; + const Variable encoding; + + CurlPtr newCurl() const; + bool asHtml() const; + bool withWarnings() const; +}; + +#endif + diff --git a/project2/xml/xmlMemCache.cpp b/project2/xml/xmlMemCache.cpp deleted file mode 100644 index 9fe0a6f..0000000 --- a/project2/xml/xmlMemCache.cpp +++ /dev/null @@ -1,121 +0,0 @@ -#include -#include "cache.h" -#include "logger.h" -#include "xmlObjectLoader.h" -#include "iHaveParameters.h" -#include "xmlRawRows.h" -#include "xmlPresenter.h" -#include -#include -#include -#include -#include -#include -#include - -class XmlMemCache : public Cache { - public: - class CacheEntry : public IntrusivePtrBase { - public: - std::vector key; - time_t createdAt; - boost::shared_ptr doc; - }; - typedef boost::intrusive_ptr CacheEntryPtr; - typedef boost::intrusive_ptr CacheEntryCPtr; - - struct IndexByKey { }; - struct IndexByTime { }; - typedef boost::multi_index::multi_index_container< - boost::intrusive_ptr, - boost::multi_index::indexed_by< - boost::multi_index::ordered_unique< - boost::multi_index::tag, BOOST_MULTI_INDEX_MEMBER(CacheEntry, const std::vector, key)>, - boost::multi_index::ordered_non_unique< - boost::multi_index::tag, BOOST_MULTI_INDEX_MEMBER(CacheEntry, const time_t, createdAt)> - > > CacheStore; - - XmlMemCache(const xmlpp::Element * p) : - Cache(p) - { - } - - void loadComplete(const CommonObjects*) - { - } - - RowSetCPtr getCachedRowSet(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps) const - { - std::vector key; - key.push_back(n); - key.push_back(f); - applyKeys(boost::bind(&std::vector::push_back, &key, _2), ps); - CacheStore::index::type::const_iterator i = Store.get().find(key); - if (i == Store.get().end()) { - return NULL; - } - if ((*i)->createdAt < (time(NULL) - CacheLife)) { - Store.erase(i); - return NULL; - } - return new XmlMemRawRows((*i)->doc); - } - - PresenterPtr openFor(const Glib::ustring & n, const Glib::ustring &, const IHaveParameters *) - { - writeTo = new XmlPresenter(n, Glib::ustring(), Glib::ustring()); - return writeTo; - } - - void close(const Glib::ustring & n, const Glib::ustring & f, const IHaveParameters * ps) - { - CacheEntryPtr ce = new CacheEntry(); - ce->key.push_back(n); - ce->key.push_back(f); - applyKeys(boost::bind(&std::vector::push_back, &ce->key, _2), ps); - time(&ce->createdAt); - ce->doc = *(const boost::shared_ptr*)(*writeTo); - Store.insert(ce); - } - - private: - XmlPresenterPtr writeTo; - - friend class CustomXmlMemCacheLoader; - static time_t CacheLife; - static CacheStore Store; -}; - -time_t XmlMemCache::CacheLife; -XmlMemCache::CacheStore XmlMemCache::Store; - -namespace po = boost::program_options; -class CustomXmlMemCacheLoader : public ElementLoaderImpl { - public: - CustomXmlMemCacheLoader() : - opts("XML Memory Cache options") - { - opts.add_options() - ("cache.xmlmem.life", po::value(&XmlMemCache::CacheLife)->default_value(3600), - "The age of cache entries after which they are removed (seconds)") - ; - } - - po::options_description * options() - { - return &opts; - } - - void onIdle() - { - typedef XmlMemCache::CacheStore::index::type::iterator iter; - iter x = XmlMemCache::Store.get().begin(); - iter y = XmlMemCache::Store.get().upper_bound(time(NULL) - XmlMemCache::CacheLife); - XmlMemCache::Store.get().erase(x, y); - } - - private: - po::options_description opts; -}; -DECLARE_CUSTOM_LOADER("xmlmemcache", CustomXmlMemCacheLoader); - diff --git a/project2/xml/xpathRows.cpp b/project2/xml/xpathRows.cpp new file mode 100644 index 0000000..24283dd --- /dev/null +++ b/project2/xml/xpathRows.cpp @@ -0,0 +1,160 @@ +#include +#include "xpathRows.h" +#include "safeMapFind.h" +#include "rowProcessor.h" +#include "logger.h" +#include "xml.h" +#include "exceptions.h" +#include "xmlObjectLoader.h" +#include +#include +#include +#include "../libmisc/curlsup.h" +#include + +DECLARE_LOADER("xpathrows", XPathRows); + +SimpleMessageException(XpathInitError); +SimpleMessageException(XpathEvalError); + +XPathRows::XPathRows(const xmlpp::Element * p) : + RowSet(p), + VariableCurlHelper(p), + html(p->get_attribute_value("html") == "true"), + warnings(p->get_attribute_value("warnings") != "false"), + encoding(p, "encoding", false) +{ + BOOST_FOREACH(const xmlpp::Node * node, p->find("filterview")) { + const xmlpp::Element * elem = dynamic_cast(node); + if (elem) { + FilterViewPtr fv = new FilterView(elem); + fvs[fv->name] = fv; + } + } + BOOST_FOREACH(const xmlpp::Node * node, p->find("namespace")) { + const xmlpp::Element * elem = dynamic_cast(node); + if (elem) { + namespaces[elem->get_attribute_value("prefix")] = elem->get_attribute_value("url"); + } + } +} + +XPathRows::~XPathRows() +{ +} + +void +XPathRows::loadComplete(const CommonObjects *) +{ +} + +bool +XPathRows::asHtml() const +{ + return html; +} + +bool +XPathRows::withWarnings() const +{ + return warnings; +} + +CurlPtr +XPathRows::newCurl() const +{ + return VariableCurlHelper::newCurl(); +} + +void +XPathRows::execute(const Glib::ustring & filter, const RowProcessor * rp) const +{ + FilterViewPtr fv = safeMapFind(fvs, filter)->second; + + typedef boost::shared_ptr xmlXPathObjectSPtr; + typedef boost::shared_ptr xmlXPathContextSPtr; + xmlDocPtr doc = getDocument(url(), encoding()); + xmlXPathContextSPtr xpathCtx = xmlXPathContextSPtr(xmlXPathNewContext(doc), xmlXPathFreeContext); + if (!xpathCtx) { + throw XpathInitError(xmlGetLastError()->message); + } + BOOST_FOREACH(const Namespaces::value_type & ns, namespaces) { + xmlXPathRegisterNs(xpathCtx.get(), BAD_CAST ns.first.c_str(), BAD_CAST ns.second.c_str()); + } + xmlXPathObjectSPtr xpathObj = xmlXPathObjectSPtr(xmlXPathEvalExpression(fv->root(), xpathCtx.get()), xmlXPathFreeObject); + if (!xpathObj || !xpathObj->nodesetval) { + throw XpathEvalError(xmlGetLastError()->message); + } + Logger()->messagef(LOG_INFO, "%d nodes matched %s", xpathObj->nodesetval->nodeNr, (const char *)(fv->root())); + XPathState xs(fv); + for (int row = 0; row < xpathObj->nodesetval->nodeNr; row += 1) { + xmlNodePtr rowRoot = xpathObj->nodesetval->nodeTab[row]; + xpathCtx->node = rowRoot; + BOOST_FOREACH(const Columns::value_type & _xp, fv->columns.get()) { + const FilterViewColumn * xp = static_cast(_xp.get()); + const VariableType & path(xp->path); + if (boost::get(&path)) { + continue; + } + xmlXPathObjectSPtr xpathObjI = xmlXPathObjectSPtr(xmlXPathEvalExpression(path, xpathCtx.get()), xmlXPathFreeObject); + if (!xpathObjI) { + throw XpathEvalError(xmlGetLastError()->message); + } + if (xpathObjI->floatval) { + xs.fields[xp->idx] = xpathObjI->floatval; + } + else if (xpathObjI->stringval) { + xs.fields[xp->idx] = Glib::ustring((const char *)xpathObjI->stringval); + } + else if (xpathObjI->nodesetval) { + Glib::ustring str; + for (int i = 0; i < xpathObjI->nodesetval->nodeNr; i += 1) { + xmlNodePtr n = xpathObjI->nodesetval->nodeTab[i]; + if (n->content) { + str += (const char *)n->content; + } + for (n = n->children; n; n = n->next) { + xmlChar * val = n->content; + if (val) { + str += (const char *)val; + } + } + } + xs.fields[xp->idx] = str; + } + } + xs.process(rp); + } +} + +XPathRows::FilterView::FilterView(const xmlpp::Element * p) : + DefinedColumns(p, "field", boost::bind(XPathRows::FilterViewColumn::make, _1, _2)), + name(p->get_attribute_value("name")), + root(p, "root") +{ +} + +XPathRows::FilterViewColumn::FilterViewColumn(unsigned int idx, const xmlpp::Element * p) : + Column(idx, p), + path(p, "xpath") +{ +} + +XPathRows::FilterViewColumn * +XPathRows::FilterViewColumn::make(unsigned int idx, const xmlpp::Element * p) +{ + return new FilterViewColumn(idx, p); +} + +XPathRows::XPathState::XPathState(FilterViewCPtr f) : + fv(f) +{ + fields.resize(f->columns.size()); +} + +const Columns & +XPathRows::XPathState::getColumns() const +{ + return fv->columns; +} + diff --git a/project2/xml/xpathRows.h b/project2/xml/xpathRows.h new file mode 100644 index 0000000..4b9e008 --- /dev/null +++ b/project2/xml/xpathRows.h @@ -0,0 +1,64 @@ +#ifndef XPATHROWS_H +#define XPATHROWS_H + +#include +#include +#include +#include +#include "rowSet.h" +#include "variables.h" +#include "xmlDocumentCache.h" +#include "curlHelper.h" +#include "definedColumns.h" + +/// Project2 component to create a row set based on the contents of an XML resource and specific XPaths with its hierarchy +class XPathRows : public RowSet, XmlDocumentCache, VariableCurlHelper { + public: + XPathRows(const xmlpp::Element * p); + ~XPathRows(); + + void execute(const Glib::ustring &, const RowProcessor *) const; + virtual void loadComplete(const CommonObjects *); + + const bool html; + const bool warnings; + + private: + class FilterViewColumn : public Column { + public: + FilterViewColumn(unsigned int, const xmlpp::Element *); + static FilterViewColumn * make(unsigned int, const xmlpp::Element *); + const Variable path; + }; + class FilterView : public DefinedColumns, public virtual IntrusivePtrBase { + public: + typedef std::map XPaths; + + FilterView(const xmlpp::Element * p); + + const Glib::ustring name; + const Variable root; + }; + typedef boost::intrusive_ptr FilterViewPtr; + typedef boost::intrusive_ptr FilterViewCPtr; + typedef std::map FilterViews; + FilterViews fvs; + + virtual CurlPtr newCurl() const; + virtual bool asHtml() const; + virtual bool withWarnings() const; + + typedef std::map Namespaces; + Namespaces namespaces; + class XPathState : public RowState { + public: + XPathState(FilterViewCPtr); + const Columns & getColumns() const; + private: + const FilterViewCPtr fv; + }; + const Variable encoding; +}; + +#endif + diff --git a/project2/xml/xslPreFetch.cpp b/project2/xml/xslPreFetch.cpp deleted file mode 100644 index be72fb5..0000000 --- a/project2/xml/xslPreFetch.cpp +++ /dev/null @@ -1,57 +0,0 @@ -#include -#include "xslPreFetch.h" -#include "xmlObjectLoader.h" - -DECLARE_LOADER("xslprefetch", XslPreFetch); - -XslPreFetch::XslPreFetch(const xmlpp::Element * p) : - SourceObject(p), - View(p), - Task(p), - VariableCurlHelper(p), - html(p->get_attribute_value("html") == "true"), - warnings(p->get_attribute_value("warnings") != "false"), - encoding(p, "encoding", false) -{ -} - -XslPreFetch::~XslPreFetch() -{ -} - -void -XslPreFetch::execute(const Presenter*) const -{ - execute(); -} - -void -XslPreFetch::execute() const -{ - queue(url(), encoding()); -} - -void -XslPreFetch::loadComplete(const CommonObjects *) -{ -} - - -CurlPtr -XslPreFetch::newCurl() const -{ - return VariableCurlHelper::newCurl(); -} - -bool -XslPreFetch::asHtml() const -{ - return html; -} - -bool -XslPreFetch::withWarnings() const -{ - return warnings; -} - diff --git a/project2/xml/xslPreFetch.h b/project2/xml/xslPreFetch.h deleted file mode 100644 index fb453d5..0000000 --- a/project2/xml/xslPreFetch.h +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef XSLPREFETCH_H -#define XSLPREFETCH_H - -#include "xslRowsCache.h" -#include "curlHelper.h" -#include "view.h" -#include "task.h" -#include - -/// Project2 component to queue up CURL objects to be downloaded -class XslPreFetch : public View, public Task, XslRowsCache, VariableCurlHelper { - public: - XslPreFetch(const xmlpp::Element * p); - ~XslPreFetch(); - - void execute(const Presenter*) const; - void execute() const; - void loadComplete(const CommonObjects *); - - const bool html; - const bool warnings; - const Variable encoding; - - CurlPtr newCurl() const; - bool asHtml() const; - bool withWarnings() const; -}; - -#endif - diff --git a/project2/xml/xslRows.cpp b/project2/xml/xslRows.cpp deleted file mode 100644 index 31d0e0b..0000000 --- a/project2/xml/xslRows.cpp +++ /dev/null @@ -1,160 +0,0 @@ -#include -#include "xslRows.h" -#include "safeMapFind.h" -#include "rowProcessor.h" -#include "logger.h" -#include "xml.h" -#include "exceptions.h" -#include "xmlObjectLoader.h" -#include -#include -#include -#include "../libmisc/curlsup.h" -#include - -DECLARE_LOADER("xslrows", XslRows); - -SimpleMessageException(XpathInitError); -SimpleMessageException(XpathEvalError); - -XslRows::XslRows(const xmlpp::Element * p) : - RowSet(p), - VariableCurlHelper(p), - html(p->get_attribute_value("html") == "true"), - warnings(p->get_attribute_value("warnings") != "false"), - encoding(p, "encoding", false) -{ - BOOST_FOREACH(const xmlpp::Node * node, p->find("filterview")) { - const xmlpp::Element * elem = dynamic_cast(node); - if (elem) { - FilterViewPtr fv = new FilterView(elem); - fvs[fv->name] = fv; - } - } - BOOST_FOREACH(const xmlpp::Node * node, p->find("namespace")) { - const xmlpp::Element * elem = dynamic_cast(node); - if (elem) { - namespaces[elem->get_attribute_value("prefix")] = elem->get_attribute_value("url"); - } - } -} - -XslRows::~XslRows() -{ -} - -void -XslRows::loadComplete(const CommonObjects *) -{ -} - -bool -XslRows::asHtml() const -{ - return html; -} - -bool -XslRows::withWarnings() const -{ - return warnings; -} - -CurlPtr -XslRows::newCurl() const -{ - return VariableCurlHelper::newCurl(); -} - -void -XslRows::execute(const Glib::ustring & filter, const RowProcessor * rp) const -{ - FilterViewPtr fv = safeMapFind(fvs, filter)->second; - - typedef boost::shared_ptr xmlXPathObjectSPtr; - typedef boost::shared_ptr xmlXPathContextSPtr; - xmlDocPtr doc = getDocument(url(), encoding()); - xmlXPathContextSPtr xpathCtx = xmlXPathContextSPtr(xmlXPathNewContext(doc), xmlXPathFreeContext); - if (!xpathCtx) { - throw XpathInitError(xmlGetLastError()->message); - } - BOOST_FOREACH(const Namespaces::value_type & ns, namespaces) { - xmlXPathRegisterNs(xpathCtx.get(), BAD_CAST ns.first.c_str(), BAD_CAST ns.second.c_str()); - } - xmlXPathObjectSPtr xpathObj = xmlXPathObjectSPtr(xmlXPathEvalExpression(fv->root(), xpathCtx.get()), xmlXPathFreeObject); - if (!xpathObj || !xpathObj->nodesetval) { - throw XpathEvalError(xmlGetLastError()->message); - } - Logger()->messagef(LOG_INFO, "%d nodes matched %s", xpathObj->nodesetval->nodeNr, (const char *)(fv->root())); - XslState xs(fv); - for (int row = 0; row < xpathObj->nodesetval->nodeNr; row += 1) { - xmlNodePtr rowRoot = xpathObj->nodesetval->nodeTab[row]; - xpathCtx->node = rowRoot; - BOOST_FOREACH(const Columns::value_type & _xp, fv->columns.get()) { - const FilterViewColumn * xp = static_cast(_xp.get()); - const VariableType & path(xp->path); - if (boost::get(&path)) { - continue; - } - xmlXPathObjectSPtr xpathObjI = xmlXPathObjectSPtr(xmlXPathEvalExpression(path, xpathCtx.get()), xmlXPathFreeObject); - if (!xpathObjI) { - throw XpathEvalError(xmlGetLastError()->message); - } - if (xpathObjI->floatval) { - xs.fields[xp->idx] = xpathObjI->floatval; - } - else if (xpathObjI->stringval) { - xs.fields[xp->idx] = Glib::ustring((const char *)xpathObjI->stringval); - } - else if (xpathObjI->nodesetval) { - Glib::ustring str; - for (int i = 0; i < xpathObjI->nodesetval->nodeNr; i += 1) { - xmlNodePtr n = xpathObjI->nodesetval->nodeTab[i]; - if (n->content) { - str += (const char *)n->content; - } - for (n = n->children; n; n = n->next) { - xmlChar * val = n->content; - if (val) { - str += (const char *)val; - } - } - } - xs.fields[xp->idx] = str; - } - } - xs.process(rp); - } -} - -XslRows::FilterView::FilterView(const xmlpp::Element * p) : - DefinedColumns(p, "field", boost::bind(XslRows::FilterViewColumn::make, _1, _2)), - name(p->get_attribute_value("name")), - root(p, "root") -{ -} - -XslRows::FilterViewColumn::FilterViewColumn(unsigned int idx, const xmlpp::Element * p) : - Column(idx, p), - path(p, "xpath") -{ -} - -XslRows::FilterViewColumn * -XslRows::FilterViewColumn::make(unsigned int idx, const xmlpp::Element * p) -{ - return new FilterViewColumn(idx, p); -} - -XslRows::XslState::XslState(FilterViewCPtr f) : - fv(f) -{ - fields.resize(f->columns.size()); -} - -const Columns & -XslRows::XslState::getColumns() const -{ - return fv->columns; -} - diff --git a/project2/xml/xslRows.h b/project2/xml/xslRows.h deleted file mode 100644 index 9611ca4..0000000 --- a/project2/xml/xslRows.h +++ /dev/null @@ -1,64 +0,0 @@ -#ifndef XSLROWS_H -#define XSLROWS_H - -#include -#include -#include -#include -#include "rowSet.h" -#include "variables.h" -#include "xslRowsCache.h" -#include "curlHelper.h" -#include "definedColumns.h" - -/// Project2 component to create a row set based on the contents of an XML resource and specific XPaths with its hierarchy -class XslRows : public RowSet, XslRowsCache, VariableCurlHelper { - public: - XslRows(const xmlpp::Element * p); - ~XslRows(); - - void execute(const Glib::ustring &, const RowProcessor *) const; - virtual void loadComplete(const CommonObjects *); - - const bool html; - const bool warnings; - - private: - class FilterViewColumn : public Column { - public: - FilterViewColumn(unsigned int, const xmlpp::Element *); - static FilterViewColumn * make(unsigned int, const xmlpp::Element *); - const Variable path; - }; - class FilterView : public DefinedColumns, public virtual IntrusivePtrBase { - public: - typedef std::map XPaths; - - FilterView(const xmlpp::Element * p); - - const Glib::ustring name; - const Variable root; - }; - typedef boost::intrusive_ptr FilterViewPtr; - typedef boost::intrusive_ptr FilterViewCPtr; - typedef std::map FilterViews; - FilterViews fvs; - - virtual CurlPtr newCurl() const; - virtual bool asHtml() const; - virtual bool withWarnings() const; - - typedef std::map Namespaces; - Namespaces namespaces; - class XslState : public RowState { - public: - XslState(FilterViewCPtr); - const Columns & getColumns() const; - private: - const FilterViewCPtr fv; - }; - const Variable encoding; -}; - -#endif - diff --git a/project2/xml/xslRowsCache.cpp b/project2/xml/xslRowsCache.cpp deleted file mode 100644 index bd017d7..0000000 --- a/project2/xml/xslRowsCache.cpp +++ /dev/null @@ -1,107 +0,0 @@ -#include -#include -#include "xslRowsCache.h" -#include -#include -#include "exceptions.h" -#include "curlHelper.h" -#include "safeMapFind.h" - -XslRowsCache::Documents XslRowsCache::documents; -XslRowsCache::Queued XslRowsCache::queued; -CurlBulkFetcher XslRowsCache::cbf; - -SimpleMessageException(XmlParseError); -SimpleMessageException(DownloadFailed); - -template -static XslRowsCache::DocumentPtr helperThrow(const std::string & msg) { - throw Exception(msg); -} -static XslRowsCache::DocumentPtr helperReturnDocument(XslRowsCache::DocumentPtr dp) { - return dp; -} - -class XslCachePopulator : public CurlCompleteCallback { - public: - XslCachePopulator(CurlPtr ch, const Glib::ustring & u, bool h, bool w, const char * e) : - CurlCompleteCallback(ch), - handler(boost::bind(&XslCachePopulator::append, this, _1, _2)), - url(u), - html(h), - warnings(w), - encoding(e ? strdup(e) : NULL) - { - ch->setReadHandler(handler); - } - ~XslCachePopulator() - { - free(encoding); - } - void call(CurlBulkFetcher *) - { - int flags = 0; - flags |= warnings ? 0 : XML_PARSE_NOWARNING | XML_PARSE_NOERROR; - xmlDocPtr doc = html ? - htmlReadMemory(buf.c_str(), buf.length(), url.c_str(), encoding, flags) : - xmlReadMemory(buf.c_str(), buf.length(), url.c_str(), encoding, flags); - if (!doc) { - Logger()->messagef(LOG_DEBUG, "Download of '%s' succeeded, but parsing failed with error '%s'", url.c_str(), xmlGetLastError()->message); - XslRowsCache::documents.insert(XslRowsCache::Documents::value_type(url, - boost::bind(helperThrow, std::string(xmlGetLastError()->message)))); - } - XslRowsCache::documents.insert(XslRowsCache::Documents::value_type(url, - boost::bind(helperReturnDocument, XslRowsCache::DocumentPtr(doc, xmlFreeDoc)))); - Logger()->messagef(LOG_DEBUG, "Download of '%s' completed, stored", url.c_str()); - } - void error(CurlBulkFetcher *, const char * error) - { - Logger()->messagef(LOG_DEBUG, "Download of '%s' failed with error '%s'", url.c_str(), error); - XslRowsCache::documents.insert(XslRowsCache::Documents::value_type(url, - boost::bind(helperThrow, std::string(error)))); - } - size_t append(const char * c, size_t b) - { - buf.append(c, b); - return b; - } - - Curl::ReadHandler handler; - std::string buf; - const Glib::ustring url; - const bool html; - const bool warnings; - char * encoding; -}; - -xmlDocPtr -XslRowsCache::getDocument(const Glib::ustring & url, const char * encoding) const -{ - Documents::const_iterator i = documents.find(url); - if (i == documents.end()) { - queue(url, encoding); - cbf.perform(); - queued.clear(); - } - return safeMapFind(documents, url)->second().get(); -} - -void -XslRowsCache::queue(const Glib::ustring & url, const char * encoding) const -{ - if (queued.find(url) == queued.end()) { - cbf.curls.insert(new XslCachePopulator(newCurl(), url, asHtml(), withWarnings(), encoding)); - queued.insert(url); - } -} - -class XslCacheClearer : public ComponentLoader { - public: - void onIteration() - { - Logger()->messagef(LOG_DEBUG, "%s: Clearing XSL row document cache", __PRETTY_FUNCTION__); - XslRowsCache::documents.clear(); - } -}; -DECLARE_CUSTOM_COMPONENT_LOADER("XslCacheClearer", XslCacheClearer, XslCacheClearer, ComponentLoader); - diff --git a/project2/xml/xslRowsCache.h b/project2/xml/xslRowsCache.h deleted file mode 100644 index 14362b5..0000000 --- a/project2/xml/xslRowsCache.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef XSLROWSCACHE_H -#define XSLROWSCACHE_H - -#include -#include -#include -#include -#include -#include - -class XslRowsCache { - public: - typedef std::set Queued; - typedef boost::shared_ptr DocumentPtr; - typedef boost::function0 ReturnDocument; - typedef std::map Documents; - - protected: - static Queued queued; - static Documents documents; - - void queue(const Glib::ustring & url, const char * encoding) const; - - virtual CurlPtr newCurl() const = 0; - virtual bool asHtml() const = 0; - virtual bool withWarnings() const = 0; - - protected: - xmlDocPtr getDocument(const Glib::ustring & url, const char * encoding) const; - - private: - static CurlBulkFetcher cbf; - - friend class XslCachePopulator; - friend class XslCacheClearer; -}; - -#endif -- cgit v1.2.3