From d04e24bc2f35951219d42f1c9c0d57236ae6a026 Mon Sep 17 00:00:00 2001 From: Dan Goodliffe Date: Wed, 20 Apr 2016 23:21:21 +0100 Subject: Add file processing framework and category meta processor --- gentoobrowse-api/service/Jamfile.jam | 2 +- .../service/maintenance/categoryMetaProcessor.cpp | 47 ++++++++++++++++++ .../service/maintenance/categoryMetaProcessor.h | 26 ++++++++++ gentoobrowse-api/service/maintenanceimpl.cpp | 55 ++++++++++++++++++---- gentoobrowse-api/service/maintenanceimpl.h | 18 +++++++ gentoobrowse-api/unittests/testMaintenance.cpp | 9 ++-- 6 files changed, 144 insertions(+), 13 deletions(-) create mode 100644 gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp create mode 100644 gentoobrowse-api/service/maintenance/categoryMetaProcessor.h diff --git a/gentoobrowse-api/service/Jamfile.jam b/gentoobrowse-api/service/Jamfile.jam index c5a19a2..2f4279a 100644 --- a/gentoobrowse-api/service/Jamfile.jam +++ b/gentoobrowse-api/service/Jamfile.jam @@ -4,7 +4,7 @@ import icetray ; lib icetray : : : : /usr/include/icetray ; lib gentoobrowse-service : - [ glob *.cpp ] + [ glob-tree *.cpp : bin ] [ glob *.ll ] [ glob-tree *.sql ] [ glob ../domain/*.ice ] diff --git a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp new file mode 100644 index 0000000..01f27f1 --- /dev/null +++ b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp @@ -0,0 +1,47 @@ +#include "categoryMetaProcessor.h" +#include +#include "fileUtils.h" +#include "xmlUtils.h" +#include "dbUtils.h" + +namespace U = Gentoo::Utils; +using namespace Gentoo::Utils::File; + +namespace Gentoo { + namespace Service { + const int CategoryMetaProcessor::FILETYPEID = 10; + + void + CategoryMetaProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const + { + upsert(dbc->modify("INSERT INTO gentoobrowse.categories(summary, name) VALUES(?, ?)"), fn, path); + } + + void + CategoryMetaProcessor::upsert(DB::ModifyCommandPtr m, const boost::filesystem::path & fn, const boost::filesystem::path & path) const + { + U::XmlDoc md(path); + U::Database::bindOptionalsS(m, 0, { + md.getXPathValue("/catmetadata/longdescription[@lang='en']"), + md.getXPathValue("/catmetadata/longdescription[1]") + }); + m->bindParamS(1, (fn / 1).string()); + m->execute(); + } + + void + CategoryMetaProcessor::modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const + { + upsert(dbc->modify("UPDATE gentoobrowse.categories SET summary = ? WHERE name = ?)"), fn, path); + } + + void + CategoryMetaProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const + { + auto m = dbc->modify("DELETE FROM gentoobrowse.categories WHERE name = ?"); + m->bindParamS(0, (fn / 1).string()); + m->execute(); + } + } +} + diff --git a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.h b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.h new file mode 100644 index 0000000..b8c457e --- /dev/null +++ b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.h @@ -0,0 +1,26 @@ +#ifndef GENTOOBROWSE_API_SERVICE_MAINTENANCE_CATEGORYMETAPROC_H +#define GENTOOBROWSE_API_SERVICE_MAINTENANCE_CATEGORYMETAPROC_H + +#include "../maintenanceimpl.h" +#include +#include +#include + +namespace Gentoo { + namespace Service { + class CategoryMetaProcessor : public Maintenance::FileProcessor { + public: + static const int FILETYPEID; + + void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const; + void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const; + void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const; + + private: + void upsert(DB::ModifyCommandPtr m, const boost::filesystem::path & fn, const boost::filesystem::path & path) const; + }; + } +} + +#endif + diff --git a/gentoobrowse-api/service/maintenanceimpl.cpp b/gentoobrowse-api/service/maintenanceimpl.cpp index 8592313..9b8b550 100644 --- a/gentoobrowse-api/service/maintenanceimpl.cpp +++ b/gentoobrowse-api/service/maintenanceimpl.cpp @@ -6,6 +6,7 @@ #include #include #include +#include "maintenance/categoryMetaProcessor.h" /* 1 package metadata {"(1,metadata)","(2,md5-cache)"} @@ -23,36 +24,72 @@ namespace Gentoo { namespace Service { + Maintenance::FileProcessor::~FileProcessor() + { + } + + void + Maintenance::FileProcessor::created(DB::Connection *, const boost::filesystem::path &, const boost::filesystem::path &) const + { + } + + void + Maintenance::FileProcessor::modified(DB::Connection *, const boost::filesystem::path &, const boost::filesystem::path &) const + { + } + + void + Maintenance::FileProcessor::deleted(DB::Connection *, const boost::filesystem::path &) const + { + } + Maintenance::Maintenance(IceTray::DatabasePoolPtr d) : IceTray::AbstractDatabaseClient(d) { + fps[CategoryMetaProcessor::FILETYPEID] = new CategoryMetaProcessor(); + } + + Maintenance::~Maintenance() + { + for(const auto & fp : fps) { + delete fp.second; + } } void Maintenance::fileDeleted(DB::Connection * dbc, const boost::filesystem::path & tmp, DB::SelectCommandPtr s) { // b.filename, b.filesize, b.filetypeid, b.moddate, b.pathparts, b.repoid - s->forEachRow([dbc,&tmp](auto fn, auto, auto, auto, auto, auto) { - fprintf(stderr, "deleted file: %s\n", fn.c_str()); - }); + s->forEachRow([dbc,&tmp,this](auto fn, auto, auto ft, auto, auto, auto) { + this->fileHandle(ft, boost::bind(&FileProcessor::deleted, _1, dbc, fn)); + }); } void Maintenance::fileChanged(DB::Connection * dbc, const boost::filesystem::path & tmp, DB::SelectCommandPtr s) { // a.filename, a.repoid, a.filesize old_filesize, a.filetypeid old_filetypeid, a.moddate old_moddate, a.pathparts old_pathparts, b.filesize new_filesize, b.filetypeid new_filetypeid, b.moddate new_moddate, b.pathparts new_pathparts - s->forEachRow([dbc,&tmp](auto fn, auto, auto, auto, auto, auto, auto, auto, auto, auto) { - fprintf(stderr, "updated file: %s\n", fn.c_str()); - }); + s->forEachRow([dbc,&tmp,this](auto fn, auto, auto, auto ft, auto, auto, auto, auto, auto, auto) { + this->fileHandle(ft, boost::bind(&FileProcessor::modified, _1, dbc, fn, tmp / fn)); + }); } void Maintenance::fileCreated(DB::Connection * dbc, const boost::filesystem::path & tmp, DB::SelectCommandPtr s) { // b.filename, b.filesize, b.filetypeid, b.moddate, b.pathparts, b.repoid - s->forEachRow([dbc,&tmp](auto fn, auto, auto, auto, auto, auto) { - fprintf(stderr, "new file: %s\n", fn.c_str()); - }); + s->forEachRow([dbc,&tmp,this](auto fn, auto, auto ft, auto, auto, auto) { + this->fileHandle(ft, boost::bind(&FileProcessor::created, _1, dbc, fn, tmp / fn)); + }); + } + + void + Maintenance::fileHandle(int64_t ft, const FileHandleFunc & f) const + { + auto i = fps.find(ft); + if (i != fps.end()) { + f(i->second); + } } void diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h index baa8783..846507b 100644 --- a/gentoobrowse-api/service/maintenanceimpl.h +++ b/gentoobrowse-api/service/maintenanceimpl.h @@ -4,12 +4,27 @@ #include #include #include +#include +#include namespace Gentoo { namespace Service { class DLL_PUBLIC Maintenance : public Gentoo::Maintenance, IceTray::AbstractDatabaseClient { public: + class DLL_PRIVATE FileProcessor { + public: + virtual ~FileProcessor() = 0; + + virtual void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const; + virtual void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const; + virtual void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const; + }; + + typedef std::map FileProcessors; + typedef boost::function FileHandleFunc; + Maintenance(IceTray::DatabasePoolPtr d); + ~Maintenance(); void refreshPackageTree(const Ice::Current &) override; @@ -19,6 +34,9 @@ namespace Gentoo { void fileDeleted(DB::Connection * dbc, const boost::filesystem::path &, DB::SelectCommandPtr s); void fileChanged(DB::Connection * dbc, const boost::filesystem::path &, DB::SelectCommandPtr s); void fileCreated(DB::Connection * dbc, const boost::filesystem::path & tmp, DB::SelectCommandPtr s); + void fileHandle(int64_t filetypeId, const FileHandleFunc &) const; + + FileProcessors fps; }; } } diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp index bb8de21..b8bb5d4 100644 --- a/gentoobrowse-api/unittests/testMaintenance.cpp +++ b/gentoobrowse-api/unittests/testMaintenance.cpp @@ -37,6 +37,9 @@ class SampleData { } }; +#define SQL_REQUIRE_EQUAL(sql, type, expected) \ + db->select(sql)->forEachRow([](const auto & n) { BOOST_REQUIRE_EQUAL(expected, n); }); + BOOST_FIXTURE_TEST_SUITE(tp, TestClient) BOOST_AUTO_TEST_CASE( refreshPackageTree ) @@ -52,9 +55,9 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree ) m->refreshPackageTree(); sd.clean(); - db->select("SELECT COUNT(*) FROM gentoobrowse.files")->forEachRow([](auto n) { - BOOST_REQUIRE_EQUAL(2084, n); - }); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.files", int64_t, 2084); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, 5); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories WHERE name = 'dev-vcs'", int64_t, 1); sd.extract("756569aa764177340726dd3d40b41d89b11b20c7", "gentoo"); m->refreshPackageTree(); -- cgit v1.2.3