From 8330078325a5cd988c27bd31a614bcc961e66d70 Mon Sep 17 00:00:00 2001 From: Dan Goodliffe Date: Mon, 11 Jan 2016 20:25:01 +0000 Subject: WIP GB API Maint --- gentoobrowse-api/api/maintenance.ice | 6 ++ gentoobrowse-api/db/schema.sql | 3 +- gentoobrowse-api/service/Jamfile.jam | 1 + gentoobrowse-api/service/main.cpp | 2 + gentoobrowse-api/service/maintenanceimpl.cpp | 76 ++++++++++++++++++++++ gentoobrowse-api/service/maintenanceimpl.h | 28 ++++++++ .../sql/maintenance/createTempFileListTable.sql | 5 ++ .../sql/maintenance/createTempFileListView.sql | 11 ++++ gentoobrowse-api/unittests/Jamfile.jam | 9 +++ gentoobrowse-api/unittests/data.sql | 2 +- gentoobrowse-api/unittests/fixtures/repos.dat | 4 +- gentoobrowse-api/unittests/mockDefs.cpp | 1 + gentoobrowse-api/unittests/mockDefs.h | 2 + gentoobrowse-api/unittests/testMaintenance.cpp | 21 ++++++ 14 files changed, 167 insertions(+), 4 deletions(-) create mode 100644 gentoobrowse-api/api/maintenance.ice create mode 100644 gentoobrowse-api/service/maintenanceimpl.cpp create mode 100644 gentoobrowse-api/service/maintenanceimpl.h create mode 100644 gentoobrowse-api/service/sql/maintenance/createTempFileListTable.sql create mode 100644 gentoobrowse-api/service/sql/maintenance/createTempFileListView.sql create mode 100644 gentoobrowse-api/unittests/testMaintenance.cpp diff --git a/gentoobrowse-api/api/maintenance.ice b/gentoobrowse-api/api/maintenance.ice new file mode 100644 index 0000000..55389b2 --- /dev/null +++ b/gentoobrowse-api/api/maintenance.ice @@ -0,0 +1,6 @@ +module Gentoo { + interface Maintenance { + idempotent void refreshPackageTree(); + }; +}; + diff --git a/gentoobrowse-api/db/schema.sql b/gentoobrowse-api/db/schema.sql index 119d5bc..a2e66c4 100644 --- a/gentoobrowse-api/db/schema.sql +++ b/gentoobrowse-api/db/schema.sql @@ -437,7 +437,8 @@ COMMENT ON TABLE package_urls IS 'URLs associated with a given package'; -- Name: repos; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: CREATE TABLE repos ( repoid integer NOT NULL, - name text NOT NULL + name text NOT NULL, + path text NOT NULL ); ALTER TABLE repos OWNER TO gentoo; -- Name: repos_repoid_seq; Type: SEQUENCE; Schema: gentoobrowse; Owner: gentoo diff --git a/gentoobrowse-api/service/Jamfile.jam b/gentoobrowse-api/service/Jamfile.jam index 5bad606..c4b1bad 100644 --- a/gentoobrowse-api/service/Jamfile.jam +++ b/gentoobrowse-api/service/Jamfile.jam @@ -20,6 +20,7 @@ lib gentoobrowse-service : ../api//gentoobrowse-api ..//boost_system ..//boost_thread + ..//boost_filesystem ..//boost_date_time ../..//glibmm Gentoo::Service diff --git a/gentoobrowse-api/service/main.cpp b/gentoobrowse-api/service/main.cpp index 1429a72..a936b74 100644 --- a/gentoobrowse-api/service/main.cpp +++ b/gentoobrowse-api/service/main.cpp @@ -2,6 +2,7 @@ #include #include #include "portageimpl.h" +#include "maintenanceimpl.h" namespace Gentoo { namespace Service { @@ -11,6 +12,7 @@ class Api : public IceTray::Service { { auto db = getConnectionPool(ic, "postgresql", "GentooBrowseAPI"); adp->add(new Portage(db), ic->stringToIdentity("portage")); + adp->add(new Maintenance(db), ic->stringToIdentity("maintenance")); } }; diff --git a/gentoobrowse-api/service/maintenanceimpl.cpp b/gentoobrowse-api/service/maintenanceimpl.cpp new file mode 100644 index 0000000..b9a83b4 --- /dev/null +++ b/gentoobrowse-api/service/maintenanceimpl.cpp @@ -0,0 +1,76 @@ +#include "maintenanceimpl.h" +#include +#include +#include +#include +#include +#include + +namespace Gentoo { + namespace Service { + Maintenance::Maintenance(IceTray::DatabasePoolPtr d) : + IceTray::AbstractDatabaseClient(d) + { + } + + FilePtr + Maintenance::findInPath(const std::string & p) + { + AdHoc::Buffer find; + find.appendbf("(cd %s && find * -type f -printf \"%%p\t%%s\t%%TFT%%TT\\n\" -follow)", p); + return FilePtr(popen(find, "r"), pclose); + } + + void + Maintenance::createTempFileList(DB::Connection * dbc) + { + auto tmp = boost::filesystem::temp_directory_path() / "import"; + boost::filesystem::remove_all(tmp); + boost::filesystem::create_directories(tmp); + dbc->select("SELECT name, path FROM gentoobrowse.repos")->forEachRow([&tmp](auto n, auto p) { + boost::filesystem::create_symlink(p, tmp / n); + }); + dbc->execute(sql::maintenance::createTempFileListTable::sql); + dbc->beginBulkUpload("filelist", ""); + dbc->bulkUploadData(findInPath(tmp.string()).get()); + dbc->endBulkUpload(nullptr); + boost::filesystem::remove_all(tmp); + } + + void + Maintenance::createTempFileListView(DB::Connection * dbc) + { + dbc->execute(sql::maintenance::createTempFileListView::sql); + } + + void + Maintenance::mergeTempFileList(DB::Connection * dbc) + { + DB::TablePatch tp; + tp.src = "tmp"; + tp.dest = "gentoobrowse.files"; + tp.pk.insert("repoid"); + tp.pk.insert("filename"); + tp.cols.insert("repoid"); + tp.cols.insert("filename"); + tp.cols.insert("filetypeid"); + tp.cols.insert("pathparts"); + tp.cols.insert("filesize"); + tp.cols.insert("moddate"); + dbc->patchTable(&tp); + } + + void + Maintenance::refreshPackageTree(const Ice::Current &) + { + auto dbc = db->get(); + dbc->execute("SET search_path = gentoobrowse, pg_catalog"); + DB::TransactionScope tx(dbc.get()); + createTempFileList(dbc.get()); + createTempFileListView(dbc.get()); + mergeTempFileList(dbc.get()); + dbc->execute("SET search_path = public, pg_catalog"); + } + } +} + diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h new file mode 100644 index 0000000..b7f3b3e --- /dev/null +++ b/gentoobrowse-api/service/maintenanceimpl.h @@ -0,0 +1,28 @@ +#ifndef MAINTENANCEIMPL_H +#define MAINTENANCEIMPL_H + +#include +#include +#include + +namespace Gentoo { + typedef boost::shared_ptr FilePtr; + + namespace Service { + class DLL_PUBLIC Maintenance : public Gentoo::Maintenance, IceTray::AbstractDatabaseClient { + public: + Maintenance(IceTray::DatabasePoolPtr d); + + void refreshPackageTree(const Ice::Current &) override; + + private: + static void createTempFileList(DB::Connection *); + static void createTempFileListView(DB::Connection *); + static void mergeTempFileList(DB::Connection *); + static FilePtr findInPath(const std::string &); + }; + } +} + +#endif + diff --git a/gentoobrowse-api/service/sql/maintenance/createTempFileListTable.sql b/gentoobrowse-api/service/sql/maintenance/createTempFileListTable.sql new file mode 100644 index 0000000..dd6f33c --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/createTempFileListTable.sql @@ -0,0 +1,5 @@ +CREATE TEMPORARY TABLE filelist( + filename TEXT NOT NULL, + filesize INTEGER NOT NULL, + filemtime TIMESTAMP WITHOUT TIME ZONE NOT NULL, + PRIMARY KEY(filename)) diff --git a/gentoobrowse-api/service/sql/maintenance/createTempFileListView.sql b/gentoobrowse-api/service/sql/maintenance/createTempFileListView.sql new file mode 100644 index 0000000..91dd062 --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/createTempFileListView.sql @@ -0,0 +1,11 @@ +CREATE TEMPORARY VIEW tmp AS +SELECT + r.repoid, + fl.filename, + ft.filetypeid, + (STRING_TO_ARRAY(fl.filename, '/'))[2:100] pathparts, + fl.filesize, + fl.filemtime AT TIME ZONE 'utc' moddate +FROM filelist fl, gentoobrowse.repos r, gentoobrowse.filetypes ft +WHERE (STRING_TO_ARRAY(fl.filename, '/'))[1] = r.name +AND (STRING_TO_ARRAY(fl.filename, '/'))[2:100] ~ ft.spec diff --git a/gentoobrowse-api/unittests/Jamfile.jam b/gentoobrowse-api/unittests/Jamfile.jam index fb6abf7..256ccb5 100644 --- a/gentoobrowse-api/unittests/Jamfile.jam +++ b/gentoobrowse-api/unittests/Jamfile.jam @@ -43,6 +43,15 @@ lib testCommon : ; +run + testMaintenance.cpp + : : + ../db/schema.sql + : + BOOST_TEST_DYN_LINK + testCommon + : testMaintenance ; + run testPortage.cpp : : diff --git a/gentoobrowse-api/unittests/data.sql b/gentoobrowse-api/unittests/data.sql index 439f257..069fd25 100644 --- a/gentoobrowse-api/unittests/data.sql +++ b/gentoobrowse-api/unittests/data.sql @@ -9,7 +9,7 @@ SET search_path = gentoobrowse, pg_catalog; COPY gentoobrowse.bugs (bugid, severity, status, summary, firstseen) FROM '$SCRIPTDIR/fixtures/bugs.dat'; COPY gentoobrowse.categories (categoryid, name, summary) FROM '$SCRIPTDIR/fixtures/categories.dat'; COPY gentoobrowse.packages (packageid, categoryid, name, firstseen, description, summary, maintainer, herd, image, maintainername) FROM '$SCRIPTDIR/fixtures/packages.dat'; -COPY gentoobrowse.repos (repoid, name) FROM '$SCRIPTDIR/fixtures/repos.dat'; +COPY gentoobrowse.repos (repoid, name, path) FROM '$SCRIPTDIR/fixtures/repos.dat'; COPY gentoobrowse.ebuilds (ebuildid, packageid, version, versioninst, slot, license, firstseen, moddate, repoid) FROM '$SCRIPTDIR/fixtures/ebuilds.dat'; COPY gentoobrowse.ebuild_archs (ebuildid, arch) FROM '$SCRIPTDIR/fixtures/ebuild_archs.dat'; COPY gentoobrowse.ebuild_deps (ebuildid, packageid, versionspec, flags, op, slot) FROM '$SCRIPTDIR/fixtures/ebuild_deps.dat'; diff --git a/gentoobrowse-api/unittests/fixtures/repos.dat b/gentoobrowse-api/unittests/fixtures/repos.dat index d1f6df9..30d0932 100644 --- a/gentoobrowse-api/unittests/fixtures/repos.dat +++ b/gentoobrowse-api/unittests/fixtures/repos.dat @@ -1,2 +1,2 @@ -17 gentoo -18 RandomLAN +17 gentoo /usr/portage +18 RandomLAN /home/randomdan/dev/portage diff --git a/gentoobrowse-api/unittests/mockDefs.cpp b/gentoobrowse-api/unittests/mockDefs.cpp index 35032e2..e5bce72 100644 --- a/gentoobrowse-api/unittests/mockDefs.cpp +++ b/gentoobrowse-api/unittests/mockDefs.cpp @@ -9,6 +9,7 @@ Service::Service() : } TestClient::TestClient() : + m(getProxy("maintenance")), p(getProxy("portage")) { } diff --git a/gentoobrowse-api/unittests/mockDefs.h b/gentoobrowse-api/unittests/mockDefs.h index 8580fed..0e68be5 100644 --- a/gentoobrowse-api/unittests/mockDefs.h +++ b/gentoobrowse-api/unittests/mockDefs.h @@ -5,6 +5,7 @@ #include #include #include +#include class DLL_PUBLIC Service : public IceTray::DryIce, PQ::Mock { public: @@ -15,6 +16,7 @@ class DLL_PUBLIC TestClient : public IceTray::DryIceClient { public: TestClient(); + Gentoo::MaintenancePrx m; Gentoo::PortagePrx p; }; diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp new file mode 100644 index 0000000..383f6cc --- /dev/null +++ b/gentoobrowse-api/unittests/testMaintenance.cpp @@ -0,0 +1,21 @@ +#define BOOST_TEST_MODULE TestMaintenance +#include +#include + +#include "mockDefs.h" + +BOOST_GLOBAL_FIXTURE( Service ); + +BOOST_FIXTURE_TEST_SUITE(tp, TestClient) + +BOOST_AUTO_TEST_CASE( refreshPackageTree ) +{ + m->refreshPackageTree(); + auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("GentooBrowseAPI")); + db->select("SELECT COUNT(*) FROM gentoobrowse.files")->forEachRow([](auto n) { + BOOST_REQUIRE_EQUAL(96645, n); + }); +} + +BOOST_AUTO_TEST_SUITE_END(); + -- cgit v1.2.3