WIP GB API Maint

14 files changed, 167 insertions, 4 deletions

diff --git a/gentoobrowse-api/api/maintenance.ice b/gentoobrowse-api/api/maintenance.ice
new file mode 100644
index 0000000..55389b2
--- /dev/null
+++ b/gentoobrowse-api/api/maintenance.ice
@@ -0,0 +1,6 @@
+module Gentoo {
+	interface Maintenance {
+		idempotent void refreshPackageTree();
+	};
+};
+
diff --git a/gentoobrowse-api/db/schema.sql b/gentoobrowse-api/db/schema.sql
index 119d5bc..a2e66c4 100644
--- a/gentoobrowse-api/db/schema.sql
+++ b/gentoobrowse-api/db/schema.sql
@@ -437,7 +437,8 @@ COMMENT ON TABLE package_urls IS 'URLs associated with a given package';
 -- Name: repos; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: 
 CREATE TABLE repos (
     repoid integer NOT NULL,
-    name text NOT NULL
+    name text NOT NULL,
+		path text NOT NULL
 );
 ALTER TABLE repos OWNER TO gentoo;
 -- Name: repos_repoid_seq; Type: SEQUENCE; Schema: gentoobrowse; Owner: gentoo
diff --git a/gentoobrowse-api/service/Jamfile.jam b/gentoobrowse-api/service/Jamfile.jam
index 5bad606..c4b1bad 100644
--- a/gentoobrowse-api/service/Jamfile.jam
+++ b/gentoobrowse-api/service/Jamfile.jam
@@ -20,6 +20,7 @@ lib gentoobrowse-service :
 	<implicit-dependency>../api//gentoobrowse-api
 	<library>..//boost_system
 	<library>..//boost_thread
+	<library>..//boost_filesystem
 	<library>..//boost_date_time
 	<library>../..//glibmm
 	<icetray.sql.namespace>Gentoo::Service
diff --git a/gentoobrowse-api/service/main.cpp b/gentoobrowse-api/service/main.cpp
index 1429a72..a936b74 100644
--- a/gentoobrowse-api/service/main.cpp
+++ b/gentoobrowse-api/service/main.cpp
@@ -2,6 +2,7 @@
 #include <Ice/ObjectAdapter.h>
 #include <icetrayService.h>
 #include "portageimpl.h"
+#include "maintenanceimpl.h"
 
 namespace Gentoo {
 namespace Service {
@@ -11,6 +12,7 @@ class Api : public IceTray::Service {
 		{
 			auto db = getConnectionPool(ic, "postgresql", "GentooBrowseAPI");
 			adp->add(new Portage(db), ic->stringToIdentity("portage"));
+			adp->add(new Maintenance(db), ic->stringToIdentity("maintenance"));
 		}
 };
 
diff --git a/gentoobrowse-api/service/maintenanceimpl.cpp b/gentoobrowse-api/service/maintenanceimpl.cpp
new file mode 100644
index 0000000..b9a83b4
--- /dev/null
+++ b/gentoobrowse-api/service/maintenanceimpl.cpp
@@ -0,0 +1,76 @@
+#include "maintenanceimpl.h"
+#include <selectcommandUtil.impl.h>
+#include <tablepatch.h>
+#include <buffer.h>
+#include <boost/filesystem/operations.hpp>
+#include <sql/maintenance/createTempFileListTable.sql.h>
+#include <sql/maintenance/createTempFileListView.sql.h>
+
+namespace Gentoo {
+	namespace Service {
+		Maintenance::Maintenance(IceTray::DatabasePoolPtr d) :
+			IceTray::AbstractDatabaseClient(d)
+		{
+		}
+
+		FilePtr
+		Maintenance::findInPath(const std::string & p)
+		{
+			AdHoc::Buffer find;
+			find.appendbf("(cd %s && find * -type f -printf \"%%p\t%%s\t%%TFT%%TT\\n\" -follow)", p);
+			return FilePtr(popen(find, "r"), pclose);
+		}
+
+		void
+		Maintenance::createTempFileList(DB::Connection * dbc)
+		{
+			auto tmp = boost::filesystem::temp_directory_path() / "import";
+			boost::filesystem::remove_all(tmp);
+			boost::filesystem::create_directories(tmp);
+			dbc->select("SELECT name, path FROM gentoobrowse.repos")->forEachRow<std::string, std::string>([&tmp](auto n, auto p) {
+					boost::filesystem::create_symlink(p, tmp / n);
+				});
+			dbc->execute(sql::maintenance::createTempFileListTable::sql);
+			dbc->beginBulkUpload("filelist", "");
+			dbc->bulkUploadData(findInPath(tmp.string()).get());
+			dbc->endBulkUpload(nullptr);
+			boost::filesystem::remove_all(tmp);
+		}
+
+		void
+		Maintenance::createTempFileListView(DB::Connection * dbc)
+		{
+			dbc->execute(sql::maintenance::createTempFileListView::sql);
+		}
+
+		void
+		Maintenance::mergeTempFileList(DB::Connection * dbc)
+		{
+			DB::TablePatch tp;
+			tp.src = "tmp";
+			tp.dest = "gentoobrowse.files";
+			tp.pk.insert("repoid");
+			tp.pk.insert("filename");
+			tp.cols.insert("repoid");
+			tp.cols.insert("filename");
+			tp.cols.insert("filetypeid");
+			tp.cols.insert("pathparts");
+			tp.cols.insert("filesize");
+			tp.cols.insert("moddate");
+			dbc->patchTable(&tp);
+		}
+
+		void
+		Maintenance::refreshPackageTree(const Ice::Current &)
+		{
+			auto dbc = db->get();
+			dbc->execute("SET search_path = gentoobrowse, pg_catalog");
+			DB::TransactionScope tx(dbc.get());
+			createTempFileList(dbc.get());
+			createTempFileListView(dbc.get());
+			mergeTempFileList(dbc.get());
+			dbc->execute("SET search_path = public, pg_catalog");
+		}
+	}
+}
+
diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h
new file mode 100644
index 0000000..b7f3b3e
--- /dev/null
+++ b/gentoobrowse-api/service/maintenanceimpl.h
@@ -0,0 +1,28 @@
+#ifndef MAINTENANCEIMPL_H
+#define MAINTENANCEIMPL_H
+
+#include <maintenance.h>
+#include <visibility.h>
+#include <abstractDatabaseClient.h>
+
+namespace Gentoo {
+	typedef boost::shared_ptr<FILE> FilePtr;
+
+	namespace Service {
+		class DLL_PUBLIC Maintenance : public Gentoo::Maintenance, IceTray::AbstractDatabaseClient {
+			public:
+				Maintenance(IceTray::DatabasePoolPtr d);
+
+				void refreshPackageTree(const Ice::Current &) override;
+
+			private:
+				static void createTempFileList(DB::Connection *);
+				static void createTempFileListView(DB::Connection *);
+				static void mergeTempFileList(DB::Connection *);
+				static FilePtr findInPath(const std::string &);
+		};
+	}
+}
+
+#endif
+
diff --git a/gentoobrowse-api/service/sql/maintenance/createTempFileListTable.sql b/gentoobrowse-api/service/sql/maintenance/createTempFileListTable.sql
new file mode 100644
index 0000000..dd6f33c
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/createTempFileListTable.sql
@@ -0,0 +1,5 @@
+CREATE TEMPORARY TABLE filelist(
+		filename TEXT NOT NULL,
+		filesize INTEGER NOT NULL,
+		filemtime TIMESTAMP WITHOUT TIME ZONE NOT NULL,
+		PRIMARY KEY(filename))
diff --git a/gentoobrowse-api/service/sql/maintenance/createTempFileListView.sql b/gentoobrowse-api/service/sql/maintenance/createTempFileListView.sql
new file mode 100644
index 0000000..91dd062
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/createTempFileListView.sql
@@ -0,0 +1,11 @@
+CREATE TEMPORARY VIEW tmp AS
+SELECT
+	r.repoid,
+	fl.filename,
+	ft.filetypeid,
+	(STRING_TO_ARRAY(fl.filename, '/'))[2:100] pathparts,
+	fl.filesize,
+	fl.filemtime AT TIME ZONE 'utc' moddate
+FROM filelist fl, gentoobrowse.repos r, gentoobrowse.filetypes ft
+WHERE (STRING_TO_ARRAY(fl.filename, '/'))[1] = r.name
+AND (STRING_TO_ARRAY(fl.filename, '/'))[2:100] ~ ft.spec
diff --git a/gentoobrowse-api/unittests/Jamfile.jam b/gentoobrowse-api/unittests/Jamfile.jam
index fb6abf7..256ccb5 100644
--- a/gentoobrowse-api/unittests/Jamfile.jam
+++ b/gentoobrowse-api/unittests/Jamfile.jam
@@ -44,6 +44,15 @@ lib testCommon :
 
 
 run
+	testMaintenance.cpp
+	: :
+	../db/schema.sql
+	:
+	<define>BOOST_TEST_DYN_LINK
+	<library>testCommon
+	: testMaintenance ;
+
+run
 	testPortage.cpp
 	: :
 	../db/schema.sql
diff --git a/gentoobrowse-api/unittests/data.sql b/gentoobrowse-api/unittests/data.sql
index 439f257..069fd25 100644
--- a/gentoobrowse-api/unittests/data.sql
+++ b/gentoobrowse-api/unittests/data.sql
@@ -9,7 +9,7 @@ SET search_path = gentoobrowse, pg_catalog;
 COPY gentoobrowse.bugs (bugid, severity, status, summary, firstseen) FROM '$SCRIPTDIR/fixtures/bugs.dat';
 COPY gentoobrowse.categories (categoryid, name, summary) FROM '$SCRIPTDIR/fixtures/categories.dat';
 COPY gentoobrowse.packages (packageid, categoryid, name, firstseen, description, summary, maintainer, herd, image, maintainername) FROM '$SCRIPTDIR/fixtures/packages.dat';
-COPY gentoobrowse.repos (repoid, name) FROM '$SCRIPTDIR/fixtures/repos.dat';
+COPY gentoobrowse.repos (repoid, name, path) FROM '$SCRIPTDIR/fixtures/repos.dat';
 COPY gentoobrowse.ebuilds (ebuildid, packageid, version, versioninst, slot, license, firstseen, moddate, repoid) FROM '$SCRIPTDIR/fixtures/ebuilds.dat';
 COPY gentoobrowse.ebuild_archs (ebuildid, arch) FROM '$SCRIPTDIR/fixtures/ebuild_archs.dat';
 COPY gentoobrowse.ebuild_deps (ebuildid, packageid, versionspec, flags, op, slot) FROM '$SCRIPTDIR/fixtures/ebuild_deps.dat';
diff --git a/gentoobrowse-api/unittests/fixtures/repos.dat b/gentoobrowse-api/unittests/fixtures/repos.dat
index d1f6df9..30d0932 100644
--- a/gentoobrowse-api/unittests/fixtures/repos.dat
+++ b/gentoobrowse-api/unittests/fixtures/repos.dat
@@ -1,2 +1,2 @@
-17	gentoo
-18	RandomLAN
+17	gentoo	/usr/portage
+18	RandomLAN	/home/randomdan/dev/portage
diff --git a/gentoobrowse-api/unittests/mockDefs.cpp b/gentoobrowse-api/unittests/mockDefs.cpp
index 35032e2..e5bce72 100644
--- a/gentoobrowse-api/unittests/mockDefs.cpp
+++ b/gentoobrowse-api/unittests/mockDefs.cpp
@@ -9,6 +9,7 @@ Service::Service() :
 }
 
 TestClient::TestClient() :
+	m(getProxy<Gentoo::MaintenancePrx>("maintenance")),
 	p(getProxy<Gentoo::PortagePrx>("portage"))
 {
 }
diff --git a/gentoobrowse-api/unittests/mockDefs.h b/gentoobrowse-api/unittests/mockDefs.h
index 8580fed..0e68be5 100644
--- a/gentoobrowse-api/unittests/mockDefs.h
+++ b/gentoobrowse-api/unittests/mockDefs.h
@@ -5,6 +5,7 @@
 #include <visibility.h>
 #include <dryice.h>
 #include <portage.h>
+#include <maintenance.h>
 
 class DLL_PUBLIC Service : public IceTray::DryIce, PQ::Mock {
 	public:
@@ -15,6 +16,7 @@ class DLL_PUBLIC TestClient : public IceTray::DryIceClient {
 	public:
 		TestClient();
 
+		Gentoo::MaintenancePrx m;
 		Gentoo::PortagePrx p;
 };
 
diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp
new file mode 100644
index 0000000..383f6cc
--- /dev/null
+++ b/gentoobrowse-api/unittests/testMaintenance.cpp
@@ -0,0 +1,21 @@
+#define BOOST_TEST_MODULE TestMaintenance
+#include <boost/test/unit_test.hpp>
+#include <selectcommandUtil.impl.h>
+
+#include "mockDefs.h"
+
+BOOST_GLOBAL_FIXTURE( Service );
+
+BOOST_FIXTURE_TEST_SUITE(tp, TestClient)
+
+BOOST_AUTO_TEST_CASE( refreshPackageTree )
+{
+	m->refreshPackageTree();
+	auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("GentooBrowseAPI"));
+	db->select("SELECT COUNT(*) FROM gentoobrowse.files")->forEachRow<int64_t>([](auto n) {
+			BOOST_REQUIRE_EQUAL(96645, n);
+		});
+}
+
+BOOST_AUTO_TEST_SUITE_END();
+