summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--gentoobrowse-api/api/maintenance.ice6
-rw-r--r--gentoobrowse-api/db/schema.sql3
-rw-r--r--gentoobrowse-api/service/Jamfile.jam1
-rw-r--r--gentoobrowse-api/service/main.cpp2
-rw-r--r--gentoobrowse-api/service/maintenanceimpl.cpp76
-rw-r--r--gentoobrowse-api/service/maintenanceimpl.h28
-rw-r--r--gentoobrowse-api/service/sql/maintenance/createTempFileListTable.sql5
-rw-r--r--gentoobrowse-api/service/sql/maintenance/createTempFileListView.sql11
-rw-r--r--gentoobrowse-api/unittests/Jamfile.jam9
-rw-r--r--gentoobrowse-api/unittests/data.sql2
-rw-r--r--gentoobrowse-api/unittests/fixtures/repos.dat4
-rw-r--r--gentoobrowse-api/unittests/mockDefs.cpp1
-rw-r--r--gentoobrowse-api/unittests/mockDefs.h2
-rw-r--r--gentoobrowse-api/unittests/testMaintenance.cpp21
14 files changed, 167 insertions, 4 deletions
diff --git a/gentoobrowse-api/api/maintenance.ice b/gentoobrowse-api/api/maintenance.ice
new file mode 100644
index 0000000..55389b2
--- /dev/null
+++ b/gentoobrowse-api/api/maintenance.ice
@@ -0,0 +1,6 @@
+module Gentoo {
+ interface Maintenance {
+ idempotent void refreshPackageTree();
+ };
+};
+
diff --git a/gentoobrowse-api/db/schema.sql b/gentoobrowse-api/db/schema.sql
index 119d5bc..a2e66c4 100644
--- a/gentoobrowse-api/db/schema.sql
+++ b/gentoobrowse-api/db/schema.sql
@@ -437,7 +437,8 @@ COMMENT ON TABLE package_urls IS 'URLs associated with a given package';
-- Name: repos; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace:
CREATE TABLE repos (
repoid integer NOT NULL,
- name text NOT NULL
+ name text NOT NULL,
+ path text NOT NULL
);
ALTER TABLE repos OWNER TO gentoo;
-- Name: repos_repoid_seq; Type: SEQUENCE; Schema: gentoobrowse; Owner: gentoo
diff --git a/gentoobrowse-api/service/Jamfile.jam b/gentoobrowse-api/service/Jamfile.jam
index 5bad606..c4b1bad 100644
--- a/gentoobrowse-api/service/Jamfile.jam
+++ b/gentoobrowse-api/service/Jamfile.jam
@@ -20,6 +20,7 @@ lib gentoobrowse-service :
<implicit-dependency>../api//gentoobrowse-api
<library>..//boost_system
<library>..//boost_thread
+ <library>..//boost_filesystem
<library>..//boost_date_time
<library>../..//glibmm
<icetray.sql.namespace>Gentoo::Service
diff --git a/gentoobrowse-api/service/main.cpp b/gentoobrowse-api/service/main.cpp
index 1429a72..a936b74 100644
--- a/gentoobrowse-api/service/main.cpp
+++ b/gentoobrowse-api/service/main.cpp
@@ -2,6 +2,7 @@
#include <Ice/ObjectAdapter.h>
#include <icetrayService.h>
#include "portageimpl.h"
+#include "maintenanceimpl.h"
namespace Gentoo {
namespace Service {
@@ -11,6 +12,7 @@ class Api : public IceTray::Service {
{
auto db = getConnectionPool(ic, "postgresql", "GentooBrowseAPI");
adp->add(new Portage(db), ic->stringToIdentity("portage"));
+ adp->add(new Maintenance(db), ic->stringToIdentity("maintenance"));
}
};
diff --git a/gentoobrowse-api/service/maintenanceimpl.cpp b/gentoobrowse-api/service/maintenanceimpl.cpp
new file mode 100644
index 0000000..b9a83b4
--- /dev/null
+++ b/gentoobrowse-api/service/maintenanceimpl.cpp
@@ -0,0 +1,76 @@
+#include "maintenanceimpl.h"
+#include <selectcommandUtil.impl.h>
+#include <tablepatch.h>
+#include <buffer.h>
+#include <boost/filesystem/operations.hpp>
+#include <sql/maintenance/createTempFileListTable.sql.h>
+#include <sql/maintenance/createTempFileListView.sql.h>
+
+namespace Gentoo {
+ namespace Service {
+ Maintenance::Maintenance(IceTray::DatabasePoolPtr d) :
+ IceTray::AbstractDatabaseClient(d)
+ {
+ }
+
+ FilePtr
+ Maintenance::findInPath(const std::string & p)
+ {
+ AdHoc::Buffer find;
+ find.appendbf("(cd %s && find * -type f -printf \"%%p\t%%s\t%%TFT%%TT\\n\" -follow)", p);
+ return FilePtr(popen(find, "r"), pclose);
+ }
+
+ void
+ Maintenance::createTempFileList(DB::Connection * dbc)
+ {
+ auto tmp = boost::filesystem::temp_directory_path() / "import";
+ boost::filesystem::remove_all(tmp);
+ boost::filesystem::create_directories(tmp);
+ dbc->select("SELECT name, path FROM gentoobrowse.repos")->forEachRow<std::string, std::string>([&tmp](auto n, auto p) {
+ boost::filesystem::create_symlink(p, tmp / n);
+ });
+ dbc->execute(sql::maintenance::createTempFileListTable::sql);
+ dbc->beginBulkUpload("filelist", "");
+ dbc->bulkUploadData(findInPath(tmp.string()).get());
+ dbc->endBulkUpload(nullptr);
+ boost::filesystem::remove_all(tmp);
+ }
+
+ void
+ Maintenance::createTempFileListView(DB::Connection * dbc)
+ {
+ dbc->execute(sql::maintenance::createTempFileListView::sql);
+ }
+
+ void
+ Maintenance::mergeTempFileList(DB::Connection * dbc)
+ {
+ DB::TablePatch tp;
+ tp.src = "tmp";
+ tp.dest = "gentoobrowse.files";
+ tp.pk.insert("repoid");
+ tp.pk.insert("filename");
+ tp.cols.insert("repoid");
+ tp.cols.insert("filename");
+ tp.cols.insert("filetypeid");
+ tp.cols.insert("pathparts");
+ tp.cols.insert("filesize");
+ tp.cols.insert("moddate");
+ dbc->patchTable(&tp);
+ }
+
+ void
+ Maintenance::refreshPackageTree(const Ice::Current &)
+ {
+ auto dbc = db->get();
+ dbc->execute("SET search_path = gentoobrowse, pg_catalog");
+ DB::TransactionScope tx(dbc.get());
+ createTempFileList(dbc.get());
+ createTempFileListView(dbc.get());
+ mergeTempFileList(dbc.get());
+ dbc->execute("SET search_path = public, pg_catalog");
+ }
+ }
+}
+
diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h
new file mode 100644
index 0000000..b7f3b3e
--- /dev/null
+++ b/gentoobrowse-api/service/maintenanceimpl.h
@@ -0,0 +1,28 @@
+#ifndef MAINTENANCEIMPL_H
+#define MAINTENANCEIMPL_H
+
+#include <maintenance.h>
+#include <visibility.h>
+#include <abstractDatabaseClient.h>
+
+namespace Gentoo {
+ typedef boost::shared_ptr<FILE> FilePtr;
+
+ namespace Service {
+ class DLL_PUBLIC Maintenance : public Gentoo::Maintenance, IceTray::AbstractDatabaseClient {
+ public:
+ Maintenance(IceTray::DatabasePoolPtr d);
+
+ void refreshPackageTree(const Ice::Current &) override;
+
+ private:
+ static void createTempFileList(DB::Connection *);
+ static void createTempFileListView(DB::Connection *);
+ static void mergeTempFileList(DB::Connection *);
+ static FilePtr findInPath(const std::string &);
+ };
+ }
+}
+
+#endif
+
diff --git a/gentoobrowse-api/service/sql/maintenance/createTempFileListTable.sql b/gentoobrowse-api/service/sql/maintenance/createTempFileListTable.sql
new file mode 100644
index 0000000..dd6f33c
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/createTempFileListTable.sql
@@ -0,0 +1,5 @@
+CREATE TEMPORARY TABLE filelist(
+ filename TEXT NOT NULL,
+ filesize INTEGER NOT NULL,
+ filemtime TIMESTAMP WITHOUT TIME ZONE NOT NULL,
+ PRIMARY KEY(filename))
diff --git a/gentoobrowse-api/service/sql/maintenance/createTempFileListView.sql b/gentoobrowse-api/service/sql/maintenance/createTempFileListView.sql
new file mode 100644
index 0000000..91dd062
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/createTempFileListView.sql
@@ -0,0 +1,11 @@
+CREATE TEMPORARY VIEW tmp AS
+SELECT
+ r.repoid,
+ fl.filename,
+ ft.filetypeid,
+ (STRING_TO_ARRAY(fl.filename, '/'))[2:100] pathparts,
+ fl.filesize,
+ fl.filemtime AT TIME ZONE 'utc' moddate
+FROM filelist fl, gentoobrowse.repos r, gentoobrowse.filetypes ft
+WHERE (STRING_TO_ARRAY(fl.filename, '/'))[1] = r.name
+AND (STRING_TO_ARRAY(fl.filename, '/'))[2:100] ~ ft.spec
diff --git a/gentoobrowse-api/unittests/Jamfile.jam b/gentoobrowse-api/unittests/Jamfile.jam
index fb6abf7..256ccb5 100644
--- a/gentoobrowse-api/unittests/Jamfile.jam
+++ b/gentoobrowse-api/unittests/Jamfile.jam
@@ -44,6 +44,15 @@ lib testCommon :
run
+ testMaintenance.cpp
+ : :
+ ../db/schema.sql
+ :
+ <define>BOOST_TEST_DYN_LINK
+ <library>testCommon
+ : testMaintenance ;
+
+run
testPortage.cpp
: :
../db/schema.sql
diff --git a/gentoobrowse-api/unittests/data.sql b/gentoobrowse-api/unittests/data.sql
index 439f257..069fd25 100644
--- a/gentoobrowse-api/unittests/data.sql
+++ b/gentoobrowse-api/unittests/data.sql
@@ -9,7 +9,7 @@ SET search_path = gentoobrowse, pg_catalog;
COPY gentoobrowse.bugs (bugid, severity, status, summary, firstseen) FROM '$SCRIPTDIR/fixtures/bugs.dat';
COPY gentoobrowse.categories (categoryid, name, summary) FROM '$SCRIPTDIR/fixtures/categories.dat';
COPY gentoobrowse.packages (packageid, categoryid, name, firstseen, description, summary, maintainer, herd, image, maintainername) FROM '$SCRIPTDIR/fixtures/packages.dat';
-COPY gentoobrowse.repos (repoid, name) FROM '$SCRIPTDIR/fixtures/repos.dat';
+COPY gentoobrowse.repos (repoid, name, path) FROM '$SCRIPTDIR/fixtures/repos.dat';
COPY gentoobrowse.ebuilds (ebuildid, packageid, version, versioninst, slot, license, firstseen, moddate, repoid) FROM '$SCRIPTDIR/fixtures/ebuilds.dat';
COPY gentoobrowse.ebuild_archs (ebuildid, arch) FROM '$SCRIPTDIR/fixtures/ebuild_archs.dat';
COPY gentoobrowse.ebuild_deps (ebuildid, packageid, versionspec, flags, op, slot) FROM '$SCRIPTDIR/fixtures/ebuild_deps.dat';
diff --git a/gentoobrowse-api/unittests/fixtures/repos.dat b/gentoobrowse-api/unittests/fixtures/repos.dat
index d1f6df9..30d0932 100644
--- a/gentoobrowse-api/unittests/fixtures/repos.dat
+++ b/gentoobrowse-api/unittests/fixtures/repos.dat
@@ -1,2 +1,2 @@
-17 gentoo
-18 RandomLAN
+17 gentoo /usr/portage
+18 RandomLAN /home/randomdan/dev/portage
diff --git a/gentoobrowse-api/unittests/mockDefs.cpp b/gentoobrowse-api/unittests/mockDefs.cpp
index 35032e2..e5bce72 100644
--- a/gentoobrowse-api/unittests/mockDefs.cpp
+++ b/gentoobrowse-api/unittests/mockDefs.cpp
@@ -9,6 +9,7 @@ Service::Service() :
}
TestClient::TestClient() :
+ m(getProxy<Gentoo::MaintenancePrx>("maintenance")),
p(getProxy<Gentoo::PortagePrx>("portage"))
{
}
diff --git a/gentoobrowse-api/unittests/mockDefs.h b/gentoobrowse-api/unittests/mockDefs.h
index 8580fed..0e68be5 100644
--- a/gentoobrowse-api/unittests/mockDefs.h
+++ b/gentoobrowse-api/unittests/mockDefs.h
@@ -5,6 +5,7 @@
#include <visibility.h>
#include <dryice.h>
#include <portage.h>
+#include <maintenance.h>
class DLL_PUBLIC Service : public IceTray::DryIce, PQ::Mock {
public:
@@ -15,6 +16,7 @@ class DLL_PUBLIC TestClient : public IceTray::DryIceClient {
public:
TestClient();
+ Gentoo::MaintenancePrx m;
Gentoo::PortagePrx p;
};
diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp
new file mode 100644
index 0000000..383f6cc
--- /dev/null
+++ b/gentoobrowse-api/unittests/testMaintenance.cpp
@@ -0,0 +1,21 @@
+#define BOOST_TEST_MODULE TestMaintenance
+#include <boost/test/unit_test.hpp>
+#include <selectcommandUtil.impl.h>
+
+#include "mockDefs.h"
+
+BOOST_GLOBAL_FIXTURE( Service );
+
+BOOST_FIXTURE_TEST_SUITE(tp, TestClient)
+
+BOOST_AUTO_TEST_CASE( refreshPackageTree )
+{
+ m->refreshPackageTree();
+ auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("GentooBrowseAPI"));
+ db->select("SELECT COUNT(*) FROM gentoobrowse.files")->forEachRow<int64_t>([](auto n) {
+ BOOST_REQUIRE_EQUAL(96645, n);
+ });
+}
+
+BOOST_AUTO_TEST_SUITE_END();
+
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 16:15:18 +0000