summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--gentoobrowse-api/api/maintenance.ice3
-rw-r--r--gentoobrowse-api/db/schema.sql26
-rw-r--r--gentoobrowse-api/service/maintenance/updatesProcessor.cpp19
-rw-r--r--gentoobrowse-api/service/maintenanceCommon.cpp11
-rw-r--r--gentoobrowse-api/service/maintenanceGitOperations.cpp32
-rw-r--r--gentoobrowse-api/service/maintenancePackageTree.cpp112
-rw-r--r--gentoobrowse-api/service/maintenanceimpl.h13
-rw-r--r--gentoobrowse-api/service/sql/maintenance/updatesMD5CacheFiles.sql6
-rw-r--r--gentoobrowse-api/service/sql/maintenance/updatesPackageDirFiles.sql4
-rw-r--r--gentoobrowse-api/unittests/Jamfile.jam7
-rw-r--r--gentoobrowse-api/unittests/data.sql3
-rw-r--r--gentoobrowse-api/unittests/fixtures/repos.dat4
-rw-r--r--gentoobrowse-api/unittests/testMaintenance.cpp181
-rw-r--r--gentoobrowse-api/util/update.cpp7
14 files changed, 183 insertions, 245 deletions
diff --git a/gentoobrowse-api/api/maintenance.ice b/gentoobrowse-api/api/maintenance.ice
index f68d371..be88569 100644
--- a/gentoobrowse-api/api/maintenance.ice
+++ b/gentoobrowse-api/api/maintenance.ice
@@ -10,9 +10,8 @@ module Gentoo {
};
interface Maintenance {
- idempotent void refreshPackageTree();
+ idempotent void refreshPackageTree() throws GitError;
idempotent void refreshBugs();
- idempotent void refreshPackageTreeGit() throws GitError;
idempotent void updateRepositories() throws GitError;
UserNews getUserNews();
void sendNotifications();
diff --git a/gentoobrowse-api/db/schema.sql b/gentoobrowse-api/db/schema.sql
index acf0635..25a16b3 100644
--- a/gentoobrowse-api/db/schema.sql
+++ b/gentoobrowse-api/db/schema.sql
@@ -383,15 +383,6 @@ CREATE TABLE ebuilds (
ALTER TABLE ebuilds OWNER TO gentoo;
-- Name: TABLE ebuilds; Type: COMMENT; Schema: gentoobrowse; Owner: gentoo
COMMENT ON TABLE ebuilds IS 'Ebuilds :-)';
--- Name: files; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace:
-CREATE TABLE files (
- moddate timestamp without time zone NOT NULL,
- filetypeid integer NOT NULL,
- repoid integer NOT NULL,
- filesize integer NOT NULL,
- pathparts text[] NOT NULL
-);
-ALTER TABLE files OWNER TO gentoo;
-- Name: filetypes; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace:
CREATE TABLE filetypes (
filetypeid integer NOT NULL,
@@ -474,9 +465,9 @@ CREATE TABLE repos (
repoid integer NOT NULL,
name text NOT NULL,
path text NOT NULL,
- upstream text,
- branch text,
- lastcommit character(40)
+ upstream text NOT NULL,
+ branch text NOT NULL,
+ lastcommit character(40) NOT NULL
);
ALTER TABLE repos OWNER TO gentoo;
-- Name: repos_repoid_seq; Type: SEQUENCE; Schema: gentoobrowse; Owner: gentoo
@@ -663,9 +654,6 @@ ALTER TABLE ebuild_uses CLUSTER ON pk_ebuilduses;
-- Name: pk_fileclass; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace:
ALTER TABLE ONLY filetypes
ADD CONSTRAINT pk_fileclass PRIMARY KEY (filetypeid);
--- Name: pk_files; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace:
-ALTER TABLE ONLY files
- ADD CONSTRAINT pk_files PRIMARY KEY (repoid, pathparts);
-- Name: pk_news; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace:
ALTER TABLE ONLY news
ADD CONSTRAINT pk_news PRIMARY KEY (newsid);
@@ -806,9 +794,6 @@ ALTER TABLE ONLY ebuild_rdeps
-- Name: fk_ebuilduses_file; Type: FK CONSTRAINT; Schema: gentoobrowse; Owner: gentoo
ALTER TABLE ONLY ebuild_uses
ADD CONSTRAINT fk_ebuilduses_file FOREIGN KEY (ebuildid) REFERENCES ebuilds(ebuildid) ON UPDATE CASCADE ON DELETE CASCADE;
--- Name: fk_files_type; Type: FK CONSTRAINT; Schema: gentoobrowse; Owner: gentoo
-ALTER TABLE ONLY files
- ADD CONSTRAINT fk_files_type FOREIGN KEY (filetypeid) REFERENCES filetypes(filetypeid) ON UPDATE CASCADE;
-- Name: fk_packageurls_package; Type: FK CONSTRAINT; Schema: gentoobrowse; Owner: gentoo
ALTER TABLE ONLY package_urls
ADD CONSTRAINT fk_packageurls_package FOREIGN KEY (packageid) REFERENCES packages(packageid) ON UPDATE CASCADE ON DELETE CASCADE;
@@ -888,11 +873,6 @@ REVOKE ALL ON TABLE ebuilds FROM PUBLIC;
REVOKE ALL ON TABLE ebuilds FROM gentoo;
GRANT ALL ON TABLE ebuilds TO gentoo;
GRANT SELECT ON TABLE ebuilds TO apache;
--- Name: files; Type: ACL; Schema: gentoobrowse; Owner: gentoo
-REVOKE ALL ON TABLE files FROM PUBLIC;
-REVOKE ALL ON TABLE files FROM gentoo;
-GRANT ALL ON TABLE files TO gentoo;
-GRANT SELECT ON TABLE files TO apache;
-- Name: filetypes; Type: ACL; Schema: gentoobrowse; Owner: gentoo
REVOKE ALL ON TABLE filetypes FROM PUBLIC;
REVOKE ALL ON TABLE filetypes FROM gentoo;
diff --git a/gentoobrowse-api/service/maintenance/updatesProcessor.cpp b/gentoobrowse-api/service/maintenance/updatesProcessor.cpp
index 5765c17..30e93c1 100644
--- a/gentoobrowse-api/service/maintenance/updatesProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/updatesProcessor.cpp
@@ -9,8 +9,6 @@
#include <fstream>
#include <boost/bind.hpp>
#include <sql/maintenance/categoryInsert.sql.h>
-#include <sql/maintenance/updatesPackageDirFiles.sql.h>
-#include <sql/maintenance/updatesMD5CacheFiles.sql.h>
#include <sql/maintenance/updatesMovePackages.sql.h>
#include <sql/maintenance/updatesMoveUserPackages.sql.h>
#include <sql/maintenance/updatesMoveFilelist.sql.h>
@@ -36,8 +34,6 @@ namespace Gentoo {
UpdatesPatch(DB::Connection * db, bool v) :
vcsMode(v),
categoryInsert(sql::maintenance::categoryInsert.modify(db)),
- movePackageDirFiles(sql::maintenance::updatesPackageDirFiles.modify(db)),
- moveMD5CacheFiles(sql::maintenance::updatesMD5CacheFiles.modify(db)),
findPackage(sql::portage::findPackage.select(db)),
movePackages(sql::maintenance::updatesMovePackages.modify(db)),
moveUserPackages(sql::maintenance::updatesMoveUserPackages.modify(db)),
@@ -69,8 +65,6 @@ namespace Gentoo {
const bool vcsMode;
DB::ModifyCommandPtr drop;
DB::ModifyCommandPtr categoryInsert;
- DB::ModifyCommandPtr movePackageDirFiles;
- DB::ModifyCommandPtr moveMD5CacheFiles;
DB::SelectCommandPtr findPackage;
DB::ModifyCommandPtr movePackages;
DB::ModifyCommandPtr moveUserPackages;
@@ -93,17 +87,6 @@ namespace Gentoo {
}
else {
// Move
- movePackageDirFiles->bindParamS(0, catto);
- movePackageDirFiles->bindParamS(1, nameto);
- movePackageDirFiles->bindParamS(2, catfrom);
- movePackageDirFiles->bindParamS(3, namefrom);
- movePackageDirFiles->execute();
- moveMD5CacheFiles->bindParamS(0, catto);
- moveMD5CacheFiles->bindParamS(1, namefrom);
- moveMD5CacheFiles->bindParamS(2, nameto);
- moveMD5CacheFiles->bindParamS(3, catfrom);
- moveMD5CacheFiles->bindParamS(4, namefrom);
- moveMD5CacheFiles->execute();
categoryInsert->bindParamS(0, catto);
categoryInsert->bindParamS(1, catto);
categoryInsert->execute();
@@ -112,8 +95,6 @@ namespace Gentoo {
movePackages->bindParamS(2, catfrom);
movePackages->bindParamS(3, catto);
movePackages->execute();
- }
- if (vcsMode) {
moveFilelist->bindParamS(0, catto);
moveFilelist->bindParamS(1, namefrom);
moveFilelist->bindParamS(2, nameto);
diff --git a/gentoobrowse-api/service/maintenanceCommon.cpp b/gentoobrowse-api/service/maintenanceCommon.cpp
index 4085e85..63bb13d 100644
--- a/gentoobrowse-api/service/maintenanceCommon.cpp
+++ b/gentoobrowse-api/service/maintenanceCommon.cpp
@@ -18,7 +18,7 @@ namespace Gentoo {
m->ice_ping();
auto rb = m->begin_refreshBugs();
m->updateRepositories();
- m->refreshPackageTreeGit();
+ m->refreshPackageTree();
m->sendNotifications();
m->end_refreshBugs(rb);
}
@@ -28,15 +28,20 @@ namespace Gentoo {
};
Maintenance::Maintenance(IceTray::DatabasePoolPtr d, Ice::CommunicatorPtr ic, Ice::PropertiesPtr p) :
- IceTray::AbstractDatabaseClient(d)
+ Maintenance::Maintenance(d)
{
- setupFileProcessors();
if (auto sync = p->getPropertyAsIntWithDefault("GentooBrowseAPI.SyncTime", 0) * 60) {
taskRunner = new IceUtil::Timer();
taskRunner->scheduleRepeated(new AutoUpdatePackages(ic), IceUtil::Time::seconds(sync));
}
}
+ Maintenance::Maintenance(IceTray::DatabasePoolPtr d) :
+ IceTray::AbstractDatabaseClient(d)
+ {
+ setupFileProcessors();
+ }
+
Maintenance::~Maintenance()
{
if (taskRunner) {
diff --git a/gentoobrowse-api/service/maintenanceGitOperations.cpp b/gentoobrowse-api/service/maintenanceGitOperations.cpp
index b178100..16c51d3 100644
--- a/gentoobrowse-api/service/maintenanceGitOperations.cpp
+++ b/gentoobrowse-api/service/maintenanceGitOperations.cpp
@@ -166,7 +166,7 @@ namespace Gentoo {
}
void
- Maintenance::refreshPackageTreeGit(const Ice::Current & c)
+ Maintenance::refreshPackageTree(const Ice::Current & c)
{
auto properties = c.adapter->getCommunicator()->getProperties();
auto repoRoot = boost::filesystem::path(properties->getPropertyWithDefault("GentooBrowseAPI.RepoRoot", "/"));
@@ -181,25 +181,35 @@ namespace Gentoo {
sql::maintenance::gitListCreate.modify(dbc.get())->execute();
sql::maintenance::gitListCreateIdx.modify(dbc.get())->execute();
+ applyFileChanges(dbc.get(), repoRoot);
+
+ dbc->execute("DROP TABLE filelist");
+ dbc->execute("DROP TABLE filelistraw");
+ dbc->execute("SET search_path = public, pg_catalog");
+ }
+
+ void
+ Maintenance::applyFileChanges(DB::Connection * dbc, const boost::filesystem::path & repoRoot) const
+ {
FileProcessors fps;
for (const auto & fpf : fpfs) {
fps[fpf.first] = fpf.second(true);
}
for (const auto & fp : fps) {
- fp.second->prepare(dbc.get());
+ fp.second->prepare(dbc);
}
RepoMap repos;
dbc->select("SELECT path, repoid FROM gentoobrowse.repos")->forEachRow<std::string, int64_t>([&repos](auto path, auto id) {
repos[id] = path;
});
std::map<std::string, FileHandleFunc> funcs;
- funcs["D"] = boost::bind(&AbstractFileProcessor::deleted, _1, dbc.get(), _2, _3);
- funcs["M"] = boost::bind(&AbstractFileProcessor::modified, _1, dbc.get(), _2, _3, _4);
- funcs["A"] = boost::bind(&AbstractFileProcessor::created, _1, dbc.get(), _2, _3, _4);
- auto phases = sql::maintenance::filelistPhases.select(dbc.get());
+ funcs["D"] = boost::bind(&AbstractFileProcessor::deleted, _1, dbc, _2, _3);
+ funcs["M"] = boost::bind(&AbstractFileProcessor::modified, _1, dbc, _2, _3, _4);
+ funcs["A"] = boost::bind(&AbstractFileProcessor::created, _1, dbc, _2, _3, _4);
+ auto phases = sql::maintenance::filelistPhases.select(dbc);
for (const auto & phaseRow : phases->as<int64_t>()) {
const auto & phase = phaseRow.value<0>();
- auto files = sql::maintenance::gitListChangesInPhase.select(dbc.get());
+ auto files = sql::maintenance::gitListChangesInPhase.select(dbc);
files->bindParamI(0, phase);
std::set<int64_t> usedTypes;
for (const auto & fileRow : files->as<int64_t, int64_t, std::string, std::string>()) {
@@ -213,16 +223,12 @@ namespace Gentoo {
}
}
for (const auto & fp : usedTypes) {
- fps[fp]->apply(dbc.get());
+ fps[fp]->apply(dbc);
}
}
-
- dbc->execute("DROP TABLE filelist");
- dbc->execute("DROP TABLE filelistraw");
for (const auto & fp : fps) {
- fp.second->tidy(dbc.get());
+ fp.second->tidy(dbc);
}
- dbc->execute("SET search_path = public, pg_catalog");
}
}
diff --git a/gentoobrowse-api/service/maintenancePackageTree.cpp b/gentoobrowse-api/service/maintenancePackageTree.cpp
index dbc72a5..9862f28 100644
--- a/gentoobrowse-api/service/maintenancePackageTree.cpp
+++ b/gentoobrowse-api/service/maintenancePackageTree.cpp
@@ -9,9 +9,6 @@
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/algorithm/string/join.hpp>
-#include <sql/maintenance/fileListCreateRaw.sql.h>
-#include <sql/maintenance/fileListCreate.sql.h>
-#include <sql/maintenance/fileListCreatePk.sql.h>
#include "maintenance/categoryMetaProcessor.h"
#include "maintenance/packageMetaProcessor.h"
#include "maintenance/ebuildMetaProcessor.h"
@@ -105,35 +102,6 @@ R"DATA( 10 category metadata 2 2 {"(2,metadata.xml)"}
}
}
- Maintenance::RepoMap
- Maintenance::createTempFileList(DB::Connection * dbc, const boost::filesystem::path & tmp)
- {
- boost::filesystem::remove_all(tmp);
- boost::filesystem::create_directories(tmp);
- RepoMap repos;
- dbc->select("SELECT name, path, repoid FROM gentoobrowse.repos")->forEachRow<std::string, std::string, int64_t>([&tmp,&repos](auto n, auto p, auto id) {
- boost::filesystem::create_symlink(p, tmp / n);
- repos[id] = n;
- });
- dbc->execute(sql::maintenance::fileListCreateRaw.getSql());
- dbc->beginBulkUpload("filelistraw", "");
- char buf[BUFSIZ];
- for (boost::filesystem::recursive_directory_iterator d(tmp, boost::filesystem::symlink_option::recurse);
- d != boost::filesystem::recursive_directory_iterator(); d++) {
- if (boost::filesystem::is_regular_file(d->status()) &&
- !boost::algorithm::ends_with(d->path().string(), ".ebuild")) {
- auto len = snprintf(buf, BUFSIZ, "%s\t%ju\t%s\n",
- d->path().lexically_relative(tmp).c_str(),
- boost::filesystem::file_size(*d),
- boost::posix_time::to_iso_extended_string(
- boost::posix_time::from_time_t(boost::filesystem::last_write_time(*d))).c_str());
- dbc->bulkUploadData(buf, len);
- }
- }
- dbc->endBulkUpload(nullptr);
- return repos;
- }
-
void Maintenance::updateFileTypes(DB::Connection * dbc) const
{
DB::TablePatch p;
@@ -148,86 +116,6 @@ R"DATA( 10 category metadata 2 2 {"(2,metadata.xml)"}
dbc->patchTable(&p);
Utils::Database::drop(dbc, p.src);
}
-
- class PhaseSourceFilter : public Utils::EntityWhereFilter<int64_t> {
- public:
- PhaseSourceFilter() :
- Utils::EntityWhereFilter<int64_t>("filetypeId")
- {
- }
-
- void
- writeSql(AdHoc::Buffer & sql) override
- {
- sql.append("( SELECT * FROM filelist a WHERE ");
- Utils::EntityWhereFilter<int64_t>::writeSql(sql);
- sql.append(")");
- }
- };
-
- void
- Maintenance::processChanges(DB::Connection * dbc, const boost::filesystem::path & tmp, const RepoMap & repos)
- {
- FileProcessors fps;
- for (const auto & fpf : fpfs) {
- fps[fpf.first] = fpf.second(false);
- }
- for (const auto & fp : fps) {
- fp.second->prepare(dbc);
- }
- dbc->execute(sql::maintenance::fileListCreate.getSql());
- dbc->execute(sql::maintenance::fileListCreatePk.getSql());
- DB::TablePatch tp;
- tp.dest = "gentoobrowse.files";
- tp.pk = {"pathparts", "repoid"};
- tp.cols = {"pathparts", "repoid", "filetypeid", "filesize", "moddate"};
- tp.beforeDelete = boost::bind(&Maintenance::fileDeleted, this, dbc, &fps, tmp, &repos, _1);
- tp.beforeUpdate = boost::bind(&Maintenance::fileChanged, this, dbc, &fps, tmp, &repos, _1);
- tp.beforeInsert = boost::bind(&Maintenance::fileCreated, this, dbc, &fps, tmp, &repos, _1);
- DB::StaticSqlWriter obpo("b.updateOrder NULLS LAST");
- tp.order = &obpo;
- auto phases = dbc->select("SELECT phase FROM gentoobrowse.filetypes GROUP BY phase ORDER BY phase");
- for (const auto & phase : phases->as<int64_t>()) {
- Utils::EntityWhereFilter<int64_t> pf("filetypeId");
- PhaseSourceFilter psf;
- auto getFts = dbc->select("SELECT filetypeId FROM gentoobrowse.filetypes WHERE phase = ?");
- getFts->bindParamI(0, phase.value<0>());
- for (const auto & ft : getFts->as<int64_t>()) {
- pf.entityIds.insert(ft.value<0>());
- psf.entityIds.insert(ft.value<0>());
- }
- tp.srcExpr = &psf;
- tp.where = &pf;
- dbc->patchTable(&tp);
- for (const auto & fp : pf.entityIds) {
- if (fps.find(fp) != fps.end()) {
- fps[fp]->apply(dbc);
- }
- }
- }
- dbc->execute("DROP TABLE filelist");
- dbc->execute("DROP TABLE filelistraw");
- for (const auto & fp : fps) {
- fp.second->tidy(dbc);
- }
- }
-
- void
- Maintenance::refreshPackageTree(const Ice::Current &)
- {
- auto tmp = boost::filesystem::temp_directory_path() / "import";
- AdHoc::ScopeExit tidyTmp([&tmp]{
- boost::system::error_code ec;
- boost::filesystem::remove_all(tmp, ec);
- });
- auto dbc = db->get();
- dbc->execute("SET search_path = gentoobrowse, pg_catalog");
- DB::TransactionScope tx(dbc.get());
- updateFileTypes(dbc.get());
- auto repos = createTempFileList(dbc.get(), tmp);
- processChanges(dbc.get(), tmp, repos);
- dbc->execute("SET search_path = public, pg_catalog");
- }
}
}
diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h
index d82adc9..ac74f39 100644
--- a/gentoobrowse-api/service/maintenanceimpl.h
+++ b/gentoobrowse-api/service/maintenanceimpl.h
@@ -13,7 +13,7 @@
namespace Gentoo {
namespace Service {
- class DLL_PUBLIC Maintenance : public Gentoo::Maintenance, IceTray::AbstractDatabaseClient {
+ class DLL_PUBLIC Maintenance : public Gentoo::Maintenance, public IceTray::AbstractDatabaseClient {
public:
typedef std::map<int64_t, boost::function<FileProcessorPtr(bool)>> FileProcessorFactories;
typedef std::map<int64_t, FileProcessorPtr> FileProcessors;
@@ -21,20 +21,21 @@ namespace Gentoo {
typedef std::map<int64_t, std::string> RepoMap;
Maintenance(IceTray::DatabasePoolPtr d, Ice::CommunicatorPtr ic, Ice::PropertiesPtr p);
- ~Maintenance();
+ virtual ~Maintenance();
void refreshPackageTree(const Ice::Current &) override;
- void refreshPackageTreeGit(const Ice::Current &) override;
void refreshBugs(const Ice::Current &) override;
void updateRepositories(const Ice::Current &) override;
Gentoo::UserNews getUserNews(const Ice::Current &) override;
void sendNotifications(const Ice::Current &) override;
+ protected:
+ Maintenance(IceTray::DatabasePoolPtr d);
+ void applyFileChanges(DB::Connection *, const boost::filesystem::path &) const;
+ void updateFileTypes(DB::Connection *) const;
+
private:
void setupFileProcessors();
- void updateFileTypes(DB::Connection *) const;
- static RepoMap createTempFileList(DB::Connection *, const boost::filesystem::path &);
- void processChanges(DB::Connection *, const boost::filesystem::path &, const RepoMap & repos);
void fileDeleted(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path &, const RepoMap *, DB::SelectCommandPtr s);
void fileChanged(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path &, const RepoMap *, DB::SelectCommandPtr s);
void fileCreated(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path & tmp, const RepoMap *, DB::SelectCommandPtr s);
diff --git a/gentoobrowse-api/service/sql/maintenance/updatesMD5CacheFiles.sql b/gentoobrowse-api/service/sql/maintenance/updatesMD5CacheFiles.sql
deleted file mode 100644
index 1fcbb7e..0000000
--- a/gentoobrowse-api/service/sql/maintenance/updatesMD5CacheFiles.sql
+++ /dev/null
@@ -1,6 +0,0 @@
-UPDATE gentoobrowse.files SET
- pathparts[3] = ?,
- pathparts[4] = REGEXP_REPLACE(pathparts[4], CONCAT('^', REPLACE(?, '+', '\+'), '(-[0-9])'), CONCAT(?::text, '\1'))
-WHERE pathparts[3] = ?
-AND pathparts[4] ~ CONCAT('^', replace(?, '+', '\+'), '-[0-9]')
-AND filetypeId = 1
diff --git a/gentoobrowse-api/service/sql/maintenance/updatesPackageDirFiles.sql b/gentoobrowse-api/service/sql/maintenance/updatesPackageDirFiles.sql
deleted file mode 100644
index 307484a..0000000
--- a/gentoobrowse-api/service/sql/maintenance/updatesPackageDirFiles.sql
+++ /dev/null
@@ -1,4 +0,0 @@
-UPDATE gentoobrowse.files SET
- pathparts[1:2] = ARRAY[?, ?]
-WHERE pathparts[1:2] = ARRAY[?, ?]
-AND filetypeId IN (4, 8)
diff --git a/gentoobrowse-api/unittests/Jamfile.jam b/gentoobrowse-api/unittests/Jamfile.jam
index 53d9289..74a67df 100644
--- a/gentoobrowse-api/unittests/Jamfile.jam
+++ b/gentoobrowse-api/unittests/Jamfile.jam
@@ -80,12 +80,19 @@ run
run
testMaintenance.cpp
+ ../service/sql/maintenance/gitListCreate.sql
+ ../service/sql/maintenance/gitListCreateRaw.sql
+ ../service/sql/maintenance/gitListCreateIdx.sql
: : :
<dependency>../db/schema.sql
<define>BOOST_TEST_DYN_LINK
<library>testCommon
<library>git2
+ <library>../service//icetray
<implicit-dependency>testCommon
+ <icetray.sql.namespace>Gentoo::Service
+ <icetray.sql.connector>postgresql
+ <icetray.sql.basedir>../service
: testMaintenance ;
run
diff --git a/gentoobrowse-api/unittests/data.sql b/gentoobrowse-api/unittests/data.sql
index aedf5c6..4120934 100644
--- a/gentoobrowse-api/unittests/data.sql
+++ b/gentoobrowse-api/unittests/data.sql
@@ -9,7 +9,7 @@ SET search_path = gentoobrowse, pg_catalog;
COPY gentoobrowse.bugs (bugid, severity, status, summary, firstseen) FROM '$SCRIPTDIR/fixtures/bugs.dat';
COPY gentoobrowse.categories (categoryid, name, summary) FROM '$SCRIPTDIR/fixtures/categories.dat';
COPY gentoobrowse.packages (packageid, categoryid, name, firstseen, description, summary, maintainer, herd, image, maintainername) FROM '$SCRIPTDIR/fixtures/packages.dat';
-COPY gentoobrowse.repos (repoid, name, path) FROM '$SCRIPTDIR/fixtures/repos.dat';
+COPY gentoobrowse.repos (repoid, name, path, upstream, branch, lastcommit) FROM '$SCRIPTDIR/fixtures/repos.dat';
COPY gentoobrowse.ebuilds (ebuildid, packageid, version, versioninst, slot, license, firstseen, moddate, repoid) FROM '$SCRIPTDIR/fixtures/ebuilds.dat';
COPY gentoobrowse.ebuild_archs (ebuildid, arch) FROM '$SCRIPTDIR/fixtures/ebuild_archs.dat';
COPY gentoobrowse.ebuild_deps (ebuildid, packageid, versionspec, flags, op, slot) FROM '$SCRIPTDIR/fixtures/ebuild_deps.dat';
@@ -18,7 +18,6 @@ COPY gentoobrowse.ebuild_masks (setno, ebuildid) FROM '$SCRIPTDIR/fixtures/ebuil
COPY gentoobrowse.ebuild_rdeps (ebuildid, packageid, versionspec, flags, op, slot) FROM '$SCRIPTDIR/fixtures/ebuild_rdeps.dat';
COPY gentoobrowse.ebuild_uses (ebuildid, use) FROM '$SCRIPTDIR/fixtures/ebuild_uses.dat';
COPY gentoobrowse.filetypes (filetypeid, description, spec, phase, updateOrder) FROM '$SCRIPTDIR/fixtures/filetypes.dat';
-COPY gentoobrowse.files (moddate, filetypeid, repoid, filesize, pathparts) FROM '$SCRIPTDIR/fixtures/files.dat';
COPY gentoobrowse.license (name, legalbumph) FROM '$SCRIPTDIR/fixtures/license.dat';
COPY gentoobrowse.news (newsid, title, posted, authorname, authoremail, atomspec, body, urls) FROM '$SCRIPTDIR/fixtures/news.dat';
COPY gentoobrowse.package_urls (packageid, url) FROM '$SCRIPTDIR/fixtures/package_urls.dat';
diff --git a/gentoobrowse-api/unittests/fixtures/repos.dat b/gentoobrowse-api/unittests/fixtures/repos.dat
index 30d0932..072a029 100644
--- a/gentoobrowse-api/unittests/fixtures/repos.dat
+++ b/gentoobrowse-api/unittests/fixtures/repos.dat
@@ -1,2 +1,2 @@
-17 gentoo /usr/portage
-18 RandomLAN /home/randomdan/dev/portage
+17 gentoo /usr/portage origin master 87ebb8523b2890c641a27f29007ba8eaa6aa834e
+18 RandomLAN /home/randomdan/dev/portage origin master 4321e561074c32ec3015417e517c6cbdb2120fcc
diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp
index e05d9ad..21b77db 100644
--- a/gentoobrowse-api/unittests/testMaintenance.cpp
+++ b/gentoobrowse-api/unittests/testMaintenance.cpp
@@ -7,56 +7,27 @@
#include <modifycommand.h>
#include <git2.h>
#include <fstream>
+#include <maintenanceimpl.h>
+#include <sql/maintenance/gitListCreateRaw.sql.h>
+#include <sql/maintenance/gitListCreate.sql.h>
+#include <sql/maintenance/gitListCreateIdx.sql.h>
class MaintenanceClientCombined : public Maintenance, public TestClient { };
BOOST_FIXTURE_TEST_SUITE(maintenance, MaintenanceClientCombined)
-const boost::filesystem::path treeDir(binDir / "tree" / "gentoobrowse");
+const boost::filesystem::path treeDir(binDir / "tree");
const boost::filesystem::path fixturesDir(rootDir / "fixtures");
-class SampleData {
- public:
- SampleData()
- {
- clean();
- }
-
- ~SampleData()
- {
- clean();
- }
-
- void clean()
- {
- BOOST_TEST_CHECKPOINT("Cleaning sample data dir " << treeDir);
- boost::filesystem::remove_all(treeDir);
- }
-
- void extract(const std::string & archive, const std::string & repo)
- {
- boost::filesystem::create_directories(treeDir);
- boost::filesystem::create_symlink(fixturesDir / archive, treeDir / repo);
- }
-};
-
void dumpDb(DB::ConnectionPtr db);
void
-doRefreshPackageTree(SampleData & sd, DB::ConnectionPtr db, const std::string & archive, const std::string & dir,
- Gentoo::MaintenancePrx m, int64_t files, int64_t cats, int64_t devvcs, int64_t pkgs, int64_t ebs, int64_t ebus,
+doRefreshPackageTree(DB::ConnectionPtr db, int64_t cats, int64_t devvcs, int64_t pkgs, int64_t ebs, int64_t ebus,
int64_t ebas, int64_t pus, int64_t ug, int64_t ul, int64_t ugs, int64_t ugds, int64_t deps, int64_t rdeps,
int64_t news, int64_t masks)
{
- BOOST_TEST_INFO("archive: " << archive);
- if (!archive.empty()) {
- sd.extract(archive, dir);
- }
- m->refreshPackageTree();
- sd.clean();
dumpDb(db);
- SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.files", int64_t, files);
SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, cats);
SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories WHERE name = 'dev-vcs'", int64_t, devvcs);
SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, pkgs);
@@ -97,23 +68,130 @@ dumpDb(DB::ConnectionPtr db)
#endif
}
+class M2 : public Gentoo::Service::Maintenance {
+ public:
+ typedef std::map<boost::filesystem::path, size_t> FileMap;
+
+ M2(IceTray::DatabasePoolPtr d) :
+ Gentoo::Service::Maintenance(d)
+ {
+ auto dbc = db->get();
+ DB::TransactionScope tx(dbc.get());
+ updateFileTypes(dbc.get());
+ }
+
+ void applyDiffOfFolders(int64_t repoId, const boost::filesystem::path & from, const boost::filesystem::path & to) const
+ {
+ fprintf(stderr, "%s\n", __PRETTY_FUNCTION__);
+ auto dbc = db->get();
+ auto fromFiles = fileSet(from);
+ auto toFiles = fileSet(to);
+ dbc->execute("SET search_path = gentoobrowse, pg_catalog");
+ Gentoo::Service::sql::maintenance::gitListCreateRaw.modify(dbc.get())->execute();
+ auto ins = dbc->modify("INSERT INTO filelistraw(repoid, status, filename) VALUES(?, ?, ?)");
+ ins->bindParamI(0, repoId);
+ newFiles(ins, fromFiles, toFiles);
+ removedFiles(ins, fromFiles, toFiles);
+ changedFiles(ins, fromFiles, toFiles);
+ Gentoo::Service::sql::maintenance::gitListCreate.modify(dbc.get())->execute();
+ Gentoo::Service::sql::maintenance::gitListCreateIdx.modify(dbc.get())->execute();
+ boost::filesystem::remove(treeDir);
+ boost::filesystem::create_symlink(to, treeDir);
+ applyFileChanges(dbc.get(), "/");
+ boost::filesystem::remove(treeDir);
+ dbc->execute("DROP TABLE filelist");
+ dbc->execute("DROP TABLE filelistraw");
+ dbc->execute("SET search_path = public, pg_catalog");
+ }
+
+ static FileMap fileSet(const boost::filesystem::path & p)
+ {
+ FileMap found;
+ for (boost::filesystem::recursive_directory_iterator r(p); r != boost::filesystem::recursive_directory_iterator(); r++) {
+ if (boost::filesystem::is_regular(r->status())) {
+ found[boost::filesystem::relative(*r, p)] = boost::filesystem::file_size(*r);
+ }
+ }
+ return found;
+ }
+ static void newFiles(DB::ModifyCommandPtr ins, const FileMap & from, const FileMap & to)
+ {
+ ins->bindParamS(1, "A");
+ for(const auto & f : to) {
+ if (from.find(f.first) == from.end()) {
+ fprintf(stderr, "Added %s\n", f.first.c_str());
+ ins->bindParamS(2, f.first.string());
+ ins->execute();
+ }
+ }
+ }
+ static void removedFiles(DB::ModifyCommandPtr ins, const FileMap & from, const FileMap & to)
+ {
+ ins->bindParamS(1, "D");
+ for(const auto & f : from) {
+ if (to.find(f.first) == to.end()) {
+ fprintf(stderr, "Removed %s\n", f.first.c_str());
+ ins->bindParamS(2, f.first.string());
+ ins->execute();
+ }
+ }
+ }
+ static void changedFiles(DB::ModifyCommandPtr ins, const FileMap & from, const FileMap & to)
+ {
+ ins->bindParamS(1, "M");
+ for(const auto & f : to) {
+ auto i = from.find(f.first);
+ if (i != from.end() && i->second != f.second) {
+ fprintf(stderr, "Changed %s\n", f.first.c_str());
+ ins->bindParamS(2, f.first.string());
+ ins->execute();
+ }
+ }
+ }
+};
+
+class MockPool : public IceTray::DatabasePool {
+ public:
+ MockPool() : IceTray::DatabasePool(10, 2) {}
+ DB::Connection * createResource() const override
+ {
+ return DB::MockDatabase::openConnectionTo("GentooBrowseAPI");
+ }
+};
+
BOOST_AUTO_TEST_CASE( refreshPackageTree )
{
- SampleData sd;
auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("GentooBrowseAPI"));
- auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path) VALUES(?, ?)");
+ auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, upstream, branch, lastcommit) VALUES(?, ?, ?, ?, ?)");
insRepo->bindParamS(0, "gentoo");
- insRepo->bindParamS(1, (treeDir / "gentoo").string());
+ insRepo->bindParamS(1, treeDir.string());
+ insRepo->bindParamS(2, "origin");
+ insRepo->bindParamS(3, "master");
+ insRepo->bindParamS(4, "8292397bf6a8c91215b03a558e8bc843aff25b64");
insRepo->execute();
- doRefreshPackageTree(sd, db, "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d", "gentoo",
- m, 2080, 5, 1, 482, 981, 3626, 4593, 501, 393, 238, 50, 1573, 2008, 1543, 81, 152);
+ boost::filesystem::remove_all(binDir / "empty");
+ boost::filesystem::create_directory(binDir / "empty");
+ auto p = IceTray::DatabasePoolPtr(new MockPool());
+ M2 m2(p);
+
+ BOOST_TEST_CONTEXT("4156eb45cf3b0ce1d7125b84efd8688c2d6e831d") {
+ m2.applyDiffOfFolders(1, binDir / "empty", rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d");
+ doRefreshPackageTree(db,
+ 5, 1, 482, 981, 3626, 4593, 501, 393, 238, 50, 1573, 2008, 1543, 81, 152);
+ }
- doRefreshPackageTree(sd, db, "756569aa764177340726dd3d40b41d89b11b20c7", "gentoo",
- m, 2082, 5, 1, 483, 982, 3638, 4599, 502, 393, 238, 50, 1573, 2009, 1546, 79, 152);
+ BOOST_TEST_CONTEXT("756569aa764177340726dd3d40b41d89b11b20c7") {
+ m2.applyDiffOfFolders(1, rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d", rootDir / "fixtures" / "756569aa764177340726dd3d40b41d89b11b20c7");
+ doRefreshPackageTree(db,
+ 5, 1, 483, 982, 3638, 4599, 502, 393, 238, 50, 1573, 2009, 1546, 79, 152);
+ }
- doRefreshPackageTree(sd, db, "", "gentoo",
- m, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ BOOST_TEST_CONTEXT("empty") {
+ m2.applyDiffOfFolders(1, rootDir / "fixtures" / "756569aa764177340726dd3d40b41d89b11b20c7", binDir / "empty");
+ doRefreshPackageTree(db,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+ }
}
static
@@ -150,15 +228,18 @@ BOOST_AUTO_TEST_CASE( testUpdateGitRepository )
git_remote * origin;
BOOST_REQUIRE_EQUAL(0, git_remote_create(&origin, repo, "origin", "http://git.randomdan.homeip.net/git/portage"));
git_remote_free(origin);
- create_initial_commit(repo);
+ auto commitOid = create_initial_commit(repo);
+ char commit[GIT_OID_HEXSZ + 1];
+ git_oid_tostr(commit, GIT_OID_HEXSZ, &commitOid);
git_repository_free(repo);
git_libgit2_shutdown();
// Reference it
- auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, upstream, branch) VALUES(?, ?, ?, ?)");
+ auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, upstream, branch, lastcommit) VALUES(?, ?, ?, ?, ?)");
insRepo->bindParamS(0, "testrepo");
insRepo->bindParamS(1, testRepo.string());
insRepo->bindParamS(2, "origin");
insRepo->bindParamS(3, "master");
+ insRepo->bindParamS(4, commit);
insRepo->execute();
// Update
@@ -175,10 +256,12 @@ BOOST_AUTO_TEST_CASE( testUpdateGitRepository )
BOOST_AUTO_TEST_CASE( testRefreshGitRepository )
{
auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("GentooBrowseAPI"));
- auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, lastcommit) VALUES(?, ?, ?)");
+ auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, upstream, branch, lastcommit) VALUES(?, ?, ?, ?, ?)");
insRepo->bindParamS(0, "gentoo");
insRepo->bindParamS(1, "/usr/portage");
- insRepo->bindParamS(2, "8292397bf6a8c91215b03a558e8bc843aff25b64");
+ insRepo->bindParamS(2, "origin");
+ insRepo->bindParamS(3, "master");
+ insRepo->bindParamS(4, "8292397bf6a8c91215b03a558e8bc843aff25b64");
insRepo->execute();
std::ifstream data((rootDir / "gitdata.sql").string());
db->executeScript(data, rootDir);
@@ -193,7 +276,7 @@ BOOST_AUTO_TEST_CASE( testRefreshGitRepository )
}
// Import it
- m->refreshPackageTreeGit();
+ m->refreshPackageTree();
SQL_REQUIRE_EQUAL(R"SQL(SELECT lastCommit FROM gentoobrowse.repos)SQL", std::string, "40539afe6705aee26a55bb861f5e892ae7240057");
BOOST_TEST_CONTEXT("libkgapi was moved correctly") {
diff --git a/gentoobrowse-api/util/update.cpp b/gentoobrowse-api/util/update.cpp
index 2d25530..f14df6e 100644
--- a/gentoobrowse-api/util/update.cpp
+++ b/gentoobrowse-api/util/update.cpp
@@ -8,7 +8,7 @@ int
main(int c, char ** v)
{
std::string endpoint;
- bool background, tree, bugs, sendNotifications, pull, git;
+ bool background, tree, bugs, sendNotifications, pull;
po::options_description opts("Gentoo Browse Util::Update options");
opts.add_options()
("endpoint", po::value(&endpoint)->default_value("tcp -p 9001"), "Service endpoint")
@@ -16,7 +16,6 @@ main(int c, char ** v)
("bugs", po::value(&bugs)->default_value(true), "Update bugs")
("tree", po::value(&tree)->default_value(true), "Update tree")
("pull", po::value(&pull)->default_value(true), "Perform a git pull on repositories")
- ("git", po::value(&git)->default_value(true), "Update tree using git")
("notifications", po::value(&sendNotifications)->default_value(true), "Send notification emails")
("help,h", "Show help")
;
@@ -35,9 +34,9 @@ main(int c, char ** v)
m->ice_ping();
std::set<Ice::AsyncResultPtr> jobs;
- if (pull) m->updateRepositories();
- if (tree) jobs.insert(git ? m->begin_refreshPackageTreeGit() : m->begin_refreshPackageTree());
if (bugs) jobs.insert(m->begin_refreshBugs());
+ if (pull) m->updateRepositories();
+ if (tree) jobs.insert(m->begin_refreshPackageTree());
std::for_each(jobs.begin(), jobs.end(), [](const auto & j) { j->waitForSent(); });
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 04:54:25 +0000