From 7ab54c49b3e869cd364a542fe51df8d638565584 Mon Sep 17 00:00:00 2001 From: Dan Goodliffe Date: Mon, 1 May 2017 18:37:32 +0100 Subject: Remove support for tree comparison updates (unittests have their own variant) --- gentoobrowse-api/api/maintenance.ice | 3 +- gentoobrowse-api/db/schema.sql | 26 +-- .../service/maintenance/updatesProcessor.cpp | 19 --- gentoobrowse-api/service/maintenanceCommon.cpp | 11 +- .../service/maintenanceGitOperations.cpp | 32 ++-- .../service/maintenancePackageTree.cpp | 112 ------------- gentoobrowse-api/service/maintenanceimpl.h | 13 +- .../sql/maintenance/updatesMD5CacheFiles.sql | 6 - .../sql/maintenance/updatesPackageDirFiles.sql | 4 - gentoobrowse-api/unittests/Jamfile.jam | 7 + gentoobrowse-api/unittests/data.sql | 3 +- gentoobrowse-api/unittests/fixtures/repos.dat | 4 +- gentoobrowse-api/unittests/testMaintenance.cpp | 181 +++++++++++++++------ gentoobrowse-api/util/update.cpp | 7 +- 14 files changed, 183 insertions(+), 245 deletions(-) delete mode 100644 gentoobrowse-api/service/sql/maintenance/updatesMD5CacheFiles.sql delete mode 100644 gentoobrowse-api/service/sql/maintenance/updatesPackageDirFiles.sql diff --git a/gentoobrowse-api/api/maintenance.ice b/gentoobrowse-api/api/maintenance.ice index f68d371..be88569 100644 --- a/gentoobrowse-api/api/maintenance.ice +++ b/gentoobrowse-api/api/maintenance.ice @@ -10,9 +10,8 @@ module Gentoo { }; interface Maintenance { - idempotent void refreshPackageTree(); + idempotent void refreshPackageTree() throws GitError; idempotent void refreshBugs(); - idempotent void refreshPackageTreeGit() throws GitError; idempotent void updateRepositories() throws GitError; UserNews getUserNews(); void sendNotifications(); diff --git a/gentoobrowse-api/db/schema.sql b/gentoobrowse-api/db/schema.sql index acf0635..25a16b3 100644 --- a/gentoobrowse-api/db/schema.sql +++ b/gentoobrowse-api/db/schema.sql @@ -383,15 +383,6 @@ CREATE TABLE ebuilds ( ALTER TABLE ebuilds OWNER TO gentoo; -- Name: TABLE ebuilds; Type: COMMENT; Schema: gentoobrowse; Owner: gentoo COMMENT ON TABLE ebuilds IS 'Ebuilds :-)'; --- Name: files; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: -CREATE TABLE files ( - moddate timestamp without time zone NOT NULL, - filetypeid integer NOT NULL, - repoid integer NOT NULL, - filesize integer NOT NULL, - pathparts text[] NOT NULL -); -ALTER TABLE files OWNER TO gentoo; -- Name: filetypes; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: CREATE TABLE filetypes ( filetypeid integer NOT NULL, @@ -474,9 +465,9 @@ CREATE TABLE repos ( repoid integer NOT NULL, name text NOT NULL, path text NOT NULL, - upstream text, - branch text, - lastcommit character(40) + upstream text NOT NULL, + branch text NOT NULL, + lastcommit character(40) NOT NULL ); ALTER TABLE repos OWNER TO gentoo; -- Name: repos_repoid_seq; Type: SEQUENCE; Schema: gentoobrowse; Owner: gentoo @@ -663,9 +654,6 @@ ALTER TABLE ebuild_uses CLUSTER ON pk_ebuilduses; -- Name: pk_fileclass; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace: ALTER TABLE ONLY filetypes ADD CONSTRAINT pk_fileclass PRIMARY KEY (filetypeid); --- Name: pk_files; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace: -ALTER TABLE ONLY files - ADD CONSTRAINT pk_files PRIMARY KEY (repoid, pathparts); -- Name: pk_news; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace: ALTER TABLE ONLY news ADD CONSTRAINT pk_news PRIMARY KEY (newsid); @@ -806,9 +794,6 @@ ALTER TABLE ONLY ebuild_rdeps -- Name: fk_ebuilduses_file; Type: FK CONSTRAINT; Schema: gentoobrowse; Owner: gentoo ALTER TABLE ONLY ebuild_uses ADD CONSTRAINT fk_ebuilduses_file FOREIGN KEY (ebuildid) REFERENCES ebuilds(ebuildid) ON UPDATE CASCADE ON DELETE CASCADE; --- Name: fk_files_type; Type: FK CONSTRAINT; Schema: gentoobrowse; Owner: gentoo -ALTER TABLE ONLY files - ADD CONSTRAINT fk_files_type FOREIGN KEY (filetypeid) REFERENCES filetypes(filetypeid) ON UPDATE CASCADE; -- Name: fk_packageurls_package; Type: FK CONSTRAINT; Schema: gentoobrowse; Owner: gentoo ALTER TABLE ONLY package_urls ADD CONSTRAINT fk_packageurls_package FOREIGN KEY (packageid) REFERENCES packages(packageid) ON UPDATE CASCADE ON DELETE CASCADE; @@ -888,11 +873,6 @@ REVOKE ALL ON TABLE ebuilds FROM PUBLIC; REVOKE ALL ON TABLE ebuilds FROM gentoo; GRANT ALL ON TABLE ebuilds TO gentoo; GRANT SELECT ON TABLE ebuilds TO apache; --- Name: files; Type: ACL; Schema: gentoobrowse; Owner: gentoo -REVOKE ALL ON TABLE files FROM PUBLIC; -REVOKE ALL ON TABLE files FROM gentoo; -GRANT ALL ON TABLE files TO gentoo; -GRANT SELECT ON TABLE files TO apache; -- Name: filetypes; Type: ACL; Schema: gentoobrowse; Owner: gentoo REVOKE ALL ON TABLE filetypes FROM PUBLIC; REVOKE ALL ON TABLE filetypes FROM gentoo; diff --git a/gentoobrowse-api/service/maintenance/updatesProcessor.cpp b/gentoobrowse-api/service/maintenance/updatesProcessor.cpp index 5765c17..30e93c1 100644 --- a/gentoobrowse-api/service/maintenance/updatesProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/updatesProcessor.cpp @@ -9,8 +9,6 @@ #include #include #include -#include -#include #include #include #include @@ -36,8 +34,6 @@ namespace Gentoo { UpdatesPatch(DB::Connection * db, bool v) : vcsMode(v), categoryInsert(sql::maintenance::categoryInsert.modify(db)), - movePackageDirFiles(sql::maintenance::updatesPackageDirFiles.modify(db)), - moveMD5CacheFiles(sql::maintenance::updatesMD5CacheFiles.modify(db)), findPackage(sql::portage::findPackage.select(db)), movePackages(sql::maintenance::updatesMovePackages.modify(db)), moveUserPackages(sql::maintenance::updatesMoveUserPackages.modify(db)), @@ -69,8 +65,6 @@ namespace Gentoo { const bool vcsMode; DB::ModifyCommandPtr drop; DB::ModifyCommandPtr categoryInsert; - DB::ModifyCommandPtr movePackageDirFiles; - DB::ModifyCommandPtr moveMD5CacheFiles; DB::SelectCommandPtr findPackage; DB::ModifyCommandPtr movePackages; DB::ModifyCommandPtr moveUserPackages; @@ -93,17 +87,6 @@ namespace Gentoo { } else { // Move - movePackageDirFiles->bindParamS(0, catto); - movePackageDirFiles->bindParamS(1, nameto); - movePackageDirFiles->bindParamS(2, catfrom); - movePackageDirFiles->bindParamS(3, namefrom); - movePackageDirFiles->execute(); - moveMD5CacheFiles->bindParamS(0, catto); - moveMD5CacheFiles->bindParamS(1, namefrom); - moveMD5CacheFiles->bindParamS(2, nameto); - moveMD5CacheFiles->bindParamS(3, catfrom); - moveMD5CacheFiles->bindParamS(4, namefrom); - moveMD5CacheFiles->execute(); categoryInsert->bindParamS(0, catto); categoryInsert->bindParamS(1, catto); categoryInsert->execute(); @@ -112,8 +95,6 @@ namespace Gentoo { movePackages->bindParamS(2, catfrom); movePackages->bindParamS(3, catto); movePackages->execute(); - } - if (vcsMode) { moveFilelist->bindParamS(0, catto); moveFilelist->bindParamS(1, namefrom); moveFilelist->bindParamS(2, nameto); diff --git a/gentoobrowse-api/service/maintenanceCommon.cpp b/gentoobrowse-api/service/maintenanceCommon.cpp index 4085e85..63bb13d 100644 --- a/gentoobrowse-api/service/maintenanceCommon.cpp +++ b/gentoobrowse-api/service/maintenanceCommon.cpp @@ -18,7 +18,7 @@ namespace Gentoo { m->ice_ping(); auto rb = m->begin_refreshBugs(); m->updateRepositories(); - m->refreshPackageTreeGit(); + m->refreshPackageTree(); m->sendNotifications(); m->end_refreshBugs(rb); } @@ -28,15 +28,20 @@ namespace Gentoo { }; Maintenance::Maintenance(IceTray::DatabasePoolPtr d, Ice::CommunicatorPtr ic, Ice::PropertiesPtr p) : - IceTray::AbstractDatabaseClient(d) + Maintenance::Maintenance(d) { - setupFileProcessors(); if (auto sync = p->getPropertyAsIntWithDefault("GentooBrowseAPI.SyncTime", 0) * 60) { taskRunner = new IceUtil::Timer(); taskRunner->scheduleRepeated(new AutoUpdatePackages(ic), IceUtil::Time::seconds(sync)); } } + Maintenance::Maintenance(IceTray::DatabasePoolPtr d) : + IceTray::AbstractDatabaseClient(d) + { + setupFileProcessors(); + } + Maintenance::~Maintenance() { if (taskRunner) { diff --git a/gentoobrowse-api/service/maintenanceGitOperations.cpp b/gentoobrowse-api/service/maintenanceGitOperations.cpp index b178100..16c51d3 100644 --- a/gentoobrowse-api/service/maintenanceGitOperations.cpp +++ b/gentoobrowse-api/service/maintenanceGitOperations.cpp @@ -166,7 +166,7 @@ namespace Gentoo { } void - Maintenance::refreshPackageTreeGit(const Ice::Current & c) + Maintenance::refreshPackageTree(const Ice::Current & c) { auto properties = c.adapter->getCommunicator()->getProperties(); auto repoRoot = boost::filesystem::path(properties->getPropertyWithDefault("GentooBrowseAPI.RepoRoot", "/")); @@ -181,25 +181,35 @@ namespace Gentoo { sql::maintenance::gitListCreate.modify(dbc.get())->execute(); sql::maintenance::gitListCreateIdx.modify(dbc.get())->execute(); + applyFileChanges(dbc.get(), repoRoot); + + dbc->execute("DROP TABLE filelist"); + dbc->execute("DROP TABLE filelistraw"); + dbc->execute("SET search_path = public, pg_catalog"); + } + + void + Maintenance::applyFileChanges(DB::Connection * dbc, const boost::filesystem::path & repoRoot) const + { FileProcessors fps; for (const auto & fpf : fpfs) { fps[fpf.first] = fpf.second(true); } for (const auto & fp : fps) { - fp.second->prepare(dbc.get()); + fp.second->prepare(dbc); } RepoMap repos; dbc->select("SELECT path, repoid FROM gentoobrowse.repos")->forEachRow([&repos](auto path, auto id) { repos[id] = path; }); std::map funcs; - funcs["D"] = boost::bind(&AbstractFileProcessor::deleted, _1, dbc.get(), _2, _3); - funcs["M"] = boost::bind(&AbstractFileProcessor::modified, _1, dbc.get(), _2, _3, _4); - funcs["A"] = boost::bind(&AbstractFileProcessor::created, _1, dbc.get(), _2, _3, _4); - auto phases = sql::maintenance::filelistPhases.select(dbc.get()); + funcs["D"] = boost::bind(&AbstractFileProcessor::deleted, _1, dbc, _2, _3); + funcs["M"] = boost::bind(&AbstractFileProcessor::modified, _1, dbc, _2, _3, _4); + funcs["A"] = boost::bind(&AbstractFileProcessor::created, _1, dbc, _2, _3, _4); + auto phases = sql::maintenance::filelistPhases.select(dbc); for (const auto & phaseRow : phases->as()) { const auto & phase = phaseRow.value<0>(); - auto files = sql::maintenance::gitListChangesInPhase.select(dbc.get()); + auto files = sql::maintenance::gitListChangesInPhase.select(dbc); files->bindParamI(0, phase); std::set usedTypes; for (const auto & fileRow : files->as()) { @@ -213,16 +223,12 @@ namespace Gentoo { } } for (const auto & fp : usedTypes) { - fps[fp]->apply(dbc.get()); + fps[fp]->apply(dbc); } } - - dbc->execute("DROP TABLE filelist"); - dbc->execute("DROP TABLE filelistraw"); for (const auto & fp : fps) { - fp.second->tidy(dbc.get()); + fp.second->tidy(dbc); } - dbc->execute("SET search_path = public, pg_catalog"); } } diff --git a/gentoobrowse-api/service/maintenancePackageTree.cpp b/gentoobrowse-api/service/maintenancePackageTree.cpp index dbc72a5..9862f28 100644 --- a/gentoobrowse-api/service/maintenancePackageTree.cpp +++ b/gentoobrowse-api/service/maintenancePackageTree.cpp @@ -9,9 +9,6 @@ #include #include #include -#include -#include -#include #include "maintenance/categoryMetaProcessor.h" #include "maintenance/packageMetaProcessor.h" #include "maintenance/ebuildMetaProcessor.h" @@ -105,35 +102,6 @@ R"DATA( 10 category metadata 2 2 {"(2,metadata.xml)"} } } - Maintenance::RepoMap - Maintenance::createTempFileList(DB::Connection * dbc, const boost::filesystem::path & tmp) - { - boost::filesystem::remove_all(tmp); - boost::filesystem::create_directories(tmp); - RepoMap repos; - dbc->select("SELECT name, path, repoid FROM gentoobrowse.repos")->forEachRow([&tmp,&repos](auto n, auto p, auto id) { - boost::filesystem::create_symlink(p, tmp / n); - repos[id] = n; - }); - dbc->execute(sql::maintenance::fileListCreateRaw.getSql()); - dbc->beginBulkUpload("filelistraw", ""); - char buf[BUFSIZ]; - for (boost::filesystem::recursive_directory_iterator d(tmp, boost::filesystem::symlink_option::recurse); - d != boost::filesystem::recursive_directory_iterator(); d++) { - if (boost::filesystem::is_regular_file(d->status()) && - !boost::algorithm::ends_with(d->path().string(), ".ebuild")) { - auto len = snprintf(buf, BUFSIZ, "%s\t%ju\t%s\n", - d->path().lexically_relative(tmp).c_str(), - boost::filesystem::file_size(*d), - boost::posix_time::to_iso_extended_string( - boost::posix_time::from_time_t(boost::filesystem::last_write_time(*d))).c_str()); - dbc->bulkUploadData(buf, len); - } - } - dbc->endBulkUpload(nullptr); - return repos; - } - void Maintenance::updateFileTypes(DB::Connection * dbc) const { DB::TablePatch p; @@ -148,86 +116,6 @@ R"DATA( 10 category metadata 2 2 {"(2,metadata.xml)"} dbc->patchTable(&p); Utils::Database::drop(dbc, p.src); } - - class PhaseSourceFilter : public Utils::EntityWhereFilter { - public: - PhaseSourceFilter() : - Utils::EntityWhereFilter("filetypeId") - { - } - - void - writeSql(AdHoc::Buffer & sql) override - { - sql.append("( SELECT * FROM filelist a WHERE "); - Utils::EntityWhereFilter::writeSql(sql); - sql.append(")"); - } - }; - - void - Maintenance::processChanges(DB::Connection * dbc, const boost::filesystem::path & tmp, const RepoMap & repos) - { - FileProcessors fps; - for (const auto & fpf : fpfs) { - fps[fpf.first] = fpf.second(false); - } - for (const auto & fp : fps) { - fp.second->prepare(dbc); - } - dbc->execute(sql::maintenance::fileListCreate.getSql()); - dbc->execute(sql::maintenance::fileListCreatePk.getSql()); - DB::TablePatch tp; - tp.dest = "gentoobrowse.files"; - tp.pk = {"pathparts", "repoid"}; - tp.cols = {"pathparts", "repoid", "filetypeid", "filesize", "moddate"}; - tp.beforeDelete = boost::bind(&Maintenance::fileDeleted, this, dbc, &fps, tmp, &repos, _1); - tp.beforeUpdate = boost::bind(&Maintenance::fileChanged, this, dbc, &fps, tmp, &repos, _1); - tp.beforeInsert = boost::bind(&Maintenance::fileCreated, this, dbc, &fps, tmp, &repos, _1); - DB::StaticSqlWriter obpo("b.updateOrder NULLS LAST"); - tp.order = &obpo; - auto phases = dbc->select("SELECT phase FROM gentoobrowse.filetypes GROUP BY phase ORDER BY phase"); - for (const auto & phase : phases->as()) { - Utils::EntityWhereFilter pf("filetypeId"); - PhaseSourceFilter psf; - auto getFts = dbc->select("SELECT filetypeId FROM gentoobrowse.filetypes WHERE phase = ?"); - getFts->bindParamI(0, phase.value<0>()); - for (const auto & ft : getFts->as()) { - pf.entityIds.insert(ft.value<0>()); - psf.entityIds.insert(ft.value<0>()); - } - tp.srcExpr = &psf; - tp.where = &pf; - dbc->patchTable(&tp); - for (const auto & fp : pf.entityIds) { - if (fps.find(fp) != fps.end()) { - fps[fp]->apply(dbc); - } - } - } - dbc->execute("DROP TABLE filelist"); - dbc->execute("DROP TABLE filelistraw"); - for (const auto & fp : fps) { - fp.second->tidy(dbc); - } - } - - void - Maintenance::refreshPackageTree(const Ice::Current &) - { - auto tmp = boost::filesystem::temp_directory_path() / "import"; - AdHoc::ScopeExit tidyTmp([&tmp]{ - boost::system::error_code ec; - boost::filesystem::remove_all(tmp, ec); - }); - auto dbc = db->get(); - dbc->execute("SET search_path = gentoobrowse, pg_catalog"); - DB::TransactionScope tx(dbc.get()); - updateFileTypes(dbc.get()); - auto repos = createTempFileList(dbc.get(), tmp); - processChanges(dbc.get(), tmp, repos); - dbc->execute("SET search_path = public, pg_catalog"); - } } } diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h index d82adc9..ac74f39 100644 --- a/gentoobrowse-api/service/maintenanceimpl.h +++ b/gentoobrowse-api/service/maintenanceimpl.h @@ -13,7 +13,7 @@ namespace Gentoo { namespace Service { - class DLL_PUBLIC Maintenance : public Gentoo::Maintenance, IceTray::AbstractDatabaseClient { + class DLL_PUBLIC Maintenance : public Gentoo::Maintenance, public IceTray::AbstractDatabaseClient { public: typedef std::map> FileProcessorFactories; typedef std::map FileProcessors; @@ -21,20 +21,21 @@ namespace Gentoo { typedef std::map RepoMap; Maintenance(IceTray::DatabasePoolPtr d, Ice::CommunicatorPtr ic, Ice::PropertiesPtr p); - ~Maintenance(); + virtual ~Maintenance(); void refreshPackageTree(const Ice::Current &) override; - void refreshPackageTreeGit(const Ice::Current &) override; void refreshBugs(const Ice::Current &) override; void updateRepositories(const Ice::Current &) override; Gentoo::UserNews getUserNews(const Ice::Current &) override; void sendNotifications(const Ice::Current &) override; + protected: + Maintenance(IceTray::DatabasePoolPtr d); + void applyFileChanges(DB::Connection *, const boost::filesystem::path &) const; + void updateFileTypes(DB::Connection *) const; + private: void setupFileProcessors(); - void updateFileTypes(DB::Connection *) const; - static RepoMap createTempFileList(DB::Connection *, const boost::filesystem::path &); - void processChanges(DB::Connection *, const boost::filesystem::path &, const RepoMap & repos); void fileDeleted(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path &, const RepoMap *, DB::SelectCommandPtr s); void fileChanged(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path &, const RepoMap *, DB::SelectCommandPtr s); void fileCreated(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path & tmp, const RepoMap *, DB::SelectCommandPtr s); diff --git a/gentoobrowse-api/service/sql/maintenance/updatesMD5CacheFiles.sql b/gentoobrowse-api/service/sql/maintenance/updatesMD5CacheFiles.sql deleted file mode 100644 index 1fcbb7e..0000000 --- a/gentoobrowse-api/service/sql/maintenance/updatesMD5CacheFiles.sql +++ /dev/null @@ -1,6 +0,0 @@ -UPDATE gentoobrowse.files SET - pathparts[3] = ?, - pathparts[4] = REGEXP_REPLACE(pathparts[4], CONCAT('^', REPLACE(?, '+', '\+'), '(-[0-9])'), CONCAT(?::text, '\1')) -WHERE pathparts[3] = ? -AND pathparts[4] ~ CONCAT('^', replace(?, '+', '\+'), '-[0-9]') -AND filetypeId = 1 diff --git a/gentoobrowse-api/service/sql/maintenance/updatesPackageDirFiles.sql b/gentoobrowse-api/service/sql/maintenance/updatesPackageDirFiles.sql deleted file mode 100644 index 307484a..0000000 --- a/gentoobrowse-api/service/sql/maintenance/updatesPackageDirFiles.sql +++ /dev/null @@ -1,4 +0,0 @@ -UPDATE gentoobrowse.files SET - pathparts[1:2] = ARRAY[?, ?] -WHERE pathparts[1:2] = ARRAY[?, ?] -AND filetypeId IN (4, 8) diff --git a/gentoobrowse-api/unittests/Jamfile.jam b/gentoobrowse-api/unittests/Jamfile.jam index 53d9289..74a67df 100644 --- a/gentoobrowse-api/unittests/Jamfile.jam +++ b/gentoobrowse-api/unittests/Jamfile.jam @@ -80,12 +80,19 @@ run run testMaintenance.cpp + ../service/sql/maintenance/gitListCreate.sql + ../service/sql/maintenance/gitListCreateRaw.sql + ../service/sql/maintenance/gitListCreateIdx.sql : : : ../db/schema.sql BOOST_TEST_DYN_LINK testCommon git2 + ../service//icetray testCommon + Gentoo::Service + postgresql + ../service : testMaintenance ; run diff --git a/gentoobrowse-api/unittests/data.sql b/gentoobrowse-api/unittests/data.sql index aedf5c6..4120934 100644 --- a/gentoobrowse-api/unittests/data.sql +++ b/gentoobrowse-api/unittests/data.sql @@ -9,7 +9,7 @@ SET search_path = gentoobrowse, pg_catalog; COPY gentoobrowse.bugs (bugid, severity, status, summary, firstseen) FROM '$SCRIPTDIR/fixtures/bugs.dat'; COPY gentoobrowse.categories (categoryid, name, summary) FROM '$SCRIPTDIR/fixtures/categories.dat'; COPY gentoobrowse.packages (packageid, categoryid, name, firstseen, description, summary, maintainer, herd, image, maintainername) FROM '$SCRIPTDIR/fixtures/packages.dat'; -COPY gentoobrowse.repos (repoid, name, path) FROM '$SCRIPTDIR/fixtures/repos.dat'; +COPY gentoobrowse.repos (repoid, name, path, upstream, branch, lastcommit) FROM '$SCRIPTDIR/fixtures/repos.dat'; COPY gentoobrowse.ebuilds (ebuildid, packageid, version, versioninst, slot, license, firstseen, moddate, repoid) FROM '$SCRIPTDIR/fixtures/ebuilds.dat'; COPY gentoobrowse.ebuild_archs (ebuildid, arch) FROM '$SCRIPTDIR/fixtures/ebuild_archs.dat'; COPY gentoobrowse.ebuild_deps (ebuildid, packageid, versionspec, flags, op, slot) FROM '$SCRIPTDIR/fixtures/ebuild_deps.dat'; @@ -18,7 +18,6 @@ COPY gentoobrowse.ebuild_masks (setno, ebuildid) FROM '$SCRIPTDIR/fixtures/ebuil COPY gentoobrowse.ebuild_rdeps (ebuildid, packageid, versionspec, flags, op, slot) FROM '$SCRIPTDIR/fixtures/ebuild_rdeps.dat'; COPY gentoobrowse.ebuild_uses (ebuildid, use) FROM '$SCRIPTDIR/fixtures/ebuild_uses.dat'; COPY gentoobrowse.filetypes (filetypeid, description, spec, phase, updateOrder) FROM '$SCRIPTDIR/fixtures/filetypes.dat'; -COPY gentoobrowse.files (moddate, filetypeid, repoid, filesize, pathparts) FROM '$SCRIPTDIR/fixtures/files.dat'; COPY gentoobrowse.license (name, legalbumph) FROM '$SCRIPTDIR/fixtures/license.dat'; COPY gentoobrowse.news (newsid, title, posted, authorname, authoremail, atomspec, body, urls) FROM '$SCRIPTDIR/fixtures/news.dat'; COPY gentoobrowse.package_urls (packageid, url) FROM '$SCRIPTDIR/fixtures/package_urls.dat'; diff --git a/gentoobrowse-api/unittests/fixtures/repos.dat b/gentoobrowse-api/unittests/fixtures/repos.dat index 30d0932..072a029 100644 --- a/gentoobrowse-api/unittests/fixtures/repos.dat +++ b/gentoobrowse-api/unittests/fixtures/repos.dat @@ -1,2 +1,2 @@ -17 gentoo /usr/portage -18 RandomLAN /home/randomdan/dev/portage +17 gentoo /usr/portage origin master 87ebb8523b2890c641a27f29007ba8eaa6aa834e +18 RandomLAN /home/randomdan/dev/portage origin master 4321e561074c32ec3015417e517c6cbdb2120fcc diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp index e05d9ad..21b77db 100644 --- a/gentoobrowse-api/unittests/testMaintenance.cpp +++ b/gentoobrowse-api/unittests/testMaintenance.cpp @@ -7,56 +7,27 @@ #include #include #include +#include +#include +#include +#include class MaintenanceClientCombined : public Maintenance, public TestClient { }; BOOST_FIXTURE_TEST_SUITE(maintenance, MaintenanceClientCombined) -const boost::filesystem::path treeDir(binDir / "tree" / "gentoobrowse"); +const boost::filesystem::path treeDir(binDir / "tree"); const boost::filesystem::path fixturesDir(rootDir / "fixtures"); -class SampleData { - public: - SampleData() - { - clean(); - } - - ~SampleData() - { - clean(); - } - - void clean() - { - BOOST_TEST_CHECKPOINT("Cleaning sample data dir " << treeDir); - boost::filesystem::remove_all(treeDir); - } - - void extract(const std::string & archive, const std::string & repo) - { - boost::filesystem::create_directories(treeDir); - boost::filesystem::create_symlink(fixturesDir / archive, treeDir / repo); - } -}; - void dumpDb(DB::ConnectionPtr db); void -doRefreshPackageTree(SampleData & sd, DB::ConnectionPtr db, const std::string & archive, const std::string & dir, - Gentoo::MaintenancePrx m, int64_t files, int64_t cats, int64_t devvcs, int64_t pkgs, int64_t ebs, int64_t ebus, +doRefreshPackageTree(DB::ConnectionPtr db, int64_t cats, int64_t devvcs, int64_t pkgs, int64_t ebs, int64_t ebus, int64_t ebas, int64_t pus, int64_t ug, int64_t ul, int64_t ugs, int64_t ugds, int64_t deps, int64_t rdeps, int64_t news, int64_t masks) { - BOOST_TEST_INFO("archive: " << archive); - if (!archive.empty()) { - sd.extract(archive, dir); - } - m->refreshPackageTree(); - sd.clean(); dumpDb(db); - SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.files", int64_t, files); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, cats); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories WHERE name = 'dev-vcs'", int64_t, devvcs); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, pkgs); @@ -97,23 +68,130 @@ dumpDb(DB::ConnectionPtr db) #endif } +class M2 : public Gentoo::Service::Maintenance { + public: + typedef std::map FileMap; + + M2(IceTray::DatabasePoolPtr d) : + Gentoo::Service::Maintenance(d) + { + auto dbc = db->get(); + DB::TransactionScope tx(dbc.get()); + updateFileTypes(dbc.get()); + } + + void applyDiffOfFolders(int64_t repoId, const boost::filesystem::path & from, const boost::filesystem::path & to) const + { + fprintf(stderr, "%s\n", __PRETTY_FUNCTION__); + auto dbc = db->get(); + auto fromFiles = fileSet(from); + auto toFiles = fileSet(to); + dbc->execute("SET search_path = gentoobrowse, pg_catalog"); + Gentoo::Service::sql::maintenance::gitListCreateRaw.modify(dbc.get())->execute(); + auto ins = dbc->modify("INSERT INTO filelistraw(repoid, status, filename) VALUES(?, ?, ?)"); + ins->bindParamI(0, repoId); + newFiles(ins, fromFiles, toFiles); + removedFiles(ins, fromFiles, toFiles); + changedFiles(ins, fromFiles, toFiles); + Gentoo::Service::sql::maintenance::gitListCreate.modify(dbc.get())->execute(); + Gentoo::Service::sql::maintenance::gitListCreateIdx.modify(dbc.get())->execute(); + boost::filesystem::remove(treeDir); + boost::filesystem::create_symlink(to, treeDir); + applyFileChanges(dbc.get(), "/"); + boost::filesystem::remove(treeDir); + dbc->execute("DROP TABLE filelist"); + dbc->execute("DROP TABLE filelistraw"); + dbc->execute("SET search_path = public, pg_catalog"); + } + + static FileMap fileSet(const boost::filesystem::path & p) + { + FileMap found; + for (boost::filesystem::recursive_directory_iterator r(p); r != boost::filesystem::recursive_directory_iterator(); r++) { + if (boost::filesystem::is_regular(r->status())) { + found[boost::filesystem::relative(*r, p)] = boost::filesystem::file_size(*r); + } + } + return found; + } + static void newFiles(DB::ModifyCommandPtr ins, const FileMap & from, const FileMap & to) + { + ins->bindParamS(1, "A"); + for(const auto & f : to) { + if (from.find(f.first) == from.end()) { + fprintf(stderr, "Added %s\n", f.first.c_str()); + ins->bindParamS(2, f.first.string()); + ins->execute(); + } + } + } + static void removedFiles(DB::ModifyCommandPtr ins, const FileMap & from, const FileMap & to) + { + ins->bindParamS(1, "D"); + for(const auto & f : from) { + if (to.find(f.first) == to.end()) { + fprintf(stderr, "Removed %s\n", f.first.c_str()); + ins->bindParamS(2, f.first.string()); + ins->execute(); + } + } + } + static void changedFiles(DB::ModifyCommandPtr ins, const FileMap & from, const FileMap & to) + { + ins->bindParamS(1, "M"); + for(const auto & f : to) { + auto i = from.find(f.first); + if (i != from.end() && i->second != f.second) { + fprintf(stderr, "Changed %s\n", f.first.c_str()); + ins->bindParamS(2, f.first.string()); + ins->execute(); + } + } + } +}; + +class MockPool : public IceTray::DatabasePool { + public: + MockPool() : IceTray::DatabasePool(10, 2) {} + DB::Connection * createResource() const override + { + return DB::MockDatabase::openConnectionTo("GentooBrowseAPI"); + } +}; + BOOST_AUTO_TEST_CASE( refreshPackageTree ) { - SampleData sd; auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("GentooBrowseAPI")); - auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path) VALUES(?, ?)"); + auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, upstream, branch, lastcommit) VALUES(?, ?, ?, ?, ?)"); insRepo->bindParamS(0, "gentoo"); - insRepo->bindParamS(1, (treeDir / "gentoo").string()); + insRepo->bindParamS(1, treeDir.string()); + insRepo->bindParamS(2, "origin"); + insRepo->bindParamS(3, "master"); + insRepo->bindParamS(4, "8292397bf6a8c91215b03a558e8bc843aff25b64"); insRepo->execute(); - doRefreshPackageTree(sd, db, "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d", "gentoo", - m, 2080, 5, 1, 482, 981, 3626, 4593, 501, 393, 238, 50, 1573, 2008, 1543, 81, 152); + boost::filesystem::remove_all(binDir / "empty"); + boost::filesystem::create_directory(binDir / "empty"); + auto p = IceTray::DatabasePoolPtr(new MockPool()); + M2 m2(p); + + BOOST_TEST_CONTEXT("4156eb45cf3b0ce1d7125b84efd8688c2d6e831d") { + m2.applyDiffOfFolders(1, binDir / "empty", rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d"); + doRefreshPackageTree(db, + 5, 1, 482, 981, 3626, 4593, 501, 393, 238, 50, 1573, 2008, 1543, 81, 152); + } - doRefreshPackageTree(sd, db, "756569aa764177340726dd3d40b41d89b11b20c7", "gentoo", - m, 2082, 5, 1, 483, 982, 3638, 4599, 502, 393, 238, 50, 1573, 2009, 1546, 79, 152); + BOOST_TEST_CONTEXT("756569aa764177340726dd3d40b41d89b11b20c7") { + m2.applyDiffOfFolders(1, rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d", rootDir / "fixtures" / "756569aa764177340726dd3d40b41d89b11b20c7"); + doRefreshPackageTree(db, + 5, 1, 483, 982, 3638, 4599, 502, 393, 238, 50, 1573, 2009, 1546, 79, 152); + } - doRefreshPackageTree(sd, db, "", "gentoo", - m, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + BOOST_TEST_CONTEXT("empty") { + m2.applyDiffOfFolders(1, rootDir / "fixtures" / "756569aa764177340726dd3d40b41d89b11b20c7", binDir / "empty"); + doRefreshPackageTree(db, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + } } static @@ -150,15 +228,18 @@ BOOST_AUTO_TEST_CASE( testUpdateGitRepository ) git_remote * origin; BOOST_REQUIRE_EQUAL(0, git_remote_create(&origin, repo, "origin", "http://git.randomdan.homeip.net/git/portage")); git_remote_free(origin); - create_initial_commit(repo); + auto commitOid = create_initial_commit(repo); + char commit[GIT_OID_HEXSZ + 1]; + git_oid_tostr(commit, GIT_OID_HEXSZ, &commitOid); git_repository_free(repo); git_libgit2_shutdown(); // Reference it - auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, upstream, branch) VALUES(?, ?, ?, ?)"); + auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, upstream, branch, lastcommit) VALUES(?, ?, ?, ?, ?)"); insRepo->bindParamS(0, "testrepo"); insRepo->bindParamS(1, testRepo.string()); insRepo->bindParamS(2, "origin"); insRepo->bindParamS(3, "master"); + insRepo->bindParamS(4, commit); insRepo->execute(); // Update @@ -175,10 +256,12 @@ BOOST_AUTO_TEST_CASE( testUpdateGitRepository ) BOOST_AUTO_TEST_CASE( testRefreshGitRepository ) { auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("GentooBrowseAPI")); - auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, lastcommit) VALUES(?, ?, ?)"); + auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, upstream, branch, lastcommit) VALUES(?, ?, ?, ?, ?)"); insRepo->bindParamS(0, "gentoo"); insRepo->bindParamS(1, "/usr/portage"); - insRepo->bindParamS(2, "8292397bf6a8c91215b03a558e8bc843aff25b64"); + insRepo->bindParamS(2, "origin"); + insRepo->bindParamS(3, "master"); + insRepo->bindParamS(4, "8292397bf6a8c91215b03a558e8bc843aff25b64"); insRepo->execute(); std::ifstream data((rootDir / "gitdata.sql").string()); db->executeScript(data, rootDir); @@ -193,7 +276,7 @@ BOOST_AUTO_TEST_CASE( testRefreshGitRepository ) } // Import it - m->refreshPackageTreeGit(); + m->refreshPackageTree(); SQL_REQUIRE_EQUAL(R"SQL(SELECT lastCommit FROM gentoobrowse.repos)SQL", std::string, "40539afe6705aee26a55bb861f5e892ae7240057"); BOOST_TEST_CONTEXT("libkgapi was moved correctly") { diff --git a/gentoobrowse-api/util/update.cpp b/gentoobrowse-api/util/update.cpp index 2d25530..f14df6e 100644 --- a/gentoobrowse-api/util/update.cpp +++ b/gentoobrowse-api/util/update.cpp @@ -8,7 +8,7 @@ int main(int c, char ** v) { std::string endpoint; - bool background, tree, bugs, sendNotifications, pull, git; + bool background, tree, bugs, sendNotifications, pull; po::options_description opts("Gentoo Browse Util::Update options"); opts.add_options() ("endpoint", po::value(&endpoint)->default_value("tcp -p 9001"), "Service endpoint") @@ -16,7 +16,6 @@ main(int c, char ** v) ("bugs", po::value(&bugs)->default_value(true), "Update bugs") ("tree", po::value(&tree)->default_value(true), "Update tree") ("pull", po::value(&pull)->default_value(true), "Perform a git pull on repositories") - ("git", po::value(&git)->default_value(true), "Update tree using git") ("notifications", po::value(&sendNotifications)->default_value(true), "Send notification emails") ("help,h", "Show help") ; @@ -35,9 +34,9 @@ main(int c, char ** v) m->ice_ping(); std::set jobs; - if (pull) m->updateRepositories(); - if (tree) jobs.insert(git ? m->begin_refreshPackageTreeGit() : m->begin_refreshPackageTree()); if (bugs) jobs.insert(m->begin_refreshBugs()); + if (pull) m->updateRepositories(); + if (tree) jobs.insert(m->begin_refreshPackageTree()); std::for_each(jobs.begin(), jobs.end(), [](const auto & j) { j->waitForSent(); }); -- cgit v1.2.3