diff options
12 files changed, 207 insertions, 3 deletions
diff --git a/gentoobrowse-api/api/maintenance.ice b/gentoobrowse-api/api/maintenance.ice index 3a4c038..1ac95af 100644 --- a/gentoobrowse-api/api/maintenance.ice +++ b/gentoobrowse-api/api/maintenance.ice @@ -13,6 +13,7 @@ module Gentoo { idempotent void refreshPackageTree(); idempotent void refreshBugs(); idempotent void refreshChangeLogs() throws GitError; + idempotent void refreshPackageTreeGit() throws GitError; idempotent void updateRepositories() throws GitError; UserNews getUserNews(); void sendNotifications(); diff --git a/gentoobrowse-api/db/schema.sql b/gentoobrowse-api/db/schema.sql index f1bee54..d79a3dd 100644 --- a/gentoobrowse-api/db/schema.sql +++ b/gentoobrowse-api/db/schema.sql @@ -475,7 +475,8 @@ CREATE TABLE repos ( name text NOT NULL, path text NOT NULL, upstream text, - branch text + branch text, + lastcommit character(40) ); ALTER TABLE repos OWNER TO gentoo; -- Name: repos_repoid_seq; Type: SEQUENCE; Schema: gentoobrowse; Owner: gentoo diff --git a/gentoobrowse-api/service/maintenance/updatesProcessor.cpp b/gentoobrowse-api/service/maintenance/updatesProcessor.cpp index 8fbddd6..3cd4f7c 100644 --- a/gentoobrowse-api/service/maintenance/updatesProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/updatesProcessor.cpp @@ -11,6 +11,7 @@ #include <sql/maintenance/updatesMD5CacheFiles.sql.h> #include <sql/maintenance/updatesMovePackages.sql.h> #include <sql/maintenance/updatesMoveUserPackages.sql.h> +#include <sql/maintenance/updatesMoveFilelist.sql.h> #include <sql/portage/findPackage.sql.h> using namespace AdHoc::FileUtils; @@ -29,12 +30,14 @@ namespace Gentoo { class UpdatesPatch : public DB::TablePatch, public DB::SqlWriter { public: - UpdatesPatch(DB::Connection * db) : + UpdatesPatch(DB::Connection * db, bool v) : + vcsMode(v), movePackageDirFiles(sql::maintenance::updatesPackageDirFiles.modify(db)), moveMD5CacheFiles(sql::maintenance::updatesMD5CacheFiles.modify(db)), findPackage(sql::portage::findPackage.select(db)), movePackages(sql::maintenance::updatesMovePackages.modify(db)), moveUserPackages(sql::maintenance::updatesMoveUserPackages.modify(db)), + moveFilelist(sql::maintenance::updatesMoveFilelist.modify(db)), updatePacks("updatePack") { this->src = Utils::Database::emptyClone(db, "gentoobrowse.updates"); @@ -59,12 +62,14 @@ namespace Gentoo { sql.append("SPLIT_PART(b.updatePack, '-', 2)::INT, SUBSTR(b.updatePack, 1, 1)::INT"); } + const bool vcsMode; DB::ModifyCommandPtr drop; DB::ModifyCommandPtr movePackageDirFiles; DB::ModifyCommandPtr moveMD5CacheFiles; DB::SelectCommandPtr findPackage; DB::ModifyCommandPtr movePackages; DB::ModifyCommandPtr moveUserPackages; + DB::ModifyCommandPtr moveFilelist; Utils::EntityWhereFilter<std::string> updatePacks; void @@ -100,6 +105,14 @@ namespace Gentoo { movePackages->bindParamS(3, catto); movePackages->execute(); } + if (vcsMode) { + moveFilelist->bindParamS(0, catto); + moveFilelist->bindParamS(1, namefrom); + moveFilelist->bindParamS(2, nameto); + moveFilelist->bindParamS(3, catfrom); + moveFilelist->bindParamS(4, namefrom); + moveFilelist->execute(); + } }); } }; @@ -107,7 +120,7 @@ namespace Gentoo { void UpdatesProcessor::prepare(DB::Connection * dbc) { - up = new UpdatesPatch(dbc); + up = new UpdatesPatch(dbc, vcsMode); } void diff --git a/gentoobrowse-api/service/maintenanceGitOperations.cpp b/gentoobrowse-api/service/maintenanceGitOperations.cpp index 5aa3ae4..0f8adb1 100644 --- a/gentoobrowse-api/service/maintenanceGitOperations.cpp +++ b/gentoobrowse-api/service/maintenanceGitOperations.cpp @@ -10,6 +10,10 @@ #include <sql/maintenance/changeLogInsert.sql.h> #include <sql/maintenance/changeLogRepoCommits.sql.h> #include <sql/maintenance/reposToUpdate.sql.h> +#include <sql/maintenance/reposToGitRefresh.sql.h> +#include <sql/maintenance/gitListCreateRaw.sql.h> +#include <sql/maintenance/gitListCreate.sql.h> +#include <sql/maintenance/gitListCreateIdx.sql.h> #include <portage-models.h> #include "utils/git.h" #include "converters.h" @@ -126,6 +130,117 @@ namespace Gentoo { sql::maintenance::reposToUpdate.select(dbc.get())->forEachRow<std::string, std::string, std::string>(&updateRepository); } + int + insertFileChange(const git_diff_delta * delta, float, void * ptr) + { + auto ins = static_cast<DB::ModifyCommand *>(ptr); + switch (delta->status) { + case GIT_DELTA_ADDED: + ins->bindParamS(1, "A"); + ins->bindParamS(2, delta->new_file.path); + break; + case GIT_DELTA_DELETED: + ins->bindParamS(1, "D"); + ins->bindParamS(2, delta->old_file.path); + break; + case GIT_DELTA_MODIFIED: + ins->bindParamS(1, "M"); + ins->bindParamS(2, delta->new_file.path); + break; + default: + throw GitError("Insert Git file changes", 0, 0, "Unexpected change status."); + } + // fprintf(stderr, "%d %s\n", delta->status, delta->new_file.path); + ins->execute(); + return 0; + } + + void + refreshRepository(DB::Connection * db, Ice::PropertiesPtr properties, int64_t repoId, const std::string & repoName, const std::string & path, const std::string & lastCommitId) + { + auto targetRef = properties->getProperty("GentooBrowseAPI.RefreshTarget." + repoName); + auto repo = gitSafeGet(git_repository_open, git_repository_free, path.c_str()); + auto lastCommitOid = gitSafeGet(git_oid_fromstr, lastCommitId.c_str()); + auto lastCommit = gitSafeGet(git_commit_lookup, git_commit_free, repo.get(), &lastCommitOid); + auto headCommitOid = targetRef.empty() ? + gitSafeGet(git_reference_name_to_id, repo.get(), "HEAD") : + gitSafeGet(git_oid_fromstr, targetRef.c_str()); + auto headCommit = gitSafeGet(git_commit_lookup, git_commit_free, repo.get(), &headCommitOid); + + auto lastTree = gitSafeGet(git_commit_tree, git_tree_free, lastCommit.get()); + auto headTree = gitSafeGet(git_commit_tree, git_tree_free, headCommit.get()); + auto diff = gitSafeGet(git_diff_tree_to_tree, git_diff_free, repo.get(), lastTree.get(), headTree.get(), nullptr); + auto ins = db->modify("INSERT INTO filelistraw(repoid, status, filename) VALUES(?, ?, ?)"); + ins->bindParamI(0, repoId); + gitSafe(git_diff_foreach, diff.get(), insertFileChange, nullptr, nullptr, nullptr, ins.get()); + } + + void + Maintenance::refreshPackageTreeGit(const Ice::Current & c) + { + git_libgit2_init(); + AdHoc::ScopeExit shutdownlibgit2(&git_libgit2_shutdown); + + auto properties = c.adapter->getCommunicator()->getProperties(); + auto repoRoot = boost::filesystem::path(properties->getPropertyWithDefault("GentooBrowseAPI.RepoRoot", "/")); + auto dbc = db->get(); + dbc->execute("SET search_path = gentoobrowse, pg_catalog"); + DB::TransactionScope tx(dbc.get()); + updateFileTypes(dbc.get()); + sql::maintenance::gitListCreateRaw.modify(dbc.get())->execute(); + sql::maintenance::reposToGitRefresh.select(dbc.get()) + ->forEachRow<int64_t, std::string, std::string, std::string>( + boost::bind(&refreshRepository, dbc.get(), properties, _1, _2, _3, _4)); + sql::maintenance::gitListCreate.modify(dbc.get())->execute(); + sql::maintenance::gitListCreateIdx.modify(dbc.get())->execute(); + + FileProcessors fps; + for (const auto & fpf : fpfs) { + fps[fpf.first] = fpf.second(true); + } + for (const auto & fp : fps) { + fp.second->prepare(dbc.get()); + } + RepoMap repos; + dbc->select("SELECT name, repoid FROM gentoobrowse.repos")->forEachRow<std::string, int64_t>([&repos](auto name, auto id) { + repos[id] = name; + }); + std::map<std::string, FileHandleFunc> funcs; + funcs["D"] = boost::bind(&FileProcessor::deleted, _1, dbc.get(), _2, _3); + funcs["M"] = boost::bind(&FileProcessor::modified, _1, dbc.get(), _2, _3, _4); + funcs["A"] = boost::bind(&FileProcessor::created, _1, dbc.get(), _2, _3, _4); + auto phases = dbc->select("SELECT phase FROM filelist GROUP BY phase ORDER BY phase"); + for (const auto & phaseRow : phases->as<int64_t>()) { + const auto & phase = phaseRow.value<0>(); + fprintf(stderr, "Phase %ld\n", phase); + auto files = dbc->select("SELECT repoId, filename, fileTypeId, pathParts, status FROM filelist WHERE phase = ? ORDER BY POSITION(status IN 'DMA'), updateOrder NULLS LAST"); + files->bindParamI(0, phase); + std::set<int64_t> usedTypes; + for (const auto & fileRow : files->as<int64_t, std::string, int64_t, std::string, std::string>()) { + const auto & fileTypeId = fileRow.value<2>(); + if (fps.find(fileTypeId) != fps.end()) { + const auto & repoId = fileRow.value<0>(); + const auto & filename = fileRow.value<1>(); + const auto & pathParts = fileRow.value<3>(); + const auto & status = fileRow.value<4>(); + usedTypes.insert(fileRow.value<2>()); + fprintf(stderr, "%s: %s\n", status.c_str(), filename.c_str()); + this->fileHandle(fileTypeId, &fps, &repos, repoId, pathParts, repoRoot, funcs[status]); + } + } + for (const auto & fp : usedTypes) { + fps[fp]->apply(dbc.get()); + } + } + + dbc->execute("DROP TABLE filelist"); + dbc->execute("DROP TABLE filelistraw"); + for (const auto & fp : fps) { + fp.second->tidy(dbc.get()); + } + dbc->execute("SET search_path = public, pg_catalog"); + } + } } diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h index c55d74b..33cccd3 100644 --- a/gentoobrowse-api/service/maintenanceimpl.h +++ b/gentoobrowse-api/service/maintenanceimpl.h @@ -40,6 +40,7 @@ namespace Gentoo { ~Maintenance(); void refreshPackageTree(const Ice::Current &) override; + void refreshPackageTreeGit(const Ice::Current &) override; void refreshBugs(const Ice::Current &) override; void refreshChangeLogs(const Ice::Current &) override; void updateRepositories(const Ice::Current &) override; diff --git a/gentoobrowse-api/service/sql/maintenance/gitListCreate.sql b/gentoobrowse-api/service/sql/maintenance/gitListCreate.sql new file mode 100644 index 0000000..df9f947 --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/gitListCreate.sql @@ -0,0 +1,11 @@ +CREATE TEMPORARY TABLE filelist AS +SELECT + fl.repoid, + fl.status, + fl.filename, + ft.filetypeid, + ft.updateOrder, + ft.phase, + (STRING_TO_ARRAY(fl.filename, '/')) pathparts +FROM filelistraw fl, gentoobrowse.filetypes ft +WHERE (STRING_TO_ARRAY(fl.filename, '/')) ~ ft.spec diff --git a/gentoobrowse-api/service/sql/maintenance/gitListCreateIdx.sql b/gentoobrowse-api/service/sql/maintenance/gitListCreateIdx.sql new file mode 100644 index 0000000..1b3a6ca --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/gitListCreateIdx.sql @@ -0,0 +1 @@ +CREATE INDEX idx_filelist_mode ON filelist(phase, status) diff --git a/gentoobrowse-api/service/sql/maintenance/gitListCreateRaw.sql b/gentoobrowse-api/service/sql/maintenance/gitListCreateRaw.sql new file mode 100644 index 0000000..3007cd8 --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/gitListCreateRaw.sql @@ -0,0 +1,4 @@ +CREATE TEMPORARY TABLE filelistraw ( + repoid INT NOT NULL, + filename TEXT NOT NULL, + status CHARACTER(1) NOT NULL) diff --git a/gentoobrowse-api/service/sql/maintenance/reposToGitRefresh.sql b/gentoobrowse-api/service/sql/maintenance/reposToGitRefresh.sql new file mode 100644 index 0000000..068ddb9 --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/reposToGitRefresh.sql @@ -0,0 +1,4 @@ +SELECT repoid, name, path, lastcommit +FROM gentoobrowse.repos +WHERE lastcommit IS NOT NULL + diff --git a/gentoobrowse-api/service/sql/maintenance/updatesMoveFilelist.sql b/gentoobrowse-api/service/sql/maintenance/updatesMoveFilelist.sql new file mode 100644 index 0000000..0a34c82 --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/updatesMoveFilelist.sql @@ -0,0 +1,6 @@ +UPDATE filelist SET + pathparts[3] = ?, + pathparts[4] = REGEXP_REPLACE(pathparts[4], CONCAT('^', REPLACE(?, '+', '\+'), '(-[0-9])'), CONCAT(?::text, '\1')) +WHERE pathparts[3] = ? +AND pathparts[4] ~ CONCAT('^', replace(?, '+', '\+'), '-[0-9]') +AND filetypeId = 1 diff --git a/gentoobrowse-api/unittests/mockDefs.cpp b/gentoobrowse-api/unittests/mockDefs.cpp index cfe426b..ffc2f2f 100644 --- a/gentoobrowse-api/unittests/mockDefs.cpp +++ b/gentoobrowse-api/unittests/mockDefs.cpp @@ -25,6 +25,8 @@ Service::Service() : Maintenance::Maintenance() : IceTray::DryIce({ "--GentooBrowseAPI.ChangeLogStart.gentoo=68190573ce9846f82b8cb4e8a6376758c8257184", + "--GentooBrowseAPI.RefreshTarget.gentoo=40539afe6705aee26a55bb861f5e892ae7240057", + "--GentooBrowseAPI.RepoRoot=" + (rootDir / "fixtures" / "gitrepo").string(), "--GentooBrowseAPI.BugRoot=file://" + (rootDir / "fixtures" / "bugs").string() }), PQ::Mock("user=postgres dbname=postgres", "GentooBrowseAPI", { diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp index 1ee94bc..7c016fc 100644 --- a/gentoobrowse-api/unittests/testMaintenance.cpp +++ b/gentoobrowse-api/unittests/testMaintenance.cpp @@ -6,6 +6,7 @@ #include <buffer.h> #include <modifycommand.h> #include <git2.h> +#include <fstream> class MaintenanceClientCombined : public Maintenance, public TestClient { }; @@ -171,5 +172,49 @@ BOOST_AUTO_TEST_CASE( testUpdateGitRepository ) BOOST_REQUIRE(boost::filesystem::is_regular_file(testRepo / "net-misc" / "gentoobrowse-api" / "Manifest")); } +BOOST_AUTO_TEST_CASE( testRefreshGitRepository ) +{ + auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("GentooBrowseAPI")); + auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, lastcommit) VALUES(?, ?, ?)"); + insRepo->bindParamS(0, "gentoo"); + insRepo->bindParamS(1, "/usr/portage"); + insRepo->bindParamS(2, "8292397bf6a8c91215b03a558e8bc843aff25b64"); + insRepo->execute(); + std::ifstream data((rootDir / "gitdata.sql").string()); + db->executeScript(data, rootDir); + + BOOST_TEST_CONTEXT("Intiial values") { + SQL_REQUIRE_EQUAL(R"SQL(SELECT name FROM gentoobrowse.packages p WHERE p.packageid = 9)SQL", std::string, "libkgapi"); + SQL_REQUIRE_EQUAL(R"SQL(SELECT p.categoryid FROM gentoobrowse.categories c, gentoobrowse.packages p + WHERE c.categoryid = p.categoryid AND p.name = 'libkgapi' AND c.name='net-libs')SQL", int64_t, 6); + SQL_REQUIRE_EQUAL(R"SQL(SELECT COUNT(*) FROM gentoobrowse.ebuilds e WHERE e.packageid = 9)SQL", int64_t, 2); + SQL_REQUIRE_EQUAL(R"SQL(SELECT COUNT(*) FROM gentoobrowse.ebuilds e WHERE e.ebuildid in (12, 13))SQL", int64_t, 2); + SQL_REQUIRE_EQUAL(R"SQL(SELECT MIN(e.packageid) FROM gentoobrowse.ebuilds e WHERE e.ebuildid in (12, 13))SQL", int64_t, 9); + } + + // Import it + m->refreshPackageTreeGit(); + + BOOST_TEST_CONTEXT("libkgapi was moved correctly") { + BOOST_TEST_CONTEXT("from net-libs to kde-apps") { + SQL_REQUIRE_EQUAL(R"SQL(SELECT COUNT(*) FROM gentoobrowse.categories c, gentoobrowse.packages p + WHERE c.categoryid = p.categoryid AND p.name = 'libkgapi' AND c.name='kde-apps')SQL", int64_t, 1); + SQL_REQUIRE_EQUAL(R"SQL(SELECT COUNT(*) FROM gentoobrowse.categories c, gentoobrowse.packages p + WHERE c.categoryid = p.categoryid AND p.name = 'libkgapi' AND c.name='net-libs')SQL", int64_t, 0); + } + BOOST_TEST_CONTEXT("And it maintained its original packageId, but changed categoryId") { + SQL_REQUIRE_EQUAL(R"SQL(SELECT p.packageid FROM gentoobrowse.categories c, gentoobrowse.packages p + WHERE c.categoryid = p.categoryid AND p.name = 'libkgapi' AND c.name='net-libs')SQL", int64_t, 9); + SQL_REQUIRE_EQUAL(R"SQL(SELECT p.categoryid FROM gentoobrowse.categories c, gentoobrowse.packages p + WHERE c.categoryid = p.categoryid AND p.name = 'libkgapi' AND c.name='net-libs')SQL", int64_t, 8); + } + BOOST_TEST_CONTEXT("Original ebuilds in moved package were not deleted and replaced") { + SQL_REQUIRE_EQUAL(R"SQL(SELECT COUNT(*) FROM gentoobrowse.ebuilds e WHERE e.packageid = 9)SQL", int64_t, 2); + // SQL_REQUIRE_EQUAL(R"SQL(SELECT COUNT(*) FROM gentoobrowse.ebuilds e WHERE e.ebuildid in (12, 13))SQL", int64_t, 2); + // SQL_REQUIRE_EQUAL(R"SQL(SELECT MIN(e.packageid) FROM gentoobrowse.ebuilds e WHERE e.ebuildid in (12, 13))SQL", int64_t, 9); + } + } +} + BOOST_AUTO_TEST_SUITE_END(); |