diff options
5 files changed, 21 insertions, 17 deletions
diff --git a/gentoobrowse-api/service/maintenanceChangeLogs.cpp b/gentoobrowse-api/service/maintenanceChangeLogs.cpp index 027429e..4a6a278 100644 --- a/gentoobrowse-api/service/maintenanceChangeLogs.cpp +++ b/gentoobrowse-api/service/maintenanceChangeLogs.cpp @@ -8,6 +8,7 @@ #include <scopeExit.h> #include <sql/maintenance/changeLogRoots.sql.h> #include <sql/maintenance/changeLogInsert.sql.h> +#include <sql/maintenance/changeLogRepoCommits.sql.h> #include <portage-models.h> #include "converters.h" @@ -60,7 +61,13 @@ namespace Gentoo { DB::TransactionScope tx(dbc.get()); auto cli = dbc->modify(sql::maintenance::changeLogInsert.getSql()); - dbc->select(sql::maintenance::changeLogRoots.getSql())->forEachRow<int64_t, std::string, std::string, boost::optional<std::string>>([&cli,&c](auto repoId, const auto & repoName, const auto & repoRoot, const auto & lastCommitId) { + dbc->select(sql::maintenance::changeLogRoots.getSql())->forEachRow<int64_t, std::string, std::string>([&cli,&c,&dbc](auto repoId, const auto & repoName, const auto & repoRoot) { + std::set<std::string> processedChanges; + { + auto changes = dbc->select(sql::maintenance::changeLogRepoCommits.getSql()); + changes->bindParamI(0, repoId); + changes->forEachRow<std::string>([&processedChanges](const auto & c) { processedChanges.insert(c); }); + } cli->bindParamI(0, repoId); // Open repository auto repo = gitSafeGet(git_repository_open_ext, git_repository_free, repoRoot.c_str(), 0, nullptr); @@ -83,8 +90,10 @@ namespace Gentoo { // Walk through revisions for (; !git_revwalk_next(&oid, walker.get()); ) { git_oid_tostr(str, sizeof(str), &oid); - if (lastCommitId && *lastCommitId == str) { - break; + auto i = processedChanges.find(str); + if (i != processedChanges.end()) { + processedChanges.erase(i); + continue; } // Get commit auto commit = gitSafeGet(git_commit_lookup, git_commit_free, repo.get(), &oid); diff --git a/gentoobrowse-api/service/sql/maintenance/changeLogRepoCommits.sql b/gentoobrowse-api/service/sql/maintenance/changeLogRepoCommits.sql new file mode 100644 index 0000000..99f5db8 --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/changeLogRepoCommits.sql @@ -0,0 +1,3 @@ +SELECT commitid +FROM gentoobrowse.changelog +WHERE repoId = ? diff --git a/gentoobrowse-api/service/sql/maintenance/changeLogRoots.sql b/gentoobrowse-api/service/sql/maintenance/changeLogRoots.sql index ce5d3a3..ac05611 100644 --- a/gentoobrowse-api/service/sql/maintenance/changeLogRoots.sql +++ b/gentoobrowse-api/service/sql/maintenance/changeLogRoots.sql @@ -1,8 +1,2 @@ -SELECT r.repoid, r.name, r.path, cl.commitid +SELECT r.repoid, r.name, r.path FROM gentoobrowse.repos r - LEFT JOIN LATERAL ( - SELECT commitid - FROM gentoobrowse.changelog cl - WHERE cl.repoid = r.repoid - ORDER BY committime DESC - limit 1) cl ON TRUE diff --git a/gentoobrowse-api/unittests/mockDefs.cpp b/gentoobrowse-api/unittests/mockDefs.cpp index 3d53b42..91b8164 100644 --- a/gentoobrowse-api/unittests/mockDefs.cpp +++ b/gentoobrowse-api/unittests/mockDefs.cpp @@ -10,7 +10,7 @@ Service::Service() : Maintenance::Maintenance() : IceTray::DryIce({ - "--GentooBrowseAPI.ChangeLogStart.gentoo=f8e9b2f0044232213a6e0299a3f31752e8dd9d24", + "--GentooBrowseAPI.ChangeLogStart.gentoo=68190573ce9846f82b8cb4e8a6376758c8257184", "--GentooBrowseAPI.BugRoot=file://" + (rootDir / "fixtures" / "bugs").string() }), PQ::Mock("user=postgres dbname=postgres", "GentooBrowseAPI", { diff --git a/gentoobrowse-api/unittests/testChangeLog.cpp b/gentoobrowse-api/unittests/testChangeLog.cpp index df01a7b..6cd814a 100644 --- a/gentoobrowse-api/unittests/testChangeLog.cpp +++ b/gentoobrowse-api/unittests/testChangeLog.cpp @@ -16,17 +16,15 @@ BOOST_AUTO_TEST_CASE( importChangeLogs ) insRepo->bindParamS(0, "gentoo"); insRepo->bindParamS(1, "/usr/portage"); insRepo->execute(); - insRepo = db->modify("INSERT INTO gentoobrowse.changelog(repoid, commitid, committime, authorname, authoremail, summary, files) VALUES(1, ?, NOW(), '', '', '', ARRAY[]::TEXT[])"); - // this an arbitrary commit from history to test we don't report things again (i.e. we should stop here) - insRepo->bindParamS(0, "ce9d693cedc899ee85351e8bc41b7f85db6b0d39"); - insRepo->execute(); const int64_t expectedLogCount = 125; - const int64_t expectedFilesCount = 9; + const int64_t expectedFilesCommitCount = 1; + const int64_t expectedFilesCount = 2; m->refreshChangeLogs(); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.changelog", int64_t, expectedLogCount); - SQL_REQUIRE_EQUAL("SELECT ARRAY_LENGTH(files, 1) FROM gentoobrowse.changelog WHERE commitid = 'f8e9b2f0044232213a6e0299a3f31752e8dd9d24'", int64_t, expectedFilesCount); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.changelog WHERE commitid = '599aeeacf88dbe805a8310ba1c4c9dd70aa2baf4'", int64_t, expectedFilesCommitCount); + SQL_REQUIRE_EQUAL("SELECT ARRAY_LENGTH(files, 1) FROM gentoobrowse.changelog WHERE commitid = '599aeeacf88dbe805a8310ba1c4c9dd70aa2baf4'", int64_t, expectedFilesCount); db->execute("COPY gentoobrowse.changelog TO '/tmp/changelog.tsv'"); } |