Import all commits regardless of age

5 files changed, 21 insertions, 17 deletions

diff --git a/gentoobrowse-api/service/maintenanceChangeLogs.cpp b/gentoobrowse-api/service/maintenanceChangeLogs.cpp
index 027429e..4a6a278 100644
--- a/gentoobrowse-api/service/maintenanceChangeLogs.cpp
+++ b/gentoobrowse-api/service/maintenanceChangeLogs.cpp
@@ -8,6 +8,7 @@
 #include <scopeExit.h>
 #include <sql/maintenance/changeLogRoots.sql.h>
 #include <sql/maintenance/changeLogInsert.sql.h>
+#include <sql/maintenance/changeLogRepoCommits.sql.h>
 #include <portage-models.h>
 #include "converters.h"
 
@@ -60,7 +61,13 @@ namespace Gentoo {
 			DB::TransactionScope tx(dbc.get());
 			auto cli = dbc->modify(sql::maintenance::changeLogInsert.getSql());
 
-			dbc->select(sql::maintenance::changeLogRoots.getSql())->forEachRow<int64_t, std::string, std::string, boost::optional<std::string>>([&cli,&c](auto repoId, const auto & repoName, const auto & repoRoot, const auto & lastCommitId) {
+			dbc->select(sql::maintenance::changeLogRoots.getSql())->forEachRow<int64_t, std::string, std::string>([&cli,&c,&dbc](auto repoId, const auto & repoName, const auto & repoRoot) {
+					std::set<std::string> processedChanges;
+					{
+						auto changes = dbc->select(sql::maintenance::changeLogRepoCommits.getSql());
+						changes->bindParamI(0, repoId);
+						changes->forEachRow<std::string>([&processedChanges](const auto & c) { processedChanges.insert(c); });
+					}
 					cli->bindParamI(0, repoId);
 					// Open repository
 					auto repo = gitSafeGet(git_repository_open_ext, git_repository_free, repoRoot.c_str(), 0, nullptr);
@@ -83,8 +90,10 @@ namespace Gentoo {
 					// Walk through revisions
 					for (; !git_revwalk_next(&oid, walker.get()); ) {
 						git_oid_tostr(str, sizeof(str), &oid);
-						if (lastCommitId && *lastCommitId == str) {
-							break;
+						auto i = processedChanges.find(str);
+						if (i != processedChanges.end()) {
+							processedChanges.erase(i);
+							continue;
 						}
 						// Get commit
 						auto commit = gitSafeGet(git_commit_lookup, git_commit_free, repo.get(), &oid);
diff --git a/gentoobrowse-api/service/sql/maintenance/changeLogRepoCommits.sql b/gentoobrowse-api/service/sql/maintenance/changeLogRepoCommits.sql
new file mode 100644
index 0000000..99f5db8
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/changeLogRepoCommits.sql
@@ -0,0 +1,3 @@
+SELECT commitid
+FROM gentoobrowse.changelog
+WHERE repoId = ?
diff --git a/gentoobrowse-api/service/sql/maintenance/changeLogRoots.sql b/gentoobrowse-api/service/sql/maintenance/changeLogRoots.sql
index ce5d3a3..ac05611 100644
--- a/gentoobrowse-api/service/sql/maintenance/changeLogRoots.sql
+++ b/gentoobrowse-api/service/sql/maintenance/changeLogRoots.sql
@@ -1,8 +1,2 @@
-SELECT r.repoid, r.name, r.path, cl.commitid
+SELECT r.repoid, r.name, r.path
 FROM gentoobrowse.repos r
-	LEFT JOIN LATERAL (
-		SELECT commitid
-		FROM gentoobrowse.changelog cl
-		WHERE cl.repoid = r.repoid
-		ORDER BY committime DESC
-		limit 1) cl ON TRUE
diff --git a/gentoobrowse-api/unittests/mockDefs.cpp b/gentoobrowse-api/unittests/mockDefs.cpp
index 3d53b42..91b8164 100644
--- a/gentoobrowse-api/unittests/mockDefs.cpp
+++ b/gentoobrowse-api/unittests/mockDefs.cpp
@@ -10,7 +10,7 @@ Service::Service() :
 
 Maintenance::Maintenance() :
 	IceTray::DryIce({
-			"--GentooBrowseAPI.ChangeLogStart.gentoo=f8e9b2f0044232213a6e0299a3f31752e8dd9d24",
+			"--GentooBrowseAPI.ChangeLogStart.gentoo=68190573ce9846f82b8cb4e8a6376758c8257184",
 			"--GentooBrowseAPI.BugRoot=file://" + (rootDir / "fixtures" / "bugs").string()
 		}),
 	PQ::Mock("user=postgres dbname=postgres", "GentooBrowseAPI", {
diff --git a/gentoobrowse-api/unittests/testChangeLog.cpp b/gentoobrowse-api/unittests/testChangeLog.cpp
index df01a7b..6cd814a 100644
--- a/gentoobrowse-api/unittests/testChangeLog.cpp
+++ b/gentoobrowse-api/unittests/testChangeLog.cpp
@@ -16,17 +16,15 @@ BOOST_AUTO_TEST_CASE( importChangeLogs )
 	insRepo->bindParamS(0, "gentoo");
 	insRepo->bindParamS(1, "/usr/portage");
 	insRepo->execute();
-	insRepo = db->modify("INSERT INTO gentoobrowse.changelog(repoid, commitid, committime, authorname, authoremail, summary, files) VALUES(1, ?, NOW(), '', '', '', ARRAY[]::TEXT[])");
-	// this an arbitrary commit from history to test we don't report things again (i.e. we should stop here)
-	insRepo->bindParamS(0, "ce9d693cedc899ee85351e8bc41b7f85db6b0d39");
-	insRepo->execute();
 
 	const int64_t expectedLogCount = 125;
-	const int64_t expectedFilesCount = 9;
+	const int64_t expectedFilesCommitCount = 1;
+	const int64_t expectedFilesCount = 2;
 	m->refreshChangeLogs();
 
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.changelog", int64_t, expectedLogCount);
-	SQL_REQUIRE_EQUAL("SELECT ARRAY_LENGTH(files, 1) FROM gentoobrowse.changelog WHERE commitid = 'f8e9b2f0044232213a6e0299a3f31752e8dd9d24'", int64_t, expectedFilesCount);
+	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.changelog WHERE commitid = '599aeeacf88dbe805a8310ba1c4c9dd70aa2baf4'", int64_t, expectedFilesCommitCount);
+	SQL_REQUIRE_EQUAL("SELECT ARRAY_LENGTH(files, 1) FROM gentoobrowse.changelog WHERE commitid = '599aeeacf88dbe805a8310ba1c4c9dd70aa2baf4'", int64_t, expectedFilesCount);
 	db->execute("COPY gentoobrowse.changelog TO '/tmp/changelog.tsv'");
 }