Update change log along with package tree

11 files changed, 60 insertions, 132 deletions

diff --git a/gentoobrowse-api/api/maintenance.ice b/gentoobrowse-api/api/maintenance.ice
index 1ac95af..f68d371 100644
--- a/gentoobrowse-api/api/maintenance.ice
+++ b/gentoobrowse-api/api/maintenance.ice
@@ -12,7 +12,6 @@ module Gentoo {
 	interface Maintenance {
 		idempotent void refreshPackageTree();
 		idempotent void refreshBugs();
-		idempotent void refreshChangeLogs() throws GitError;
 		idempotent void refreshPackageTreeGit() throws GitError;
 		idempotent void updateRepositories() throws GitError;
 		UserNews getUserNews();
diff --git a/gentoobrowse-api/service/maintenanceCommon.cpp b/gentoobrowse-api/service/maintenanceCommon.cpp
index e7590e3..34a69a3 100644
--- a/gentoobrowse-api/service/maintenanceCommon.cpp
+++ b/gentoobrowse-api/service/maintenanceCommon.cpp
@@ -19,10 +19,8 @@ namespace Gentoo {
 					auto rb = m->begin_refreshBugs();
 					m->updateRepositories();
 					auto rptg = m->begin_refreshPackageTreeGit();
-					auto rcl = m->begin_refreshChangeLogs();
 					m->end_refreshPackageTreeGit(rptg);
 					m->sendNotifications();
-					m->end_refreshChangeLogs(rcl);
 					m->end_refreshBugs(rb);
 				}
 
diff --git a/gentoobrowse-api/service/maintenanceGitOperations.cpp b/gentoobrowse-api/service/maintenanceGitOperations.cpp
index 9d9ae61..8941dc2 100644
--- a/gentoobrowse-api/service/maintenanceGitOperations.cpp
+++ b/gentoobrowse-api/service/maintenanceGitOperations.cpp
@@ -5,9 +5,7 @@
 #include <selectcommandUtil.impl.h>
 #include <utils/dbUtils.h>
 #include <git2.h>
-#include <sql/maintenance/changeLogRoots.sql.h>
 #include <sql/maintenance/changeLogInsert.sql.h>
-#include <sql/maintenance/changeLogRepoCommits.sql.h>
 #include <sql/maintenance/reposToUpdate.sql.h>
 #include <sql/maintenance/reposToGitRefresh.sql.h>
 #include <sql/maintenance/gitListCreateRaw.sql.h>
@@ -41,83 +39,57 @@ namespace Gentoo {
 			return 0;
 		}
 
+		static
 		void
-		Maintenance::refreshChangeLogs(const Ice::Current & c)
+		writeChangeLog(DB::Connection * db, int64_t repoId, git_repository * repo, const git_oid & lastCommitOid, const git_oid & headCommitOid)
 		{
-			libgit2 git;
-
-			auto dbc = db->get();
-			DB::TransactionScope tx(dbc.get());
-			auto cli = sql::maintenance::changeLogInsert.modify(dbc.get());
-
-			sql::maintenance::changeLogRoots.select(dbc.get())->forEachRow<int64_t, std::string, std::string>([&cli,&c,&dbc](auto repoId, const auto & repoName, const auto & repoRoot) {
-					std::set<std::string> processedChanges;
-					{
-						auto changes = sql::maintenance::changeLogRepoCommits.select(dbc.get());
-						changes->bindParamI(0, repoId);
-						changes->forEachRow<std::string>([&processedChanges](const auto & c) { processedChanges.insert(c); });
-					}
-					cli->bindParamI(0, repoId);
-					// Open repository
-					auto repo = gitSafeGet(git_repository_open_ext, git_repository_free, repoRoot.c_str(), 0, nullptr);
-					// Set up walker
-					auto walker = gitSafeGet(git_revwalk_new, git_revwalk_free, repo.get());
-					auto startref = c.adapter->getCommunicator()->getProperties()
-							->getProperty("GentooBrowseAPI.ChangeLogStart." + repoName);
-					if (startref.empty()) {
-						gitSafe(git_revwalk_push_head, walker.get());
-					}
-					else {
-						git_oid oid;
-						gitSafe(git_oid_fromstr, &oid, startref.c_str());
-						gitSafe(git_revwalk_push, walker.get(), &oid);
-					}
-					git_revwalk_sorting(walker.get(), GIT_SORT_TIME);
-
-					git_oid oid;
-					char str[GIT_OID_HEXSZ + 1];
-					// Walk through revisions
-					for (; !git_revwalk_next(&oid, walker.get()); ) {
-						git_oid_tostr(str, sizeof(str), &oid);
-						auto i = processedChanges.find(str);
-						if (i != processedChanges.end()) {
-							processedChanges.erase(i);
-							continue;
-						}
-						// Get commit
-						auto commit = gitSafeGet(git_commit_lookup, git_commit_free, repo.get(), &oid);
-
-						// Get commit's tree
-						auto currentTree = gitSafeGet(git_commit_tree, git_tree_free, commit.get());
-
-						// Collect all files change in commit from all parents
-						std::unique_ptr<git_tree, void(*)(git_tree*)> parentTree(nullptr, git_tree_free);
-						if (git_commit_parentcount(commit.get()) > 0) {
-							auto parentCommit = gitSafeGet(git_commit_parent, git_commit_free, commit.get(), 0);
-							// Get parent tree
-							parentTree = gitSafeGet(git_commit_tree, git_tree_free, parentCommit.get());
-						}
-						// Get tree to tree diff
-						auto diff = gitSafeGet(git_diff_tree_to_tree, git_diff_free, repo.get(), currentTree.get(), parentTree.get(), nullptr);
-						// Compare trees
-						StringList fs;
-						git_diff_foreach(diff.get(), onFile, onBinaryFile, nullptr, nullptr, &fs);
-						// Remove duplicate mentions of files
-						std::sort(fs.begin(), fs.end());
-						fs.erase(std::unique(fs.begin(), fs.end()), fs.end());
-
-						// Insert commit into DB
-						cli->bindParamS(1, str);
-						auto sig = git_commit_author(commit.get());
-						cli->bindParamT(2, boost::posix_time::from_time_t(sig->when.time));
-						cli->bindParamS(3, git_commit_summary(commit.get()));
-						cli->bindParamS(4, git_commit_body(commit.get()));
-						cli->bindParamS(5, sig->name);
-						cli->bindParamS(6, sig->email);
-						cli->bindParamS(7, Slicer::packPqTextArray(fs));
-						cli->execute();
-					}
-				});
+			auto cli = sql::maintenance::changeLogInsert.modify(db);
+			cli->bindParamI(0, repoId);
+			// Set up walker
+			auto walker = gitSafeGet(git_revwalk_new, git_revwalk_free, repo);
+			gitSafe(git_revwalk_push, walker.get(), &headCommitOid);
+
+			git_oid oid;
+			char str[GIT_OID_HEXSZ + 1];
+			// Walk through revisions
+			while (!git_revwalk_next(&oid, walker.get())) {
+				git_oid_tostr(str, sizeof(str), &oid);
+				if (git_oid_equal(&oid, &lastCommitOid)) {
+					break;
+				}
+				// Get commit
+				auto commit = gitSafeGet(git_commit_lookup, git_commit_free, repo, &oid);
+
+				// Get commit's tree
+				auto currentTree = gitSafeGet(git_commit_tree, git_tree_free, commit.get());
+
+				// Collect all files change in commit from all parents
+				std::unique_ptr<git_tree, void(*)(git_tree*)> parentTree(nullptr, git_tree_free);
+				if (git_commit_parentcount(commit.get()) > 0) {
+					auto parentCommit = gitSafeGet(git_commit_parent, git_commit_free, commit.get(), 0);
+					// Get parent tree
+					parentTree = gitSafeGet(git_commit_tree, git_tree_free, parentCommit.get());
+				}
+				// Get tree to tree diff
+				auto diff = gitSafeGet(git_diff_tree_to_tree, git_diff_free, repo, currentTree.get(), parentTree.get(), nullptr);
+				// Compare trees
+				StringList fs;
+				git_diff_foreach(diff.get(), onFile, onBinaryFile, nullptr, nullptr, &fs);
+				// Remove duplicate mentions of files
+				std::sort(fs.begin(), fs.end());
+				fs.erase(std::unique(fs.begin(), fs.end()), fs.end());
+
+				// Insert commit into DB
+				cli->bindParamS(1, str);
+				auto sig = git_commit_author(commit.get());
+				cli->bindParamT(2, boost::posix_time::from_time_t(sig->when.time));
+				cli->bindParamS(3, git_commit_summary(commit.get()));
+				cli->bindParamS(4, git_commit_body(commit.get()));
+				cli->bindParamS(5, sig->name);
+				cli->bindParamS(6, sig->email);
+				cli->bindParamS(7, Slicer::packPqTextArray(fs));
+				cli->execute();
+			}
 		}
 
 		void
@@ -173,6 +145,7 @@ namespace Gentoo {
 			updateLastCommit->bindParamI(1, repoId);
 			updateLastCommit->execute();
 
+			writeChangeLog(db, repoId, repo.get(), lastCommitOid, headCommitOid);
 			writeChangesToFileList(db, repoId, repo.get(), lastCommitOid, headCommitOid);
 		}
 
diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h
index a1f474b..d82adc9 100644
--- a/gentoobrowse-api/service/maintenanceimpl.h
+++ b/gentoobrowse-api/service/maintenanceimpl.h
@@ -26,7 +26,6 @@ namespace Gentoo {
 				void refreshPackageTree(const Ice::Current &) override;
 				void refreshPackageTreeGit(const Ice::Current &) override;
 				void refreshBugs(const Ice::Current &) override;
-				void refreshChangeLogs(const Ice::Current &) override;
 				void updateRepositories(const Ice::Current &) override;
 				Gentoo::UserNews getUserNews(const Ice::Current &) override;
 				void sendNotifications(const Ice::Current &) override;
diff --git a/gentoobrowse-api/service/sql/maintenance/changeLogRepoCommits.sql b/gentoobrowse-api/service/sql/maintenance/changeLogRepoCommits.sql
deleted file mode 100644
index 99f5db8..0000000
--- a/gentoobrowse-api/service/sql/maintenance/changeLogRepoCommits.sql
+++ /dev/null
@@ -1,3 +0,0 @@
-SELECT commitid
-FROM gentoobrowse.changelog
-WHERE repoId = ?
diff --git a/gentoobrowse-api/service/sql/maintenance/changeLogRoots.sql b/gentoobrowse-api/service/sql/maintenance/changeLogRoots.sql
deleted file mode 100644
index 622e82f..0000000
--- a/gentoobrowse-api/service/sql/maintenance/changeLogRoots.sql
+++ /dev/null
@@ -1,3 +0,0 @@
--- libdbpp:no-cursor
-SELECT r.repoid, r.name, r.path
-FROM gentoobrowse.repos r
diff --git a/gentoobrowse-api/unittests/Jamfile.jam b/gentoobrowse-api/unittests/Jamfile.jam
index 1876d46..53d9289 100644
--- a/gentoobrowse-api/unittests/Jamfile.jam
+++ b/gentoobrowse-api/unittests/Jamfile.jam
@@ -71,15 +71,6 @@ run
 	: testDependAll ;
 
 run
-	testChangeLog.cpp
-	: : :
-	<dependency>../db/schema.sql
-	<define>BOOST_TEST_DYN_LINK
-	<library>testCommon
-	<implicit-dependency>testCommon
-	: testChangeLog ;
-
-run
 	testNews.cpp ../service/news.cpp
 	: : :
 	<define>BOOST_TEST_DYN_LINK
diff --git a/gentoobrowse-api/unittests/mockDefs.cpp b/gentoobrowse-api/unittests/mockDefs.cpp
index ffc2f2f..a067b01 100644
--- a/gentoobrowse-api/unittests/mockDefs.cpp
+++ b/gentoobrowse-api/unittests/mockDefs.cpp
@@ -24,7 +24,6 @@ Service::Service() :
 
 Maintenance::Maintenance() :
 	IceTray::DryIce({
-			"--GentooBrowseAPI.ChangeLogStart.gentoo=68190573ce9846f82b8cb4e8a6376758c8257184",
 			"--GentooBrowseAPI.RefreshTarget.gentoo=40539afe6705aee26a55bb861f5e892ae7240057",
 			"--GentooBrowseAPI.RepoRoot=" + (rootDir / "fixtures" / "gitrepo").string(),
 			"--GentooBrowseAPI.BugRoot=file://" + (rootDir / "fixtures" / "bugs").string()
diff --git a/gentoobrowse-api/unittests/testChangeLog.cpp b/gentoobrowse-api/unittests/testChangeLog.cpp
deleted file mode 100644
index 6cd814a..0000000
--- a/gentoobrowse-api/unittests/testChangeLog.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-#define BOOST_TEST_MODULE TestChangeLog
-#include <boost/test/unit_test.hpp>
-
-#include "mockDefs.h"
-#include <definedDirs.h>
-#include <modifycommand.h>
-
-BOOST_GLOBAL_FIXTURE( Maintenance );
-
-BOOST_FIXTURE_TEST_SUITE(tp, TestClient)
-
-BOOST_AUTO_TEST_CASE( importChangeLogs )
-{
-	auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("GentooBrowseAPI"));
-	auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path) VALUES(?, ?)");
-	insRepo->bindParamS(0, "gentoo");
-	insRepo->bindParamS(1, "/usr/portage");
-	insRepo->execute();
-
-	const int64_t expectedLogCount = 125;
-	const int64_t expectedFilesCommitCount = 1;
-	const int64_t expectedFilesCount = 2;
-	m->refreshChangeLogs();
-
-	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.changelog", int64_t, expectedLogCount);
-	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.changelog WHERE commitid = '599aeeacf88dbe805a8310ba1c4c9dd70aa2baf4'", int64_t, expectedFilesCommitCount);
-	SQL_REQUIRE_EQUAL("SELECT ARRAY_LENGTH(files, 1) FROM gentoobrowse.changelog WHERE commitid = '599aeeacf88dbe805a8310ba1c4c9dd70aa2baf4'", int64_t, expectedFilesCount);
-	db->execute("COPY gentoobrowse.changelog TO '/tmp/changelog.tsv'");
-}
-
-BOOST_AUTO_TEST_SUITE_END();
-
diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp
index eab8090..e05d9ad 100644
--- a/gentoobrowse-api/unittests/testMaintenance.cpp
+++ b/gentoobrowse-api/unittests/testMaintenance.cpp
@@ -215,6 +215,15 @@ BOOST_AUTO_TEST_CASE( testRefreshGitRepository )
 			SQL_REQUIRE_EQUAL(R"SQL(SELECT MIN(e.packageid) FROM gentoobrowse.ebuilds e WHERE e.ebuildid in (12, 13))SQL", int64_t, 9);
 		}
 	}
+	BOOST_TEST_CONTEXT("Change log was imported correctly") {
+		SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.changelog", int64_t, 43);
+		SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.changelog WHERE commitid = 'ca4dc75c068fcffad6ff46678dfff3fcbdf8f1d2'", int64_t, 1);
+		SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.changelog WHERE commitid = '8292397bf6a8c91215b03a558e8bc843aff25b64'", int64_t, 0);
+		SQL_REQUIRE_EQUAL("SELECT ARRAY_LENGTH(files, 1) FROM gentoobrowse.changelog WHERE commitid = 'ca4dc75c068fcffad6ff46678dfff3fcbdf8f1d2'", int64_t, 5);
+#if DUMBDB
+		db->execute("COPY gentoobrowse.changelog TO '/tmp/changelog.tsv'");
+#endif
+	}
 }
 
 BOOST_AUTO_TEST_SUITE_END();
diff --git a/gentoobrowse-api/util/update.cpp b/gentoobrowse-api/util/update.cpp
index 0bf2a1c..c54b882 100644
--- a/gentoobrowse-api/util/update.cpp
+++ b/gentoobrowse-api/util/update.cpp
@@ -8,7 +8,7 @@ int
 main(int c, char ** v)
 {
 	std::string endpoint;
-	bool background, tree, bugs, changeLog, sendNotifications, pull, git;
+	bool background, tree, bugs, sendNotifications, pull, git;
 	po::options_description opts("Gentoo Browse Util::Update options");
 	opts.add_options()
 		("endpoint", po::value(&endpoint)->default_value("tcp -p 9001"), "Service endpoint")
@@ -17,7 +17,6 @@ main(int c, char ** v)
 		("tree", po::value(&tree)->default_value(true), "Update tree")
 		("pull", po::value(&pull)->default_value(true), "Perform a git pull on repositories")
 		("git", po::value(&git)->default_value(true), "Update tree using git")
-		("changeLog", po::value(&changeLog)->default_value(true), "Update change log")
 		("notifications", po::value(&sendNotifications)->default_value(true), "Send notification emails")
 		("help,h", "Show help")
 		;
@@ -39,7 +38,6 @@ main(int c, char ** v)
 	if (pull) m->updateRepositories();
 	if (tree) jobs.insert(git ? m->begin_refreshPackageTreeGit() : m->begin_refreshPackageTree());
 	if (bugs) jobs.insert(m->begin_refreshBugs());
-	if (changeLog) jobs.insert(m->begin_refreshChangeLogs());
 
 	std::for_each(jobs.begin(), jobs.end(), [](const auto & j) { j->waitForSent(); });