Initial WIP commit of refreshPackageTreeGit, some tests still fail

12 files changed, 207 insertions, 3 deletions

diff --git a/gentoobrowse-api/api/maintenance.ice b/gentoobrowse-api/api/maintenance.ice
index 3a4c038..1ac95af 100644
--- a/gentoobrowse-api/api/maintenance.ice
+++ b/gentoobrowse-api/api/maintenance.ice
@@ -13,6 +13,7 @@ module Gentoo {
 		idempotent void refreshPackageTree();
 		idempotent void refreshBugs();
 		idempotent void refreshChangeLogs() throws GitError;
+		idempotent void refreshPackageTreeGit() throws GitError;
 		idempotent void updateRepositories() throws GitError;
 		UserNews getUserNews();
 		void sendNotifications();
diff --git a/gentoobrowse-api/db/schema.sql b/gentoobrowse-api/db/schema.sql
index f1bee54..d79a3dd 100644
--- a/gentoobrowse-api/db/schema.sql
+++ b/gentoobrowse-api/db/schema.sql
@@ -475,7 +475,8 @@ CREATE TABLE repos (
     name text NOT NULL,
 		path text NOT NULL,
 		upstream text,
-		branch text
+		branch text,
+		lastcommit character(40)
 );
 ALTER TABLE repos OWNER TO gentoo;
 -- Name: repos_repoid_seq; Type: SEQUENCE; Schema: gentoobrowse; Owner: gentoo
diff --git a/gentoobrowse-api/service/maintenance/updatesProcessor.cpp b/gentoobrowse-api/service/maintenance/updatesProcessor.cpp
index 8fbddd6..3cd4f7c 100644
--- a/gentoobrowse-api/service/maintenance/updatesProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/updatesProcessor.cpp
@@ -11,6 +11,7 @@
 #include <sql/maintenance/updatesMD5CacheFiles.sql.h>
 #include <sql/maintenance/updatesMovePackages.sql.h>
 #include <sql/maintenance/updatesMoveUserPackages.sql.h>
+#include <sql/maintenance/updatesMoveFilelist.sql.h>
 #include <sql/portage/findPackage.sql.h>
 
 using namespace AdHoc::FileUtils;
@@ -29,12 +30,14 @@ namespace Gentoo {
 
 		class UpdatesPatch : public DB::TablePatch, public DB::SqlWriter {
 			public:
-				UpdatesPatch(DB::Connection * db) :
+				UpdatesPatch(DB::Connection * db, bool v) :
+					vcsMode(v),
 					movePackageDirFiles(sql::maintenance::updatesPackageDirFiles.modify(db)),
 					moveMD5CacheFiles(sql::maintenance::updatesMD5CacheFiles.modify(db)),
 					findPackage(sql::portage::findPackage.select(db)),
 					movePackages(sql::maintenance::updatesMovePackages.modify(db)),
 					moveUserPackages(sql::maintenance::updatesMoveUserPackages.modify(db)),
+					moveFilelist(sql::maintenance::updatesMoveFilelist.modify(db)),
 					updatePacks("updatePack")
 				{
 					this->src = Utils::Database::emptyClone(db, "gentoobrowse.updates");
@@ -59,12 +62,14 @@ namespace Gentoo {
 					sql.append("SPLIT_PART(b.updatePack, '-', 2)::INT, SUBSTR(b.updatePack, 1, 1)::INT");
 				}
 
+				const bool vcsMode;
 				DB::ModifyCommandPtr drop;
 				DB::ModifyCommandPtr movePackageDirFiles;
 				DB::ModifyCommandPtr moveMD5CacheFiles;
 				DB::SelectCommandPtr findPackage;
 				DB::ModifyCommandPtr movePackages;
 				DB::ModifyCommandPtr moveUserPackages;
+				DB::ModifyCommandPtr moveFilelist;
 				Utils::EntityWhereFilter<std::string> updatePacks;
 
 				void
@@ -100,6 +105,14 @@ namespace Gentoo {
 							movePackages->bindParamS(3, catto);
 							movePackages->execute();
 						}
+						if (vcsMode) {
+							moveFilelist->bindParamS(0, catto);
+							moveFilelist->bindParamS(1, namefrom);
+							moveFilelist->bindParamS(2, nameto);
+							moveFilelist->bindParamS(3, catfrom);
+							moveFilelist->bindParamS(4, namefrom);
+							moveFilelist->execute();
+						}
 					});
 				}
 		};
@@ -107,7 +120,7 @@ namespace Gentoo {
 		void
 		UpdatesProcessor::prepare(DB::Connection * dbc)
 		{
-			up = new UpdatesPatch(dbc);
+			up = new UpdatesPatch(dbc, vcsMode);
 		}
 
 		void
diff --git a/gentoobrowse-api/service/maintenanceGitOperations.cpp b/gentoobrowse-api/service/maintenanceGitOperations.cpp
index 5aa3ae4..0f8adb1 100644
--- a/gentoobrowse-api/service/maintenanceGitOperations.cpp
+++ b/gentoobrowse-api/service/maintenanceGitOperations.cpp
@@ -10,6 +10,10 @@
 #include <sql/maintenance/changeLogInsert.sql.h>
 #include <sql/maintenance/changeLogRepoCommits.sql.h>
 #include <sql/maintenance/reposToUpdate.sql.h>
+#include <sql/maintenance/reposToGitRefresh.sql.h>
+#include <sql/maintenance/gitListCreateRaw.sql.h>
+#include <sql/maintenance/gitListCreate.sql.h>
+#include <sql/maintenance/gitListCreateIdx.sql.h>
 #include <portage-models.h>
 #include "utils/git.h"
 #include "converters.h"
@@ -126,6 +130,117 @@ namespace Gentoo {
 			sql::maintenance::reposToUpdate.select(dbc.get())->forEachRow<std::string, std::string, std::string>(&updateRepository);
 		}
 
+		int
+		insertFileChange(const git_diff_delta * delta, float, void * ptr)
+		{
+			auto ins = static_cast<DB::ModifyCommand *>(ptr);
+			switch (delta->status) {
+				case GIT_DELTA_ADDED:
+					ins->bindParamS(1, "A");
+					ins->bindParamS(2, delta->new_file.path);
+					break;
+				case GIT_DELTA_DELETED:
+					ins->bindParamS(1, "D");
+					ins->bindParamS(2, delta->old_file.path);
+					break;
+				case GIT_DELTA_MODIFIED:
+					ins->bindParamS(1, "M");
+					ins->bindParamS(2, delta->new_file.path);
+					break;
+				default:
+					throw GitError("Insert Git file changes", 0, 0, "Unexpected change status.");
+			}
+			// fprintf(stderr, "%d %s\n", delta->status, delta->new_file.path);
+			ins->execute();
+			return 0;
+		}
+
+		void
+		refreshRepository(DB::Connection * db, Ice::PropertiesPtr properties, int64_t repoId, const std::string & repoName, const std::string & path, const std::string & lastCommitId)
+		{
+			auto targetRef = properties->getProperty("GentooBrowseAPI.RefreshTarget." + repoName);
+			auto repo = gitSafeGet(git_repository_open, git_repository_free, path.c_str());
+			auto lastCommitOid = gitSafeGet(git_oid_fromstr, lastCommitId.c_str());
+			auto lastCommit = gitSafeGet(git_commit_lookup, git_commit_free, repo.get(), &lastCommitOid);
+			auto headCommitOid = targetRef.empty() ?
+				gitSafeGet(git_reference_name_to_id, repo.get(), "HEAD") :
+				gitSafeGet(git_oid_fromstr, targetRef.c_str());
+			auto headCommit = gitSafeGet(git_commit_lookup, git_commit_free, repo.get(), &headCommitOid);
+
+			auto lastTree = gitSafeGet(git_commit_tree, git_tree_free, lastCommit.get());
+			auto headTree = gitSafeGet(git_commit_tree, git_tree_free, headCommit.get());
+			auto diff = gitSafeGet(git_diff_tree_to_tree, git_diff_free, repo.get(), lastTree.get(), headTree.get(), nullptr);
+			auto ins = db->modify("INSERT INTO filelistraw(repoid, status, filename) VALUES(?, ?, ?)");
+			ins->bindParamI(0, repoId);
+			gitSafe(git_diff_foreach, diff.get(), insertFileChange, nullptr, nullptr, nullptr, ins.get());
+		}
+
+		void
+		Maintenance::refreshPackageTreeGit(const Ice::Current & c)
+		{
+			git_libgit2_init();
+			AdHoc::ScopeExit shutdownlibgit2(&git_libgit2_shutdown);
+
+			auto properties = c.adapter->getCommunicator()->getProperties();
+			auto repoRoot = boost::filesystem::path(properties->getPropertyWithDefault("GentooBrowseAPI.RepoRoot", "/"));
+			auto dbc = db->get();
+			dbc->execute("SET search_path = gentoobrowse, pg_catalog");
+			DB::TransactionScope tx(dbc.get());
+			updateFileTypes(dbc.get());
+			sql::maintenance::gitListCreateRaw.modify(dbc.get())->execute();
+			sql::maintenance::reposToGitRefresh.select(dbc.get())
+				->forEachRow<int64_t, std::string, std::string, std::string>(
+						boost::bind(&refreshRepository, dbc.get(), properties, _1, _2, _3, _4));
+			sql::maintenance::gitListCreate.modify(dbc.get())->execute();
+			sql::maintenance::gitListCreateIdx.modify(dbc.get())->execute();
+
+			FileProcessors fps;
+			for (const auto & fpf : fpfs) {
+				fps[fpf.first] = fpf.second(true);
+			}
+			for (const auto & fp : fps) {
+				fp.second->prepare(dbc.get());
+			}
+			RepoMap repos;
+			dbc->select("SELECT name, repoid FROM gentoobrowse.repos")->forEachRow<std::string, int64_t>([&repos](auto name, auto id) {
+					repos[id] = name;
+				});
+			std::map<std::string, FileHandleFunc> funcs;
+			funcs["D"] = boost::bind(&FileProcessor::deleted, _1, dbc.get(), _2, _3);
+			funcs["M"] = boost::bind(&FileProcessor::modified, _1, dbc.get(), _2, _3, _4);
+			funcs["A"] = boost::bind(&FileProcessor::created, _1, dbc.get(), _2, _3, _4);
+			auto phases = dbc->select("SELECT phase FROM filelist GROUP BY phase ORDER BY phase");
+			for (const auto & phaseRow : phases->as<int64_t>()) {
+				const auto & phase = phaseRow.value<0>();
+				fprintf(stderr, "Phase %ld\n", phase);
+				auto files = dbc->select("SELECT repoId, filename, fileTypeId, pathParts, status FROM filelist WHERE phase = ? ORDER BY POSITION(status IN 'DMA'), updateOrder NULLS LAST");
+				files->bindParamI(0, phase);
+				std::set<int64_t> usedTypes;
+				for (const auto & fileRow : files->as<int64_t, std::string, int64_t, std::string, std::string>()) {
+					const auto & fileTypeId = fileRow.value<2>();
+					if (fps.find(fileTypeId) != fps.end()) {
+						const auto & repoId = fileRow.value<0>();
+						const auto & filename = fileRow.value<1>();
+						const auto & pathParts = fileRow.value<3>();
+						const auto & status = fileRow.value<4>();
+						usedTypes.insert(fileRow.value<2>());
+						fprintf(stderr, "%s: %s\n", status.c_str(), filename.c_str());
+						this->fileHandle(fileTypeId, &fps, &repos, repoId, pathParts, repoRoot, funcs[status]);
+					}
+				}
+				for (const auto & fp : usedTypes) {
+					fps[fp]->apply(dbc.get());
+				}
+			}
+
+			dbc->execute("DROP TABLE filelist");
+			dbc->execute("DROP TABLE filelistraw");
+			for (const auto & fp : fps) {
+				fp.second->tidy(dbc.get());
+			}
+			dbc->execute("SET search_path = public, pg_catalog");
+		}
+
 	}
 }
 
diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h
index c55d74b..33cccd3 100644
--- a/gentoobrowse-api/service/maintenanceimpl.h
+++ b/gentoobrowse-api/service/maintenanceimpl.h
@@ -40,6 +40,7 @@ namespace Gentoo {
 				~Maintenance();
 
 				void refreshPackageTree(const Ice::Current &) override;
+				void refreshPackageTreeGit(const Ice::Current &) override;
 				void refreshBugs(const Ice::Current &) override;
 				void refreshChangeLogs(const Ice::Current &) override;
 				void updateRepositories(const Ice::Current &) override;
diff --git a/gentoobrowse-api/service/sql/maintenance/gitListCreate.sql b/gentoobrowse-api/service/sql/maintenance/gitListCreate.sql
new file mode 100644
index 0000000..df9f947
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/gitListCreate.sql
@@ -0,0 +1,11 @@
+CREATE TEMPORARY TABLE filelist AS
+SELECT
+	fl.repoid,
+	fl.status,
+	fl.filename,
+	ft.filetypeid,
+	ft.updateOrder,
+	ft.phase,
+	(STRING_TO_ARRAY(fl.filename, '/')) pathparts
+FROM filelistraw fl, gentoobrowse.filetypes ft
+WHERE (STRING_TO_ARRAY(fl.filename, '/')) ~ ft.spec
diff --git a/gentoobrowse-api/service/sql/maintenance/gitListCreateIdx.sql b/gentoobrowse-api/service/sql/maintenance/gitListCreateIdx.sql
new file mode 100644
index 0000000..1b3a6ca
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/gitListCreateIdx.sql
@@ -0,0 +1 @@
+CREATE INDEX idx_filelist_mode ON filelist(phase, status)
diff --git a/gentoobrowse-api/service/sql/maintenance/gitListCreateRaw.sql b/gentoobrowse-api/service/sql/maintenance/gitListCreateRaw.sql
new file mode 100644
index 0000000..3007cd8
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/gitListCreateRaw.sql
@@ -0,0 +1,4 @@
+CREATE TEMPORARY TABLE filelistraw (
+		repoid INT NOT NULL,
+		filename TEXT NOT NULL,
+		status CHARACTER(1) NOT NULL)
diff --git a/gentoobrowse-api/service/sql/maintenance/reposToGitRefresh.sql b/gentoobrowse-api/service/sql/maintenance/reposToGitRefresh.sql
new file mode 100644
index 0000000..068ddb9
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/reposToGitRefresh.sql
@@ -0,0 +1,4 @@
+SELECT repoid, name, path, lastcommit
+FROM gentoobrowse.repos
+WHERE lastcommit IS NOT NULL
+
diff --git a/gentoobrowse-api/service/sql/maintenance/updatesMoveFilelist.sql b/gentoobrowse-api/service/sql/maintenance/updatesMoveFilelist.sql
new file mode 100644
index 0000000..0a34c82
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/updatesMoveFilelist.sql
@@ -0,0 +1,6 @@
+UPDATE filelist SET
+	pathparts[3] = ?,
+	pathparts[4] = REGEXP_REPLACE(pathparts[4], CONCAT('^', REPLACE(?, '+', '\+'), '(-[0-9])'), CONCAT(?::text, '\1'))
+WHERE pathparts[3] = ?
+AND pathparts[4] ~ CONCAT('^', replace(?, '+', '\+'), '-[0-9]')
+AND filetypeId = 1
diff --git a/gentoobrowse-api/unittests/mockDefs.cpp b/gentoobrowse-api/unittests/mockDefs.cpp
index cfe426b..ffc2f2f 100644
--- a/gentoobrowse-api/unittests/mockDefs.cpp
+++ b/gentoobrowse-api/unittests/mockDefs.cpp
@@ -25,6 +25,8 @@ Service::Service() :
 Maintenance::Maintenance() :
 	IceTray::DryIce({
 			"--GentooBrowseAPI.ChangeLogStart.gentoo=68190573ce9846f82b8cb4e8a6376758c8257184",
+			"--GentooBrowseAPI.RefreshTarget.gentoo=40539afe6705aee26a55bb861f5e892ae7240057",
+			"--GentooBrowseAPI.RepoRoot=" + (rootDir / "fixtures" / "gitrepo").string(),
 			"--GentooBrowseAPI.BugRoot=file://" + (rootDir / "fixtures" / "bugs").string()
 		}),
 	PQ::Mock("user=postgres dbname=postgres", "GentooBrowseAPI", {
diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp
index 1ee94bc..7c016fc 100644
--- a/gentoobrowse-api/unittests/testMaintenance.cpp
+++ b/gentoobrowse-api/unittests/testMaintenance.cpp
@@ -6,6 +6,7 @@
 #include <buffer.h>
 #include <modifycommand.h>
 #include <git2.h>
+#include <fstream>
 
 class MaintenanceClientCombined : public Maintenance, public TestClient { };
 
@@ -171,5 +172,49 @@ BOOST_AUTO_TEST_CASE( testUpdateGitRepository )
 	BOOST_REQUIRE(boost::filesystem::is_regular_file(testRepo / "net-misc" / "gentoobrowse-api" / "Manifest"));
 }
 
+BOOST_AUTO_TEST_CASE( testRefreshGitRepository )
+{
+	auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("GentooBrowseAPI"));
+	auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, lastcommit) VALUES(?, ?, ?)");
+	insRepo->bindParamS(0, "gentoo");
+	insRepo->bindParamS(1, "/usr/portage");
+	insRepo->bindParamS(2, "8292397bf6a8c91215b03a558e8bc843aff25b64");
+	insRepo->execute();
+	std::ifstream data((rootDir / "gitdata.sql").string());
+	db->executeScript(data, rootDir);
+
+	BOOST_TEST_CONTEXT("Intiial values") {
+		SQL_REQUIRE_EQUAL(R"SQL(SELECT name FROM gentoobrowse.packages p WHERE p.packageid = 9)SQL", std::string, "libkgapi");
+		SQL_REQUIRE_EQUAL(R"SQL(SELECT p.categoryid FROM gentoobrowse.categories c, gentoobrowse.packages p
+			WHERE c.categoryid = p.categoryid AND p.name = 'libkgapi' AND c.name='net-libs')SQL", int64_t, 6);
+		SQL_REQUIRE_EQUAL(R"SQL(SELECT COUNT(*) FROM gentoobrowse.ebuilds e WHERE e.packageid = 9)SQL", int64_t, 2);
+		SQL_REQUIRE_EQUAL(R"SQL(SELECT COUNT(*) FROM gentoobrowse.ebuilds e WHERE e.ebuildid in (12, 13))SQL", int64_t, 2);
+		SQL_REQUIRE_EQUAL(R"SQL(SELECT MIN(e.packageid) FROM gentoobrowse.ebuilds e WHERE e.ebuildid in (12, 13))SQL", int64_t, 9);
+	}
+
+	// Import it
+	m->refreshPackageTreeGit();
+
+	BOOST_TEST_CONTEXT("libkgapi was moved correctly") {
+		BOOST_TEST_CONTEXT("from net-libs to kde-apps") {
+			SQL_REQUIRE_EQUAL(R"SQL(SELECT COUNT(*) FROM gentoobrowse.categories c, gentoobrowse.packages p
+				WHERE c.categoryid = p.categoryid AND p.name = 'libkgapi' AND c.name='kde-apps')SQL", int64_t, 1);
+			SQL_REQUIRE_EQUAL(R"SQL(SELECT COUNT(*) FROM gentoobrowse.categories c, gentoobrowse.packages p
+				WHERE c.categoryid = p.categoryid AND p.name = 'libkgapi' AND c.name='net-libs')SQL", int64_t, 0);
+		}
+		BOOST_TEST_CONTEXT("And it maintained its original packageId, but changed categoryId") {
+			SQL_REQUIRE_EQUAL(R"SQL(SELECT p.packageid FROM gentoobrowse.categories c, gentoobrowse.packages p
+				WHERE c.categoryid = p.categoryid AND p.name = 'libkgapi' AND c.name='net-libs')SQL", int64_t, 9);
+			SQL_REQUIRE_EQUAL(R"SQL(SELECT p.categoryid FROM gentoobrowse.categories c, gentoobrowse.packages p
+				WHERE c.categoryid = p.categoryid AND p.name = 'libkgapi' AND c.name='net-libs')SQL", int64_t, 8);
+		}
+		BOOST_TEST_CONTEXT("Original ebuilds in moved package were not deleted and replaced") {
+			SQL_REQUIRE_EQUAL(R"SQL(SELECT COUNT(*) FROM gentoobrowse.ebuilds e WHERE e.packageid = 9)SQL", int64_t, 2);
+			// SQL_REQUIRE_EQUAL(R"SQL(SELECT COUNT(*) FROM gentoobrowse.ebuilds e WHERE e.ebuildid in (12, 13))SQL", int64_t, 2);
+			// SQL_REQUIRE_EQUAL(R"SQL(SELECT MIN(e.packageid) FROM gentoobrowse.ebuilds e WHERE e.ebuildid in (12, 13))SQL", int64_t, 9);
+		}
+	}
+}
+
 BOOST_AUTO_TEST_SUITE_END();