Remove support for tree comparison updates (unittests have their own variant)

14 files changed, 183 insertions, 245 deletions

diff --git a/gentoobrowse-api/api/maintenance.ice b/gentoobrowse-api/api/maintenance.ice
index f68d371..be88569 100644
--- a/gentoobrowse-api/api/maintenance.ice
+++ b/gentoobrowse-api/api/maintenance.ice
@@ -10,9 +10,8 @@ module Gentoo {
 	};
 
 	interface Maintenance {
-		idempotent void refreshPackageTree();
+		idempotent void refreshPackageTree() throws GitError;
 		idempotent void refreshBugs();
-		idempotent void refreshPackageTreeGit() throws GitError;
 		idempotent void updateRepositories() throws GitError;
 		UserNews getUserNews();
 		void sendNotifications();
diff --git a/gentoobrowse-api/db/schema.sql b/gentoobrowse-api/db/schema.sql
index acf0635..25a16b3 100644
--- a/gentoobrowse-api/db/schema.sql
+++ b/gentoobrowse-api/db/schema.sql
@@ -383,15 +383,6 @@ CREATE TABLE ebuilds (
 ALTER TABLE ebuilds OWNER TO gentoo;
 -- Name: TABLE ebuilds; Type: COMMENT; Schema: gentoobrowse; Owner: gentoo
 COMMENT ON TABLE ebuilds IS 'Ebuilds :-)';
--- Name: files; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: 
-CREATE TABLE files (
-    moddate timestamp without time zone NOT NULL,
-    filetypeid integer NOT NULL,
-    repoid integer NOT NULL,
-    filesize integer NOT NULL,
-    pathparts text[] NOT NULL
-);
-ALTER TABLE files OWNER TO gentoo;
 -- Name: filetypes; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: 
 CREATE TABLE filetypes (
     filetypeid integer NOT NULL,
@@ -474,9 +465,9 @@ CREATE TABLE repos (
     repoid integer NOT NULL,
     name text NOT NULL,
 		path text NOT NULL,
-		upstream text,
-		branch text,
-		lastcommit character(40)
+		upstream text NOT NULL,
+		branch text NOT NULL,
+		lastcommit character(40) NOT NULL
 );
 ALTER TABLE repos OWNER TO gentoo;
 -- Name: repos_repoid_seq; Type: SEQUENCE; Schema: gentoobrowse; Owner: gentoo
@@ -663,9 +654,6 @@ ALTER TABLE ebuild_uses CLUSTER ON pk_ebuilduses;
 -- Name: pk_fileclass; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace: 
 ALTER TABLE ONLY filetypes
     ADD CONSTRAINT pk_fileclass PRIMARY KEY (filetypeid);
--- Name: pk_files; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace: 
-ALTER TABLE ONLY files
-    ADD CONSTRAINT pk_files PRIMARY KEY (repoid, pathparts);
 -- Name: pk_news; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace: 
 ALTER TABLE ONLY news
     ADD CONSTRAINT pk_news PRIMARY KEY (newsid);
@@ -806,9 +794,6 @@ ALTER TABLE ONLY ebuild_rdeps
 -- Name: fk_ebuilduses_file; Type: FK CONSTRAINT; Schema: gentoobrowse; Owner: gentoo
 ALTER TABLE ONLY ebuild_uses
     ADD CONSTRAINT fk_ebuilduses_file FOREIGN KEY (ebuildid) REFERENCES ebuilds(ebuildid) ON UPDATE CASCADE ON DELETE CASCADE;
--- Name: fk_files_type; Type: FK CONSTRAINT; Schema: gentoobrowse; Owner: gentoo
-ALTER TABLE ONLY files
-    ADD CONSTRAINT fk_files_type FOREIGN KEY (filetypeid) REFERENCES filetypes(filetypeid) ON UPDATE CASCADE;
 -- Name: fk_packageurls_package; Type: FK CONSTRAINT; Schema: gentoobrowse; Owner: gentoo
 ALTER TABLE ONLY package_urls
     ADD CONSTRAINT fk_packageurls_package FOREIGN KEY (packageid) REFERENCES packages(packageid) ON UPDATE CASCADE ON DELETE CASCADE;
@@ -888,11 +873,6 @@ REVOKE ALL ON TABLE ebuilds FROM PUBLIC;
 REVOKE ALL ON TABLE ebuilds FROM gentoo;
 GRANT ALL ON TABLE ebuilds TO gentoo;
 GRANT SELECT ON TABLE ebuilds TO apache;
--- Name: files; Type: ACL; Schema: gentoobrowse; Owner: gentoo
-REVOKE ALL ON TABLE files FROM PUBLIC;
-REVOKE ALL ON TABLE files FROM gentoo;
-GRANT ALL ON TABLE files TO gentoo;
-GRANT SELECT ON TABLE files TO apache;
 -- Name: filetypes; Type: ACL; Schema: gentoobrowse; Owner: gentoo
 REVOKE ALL ON TABLE filetypes FROM PUBLIC;
 REVOKE ALL ON TABLE filetypes FROM gentoo;
diff --git a/gentoobrowse-api/service/maintenance/updatesProcessor.cpp b/gentoobrowse-api/service/maintenance/updatesProcessor.cpp
index 5765c17..30e93c1 100644
--- a/gentoobrowse-api/service/maintenance/updatesProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/updatesProcessor.cpp
@@ -9,8 +9,6 @@
 #include <fstream>
 #include <boost/bind.hpp>
 #include <sql/maintenance/categoryInsert.sql.h>
-#include <sql/maintenance/updatesPackageDirFiles.sql.h>
-#include <sql/maintenance/updatesMD5CacheFiles.sql.h>
 #include <sql/maintenance/updatesMovePackages.sql.h>
 #include <sql/maintenance/updatesMoveUserPackages.sql.h>
 #include <sql/maintenance/updatesMoveFilelist.sql.h>
@@ -36,8 +34,6 @@ namespace Gentoo {
 				UpdatesPatch(DB::Connection * db, bool v) :
 					vcsMode(v),
 					categoryInsert(sql::maintenance::categoryInsert.modify(db)),
-					movePackageDirFiles(sql::maintenance::updatesPackageDirFiles.modify(db)),
-					moveMD5CacheFiles(sql::maintenance::updatesMD5CacheFiles.modify(db)),
 					findPackage(sql::portage::findPackage.select(db)),
 					movePackages(sql::maintenance::updatesMovePackages.modify(db)),
 					moveUserPackages(sql::maintenance::updatesMoveUserPackages.modify(db)),
@@ -69,8 +65,6 @@ namespace Gentoo {
 				const bool vcsMode;
 				DB::ModifyCommandPtr drop;
 				DB::ModifyCommandPtr categoryInsert;
-				DB::ModifyCommandPtr movePackageDirFiles;
-				DB::ModifyCommandPtr moveMD5CacheFiles;
 				DB::SelectCommandPtr findPackage;
 				DB::ModifyCommandPtr movePackages;
 				DB::ModifyCommandPtr moveUserPackages;
@@ -93,17 +87,6 @@ namespace Gentoo {
 						}
 						else {
 							// Move
-							movePackageDirFiles->bindParamS(0, catto);
-							movePackageDirFiles->bindParamS(1, nameto);
-							movePackageDirFiles->bindParamS(2, catfrom);
-							movePackageDirFiles->bindParamS(3, namefrom);
-							movePackageDirFiles->execute();
-							moveMD5CacheFiles->bindParamS(0, catto);
-							moveMD5CacheFiles->bindParamS(1, namefrom);
-							moveMD5CacheFiles->bindParamS(2, nameto);
-							moveMD5CacheFiles->bindParamS(3, catfrom);
-							moveMD5CacheFiles->bindParamS(4, namefrom);
-							moveMD5CacheFiles->execute();
 							categoryInsert->bindParamS(0, catto);
 							categoryInsert->bindParamS(1, catto);
 							categoryInsert->execute();
@@ -112,8 +95,6 @@ namespace Gentoo {
 							movePackages->bindParamS(2, catfrom);
 							movePackages->bindParamS(3, catto);
 							movePackages->execute();
-						}
-						if (vcsMode) {
 							moveFilelist->bindParamS(0, catto);
 							moveFilelist->bindParamS(1, namefrom);
 							moveFilelist->bindParamS(2, nameto);
diff --git a/gentoobrowse-api/service/maintenanceCommon.cpp b/gentoobrowse-api/service/maintenanceCommon.cpp
index 4085e85..63bb13d 100644
--- a/gentoobrowse-api/service/maintenanceCommon.cpp
+++ b/gentoobrowse-api/service/maintenanceCommon.cpp
@@ -18,7 +18,7 @@ namespace Gentoo {
 					m->ice_ping();
 					auto rb = m->begin_refreshBugs();
 					m->updateRepositories();
-					m->refreshPackageTreeGit();
+					m->refreshPackageTree();
 					m->sendNotifications();
 					m->end_refreshBugs(rb);
 				}
@@ -28,15 +28,20 @@ namespace Gentoo {
 		};
 
 		Maintenance::Maintenance(IceTray::DatabasePoolPtr d, Ice::CommunicatorPtr ic, Ice::PropertiesPtr p) :
-			IceTray::AbstractDatabaseClient(d)
+			Maintenance::Maintenance(d)
 		{
-			setupFileProcessors();
 			if (auto sync = p->getPropertyAsIntWithDefault("GentooBrowseAPI.SyncTime", 0) * 60) {
 				taskRunner = new IceUtil::Timer();
 				taskRunner->scheduleRepeated(new AutoUpdatePackages(ic), IceUtil::Time::seconds(sync));
 			}
 		}
 
+		Maintenance::Maintenance(IceTray::DatabasePoolPtr d) :
+			IceTray::AbstractDatabaseClient(d)
+		{
+			setupFileProcessors();
+		}
+
 		Maintenance::~Maintenance()
 		{
 			if (taskRunner) {
diff --git a/gentoobrowse-api/service/maintenanceGitOperations.cpp b/gentoobrowse-api/service/maintenanceGitOperations.cpp
index b178100..16c51d3 100644
--- a/gentoobrowse-api/service/maintenanceGitOperations.cpp
+++ b/gentoobrowse-api/service/maintenanceGitOperations.cpp
@@ -166,7 +166,7 @@ namespace Gentoo {
 		}
 
 		void
-		Maintenance::refreshPackageTreeGit(const Ice::Current & c)
+		Maintenance::refreshPackageTree(const Ice::Current & c)
 		{
 			auto properties = c.adapter->getCommunicator()->getProperties();
 			auto repoRoot = boost::filesystem::path(properties->getPropertyWithDefault("GentooBrowseAPI.RepoRoot", "/"));
@@ -181,25 +181,35 @@ namespace Gentoo {
 			sql::maintenance::gitListCreate.modify(dbc.get())->execute();
 			sql::maintenance::gitListCreateIdx.modify(dbc.get())->execute();
 
+			applyFileChanges(dbc.get(), repoRoot);
+
+			dbc->execute("DROP TABLE filelist");
+			dbc->execute("DROP TABLE filelistraw");
+			dbc->execute("SET search_path = public, pg_catalog");
+		}
+
+		void
+		Maintenance::applyFileChanges(DB::Connection * dbc, const boost::filesystem::path & repoRoot) const
+		{
 			FileProcessors fps;
 			for (const auto & fpf : fpfs) {
 				fps[fpf.first] = fpf.second(true);
 			}
 			for (const auto & fp : fps) {
-				fp.second->prepare(dbc.get());
+				fp.second->prepare(dbc);
 			}
 			RepoMap repos;
 			dbc->select("SELECT path, repoid FROM gentoobrowse.repos")->forEachRow<std::string, int64_t>([&repos](auto path, auto id) {
 					repos[id] = path;
 				});
 			std::map<std::string, FileHandleFunc> funcs;
-			funcs["D"] = boost::bind(&AbstractFileProcessor::deleted, _1, dbc.get(), _2, _3);
-			funcs["M"] = boost::bind(&AbstractFileProcessor::modified, _1, dbc.get(), _2, _3, _4);
-			funcs["A"] = boost::bind(&AbstractFileProcessor::created, _1, dbc.get(), _2, _3, _4);
-			auto phases = sql::maintenance::filelistPhases.select(dbc.get());
+			funcs["D"] = boost::bind(&AbstractFileProcessor::deleted, _1, dbc, _2, _3);
+			funcs["M"] = boost::bind(&AbstractFileProcessor::modified, _1, dbc, _2, _3, _4);
+			funcs["A"] = boost::bind(&AbstractFileProcessor::created, _1, dbc, _2, _3, _4);
+			auto phases = sql::maintenance::filelistPhases.select(dbc);
 			for (const auto & phaseRow : phases->as<int64_t>()) {
 				const auto & phase = phaseRow.value<0>();
-				auto files = sql::maintenance::gitListChangesInPhase.select(dbc.get());
+				auto files = sql::maintenance::gitListChangesInPhase.select(dbc);
 				files->bindParamI(0, phase);
 				std::set<int64_t> usedTypes;
 				for (const auto & fileRow : files->as<int64_t, int64_t, std::string, std::string>()) {
@@ -213,16 +223,12 @@ namespace Gentoo {
 					}
 				}
 				for (const auto & fp : usedTypes) {
-					fps[fp]->apply(dbc.get());
+					fps[fp]->apply(dbc);
 				}
 			}
-
-			dbc->execute("DROP TABLE filelist");
-			dbc->execute("DROP TABLE filelistraw");
 			for (const auto & fp : fps) {
-				fp.second->tidy(dbc.get());
+				fp.second->tidy(dbc);
 			}
-			dbc->execute("SET search_path = public, pg_catalog");
 		}
 
 	}
diff --git a/gentoobrowse-api/service/maintenancePackageTree.cpp b/gentoobrowse-api/service/maintenancePackageTree.cpp
index dbc72a5..9862f28 100644
--- a/gentoobrowse-api/service/maintenancePackageTree.cpp
+++ b/gentoobrowse-api/service/maintenancePackageTree.cpp
@@ -9,9 +9,6 @@
 #include <boost/date_time/posix_time/posix_time.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/algorithm/string/join.hpp>
-#include <sql/maintenance/fileListCreateRaw.sql.h>
-#include <sql/maintenance/fileListCreate.sql.h>
-#include <sql/maintenance/fileListCreatePk.sql.h>
 #include "maintenance/categoryMetaProcessor.h"
 #include "maintenance/packageMetaProcessor.h"
 #include "maintenance/ebuildMetaProcessor.h"
@@ -105,35 +102,6 @@ R"DATA( 10	category metadata	2	2	{"(2,metadata.xml)"}
 			}
 		}
 
-		Maintenance::RepoMap
-		Maintenance::createTempFileList(DB::Connection * dbc, const boost::filesystem::path & tmp)
-		{
-			boost::filesystem::remove_all(tmp);
-			boost::filesystem::create_directories(tmp);
-			RepoMap repos;
-			dbc->select("SELECT name, path, repoid FROM gentoobrowse.repos")->forEachRow<std::string, std::string, int64_t>([&tmp,&repos](auto n, auto p, auto id) {
-					boost::filesystem::create_symlink(p, tmp / n);
-					repos[id] = n;
-				});
-			dbc->execute(sql::maintenance::fileListCreateRaw.getSql());
-			dbc->beginBulkUpload("filelistraw", "");
-			char buf[BUFSIZ];
-			for (boost::filesystem::recursive_directory_iterator d(tmp, boost::filesystem::symlink_option::recurse);
-					d != boost::filesystem::recursive_directory_iterator(); d++) {
-				if (boost::filesystem::is_regular_file(d->status()) &&
-						!boost::algorithm::ends_with(d->path().string(), ".ebuild")) {
-					auto len = snprintf(buf, BUFSIZ, "%s\t%ju\t%s\n",
-						d->path().lexically_relative(tmp).c_str(),
-						boost::filesystem::file_size(*d),
-						boost::posix_time::to_iso_extended_string(
-							boost::posix_time::from_time_t(boost::filesystem::last_write_time(*d))).c_str());
-					dbc->bulkUploadData(buf, len);
-				}
-			}
-			dbc->endBulkUpload(nullptr);
-			return repos;
-		}
-
 		void Maintenance::updateFileTypes(DB::Connection * dbc) const
 		{
 			DB::TablePatch p;
@@ -148,86 +116,6 @@ R"DATA( 10	category metadata	2	2	{"(2,metadata.xml)"}
 			dbc->patchTable(&p);
 			Utils::Database::drop(dbc, p.src);
 		}
-
-		class PhaseSourceFilter : public Utils::EntityWhereFilter<int64_t> {
-			public:
-				PhaseSourceFilter() :
-					Utils::EntityWhereFilter<int64_t>("filetypeId")
-				{
-				}
-
-				void
-				writeSql(AdHoc::Buffer & sql) override
-				{
-					sql.append("( SELECT * FROM filelist a WHERE ");
-					Utils::EntityWhereFilter<int64_t>::writeSql(sql);
-					sql.append(")");
-				}
-		};
-
-		void
-		Maintenance::processChanges(DB::Connection * dbc, const boost::filesystem::path & tmp, const RepoMap & repos)
-		{
-			FileProcessors fps;
-			for (const auto & fpf : fpfs) {
-				fps[fpf.first] = fpf.second(false);
-			}
-			for (const auto & fp : fps) {
-				fp.second->prepare(dbc);
-			}
-			dbc->execute(sql::maintenance::fileListCreate.getSql());
-			dbc->execute(sql::maintenance::fileListCreatePk.getSql());
-			DB::TablePatch tp;
-			tp.dest = "gentoobrowse.files";
-			tp.pk = {"pathparts", "repoid"};
-			tp.cols = {"pathparts", "repoid", "filetypeid", "filesize", "moddate"};
-			tp.beforeDelete = boost::bind(&Maintenance::fileDeleted, this, dbc, &fps, tmp, &repos, _1);
-			tp.beforeUpdate = boost::bind(&Maintenance::fileChanged, this, dbc, &fps, tmp, &repos, _1);
-			tp.beforeInsert = boost::bind(&Maintenance::fileCreated, this, dbc, &fps, tmp, &repos, _1);
-			DB::StaticSqlWriter obpo("b.updateOrder NULLS LAST");
-			tp.order = &obpo;
-			auto phases = dbc->select("SELECT phase FROM gentoobrowse.filetypes GROUP BY phase ORDER BY phase");
-			for (const auto & phase : phases->as<int64_t>()) {
-				Utils::EntityWhereFilter<int64_t> pf("filetypeId");
-				PhaseSourceFilter psf;
-				auto getFts = dbc->select("SELECT filetypeId FROM gentoobrowse.filetypes WHERE phase = ?");
-				getFts->bindParamI(0, phase.value<0>());
-				for (const auto & ft : getFts->as<int64_t>()) {
-					pf.entityIds.insert(ft.value<0>());
-					psf.entityIds.insert(ft.value<0>());
-				}
-				tp.srcExpr = &psf;
-				tp.where = &pf;
-				dbc->patchTable(&tp);
-				for (const auto & fp : pf.entityIds) {
-					if (fps.find(fp) != fps.end()) {
-						fps[fp]->apply(dbc);
-					}
-				}
-			}
-			dbc->execute("DROP TABLE filelist");
-			dbc->execute("DROP TABLE filelistraw");
-			for (const auto & fp : fps) {
-				fp.second->tidy(dbc);
-			}
-		}
-
-		void
-		Maintenance::refreshPackageTree(const Ice::Current &)
-		{
-			auto tmp = boost::filesystem::temp_directory_path() / "import";
-			AdHoc::ScopeExit tidyTmp([&tmp]{
-					boost::system::error_code ec;
-					boost::filesystem::remove_all(tmp, ec);
-				});
-			auto dbc = db->get();
-			dbc->execute("SET search_path = gentoobrowse, pg_catalog");
-			DB::TransactionScope tx(dbc.get());
-			updateFileTypes(dbc.get());
-			auto repos = createTempFileList(dbc.get(), tmp);
-			processChanges(dbc.get(), tmp, repos);
-			dbc->execute("SET search_path = public, pg_catalog");
-		}
 	}
 }
 
diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h
index d82adc9..ac74f39 100644
--- a/gentoobrowse-api/service/maintenanceimpl.h
+++ b/gentoobrowse-api/service/maintenanceimpl.h
@@ -13,7 +13,7 @@
 
 namespace Gentoo {
 	namespace Service {
-		class DLL_PUBLIC Maintenance : public Gentoo::Maintenance, IceTray::AbstractDatabaseClient {
+		class DLL_PUBLIC Maintenance : public Gentoo::Maintenance, public IceTray::AbstractDatabaseClient {
 			public:
 				typedef std::map<int64_t, boost::function<FileProcessorPtr(bool)>> FileProcessorFactories;
 				typedef std::map<int64_t, FileProcessorPtr> FileProcessors;
@@ -21,20 +21,21 @@ namespace Gentoo {
 				typedef std::map<int64_t, std::string> RepoMap;
 
 				Maintenance(IceTray::DatabasePoolPtr d, Ice::CommunicatorPtr ic, Ice::PropertiesPtr p);
-				~Maintenance();
+				virtual ~Maintenance();
 
 				void refreshPackageTree(const Ice::Current &) override;
-				void refreshPackageTreeGit(const Ice::Current &) override;
 				void refreshBugs(const Ice::Current &) override;
 				void updateRepositories(const Ice::Current &) override;
 				Gentoo::UserNews getUserNews(const Ice::Current &) override;
 				void sendNotifications(const Ice::Current &) override;
 
+			protected:
+				Maintenance(IceTray::DatabasePoolPtr d);
+				void applyFileChanges(DB::Connection *, const boost::filesystem::path &) const;
+				void updateFileTypes(DB::Connection *) const;
+
 			private:
 				void setupFileProcessors();
-				void updateFileTypes(DB::Connection *) const;
-				static RepoMap createTempFileList(DB::Connection *, const boost::filesystem::path &);
-				void processChanges(DB::Connection *, const boost::filesystem::path &, const RepoMap & repos);
 				void fileDeleted(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path &, const RepoMap *, DB::SelectCommandPtr s);
 				void fileChanged(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path &, const RepoMap *, DB::SelectCommandPtr s);
 				void fileCreated(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path & tmp, const RepoMap *, DB::SelectCommandPtr s);
diff --git a/gentoobrowse-api/service/sql/maintenance/updatesMD5CacheFiles.sql b/gentoobrowse-api/service/sql/maintenance/updatesMD5CacheFiles.sql
deleted file mode 100644
index 1fcbb7e..0000000
--- a/gentoobrowse-api/service/sql/maintenance/updatesMD5CacheFiles.sql
+++ /dev/null
@@ -1,6 +0,0 @@
-UPDATE gentoobrowse.files SET
-	pathparts[3] = ?,
-	pathparts[4] = REGEXP_REPLACE(pathparts[4], CONCAT('^', REPLACE(?, '+', '\+'), '(-[0-9])'), CONCAT(?::text, '\1'))
-WHERE pathparts[3] = ?
-AND pathparts[4] ~ CONCAT('^', replace(?, '+', '\+'), '-[0-9]')
-AND filetypeId = 1
diff --git a/gentoobrowse-api/service/sql/maintenance/updatesPackageDirFiles.sql b/gentoobrowse-api/service/sql/maintenance/updatesPackageDirFiles.sql
deleted file mode 100644
index 307484a..0000000
--- a/gentoobrowse-api/service/sql/maintenance/updatesPackageDirFiles.sql
+++ /dev/null
@@ -1,4 +0,0 @@
-UPDATE gentoobrowse.files SET
-	pathparts[1:2] = ARRAY[?, ?]
-WHERE pathparts[1:2] = ARRAY[?, ?]
-AND filetypeId IN (4, 8)
diff --git a/gentoobrowse-api/unittests/Jamfile.jam b/gentoobrowse-api/unittests/Jamfile.jam
index 53d9289..74a67df 100644
--- a/gentoobrowse-api/unittests/Jamfile.jam
+++ b/gentoobrowse-api/unittests/Jamfile.jam
@@ -80,12 +80,19 @@ run
 
 run
 	testMaintenance.cpp
+	../service/sql/maintenance/gitListCreate.sql
+	../service/sql/maintenance/gitListCreateRaw.sql
+	../service/sql/maintenance/gitListCreateIdx.sql
 	: : :
 	<dependency>../db/schema.sql
 	<define>BOOST_TEST_DYN_LINK
 	<library>testCommon
 	<library>git2
+	<library>../service//icetray
 	<implicit-dependency>testCommon
+	<icetray.sql.namespace>Gentoo::Service
+	<icetray.sql.connector>postgresql
+	<icetray.sql.basedir>../service
 	: testMaintenance ;
 
 run
diff --git a/gentoobrowse-api/unittests/data.sql b/gentoobrowse-api/unittests/data.sql
index aedf5c6..4120934 100644
--- a/gentoobrowse-api/unittests/data.sql
+++ b/gentoobrowse-api/unittests/data.sql
@@ -9,7 +9,7 @@ SET search_path = gentoobrowse, pg_catalog;
 COPY gentoobrowse.bugs (bugid, severity, status, summary, firstseen) FROM '$SCRIPTDIR/fixtures/bugs.dat';
 COPY gentoobrowse.categories (categoryid, name, summary) FROM '$SCRIPTDIR/fixtures/categories.dat';
 COPY gentoobrowse.packages (packageid, categoryid, name, firstseen, description, summary, maintainer, herd, image, maintainername) FROM '$SCRIPTDIR/fixtures/packages.dat';
-COPY gentoobrowse.repos (repoid, name, path) FROM '$SCRIPTDIR/fixtures/repos.dat';
+COPY gentoobrowse.repos (repoid, name, path, upstream, branch, lastcommit) FROM '$SCRIPTDIR/fixtures/repos.dat';
 COPY gentoobrowse.ebuilds (ebuildid, packageid, version, versioninst, slot, license, firstseen, moddate, repoid) FROM '$SCRIPTDIR/fixtures/ebuilds.dat';
 COPY gentoobrowse.ebuild_archs (ebuildid, arch) FROM '$SCRIPTDIR/fixtures/ebuild_archs.dat';
 COPY gentoobrowse.ebuild_deps (ebuildid, packageid, versionspec, flags, op, slot) FROM '$SCRIPTDIR/fixtures/ebuild_deps.dat';
@@ -18,7 +18,6 @@ COPY gentoobrowse.ebuild_masks (setno, ebuildid) FROM '$SCRIPTDIR/fixtures/ebuil
 COPY gentoobrowse.ebuild_rdeps (ebuildid, packageid, versionspec, flags, op, slot) FROM '$SCRIPTDIR/fixtures/ebuild_rdeps.dat';
 COPY gentoobrowse.ebuild_uses (ebuildid, use) FROM '$SCRIPTDIR/fixtures/ebuild_uses.dat';
 COPY gentoobrowse.filetypes (filetypeid, description, spec, phase, updateOrder) FROM '$SCRIPTDIR/fixtures/filetypes.dat';
-COPY gentoobrowse.files (moddate, filetypeid, repoid, filesize, pathparts) FROM '$SCRIPTDIR/fixtures/files.dat';
 COPY gentoobrowse.license (name, legalbumph) FROM '$SCRIPTDIR/fixtures/license.dat';
 COPY gentoobrowse.news (newsid, title, posted, authorname, authoremail, atomspec, body, urls) FROM '$SCRIPTDIR/fixtures/news.dat';
 COPY gentoobrowse.package_urls (packageid, url) FROM '$SCRIPTDIR/fixtures/package_urls.dat';
diff --git a/gentoobrowse-api/unittests/fixtures/repos.dat b/gentoobrowse-api/unittests/fixtures/repos.dat
index 30d0932..072a029 100644
--- a/gentoobrowse-api/unittests/fixtures/repos.dat
+++ b/gentoobrowse-api/unittests/fixtures/repos.dat
@@ -1,2 +1,2 @@
-17	gentoo	/usr/portage
-18	RandomLAN	/home/randomdan/dev/portage
+17	gentoo	/usr/portage	origin	master	87ebb8523b2890c641a27f29007ba8eaa6aa834e
+18	RandomLAN	/home/randomdan/dev/portage	origin	master	4321e561074c32ec3015417e517c6cbdb2120fcc
diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp
index e05d9ad..21b77db 100644
--- a/gentoobrowse-api/unittests/testMaintenance.cpp
+++ b/gentoobrowse-api/unittests/testMaintenance.cpp
@@ -7,56 +7,27 @@
 #include <modifycommand.h>
 #include <git2.h>
 #include <fstream>
+#include <maintenanceimpl.h>
+#include <sql/maintenance/gitListCreateRaw.sql.h>
+#include <sql/maintenance/gitListCreate.sql.h>
+#include <sql/maintenance/gitListCreateIdx.sql.h>
 
 class MaintenanceClientCombined : public Maintenance, public TestClient { };
 
 BOOST_FIXTURE_TEST_SUITE(maintenance, MaintenanceClientCombined)
 
-const boost::filesystem::path treeDir(binDir / "tree" / "gentoobrowse");
+const boost::filesystem::path treeDir(binDir / "tree");
 const boost::filesystem::path fixturesDir(rootDir / "fixtures");
 
-class SampleData {
-	public:
-		SampleData()
-		{
-			clean();
-		}
-
-		~SampleData()
-		{
-			clean();
-		}
-
-		void clean()
-		{
-			BOOST_TEST_CHECKPOINT("Cleaning sample data dir " << treeDir);
-			boost::filesystem::remove_all(treeDir);
-		}
-
-		void extract(const std::string & archive, const std::string & repo)
-		{
-			boost::filesystem::create_directories(treeDir);
-			boost::filesystem::create_symlink(fixturesDir / archive, treeDir / repo);
-		}
-};
-
 void dumpDb(DB::ConnectionPtr db);
 
 void
-doRefreshPackageTree(SampleData & sd, DB::ConnectionPtr db, const std::string & archive, const std::string & dir,
-		Gentoo::MaintenancePrx m, int64_t files, int64_t cats, int64_t devvcs, int64_t pkgs, int64_t ebs, int64_t ebus,
+doRefreshPackageTree(DB::ConnectionPtr db, int64_t cats, int64_t devvcs, int64_t pkgs, int64_t ebs, int64_t ebus,
 		int64_t ebas, int64_t pus, int64_t ug, int64_t ul, int64_t ugs, int64_t ugds, int64_t deps, int64_t rdeps,
 		int64_t news, int64_t masks)
 {
-	BOOST_TEST_INFO("archive: " << archive);
-	if (!archive.empty()) {
-		sd.extract(archive, dir);
-	}
-	m->refreshPackageTree();
-	sd.clean();
 	dumpDb(db);
 
-	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.files", int64_t, files);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, cats);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories WHERE name = 'dev-vcs'", int64_t, devvcs);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, pkgs);
@@ -97,23 +68,130 @@ dumpDb(DB::ConnectionPtr db)
 #endif
 }
 
+class M2 : public Gentoo::Service::Maintenance {
+	public:
+		typedef std::map<boost::filesystem::path, size_t> FileMap;
+
+		M2(IceTray::DatabasePoolPtr d) :
+			Gentoo::Service::Maintenance(d)
+		{
+			auto dbc = db->get();
+			DB::TransactionScope tx(dbc.get());
+			updateFileTypes(dbc.get());
+		}
+
+		void applyDiffOfFolders(int64_t repoId, const boost::filesystem::path & from, const boost::filesystem::path & to) const
+		{
+			fprintf(stderr, "%s\n", __PRETTY_FUNCTION__);
+			auto dbc = db->get();
+			auto fromFiles = fileSet(from);
+			auto toFiles = fileSet(to);
+			dbc->execute("SET search_path = gentoobrowse, pg_catalog");
+			Gentoo::Service::sql::maintenance::gitListCreateRaw.modify(dbc.get())->execute();
+			auto ins = dbc->modify("INSERT INTO filelistraw(repoid, status, filename) VALUES(?, ?, ?)");
+			ins->bindParamI(0, repoId);
+			newFiles(ins, fromFiles, toFiles);
+			removedFiles(ins, fromFiles, toFiles);
+			changedFiles(ins, fromFiles, toFiles);
+			Gentoo::Service::sql::maintenance::gitListCreate.modify(dbc.get())->execute();
+			Gentoo::Service::sql::maintenance::gitListCreateIdx.modify(dbc.get())->execute();
+			boost::filesystem::remove(treeDir);
+			boost::filesystem::create_symlink(to, treeDir);
+			applyFileChanges(dbc.get(), "/");
+			boost::filesystem::remove(treeDir);
+			dbc->execute("DROP TABLE filelist");
+			dbc->execute("DROP TABLE filelistraw");
+			dbc->execute("SET search_path = public, pg_catalog");
+		}
+
+		static FileMap fileSet(const boost::filesystem::path & p)
+		{
+			FileMap found;
+			for (boost::filesystem::recursive_directory_iterator r(p); r != boost::filesystem::recursive_directory_iterator(); r++) {
+				if (boost::filesystem::is_regular(r->status())) {
+					found[boost::filesystem::relative(*r, p)] = boost::filesystem::file_size(*r);
+				}
+			}
+			return found;
+		}
+		static void newFiles(DB::ModifyCommandPtr ins, const FileMap & from, const FileMap & to)
+		{
+			ins->bindParamS(1, "A");
+			for(const auto & f : to) {
+				if (from.find(f.first) == from.end()) {
+					fprintf(stderr, "Added %s\n", f.first.c_str());
+					ins->bindParamS(2, f.first.string());
+					ins->execute();
+				}
+			}
+		}
+		static void removedFiles(DB::ModifyCommandPtr ins, const FileMap & from, const FileMap & to)
+		{
+			ins->bindParamS(1, "D");
+			for(const auto & f : from) {
+				if (to.find(f.first) == to.end()) {
+					fprintf(stderr, "Removed %s\n", f.first.c_str());
+					ins->bindParamS(2, f.first.string());
+					ins->execute();
+				}
+			}
+		}
+		static void changedFiles(DB::ModifyCommandPtr ins, const FileMap & from, const FileMap & to)
+		{
+			ins->bindParamS(1, "M");
+			for(const auto & f : to) {
+				auto i = from.find(f.first);
+				if (i != from.end() && i->second != f.second) {
+					fprintf(stderr, "Changed %s\n", f.first.c_str());
+					ins->bindParamS(2, f.first.string());
+					ins->execute();
+				}
+			}
+		}
+};
+
+class MockPool : public IceTray::DatabasePool {
+	public:
+		MockPool() : IceTray::DatabasePool(10, 2) {}
+		DB::Connection * createResource() const override
+		{
+			return DB::MockDatabase::openConnectionTo("GentooBrowseAPI");
+		}
+};
+
 BOOST_AUTO_TEST_CASE( refreshPackageTree )
 {
-	SampleData sd;
 	auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("GentooBrowseAPI"));
-	auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path) VALUES(?, ?)");
+	auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, upstream, branch, lastcommit) VALUES(?, ?, ?, ?, ?)");
 	insRepo->bindParamS(0, "gentoo");
-	insRepo->bindParamS(1, (treeDir / "gentoo").string());
+	insRepo->bindParamS(1, treeDir.string());
+	insRepo->bindParamS(2, "origin");
+	insRepo->bindParamS(3, "master");
+	insRepo->bindParamS(4, "8292397bf6a8c91215b03a558e8bc843aff25b64");
 	insRepo->execute();
 
-	doRefreshPackageTree(sd, db, "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d", "gentoo",
-			m, 2080, 5, 1, 482, 981, 3626, 4593, 501, 393, 238, 50, 1573, 2008, 1543, 81, 152);
+	boost::filesystem::remove_all(binDir / "empty");
+	boost::filesystem::create_directory(binDir / "empty");
+	auto p = IceTray::DatabasePoolPtr(new MockPool());
+	M2 m2(p);
+
+	BOOST_TEST_CONTEXT("4156eb45cf3b0ce1d7125b84efd8688c2d6e831d") {
+		m2.applyDiffOfFolders(1, binDir / "empty", rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d");
+		doRefreshPackageTree(db,
+				5, 1, 482, 981, 3626, 4593, 501, 393, 238, 50, 1573, 2008, 1543, 81, 152);
+	}
 
-	doRefreshPackageTree(sd, db, "756569aa764177340726dd3d40b41d89b11b20c7", "gentoo",
-			m, 2082, 5, 1, 483, 982, 3638, 4599, 502, 393, 238, 50, 1573, 2009, 1546, 79, 152);
+	BOOST_TEST_CONTEXT("756569aa764177340726dd3d40b41d89b11b20c7") {
+		m2.applyDiffOfFolders(1, rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d", rootDir / "fixtures" / "756569aa764177340726dd3d40b41d89b11b20c7");
+		doRefreshPackageTree(db,
+				5, 1, 483, 982, 3638, 4599, 502, 393, 238, 50, 1573, 2009, 1546, 79, 152);
+	}
 
-	doRefreshPackageTree(sd, db, "", "gentoo",
-			m, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+	BOOST_TEST_CONTEXT("empty") {
+		m2.applyDiffOfFolders(1, rootDir / "fixtures" / "756569aa764177340726dd3d40b41d89b11b20c7", binDir / "empty");
+		doRefreshPackageTree(db,
+				0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+	}
 }
 
 static
@@ -150,15 +228,18 @@ BOOST_AUTO_TEST_CASE( testUpdateGitRepository )
 	git_remote * origin;
 	BOOST_REQUIRE_EQUAL(0, git_remote_create(&origin, repo, "origin", "http://git.randomdan.homeip.net/git/portage"));
 	git_remote_free(origin);
-	create_initial_commit(repo);
+	auto commitOid = create_initial_commit(repo);
+	char commit[GIT_OID_HEXSZ + 1];
+	git_oid_tostr(commit, GIT_OID_HEXSZ, &commitOid);
 	git_repository_free(repo);
 	git_libgit2_shutdown();
 	// Reference it
-	auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, upstream, branch) VALUES(?, ?, ?, ?)");
+	auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, upstream, branch, lastcommit) VALUES(?, ?, ?, ?, ?)");
 	insRepo->bindParamS(0, "testrepo");
 	insRepo->bindParamS(1, testRepo.string());
 	insRepo->bindParamS(2, "origin");
 	insRepo->bindParamS(3, "master");
+	insRepo->bindParamS(4, commit);
 	insRepo->execute();
 
 	// Update
@@ -175,10 +256,12 @@ BOOST_AUTO_TEST_CASE( testUpdateGitRepository )
 BOOST_AUTO_TEST_CASE( testRefreshGitRepository )
 {
 	auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("GentooBrowseAPI"));
-	auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, lastcommit) VALUES(?, ?, ?)");
+	auto insRepo = db->modify("INSERT INTO gentoobrowse.repos(name, path, upstream, branch, lastcommit) VALUES(?, ?, ?, ?, ?)");
 	insRepo->bindParamS(0, "gentoo");
 	insRepo->bindParamS(1, "/usr/portage");
-	insRepo->bindParamS(2, "8292397bf6a8c91215b03a558e8bc843aff25b64");
+	insRepo->bindParamS(2, "origin");
+	insRepo->bindParamS(3, "master");
+	insRepo->bindParamS(4, "8292397bf6a8c91215b03a558e8bc843aff25b64");
 	insRepo->execute();
 	std::ifstream data((rootDir / "gitdata.sql").string());
 	db->executeScript(data, rootDir);
@@ -193,7 +276,7 @@ BOOST_AUTO_TEST_CASE( testRefreshGitRepository )
 	}
 
 	// Import it
-	m->refreshPackageTreeGit();
+	m->refreshPackageTree();
 
 	SQL_REQUIRE_EQUAL(R"SQL(SELECT lastCommit FROM gentoobrowse.repos)SQL", std::string, "40539afe6705aee26a55bb861f5e892ae7240057");
 	BOOST_TEST_CONTEXT("libkgapi was moved correctly") {
diff --git a/gentoobrowse-api/util/update.cpp b/gentoobrowse-api/util/update.cpp
index 2d25530..f14df6e 100644
--- a/gentoobrowse-api/util/update.cpp
+++ b/gentoobrowse-api/util/update.cpp
@@ -8,7 +8,7 @@ int
 main(int c, char ** v)
 {
 	std::string endpoint;
-	bool background, tree, bugs, sendNotifications, pull, git;
+	bool background, tree, bugs, sendNotifications, pull;
 	po::options_description opts("Gentoo Browse Util::Update options");
 	opts.add_options()
 		("endpoint", po::value(&endpoint)->default_value("tcp -p 9001"), "Service endpoint")
@@ -16,7 +16,6 @@ main(int c, char ** v)
 		("bugs", po::value(&bugs)->default_value(true), "Update bugs")
 		("tree", po::value(&tree)->default_value(true), "Update tree")
 		("pull", po::value(&pull)->default_value(true), "Perform a git pull on repositories")
-		("git", po::value(&git)->default_value(true), "Update tree using git")
 		("notifications", po::value(&sendNotifications)->default_value(true), "Send notification emails")
 		("help,h", "Show help")
 		;
@@ -35,9 +34,9 @@ main(int c, char ** v)
 	m->ice_ping();
 
 	std::set<Ice::AsyncResultPtr> jobs;
-	if (pull) m->updateRepositories();
-	if (tree) jobs.insert(git ? m->begin_refreshPackageTreeGit() : m->begin_refreshPackageTree());
 	if (bugs) jobs.insert(m->begin_refreshBugs());
+	if (pull) m->updateRepositories();
+	if (tree) jobs.insert(m->begin_refreshPackageTree());
 
 	std::for_each(jobs.begin(), jobs.end(), [](const auto & j) { j->waitForSent(); });