Replace the database file types and file list comparison logic with one that's purely code

40 files changed, 254 insertions, 382 deletions

diff --git a/gentoobrowse-api/db/schema.sql b/gentoobrowse-api/db/schema.sql
index 25a16b3..b7f8209 100644
--- a/gentoobrowse-api/db/schema.sql
+++ b/gentoobrowse-api/db/schema.sql
@@ -274,27 +274,6 @@ BEGIN
 END
 $$;
 ALTER FUNCTION gentoobrowse.packagefts(p packages) OWNER TO gentoo;
--- Name: pathpartsmatchesspecs(text[], filterspec[]); Type: FUNCTION; Schema: gentoobrowse; Owner: gentoo
-CREATE FUNCTION pathpartsmatchesspecs(pathparts text[], spec filterspec[]) RETURNS boolean
-    LANGUAGE plpgsql
-    AS $$
-DECLARE
-	i integer;
-	p integer;
-BEGIN
-	p := array_upper(pathparts, 1);
-	FOR i IN 1 .. array_upper(spec, 1) LOOP
-		IF (p < spec[i].part) THEN RETURN FALSE; END IF;
-		IF (spec[i].part <= 0) THEN
-			IF (pathparts[p - spec[i].part] NOT LIKE spec[i].pattern) THEN RETURN FALSE; END IF;
-		ELSE
-			IF (pathparts[spec[i].part] NOT LIKE spec[i].pattern) THEN RETURN FALSE; END IF;
-		END IF;
-	END LOOP;
-	RETURN TRUE;
-END;
-$$;
-ALTER FUNCTION gentoobrowse.pathpartsmatchesspecs(pathparts text[], spec filterspec[]) OWNER TO gentoo;
 -- Name: sum(tsvector); Type: AGGREGATE; Schema: gentoobrowse; Owner: gentoo
 CREATE AGGREGATE sum(tsvector) (
     SFUNC = tsvector_concat,
@@ -308,13 +287,6 @@ CREATE OPERATOR ~ (
     RIGHTARG = ebuildversion
 );
 ALTER OPERATOR gentoobrowse.~ (ebuildversion, ebuildversion) OWNER TO gentoo;
--- Name: ~; Type: OPERATOR; Schema: gentoobrowse; Owner: gentoo
-CREATE OPERATOR ~ (
-    PROCEDURE = pathpartsmatchesspecs,
-    LEFTARG = text[],
-    RIGHTARG = filterspec[]
-);
-ALTER OPERATOR gentoobrowse.~ (text[], filterspec[]) OWNER TO gentoo;
 -- Name: categories; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: 
 CREATE TABLE categories (
     categoryid integer NOT NULL,
@@ -383,17 +355,6 @@ CREATE TABLE ebuilds (
 ALTER TABLE ebuilds OWNER TO gentoo;
 -- Name: TABLE ebuilds; Type: COMMENT; Schema: gentoobrowse; Owner: gentoo
 COMMENT ON TABLE ebuilds IS 'Ebuilds :-)';
--- Name: filetypes; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: 
-CREATE TABLE filetypes (
-    filetypeid integer NOT NULL,
-    description text NOT NULL,
-		phase integer NOT NULL,
-		updateOrder integer,
-    spec filterspec[] NOT NULL
-);
-ALTER TABLE filetypes OWNER TO gentoo;
--- Name: TABLE filetypes; Type: COMMENT; Schema: gentoobrowse; Owner: gentoo
-COMMENT ON TABLE filetypes IS 'What kind of file is this?... metadata, changelog, etc';
 -- Name: license; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: 
 CREATE TABLE license (
     name text NOT NULL,
@@ -651,9 +612,6 @@ ALTER TABLE ebuild_rdeps CLUSTER ON pk_ebuildrdeps;
 ALTER TABLE ONLY ebuild_uses
     ADD CONSTRAINT pk_ebuilduses PRIMARY KEY (ebuildid, use);
 ALTER TABLE ebuild_uses CLUSTER ON pk_ebuilduses;
--- Name: pk_fileclass; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace: 
-ALTER TABLE ONLY filetypes
-    ADD CONSTRAINT pk_fileclass PRIMARY KEY (filetypeid);
 -- Name: pk_news; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace: 
 ALTER TABLE ONLY news
     ADD CONSTRAINT pk_news PRIMARY KEY (newsid);
@@ -873,11 +831,6 @@ REVOKE ALL ON TABLE ebuilds FROM PUBLIC;
 REVOKE ALL ON TABLE ebuilds FROM gentoo;
 GRANT ALL ON TABLE ebuilds TO gentoo;
 GRANT SELECT ON TABLE ebuilds TO apache;
--- Name: filetypes; Type: ACL; Schema: gentoobrowse; Owner: gentoo
-REVOKE ALL ON TABLE filetypes FROM PUBLIC;
-REVOKE ALL ON TABLE filetypes FROM gentoo;
-GRANT ALL ON TABLE filetypes TO gentoo;
-GRANT SELECT ON TABLE filetypes TO apache;
 -- Name: masksets; Type: ACL; Schema: gentoobrowse; Owner: gentoo
 REVOKE ALL ON TABLE masksets FROM PUBLIC;
 REVOKE ALL ON TABLE masksets FROM gentoo;
diff --git a/gentoobrowse-api/service/changeSet.h b/gentoobrowse-api/service/changeSet.h
new file mode 100644
index 0000000..321de4e
--- /dev/null
+++ b/gentoobrowse-api/service/changeSet.h
@@ -0,0 +1,20 @@
+#ifndef CHANGESET_H
+#define CHANGESET_H
+
+#include <git2/diff.h>
+#include <portage-models.h>
+#include <boost/filesystem/path.hpp>
+#include <map>
+
+namespace Gentoo {
+	namespace Service {
+		struct ChangeDetails {
+			StringList pathParts;
+			git_delta_t changeType;
+		};
+		typedef std::map<boost::filesystem::path, ChangeDetails> ChangeSet;
+	}
+}
+
+#endif
+
diff --git a/gentoobrowse-api/service/maintenance/abstractFileProcessor.cpp b/gentoobrowse-api/service/maintenance/abstractFileProcessor.cpp
index cf453b6..52fa266 100644
--- a/gentoobrowse-api/service/maintenance/abstractFileProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/abstractFileProcessor.cpp
@@ -1,4 +1,7 @@
 #include "abstractFileProcessor.h"
+#include <factory.impl.h>
+
+INSTANTIATEVOIDFACTORY(Gentoo::Service::AbstractFileProcessor);
 
 namespace Gentoo {
 	namespace Service {
@@ -12,7 +15,7 @@ namespace Gentoo {
 		}
 
 		void
-		AbstractFileProcessor::apply(DB::Connection *)
+		AbstractFileProcessor::apply(DB::Connection *, ChangeSet &)
 		{
 		}
 
diff --git a/gentoobrowse-api/service/maintenance/abstractFileProcessor.h b/gentoobrowse-api/service/maintenance/abstractFileProcessor.h
index 3447d25..d25fb70 100644
--- a/gentoobrowse-api/service/maintenance/abstractFileProcessor.h
+++ b/gentoobrowse-api/service/maintenance/abstractFileProcessor.h
@@ -4,15 +4,23 @@
 #include <connection.h>
 #include <maintenance.h>
 #include <boost/filesystem/path.hpp>
+#include <factory.h>
+#include "changeSet.h"
 
 namespace Gentoo {
 	namespace Service {
+		typedef std::vector<std::string> PathParts;
+
 		class AbstractFileProcessor {
 			public:
 				virtual ~AbstractFileProcessor() = 0;
 
+				virtual unsigned char phase() const = 0;
+				virtual unsigned char order() const = 0;
+				virtual bool match(const PathParts & pp) const = 0;
+
 				virtual void prepare(DB::Connection *);
-				virtual void apply(DB::Connection *);
+				virtual void apply(DB::Connection *, ChangeSet &);
 				virtual void tidy(DB::Connection *);
 
 				virtual void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) = 0;
@@ -20,6 +28,7 @@ namespace Gentoo {
 				virtual void deleted(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn) = 0;
 		};
 		typedef boost::shared_ptr<AbstractFileProcessor> FileProcessorPtr;
+		typedef AdHoc::Factory<AbstractFileProcessor> FileProcessorFactory;
 	}
 }
 
diff --git a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp
index ad3fb0b..552448f 100644
--- a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp
@@ -12,7 +12,12 @@ using namespace AdHoc::FileUtils;
 
 namespace Gentoo {
 	namespace Service {
-		const int CategoryMetaProcessor::FILETYPEID = 10;
+		unsigned char CategoryMetaProcessor::phase() const { return 2; }
+		unsigned char CategoryMetaProcessor::order() const { return 2; }
+		bool CategoryMetaProcessor::match(const PathParts & pp) const
+		{
+			return (pp.size() == 2 && pp[1] == "metadata.xml");
+		}
 
 		void
 		CategoryMetaProcessor::created(DB::Connection * dbc, int64_t r, const StringList & fn, const boost::filesystem::path & path)
@@ -42,4 +47,5 @@ namespace Gentoo {
 		}
 	}
 }
+FACTORY(Gentoo::Service::CategoryMetaProcessor, Gentoo::Service::FileProcessorFactory);
 
diff --git a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.h b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.h
index 46bb691..570faef 100644
--- a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.h
+++ b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.h
@@ -8,7 +8,9 @@ namespace Gentoo {
 	namespace Service {
 		class CategoryMetaProcessor : public AbstractFileProcessor {
 			public:
-				static const int FILETYPEID;
+				unsigned char phase() const override;
+				unsigned char order() const override;
+				bool match(const PathParts & pp) const override;
 
 				void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
 				void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
diff --git a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp
index fad96b6..35a1bbf 100644
--- a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp
@@ -34,14 +34,18 @@ static Glib::RefPtr<Glib::Regex> packageVersion = Glib::Regex::create("^(.+)-([0
 
 namespace Gentoo {
 	namespace Service {
-
-		const int EbuildMetaProcessor::FILETYPEID = 1;
-
 		EbuildMetaProcessor::EbuildMetaProcessor() :
 			ebuildIds("ebuildId")
 		{
 		}
 
+		unsigned char EbuildMetaProcessor::phase() const { return 2; }
+		unsigned char EbuildMetaProcessor::order() const { return 1; }
+		bool EbuildMetaProcessor::match(const PathParts & pp) const
+		{
+			return (pp.size() == 4 && pp[0] == "metadata" && pp[1] == "md5-cache");
+		}
+
 		void
 		EbuildMetaProcessor::created(DB::Connection * dbc, int64_t repoId, const StringList & fn, const boost::filesystem::path & path)
 		{
@@ -128,7 +132,7 @@ namespace Gentoo {
 		}
 
 		void
-		EbuildMetaProcessor::apply(DB::Connection * dbc)
+		EbuildMetaProcessor::apply(DB::Connection * dbc, ChangeSet &)
 		{
 			if (!ebuildIds.entityIds.empty()) {
 				{
@@ -279,4 +283,5 @@ namespace Gentoo {
 		}
 	}
 }
+FACTORY(Gentoo::Service::EbuildMetaProcessor, Gentoo::Service::FileProcessorFactory);
 
diff --git a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h
index 1feb3aa..471a7d7 100644
--- a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h
+++ b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h
@@ -13,7 +13,9 @@ namespace Gentoo {
 	namespace Service {
 		class EbuildMetaProcessor : public AbstractFileProcessor {
 			public:
-				static const int FILETYPEID;
+				unsigned char phase() const override;
+				unsigned char order() const override;
+				bool match(const PathParts & pp) const override;
 
 				EbuildMetaProcessor();
 
@@ -23,7 +25,7 @@ namespace Gentoo {
 
 			private:
 				void prepare(DB::Connection * dbc) override;
-				void apply(DB::Connection * dbc) override;
+				void apply(DB::Connection * dbc, ChangeSet &) override;
 				void tidy(DB::Connection * dbc) override;
 
 				void perEbuildUpdates(DB::Connection * dbc, const Utils::EbuildCacheParser & ecp, int64_t ebuildId, bool newest, int64_t packageId);
diff --git a/gentoobrowse-api/service/maintenance/masksProcessor.cpp b/gentoobrowse-api/service/maintenance/masksProcessor.cpp
index e56153e..5126fd4 100644
--- a/gentoobrowse-api/service/maintenance/masksProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/masksProcessor.cpp
@@ -22,7 +22,12 @@ static AdHoc::Lexer::PatternPtr discard = AdHoc::LexerMatchers::regex("^([^\n]*)
 
 namespace Gentoo {
 	namespace Service {
-		const int MasksProcessor::FILETYPEID = 3;
+		unsigned char MasksProcessor::phase() const { return 2; }
+		unsigned char MasksProcessor::order() const { return 10; }
+		bool MasksProcessor::match(const PathParts & pp) const
+		{
+			return (pp.size() == 2 && pp[0] == "profiles" && pp[1] == "package.mask");
+		}
 
 		void
 		MasksProcessor::created(DB::Connection * dbc, int64_t r, const StringList & fn, const boost::filesystem::path & path)
@@ -94,7 +99,7 @@ namespace Gentoo {
 		}
 
 		void
-		MasksProcessor::apply(DB::Connection * dbc)
+		MasksProcessor::apply(DB::Connection * dbc, ChangeSet &)
 		{
 			DB::TablePatch e;
 			e.dest = "gentoobrowse.ebuild_masks";
@@ -112,4 +117,5 @@ namespace Gentoo {
 		}
 	}
 }
+FACTORY(Gentoo::Service::MasksProcessor, Gentoo::Service::FileProcessorFactory);
 
diff --git a/gentoobrowse-api/service/maintenance/masksProcessor.h b/gentoobrowse-api/service/maintenance/masksProcessor.h
index d076a26..3a07036 100644
--- a/gentoobrowse-api/service/maintenance/masksProcessor.h
+++ b/gentoobrowse-api/service/maintenance/masksProcessor.h
@@ -10,13 +10,15 @@ namespace Gentoo {
 	namespace Service {
 		class MasksProcessor : public AbstractFileProcessor {
 			public:
-				static const int FILETYPEID;
+				unsigned char phase() const override;
+				unsigned char order() const override;
+				bool match(const PathParts & pp) const override;
 
 				void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
 				void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
 				void deleted(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn) override;
 
-				void apply(DB::Connection *) override;
+				void apply(DB::Connection *, ChangeSet &) override;
 		};
 	}
 }
diff --git a/gentoobrowse-api/service/maintenance/newsProcessor.cpp b/gentoobrowse-api/service/maintenance/newsProcessor.cpp
index 07230ba..68e2b57 100644
--- a/gentoobrowse-api/service/maintenance/newsProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/newsProcessor.cpp
@@ -6,10 +6,16 @@
 #include <db/sqlInsertSerializer.h>
 #include <db/sqlUpdateSerializer.h>
 #include <sql/maintenance/newsDelete.sql.h>
+#include <boost/algorithm/string/predicate.hpp>
 
 namespace Gentoo {
 	namespace Service {
-		const int NewsProcessor::FILETYPEID = 11;
+		unsigned char NewsProcessor::phase() const { return 2; }
+		unsigned char NewsProcessor::order() const { return 10; }
+		bool NewsProcessor::match(const PathParts & pp) const
+		{
+			return (pp.size() == 4 && pp[0] == "metadata" && pp[1] == "news" && boost::algorithm::ends_with(pp[3], ".txt"));
+		}
 
 		void
 		NewsProcessor::created(DB::Connection * dbc, int64_t, const StringList &, const boost::filesystem::path & path)
@@ -42,4 +48,5 @@ namespace Gentoo {
 		}
 	}
 }
+FACTORY(Gentoo::Service::NewsProcessor, Gentoo::Service::FileProcessorFactory);
 
diff --git a/gentoobrowse-api/service/maintenance/newsProcessor.h b/gentoobrowse-api/service/maintenance/newsProcessor.h
index 0e2c956..b7a34ff 100644
--- a/gentoobrowse-api/service/maintenance/newsProcessor.h
+++ b/gentoobrowse-api/service/maintenance/newsProcessor.h
@@ -10,7 +10,9 @@ namespace Gentoo {
 	namespace Service {
 		class NewsProcessor : public AbstractFileProcessor {
 			public:
-				static const int FILETYPEID;
+				unsigned char phase() const override;
+				unsigned char order() const override;
+				bool match(const PathParts & pp) const override;
 
 				void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
 				void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
diff --git a/gentoobrowse-api/service/maintenance/packageMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/packageMetaProcessor.cpp
index 653db7f..f77c2bb 100644
--- a/gentoobrowse-api/service/maintenance/packageMetaProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/packageMetaProcessor.cpp
@@ -12,7 +12,13 @@ using namespace AdHoc::FileUtils;
 
 namespace Gentoo {
 	namespace Service {
-		const int PackageMetaProcessor::FILETYPEID = 4;
+		unsigned char PackageMetaProcessor::phase() const { return 2; }
+		unsigned char PackageMetaProcessor::order() const { return 3; }
+		bool PackageMetaProcessor::match(const PathParts & pp) const
+		{
+			return (pp.size() == 3 && pp[2] == "metadata.xml");
+		}
+
 
 		void
 		PackageMetaProcessor::created(DB::Connection * dbc, int64_t r, const StringList & fn, const boost::filesystem::path & path)
@@ -46,4 +52,5 @@ namespace Gentoo {
 		}
 	}
 }
+FACTORY(Gentoo::Service::PackageMetaProcessor, Gentoo::Service::FileProcessorFactory);
 
diff --git a/gentoobrowse-api/service/maintenance/packageMetaProcessor.h b/gentoobrowse-api/service/maintenance/packageMetaProcessor.h
index 02b5f84..84ce4e4 100644
--- a/gentoobrowse-api/service/maintenance/packageMetaProcessor.h
+++ b/gentoobrowse-api/service/maintenance/packageMetaProcessor.h
@@ -9,7 +9,9 @@ namespace Gentoo {
 	namespace Service {
 		class PackageMetaProcessor : public AbstractFileProcessor {
 			public:
-				static const int FILETYPEID;
+				unsigned char phase() const override;
+				unsigned char order() const override;
+				bool match(const PathParts & pp) const override;
 
 				void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
 				void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
diff --git a/gentoobrowse-api/service/maintenance/updatesProcessor.cpp b/gentoobrowse-api/service/maintenance/updatesProcessor.cpp
index e9bdda2..4717a95 100644
--- a/gentoobrowse-api/service/maintenance/updatesProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/updatesProcessor.cpp
@@ -11,9 +11,9 @@
 #include <sql/maintenance/categoryInsert.sql.h>
 #include <sql/maintenance/updatesMovePackages.sql.h>
 #include <sql/maintenance/updatesMoveUserPackages.sql.h>
-#include <sql/maintenance/updatesMoveFilelist.sql.h>
-#include <sql/maintenance/updatesMergeFilelistEntries.sql.h>
 #include <sql/portage/findPackage.sql.h>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/algorithm/string/join.hpp>
 
 using namespace AdHoc::FileUtils;
 
@@ -22,7 +22,12 @@ static Glib::RefPtr<Glib::Regex> move = Glib::Regex::create(
 
 namespace Gentoo {
 	namespace Service {
-		const int UpdatesProcessor::FILETYPEID = 12;
+		unsigned char UpdatesProcessor::phase() const { return 1; }
+		unsigned char UpdatesProcessor::order() const { return 1; }
+		bool UpdatesProcessor::match(const PathParts & pp) const
+		{
+			return (pp.size() == 3 && pp[0] == "profiles" && pp[1] == "updates" && pp[2].substr(1, 4) == "Q-20");
+		}
 
 		class UpdatesPatch : public DB::TablePatch, public DB::SqlWriter {
 			public:
@@ -31,7 +36,7 @@ namespace Gentoo {
 					findPackage(sql::portage::findPackage.select(db)),
 					movePackages(sql::maintenance::updatesMovePackages.modify(db)),
 					moveUserPackages(sql::maintenance::updatesMoveUserPackages.modify(db)),
-					moveFilelist(sql::maintenance::updatesMoveFilelist.modify(db)),
+					changes(nullptr),
 					updatePacks("updatePack")
 				{
 					this->src = Utils::Database::emptyClone(db, "gentoobrowse.updates");
@@ -61,7 +66,7 @@ namespace Gentoo {
 				DB::SelectCommandPtr findPackage;
 				DB::ModifyCommandPtr movePackages;
 				DB::ModifyCommandPtr moveUserPackages;
-				DB::ModifyCommandPtr moveFilelist;
+				ChangeSet * changes;
 				Utils::EntityWhereFilter<std::string> updatePacks;
 
 				void
@@ -88,12 +93,27 @@ namespace Gentoo {
 							movePackages->bindParamS(2, catfrom);
 							movePackages->bindParamS(3, catto);
 							movePackages->execute();
-							moveFilelist->bindParamS(0, catto);
-							moveFilelist->bindParamS(1, namefrom);
-							moveFilelist->bindParamS(2, nameto);
-							moveFilelist->bindParamS(3, catfrom);
-							moveFilelist->bindParamS(4, namefrom);
-							moveFilelist->execute();
+							// Move and/or merge file list entries
+							for(auto changei = changes->begin(); changei != changes->end();) {
+								auto & pp = changei->second.pathParts;
+								if (pp.size() == 4 && pp[0] == "metadata" && pp[1] == "md5-cache" && pp[2] == catfrom && boost::algorithm::starts_with(pp[3], namefrom + "-")) {
+									auto cd = changei->second;
+									cd.pathParts[2] = catto;
+									cd.pathParts[3] = nameto + cd.pathParts[3].substr(namefrom.length());
+									auto path = boost::algorithm::join(cd.pathParts, "/");
+									auto target = changes->find(path);
+									if (target == changes->end()) {
+										changes->insert({ path, cd });
+									}
+									else if (cd.changeType == GIT_DELTA_DELETED && target->second.changeType == GIT_DELTA_ADDED) {
+										target->second.changeType = GIT_DELTA_MODIFIED;
+									}
+									changei = changes->erase(changei);
+								}
+								else {
+									changei++;
+								}
+							}
 						}
 					});
 				}
@@ -106,11 +126,11 @@ namespace Gentoo {
 		}
 
 		void
-		UpdatesProcessor::apply(DB::Connection * dbc)
+		UpdatesProcessor::apply(DB::Connection * dbc, ChangeSet & changes)
 		{
 			if (!up->updatePacks.entityIds.empty()) {
+				up->changes = &changes;
 				dbc->patchTable(up);
-				sql::maintenance::updatesMergeFilelistEntries.modify(dbc)->execute();
 			}
 		}
 
@@ -152,4 +172,5 @@ namespace Gentoo {
 		}
 	}
 }
+FACTORY(Gentoo::Service::UpdatesProcessor, Gentoo::Service::FileProcessorFactory);
 
diff --git a/gentoobrowse-api/service/maintenance/updatesProcessor.h b/gentoobrowse-api/service/maintenance/updatesProcessor.h
index c69267c..7d7f258 100644
--- a/gentoobrowse-api/service/maintenance/updatesProcessor.h
+++ b/gentoobrowse-api/service/maintenance/updatesProcessor.h
@@ -11,14 +11,16 @@ namespace Gentoo {
 		class UpdatesPatch;
 		class UpdatesProcessor : public AbstractFileProcessor {
 			public:
-				static const int FILETYPEID;
+				unsigned char phase() const override;
+				unsigned char order() const override;
+				bool match(const PathParts & pp) const override;
 
 				void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
 				void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
 				void deleted(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn) override;
 
 				void prepare(DB::Connection * dbc) override;
-				void apply(DB::Connection * dbc) override;
+				void apply(DB::Connection * dbc, ChangeSet &) override;
 				void tidy(DB::Connection * dbc) override;
 
 			private:
diff --git a/gentoobrowse-api/service/maintenance/useGlobalProcessor.cpp b/gentoobrowse-api/service/maintenance/useGlobalProcessor.cpp
index 243d11b..7a3728e 100644
--- a/gentoobrowse-api/service/maintenance/useGlobalProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/useGlobalProcessor.cpp
@@ -11,7 +11,12 @@ static Glib::RefPtr<Glib::Regex> useDesc = Glib::Regex::create("^([^#\\s][^ ]*)\
 
 namespace Gentoo {
 	namespace Service {
-		const int UseGlobalProcessor::FILETYPEID = 5;
+		unsigned char UseGlobalProcessor::phase() const { return 2; }
+		unsigned char UseGlobalProcessor::order() const { return 10; }
+		bool UseGlobalProcessor::match(const PathParts & pp) const
+		{
+			return (pp.size() == 2 && pp[0] == "profiles" && pp[1] == "use.desc");
+		}
 
 		void
 		UseGlobalProcessor::created(DB::Connection * dbc, int64_t r, const StringList & fn, const boost::filesystem::path & path)
@@ -49,4 +54,5 @@ namespace Gentoo {
 		}
 	}
 }
+FACTORY(Gentoo::Service::UseGlobalProcessor, Gentoo::Service::FileProcessorFactory);
 
diff --git a/gentoobrowse-api/service/maintenance/useGlobalProcessor.h b/gentoobrowse-api/service/maintenance/useGlobalProcessor.h
index 2d2d6c7..fd54faa 100644
--- a/gentoobrowse-api/service/maintenance/useGlobalProcessor.h
+++ b/gentoobrowse-api/service/maintenance/useGlobalProcessor.h
@@ -10,7 +10,9 @@ namespace Gentoo {
 	namespace Service {
 		class UseGlobalProcessor : public AbstractFileProcessor {
 			public:
-				static const int FILETYPEID;
+				unsigned char phase() const override;
+				unsigned char order() const override;
+				bool match(const PathParts & pp) const override;
 
 				void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
 				void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
diff --git a/gentoobrowse-api/service/maintenance/useGroupProcessor.cpp b/gentoobrowse-api/service/maintenance/useGroupProcessor.cpp
index b21a701..757638b 100644
--- a/gentoobrowse-api/service/maintenance/useGroupProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/useGroupProcessor.cpp
@@ -9,6 +9,7 @@
 #include "sql/maintenance/useGroupsDelete.sql.h"
 #include "sql/maintenance/useGroupsInsert.sql.h"
 #include "sql/maintenance/useGroupsGetId.sql.h"
+#include <boost/algorithm/string/predicate.hpp>
 
 namespace U = Gentoo::Utils;
 
@@ -16,7 +17,12 @@ static Glib::RefPtr<Glib::Regex> useDesc = Glib::Regex::create("^([^#\\s][^ ]*)\
 
 namespace Gentoo {
 	namespace Service {
-		const int UseGroupProcessor::FILETYPEID = 9;
+		unsigned char UseGroupProcessor::phase() const { return 2; }
+		unsigned char UseGroupProcessor::order() const { return 10; }
+		bool UseGroupProcessor::match(const PathParts & pp) const
+		{
+			return (pp.size() == 3 && pp[0] == "profiles" && pp[1] == "desc" && boost::algorithm::ends_with(pp[2], ".desc"));
+		}
 
 		void
 		UseGroupProcessor::created(DB::Connection * dbc, int64_t, const StringList &, const boost::filesystem::path & path)
@@ -78,4 +84,5 @@ namespace Gentoo {
 		}
 	}
 }
+FACTORY(Gentoo::Service::UseGroupProcessor, Gentoo::Service::FileProcessorFactory);
 
diff --git a/gentoobrowse-api/service/maintenance/useGroupProcessor.h b/gentoobrowse-api/service/maintenance/useGroupProcessor.h
index 0e02947..2cb066e 100644
--- a/gentoobrowse-api/service/maintenance/useGroupProcessor.h
+++ b/gentoobrowse-api/service/maintenance/useGroupProcessor.h
@@ -10,7 +10,9 @@ namespace Gentoo {
 	namespace Service {
 		class UseGroupProcessor : public AbstractFileProcessor {
 			public:
-				static const int FILETYPEID;
+				unsigned char phase() const override;
+				unsigned char order() const override;
+				bool match(const PathParts & pp) const override;
 
 				void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
 				void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
diff --git a/gentoobrowse-api/service/maintenance/useLocalProcessor.cpp b/gentoobrowse-api/service/maintenance/useLocalProcessor.cpp
index 2b82616..bf961f5 100644
--- a/gentoobrowse-api/service/maintenance/useLocalProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/useLocalProcessor.cpp
@@ -13,7 +13,12 @@ static Glib::RefPtr<Glib::Regex> useDesc = Glib::Regex::create("^([^#\\s][^/]*)/
 
 namespace Gentoo {
 	namespace Service {
-		const int UseLocalProcessor::FILETYPEID = 6;
+		unsigned char UseLocalProcessor::phase() const { return 2; }
+		unsigned char UseLocalProcessor::order() const { return 10; }
+		bool UseLocalProcessor::match(const PathParts & pp) const
+		{
+			return (pp.size() == 2 && pp[0] == "profiles" && pp[1] == "use.local.desc");
+		}
 
 		void
 		UseLocalProcessor::created(DB::Connection * dbc, int64_t r, const StringList & fn, const boost::filesystem::path & path)
@@ -60,4 +65,5 @@ namespace Gentoo {
 		}
 	}
 }
+FACTORY(Gentoo::Service::UseLocalProcessor, Gentoo::Service::FileProcessorFactory);
 
diff --git a/gentoobrowse-api/service/maintenance/useLocalProcessor.h b/gentoobrowse-api/service/maintenance/useLocalProcessor.h
index d4fb59f..4e4bace 100644
--- a/gentoobrowse-api/service/maintenance/useLocalProcessor.h
+++ b/gentoobrowse-api/service/maintenance/useLocalProcessor.h
@@ -10,7 +10,9 @@ namespace Gentoo {
 	namespace Service {
 		class UseLocalProcessor : public AbstractFileProcessor {
 			public:
-				static const int FILETYPEID;
+				unsigned char phase() const override;
+				unsigned char order() const override;
+				bool match(const PathParts & pp) const override;
 
 				void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
 				void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override;
diff --git a/gentoobrowse-api/service/maintenanceCommon.cpp b/gentoobrowse-api/service/maintenanceCommon.cpp
index 63bb13d..4a11ed2 100644
--- a/gentoobrowse-api/service/maintenanceCommon.cpp
+++ b/gentoobrowse-api/service/maintenanceCommon.cpp
@@ -39,7 +39,6 @@ namespace Gentoo {
 		Maintenance::Maintenance(IceTray::DatabasePoolPtr d) :
 			IceTray::AbstractDatabaseClient(d)
 		{
-			setupFileProcessors();
 		}
 
 		Maintenance::~Maintenance()
diff --git a/gentoobrowse-api/service/maintenanceGitOperations.cpp b/gentoobrowse-api/service/maintenanceGitOperations.cpp
index 33340e8..bc66882 100644
--- a/gentoobrowse-api/service/maintenanceGitOperations.cpp
+++ b/gentoobrowse-api/service/maintenanceGitOperations.cpp
@@ -8,11 +8,6 @@
 #include <sql/maintenance/changeLogInsert.sql.h>
 #include <sql/maintenance/reposToUpdate.sql.h>
 #include <sql/maintenance/reposToGitRefresh.sql.h>
-#include <sql/maintenance/gitListCreateRaw.sql.h>
-#include <sql/maintenance/gitListCreate.sql.h>
-#include <sql/maintenance/gitListCreateIdx.sql.h>
-#include <sql/maintenance/filelistPhases.sql.h>
-#include <sql/maintenance/gitListChangesInPhase.sql.h>
 #include <portage-models.h>
 #include "utils/git.h"
 #include "converters.h"
@@ -99,28 +94,34 @@ namespace Gentoo {
 			sql::maintenance::reposToUpdate.select(dbc.get())->forEachRow<std::string, std::string, std::string>(&updateRepository);
 		}
 
+		static inline
+		PathParts
+		split(const boost::filesystem::path & path)
+		{
+			PathParts pp;
+			for (const auto & p : path) {
+				pp.push_back(p.string());
+			}
+			return pp;
+		}
+
 		static
 		int
 		insertFileChange(const git_diff_delta * delta, void * ptr)
 		{
-			auto ins = static_cast<DB::ModifyCommand *>(ptr);
+			auto cs = static_cast<ChangeSet *>(ptr);
 			switch (delta->status) {
 				case GIT_DELTA_ADDED:
-					ins->bindParamS(1, "A");
-					ins->bindParamS(2, delta->new_file.path);
-					break;
-				case GIT_DELTA_DELETED:
-					ins->bindParamS(1, "D");
-					ins->bindParamS(2, delta->old_file.path);
-					break;
 				case GIT_DELTA_MODIFIED:
-					ins->bindParamS(1, "M");
-					ins->bindParamS(2, delta->new_file.path);
+				case GIT_DELTA_DELETED:
+					{
+						boost::filesystem::path path(delta->new_file.path);
+						cs->insert({ path, { split(path), delta->status }});
+					}
 					break;
 				default:
 					throw GitError("Insert Git file changes", 0, 0, "Unexpected change status.");
 			}
-			ins->execute();
 			return 0;
 		}
 
@@ -133,10 +134,10 @@ namespace Gentoo {
 		}
 
 		static
-		void
+		ChangeSet
 		writeChangesToFileList(DB::Connection * db, int64_t repoId, git_repository * repo, const git_oid & last, const git_oid & head);
 
-		void
+		ChangeSet
 		refreshRepository(DB::Connection * db, Ice::PropertiesPtr properties, int64_t repoId, const std::string & repoName, const std::string & path, const std::string & lastCommitId)
 		{
 			auto targetRef = properties->getProperty("GentooBrowseAPI.RefreshTarget." + repoName);
@@ -153,12 +154,12 @@ namespace Gentoo {
 			updateLastCommit->execute();
 
 			writeChangeLog(db, repoId, repo.get(), lastCommitOid, headCommitOid);
-			writeChangesToFileList(db, repoId, repo.get(), lastCommitOid, headCommitOid);
+			return writeChangesToFileList(db, repoId, repo.get(), lastCommitOid, headCommitOid);
 		}
 
 		AdHocFormatter(FindingChanges, "Finding changes for repository %? with range %?...%?\n");
-		void
-		writeChangesToFileList(DB::Connection * db, int64_t repoId, git_repository * repo, const git_oid & lastCommitOid, const git_oid & headCommitOid)
+		ChangeSet
+		writeChangesToFileList(DB::Connection *, int64_t repoId, git_repository * repo, const git_oid & lastCommitOid, const git_oid & headCommitOid)
 		{
 			FindingChanges::write(std::cerr, repoId, lastCommitOid, headCommitOid);
 			auto lastCommit = gitSafeGet(git_commit_lookup, git_commit_free, repo, &lastCommitOid);
@@ -166,9 +167,9 @@ namespace Gentoo {
 			auto lastTree = gitSafeGet(git_commit_tree, git_tree_free, lastCommit.get());
 			auto headTree = gitSafeGet(git_commit_tree, git_tree_free, headCommit.get());
 			auto diff = gitSafeGet(git_diff_tree_to_tree, git_diff_free, repo, lastTree.get(), headTree.get(), nullptr);
-			auto ins = db->modify("INSERT INTO filelistraw(repoid, status, filename) VALUES(?, ?, ?)");
-			ins->bindParamI(0, repoId);
-			gitSafe(git_diff_foreach, diff.get(), insertFileChangeT<float>, insertFileChangeT<const git_diff_binary *>, nullptr, nullptr, ins.get());
+			ChangeSet changes;
+			gitSafe(git_diff_foreach, diff.get(), insertFileChangeT<float>, insertFileChangeT<const git_diff_binary *>, nullptr, nullptr, &changes);
+			return changes;
 		}
 
 		void
@@ -179,64 +180,59 @@ namespace Gentoo {
 			auto dbc = db->get();
 			dbc->execute("SET search_path = gentoobrowse, pg_catalog");
 			DB::TransactionScope tx(dbc.get());
-			updateFileTypes(dbc.get());
-			sql::maintenance::gitListCreateRaw.modify(dbc.get())->execute();
-			sql::maintenance::reposToGitRefresh.select(dbc.get())
-				->forEachRow<int64_t, std::string, std::string, std::string>(
-						boost::bind(&refreshRepository, dbc.get(), properties, _1, _2, _3, _4));
-			sql::maintenance::gitListCreate.modify(dbc.get())->execute();
-			sql::maintenance::gitListCreateIdx.modify(dbc.get())->execute();
-
-			applyFileChanges(dbc.get(), repoRoot);
-
-			dbc->execute("DROP TABLE filelist");
-			dbc->execute("DROP TABLE filelistraw");
+			sql::maintenance::reposToGitRefresh.select(dbc.get())->forEachRow<int64_t, std::string, std::string, std::string>(
+					[dbc, properties, repoRoot, this](auto repoId, auto name, auto path, auto lastCommit) {
+						auto changes = refreshRepository(dbc.get(), properties, repoId, name, path, lastCommit);
+						this->applyFileChanges(dbc.get(), repoId, path, repoRoot, changes);
+					});
 			dbc->execute("SET search_path = public, pg_catalog");
 		}
 
 		AdHocFormatter(UpdatingFileContent, "Updating file content\n");
 		AdHocFormatter(UpdatedFileContent, "Updated file content\n");
 		void
-		Maintenance::applyFileChanges(DB::Connection * dbc, const boost::filesystem::path & repoRoot) const
+		Maintenance::applyFileChanges(DB::Connection * dbc, int64_t repoId, const boost::filesystem::path & path, const boost::filesystem::path & repoRoot, ChangeSet & changes) const
 		{
 			UpdatingFileContent::write(std::cerr);
-			FileProcessors fps;
-			for (const auto & fpf : fpfs) {
-				fps[fpf.first] = fpf.second();
-			}
-			for (const auto & fp : fps) {
-				fp.second->prepare(dbc);
+			// Map of phase, to order, to processor
+			std::map<unsigned char, std::multimap<unsigned char, FileProcessorPtr>> fps;
+			for (const auto & fpf : AdHoc::PluginManager::getDefault()->getAll<FileProcessorFactory>()) {
+				FileProcessorPtr processor(fpf->implementation()->create());
+				fps[processor->phase()].insert({ processor->order(), processor })->second->prepare(dbc);
 			}
-			RepoMap repos;
-			dbc->select("SELECT path, repoid FROM gentoobrowse.repos")->forEachRow<std::string, int64_t>([&repos](auto path, auto id) {
-					repos[id] = path;
-				});
-			std::map<std::string, FileHandleFunc> funcs;
-			funcs["D"] = boost::bind(&AbstractFileProcessor::deleted, _1, dbc, _2, _3);
-			funcs["M"] = boost::bind(&AbstractFileProcessor::modified, _1, dbc, _2, _3, _4);
-			funcs["A"] = boost::bind(&AbstractFileProcessor::created, _1, dbc, _2, _3, _4);
-			auto phases = sql::maintenance::filelistPhases.select(dbc);
-			for (const auto & phaseRow : phases->as<int64_t>()) {
-				const auto & phase = phaseRow.value<0>();
-				auto files = sql::maintenance::gitListChangesInPhase.select(dbc);
-				files->bindParamI(0, phase);
-				std::set<int64_t> usedTypes;
-				for (const auto & fileRow : files->as<int64_t, int64_t, std::string, std::string>()) {
-					const auto & fileTypeId = fileRow.value<1>();
-					if (fps.find(fileTypeId) != fps.end()) {
-						const auto & repoId = fileRow.value<0>();
-						const auto & pathParts = fileRow.value<2>();
-						const auto & status = fileRow.value<3>();
-						usedTypes.insert(fileTypeId);
-						this->fileHandle(fileTypeId, &fps, &repos, repoId, pathParts, repoRoot, funcs[status]);
+			// Processor -> repoId, PathParts, Path
+			typedef boost::function<void(FileProcessorPtr, int64_t, const Gentoo::StringList &, const boost::filesystem::path &)> FileHandleFunc;
+			std::map<git_delta_t, FileHandleFunc> funcs;
+			funcs[GIT_DELTA_DELETED] = boost::bind(&AbstractFileProcessor::deleted, _1, dbc, _2, _3);
+			funcs[GIT_DELTA_MODIFIED] = boost::bind(&AbstractFileProcessor::modified, _1, dbc, _2, _3, _4);
+			funcs[GIT_DELTA_ADDED] = boost::bind(&AbstractFileProcessor::created, _1, dbc, _2, _3, _4);
+
+			// Phases
+			for (const auto & phase : fps) {
+				std::set<FileProcessorPtr> usedProcessors;
+				// Processors
+				for (const auto & processorOrder : phase.second) {
+					auto processor = processorOrder.second;
+					// Changes
+					for (auto change = changes.begin(); change != changes.end(); ) {
+						if (processor->match(change->second.pathParts)) {
+							funcs[change->second.changeType](processor, repoId, change->second.pathParts, repoRoot / path / change->first);
+							usedProcessors.insert(processor);
+							change = changes.erase(change);
+						}
+						else {
+							change++;
+						}
 					}
 				}
-				for (const auto & fp : usedTypes) {
-					fps[fp]->apply(dbc);
+				for (const auto & processor : usedProcessors) {
+					processor->apply(dbc, changes);
 				}
 			}
-			for (const auto & fp : fps) {
-				fp.second->tidy(dbc);
+			for (const auto & phase : fps) {
+				for (const auto & processor : phase.second) {
+					processor.second->tidy(dbc);
+				}
 			}
 			UpdatedFileContent::write(std::cerr);
 		}
diff --git a/gentoobrowse-api/service/maintenancePackageTree.cpp b/gentoobrowse-api/service/maintenancePackageTree.cpp
deleted file mode 100644
index 7201ac3..0000000
--- a/gentoobrowse-api/service/maintenancePackageTree.cpp
+++ /dev/null
@@ -1,121 +0,0 @@
-#include "maintenanceimpl.h"
-#include <selectcommandUtil.impl.h>
-#include <tablepatch.h>
-#include <sqlWriter.h>
-#include <buffer.h>
-#include <scopeExit.h>
-#include "utils/dbUtils.h"
-#include <boost/filesystem/operations.hpp>
-#include <boost/date_time/posix_time/posix_time.hpp>
-#include <boost/algorithm/string/predicate.hpp>
-#include <boost/algorithm/string/join.hpp>
-#include "maintenance/categoryMetaProcessor.h"
-#include "maintenance/packageMetaProcessor.h"
-#include "maintenance/ebuildMetaProcessor.h"
-#include "maintenance/useGlobalProcessor.h"
-#include "maintenance/useLocalProcessor.h"
-#include "maintenance/useGroupProcessor.h"
-#include "maintenance/masksProcessor.h"
-#include "maintenance/newsProcessor.h"
-#include "maintenance/updatesProcessor.h"
-
-namespace Gentoo {
-	const std::string fileTypes(
-R"DATA( 10	category metadata	2	2	{"(2,metadata.xml)"}
-4	package metadata.xml	2	3	{"(3,metadata.xml)"}
-1	ebuild metadata	2	1	{"(1,metadata)","(2,md5-cache)"}
-8	manifests	2	\N	{"(3,Manifest)"}
-5	use_global	2	\N	{"(1,profiles)","(2,use.desc)"}
-6	use_local	2	\N	{"(1,profiles)","(2,use.local.desc)"}
-9	use_grouped	2	\N	{"(1,profiles)","(2,desc)","(3,%.desc)"}
-3	masks	2	\N	{"(1,profiles)","(2,package.mask)"}
-11	news	2	\N	{"(1,metadata)","(2,news)","(4,%.txt)"}
-12	updates	1	\N	{"(1,profiles)","(2,updates)","(3,_Q-20__)"}
-)DATA");
-
-// 8	package manifests	{"(3,Manifest)"}
-// 7	licenses	{"(1,licenses)"}
-
-	namespace Service {
-		template<typename T>
-		FileProcessorPtr
-		Maintenance::createFileProessor()
-		{
-			return FileProcessorPtr(new T());
-		}
-
-		void
-		Maintenance::setupFileProcessors()
-		{
-			fpfs[CategoryMetaProcessor::FILETYPEID] = &createFileProessor<CategoryMetaProcessor>;
-			fpfs[PackageMetaProcessor::FILETYPEID] = &createFileProessor<PackageMetaProcessor>;
-			fpfs[EbuildMetaProcessor::FILETYPEID] = &createFileProessor<EbuildMetaProcessor>;
-			fpfs[UseGlobalProcessor::FILETYPEID] = &createFileProessor<UseGlobalProcessor>;
-			fpfs[UseLocalProcessor::FILETYPEID] = &createFileProessor<UseLocalProcessor>;
-			fpfs[UseGroupProcessor::FILETYPEID] = &createFileProessor<UseGroupProcessor>;
-			fpfs[MasksProcessor::FILETYPEID] = &createFileProessor<MasksProcessor>;
-			fpfs[NewsProcessor::FILETYPEID] = &createFileProessor<NewsProcessor>;
-			fpfs[UpdatesProcessor::FILETYPEID] = &createFileProessor<UpdatesProcessor>;
-		}
-
-		void
-		Maintenance::fileDeleted(DB::Connection * dbc, const FileProcessors * fps, const boost::filesystem::path & tmp, const RepoMap * repos, DB::SelectCommandPtr s)
-		{
-			auto f = boost::bind(&AbstractFileProcessor::deleted, _1, dbc, _2, _3);
-			// filesize, filetypeid, moddate, pathparts, repoid
-			s->forEachRow<int64_t, int64_t, boost::posix_time::ptime, std::string, int64_t>([&f,repos,dbc,&tmp,this,fps](auto, auto ft, auto, auto pp, auto repoid) {
-					this->fileHandle(ft, fps, repos, repoid, pp, tmp, f);
-				});
-		}
-
-		void
-		Maintenance::fileChanged(DB::Connection * dbc, const FileProcessors * fps, const boost::filesystem::path & tmp, const RepoMap * repos, DB::SelectCommandPtr s)
-		{
-			auto f = boost::bind(&AbstractFileProcessor::modified, _1, dbc, _2, _3, _4);
-			// pathparts, repoid, old_filesize, old_filetypeid, old_moddate, new_filesize, new_filetypeid, new_moddate
-			s->forEachRow<std::string, int64_t, int64_t, int64_t, boost::posix_time::ptime, int64_t, int64_t, boost::posix_time::ptime>([&f,repos,dbc,&tmp,this,fps](auto pp, auto repoid, auto, auto ft, auto, auto, auto, auto) {
-					this->fileHandle(ft, fps, repos, repoid, pp, tmp, f);
-				});
-		}
-
-		void
-		Maintenance::fileCreated(DB::Connection * dbc, const FileProcessors * fps, const boost::filesystem::path & tmp, const RepoMap * repos, DB::SelectCommandPtr s)
-		{
-			auto f = boost::bind(&AbstractFileProcessor::created, _1, dbc, _2, _3, _4);
-			// filesize, filetypeid, moddate, pathparts, repoid
-			s->forEachRow<int64_t, int64_t, boost::posix_time::ptime, std::string, int64_t>([&f,repos,dbc,&tmp,this,fps](auto, auto ft, auto, auto pp, auto repoid) {
-					this->fileHandle(ft, fps, repos, repoid, pp, tmp, f);
-				});
-		}
-
-		void
-		Maintenance::fileHandle(int64_t ft, const FileProcessors * fps, const RepoMap * repos, int64_t repoid, const std::string & pp, const boost::filesystem::path & tmp, const FileHandleFunc & f) const
-		{
-			auto i = fps->find(ft);
-			if (i != fps->end()) {
-				auto r = repos->find(repoid);
-				if (r != repos->end()) {
-					auto ppa = Slicer::unpackPqTextArray(pp);
-					boost::filesystem::path p = tmp / r->second / boost::algorithm::join(ppa, "/");
-					f(i->second, repoid, ppa, p);
-				}
-			}
-		}
-
-		void Maintenance::updateFileTypes(DB::Connection * dbc) const
-		{
-			DB::TablePatch p;
-			p.dest = "gentoobrowse.filetypes";
-			p.src = Utils::Database::emptyClone(dbc, p.dest);
-			p.pk = { "filetypeId" };
-			p.cols = { "filetypeId", "description", "spec", "phase", "updateOrder" };
-			std::stringstream buf(fileTypes);
-			dbc->beginBulkUpload((p.src + "(filetypeid, description, phase, updateorder, spec)").c_str(), "");
-			dbc->bulkUploadData(buf);
-			dbc->endBulkUpload(nullptr);
-			dbc->patchTable(&p);
-			Utils::Database::drop(dbc, p.src);
-		}
-	}
-}
-
diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h
index 3c43548..622a239 100644
--- a/gentoobrowse-api/service/maintenanceimpl.h
+++ b/gentoobrowse-api/service/maintenanceimpl.h
@@ -10,16 +10,12 @@
 #include <converters.h>
 #include "maintenance/abstractFileProcessor.h"
 #include <IceUtil/Timer.h>
+#include "changeSet.h"
 
 namespace Gentoo {
 	namespace Service {
 		class DLL_PUBLIC Maintenance : public Gentoo::Maintenance, public IceTray::AbstractDatabaseClient {
 			public:
-				typedef std::map<int64_t, boost::function<FileProcessorPtr()>> FileProcessorFactories;
-				typedef std::map<int64_t, FileProcessorPtr> FileProcessors;
-				typedef boost::function<void(FileProcessorPtr, int64_t, const Gentoo::StringList &, const boost::filesystem::path &)> FileHandleFunc;
-				typedef std::map<int64_t, std::string> RepoMap;
-
 				Maintenance(IceTray::DatabasePoolPtr d, Ice::CommunicatorPtr ic, Ice::PropertiesPtr p);
 				virtual ~Maintenance();
 
@@ -31,21 +27,14 @@ namespace Gentoo {
 
 			protected:
 				Maintenance(IceTray::DatabasePoolPtr d);
-				void applyFileChanges(DB::Connection *, const boost::filesystem::path &) const;
-				void updateFileTypes(DB::Connection *) const;
+				void applyFileChanges(DB::Connection *, int64_t, const boost::filesystem::path &, const boost::filesystem::path &, ChangeSet &) const;
 
 			private:
-				void setupFileProcessors();
-				void fileDeleted(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path &, const RepoMap *, DB::SelectCommandPtr s);
-				void fileChanged(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path &, const RepoMap *, DB::SelectCommandPtr s);
-				void fileCreated(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path & tmp, const RepoMap *, DB::SelectCommandPtr s);
-				void fileHandle(int64_t filetypeId, const FileProcessors *, const RepoMap *, int64_t repo, const std::string & pp, const boost::filesystem::path & tmp, const FileHandleFunc &) const;
 
 				static Ice::PropertiesPtr properties(const Ice::Current &);
 
 				template <typename T>
 				static FileProcessorPtr createFileProessor();
-				FileProcessorFactories fpfs;
 				IceUtil::TimerPtr taskRunner;
 		};
 	}
diff --git a/gentoobrowse-api/service/sql/maintenance/filelistPhases.sql b/gentoobrowse-api/service/sql/maintenance/filelistPhases.sql
deleted file mode 100644
index 5234f51..0000000
--- a/gentoobrowse-api/service/sql/maintenance/filelistPhases.sql
+++ /dev/null
@@ -1,4 +0,0 @@
-SELECT phase
-FROM filelist
-GROUP BY phase
-ORDER BY phase
diff --git a/gentoobrowse-api/service/sql/maintenance/gitListChangesInPhase.sql b/gentoobrowse-api/service/sql/maintenance/gitListChangesInPhase.sql
deleted file mode 100644
index 17dfdb3..0000000
--- a/gentoobrowse-api/service/sql/maintenance/gitListChangesInPhase.sql
+++ /dev/null
@@ -1,5 +0,0 @@
-SELECT repoId, fileTypeId, pathParts, status
-FROM filelist
-WHERE phase = ?
-AND status IS NOT NULL
-ORDER BY POSITION(status IN 'DMA'), updateOrder NULLS LAST
diff --git a/gentoobrowse-api/service/sql/maintenance/gitListCreate.sql b/gentoobrowse-api/service/sql/maintenance/gitListCreate.sql
deleted file mode 100644
index 20ca804..0000000
--- a/gentoobrowse-api/service/sql/maintenance/gitListCreate.sql
+++ /dev/null
@@ -1,10 +0,0 @@
-CREATE TEMPORARY TABLE filelist AS
-SELECT
-	fl.repoid,
-	fl.status,
-	ft.filetypeid,
-	ft.updateOrder,
-	ft.phase,
-	(STRING_TO_ARRAY(fl.filename, '/')) pathparts
-FROM filelistraw fl, gentoobrowse.filetypes ft
-WHERE (STRING_TO_ARRAY(fl.filename, '/')) ~ ft.spec
diff --git a/gentoobrowse-api/service/sql/maintenance/gitListCreateIdx.sql b/gentoobrowse-api/service/sql/maintenance/gitListCreateIdx.sql
deleted file mode 100644
index 1b3a6ca..0000000
--- a/gentoobrowse-api/service/sql/maintenance/gitListCreateIdx.sql
+++ /dev/null
@@ -1 +0,0 @@
-CREATE INDEX idx_filelist_mode ON filelist(phase, status)
diff --git a/gentoobrowse-api/service/sql/maintenance/gitListCreateRaw.sql b/gentoobrowse-api/service/sql/maintenance/gitListCreateRaw.sql
deleted file mode 100644
index 3007cd8..0000000
--- a/gentoobrowse-api/service/sql/maintenance/gitListCreateRaw.sql
+++ /dev/null
@@ -1,4 +0,0 @@
-CREATE TEMPORARY TABLE filelistraw (
-		repoid INT NOT NULL,
-		filename TEXT NOT NULL,
-		status CHARACTER(1) NOT NULL)
diff --git a/gentoobrowse-api/service/sql/maintenance/updatesMergeFilelistEntries.sql b/gentoobrowse-api/service/sql/maintenance/updatesMergeFilelistEntries.sql
deleted file mode 100644
index 68cdbee..0000000
--- a/gentoobrowse-api/service/sql/maintenance/updatesMergeFilelistEntries.sql
+++ /dev/null
@@ -1,9 +0,0 @@
-UPDATE filelist SET
-	status = CASE WHEN status = 'A' THEN 'M' END
-WHERE pathparts IN (
-		SELECT pathparts
-		FROM filelist
-		WHERE filetypeid = 1
-		AND status IN ('A', 'D')
-		GROUP BY pathparts
-		HAVING COUNT(*) = 2)
diff --git a/gentoobrowse-api/service/sql/maintenance/updatesMoveFilelist.sql b/gentoobrowse-api/service/sql/maintenance/updatesMoveFilelist.sql
deleted file mode 100644
index 0a34c82..0000000
--- a/gentoobrowse-api/service/sql/maintenance/updatesMoveFilelist.sql
+++ /dev/null
@@ -1,6 +0,0 @@
-UPDATE filelist SET
-	pathparts[3] = ?,
-	pathparts[4] = REGEXP_REPLACE(pathparts[4], CONCAT('^', REPLACE(?, '+', '\+'), '(-[0-9])'), CONCAT(?::text, '\1'))
-WHERE pathparts[3] = ?
-AND pathparts[4] ~ CONCAT('^', replace(?, '+', '\+'), '-[0-9]')
-AND filetypeId = 1
diff --git a/gentoobrowse-api/unittests/Jamfile.jam b/gentoobrowse-api/unittests/Jamfile.jam
index 74a67df..e6a7560 100644
--- a/gentoobrowse-api/unittests/Jamfile.jam
+++ b/gentoobrowse-api/unittests/Jamfile.jam
@@ -80,9 +80,6 @@ run
 
 run
 	testMaintenance.cpp
-	../service/sql/maintenance/gitListCreate.sql
-	../service/sql/maintenance/gitListCreateRaw.sql
-	../service/sql/maintenance/gitListCreateIdx.sql
 	: : :
 	<dependency>../db/schema.sql
 	<define>BOOST_TEST_DYN_LINK
@@ -90,9 +87,6 @@ run
 	<library>git2
 	<library>../service//icetray
 	<implicit-dependency>testCommon
-	<icetray.sql.namespace>Gentoo::Service
-	<icetray.sql.connector>postgresql
-	<icetray.sql.basedir>../service
 	: testMaintenance ;
 
 run
diff --git a/gentoobrowse-api/unittests/basedata.sql b/gentoobrowse-api/unittests/basedata.sql
deleted file mode 100644
index bcaf63d..0000000
--- a/gentoobrowse-api/unittests/basedata.sql
+++ /dev/null
@@ -1,9 +0,0 @@
-SET statement_timeout = 0;
-SET lock_timeout = 0;
-SET client_encoding = 'UTF8';
-SET standard_conforming_strings = on;
-SET check_function_bodies = false;
-SET client_min_messages = warning;
-SET search_path = gentoobrowse, pg_catalog;
-
-COPY gentoobrowse.filetypes (filetypeid, description, spec, phase, updateOrder) FROM '$SCRIPTDIR/fixtures/filetypes.dat';
diff --git a/gentoobrowse-api/unittests/data.sql b/gentoobrowse-api/unittests/data.sql
index 4120934..074b3d3 100644
--- a/gentoobrowse-api/unittests/data.sql
+++ b/gentoobrowse-api/unittests/data.sql
@@ -17,7 +17,6 @@ COPY gentoobrowse.masksets (setno, person, email, dateadded, message, n, atomspe
 COPY gentoobrowse.ebuild_masks (setno, ebuildid) FROM '$SCRIPTDIR/fixtures/ebuild_masks.dat';
 COPY gentoobrowse.ebuild_rdeps (ebuildid, packageid, versionspec, flags, op, slot) FROM '$SCRIPTDIR/fixtures/ebuild_rdeps.dat';
 COPY gentoobrowse.ebuild_uses (ebuildid, use) FROM '$SCRIPTDIR/fixtures/ebuild_uses.dat';
-COPY gentoobrowse.filetypes (filetypeid, description, spec, phase, updateOrder) FROM '$SCRIPTDIR/fixtures/filetypes.dat';
 COPY gentoobrowse.license (name, legalbumph) FROM '$SCRIPTDIR/fixtures/license.dat';
 COPY gentoobrowse.news (newsid, title, posted, authorname, authoremail, atomspec, body, urls) FROM '$SCRIPTDIR/fixtures/news.dat';
 COPY gentoobrowse.package_urls (packageid, url) FROM '$SCRIPTDIR/fixtures/package_urls.dat';
diff --git a/gentoobrowse-api/unittests/fixtures/files.dat b/gentoobrowse-api/unittests/fixtures/files.dat
deleted file mode 100644
index e69de29..0000000
--- a/gentoobrowse-api/unittests/fixtures/files.dat
+++ /dev/null
diff --git a/gentoobrowse-api/unittests/fixtures/filetypes.dat b/gentoobrowse-api/unittests/fixtures/filetypes.dat
deleted file mode 100644
index 34c95ee..0000000
--- a/gentoobrowse-api/unittests/fixtures/filetypes.dat
+++ /dev/null
@@ -1,2 +0,0 @@
-4	package metadata.xml	{"(3,metadata.xml)"}	1	\N
-11	news	{"(1,metadata)","(2,news)","(4,%.txt)"}	1	\N
diff --git a/gentoobrowse-api/unittests/mockDefs.cpp b/gentoobrowse-api/unittests/mockDefs.cpp
index a067b01..a865c39 100644
--- a/gentoobrowse-api/unittests/mockDefs.cpp
+++ b/gentoobrowse-api/unittests/mockDefs.cpp
@@ -29,8 +29,7 @@ Maintenance::Maintenance() :
 			"--GentooBrowseAPI.BugRoot=file://" + (rootDir / "fixtures" / "bugs").string()
 		}),
 	PQ::Mock("user=postgres dbname=postgres", "GentooBrowseAPI", {
-			rootDir.parent_path() / "db" / "schema.sql",
-			rootDir / "basedata.sql" })
+			rootDir.parent_path() / "db" / "schema.sql" })
 {
 	replace("mailserver", new MockMailServerImpl());
 }
diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp
index 1e3ae8e..0b451ca 100644
--- a/gentoobrowse-api/unittests/testMaintenance.cpp
+++ b/gentoobrowse-api/unittests/testMaintenance.cpp
@@ -8,9 +8,6 @@
 #include <git2.h>
 #include <fstream>
 #include <maintenanceimpl.h>
-#include <sql/maintenance/gitListCreateRaw.sql.h>
-#include <sql/maintenance/gitListCreate.sql.h>
-#include <sql/maintenance/gitListCreateIdx.sql.h>
 
 class MaintenanceClientCombined : public Maintenance, public TestClient { };
 
@@ -75,31 +72,23 @@ class M2 : public Gentoo::Service::Maintenance {
 		M2(IceTray::DatabasePoolPtr d) :
 			Gentoo::Service::Maintenance(d)
 		{
-			auto dbc = db->get();
-			DB::TransactionScope tx(dbc.get());
-			updateFileTypes(dbc.get());
 		}
 
-		void applyDiffOfFolders(int64_t repoId, const boost::filesystem::path & from, const boost::filesystem::path & to) const
+		void applyDiffOfFolders(const boost::filesystem::path & from, const boost::filesystem::path & to) const
 		{
 			auto dbc = db->get();
 			auto fromFiles = fileSet(from);
 			auto toFiles = fileSet(to);
 			dbc->execute("SET search_path = gentoobrowse, pg_catalog");
-			Gentoo::Service::sql::maintenance::gitListCreateRaw.modify(dbc.get())->execute();
-			auto ins = dbc->modify("INSERT INTO filelistraw(repoid, status, filename) VALUES(?, ?, ?)");
-			ins->bindParamI(0, repoId);
-			newFiles(ins, fromFiles, toFiles);
-			removedFiles(ins, fromFiles, toFiles);
-			changedFiles(ins, fromFiles, toFiles);
-			Gentoo::Service::sql::maintenance::gitListCreate.modify(dbc.get())->execute();
-			Gentoo::Service::sql::maintenance::gitListCreateIdx.modify(dbc.get())->execute();
+			DB::TransactionScope tx(dbc.get());
+			Gentoo::Service::ChangeSet changes;
+			newFiles(changes, fromFiles, toFiles);
+			removedFiles(changes, fromFiles, toFiles);
+			changedFiles(changes, fromFiles, toFiles);
 			boost::filesystem::remove(treeDir);
 			boost::filesystem::create_symlink(to, treeDir);
-			applyFileChanges(dbc.get(), "/");
+			applyFileChanges(dbc.get(), 1, "/", to, changes);
 			boost::filesystem::remove(treeDir);
-			dbc->execute("DROP TABLE filelist");
-			dbc->execute("DROP TABLE filelistraw");
 			dbc->execute("SET search_path = public, pg_catalog");
 		}
 
@@ -113,34 +102,38 @@ class M2 : public Gentoo::Service::Maintenance {
 			}
 			return found;
 		}
-		static void newFiles(DB::ModifyCommandPtr ins, const FileMap & from, const FileMap & to)
+
+		static Gentoo::Service::PathParts split(const boost::filesystem::path & path)
+		{
+			Gentoo::Service::PathParts pp;
+			for (const auto & p : path) {
+				pp.push_back(p.string());
+			}
+			return pp;
+		}
+
+		static void newFiles(Gentoo::Service::ChangeSet & changes, const FileMap & from, const FileMap & to)
 		{
-			ins->bindParamS(1, "A");
 			for(const auto & f : to) {
 				if (from.find(f.first) == from.end()) {
-					ins->bindParamS(2, f.first.string());
-					ins->execute();
+					changes[f.first] = { split(f.first), GIT_DELTA_ADDED };
 				}
 			}
 		}
-		static void removedFiles(DB::ModifyCommandPtr ins, const FileMap & from, const FileMap & to)
+		static void removedFiles(Gentoo::Service::ChangeSet & changes, const FileMap & from, const FileMap & to)
 		{
-			ins->bindParamS(1, "D");
 			for(const auto & f : from) {
 				if (to.find(f.first) == to.end()) {
-					ins->bindParamS(2, f.first.string());
-					ins->execute();
+					changes[f.first] = { split(f.first), GIT_DELTA_DELETED };
 				}
 			}
 		}
-		static void changedFiles(DB::ModifyCommandPtr ins, const FileMap & from, const FileMap & to)
+		static void changedFiles(Gentoo::Service::ChangeSet & changes, const FileMap & from, const FileMap & to)
 		{
-			ins->bindParamS(1, "M");
 			for(const auto & f : to) {
 				auto i = from.find(f.first);
 				if (i != from.end() && i->second != f.second) {
-					ins->bindParamS(2, f.first.string());
-					ins->execute();
+					changes[f.first] = { split(f.first), GIT_DELTA_MODIFIED };
 				}
 			}
 		}
@@ -172,19 +165,19 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree )
 	M2 m2(p);
 
 	BOOST_TEST_CONTEXT("4156eb45cf3b0ce1d7125b84efd8688c2d6e831d") {
-		m2.applyDiffOfFolders(1, binDir / "empty", rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d");
+		m2.applyDiffOfFolders(binDir / "empty", rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d");
 		doRefreshPackageTree(db,
 				5, 1, 482, 981, 3626, 4593, 501, 393, 238, 50, 1573, 2008, 1543, 81, 152);
 	}
 
 	BOOST_TEST_CONTEXT("756569aa764177340726dd3d40b41d89b11b20c7") {
-		m2.applyDiffOfFolders(1, rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d", rootDir / "fixtures" / "756569aa764177340726dd3d40b41d89b11b20c7");
+		m2.applyDiffOfFolders(rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d", rootDir / "fixtures" / "756569aa764177340726dd3d40b41d89b11b20c7");
 		doRefreshPackageTree(db,
 				5, 1, 483, 982, 3638, 4599, 502, 393, 238, 50, 1573, 2009, 1546, 79, 152);
 	}
 
 	BOOST_TEST_CONTEXT("empty") {
-		m2.applyDiffOfFolders(1, rootDir / "fixtures" / "756569aa764177340726dd3d40b41d89b11b20c7", binDir / "empty");
+		m2.applyDiffOfFolders(rootDir / "fixtures" / "756569aa764177340726dd3d40b41d89b11b20c7", binDir / "empty");
 		doRefreshPackageTree(db,
 				0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 	}