Add multi-phase maintenance update cycles

7 files changed, 33 insertions, 42 deletions

diff --git a/gentoobrowse-api/db/schema.sql b/gentoobrowse-api/db/schema.sql
index 25a4532..c59a6d9 100644
--- a/gentoobrowse-api/db/schema.sql
+++ b/gentoobrowse-api/db/schema.sql
@@ -396,6 +396,8 @@ ALTER TABLE files OWNER TO gentoo;
 CREATE TABLE filetypes (
     filetypeid integer NOT NULL,
     description text NOT NULL,
+		phase integer NOT NULL,
+		updateOrder integer,
     spec filterspec[] NOT NULL
 );
 ALTER TABLE filetypes OWNER TO gentoo;
diff --git a/gentoobrowse-api/service/maintenancePackageTree.cpp b/gentoobrowse-api/service/maintenancePackageTree.cpp
index 5ba85ff..d31b267 100644
--- a/gentoobrowse-api/service/maintenancePackageTree.cpp
+++ b/gentoobrowse-api/service/maintenancePackageTree.cpp
@@ -23,17 +23,15 @@
 
 namespace Gentoo {
 	const std::string fileTypes(
-R"DATA( 10	category metadata	{"(2,metadata.xml)"}
-4	package metadata.xml	{"(3,metadata.xml)"}
-1	package metadata	{"(1,metadata)","(2,md5-cache)"}
-2	changelog	{"(3,ChangeLog)"}
-7	licenses	{"(1,licenses)"}
-8	manifests	{"(3,Manifest)"}
-5	use_global	{"(1,profiles)","(2,use.desc)"}
-6	use_local	{"(1,profiles)","(2,use.local.desc)"}
-9	use_grouped	{"(1,profiles)","(2,desc)","(3,%.desc)"}
-3	masks	{"(1,profiles)","(2,package.mask)"}
-11	news	{"(1,metadata)","(2,news)","(4,%.txt)"}
+R"DATA( 10	category metadata	2	2	{"(2,metadata.xml)"}
+4	package metadata.xml	2	3	{"(3,metadata.xml)"}
+1	package metadata	2	1	{"(1,metadata)","(2,md5-cache)"}
+8	manifests	2	\N	{"(3,Manifest)"}
+5	use_global	2	\N	{"(1,profiles)","(2,use.desc)"}
+6	use_local	2	\N	{"(1,profiles)","(2,use.local.desc)"}
+9	use_grouped	2	\N	{"(1,profiles)","(2,desc)","(3,%.desc)"}
+3	masks	2	\N	{"(1,profiles)","(2,package.mask)"}
+11	news	2	\N	{"(1,metadata)","(2,news)","(4,%.txt)"}
 )DATA");
 
 // 8	package manifests	{"(3,Manifest)"}
@@ -162,7 +160,7 @@ R"DATA( 10	category metadata	{"(2,metadata.xml)"}
 			p.dest = "gentoobrowse.filetypes";
 			p.src = Utils::Database::emptyClone(dbc, p.dest);
 			p.pk = { "filetypeId" };
-			p.cols = { "filetypeId", "description", "spec" };
+			p.cols = { "filetypeId", "description", "spec", "phase", "updateOrder" };
 			std::stringstream buf(fileTypes);
 			dbc->beginBulkUpload(p.src.c_str(), "");
 			dbc->bulkUploadData(buf);
@@ -183,21 +181,19 @@ R"DATA( 10	category metadata	{"(2,metadata.xml)"}
 			}
 			dbc->execute(sql::maintenance::fileListCreate.getSql());
 			dbc->execute(sql::maintenance::fileListCreatePk.getSql());
-			DB::TablePatch tp;
-			tp.src = "filelist";
-			tp.dest = "gentoobrowse.files";
-			tp.pk = {"pathparts", "repoid"};
-			tp.cols = {"pathparts", "repoid", "filetypeid", "filesize", "moddate"};
-			tp.beforeDelete = boost::bind(&Maintenance::fileDeleted, this, dbc, &fps, tmp, &repos, _1);
-			tp.beforeUpdate = boost::bind(&Maintenance::fileChanged, this, dbc, &fps, tmp, &repos, _1);
-			tp.beforeInsert = boost::bind(&Maintenance::fileCreated, this, dbc, &fps, tmp, &repos, _1);
-			DB::StaticSqlWriter obpo("CASE \
-					WHEN b.fileTypeId = 1 THEN 1 \
-					WHEN b.fileTypeId = 10 THEN 2 \
-					WHEN b.fileTypeId = 4 THEN 3 \
-					END NULLS LAST");
-			tp.order = &obpo;
-			dbc->patchTable(&tp);
+			for (int i = 1; i < 3; i +=1 ) {
+				DB::TablePatch tp;
+				tp.src = "filelist";
+				tp.dest = "gentoobrowse.files";
+				tp.pk = {"pathparts", "repoid"};
+				tp.cols = {"pathparts", "repoid", "filetypeid", "filesize", "moddate"};
+				tp.beforeDelete = boost::bind(&Maintenance::fileDeleted, this, dbc, &fps, tmp, &repos, _1);
+				tp.beforeUpdate = boost::bind(&Maintenance::fileChanged, this, dbc, &fps, tmp, &repos, _1);
+				tp.beforeInsert = boost::bind(&Maintenance::fileCreated, this, dbc, &fps, tmp, &repos, _1);
+				DB::StaticSqlWriter obpo("b.updateOrder NULLS LAST");
+				tp.order = &obpo;
+				dbc->patchTable(&tp);
+			}
 			for (const auto & fp : fps) {
 				fp.second->apply(dbc);
 			}
diff --git a/gentoobrowse-api/service/sql/maintenance/fileListCreate.sql b/gentoobrowse-api/service/sql/maintenance/fileListCreate.sql
index 565d31a..008b154 100644
--- a/gentoobrowse-api/service/sql/maintenance/fileListCreate.sql
+++ b/gentoobrowse-api/service/sql/maintenance/fileListCreate.sql
@@ -2,6 +2,8 @@ CREATE TEMPORARY TABLE filelist AS
 SELECT
 	r.repoid,
 	ft.filetypeid,
+	ft.updateOrder,
+	ft.phase,
 	(STRING_TO_ARRAY(fl.filename, '/'))[2:100] pathparts,
 	fl.filesize,
 	fl.filemtime moddate
diff --git a/gentoobrowse-api/unittests/basedata.sql b/gentoobrowse-api/unittests/basedata.sql
index e8e3d55..bcaf63d 100644
--- a/gentoobrowse-api/unittests/basedata.sql
+++ b/gentoobrowse-api/unittests/basedata.sql
@@ -6,4 +6,4 @@ SET check_function_bodies = false;
 SET client_min_messages = warning;
 SET search_path = gentoobrowse, pg_catalog;
 
-COPY gentoobrowse.filetypes (filetypeid, description, spec) FROM '$SCRIPTDIR/fixtures/filetypes.dat';
+COPY gentoobrowse.filetypes (filetypeid, description, spec, phase, updateOrder) FROM '$SCRIPTDIR/fixtures/filetypes.dat';
diff --git a/gentoobrowse-api/unittests/data.sql b/gentoobrowse-api/unittests/data.sql
index a5ea117..d1b70de 100644
--- a/gentoobrowse-api/unittests/data.sql
+++ b/gentoobrowse-api/unittests/data.sql
@@ -17,7 +17,7 @@ COPY gentoobrowse.masksets (setno, person, email, dateadded, message, n, atomspe
 COPY gentoobrowse.ebuild_masks (setno, ebuildid) FROM '$SCRIPTDIR/fixtures/ebuild_masks.dat';
 COPY gentoobrowse.ebuild_rdeps (ebuildid, packageid, versionspec, flags, op, slot) FROM '$SCRIPTDIR/fixtures/ebuild_rdeps.dat';
 COPY gentoobrowse.ebuild_uses (ebuildid, use) FROM '$SCRIPTDIR/fixtures/ebuild_uses.dat';
-COPY gentoobrowse.filetypes (filetypeid, description, spec) FROM '$SCRIPTDIR/fixtures/filetypes.dat';
+COPY gentoobrowse.filetypes (filetypeid, description, spec, phase, updateOrder) FROM '$SCRIPTDIR/fixtures/filetypes.dat';
 COPY gentoobrowse.files (moddate, filetypeid, repoid, filesize, pathparts) FROM '$SCRIPTDIR/fixtures/files.dat';
 COPY gentoobrowse.license (name, legalbumph) FROM '$SCRIPTDIR/fixtures/license.dat';
 COPY gentoobrowse.news (newsid, title, posted, authorname, authoremail, atomspec, body, urls) FROM '$SCRIPTDIR/fixtures/news.dat';
diff --git a/gentoobrowse-api/unittests/fixtures/filetypes.dat b/gentoobrowse-api/unittests/fixtures/filetypes.dat
index 5144c64..34c95ee 100644
--- a/gentoobrowse-api/unittests/fixtures/filetypes.dat
+++ b/gentoobrowse-api/unittests/fixtures/filetypes.dat
@@ -1,11 +1,2 @@
-2	changelog	{"(3,ChangeLog)"}
-3	masks	{"(1,profiles)","(2,package.mask)"}
-7	licenses	{"(1,licenses)"}
-8	manifests	{"(3,Manifest)"}
-9	use_grouped	{"(1,profiles)","(2,desc)","(3,%.desc)"}
-5	use_global	{"(1,profiles)","(2,use.desc)"}
-6	use_local	{"(1,profiles)","(2,use.local.desc)"}
-1	ebuild metadata	{"(1,metadata)","(2,md5-cache)"}
-10	category metadata.xml	{"(2,metadata.xml)"}
-4	package metadata.xml	{"(3,metadata.xml)"}
-11	news	{"(1,metadata)","(2,news)","(4,%.txt)"}
+4	package metadata.xml	{"(3,metadata.xml)"}	1	\N
+11	news	{"(1,metadata)","(2,news)","(4,%.txt)"}	1	\N
diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp
index fe0cbe3..6439923 100644
--- a/gentoobrowse-api/unittests/testMaintenance.cpp
+++ b/gentoobrowse-api/unittests/testMaintenance.cpp
@@ -101,11 +101,11 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree )
 	insRepo->execute();
 
 	doRefreshPackageTree(sd, db, "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d", "gentoo",
-			m, 2084, 5, 1, 482, 981, 3626, 4593, 501, 393, 238, 50, 1573, 2008, 1543, 81, 152);
+			m, 2080, 5, 1, 482, 981, 3626, 4593, 501, 393, 238, 50, 1573, 2008, 1543, 81, 152);
 	dumpDb(db);
 
 	doRefreshPackageTree(sd, db, "756569aa764177340726dd3d40b41d89b11b20c7", "gentoo",
-			m, 2087, 5, 1, 484, 982, 3638, 4599, 503, 393, 238, 50, 1573, 2009, 1546, 79, 152);
+			m, 2083, 5, 1, 484, 982, 3638, 4599, 503, 393, 238, 50, 1573, 2009, 1546, 79, 152);
 	dumpDb(db);
 
 	doRefreshPackageTree(sd, db, "", "gentoo",