Build the package tree the ebuild cache

14 files changed, 92 insertions, 51 deletions

diff --git a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp
index cd4cdfe..72c41c9 100644
--- a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp
@@ -3,9 +3,7 @@
 #include "fileUtils.h"
 #include "xmlUtils.h"
 #include "dbUtils.h"
-#include <sql/maintenance/categoryInsert.sql.h>
-#include <sql/maintenance/categoryUpdate.sql.h>
-#include <sql/maintenance/categoryDelete.sql.h>
+#include <sql/maintenance/categoryMetaUpdate.sql.h>
 
 namespace U = Gentoo::Utils;
 using namespace Gentoo::Utils::File;
@@ -17,7 +15,7 @@ namespace Gentoo {
 		void
 		CategoryMetaProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const
 		{
-			upsert(dbc->modify(sql::maintenance::categoryInsert::sql), fn, path);
+			upsert(dbc->modify(sql::maintenance::categoryMetaUpdate::sql), fn, path);
 		}
 
 		void
@@ -35,14 +33,15 @@ namespace Gentoo {
 		void
 		CategoryMetaProcessor::modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const
 		{
-			upsert(dbc->modify(sql::maintenance::categoryUpdate::sql), fn, path);
+			upsert(dbc->modify(sql::maintenance::categoryMetaUpdate::sql), fn, path);
 		}
 
 		void
 		CategoryMetaProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const
 		{
-			auto m = dbc->modify(sql::maintenance::categoryDelete::sql);
-			m->bindParamS(0, (fn / 1).string());
+			auto m = dbc->modify(sql::maintenance::categoryMetaUpdate::sql);
+			m->bindNull(0);
+			m->bindParamS(1, (fn / 1).string());
 			m->execute();
 		}
 	}
diff --git a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp
index 1085b70..638d525 100644
--- a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp
@@ -5,6 +5,10 @@
 #include "fileUtils.h"
 #include "xmlUtils.h"
 #include "dbUtils.h"
+#include <sql/maintenance/categoryInsert.sql.h>
+#include <sql/maintenance/packageInsert.sql.h>
+#include <sql/maintenance/packagePrune.sql.h>
+#include <sql/maintenance/categoryPrune.sql.h>
 #include <sql/maintenance/ebuildInsert.sql.h>
 #include <sql/maintenance/ebuildUpdate.sql.h>
 #include <sql/maintenance/ebuildDelete.sql.h>
@@ -58,17 +62,35 @@ namespace Gentoo {
 			Glib::MatchInfo matches;
 			const Glib::ustring pv = (fn / 4).string();
 			if (packageVersion->match(pv, matches)) {
-				auto m = dbc->modify(sql::maintenance::ebuildInsert::sql);
 				EbuildCacheParser ecp(path);
+				const std::string repoName = (fn / 0).string();
+				const std::string categoryName = (fn / 3).string();
+				const std::string packageName = matches.fetch(1);
+				const std::string ebuildVersion = matches.fetch(2);
+
+				// Maybe create a category
+				auto ci = dbc->modify(sql::maintenance::categoryInsert::sql);
+				ci->bindParamS(0, categoryName);
+				ci->bindParamS(1, categoryName);
+				ci->execute();
+				// Maybe create a package
+				auto pi = dbc->modify(sql::maintenance::packageInsert::sql);
+				pi->bindParamS(0, packageName);
+				Utils::Database::bindOptionalsS(pi, 1, { ecp.get("DESCRIPTION") });
+				pi->bindParamS(2, packageName);
+				pi->bindParamS(3, categoryName);
+				pi->execute();
+				// Create an ebuild
+				auto m = dbc->modify(sql::maintenance::ebuildInsert::sql);
 				m->bindParamS(0, matches.fetch(2));
 				m->bindParamS(1, matches.fetch(2));
 				Utils::Database::bindOptionalsS(m, 2, { ecp.get("SLOT") });
 				Utils::Database::bindOptionalsS(m, 3, { ecp.get("LICENSE") });
 				m->bindParamT(4, boost::posix_time::from_time_t(ecp.getStat().st_mtim.tv_sec));
-				m->bindParamS(5, (fn / 0).string()); // repo
-				m->bindParamS(6, (fn / 3).string()); // category
-				m->bindParamS(7, matches.fetch(1)); // package
-				m->execute();
+				m->bindParamS(5, repoName);
+				m->bindParamS(6, categoryName);
+				m->bindParamS(7, packageName);
+				m->execute(false);
 			}
 		}
 
@@ -87,18 +109,38 @@ namespace Gentoo {
 				m->bindParamS(4, (fn / 3).string()); // category
 				m->bindParamS(5, matches.fetch(1)); // package
 				m->bindParamS(6, matches.fetch(2)); // version
-				m->execute();
+				m->execute(false);
 			}
 		}
 
 		void
 		EbuildMetaProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const
 		{
-			auto m = dbc->modify(sql::maintenance::ebuildDelete::sql);
-			m->bindParamS(0, (fn / 0).string());
-			m->bindParamS(1, (fn / 3).string());
-			m->bindParamS(2, (fn / 4).string());
-			m->execute();
+			Glib::MatchInfo matches;
+			const Glib::ustring pv = (fn / 4).string();
+			if (packageVersion->match(pv, matches)) {
+				const std::string repoName = (fn / 0).string();
+				const std::string categoryName = (fn / 3).string();
+				const std::string packageName = matches.fetch(1);
+				const std::string ebuildVersion = matches.fetch(2);
+
+				fprintf(stderr, "r = %s, c = %s, p = %s, v = %s\n",
+						repoName.c_str(), categoryName.c_str(), packageName.c_str(), ebuildVersion.c_str());
+
+				// Delete ebuild
+				auto m = dbc->modify(sql::maintenance::ebuildDelete::sql);
+				m->bindParamS(0, repoName);
+				m->bindParamS(1, categoryName);
+				m->bindParamS(2, packageName);
+				m->bindParamS(3, ebuildVersion);
+				m->execute(false);
+				// Prune packages
+				auto pp = dbc->modify(sql::maintenance::packagePrune::sql);
+				pp->bindParamS(0, categoryName);
+				pp->execute();
+				// Prune categories
+				dbc->execute(sql::maintenance::categoryPrune::sql);
+			}
 		}
 	}
 }
diff --git a/gentoobrowse-api/service/maintenance/packageManifestProcessor.cpp b/gentoobrowse-api/service/maintenance/packageManifestProcessor.cpp
index 8d6147c..a84f4a4 100644
--- a/gentoobrowse-api/service/maintenance/packageManifestProcessor.cpp
+++ b/gentoobrowse-api/service/maintenance/packageManifestProcessor.cpp
@@ -1,8 +1,6 @@
 #include "packageManifestProcessor.h"
 #include <modifycommand.h>
 #include "fileUtils.h"
-#include <sql/maintenance/packageInsert.sql.h>
-#include <sql/maintenance/packageDelete.sql.h>
 
 using namespace Gentoo::Utils::File;
 
@@ -13,20 +11,15 @@ namespace Gentoo {
 		void
 		PackageManifestProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path &) const
 		{
-			auto m = dbc->modify(sql::maintenance::packageInsert::sql);
-			m->bindParamS(0, (fn / 2).string());
-			m->bindParamS(1, fn.parent_path().string());
-			m->bindParamS(2, (fn / 1).string());
-			m->execute();
+			(void)dbc;
+			(void)fn;
 		}
 
 		void
 		PackageManifestProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const
 		{
-			auto m = dbc->modify(sql::maintenance::packageDelete::sql);
-			m->bindParamS(0, (fn / 2).string());
-			m->bindParamS(1, (fn / 1).string());
-			m->execute();
+			(void)dbc;
+			(void)fn;
 		}
 	}
 }
diff --git a/gentoobrowse-api/service/maintenanceimpl.cpp b/gentoobrowse-api/service/maintenanceimpl.cpp
index 18df8a1..5ea6b8e 100644
--- a/gentoobrowse-api/service/maintenanceimpl.cpp
+++ b/gentoobrowse-api/service/maintenanceimpl.cpp
@@ -14,9 +14,9 @@
 /*
 10	category metadata.xml	{"(2,metadata.xml)"}
 4	package metadata.xml	{"(3,metadata.xml)"}
-8	package manifests	{"(3,Manifest)"}
 1	ebuild metadata	{"(1,metadata)","(2,md5-cache)"}
 
+8	package manifests	{"(3,Manifest)"}
 2	changelog	{"(3,ChangeLog)"}
 3	masks	{"(1,profiles)","(2,package.mask)"}
 5	use_global	{"(1,profiles)","(2,use.desc)"}
diff --git a/gentoobrowse-api/service/sql/maintenance/categoryDelete.sql b/gentoobrowse-api/service/sql/maintenance/categoryDelete.sql
deleted file mode 100644
index ad961c8..0000000
--- a/gentoobrowse-api/service/sql/maintenance/categoryDelete.sql
+++ /dev/null
@@ -1,2 +0,0 @@
-DELETE FROM gentoobrowse.categories
-WHERE name = ?
diff --git a/gentoobrowse-api/service/sql/maintenance/categoryInsert.sql b/gentoobrowse-api/service/sql/maintenance/categoryInsert.sql
index fa8a1ca..27321a0 100644
--- a/gentoobrowse-api/service/sql/maintenance/categoryInsert.sql
+++ b/gentoobrowse-api/service/sql/maintenance/categoryInsert.sql
@@ -1,2 +1,3 @@
-INSERT INTO gentoobrowse.categories(summary, name)
-VALUES(gentoobrowse.textclean(?), ?)
+INSERT INTO gentoobrowse.categories(name)
+SELECT ?
+WHERE (NOT EXISTS (SELECT name FROM gentoobrowse.categories WHERE name = ?))
diff --git a/gentoobrowse-api/service/sql/maintenance/categoryUpdate.sql b/gentoobrowse-api/service/sql/maintenance/categoryMetaUpdate.sql
index 28c45b1..28c45b1 100644
--- a/gentoobrowse-api/service/sql/maintenance/categoryUpdate.sql
+++ b/gentoobrowse-api/service/sql/maintenance/categoryMetaUpdate.sql
diff --git a/gentoobrowse-api/service/sql/maintenance/categoryPrune.sql b/gentoobrowse-api/service/sql/maintenance/categoryPrune.sql
new file mode 100644
index 0000000..3b75683
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/categoryPrune.sql
@@ -0,0 +1,5 @@
+DELETE FROM gentoobrowse.categories c
+WHERE NOT EXISTS (
+	SELECT FROM gentoobrowse.packages p
+	WHERE p.categoryid = c.categoryid)
+
diff --git a/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql b/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql
index 2c33008..bd46bcd 100644
--- a/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql
+++ b/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql
@@ -5,4 +5,5 @@ AND c.categoryid = p.categoryid
 AND e.repoid = r.repoid
 AND r.name = ?
 AND c.name = ?
-AND CONCAT(p.name, '-', e.version) = ?
+AND p.name = ?
+AND e.version = ?
diff --git a/gentoobrowse-api/service/sql/maintenance/packageDelete.sql b/gentoobrowse-api/service/sql/maintenance/packageDelete.sql
deleted file mode 100644
index 5a2a0f1..0000000
--- a/gentoobrowse-api/service/sql/maintenance/packageDelete.sql
+++ /dev/null
@@ -1,4 +0,0 @@
-DELETE FROM gentoobrowse.packages p
-USING gentoobrowse.categories c
-WHERE p.name = ?
-AND c.name = ?
diff --git a/gentoobrowse-api/service/sql/maintenance/packageInsert.sql b/gentoobrowse-api/service/sql/maintenance/packageInsert.sql
index 223cdbe..8e910f8 100644
--- a/gentoobrowse-api/service/sql/maintenance/packageInsert.sql
+++ b/gentoobrowse-api/service/sql/maintenance/packageInsert.sql
@@ -1,4 +1,7 @@
 INSERT INTO gentoobrowse.packages(categoryid, name, description)
 SELECT c.categoryid, ?, ?
 FROM gentoobrowse.categories c
+	LEFT OUTER JOIN gentoobrowse.packages p
+		ON c.categoryid = p.categoryid AND p.name = ?
 WHERE c.name = ?
+AND p.packageid IS NULL
diff --git a/gentoobrowse-api/service/sql/maintenance/packagePrune.sql b/gentoobrowse-api/service/sql/maintenance/packagePrune.sql
new file mode 100644
index 0000000..510ed75
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/packagePrune.sql
@@ -0,0 +1,7 @@
+DELETE FROM gentoobrowse.packages p
+USING gentoobrowse.categories c
+WHERE p.categoryid = c.categoryid
+AND c.name = ?
+AND NOT EXISTS (
+	SELECT FROM gentoobrowse.ebuilds e
+	WHERE p.packageid = e.packageid)
diff --git a/gentoobrowse-api/unittests/fixtures/filetypes.dat b/gentoobrowse-api/unittests/fixtures/filetypes.dat
index 7f3d1dd..5d8ed02 100644
--- a/gentoobrowse-api/unittests/fixtures/filetypes.dat
+++ b/gentoobrowse-api/unittests/fixtures/filetypes.dat
@@ -1,11 +1,11 @@
 2	changelog	{"(3,ChangeLog)"}	\N
 3	masks	{"(1,profiles)","(2,package.mask)"}	\N
 7	licenses	{"(1,licenses)"}	\N
-8	manifests	{"(3,Manifest)"}	2
+8	manifests	{"(3,Manifest)"}	\N
 9	use_grouped	{"(1,profiles)","(2,desc)","(3,%.desc)"}	\N
 5	use_global	{"(1,profiles)","(2,use.desc)"}	\N
 6	use_local	{"(1,profiles)","(2,use.local.desc)"}	\N
-1	ebuild metadata	{"(1,metadata)","(2,md5-cache)"}	3
-10	category metadata	{"(2,metadata.xml)"}	1
-4	package metadata.xml	{"(3,metadata.xml)"}	\N
+1	ebuild metadata	{"(1,metadata)","(2,md5-cache)"}	1
+10	category metadata.xml	{"(2,metadata.xml)"}	2
+4	package metadata.xml	{"(3,metadata.xml)"}	3
 11	news	{"(1,metadata)","(2,news)","(4,%.txt)"}	\N
diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp
index f597f07..45819e0 100644
--- a/gentoobrowse-api/unittests/testMaintenance.cpp
+++ b/gentoobrowse-api/unittests/testMaintenance.cpp
@@ -58,10 +58,8 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree )
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.files", int64_t, 2084);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, 5);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories WHERE name = 'dev-vcs'", int64_t, 1);
-	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 478);
-	// Missing packages
-	// SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 981);
-	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 977);
+	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 482);
+	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 981);
 	db->execute("COPY gentoobrowse.categories TO '/tmp/categories1.tsv'");
 	db->execute("COPY gentoobrowse.packages TO '/tmp/packages1.tsv'");
 	db->execute("COPY gentoobrowse.ebuilds TO '/tmp/ebuilds1.tsv'");
@@ -73,10 +71,8 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree )
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.files", int64_t, 2034);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, 5);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories WHERE name = 'dev-vcs'", int64_t, 1);
-	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 480);
-	// Missing packages
-	// SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 982);
-	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 978);
+	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 484);
+	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 982);
 	db->execute("COPY gentoobrowse.categories TO '/tmp/categories2.tsv'");
 	db->execute("COPY gentoobrowse.packages TO '/tmp/packages2.tsv'");
 	db->execute("COPY gentoobrowse.ebuilds TO '/tmp/ebuilds2.tsv'");