Ingest grouped use flags

7 files changed, 130 insertions, 5 deletions

diff --git a/gentoobrowse-api/service/maintenance/useGroupProcessor.cpp b/gentoobrowse-api/service/maintenance/useGroupProcessor.cpp
new file mode 100644
index 0000000..5fbdbca
--- /dev/null
+++ b/gentoobrowse-api/service/maintenance/useGroupProcessor.cpp
@@ -0,0 +1,81 @@
+#include "useGroupProcessor.h"
+#include <modifycommand.h>
+#include <selectcommandUtil.impl.h>
+#include <tablepatch.h>
+#include "utils/fileUtils.h"
+#include "utils/dbUtils.h"
+#include "utils/entityWhereFilter.h"
+#include <glibmm/regex.h>
+#include "sql/maintenance/useGroupsDelete.sql.h"
+#include "sql/maintenance/useGroupsInsert.sql.h"
+#include "sql/maintenance/useGroupsGetId.sql.h"
+
+namespace U = Gentoo::Utils;
+
+static Glib::RefPtr<Glib::Regex> useDesc = Glib::Regex::create("^([^#\\s][^ ]*)\\s+-\\s+(.*)$", Glib::RegexCompileFlags::REGEX_MULTILINE);
+
+namespace Gentoo {
+	namespace Service {
+		const int UseGroupProcessor::FILETYPEID = 9;
+
+		void
+		UseGroupProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const
+		{
+			auto m = dbc->select(sql::maintenance::useGroupsInsert::sql);
+			m->bindParamS(0, fn.stem().string());
+			m->forEachRow<int64_t>([this, dbc, &path](auto useGroupId) {
+					this->mergeContent(dbc, path, useGroupId);
+				});
+		}
+
+		void
+		UseGroupProcessor::modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const
+		{
+			auto m = dbc->select(sql::maintenance::useGroupsGetId::sql);
+			m->bindParamS(0, fn.stem().string());
+			m->forEachRow<int64_t>([this, dbc, &path](auto useGroupId) {
+					this->mergeContent(dbc, path, useGroupId);
+				});
+		}
+
+		void
+		UseGroupProcessor::mergeContent(DB::Connection * dbc, const boost::filesystem::path & path, int64_t useGroupId) const
+		{
+			DB::TablePatch p;
+			p.dest = "gentoobrowse.use_group";
+			U::EntityWhereFilter gwf("useGroupId", useGroupId);
+			p.src = Utils::Database::emptyClone(dbc, "gentoobrowse.use_group");
+			p.pk = { "useGroupId", "use" };
+			p.cols = { "useGroupId", "use", "description" };
+			p.where = &gwf;
+
+			Utils::MemMap u(path);
+			Glib::ustring d(std::string(reinterpret_cast<const char *>(u.data), u.getStat().st_size));
+			Glib::MatchInfo matches;
+			auto i = Utils::Database::tablePatchInserter(dbc, p);
+			i->bindParamI(2, useGroupId);
+			// Dirty, but dupes exist
+			std::set<std::string> uses;
+			for (useDesc->match(d, matches); matches.get_match_count() == 3; matches.next()) {
+				if (uses.find(matches.fetch(1)) == uses.end()) {
+					i->bindParamS(1, matches.fetch(1));
+					i->bindParamS(0, matches.fetch(2));
+					i->execute();
+					uses.insert(matches.fetch(1));
+				}
+			}
+			dbc->patchTable(&p);
+
+			Utils::Database::drop(dbc, p.src);
+		}
+
+		void
+		UseGroupProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const
+		{
+			auto m = dbc->modify(sql::maintenance::useGroupsDelete::sql);
+			m->bindParamS(0, fn.stem().string());
+			m->execute();
+		}
+	}
+}
+
diff --git a/gentoobrowse-api/service/maintenance/useGroupProcessor.h b/gentoobrowse-api/service/maintenance/useGroupProcessor.h
new file mode 100644
index 0000000..dc328ce
--- /dev/null
+++ b/gentoobrowse-api/service/maintenance/useGroupProcessor.h
@@ -0,0 +1,26 @@
+#ifndef GENTOOBROWSE_API_SERVICE_MAINTENANCE_USEGROUPPROC_H
+#define GENTOOBROWSE_API_SERVICE_MAINTENANCE_USEGROUPPROC_H
+
+#include "../maintenanceimpl.h"
+#include <connection.h>
+#include <modifycommand.h>
+#include <boost/filesystem/path.hpp>
+
+namespace Gentoo {
+	namespace Service {
+		class UseGroupProcessor : public Maintenance::FileProcessor {
+			public:
+				static const int FILETYPEID;
+
+				void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const;
+				void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const;
+				void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const;
+
+			private:
+				void mergeContent(DB::Connection *, const boost::filesystem::path &, int64_t id) const;
+		};
+	}
+}
+
+#endif
+
diff --git a/gentoobrowse-api/service/maintenanceimpl.cpp b/gentoobrowse-api/service/maintenanceimpl.cpp
index d08f961..694fa56 100644
--- a/gentoobrowse-api/service/maintenanceimpl.cpp
+++ b/gentoobrowse-api/service/maintenanceimpl.cpp
@@ -12,6 +12,7 @@
 #include "maintenance/ebuildMetaProcessor.h"
 #include "maintenance/useGlobalProcessor.h"
 #include "maintenance/useLocalProcessor.h"
+#include "maintenance/useGroupProcessor.h"
 
 /*
 10	category metadata.xml	{"(2,metadata.xml)"}
@@ -19,12 +20,12 @@
 1	ebuild metadata	{"(1,metadata)","(2,md5-cache)"}
 5	use_global	{"(1,profiles)","(2,use.desc)"}
 6	use_local	{"(1,profiles)","(2,use.local.desc)"}
+9	use_grouped	{"(1,profiles)","(2,desc)","(3,%.desc)"}
 
 8	package manifests	{"(3,Manifest)"}
 2	changelog	{"(3,ChangeLog)"}
 3	masks	{"(1,profiles)","(2,package.mask)"}
 7	licenses	{"(1,licenses)"}
-9	use_grouped	{"(1,profiles)","(2,desc)","(3,%.desc)"}
 11	news	{"(1,metadata)","(2,news)","(4,%.txt)"}
 */
 
@@ -58,6 +59,7 @@ namespace Gentoo {
 			fps[EbuildMetaProcessor::FILETYPEID] = new EbuildMetaProcessor();
 			fps[UseGlobalProcessor::FILETYPEID] = new UseGlobalProcessor();
 			fps[UseLocalProcessor::FILETYPEID] = new UseLocalProcessor();
+			fps[UseGroupProcessor::FILETYPEID] = new UseGroupProcessor();
 		}
 
 		Maintenance::~Maintenance()
diff --git a/gentoobrowse-api/service/sql/maintenance/useGroupsDelete.sql b/gentoobrowse-api/service/sql/maintenance/useGroupsDelete.sql
new file mode 100644
index 0000000..7fbac23
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/useGroupsDelete.sql
@@ -0,0 +1,2 @@
+DELETE FROM gentoobrowse.use_groups
+WHERE name = ?
diff --git a/gentoobrowse-api/service/sql/maintenance/useGroupsGetId.sql b/gentoobrowse-api/service/sql/maintenance/useGroupsGetId.sql
new file mode 100644
index 0000000..56d05c2
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/useGroupsGetId.sql
@@ -0,0 +1,4 @@
+-- libdbpp:no-cursor
+SELECT usegroupid
+FROM gentoobrowse.use_groups
+WHERE name = ?
diff --git a/gentoobrowse-api/service/sql/maintenance/useGroupsInsert.sql b/gentoobrowse-api/service/sql/maintenance/useGroupsInsert.sql
new file mode 100644
index 0000000..1a99c8a
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/useGroupsInsert.sql
@@ -0,0 +1,4 @@
+-- libdbpp:no-cursor
+INSERT INTO gentoobrowse.use_groups(name)
+VALUES(?)
+RETURNING usegroupid
diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp
index d5bf738..a339524 100644
--- a/gentoobrowse-api/unittests/testMaintenance.cpp
+++ b/gentoobrowse-api/unittests/testMaintenance.cpp
@@ -45,7 +45,7 @@ BOOST_FIXTURE_TEST_SUITE(tp, TestClient)
 void
 doRefreshPackageTree(SampleData & sd, DB::ConnectionPtr db, const std::string & archive, const std::string & dir,
 		Gentoo::MaintenancePrx m, int64_t files, int64_t cats, int64_t devvcs, int64_t pkgs, int64_t ebs, int64_t ebus,
-		int64_t ebas, int64_t pus, int64_t ug, int64_t ul)
+		int64_t ebas, int64_t pus, int64_t ug, int64_t ul, int64_t ugs, int64_t ugds)
 {
 	if (!archive.empty()) {
 		sd.extract(archive, dir);
@@ -63,6 +63,8 @@ doRefreshPackageTree(SampleData & sd, DB::ConnectionPtr db, const std::string &
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.package_urls", int64_t, pus);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.use_global", int64_t, ug);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.use_local", int64_t, ul);
+	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.use_groups", int64_t, ugs);
+	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.use_group", int64_t, ugds);
 }
 
 BOOST_AUTO_TEST_CASE( refreshPackageTree )
@@ -75,7 +77,7 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree )
 	insRepo->execute();
 
 	doRefreshPackageTree(sd, db, "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d", "gentoo",
-			m, 2084, 5, 1, 482, 981, 3626, 4593, 501, 393, 238);
+			m, 2084, 5, 1, 482, 981, 3626, 4593, 501, 393, 238, 50, 1573);
 
 	db->execute("COPY gentoobrowse.categories TO '/tmp/categories1.tsv'");
 	db->execute("COPY gentoobrowse.packages TO '/tmp/packages1.tsv'");
@@ -85,9 +87,11 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree )
 	db->execute("COPY gentoobrowse.package_urls TO '/tmp/package_urls1.tsv'");
 	db->execute("COPY gentoobrowse.use_global TO '/tmp/use_global1.tsv'");
 	db->execute("COPY gentoobrowse.use_local TO '/tmp/use_local1.tsv'");
+	db->execute("COPY gentoobrowse.use_groups TO '/tmp/use_groups1.tsv'");
+	db->execute("COPY gentoobrowse.use_group TO '/tmp/use_group1.tsv'");
 
 	doRefreshPackageTree(sd, db, "756569aa764177340726dd3d40b41d89b11b20c7", "gentoo",
-			m, 2087, 5, 1, 484, 982, 3638, 4599, 503, 393, 238);
+			m, 2087, 5, 1, 484, 982, 3638, 4599, 503, 393, 238, 50, 1573);
 
 	db->execute("COPY gentoobrowse.categories TO '/tmp/categories2.tsv'");
 	db->execute("COPY gentoobrowse.packages TO '/tmp/packages2.tsv'");
@@ -97,9 +101,11 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree )
 	db->execute("COPY gentoobrowse.package_urls TO '/tmp/package_urls2.tsv'");
 	db->execute("COPY gentoobrowse.use_global TO '/tmp/use_global2.tsv'");
 	db->execute("COPY gentoobrowse.use_local TO '/tmp/use_local2.tsv'");
+	db->execute("COPY gentoobrowse.use_groups TO '/tmp/use_groups2.tsv'");
+	db->execute("COPY gentoobrowse.use_group TO '/tmp/use_group2.tsv'");
 
 	doRefreshPackageTree(sd, db, "", "gentoo",
-			m, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+			m, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 	m->refreshPackageTree();
 }