Basic ebuild ingest

8 files changed, 182 insertions, 5 deletions

diff --git a/gentoobrowse-api/db/schema.sql b/gentoobrowse-api/db/schema.sql
index 105ce3f..cb9f716 100644
--- a/gentoobrowse-api/db/schema.sql
+++ b/gentoobrowse-api/db/schema.sql
@@ -329,7 +329,7 @@ CREATE TABLE ebuilds (
     versioninst ebuildversion NOT NULL,
     slot text NOT NULL,
     license text,
-    firstseen timestamp without time zone NOT NULL,
+    firstseen timestamp without time zone NOT NULL DEFAULT now(),
     moddate timestamp without time zone NOT NULL,
     repoid integer NOT NULL
 );
@@ -491,6 +491,16 @@ CREATE SEQUENCE seq_packageid
 ALTER TABLE seq_packageid OWNER TO gentoo;
 -- Name: seq_packageid; Type: SEQUENCE OWNED BY; Schema: gentoobrowse; Owner: gentoo
 ALTER SEQUENCE seq_packageid OWNED BY packages.packageid;
+-- Name: seq_ebuildid; Type: SEQUENCE; Schema: gentoobrowse; Owner: gentoo
+CREATE SEQUENCE seq_ebuildid
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+ALTER TABLE seq_ebuildid OWNER TO gentoo;
+-- Name: seq_ebuildid; Type: SEQUENCE OWNED BY; Schema: gentoobrowse; Owner: gentoo
+ALTER SEQUENCE seq_ebuildid OWNED BY ebuilds.ebuildid;
 -- Name: use_global; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: 
 CREATE TABLE use_global (
     use text NOT NULL,
@@ -573,6 +583,8 @@ ALTER TABLE ONLY files ALTER COLUMN fileid SET DEFAULT nextval('seq_fileid'::reg
 ALTER TABLE ONLY masksets ALTER COLUMN setno SET DEFAULT nextval('masksets_setno_seq'::regclass);
 -- Name: packageid; Type: DEFAULT; Schema: gentoobrowse; Owner: gentoo
 ALTER TABLE ONLY packages ALTER COLUMN packageid SET DEFAULT nextval('seq_packageid'::regclass);
+-- Name: ebuildid; Type: DEFAULT; Schema: gentoobrowse; Owner: gentoo
+ALTER TABLE ONLY ebuilds ALTER COLUMN ebuildid SET DEFAULT nextval('seq_ebuildid'::regclass);
 -- Name: repoid; Type: DEFAULT; Schema: gentoobrowse; Owner: gentoo
 ALTER TABLE ONLY repos ALTER COLUMN repoid SET DEFAULT nextval('repos_repoid_seq'::regclass);
 -- Name: usegroupid; Type: DEFAULT; Schema: gentoobrowse; Owner: gentoo
diff --git a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp
new file mode 100644
index 0000000..1085b70
--- /dev/null
+++ b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp
@@ -0,0 +1,105 @@
+#include "ebuildMetaProcessor.h"
+#include <modifycommand.h>
+#include <boost/filesystem/operations.hpp>
+#include <glibmm/regex.h>
+#include "fileUtils.h"
+#include "xmlUtils.h"
+#include "dbUtils.h"
+#include <sql/maintenance/ebuildInsert.sql.h>
+#include <sql/maintenance/ebuildUpdate.sql.h>
+#include <sql/maintenance/ebuildDelete.sql.h>
+
+namespace U = Gentoo::Utils;
+using namespace Gentoo::Utils::File;
+
+static Glib::RefPtr<Glib::Regex> packageVersion = Glib::Regex::create("^(.+)-([0-9].*)$");
+
+namespace Gentoo {
+	namespace Service {
+		class EbuildCacheParser : public U::MemMap {
+			public:
+				typedef std::pair<const char *, const char *> Range;
+				typedef std::map<std::string, const Range> KVs;
+
+				EbuildCacheParser(const boost::filesystem::path & p) :
+					U::MemMap(p)
+				{
+					const char * chardata = (const char *)this->data;
+					while (const char * eq = strchr(chardata, '=')) {
+						if (const char * nl = strchr(eq + 1, '\n')) {
+							kvs.insert({ std::string(chardata, eq), { eq + 1, nl } });
+							chardata = nl + 1;
+						}
+						else {
+							kvs.insert({ std::string(chardata, eq), { eq + 1, (const char *)this->data + st.st_size } });
+							return;
+						}
+					}
+				}
+
+				boost::optional<Glib::ustring> get(const std::string & key) const
+				{
+					auto kvi = kvs.find(key);
+					if (kvi == kvs.end()) {
+						return boost::optional<Glib::ustring>();
+					}
+					return Glib::ustring(kvi->second.first, kvi->second.second);
+				}
+
+			private:
+				KVs kvs;
+		};
+
+		const int EbuildMetaProcessor::FILETYPEID = 1;
+
+		void
+		EbuildMetaProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const
+		{
+			Glib::MatchInfo matches;
+			const Glib::ustring pv = (fn / 4).string();
+			if (packageVersion->match(pv, matches)) {
+				auto m = dbc->modify(sql::maintenance::ebuildInsert::sql);
+				EbuildCacheParser ecp(path);
+				m->bindParamS(0, matches.fetch(2));
+				m->bindParamS(1, matches.fetch(2));
+				Utils::Database::bindOptionalsS(m, 2, { ecp.get("SLOT") });
+				Utils::Database::bindOptionalsS(m, 3, { ecp.get("LICENSE") });
+				m->bindParamT(4, boost::posix_time::from_time_t(ecp.getStat().st_mtim.tv_sec));
+				m->bindParamS(5, (fn / 0).string()); // repo
+				m->bindParamS(6, (fn / 3).string()); // category
+				m->bindParamS(7, matches.fetch(1)); // package
+				m->execute();
+			}
+		}
+
+		void
+		EbuildMetaProcessor::modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const
+		{
+			Glib::MatchInfo matches;
+			const Glib::ustring pv = (fn / 4).string();
+			if (packageVersion->match(pv, matches)) {
+				auto m = dbc->modify(sql::maintenance::ebuildUpdate::sql);
+				EbuildCacheParser ecp(path);
+				Utils::Database::bindOptionalsS(m, 0, { ecp.get("SLOT") });
+				Utils::Database::bindOptionalsS(m, 1, { ecp.get("LICENSE") });
+				m->bindParamT(2, boost::posix_time::from_time_t(ecp.getStat().st_mtim.tv_sec));
+				m->bindParamS(3, (fn / 0).string()); // repo
+				m->bindParamS(4, (fn / 3).string()); // category
+				m->bindParamS(5, matches.fetch(1)); // package
+				m->bindParamS(6, matches.fetch(2)); // version
+				m->execute();
+			}
+		}
+
+		void
+		EbuildMetaProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const
+		{
+			auto m = dbc->modify(sql::maintenance::ebuildDelete::sql);
+			m->bindParamS(0, (fn / 0).string());
+			m->bindParamS(1, (fn / 3).string());
+			m->bindParamS(2, (fn / 4).string());
+			m->execute();
+		}
+	}
+}
+
diff --git a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h
new file mode 100644
index 0000000..d06ccf9
--- /dev/null
+++ b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h
@@ -0,0 +1,23 @@
+#ifndef GENTOOBROWSE_API_SERVICE_MAINTENANCE_EBUILDMETAPROC_H
+#define GENTOOBROWSE_API_SERVICE_MAINTENANCE_EBUILDMETAPROC_H
+
+#include "../maintenanceimpl.h"
+#include <connection.h>
+#include <modifycommand.h>
+#include <boost/filesystem/path.hpp>
+
+namespace Gentoo {
+	namespace Service {
+		class EbuildMetaProcessor : public Maintenance::FileProcessor {
+			public:
+				static const int FILETYPEID;
+
+				void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const;
+				void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const;
+				void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const;
+		};
+	}
+}
+
+#endif
+
diff --git a/gentoobrowse-api/service/maintenanceimpl.cpp b/gentoobrowse-api/service/maintenanceimpl.cpp
index b81d959..18df8a1 100644
--- a/gentoobrowse-api/service/maintenanceimpl.cpp
+++ b/gentoobrowse-api/service/maintenanceimpl.cpp
@@ -9,18 +9,20 @@
 #include "maintenance/categoryMetaProcessor.h"
 #include "maintenance/packageManifestProcessor.h"
 #include "maintenance/packageMetaProcessor.h"
+#include "maintenance/ebuildMetaProcessor.h"
 
 /*
-1	package metadata	{"(1,metadata)","(2,md5-cache)"}
+10	category metadata.xml	{"(2,metadata.xml)"}
+4	package metadata.xml	{"(3,metadata.xml)"}
+8	package manifests	{"(3,Manifest)"}
+1	ebuild metadata	{"(1,metadata)","(2,md5-cache)"}
+
 2	changelog	{"(3,ChangeLog)"}
 3	masks	{"(1,profiles)","(2,package.mask)"}
-4	package metadata.xml	{"(3,metadata.xml)"}
 5	use_global	{"(1,profiles)","(2,use.desc)"}
 6	use_local	{"(1,profiles)","(2,use.local.desc)"}
 7	licenses	{"(1,licenses)"}
-8	manifests	{"(3,Manifest)"}
 9	use_grouped	{"(1,profiles)","(2,desc)","(3,%.desc)"}
-10	category metadata	{"(2,metadata.xml)"}
 11	news	{"(1,metadata)","(2,news)","(4,%.txt)"}
 */
 
@@ -51,6 +53,7 @@ namespace Gentoo {
 			fps[CategoryMetaProcessor::FILETYPEID] = new CategoryMetaProcessor();
 			fps[PackageManifestProcessor::FILETYPEID] = new PackageManifestProcessor();
 			fps[PackageMetaProcessor::FILETYPEID] = new PackageMetaProcessor();
+			fps[EbuildMetaProcessor::FILETYPEID] = new EbuildMetaProcessor();
 		}
 
 		Maintenance::~Maintenance()
diff --git a/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql b/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql
new file mode 100644
index 0000000..2c33008
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql
@@ -0,0 +1,8 @@
+DELETE FROM gentoobrowse.ebuilds e
+USING gentoobrowse.packages p, gentoobrowse.categories c, gentoobrowse.repos r
+WHERE e.packageid = p.packageid
+AND c.categoryid = p.categoryid
+AND e.repoid = r.repoid
+AND r.name = ?
+AND c.name = ?
+AND CONCAT(p.name, '-', e.version) = ?
diff --git a/gentoobrowse-api/service/sql/maintenance/ebuildInsert.sql b/gentoobrowse-api/service/sql/maintenance/ebuildInsert.sql
new file mode 100644
index 0000000..949b92a
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/ebuildInsert.sql
@@ -0,0 +1,7 @@
+INSERT INTO gentoobrowse.ebuilds(packageid, version, versioninst, slot, license, moddate, repoid)
+SELECT p.packageid, ?, gentoobrowse.ebuildversion_constructor(?), ?, ?, ?, r.repoid
+FROM gentoobrowse.packages p, gentoobrowse.categories c, gentoobrowse.repos r
+WHERE c.categoryid = p.categoryid
+AND r.name = ?
+AND c.name = ?
+AND p.name = ?
diff --git a/gentoobrowse-api/service/sql/maintenance/ebuildUpdate.sql b/gentoobrowse-api/service/sql/maintenance/ebuildUpdate.sql
new file mode 100644
index 0000000..dc8ae85
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/ebuildUpdate.sql
@@ -0,0 +1,10 @@
+UPDATE gentoobrowse.ebuilds e SET
+	slot = ?,
+	license = ?,
+	moddate = ?
+FROM gentoobrowse.packages p, gentoobrowse.categories c, gentoobrowse.repos r
+WHERE c.categoryid = p.categoryid
+AND r.name = ?
+AND c.name = ?
+AND p.name = ?
+AND e.version = ?
diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp
index 2b3a4ba..f597f07 100644
--- a/gentoobrowse-api/unittests/testMaintenance.cpp
+++ b/gentoobrowse-api/unittests/testMaintenance.cpp
@@ -59,8 +59,12 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree )
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, 5);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories WHERE name = 'dev-vcs'", int64_t, 1);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 478);
+	// Missing packages
+	// SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 981);
+	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 977);
 	db->execute("COPY gentoobrowse.categories TO '/tmp/categories1.tsv'");
 	db->execute("COPY gentoobrowse.packages TO '/tmp/packages1.tsv'");
+	db->execute("COPY gentoobrowse.ebuilds TO '/tmp/ebuilds1.tsv'");
 
 	sd.extract("756569aa764177340726dd3d40b41d89b11b20c7", "gentoo");
 	m->refreshPackageTree();
@@ -70,14 +74,19 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree )
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, 5);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories WHERE name = 'dev-vcs'", int64_t, 1);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 480);
+	// Missing packages
+	// SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 982);
+	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 978);
 	db->execute("COPY gentoobrowse.categories TO '/tmp/categories2.tsv'");
 	db->execute("COPY gentoobrowse.packages TO '/tmp/packages2.tsv'");
+	db->execute("COPY gentoobrowse.ebuilds TO '/tmp/ebuilds2.tsv'");
 
 	m->refreshPackageTree();
 
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.files", int64_t, 0);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, 0);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 0);
+	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 0);
 }
 
 BOOST_AUTO_TEST_SUITE_END();