diff options
8 files changed, 182 insertions, 5 deletions
diff --git a/gentoobrowse-api/db/schema.sql b/gentoobrowse-api/db/schema.sql index 105ce3f..cb9f716 100644 --- a/gentoobrowse-api/db/schema.sql +++ b/gentoobrowse-api/db/schema.sql @@ -329,7 +329,7 @@ CREATE TABLE ebuilds ( versioninst ebuildversion NOT NULL, slot text NOT NULL, license text, - firstseen timestamp without time zone NOT NULL, + firstseen timestamp without time zone NOT NULL DEFAULT now(), moddate timestamp without time zone NOT NULL, repoid integer NOT NULL ); @@ -491,6 +491,16 @@ CREATE SEQUENCE seq_packageid ALTER TABLE seq_packageid OWNER TO gentoo; -- Name: seq_packageid; Type: SEQUENCE OWNED BY; Schema: gentoobrowse; Owner: gentoo ALTER SEQUENCE seq_packageid OWNED BY packages.packageid; +-- Name: seq_ebuildid; Type: SEQUENCE; Schema: gentoobrowse; Owner: gentoo +CREATE SEQUENCE seq_ebuildid + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER TABLE seq_ebuildid OWNER TO gentoo; +-- Name: seq_ebuildid; Type: SEQUENCE OWNED BY; Schema: gentoobrowse; Owner: gentoo +ALTER SEQUENCE seq_ebuildid OWNED BY ebuilds.ebuildid; -- Name: use_global; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: CREATE TABLE use_global ( use text NOT NULL, @@ -573,6 +583,8 @@ ALTER TABLE ONLY files ALTER COLUMN fileid SET DEFAULT nextval('seq_fileid'::reg ALTER TABLE ONLY masksets ALTER COLUMN setno SET DEFAULT nextval('masksets_setno_seq'::regclass); -- Name: packageid; Type: DEFAULT; Schema: gentoobrowse; Owner: gentoo ALTER TABLE ONLY packages ALTER COLUMN packageid SET DEFAULT nextval('seq_packageid'::regclass); +-- Name: ebuildid; Type: DEFAULT; Schema: gentoobrowse; Owner: gentoo +ALTER TABLE ONLY ebuilds ALTER COLUMN ebuildid SET DEFAULT nextval('seq_ebuildid'::regclass); -- Name: repoid; Type: DEFAULT; Schema: gentoobrowse; Owner: gentoo ALTER TABLE ONLY repos ALTER COLUMN repoid SET DEFAULT nextval('repos_repoid_seq'::regclass); -- Name: usegroupid; Type: DEFAULT; Schema: gentoobrowse; Owner: gentoo diff --git a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp new file mode 100644 index 0000000..1085b70 --- /dev/null +++ b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp @@ -0,0 +1,105 @@ +#include "ebuildMetaProcessor.h" +#include <modifycommand.h> +#include <boost/filesystem/operations.hpp> +#include <glibmm/regex.h> +#include "fileUtils.h" +#include "xmlUtils.h" +#include "dbUtils.h" +#include <sql/maintenance/ebuildInsert.sql.h> +#include <sql/maintenance/ebuildUpdate.sql.h> +#include <sql/maintenance/ebuildDelete.sql.h> + +namespace U = Gentoo::Utils; +using namespace Gentoo::Utils::File; + +static Glib::RefPtr<Glib::Regex> packageVersion = Glib::Regex::create("^(.+)-([0-9].*)$"); + +namespace Gentoo { + namespace Service { + class EbuildCacheParser : public U::MemMap { + public: + typedef std::pair<const char *, const char *> Range; + typedef std::map<std::string, const Range> KVs; + + EbuildCacheParser(const boost::filesystem::path & p) : + U::MemMap(p) + { + const char * chardata = (const char *)this->data; + while (const char * eq = strchr(chardata, '=')) { + if (const char * nl = strchr(eq + 1, '\n')) { + kvs.insert({ std::string(chardata, eq), { eq + 1, nl } }); + chardata = nl + 1; + } + else { + kvs.insert({ std::string(chardata, eq), { eq + 1, (const char *)this->data + st.st_size } }); + return; + } + } + } + + boost::optional<Glib::ustring> get(const std::string & key) const + { + auto kvi = kvs.find(key); + if (kvi == kvs.end()) { + return boost::optional<Glib::ustring>(); + } + return Glib::ustring(kvi->second.first, kvi->second.second); + } + + private: + KVs kvs; + }; + + const int EbuildMetaProcessor::FILETYPEID = 1; + + void + EbuildMetaProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const + { + Glib::MatchInfo matches; + const Glib::ustring pv = (fn / 4).string(); + if (packageVersion->match(pv, matches)) { + auto m = dbc->modify(sql::maintenance::ebuildInsert::sql); + EbuildCacheParser ecp(path); + m->bindParamS(0, matches.fetch(2)); + m->bindParamS(1, matches.fetch(2)); + Utils::Database::bindOptionalsS(m, 2, { ecp.get("SLOT") }); + Utils::Database::bindOptionalsS(m, 3, { ecp.get("LICENSE") }); + m->bindParamT(4, boost::posix_time::from_time_t(ecp.getStat().st_mtim.tv_sec)); + m->bindParamS(5, (fn / 0).string()); // repo + m->bindParamS(6, (fn / 3).string()); // category + m->bindParamS(7, matches.fetch(1)); // package + m->execute(); + } + } + + void + EbuildMetaProcessor::modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const + { + Glib::MatchInfo matches; + const Glib::ustring pv = (fn / 4).string(); + if (packageVersion->match(pv, matches)) { + auto m = dbc->modify(sql::maintenance::ebuildUpdate::sql); + EbuildCacheParser ecp(path); + Utils::Database::bindOptionalsS(m, 0, { ecp.get("SLOT") }); + Utils::Database::bindOptionalsS(m, 1, { ecp.get("LICENSE") }); + m->bindParamT(2, boost::posix_time::from_time_t(ecp.getStat().st_mtim.tv_sec)); + m->bindParamS(3, (fn / 0).string()); // repo + m->bindParamS(4, (fn / 3).string()); // category + m->bindParamS(5, matches.fetch(1)); // package + m->bindParamS(6, matches.fetch(2)); // version + m->execute(); + } + } + + void + EbuildMetaProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const + { + auto m = dbc->modify(sql::maintenance::ebuildDelete::sql); + m->bindParamS(0, (fn / 0).string()); + m->bindParamS(1, (fn / 3).string()); + m->bindParamS(2, (fn / 4).string()); + m->execute(); + } + } +} + diff --git a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h new file mode 100644 index 0000000..d06ccf9 --- /dev/null +++ b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h @@ -0,0 +1,23 @@ +#ifndef GENTOOBROWSE_API_SERVICE_MAINTENANCE_EBUILDMETAPROC_H +#define GENTOOBROWSE_API_SERVICE_MAINTENANCE_EBUILDMETAPROC_H + +#include "../maintenanceimpl.h" +#include <connection.h> +#include <modifycommand.h> +#include <boost/filesystem/path.hpp> + +namespace Gentoo { + namespace Service { + class EbuildMetaProcessor : public Maintenance::FileProcessor { + public: + static const int FILETYPEID; + + void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const; + void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const; + void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const; + }; + } +} + +#endif + diff --git a/gentoobrowse-api/service/maintenanceimpl.cpp b/gentoobrowse-api/service/maintenanceimpl.cpp index b81d959..18df8a1 100644 --- a/gentoobrowse-api/service/maintenanceimpl.cpp +++ b/gentoobrowse-api/service/maintenanceimpl.cpp @@ -9,18 +9,20 @@ #include "maintenance/categoryMetaProcessor.h" #include "maintenance/packageManifestProcessor.h" #include "maintenance/packageMetaProcessor.h" +#include "maintenance/ebuildMetaProcessor.h" /* -1 package metadata {"(1,metadata)","(2,md5-cache)"} +10 category metadata.xml {"(2,metadata.xml)"} +4 package metadata.xml {"(3,metadata.xml)"} +8 package manifests {"(3,Manifest)"} +1 ebuild metadata {"(1,metadata)","(2,md5-cache)"} + 2 changelog {"(3,ChangeLog)"} 3 masks {"(1,profiles)","(2,package.mask)"} -4 package metadata.xml {"(3,metadata.xml)"} 5 use_global {"(1,profiles)","(2,use.desc)"} 6 use_local {"(1,profiles)","(2,use.local.desc)"} 7 licenses {"(1,licenses)"} -8 manifests {"(3,Manifest)"} 9 use_grouped {"(1,profiles)","(2,desc)","(3,%.desc)"} -10 category metadata {"(2,metadata.xml)"} 11 news {"(1,metadata)","(2,news)","(4,%.txt)"} */ @@ -51,6 +53,7 @@ namespace Gentoo { fps[CategoryMetaProcessor::FILETYPEID] = new CategoryMetaProcessor(); fps[PackageManifestProcessor::FILETYPEID] = new PackageManifestProcessor(); fps[PackageMetaProcessor::FILETYPEID] = new PackageMetaProcessor(); + fps[EbuildMetaProcessor::FILETYPEID] = new EbuildMetaProcessor(); } Maintenance::~Maintenance() diff --git a/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql b/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql new file mode 100644 index 0000000..2c33008 --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql @@ -0,0 +1,8 @@ +DELETE FROM gentoobrowse.ebuilds e +USING gentoobrowse.packages p, gentoobrowse.categories c, gentoobrowse.repos r +WHERE e.packageid = p.packageid +AND c.categoryid = p.categoryid +AND e.repoid = r.repoid +AND r.name = ? +AND c.name = ? +AND CONCAT(p.name, '-', e.version) = ? diff --git a/gentoobrowse-api/service/sql/maintenance/ebuildInsert.sql b/gentoobrowse-api/service/sql/maintenance/ebuildInsert.sql new file mode 100644 index 0000000..949b92a --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/ebuildInsert.sql @@ -0,0 +1,7 @@ +INSERT INTO gentoobrowse.ebuilds(packageid, version, versioninst, slot, license, moddate, repoid) +SELECT p.packageid, ?, gentoobrowse.ebuildversion_constructor(?), ?, ?, ?, r.repoid +FROM gentoobrowse.packages p, gentoobrowse.categories c, gentoobrowse.repos r +WHERE c.categoryid = p.categoryid +AND r.name = ? +AND c.name = ? +AND p.name = ? diff --git a/gentoobrowse-api/service/sql/maintenance/ebuildUpdate.sql b/gentoobrowse-api/service/sql/maintenance/ebuildUpdate.sql new file mode 100644 index 0000000..dc8ae85 --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/ebuildUpdate.sql @@ -0,0 +1,10 @@ +UPDATE gentoobrowse.ebuilds e SET + slot = ?, + license = ?, + moddate = ? +FROM gentoobrowse.packages p, gentoobrowse.categories c, gentoobrowse.repos r +WHERE c.categoryid = p.categoryid +AND r.name = ? +AND c.name = ? +AND p.name = ? +AND e.version = ? diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp index 2b3a4ba..f597f07 100644 --- a/gentoobrowse-api/unittests/testMaintenance.cpp +++ b/gentoobrowse-api/unittests/testMaintenance.cpp @@ -59,8 +59,12 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree ) SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, 5); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories WHERE name = 'dev-vcs'", int64_t, 1); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 478); + // Missing packages + // SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 981); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 977); db->execute("COPY gentoobrowse.categories TO '/tmp/categories1.tsv'"); db->execute("COPY gentoobrowse.packages TO '/tmp/packages1.tsv'"); + db->execute("COPY gentoobrowse.ebuilds TO '/tmp/ebuilds1.tsv'"); sd.extract("756569aa764177340726dd3d40b41d89b11b20c7", "gentoo"); m->refreshPackageTree(); @@ -70,14 +74,19 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree ) SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, 5); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories WHERE name = 'dev-vcs'", int64_t, 1); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 480); + // Missing packages + // SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 982); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 978); db->execute("COPY gentoobrowse.categories TO '/tmp/categories2.tsv'"); db->execute("COPY gentoobrowse.packages TO '/tmp/packages2.tsv'"); + db->execute("COPY gentoobrowse.ebuilds TO '/tmp/ebuilds2.tsv'"); m->refreshPackageTree(); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.files", int64_t, 0); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, 0); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 0); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 0); } BOOST_AUTO_TEST_SUITE_END(); |