diff options
25 files changed, 132 insertions, 154 deletions
diff --git a/gentoobrowse-api/db/schema.sql b/gentoobrowse-api/db/schema.sql index 6d89a05..2607185 100644 --- a/gentoobrowse-api/db/schema.sql +++ b/gentoobrowse-api/db/schema.sql @@ -385,27 +385,13 @@ ALTER TABLE ebuilds OWNER TO gentoo; COMMENT ON TABLE ebuilds IS 'Ebuilds :-)'; -- Name: files; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: CREATE TABLE files ( - filename text NOT NULL, - fileid integer NOT NULL, moddate timestamp without time zone NOT NULL, - firstseen timestamp without time zone DEFAULT now() NOT NULL, - cachedat timestamp without time zone, filetypeid integer NOT NULL, repoid integer NOT NULL, filesize integer NOT NULL, - pathparts text[] NOT NULL, - encoding text + pathparts text[] NOT NULL ); ALTER TABLE files OWNER TO gentoo; --- Name: filecontent(files); Type: FUNCTION; Schema: gentoobrowse; Owner: gentoo -CREATE FUNCTION filecontent(f files) RETURNS text - LANGUAGE plpgsql IMMUTABLE - AS $$ -begin - return CONVERT_FROM(PG_READ_BINARY_FILE(f.filename), COALESCE(f.encoding, 'utf-8')); -end -$$; -ALTER FUNCTION gentoobrowse.filecontent(f files) OWNER TO gentoo; -- Name: filetypes; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: CREATE TABLE filetypes ( filetypeid integer NOT NULL, @@ -508,16 +494,6 @@ CREATE SEQUENCE seq_categoryid ALTER TABLE seq_categoryid OWNER TO gentoo; -- Name: seq_categoryid; Type: SEQUENCE OWNED BY; Schema: gentoobrowse; Owner: gentoo ALTER SEQUENCE seq_categoryid OWNED BY categories.categoryid; --- Name: seq_fileid; Type: SEQUENCE; Schema: gentoobrowse; Owner: gentoo -CREATE SEQUENCE seq_fileid - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; -ALTER TABLE seq_fileid OWNER TO gentoo; --- Name: seq_fileid; Type: SEQUENCE OWNED BY; Schema: gentoobrowse; Owner: gentoo -ALTER SEQUENCE seq_fileid OWNED BY files.fileid; -- Name: seq_packageid; Type: SEQUENCE; Schema: gentoobrowse; Owner: gentoo CREATE SEQUENCE seq_packageid START WITH 1 @@ -614,8 +590,6 @@ ALTER TABLE users_userid_seq OWNER TO gentoo; ALTER SEQUENCE users_userid_seq OWNED BY users.userid; -- Name: categoryid; Type: DEFAULT; Schema: gentoobrowse; Owner: gentoo ALTER TABLE ONLY categories ALTER COLUMN categoryid SET DEFAULT nextval('seq_categoryid'::regclass); --- Name: fileid; Type: DEFAULT; Schema: gentoobrowse; Owner: gentoo -ALTER TABLE ONLY files ALTER COLUMN fileid SET DEFAULT nextval('seq_fileid'::regclass); -- Name: changeid; Type: DEFAULT; Schema: gentoobrowse; Owner: gentoo ALTER TABLE ONLY changelog ALTER COLUMN changeid SET DEFAULT nextval('changeid_seq'::regclass); -- Name: setno; Type: DEFAULT; Schema: gentoobrowse; Owner: gentoo @@ -666,7 +640,7 @@ ALTER TABLE ONLY filetypes ADD CONSTRAINT pk_fileclass PRIMARY KEY (filetypeid); -- Name: pk_files; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace: ALTER TABLE ONLY files - ADD CONSTRAINT pk_files PRIMARY KEY (fileid); + ADD CONSTRAINT pk_files PRIMARY KEY (repoid, pathparts); -- Name: pk_news; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace: ALTER TABLE ONLY news ADD CONSTRAINT pk_news PRIMARY KEY (newsid); @@ -706,9 +680,6 @@ ALTER TABLE ONLY changelog ALTER TABLE ONLY ebuilds ADD CONSTRAINT uni_ebuild_pkgverrepo UNIQUE (packageid, version, repoid); ALTER TABLE ebuilds CLUSTER ON uni_ebuild_pkgverrepo; --- Name: uni_files_filename; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace: -ALTER TABLE ONLY files - ADD CONSTRAINT uni_files_filename UNIQUE (filename); -- Name: uni_pkg_name; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace: ALTER TABLE ONLY packages ADD CONSTRAINT uni_pkg_name UNIQUE (categoryid, name); diff --git a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp index 6cb8b64..7a63932 100644 --- a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp @@ -15,29 +15,29 @@ namespace Gentoo { const int CategoryMetaProcessor::FILETYPEID = 10; void - CategoryMetaProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) + CategoryMetaProcessor::created(DB::Connection * dbc, int64_t r, const StringList & fn, const boost::filesystem::path & path) { - modified(dbc, fn, path); + modified(dbc, r, fn, path); } void - CategoryMetaProcessor::modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) + CategoryMetaProcessor::modified(DB::Connection * dbc, int64_t, const StringList & fn, const boost::filesystem::path & path) { DB::ModifyCommandPtr m = dbc->modify(sql::maintenance::categoryMetaUpdate.getSql()); U::XmlDoc md(path); m->bindParamS(0, md.getXPathValue("/catmetadata/longdescription[@lang='en']/text()") / md.getXPathValue("/catmetadata/longdescription[1]/text()")); - m->bindParamS(1, (fn / 1).string()); + m->bindParamS(1, fn.front()); m->execute(); } void - CategoryMetaProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) + CategoryMetaProcessor::deleted(DB::Connection * dbc, int64_t, const StringList & fn) { auto m = dbc->modify(sql::maintenance::categoryMetaUpdate.getSql()); m->bindNull(0); - m->bindParamS(1, (fn / 1).string()); + m->bindParamS(1, fn.front()); m->execute(); } } diff --git a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.h b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.h index 96224c6..98e5e8f 100644 --- a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.h +++ b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.h @@ -12,9 +12,9 @@ namespace Gentoo { public: static const int FILETYPEID; - void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) override; + void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void deleted(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn) override; }; } } diff --git a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp index 64e0cef..c88dc22 100644 --- a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp @@ -42,14 +42,13 @@ namespace Gentoo { } void - EbuildMetaProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) + EbuildMetaProcessor::created(DB::Connection * dbc, int64_t repoId, const StringList & fn, const boost::filesystem::path & path) { Glib::MatchInfo matches; - const Glib::ustring pv = (fn / 4).string(); + const Glib::ustring pv = fn[3]; if (packageVersion->match(pv, matches)) { U::EbuildCacheParser ecp(path); - const std::string repoName = (fn / 0).string(); - const std::string categoryName = (fn / 3).string(); + const std::string categoryName = fn[2]; const std::string packageName = matches.fetch(1); const std::string ebuildVersion = matches.fetch(2); @@ -72,33 +71,31 @@ namespace Gentoo { m->bindParamS(2, ecp.get("SLOT")); m->bindParamS(3, ecp.get("LICENSE")); m->bindParamT(4, boost::posix_time::from_time_t(ecp.getStat().st_mtim.tv_sec)); - m->bindParamS(5, repoName); + m->bindParamI(5, repoId); m->bindParamS(6, categoryName); m->bindParamS(7, packageName); - m->forEachRow<int64_t, bool, int64_t>([this,dbc,&ecp,&fn] (auto ebuildId, auto newest, auto packageId) { - // fprintf(stderr, "Created ebuild %ld for %s (%s)\n", ebuildId, fn.c_str(), newest ? "yes" : "no"); + m->forEachRow<int64_t, bool, int64_t>([this,dbc,&ecp] (auto ebuildId, auto newest, auto packageId) { this->perEbuildUpdates(dbc, ecp, ebuildId, newest, packageId); }); } } void - EbuildMetaProcessor::modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) + EbuildMetaProcessor::modified(DB::Connection * dbc, int64_t repoId, const StringList & fn, const boost::filesystem::path & path) { Glib::MatchInfo matches; - const Glib::ustring pv = (fn / 4).string(); + const Glib::ustring pv = fn[3]; if (packageVersion->match(pv, matches)) { auto m = dbc->select(sql::maintenance::ebuildUpdate.getSql()); U::EbuildCacheParser ecp(path); m->bindParamS(0, ecp.get("SLOT")); m->bindParamS(1, ecp.get("LICENSE")); m->bindParamT(2, boost::posix_time::from_time_t(ecp.getStat().st_mtim.tv_sec)); - m->bindParamS(3, (fn / 0).string()); // repo - m->bindParamS(4, (fn / 3).string()); // category + m->bindParamI(3, repoId); + m->bindParamS(4, fn[2]); // category m->bindParamS(5, matches.fetch(1)); // package m->bindParamS(6, matches.fetch(2)); // version - m->forEachRow<int64_t, bool, int64_t>([this,dbc,&ecp,&fn] (auto ebuildId, auto newest, auto packageId) { - // fprintf(stderr, "Updated ebuild %ld for %s (%s)\n", ebuildId, fn.c_str(), newest ? "yes" : "no"); + m->forEachRow<int64_t, bool, int64_t>([this,dbc,&ecp] (auto ebuildId, auto newest, auto packageId) { this->perEbuildUpdates(dbc, ecp, ebuildId, newest, packageId); }); } @@ -260,19 +257,18 @@ namespace Gentoo { } void - EbuildMetaProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) + EbuildMetaProcessor::deleted(DB::Connection * dbc, int64_t repoId, const StringList & fn) { Glib::MatchInfo matches; - const Glib::ustring pv = (fn / 4).string(); + const Glib::ustring pv = fn[3]; if (packageVersion->match(pv, matches)) { - const std::string repoName = (fn / 0).string(); - const std::string categoryName = (fn / 3).string(); + const std::string categoryName = fn[2]; const std::string packageName = matches.fetch(1); const std::string ebuildVersion = matches.fetch(2); // Delete ebuild auto m = dbc->modify(sql::maintenance::ebuildDelete.getSql()); - m->bindParamS(0, repoName); + m->bindParamI(0, repoId); m->bindParamS(1, categoryName); m->bindParamS(2, packageName); m->bindParamS(3, ebuildVersion); diff --git a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h index 6d597cb..eb41418 100644 --- a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h +++ b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h @@ -17,9 +17,9 @@ namespace Gentoo { EbuildMetaProcessor(); - void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) override; + void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void deleted(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn) override; private: void prepare(DB::Connection * dbc) override; diff --git a/gentoobrowse-api/service/maintenance/masksProcessor.cpp b/gentoobrowse-api/service/maintenance/masksProcessor.cpp index 484bceb..e56153e 100644 --- a/gentoobrowse-api/service/maintenance/masksProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/masksProcessor.cpp @@ -25,13 +25,13 @@ namespace Gentoo { const int MasksProcessor::FILETYPEID = 3; void - MasksProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) + MasksProcessor::created(DB::Connection * dbc, int64_t r, const StringList & fn, const boost::filesystem::path & path) { - modified(dbc, fn, path); + modified(dbc, r, fn, path); } void - MasksProcessor::modified(DB::Connection * dbc, const boost::filesystem::path &, const boost::filesystem::path & path) + MasksProcessor::modified(DB::Connection * dbc, int64_t, const StringList &, const boost::filesystem::path & path) { auto tempTable = Utils::Database::namedTemp(dbc, "tmp_masks", { { "n", "int" }, @@ -106,7 +106,7 @@ namespace Gentoo { } void - MasksProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path &) + MasksProcessor::deleted(DB::Connection * dbc, int64_t, const StringList &) { dbc->modify("DELETE FROM gentoobrowse.masksets")->execute(); } diff --git a/gentoobrowse-api/service/maintenance/masksProcessor.h b/gentoobrowse-api/service/maintenance/masksProcessor.h index a8c94a5..b92bd10 100644 --- a/gentoobrowse-api/service/maintenance/masksProcessor.h +++ b/gentoobrowse-api/service/maintenance/masksProcessor.h @@ -12,9 +12,9 @@ namespace Gentoo { public: static const int FILETYPEID; - void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) override; + void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void deleted(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn) override; void apply(DB::Connection *) override; }; diff --git a/gentoobrowse-api/service/maintenance/newsProcessor.cpp b/gentoobrowse-api/service/maintenance/newsProcessor.cpp index 848d704..a4387d4 100644 --- a/gentoobrowse-api/service/maintenance/newsProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/newsProcessor.cpp @@ -12,13 +12,13 @@ namespace Gentoo { const int NewsProcessor::FILETYPEID = 11; void - NewsProcessor::created(DB::Connection * dbc, const boost::filesystem::path &, const boost::filesystem::path & path) + NewsProcessor::created(DB::Connection * dbc, int64_t, const StringList &, const boost::filesystem::path & path) { importNews<Slicer::SqlInsertSerializer>(dbc, path); } void - NewsProcessor::modified(DB::Connection * dbc, const boost::filesystem::path &, const boost::filesystem::path & path) + NewsProcessor::modified(DB::Connection * dbc, int64_t, const StringList &, const boost::filesystem::path & path) { importNews<Slicer::SqlUpdateSerializer>(dbc, path); } @@ -34,10 +34,10 @@ namespace Gentoo { } void - NewsProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) + NewsProcessor::deleted(DB::Connection * dbc, int64_t, const StringList & fn) { auto del = dbc->modify(sql::maintenance::newsDelete.getSql()); - del->bindParamS(0, fn.parent_path().leaf().string()); + del->bindParamS(0, *++fn.rbegin()); del->execute(); } } diff --git a/gentoobrowse-api/service/maintenance/newsProcessor.h b/gentoobrowse-api/service/maintenance/newsProcessor.h index e499ed9..65c33c4 100644 --- a/gentoobrowse-api/service/maintenance/newsProcessor.h +++ b/gentoobrowse-api/service/maintenance/newsProcessor.h @@ -12,9 +12,9 @@ namespace Gentoo { public: static const int FILETYPEID; - void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) override; + void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void deleted(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn) override; private: template <typename Serializer> diff --git a/gentoobrowse-api/service/maintenance/packageMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/packageMetaProcessor.cpp index 1e3f032..5d452f5 100644 --- a/gentoobrowse-api/service/maintenance/packageMetaProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/packageMetaProcessor.cpp @@ -15,13 +15,13 @@ namespace Gentoo { const int PackageMetaProcessor::FILETYPEID = 4; void - PackageMetaProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) + PackageMetaProcessor::created(DB::Connection * dbc, int64_t r, const StringList & fn, const boost::filesystem::path & path) { - modified(dbc, fn, path); + modified(dbc, r, fn, path); } void - PackageMetaProcessor::modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) + PackageMetaProcessor::modified(DB::Connection * dbc, int64_t, const StringList & fn, const boost::filesystem::path & path) { auto m = dbc->modify(sql::maintenance::packageMetaUpdate.getSql()); U::XmlDoc md(path); @@ -29,19 +29,19 @@ namespace Gentoo { md.getXPathValue("/pkgmetadata/longdescription[@lang='en']/text()") / md.getXPathValue("/pkgmetadata/longdescription[1]/text()")); m->bindParamS(1, md.getXPathValue("/pkgmetadata/herd/text()")); - m->bindParamS(2, (fn / 1).string()); - m->bindParamS(3, (fn / 2).string()); + m->bindParamS(2, fn[0]); + m->bindParamS(3, fn[1]); m->execute(); } void - PackageMetaProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) + PackageMetaProcessor::deleted(DB::Connection * dbc, int64_t, const StringList & fn) { auto m = dbc->modify(sql::maintenance::packageMetaUpdate.getSql()); m->bindNull(0); m->bindNull(1); - m->bindParamS(2, (fn / 1).string()); - m->bindParamS(3, (fn / 2).string()); + m->bindParamS(2, fn[0]); + m->bindParamS(3, fn[1]); m->execute(); } } diff --git a/gentoobrowse-api/service/maintenance/packageMetaProcessor.h b/gentoobrowse-api/service/maintenance/packageMetaProcessor.h index c620828..55da84c 100644 --- a/gentoobrowse-api/service/maintenance/packageMetaProcessor.h +++ b/gentoobrowse-api/service/maintenance/packageMetaProcessor.h @@ -11,9 +11,9 @@ namespace Gentoo { public: static const int FILETYPEID; - void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) override; + void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void deleted(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn) override; }; } } diff --git a/gentoobrowse-api/service/maintenance/useGlobalProcessor.cpp b/gentoobrowse-api/service/maintenance/useGlobalProcessor.cpp index f3b3b5a..243d11b 100644 --- a/gentoobrowse-api/service/maintenance/useGlobalProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/useGlobalProcessor.cpp @@ -14,13 +14,13 @@ namespace Gentoo { const int UseGlobalProcessor::FILETYPEID = 5; void - UseGlobalProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) + UseGlobalProcessor::created(DB::Connection * dbc, int64_t r, const StringList & fn, const boost::filesystem::path & path) { - modified(dbc, fn, path); + modified(dbc, r, fn, path); } void - UseGlobalProcessor::modified(DB::Connection * dbc, const boost::filesystem::path &, const boost::filesystem::path & path) + UseGlobalProcessor::modified(DB::Connection * dbc, int64_t, const StringList &, const boost::filesystem::path & path) { DB::TablePatch p; p.dest = "gentoobrowse.use_global"; @@ -43,7 +43,7 @@ namespace Gentoo { } void - UseGlobalProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path &) + UseGlobalProcessor::deleted(DB::Connection * dbc, int64_t, const StringList &) { dbc->modify("DELETE FROM gentoobrowse.use_global")->execute(); } diff --git a/gentoobrowse-api/service/maintenance/useGlobalProcessor.h b/gentoobrowse-api/service/maintenance/useGlobalProcessor.h index c74af91..73d3c8a 100644 --- a/gentoobrowse-api/service/maintenance/useGlobalProcessor.h +++ b/gentoobrowse-api/service/maintenance/useGlobalProcessor.h @@ -12,9 +12,9 @@ namespace Gentoo { public: static const int FILETYPEID; - void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) override; + void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void deleted(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn) override; }; } } diff --git a/gentoobrowse-api/service/maintenance/useGroupProcessor.cpp b/gentoobrowse-api/service/maintenance/useGroupProcessor.cpp index 52e4e31..ab0f87c 100644 --- a/gentoobrowse-api/service/maintenance/useGroupProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/useGroupProcessor.cpp @@ -19,20 +19,20 @@ namespace Gentoo { const int UseGroupProcessor::FILETYPEID = 9; void - UseGroupProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) + UseGroupProcessor::created(DB::Connection * dbc, int64_t, const StringList &, const boost::filesystem::path & path) { auto m = dbc->select(sql::maintenance::useGroupsInsert.getSql()); - m->bindParamS(0, fn.stem().string()); + m->bindParamS(0, path.stem().string()); m->forEachRow<int64_t>([this, dbc, &path](auto useGroupId) { this->mergeContent(dbc, path, useGroupId); }); } void - UseGroupProcessor::modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) + UseGroupProcessor::modified(DB::Connection * dbc, int64_t, const StringList &, const boost::filesystem::path & path) { auto m = dbc->select(sql::maintenance::useGroupsGetId.getSql()); - m->bindParamS(0, fn.stem().string()); + m->bindParamS(0, path.stem().string()); m->forEachRow<int64_t>([this, dbc, &path](auto useGroupId) { this->mergeContent(dbc, path, useGroupId); }); @@ -70,10 +70,10 @@ namespace Gentoo { } void - UseGroupProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) + UseGroupProcessor::deleted(DB::Connection * dbc, int64_t, const StringList & fn) { auto m = dbc->modify(sql::maintenance::useGroupsDelete.getSql()); - m->bindParamS(0, fn.stem().string()); + m->bindParamS(0, fn.back().substr(0, fn.back().find('.'))); m->execute(); } } diff --git a/gentoobrowse-api/service/maintenance/useGroupProcessor.h b/gentoobrowse-api/service/maintenance/useGroupProcessor.h index af230df..9cc34c5 100644 --- a/gentoobrowse-api/service/maintenance/useGroupProcessor.h +++ b/gentoobrowse-api/service/maintenance/useGroupProcessor.h @@ -12,9 +12,9 @@ namespace Gentoo { public: static const int FILETYPEID; - void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) override; + void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void deleted(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn) override; private: void mergeContent(DB::Connection *, const boost::filesystem::path &, int64_t id); diff --git a/gentoobrowse-api/service/maintenance/useLocalProcessor.cpp b/gentoobrowse-api/service/maintenance/useLocalProcessor.cpp index 62b1180..2b82616 100644 --- a/gentoobrowse-api/service/maintenance/useLocalProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/useLocalProcessor.cpp @@ -16,13 +16,13 @@ namespace Gentoo { const int UseLocalProcessor::FILETYPEID = 6; void - UseLocalProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) + UseLocalProcessor::created(DB::Connection * dbc, int64_t r, const StringList & fn, const boost::filesystem::path & path) { - modified(dbc, fn, path); + modified(dbc, r, fn, path); } void - UseLocalProcessor::modified(DB::Connection * dbc, const boost::filesystem::path &, const boost::filesystem::path & path) + UseLocalProcessor::modified(DB::Connection * dbc, int64_t, const StringList &, const boost::filesystem::path & path) { DB::TablePatch p; auto tempTable = Utils::Database::namedTemp(dbc, "tmp_uselocalraw", { @@ -54,7 +54,7 @@ namespace Gentoo { } void - UseLocalProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path &) + UseLocalProcessor::deleted(DB::Connection * dbc, int64_t, const StringList &) { dbc->modify("DELETE FROM gentoobrowse.use_local")->execute(); } diff --git a/gentoobrowse-api/service/maintenance/useLocalProcessor.h b/gentoobrowse-api/service/maintenance/useLocalProcessor.h index b4cb36e..b743b15 100644 --- a/gentoobrowse-api/service/maintenance/useLocalProcessor.h +++ b/gentoobrowse-api/service/maintenance/useLocalProcessor.h @@ -12,9 +12,9 @@ namespace Gentoo { public: static const int FILETYPEID; - void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; - void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) override; + void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; + void deleted(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn) override; }; } } diff --git a/gentoobrowse-api/service/maintenancePackageTree.cpp b/gentoobrowse-api/service/maintenancePackageTree.cpp index df06de7..022730b 100644 --- a/gentoobrowse-api/service/maintenancePackageTree.cpp +++ b/gentoobrowse-api/service/maintenancePackageTree.cpp @@ -7,6 +7,7 @@ #include <boost/filesystem/operations.hpp> #include <boost/date_time/posix_time/posix_time.hpp> #include <boost/algorithm/string/predicate.hpp> +#include <boost/algorithm/string/join.hpp> #include <sql/maintenance/fileListCreateRaw.sql.h> #include <sql/maintenance/fileListCreate.sql.h> #include <sql/maintenance/fileListCreatePk.sql.h> @@ -80,48 +81,58 @@ namespace Gentoo { } void - Maintenance::fileDeleted(DB::Connection * dbc, const FileProcessors * fps, const boost::filesystem::path & tmp, DB::SelectCommandPtr s) + Maintenance::fileDeleted(DB::Connection * dbc, const FileProcessors * fps, const boost::filesystem::path & tmp, const RepoMap * repos, DB::SelectCommandPtr s) { - // b.filename, b.filesize, b.filetypeid, b.moddate, b.pathparts, b.repoid - s->forEachRow<std::string, int64_t, int64_t, boost::posix_time::ptime, std::string, int64_t>([dbc,&tmp,this,fps](auto fn, auto, auto ft, auto, auto, auto) { - this->fileHandle(ft, fps, boost::bind(&FileProcessor::deleted, _1, dbc, fn)); + auto f = boost::bind(&FileProcessor::deleted, _1, dbc, _2, _3); + // filesize, filetypeid, moddate, pathparts, repoid + s->forEachRow<int64_t, int64_t, boost::posix_time::ptime, std::string, int64_t>([&f,repos,dbc,&tmp,this,fps](auto, auto ft, auto, auto pp, auto repoid) { + this->fileHandle(ft, fps, repos, repoid, pp, tmp, f); }); } void - Maintenance::fileChanged(DB::Connection * dbc, const FileProcessors * fps, const boost::filesystem::path & tmp, DB::SelectCommandPtr s) + Maintenance::fileChanged(DB::Connection * dbc, const FileProcessors * fps, const boost::filesystem::path & tmp, const RepoMap * repos, DB::SelectCommandPtr s) { - // a.filename, a.filesize old_filesize, a.filetypeid old_filetypeid, a.moddate old_moddate, a.pathparts old_pathparts,a.repoid old_repoid, b.filesize new_filesize, b.filetypeid new_filetypeid, b.moddate new_moddate, b.pathparts new_pathparts, b.repoid new_repoid - s->forEachRow<std::string, int64_t, int64_t, boost::posix_time::ptime, std::string, int64_t, int64_t, int64_t, boost::posix_time::ptime, std::string, int64_t>([dbc,&tmp,this,fps](auto fn, auto, auto ft, auto, auto, auto, auto, auto, auto, auto, auto) { - this->fileHandle(ft, fps, boost::bind(&FileProcessor::modified, _1, dbc, fn, tmp / fn)); + auto f = boost::bind(&FileProcessor::modified, _1, dbc, _2, _3, _4); + // pathparts, repoid, old_filesize, old_filetypeid, old_moddate, new_filesize, new_filetypeid, new_moddate + s->forEachRow<std::string, int64_t, int64_t, int64_t, boost::posix_time::ptime, int64_t, int64_t, boost::posix_time::ptime>([&f,repos,dbc,&tmp,this,fps](auto pp, auto repoid, auto, auto ft, auto, auto, auto, auto) { + this->fileHandle(ft, fps, repos, repoid, pp, tmp, f); }); } void - Maintenance::fileCreated(DB::Connection * dbc, const FileProcessors * fps, const boost::filesystem::path & tmp, DB::SelectCommandPtr s) + Maintenance::fileCreated(DB::Connection * dbc, const FileProcessors * fps, const boost::filesystem::path & tmp, const RepoMap * repos, DB::SelectCommandPtr s) { - // b.filename, b.filesize, b.filetypeid, b.moddate, b.pathparts, b.repoid - s->forEachRow<std::string, int64_t, int64_t, boost::posix_time::ptime, std::string, int64_t>([dbc,&tmp,this,fps](auto fn, auto, auto ft, auto, auto, auto) { - this->fileHandle(ft, fps, boost::bind(&FileProcessor::created, _1, dbc, fn, tmp / fn)); + auto f = boost::bind(&FileProcessor::created, _1, dbc, _2, _3, _4); + // filesize, filetypeid, moddate, pathparts, repoid + s->forEachRow<int64_t, int64_t, boost::posix_time::ptime, std::string, int64_t>([&f,repos,dbc,&tmp,this,fps](auto, auto ft, auto, auto pp, auto repoid) { + this->fileHandle(ft, fps, repos, repoid, pp, tmp, f); }); } void - Maintenance::fileHandle(int64_t ft, const FileProcessors * fps, const FileHandleFunc & f) const + Maintenance::fileHandle(int64_t ft, const FileProcessors * fps, const RepoMap * repos, int64_t repoid, const std::string & pp, const boost::filesystem::path & tmp, const FileHandleFunc & f) const { auto i = fps->find(ft); if (i != fps->end()) { - f(i->second); + auto r = repos->find(repoid); + if (r != repos->end()) { + auto ppa = Slicer::unpackPqTextArray(pp); + boost::filesystem::path p = tmp / r->second / boost::algorithm::join(ppa, "/"); + f(i->second, repoid, ppa, p); + } } } - void + Maintenance::RepoMap Maintenance::createTempFileList(DB::Connection * dbc, const boost::filesystem::path & tmp) { boost::filesystem::remove_all(tmp); boost::filesystem::create_directories(tmp); - dbc->select("SELECT name, path FROM gentoobrowse.repos")->forEachRow<std::string, std::string>([&tmp](auto n, auto p) { + RepoMap repos; + dbc->select("SELECT name, path, repoid FROM gentoobrowse.repos")->forEachRow<std::string, std::string, int64_t>([&tmp,&repos](auto n, auto p, auto id) { boost::filesystem::create_symlink(p, tmp / n); + repos[id] = n; }); dbc->execute(sql::maintenance::fileListCreateRaw.getSql()); dbc->beginBulkUpload("filelistraw", ""); @@ -139,10 +150,11 @@ namespace Gentoo { } } dbc->endBulkUpload(nullptr); + return repos; } void - Maintenance::processChanges(DB::Connection * dbc, const boost::filesystem::path & tmp) + Maintenance::processChanges(DB::Connection * dbc, const boost::filesystem::path & tmp, const RepoMap & repos) { FileProcessors fps; for (const auto & fpf : fpfs) { @@ -156,11 +168,11 @@ namespace Gentoo { DB::TablePatch tp; tp.src = "filelist"; tp.dest = "gentoobrowse.files"; - tp.pk = {"filename"}; - tp.cols = {"repoid", "filename", "filetypeid", "pathparts", "filesize", "moddate"}; - tp.beforeDelete = boost::bind(&Maintenance::fileDeleted, this, dbc, &fps, tmp, _1); - tp.beforeUpdate = boost::bind(&Maintenance::fileChanged, this, dbc, &fps, tmp, _1); - tp.beforeInsert = boost::bind(&Maintenance::fileCreated, this, dbc, &fps, tmp, _1); + tp.pk = {"pathparts", "repoid"}; + tp.cols = {"pathparts", "repoid", "filetypeid", "filesize", "moddate"}; + tp.beforeDelete = boost::bind(&Maintenance::fileDeleted, this, dbc, &fps, tmp, &repos, _1); + tp.beforeUpdate = boost::bind(&Maintenance::fileChanged, this, dbc, &fps, tmp, &repos, _1); + tp.beforeInsert = boost::bind(&Maintenance::fileCreated, this, dbc, &fps, tmp, &repos, _1); DB::StaticSqlWriter obpo("CASE \ WHEN b.fileTypeId = 1 THEN 1 \ WHEN b.fileTypeId = 10 THEN 2 \ @@ -189,8 +201,8 @@ namespace Gentoo { auto dbc = db->get(); dbc->execute("SET search_path = gentoobrowse, pg_catalog"); DB::TransactionScope tx(dbc.get()); - createTempFileList(dbc.get(), tmp); - processChanges(dbc.get(), tmp); + auto repos = createTempFileList(dbc.get(), tmp); + processChanges(dbc.get(), tmp, repos); dbc->execute("SET search_path = public, pg_catalog"); } } diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h index 135ae99..8104649 100644 --- a/gentoobrowse-api/service/maintenanceimpl.h +++ b/gentoobrowse-api/service/maintenanceimpl.h @@ -7,6 +7,7 @@ #include <abstractDatabaseClient.h> #include <map> #include <boost/function.hpp> +#include <converters.h> namespace Gentoo { namespace Service { @@ -20,15 +21,16 @@ namespace Gentoo { virtual void apply(DB::Connection *); virtual void tidy(DB::Connection *); - virtual void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) = 0; - virtual void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) = 0; - virtual void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) = 0; + virtual void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) = 0; + virtual void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) = 0; + virtual void deleted(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn) = 0; }; typedef boost::shared_ptr<FileProcessor> FileProcessorPtr; typedef std::map<int64_t, boost::function<FileProcessorPtr()>> FileProcessorFactories; typedef std::map<int64_t, FileProcessorPtr> FileProcessors; - typedef boost::function<void(FileProcessorPtr)> FileHandleFunc; + typedef boost::function<void(FileProcessorPtr, int64_t, const Gentoo::StringList &, const boost::filesystem::path &)> FileHandleFunc; + typedef std::map<int64_t, std::string> RepoMap; Maintenance(IceTray::DatabasePoolPtr d); ~Maintenance(); @@ -38,12 +40,12 @@ namespace Gentoo { void refreshChangeLogs(const Ice::Current &) override; private: - static void createTempFileList(DB::Connection *, const boost::filesystem::path &); - void processChanges(DB::Connection *, const boost::filesystem::path &); - void fileDeleted(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path &, DB::SelectCommandPtr s); - void fileChanged(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path &, DB::SelectCommandPtr s); - void fileCreated(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path & tmp, DB::SelectCommandPtr s); - void fileHandle(int64_t filetypeId, const FileProcessors *, const FileHandleFunc &) const; + static RepoMap createTempFileList(DB::Connection *, const boost::filesystem::path &); + void processChanges(DB::Connection *, const boost::filesystem::path &, const RepoMap & repos); + void fileDeleted(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path &, const RepoMap *, DB::SelectCommandPtr s); + void fileChanged(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path &, const RepoMap *, DB::SelectCommandPtr s); + void fileCreated(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path & tmp, const RepoMap *, DB::SelectCommandPtr s); + void fileHandle(int64_t filetypeId, const FileProcessors *, const RepoMap *, int64_t repo, const std::string & pp, const boost::filesystem::path & tmp, const FileHandleFunc &) const; static Ice::PropertiesPtr properties(const Ice::Current &); diff --git a/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql b/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql index bd46bcd..cd7386a 100644 --- a/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql +++ b/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql @@ -1,9 +1,8 @@ DELETE FROM gentoobrowse.ebuilds e -USING gentoobrowse.packages p, gentoobrowse.categories c, gentoobrowse.repos r +USING gentoobrowse.packages p, gentoobrowse.categories c WHERE e.packageid = p.packageid AND c.categoryid = p.categoryid -AND e.repoid = r.repoid -AND r.name = ? +AND e.repoid = ? AND c.name = ? AND p.name = ? AND e.version = ? diff --git a/gentoobrowse-api/service/sql/maintenance/ebuildInsert.sql b/gentoobrowse-api/service/sql/maintenance/ebuildInsert.sql index 2a34c36..e05e5b8 100644 --- a/gentoobrowse-api/service/sql/maintenance/ebuildInsert.sql +++ b/gentoobrowse-api/service/sql/maintenance/ebuildInsert.sql @@ -1,9 +1,8 @@ -- libdbpp:no-cursor INSERT INTO gentoobrowse.ebuilds(packageid, version, versioninst, slot, license, moddate, repoid) -SELECT p.packageid, ?, gentoobrowse.ebuildversion_constructor(?), ?, ?, ?, r.repoid -FROM gentoobrowse.packages p, gentoobrowse.categories c, gentoobrowse.repos r +SELECT p.packageid, ?, gentoobrowse.ebuildversion_constructor(?), ?, ?, ?, ? +FROM gentoobrowse.packages p, gentoobrowse.categories c WHERE c.categoryid = p.categoryid -AND r.name = ? AND c.name = ? AND p.name = ? RETURNING ebuildid, NOT EXISTS ( diff --git a/gentoobrowse-api/service/sql/maintenance/ebuildUpdate.sql b/gentoobrowse-api/service/sql/maintenance/ebuildUpdate.sql index f5a42c5..acba8a9 100644 --- a/gentoobrowse-api/service/sql/maintenance/ebuildUpdate.sql +++ b/gentoobrowse-api/service/sql/maintenance/ebuildUpdate.sql @@ -3,10 +3,10 @@ UPDATE gentoobrowse.ebuilds e SET slot = ?, license = ?, moddate = ? -FROM gentoobrowse.packages p, gentoobrowse.categories c, gentoobrowse.repos r +FROM gentoobrowse.packages p, gentoobrowse.categories c WHERE c.categoryid = p.categoryid AND e.packageid = p.packageid -AND r.name = ? +AND e.repoid = ? AND c.name = ? AND p.name = ? AND e.version = ? diff --git a/gentoobrowse-api/service/sql/maintenance/fileListCreate.sql b/gentoobrowse-api/service/sql/maintenance/fileListCreate.sql index 76d5536..565d31a 100644 --- a/gentoobrowse-api/service/sql/maintenance/fileListCreate.sql +++ b/gentoobrowse-api/service/sql/maintenance/fileListCreate.sql @@ -1,7 +1,6 @@ CREATE TEMPORARY TABLE filelist AS SELECT r.repoid, - fl.filename, ft.filetypeid, (STRING_TO_ARRAY(fl.filename, '/'))[2:100] pathparts, fl.filesize, diff --git a/gentoobrowse-api/service/sql/maintenance/fileListCreatePk.sql b/gentoobrowse-api/service/sql/maintenance/fileListCreatePk.sql index 1e56d8b..20985e5 100644 --- a/gentoobrowse-api/service/sql/maintenance/fileListCreatePk.sql +++ b/gentoobrowse-api/service/sql/maintenance/fileListCreatePk.sql @@ -1 +1 @@ -ALTER TABLE filelist ADD CONSTRAINT pk_filelist PRIMARY KEY(filename) +ALTER TABLE filelist ADD CONSTRAINT pk_filelist PRIMARY KEY(repoid, pathparts) diff --git a/gentoobrowse-api/unittests/data.sql b/gentoobrowse-api/unittests/data.sql index cb63bc4..a5ea117 100644 --- a/gentoobrowse-api/unittests/data.sql +++ b/gentoobrowse-api/unittests/data.sql @@ -18,7 +18,7 @@ COPY gentoobrowse.ebuild_masks (setno, ebuildid) FROM '$SCRIPTDIR/fixtures/ebuil COPY gentoobrowse.ebuild_rdeps (ebuildid, packageid, versionspec, flags, op, slot) FROM '$SCRIPTDIR/fixtures/ebuild_rdeps.dat'; COPY gentoobrowse.ebuild_uses (ebuildid, use) FROM '$SCRIPTDIR/fixtures/ebuild_uses.dat'; COPY gentoobrowse.filetypes (filetypeid, description, spec) FROM '$SCRIPTDIR/fixtures/filetypes.dat'; -COPY gentoobrowse.files (filename, fileid, moddate, firstseen, cachedat, filetypeid, repoid, filesize, pathparts, encoding) FROM '$SCRIPTDIR/fixtures/files.dat'; +COPY gentoobrowse.files (moddate, filetypeid, repoid, filesize, pathparts) FROM '$SCRIPTDIR/fixtures/files.dat'; COPY gentoobrowse.license (name, legalbumph) FROM '$SCRIPTDIR/fixtures/license.dat'; COPY gentoobrowse.news (newsid, title, posted, authorname, authoremail, atomspec, body, urls) FROM '$SCRIPTDIR/fixtures/news.dat'; COPY gentoobrowse.package_urls (packageid, url) FROM '$SCRIPTDIR/fixtures/package_urls.dat'; |