diff options
40 files changed, 254 insertions, 382 deletions
diff --git a/gentoobrowse-api/db/schema.sql b/gentoobrowse-api/db/schema.sql index 25a16b3..b7f8209 100644 --- a/gentoobrowse-api/db/schema.sql +++ b/gentoobrowse-api/db/schema.sql @@ -274,27 +274,6 @@ BEGIN END $$; ALTER FUNCTION gentoobrowse.packagefts(p packages) OWNER TO gentoo; --- Name: pathpartsmatchesspecs(text[], filterspec[]); Type: FUNCTION; Schema: gentoobrowse; Owner: gentoo -CREATE FUNCTION pathpartsmatchesspecs(pathparts text[], spec filterspec[]) RETURNS boolean - LANGUAGE plpgsql - AS $$ -DECLARE - i integer; - p integer; -BEGIN - p := array_upper(pathparts, 1); - FOR i IN 1 .. array_upper(spec, 1) LOOP - IF (p < spec[i].part) THEN RETURN FALSE; END IF; - IF (spec[i].part <= 0) THEN - IF (pathparts[p - spec[i].part] NOT LIKE spec[i].pattern) THEN RETURN FALSE; END IF; - ELSE - IF (pathparts[spec[i].part] NOT LIKE spec[i].pattern) THEN RETURN FALSE; END IF; - END IF; - END LOOP; - RETURN TRUE; -END; -$$; -ALTER FUNCTION gentoobrowse.pathpartsmatchesspecs(pathparts text[], spec filterspec[]) OWNER TO gentoo; -- Name: sum(tsvector); Type: AGGREGATE; Schema: gentoobrowse; Owner: gentoo CREATE AGGREGATE sum(tsvector) ( SFUNC = tsvector_concat, @@ -308,13 +287,6 @@ CREATE OPERATOR ~ ( RIGHTARG = ebuildversion ); ALTER OPERATOR gentoobrowse.~ (ebuildversion, ebuildversion) OWNER TO gentoo; --- Name: ~; Type: OPERATOR; Schema: gentoobrowse; Owner: gentoo -CREATE OPERATOR ~ ( - PROCEDURE = pathpartsmatchesspecs, - LEFTARG = text[], - RIGHTARG = filterspec[] -); -ALTER OPERATOR gentoobrowse.~ (text[], filterspec[]) OWNER TO gentoo; -- Name: categories; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: CREATE TABLE categories ( categoryid integer NOT NULL, @@ -383,17 +355,6 @@ CREATE TABLE ebuilds ( ALTER TABLE ebuilds OWNER TO gentoo; -- Name: TABLE ebuilds; Type: COMMENT; Schema: gentoobrowse; Owner: gentoo COMMENT ON TABLE ebuilds IS 'Ebuilds :-)'; --- Name: filetypes; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: -CREATE TABLE filetypes ( - filetypeid integer NOT NULL, - description text NOT NULL, - phase integer NOT NULL, - updateOrder integer, - spec filterspec[] NOT NULL -); -ALTER TABLE filetypes OWNER TO gentoo; --- Name: TABLE filetypes; Type: COMMENT; Schema: gentoobrowse; Owner: gentoo -COMMENT ON TABLE filetypes IS 'What kind of file is this?... metadata, changelog, etc'; -- Name: license; Type: TABLE; Schema: gentoobrowse; Owner: gentoo; Tablespace: CREATE TABLE license ( name text NOT NULL, @@ -651,9 +612,6 @@ ALTER TABLE ebuild_rdeps CLUSTER ON pk_ebuildrdeps; ALTER TABLE ONLY ebuild_uses ADD CONSTRAINT pk_ebuilduses PRIMARY KEY (ebuildid, use); ALTER TABLE ebuild_uses CLUSTER ON pk_ebuilduses; --- Name: pk_fileclass; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace: -ALTER TABLE ONLY filetypes - ADD CONSTRAINT pk_fileclass PRIMARY KEY (filetypeid); -- Name: pk_news; Type: CONSTRAINT; Schema: gentoobrowse; Owner: gentoo; Tablespace: ALTER TABLE ONLY news ADD CONSTRAINT pk_news PRIMARY KEY (newsid); @@ -873,11 +831,6 @@ REVOKE ALL ON TABLE ebuilds FROM PUBLIC; REVOKE ALL ON TABLE ebuilds FROM gentoo; GRANT ALL ON TABLE ebuilds TO gentoo; GRANT SELECT ON TABLE ebuilds TO apache; --- Name: filetypes; Type: ACL; Schema: gentoobrowse; Owner: gentoo -REVOKE ALL ON TABLE filetypes FROM PUBLIC; -REVOKE ALL ON TABLE filetypes FROM gentoo; -GRANT ALL ON TABLE filetypes TO gentoo; -GRANT SELECT ON TABLE filetypes TO apache; -- Name: masksets; Type: ACL; Schema: gentoobrowse; Owner: gentoo REVOKE ALL ON TABLE masksets FROM PUBLIC; REVOKE ALL ON TABLE masksets FROM gentoo; diff --git a/gentoobrowse-api/service/changeSet.h b/gentoobrowse-api/service/changeSet.h new file mode 100644 index 0000000..321de4e --- /dev/null +++ b/gentoobrowse-api/service/changeSet.h @@ -0,0 +1,20 @@ +#ifndef CHANGESET_H +#define CHANGESET_H + +#include <git2/diff.h> +#include <portage-models.h> +#include <boost/filesystem/path.hpp> +#include <map> + +namespace Gentoo { + namespace Service { + struct ChangeDetails { + StringList pathParts; + git_delta_t changeType; + }; + typedef std::map<boost::filesystem::path, ChangeDetails> ChangeSet; + } +} + +#endif + diff --git a/gentoobrowse-api/service/maintenance/abstractFileProcessor.cpp b/gentoobrowse-api/service/maintenance/abstractFileProcessor.cpp index cf453b6..52fa266 100644 --- a/gentoobrowse-api/service/maintenance/abstractFileProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/abstractFileProcessor.cpp @@ -1,4 +1,7 @@ #include "abstractFileProcessor.h" +#include <factory.impl.h> + +INSTANTIATEVOIDFACTORY(Gentoo::Service::AbstractFileProcessor); namespace Gentoo { namespace Service { @@ -12,7 +15,7 @@ namespace Gentoo { } void - AbstractFileProcessor::apply(DB::Connection *) + AbstractFileProcessor::apply(DB::Connection *, ChangeSet &) { } diff --git a/gentoobrowse-api/service/maintenance/abstractFileProcessor.h b/gentoobrowse-api/service/maintenance/abstractFileProcessor.h index 3447d25..d25fb70 100644 --- a/gentoobrowse-api/service/maintenance/abstractFileProcessor.h +++ b/gentoobrowse-api/service/maintenance/abstractFileProcessor.h @@ -4,15 +4,23 @@ #include <connection.h> #include <maintenance.h> #include <boost/filesystem/path.hpp> +#include <factory.h> +#include "changeSet.h" namespace Gentoo { namespace Service { + typedef std::vector<std::string> PathParts; + class AbstractFileProcessor { public: virtual ~AbstractFileProcessor() = 0; + virtual unsigned char phase() const = 0; + virtual unsigned char order() const = 0; + virtual bool match(const PathParts & pp) const = 0; + virtual void prepare(DB::Connection *); - virtual void apply(DB::Connection *); + virtual void apply(DB::Connection *, ChangeSet &); virtual void tidy(DB::Connection *); virtual void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) = 0; @@ -20,6 +28,7 @@ namespace Gentoo { virtual void deleted(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn) = 0; }; typedef boost::shared_ptr<AbstractFileProcessor> FileProcessorPtr; + typedef AdHoc::Factory<AbstractFileProcessor> FileProcessorFactory; } } diff --git a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp index ad3fb0b..552448f 100644 --- a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp @@ -12,7 +12,12 @@ using namespace AdHoc::FileUtils; namespace Gentoo { namespace Service { - const int CategoryMetaProcessor::FILETYPEID = 10; + unsigned char CategoryMetaProcessor::phase() const { return 2; } + unsigned char CategoryMetaProcessor::order() const { return 2; } + bool CategoryMetaProcessor::match(const PathParts & pp) const + { + return (pp.size() == 2 && pp[1] == "metadata.xml"); + } void CategoryMetaProcessor::created(DB::Connection * dbc, int64_t r, const StringList & fn, const boost::filesystem::path & path) @@ -42,4 +47,5 @@ namespace Gentoo { } } } +FACTORY(Gentoo::Service::CategoryMetaProcessor, Gentoo::Service::FileProcessorFactory); diff --git a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.h b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.h index 46bb691..570faef 100644 --- a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.h +++ b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.h @@ -8,7 +8,9 @@ namespace Gentoo { namespace Service { class CategoryMetaProcessor : public AbstractFileProcessor { public: - static const int FILETYPEID; + unsigned char phase() const override; + unsigned char order() const override; + bool match(const PathParts & pp) const override; void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; diff --git a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp index fad96b6..35a1bbf 100644 --- a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp @@ -34,14 +34,18 @@ static Glib::RefPtr<Glib::Regex> packageVersion = Glib::Regex::create("^(.+)-([0 namespace Gentoo { namespace Service { - - const int EbuildMetaProcessor::FILETYPEID = 1; - EbuildMetaProcessor::EbuildMetaProcessor() : ebuildIds("ebuildId") { } + unsigned char EbuildMetaProcessor::phase() const { return 2; } + unsigned char EbuildMetaProcessor::order() const { return 1; } + bool EbuildMetaProcessor::match(const PathParts & pp) const + { + return (pp.size() == 4 && pp[0] == "metadata" && pp[1] == "md5-cache"); + } + void EbuildMetaProcessor::created(DB::Connection * dbc, int64_t repoId, const StringList & fn, const boost::filesystem::path & path) { @@ -128,7 +132,7 @@ namespace Gentoo { } void - EbuildMetaProcessor::apply(DB::Connection * dbc) + EbuildMetaProcessor::apply(DB::Connection * dbc, ChangeSet &) { if (!ebuildIds.entityIds.empty()) { { @@ -279,4 +283,5 @@ namespace Gentoo { } } } +FACTORY(Gentoo::Service::EbuildMetaProcessor, Gentoo::Service::FileProcessorFactory); diff --git a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h index 1feb3aa..471a7d7 100644 --- a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h +++ b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.h @@ -13,7 +13,9 @@ namespace Gentoo { namespace Service { class EbuildMetaProcessor : public AbstractFileProcessor { public: - static const int FILETYPEID; + unsigned char phase() const override; + unsigned char order() const override; + bool match(const PathParts & pp) const override; EbuildMetaProcessor(); @@ -23,7 +25,7 @@ namespace Gentoo { private: void prepare(DB::Connection * dbc) override; - void apply(DB::Connection * dbc) override; + void apply(DB::Connection * dbc, ChangeSet &) override; void tidy(DB::Connection * dbc) override; void perEbuildUpdates(DB::Connection * dbc, const Utils::EbuildCacheParser & ecp, int64_t ebuildId, bool newest, int64_t packageId); diff --git a/gentoobrowse-api/service/maintenance/masksProcessor.cpp b/gentoobrowse-api/service/maintenance/masksProcessor.cpp index e56153e..5126fd4 100644 --- a/gentoobrowse-api/service/maintenance/masksProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/masksProcessor.cpp @@ -22,7 +22,12 @@ static AdHoc::Lexer::PatternPtr discard = AdHoc::LexerMatchers::regex("^([^\n]*) namespace Gentoo { namespace Service { - const int MasksProcessor::FILETYPEID = 3; + unsigned char MasksProcessor::phase() const { return 2; } + unsigned char MasksProcessor::order() const { return 10; } + bool MasksProcessor::match(const PathParts & pp) const + { + return (pp.size() == 2 && pp[0] == "profiles" && pp[1] == "package.mask"); + } void MasksProcessor::created(DB::Connection * dbc, int64_t r, const StringList & fn, const boost::filesystem::path & path) @@ -94,7 +99,7 @@ namespace Gentoo { } void - MasksProcessor::apply(DB::Connection * dbc) + MasksProcessor::apply(DB::Connection * dbc, ChangeSet &) { DB::TablePatch e; e.dest = "gentoobrowse.ebuild_masks"; @@ -112,4 +117,5 @@ namespace Gentoo { } } } +FACTORY(Gentoo::Service::MasksProcessor, Gentoo::Service::FileProcessorFactory); diff --git a/gentoobrowse-api/service/maintenance/masksProcessor.h b/gentoobrowse-api/service/maintenance/masksProcessor.h index d076a26..3a07036 100644 --- a/gentoobrowse-api/service/maintenance/masksProcessor.h +++ b/gentoobrowse-api/service/maintenance/masksProcessor.h @@ -10,13 +10,15 @@ namespace Gentoo { namespace Service { class MasksProcessor : public AbstractFileProcessor { public: - static const int FILETYPEID; + unsigned char phase() const override; + unsigned char order() const override; + bool match(const PathParts & pp) const override; void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; void deleted(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn) override; - void apply(DB::Connection *) override; + void apply(DB::Connection *, ChangeSet &) override; }; } } diff --git a/gentoobrowse-api/service/maintenance/newsProcessor.cpp b/gentoobrowse-api/service/maintenance/newsProcessor.cpp index 07230ba..68e2b57 100644 --- a/gentoobrowse-api/service/maintenance/newsProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/newsProcessor.cpp @@ -6,10 +6,16 @@ #include <db/sqlInsertSerializer.h> #include <db/sqlUpdateSerializer.h> #include <sql/maintenance/newsDelete.sql.h> +#include <boost/algorithm/string/predicate.hpp> namespace Gentoo { namespace Service { - const int NewsProcessor::FILETYPEID = 11; + unsigned char NewsProcessor::phase() const { return 2; } + unsigned char NewsProcessor::order() const { return 10; } + bool NewsProcessor::match(const PathParts & pp) const + { + return (pp.size() == 4 && pp[0] == "metadata" && pp[1] == "news" && boost::algorithm::ends_with(pp[3], ".txt")); + } void NewsProcessor::created(DB::Connection * dbc, int64_t, const StringList &, const boost::filesystem::path & path) @@ -42,4 +48,5 @@ namespace Gentoo { } } } +FACTORY(Gentoo::Service::NewsProcessor, Gentoo::Service::FileProcessorFactory); diff --git a/gentoobrowse-api/service/maintenance/newsProcessor.h b/gentoobrowse-api/service/maintenance/newsProcessor.h index 0e2c956..b7a34ff 100644 --- a/gentoobrowse-api/service/maintenance/newsProcessor.h +++ b/gentoobrowse-api/service/maintenance/newsProcessor.h @@ -10,7 +10,9 @@ namespace Gentoo { namespace Service { class NewsProcessor : public AbstractFileProcessor { public: - static const int FILETYPEID; + unsigned char phase() const override; + unsigned char order() const override; + bool match(const PathParts & pp) const override; void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; diff --git a/gentoobrowse-api/service/maintenance/packageMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/packageMetaProcessor.cpp index 653db7f..f77c2bb 100644 --- a/gentoobrowse-api/service/maintenance/packageMetaProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/packageMetaProcessor.cpp @@ -12,7 +12,13 @@ using namespace AdHoc::FileUtils; namespace Gentoo { namespace Service { - const int PackageMetaProcessor::FILETYPEID = 4; + unsigned char PackageMetaProcessor::phase() const { return 2; } + unsigned char PackageMetaProcessor::order() const { return 3; } + bool PackageMetaProcessor::match(const PathParts & pp) const + { + return (pp.size() == 3 && pp[2] == "metadata.xml"); + } + void PackageMetaProcessor::created(DB::Connection * dbc, int64_t r, const StringList & fn, const boost::filesystem::path & path) @@ -46,4 +52,5 @@ namespace Gentoo { } } } +FACTORY(Gentoo::Service::PackageMetaProcessor, Gentoo::Service::FileProcessorFactory); diff --git a/gentoobrowse-api/service/maintenance/packageMetaProcessor.h b/gentoobrowse-api/service/maintenance/packageMetaProcessor.h index 02b5f84..84ce4e4 100644 --- a/gentoobrowse-api/service/maintenance/packageMetaProcessor.h +++ b/gentoobrowse-api/service/maintenance/packageMetaProcessor.h @@ -9,7 +9,9 @@ namespace Gentoo { namespace Service { class PackageMetaProcessor : public AbstractFileProcessor { public: - static const int FILETYPEID; + unsigned char phase() const override; + unsigned char order() const override; + bool match(const PathParts & pp) const override; void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; diff --git a/gentoobrowse-api/service/maintenance/updatesProcessor.cpp b/gentoobrowse-api/service/maintenance/updatesProcessor.cpp index e9bdda2..4717a95 100644 --- a/gentoobrowse-api/service/maintenance/updatesProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/updatesProcessor.cpp @@ -11,9 +11,9 @@ #include <sql/maintenance/categoryInsert.sql.h> #include <sql/maintenance/updatesMovePackages.sql.h> #include <sql/maintenance/updatesMoveUserPackages.sql.h> -#include <sql/maintenance/updatesMoveFilelist.sql.h> -#include <sql/maintenance/updatesMergeFilelistEntries.sql.h> #include <sql/portage/findPackage.sql.h> +#include <boost/algorithm/string/predicate.hpp> +#include <boost/algorithm/string/join.hpp> using namespace AdHoc::FileUtils; @@ -22,7 +22,12 @@ static Glib::RefPtr<Glib::Regex> move = Glib::Regex::create( namespace Gentoo { namespace Service { - const int UpdatesProcessor::FILETYPEID = 12; + unsigned char UpdatesProcessor::phase() const { return 1; } + unsigned char UpdatesProcessor::order() const { return 1; } + bool UpdatesProcessor::match(const PathParts & pp) const + { + return (pp.size() == 3 && pp[0] == "profiles" && pp[1] == "updates" && pp[2].substr(1, 4) == "Q-20"); + } class UpdatesPatch : public DB::TablePatch, public DB::SqlWriter { public: @@ -31,7 +36,7 @@ namespace Gentoo { findPackage(sql::portage::findPackage.select(db)), movePackages(sql::maintenance::updatesMovePackages.modify(db)), moveUserPackages(sql::maintenance::updatesMoveUserPackages.modify(db)), - moveFilelist(sql::maintenance::updatesMoveFilelist.modify(db)), + changes(nullptr), updatePacks("updatePack") { this->src = Utils::Database::emptyClone(db, "gentoobrowse.updates"); @@ -61,7 +66,7 @@ namespace Gentoo { DB::SelectCommandPtr findPackage; DB::ModifyCommandPtr movePackages; DB::ModifyCommandPtr moveUserPackages; - DB::ModifyCommandPtr moveFilelist; + ChangeSet * changes; Utils::EntityWhereFilter<std::string> updatePacks; void @@ -88,12 +93,27 @@ namespace Gentoo { movePackages->bindParamS(2, catfrom); movePackages->bindParamS(3, catto); movePackages->execute(); - moveFilelist->bindParamS(0, catto); - moveFilelist->bindParamS(1, namefrom); - moveFilelist->bindParamS(2, nameto); - moveFilelist->bindParamS(3, catfrom); - moveFilelist->bindParamS(4, namefrom); - moveFilelist->execute(); + // Move and/or merge file list entries + for(auto changei = changes->begin(); changei != changes->end();) { + auto & pp = changei->second.pathParts; + if (pp.size() == 4 && pp[0] == "metadata" && pp[1] == "md5-cache" && pp[2] == catfrom && boost::algorithm::starts_with(pp[3], namefrom + "-")) { + auto cd = changei->second; + cd.pathParts[2] = catto; + cd.pathParts[3] = nameto + cd.pathParts[3].substr(namefrom.length()); + auto path = boost::algorithm::join(cd.pathParts, "/"); + auto target = changes->find(path); + if (target == changes->end()) { + changes->insert({ path, cd }); + } + else if (cd.changeType == GIT_DELTA_DELETED && target->second.changeType == GIT_DELTA_ADDED) { + target->second.changeType = GIT_DELTA_MODIFIED; + } + changei = changes->erase(changei); + } + else { + changei++; + } + } } }); } @@ -106,11 +126,11 @@ namespace Gentoo { } void - UpdatesProcessor::apply(DB::Connection * dbc) + UpdatesProcessor::apply(DB::Connection * dbc, ChangeSet & changes) { if (!up->updatePacks.entityIds.empty()) { + up->changes = &changes; dbc->patchTable(up); - sql::maintenance::updatesMergeFilelistEntries.modify(dbc)->execute(); } } @@ -152,4 +172,5 @@ namespace Gentoo { } } } +FACTORY(Gentoo::Service::UpdatesProcessor, Gentoo::Service::FileProcessorFactory); diff --git a/gentoobrowse-api/service/maintenance/updatesProcessor.h b/gentoobrowse-api/service/maintenance/updatesProcessor.h index c69267c..7d7f258 100644 --- a/gentoobrowse-api/service/maintenance/updatesProcessor.h +++ b/gentoobrowse-api/service/maintenance/updatesProcessor.h @@ -11,14 +11,16 @@ namespace Gentoo { class UpdatesPatch; class UpdatesProcessor : public AbstractFileProcessor { public: - static const int FILETYPEID; + unsigned char phase() const override; + unsigned char order() const override; + bool match(const PathParts & pp) const override; void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; void deleted(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn) override; void prepare(DB::Connection * dbc) override; - void apply(DB::Connection * dbc) override; + void apply(DB::Connection * dbc, ChangeSet &) override; void tidy(DB::Connection * dbc) override; private: diff --git a/gentoobrowse-api/service/maintenance/useGlobalProcessor.cpp b/gentoobrowse-api/service/maintenance/useGlobalProcessor.cpp index 243d11b..7a3728e 100644 --- a/gentoobrowse-api/service/maintenance/useGlobalProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/useGlobalProcessor.cpp @@ -11,7 +11,12 @@ static Glib::RefPtr<Glib::Regex> useDesc = Glib::Regex::create("^([^#\\s][^ ]*)\ namespace Gentoo { namespace Service { - const int UseGlobalProcessor::FILETYPEID = 5; + unsigned char UseGlobalProcessor::phase() const { return 2; } + unsigned char UseGlobalProcessor::order() const { return 10; } + bool UseGlobalProcessor::match(const PathParts & pp) const + { + return (pp.size() == 2 && pp[0] == "profiles" && pp[1] == "use.desc"); + } void UseGlobalProcessor::created(DB::Connection * dbc, int64_t r, const StringList & fn, const boost::filesystem::path & path) @@ -49,4 +54,5 @@ namespace Gentoo { } } } +FACTORY(Gentoo::Service::UseGlobalProcessor, Gentoo::Service::FileProcessorFactory); diff --git a/gentoobrowse-api/service/maintenance/useGlobalProcessor.h b/gentoobrowse-api/service/maintenance/useGlobalProcessor.h index 2d2d6c7..fd54faa 100644 --- a/gentoobrowse-api/service/maintenance/useGlobalProcessor.h +++ b/gentoobrowse-api/service/maintenance/useGlobalProcessor.h @@ -10,7 +10,9 @@ namespace Gentoo { namespace Service { class UseGlobalProcessor : public AbstractFileProcessor { public: - static const int FILETYPEID; + unsigned char phase() const override; + unsigned char order() const override; + bool match(const PathParts & pp) const override; void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; diff --git a/gentoobrowse-api/service/maintenance/useGroupProcessor.cpp b/gentoobrowse-api/service/maintenance/useGroupProcessor.cpp index b21a701..757638b 100644 --- a/gentoobrowse-api/service/maintenance/useGroupProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/useGroupProcessor.cpp @@ -9,6 +9,7 @@ #include "sql/maintenance/useGroupsDelete.sql.h" #include "sql/maintenance/useGroupsInsert.sql.h" #include "sql/maintenance/useGroupsGetId.sql.h" +#include <boost/algorithm/string/predicate.hpp> namespace U = Gentoo::Utils; @@ -16,7 +17,12 @@ static Glib::RefPtr<Glib::Regex> useDesc = Glib::Regex::create("^([^#\\s][^ ]*)\ namespace Gentoo { namespace Service { - const int UseGroupProcessor::FILETYPEID = 9; + unsigned char UseGroupProcessor::phase() const { return 2; } + unsigned char UseGroupProcessor::order() const { return 10; } + bool UseGroupProcessor::match(const PathParts & pp) const + { + return (pp.size() == 3 && pp[0] == "profiles" && pp[1] == "desc" && boost::algorithm::ends_with(pp[2], ".desc")); + } void UseGroupProcessor::created(DB::Connection * dbc, int64_t, const StringList &, const boost::filesystem::path & path) @@ -78,4 +84,5 @@ namespace Gentoo { } } } +FACTORY(Gentoo::Service::UseGroupProcessor, Gentoo::Service::FileProcessorFactory); diff --git a/gentoobrowse-api/service/maintenance/useGroupProcessor.h b/gentoobrowse-api/service/maintenance/useGroupProcessor.h index 0e02947..2cb066e 100644 --- a/gentoobrowse-api/service/maintenance/useGroupProcessor.h +++ b/gentoobrowse-api/service/maintenance/useGroupProcessor.h @@ -10,7 +10,9 @@ namespace Gentoo { namespace Service { class UseGroupProcessor : public AbstractFileProcessor { public: - static const int FILETYPEID; + unsigned char phase() const override; + unsigned char order() const override; + bool match(const PathParts & pp) const override; void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; diff --git a/gentoobrowse-api/service/maintenance/useLocalProcessor.cpp b/gentoobrowse-api/service/maintenance/useLocalProcessor.cpp index 2b82616..bf961f5 100644 --- a/gentoobrowse-api/service/maintenance/useLocalProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/useLocalProcessor.cpp @@ -13,7 +13,12 @@ static Glib::RefPtr<Glib::Regex> useDesc = Glib::Regex::create("^([^#\\s][^/]*)/ namespace Gentoo { namespace Service { - const int UseLocalProcessor::FILETYPEID = 6; + unsigned char UseLocalProcessor::phase() const { return 2; } + unsigned char UseLocalProcessor::order() const { return 10; } + bool UseLocalProcessor::match(const PathParts & pp) const + { + return (pp.size() == 2 && pp[0] == "profiles" && pp[1] == "use.local.desc"); + } void UseLocalProcessor::created(DB::Connection * dbc, int64_t r, const StringList & fn, const boost::filesystem::path & path) @@ -60,4 +65,5 @@ namespace Gentoo { } } } +FACTORY(Gentoo::Service::UseLocalProcessor, Gentoo::Service::FileProcessorFactory); diff --git a/gentoobrowse-api/service/maintenance/useLocalProcessor.h b/gentoobrowse-api/service/maintenance/useLocalProcessor.h index d4fb59f..4e4bace 100644 --- a/gentoobrowse-api/service/maintenance/useLocalProcessor.h +++ b/gentoobrowse-api/service/maintenance/useLocalProcessor.h @@ -10,7 +10,9 @@ namespace Gentoo { namespace Service { class UseLocalProcessor : public AbstractFileProcessor { public: - static const int FILETYPEID; + unsigned char phase() const override; + unsigned char order() const override; + bool match(const PathParts & pp) const override; void created(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; void modified(DB::Connection * dbc, int64_t repoid, const Gentoo::StringList & fn, const boost::filesystem::path & path) override; diff --git a/gentoobrowse-api/service/maintenanceCommon.cpp b/gentoobrowse-api/service/maintenanceCommon.cpp index 63bb13d..4a11ed2 100644 --- a/gentoobrowse-api/service/maintenanceCommon.cpp +++ b/gentoobrowse-api/service/maintenanceCommon.cpp @@ -39,7 +39,6 @@ namespace Gentoo { Maintenance::Maintenance(IceTray::DatabasePoolPtr d) : IceTray::AbstractDatabaseClient(d) { - setupFileProcessors(); } Maintenance::~Maintenance() diff --git a/gentoobrowse-api/service/maintenanceGitOperations.cpp b/gentoobrowse-api/service/maintenanceGitOperations.cpp index 33340e8..bc66882 100644 --- a/gentoobrowse-api/service/maintenanceGitOperations.cpp +++ b/gentoobrowse-api/service/maintenanceGitOperations.cpp @@ -8,11 +8,6 @@ #include <sql/maintenance/changeLogInsert.sql.h> #include <sql/maintenance/reposToUpdate.sql.h> #include <sql/maintenance/reposToGitRefresh.sql.h> -#include <sql/maintenance/gitListCreateRaw.sql.h> -#include <sql/maintenance/gitListCreate.sql.h> -#include <sql/maintenance/gitListCreateIdx.sql.h> -#include <sql/maintenance/filelistPhases.sql.h> -#include <sql/maintenance/gitListChangesInPhase.sql.h> #include <portage-models.h> #include "utils/git.h" #include "converters.h" @@ -99,28 +94,34 @@ namespace Gentoo { sql::maintenance::reposToUpdate.select(dbc.get())->forEachRow<std::string, std::string, std::string>(&updateRepository); } + static inline + PathParts + split(const boost::filesystem::path & path) + { + PathParts pp; + for (const auto & p : path) { + pp.push_back(p.string()); + } + return pp; + } + static int insertFileChange(const git_diff_delta * delta, void * ptr) { - auto ins = static_cast<DB::ModifyCommand *>(ptr); + auto cs = static_cast<ChangeSet *>(ptr); switch (delta->status) { case GIT_DELTA_ADDED: - ins->bindParamS(1, "A"); - ins->bindParamS(2, delta->new_file.path); - break; - case GIT_DELTA_DELETED: - ins->bindParamS(1, "D"); - ins->bindParamS(2, delta->old_file.path); - break; case GIT_DELTA_MODIFIED: - ins->bindParamS(1, "M"); - ins->bindParamS(2, delta->new_file.path); + case GIT_DELTA_DELETED: + { + boost::filesystem::path path(delta->new_file.path); + cs->insert({ path, { split(path), delta->status }}); + } break; default: throw GitError("Insert Git file changes", 0, 0, "Unexpected change status."); } - ins->execute(); return 0; } @@ -133,10 +134,10 @@ namespace Gentoo { } static - void + ChangeSet writeChangesToFileList(DB::Connection * db, int64_t repoId, git_repository * repo, const git_oid & last, const git_oid & head); - void + ChangeSet refreshRepository(DB::Connection * db, Ice::PropertiesPtr properties, int64_t repoId, const std::string & repoName, const std::string & path, const std::string & lastCommitId) { auto targetRef = properties->getProperty("GentooBrowseAPI.RefreshTarget." + repoName); @@ -153,12 +154,12 @@ namespace Gentoo { updateLastCommit->execute(); writeChangeLog(db, repoId, repo.get(), lastCommitOid, headCommitOid); - writeChangesToFileList(db, repoId, repo.get(), lastCommitOid, headCommitOid); + return writeChangesToFileList(db, repoId, repo.get(), lastCommitOid, headCommitOid); } AdHocFormatter(FindingChanges, "Finding changes for repository %? with range %?...%?\n"); - void - writeChangesToFileList(DB::Connection * db, int64_t repoId, git_repository * repo, const git_oid & lastCommitOid, const git_oid & headCommitOid) + ChangeSet + writeChangesToFileList(DB::Connection *, int64_t repoId, git_repository * repo, const git_oid & lastCommitOid, const git_oid & headCommitOid) { FindingChanges::write(std::cerr, repoId, lastCommitOid, headCommitOid); auto lastCommit = gitSafeGet(git_commit_lookup, git_commit_free, repo, &lastCommitOid); @@ -166,9 +167,9 @@ namespace Gentoo { auto lastTree = gitSafeGet(git_commit_tree, git_tree_free, lastCommit.get()); auto headTree = gitSafeGet(git_commit_tree, git_tree_free, headCommit.get()); auto diff = gitSafeGet(git_diff_tree_to_tree, git_diff_free, repo, lastTree.get(), headTree.get(), nullptr); - auto ins = db->modify("INSERT INTO filelistraw(repoid, status, filename) VALUES(?, ?, ?)"); - ins->bindParamI(0, repoId); - gitSafe(git_diff_foreach, diff.get(), insertFileChangeT<float>, insertFileChangeT<const git_diff_binary *>, nullptr, nullptr, ins.get()); + ChangeSet changes; + gitSafe(git_diff_foreach, diff.get(), insertFileChangeT<float>, insertFileChangeT<const git_diff_binary *>, nullptr, nullptr, &changes); + return changes; } void @@ -179,64 +180,59 @@ namespace Gentoo { auto dbc = db->get(); dbc->execute("SET search_path = gentoobrowse, pg_catalog"); DB::TransactionScope tx(dbc.get()); - updateFileTypes(dbc.get()); - sql::maintenance::gitListCreateRaw.modify(dbc.get())->execute(); - sql::maintenance::reposToGitRefresh.select(dbc.get()) - ->forEachRow<int64_t, std::string, std::string, std::string>( - boost::bind(&refreshRepository, dbc.get(), properties, _1, _2, _3, _4)); - sql::maintenance::gitListCreate.modify(dbc.get())->execute(); - sql::maintenance::gitListCreateIdx.modify(dbc.get())->execute(); - - applyFileChanges(dbc.get(), repoRoot); - - dbc->execute("DROP TABLE filelist"); - dbc->execute("DROP TABLE filelistraw"); + sql::maintenance::reposToGitRefresh.select(dbc.get())->forEachRow<int64_t, std::string, std::string, std::string>( + [dbc, properties, repoRoot, this](auto repoId, auto name, auto path, auto lastCommit) { + auto changes = refreshRepository(dbc.get(), properties, repoId, name, path, lastCommit); + this->applyFileChanges(dbc.get(), repoId, path, repoRoot, changes); + }); dbc->execute("SET search_path = public, pg_catalog"); } AdHocFormatter(UpdatingFileContent, "Updating file content\n"); AdHocFormatter(UpdatedFileContent, "Updated file content\n"); void - Maintenance::applyFileChanges(DB::Connection * dbc, const boost::filesystem::path & repoRoot) const + Maintenance::applyFileChanges(DB::Connection * dbc, int64_t repoId, const boost::filesystem::path & path, const boost::filesystem::path & repoRoot, ChangeSet & changes) const { UpdatingFileContent::write(std::cerr); - FileProcessors fps; - for (const auto & fpf : fpfs) { - fps[fpf.first] = fpf.second(); - } - for (const auto & fp : fps) { - fp.second->prepare(dbc); + // Map of phase, to order, to processor + std::map<unsigned char, std::multimap<unsigned char, FileProcessorPtr>> fps; + for (const auto & fpf : AdHoc::PluginManager::getDefault()->getAll<FileProcessorFactory>()) { + FileProcessorPtr processor(fpf->implementation()->create()); + fps[processor->phase()].insert({ processor->order(), processor })->second->prepare(dbc); } - RepoMap repos; - dbc->select("SELECT path, repoid FROM gentoobrowse.repos")->forEachRow<std::string, int64_t>([&repos](auto path, auto id) { - repos[id] = path; - }); - std::map<std::string, FileHandleFunc> funcs; - funcs["D"] = boost::bind(&AbstractFileProcessor::deleted, _1, dbc, _2, _3); - funcs["M"] = boost::bind(&AbstractFileProcessor::modified, _1, dbc, _2, _3, _4); - funcs["A"] = boost::bind(&AbstractFileProcessor::created, _1, dbc, _2, _3, _4); - auto phases = sql::maintenance::filelistPhases.select(dbc); - for (const auto & phaseRow : phases->as<int64_t>()) { - const auto & phase = phaseRow.value<0>(); - auto files = sql::maintenance::gitListChangesInPhase.select(dbc); - files->bindParamI(0, phase); - std::set<int64_t> usedTypes; - for (const auto & fileRow : files->as<int64_t, int64_t, std::string, std::string>()) { - const auto & fileTypeId = fileRow.value<1>(); - if (fps.find(fileTypeId) != fps.end()) { - const auto & repoId = fileRow.value<0>(); - const auto & pathParts = fileRow.value<2>(); - const auto & status = fileRow.value<3>(); - usedTypes.insert(fileTypeId); - this->fileHandle(fileTypeId, &fps, &repos, repoId, pathParts, repoRoot, funcs[status]); + // Processor -> repoId, PathParts, Path + typedef boost::function<void(FileProcessorPtr, int64_t, const Gentoo::StringList &, const boost::filesystem::path &)> FileHandleFunc; + std::map<git_delta_t, FileHandleFunc> funcs; + funcs[GIT_DELTA_DELETED] = boost::bind(&AbstractFileProcessor::deleted, _1, dbc, _2, _3); + funcs[GIT_DELTA_MODIFIED] = boost::bind(&AbstractFileProcessor::modified, _1, dbc, _2, _3, _4); + funcs[GIT_DELTA_ADDED] = boost::bind(&AbstractFileProcessor::created, _1, dbc, _2, _3, _4); + + // Phases + for (const auto & phase : fps) { + std::set<FileProcessorPtr> usedProcessors; + // Processors + for (const auto & processorOrder : phase.second) { + auto processor = processorOrder.second; + // Changes + for (auto change = changes.begin(); change != changes.end(); ) { + if (processor->match(change->second.pathParts)) { + funcs[change->second.changeType](processor, repoId, change->second.pathParts, repoRoot / path / change->first); + usedProcessors.insert(processor); + change = changes.erase(change); + } + else { + change++; + } } } - for (const auto & fp : usedTypes) { - fps[fp]->apply(dbc); + for (const auto & processor : usedProcessors) { + processor->apply(dbc, changes); } } - for (const auto & fp : fps) { - fp.second->tidy(dbc); + for (const auto & phase : fps) { + for (const auto & processor : phase.second) { + processor.second->tidy(dbc); + } } UpdatedFileContent::write(std::cerr); } diff --git a/gentoobrowse-api/service/maintenancePackageTree.cpp b/gentoobrowse-api/service/maintenancePackageTree.cpp deleted file mode 100644 index 7201ac3..0000000 --- a/gentoobrowse-api/service/maintenancePackageTree.cpp +++ /dev/null @@ -1,121 +0,0 @@ -#include "maintenanceimpl.h" -#include <selectcommandUtil.impl.h> -#include <tablepatch.h> -#include <sqlWriter.h> -#include <buffer.h> -#include <scopeExit.h> -#include "utils/dbUtils.h" -#include <boost/filesystem/operations.hpp> -#include <boost/date_time/posix_time/posix_time.hpp> -#include <boost/algorithm/string/predicate.hpp> -#include <boost/algorithm/string/join.hpp> -#include "maintenance/categoryMetaProcessor.h" -#include "maintenance/packageMetaProcessor.h" -#include "maintenance/ebuildMetaProcessor.h" -#include "maintenance/useGlobalProcessor.h" -#include "maintenance/useLocalProcessor.h" -#include "maintenance/useGroupProcessor.h" -#include "maintenance/masksProcessor.h" -#include "maintenance/newsProcessor.h" -#include "maintenance/updatesProcessor.h" - -namespace Gentoo { - const std::string fileTypes( -R"DATA( 10 category metadata 2 2 {"(2,metadata.xml)"} -4 package metadata.xml 2 3 {"(3,metadata.xml)"} -1 ebuild metadata 2 1 {"(1,metadata)","(2,md5-cache)"} -8 manifests 2 \N {"(3,Manifest)"} -5 use_global 2 \N {"(1,profiles)","(2,use.desc)"} -6 use_local 2 \N {"(1,profiles)","(2,use.local.desc)"} -9 use_grouped 2 \N {"(1,profiles)","(2,desc)","(3,%.desc)"} -3 masks 2 \N {"(1,profiles)","(2,package.mask)"} -11 news 2 \N {"(1,metadata)","(2,news)","(4,%.txt)"} -12 updates 1 \N {"(1,profiles)","(2,updates)","(3,_Q-20__)"} -)DATA"); - -// 8 package manifests {"(3,Manifest)"} -// 7 licenses {"(1,licenses)"} - - namespace Service { - template<typename T> - FileProcessorPtr - Maintenance::createFileProessor() - { - return FileProcessorPtr(new T()); - } - - void - Maintenance::setupFileProcessors() - { - fpfs[CategoryMetaProcessor::FILETYPEID] = &createFileProessor<CategoryMetaProcessor>; - fpfs[PackageMetaProcessor::FILETYPEID] = &createFileProessor<PackageMetaProcessor>; - fpfs[EbuildMetaProcessor::FILETYPEID] = &createFileProessor<EbuildMetaProcessor>; - fpfs[UseGlobalProcessor::FILETYPEID] = &createFileProessor<UseGlobalProcessor>; - fpfs[UseLocalProcessor::FILETYPEID] = &createFileProessor<UseLocalProcessor>; - fpfs[UseGroupProcessor::FILETYPEID] = &createFileProessor<UseGroupProcessor>; - fpfs[MasksProcessor::FILETYPEID] = &createFileProessor<MasksProcessor>; - fpfs[NewsProcessor::FILETYPEID] = &createFileProessor<NewsProcessor>; - fpfs[UpdatesProcessor::FILETYPEID] = &createFileProessor<UpdatesProcessor>; - } - - void - Maintenance::fileDeleted(DB::Connection * dbc, const FileProcessors * fps, const boost::filesystem::path & tmp, const RepoMap * repos, DB::SelectCommandPtr s) - { - auto f = boost::bind(&AbstractFileProcessor::deleted, _1, dbc, _2, _3); - // filesize, filetypeid, moddate, pathparts, repoid - s->forEachRow<int64_t, int64_t, boost::posix_time::ptime, std::string, int64_t>([&f,repos,dbc,&tmp,this,fps](auto, auto ft, auto, auto pp, auto repoid) { - this->fileHandle(ft, fps, repos, repoid, pp, tmp, f); - }); - } - - void - Maintenance::fileChanged(DB::Connection * dbc, const FileProcessors * fps, const boost::filesystem::path & tmp, const RepoMap * repos, DB::SelectCommandPtr s) - { - auto f = boost::bind(&AbstractFileProcessor::modified, _1, dbc, _2, _3, _4); - // pathparts, repoid, old_filesize, old_filetypeid, old_moddate, new_filesize, new_filetypeid, new_moddate - s->forEachRow<std::string, int64_t, int64_t, int64_t, boost::posix_time::ptime, int64_t, int64_t, boost::posix_time::ptime>([&f,repos,dbc,&tmp,this,fps](auto pp, auto repoid, auto, auto ft, auto, auto, auto, auto) { - this->fileHandle(ft, fps, repos, repoid, pp, tmp, f); - }); - } - - void - Maintenance::fileCreated(DB::Connection * dbc, const FileProcessors * fps, const boost::filesystem::path & tmp, const RepoMap * repos, DB::SelectCommandPtr s) - { - auto f = boost::bind(&AbstractFileProcessor::created, _1, dbc, _2, _3, _4); - // filesize, filetypeid, moddate, pathparts, repoid - s->forEachRow<int64_t, int64_t, boost::posix_time::ptime, std::string, int64_t>([&f,repos,dbc,&tmp,this,fps](auto, auto ft, auto, auto pp, auto repoid) { - this->fileHandle(ft, fps, repos, repoid, pp, tmp, f); - }); - } - - void - Maintenance::fileHandle(int64_t ft, const FileProcessors * fps, const RepoMap * repos, int64_t repoid, const std::string & pp, const boost::filesystem::path & tmp, const FileHandleFunc & f) const - { - auto i = fps->find(ft); - if (i != fps->end()) { - auto r = repos->find(repoid); - if (r != repos->end()) { - auto ppa = Slicer::unpackPqTextArray(pp); - boost::filesystem::path p = tmp / r->second / boost::algorithm::join(ppa, "/"); - f(i->second, repoid, ppa, p); - } - } - } - - void Maintenance::updateFileTypes(DB::Connection * dbc) const - { - DB::TablePatch p; - p.dest = "gentoobrowse.filetypes"; - p.src = Utils::Database::emptyClone(dbc, p.dest); - p.pk = { "filetypeId" }; - p.cols = { "filetypeId", "description", "spec", "phase", "updateOrder" }; - std::stringstream buf(fileTypes); - dbc->beginBulkUpload((p.src + "(filetypeid, description, phase, updateorder, spec)").c_str(), ""); - dbc->bulkUploadData(buf); - dbc->endBulkUpload(nullptr); - dbc->patchTable(&p); - Utils::Database::drop(dbc, p.src); - } - } -} - diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h index 3c43548..622a239 100644 --- a/gentoobrowse-api/service/maintenanceimpl.h +++ b/gentoobrowse-api/service/maintenanceimpl.h @@ -10,16 +10,12 @@ #include <converters.h> #include "maintenance/abstractFileProcessor.h" #include <IceUtil/Timer.h> +#include "changeSet.h" namespace Gentoo { namespace Service { class DLL_PUBLIC Maintenance : public Gentoo::Maintenance, public IceTray::AbstractDatabaseClient { public: - typedef std::map<int64_t, boost::function<FileProcessorPtr()>> FileProcessorFactories; - typedef std::map<int64_t, FileProcessorPtr> FileProcessors; - typedef boost::function<void(FileProcessorPtr, int64_t, const Gentoo::StringList &, const boost::filesystem::path &)> FileHandleFunc; - typedef std::map<int64_t, std::string> RepoMap; - Maintenance(IceTray::DatabasePoolPtr d, Ice::CommunicatorPtr ic, Ice::PropertiesPtr p); virtual ~Maintenance(); @@ -31,21 +27,14 @@ namespace Gentoo { protected: Maintenance(IceTray::DatabasePoolPtr d); - void applyFileChanges(DB::Connection *, const boost::filesystem::path &) const; - void updateFileTypes(DB::Connection *) const; + void applyFileChanges(DB::Connection *, int64_t, const boost::filesystem::path &, const boost::filesystem::path &, ChangeSet &) const; private: - void setupFileProcessors(); - void fileDeleted(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path &, const RepoMap *, DB::SelectCommandPtr s); - void fileChanged(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path &, const RepoMap *, DB::SelectCommandPtr s); - void fileCreated(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path & tmp, const RepoMap *, DB::SelectCommandPtr s); - void fileHandle(int64_t filetypeId, const FileProcessors *, const RepoMap *, int64_t repo, const std::string & pp, const boost::filesystem::path & tmp, const FileHandleFunc &) const; static Ice::PropertiesPtr properties(const Ice::Current &); template <typename T> static FileProcessorPtr createFileProessor(); - FileProcessorFactories fpfs; IceUtil::TimerPtr taskRunner; }; } diff --git a/gentoobrowse-api/service/sql/maintenance/filelistPhases.sql b/gentoobrowse-api/service/sql/maintenance/filelistPhases.sql deleted file mode 100644 index 5234f51..0000000 --- a/gentoobrowse-api/service/sql/maintenance/filelistPhases.sql +++ /dev/null @@ -1,4 +0,0 @@ -SELECT phase -FROM filelist -GROUP BY phase -ORDER BY phase diff --git a/gentoobrowse-api/service/sql/maintenance/gitListChangesInPhase.sql b/gentoobrowse-api/service/sql/maintenance/gitListChangesInPhase.sql deleted file mode 100644 index 17dfdb3..0000000 --- a/gentoobrowse-api/service/sql/maintenance/gitListChangesInPhase.sql +++ /dev/null @@ -1,5 +0,0 @@ -SELECT repoId, fileTypeId, pathParts, status -FROM filelist -WHERE phase = ? -AND status IS NOT NULL -ORDER BY POSITION(status IN 'DMA'), updateOrder NULLS LAST diff --git a/gentoobrowse-api/service/sql/maintenance/gitListCreate.sql b/gentoobrowse-api/service/sql/maintenance/gitListCreate.sql deleted file mode 100644 index 20ca804..0000000 --- a/gentoobrowse-api/service/sql/maintenance/gitListCreate.sql +++ /dev/null @@ -1,10 +0,0 @@ -CREATE TEMPORARY TABLE filelist AS -SELECT - fl.repoid, - fl.status, - ft.filetypeid, - ft.updateOrder, - ft.phase, - (STRING_TO_ARRAY(fl.filename, '/')) pathparts -FROM filelistraw fl, gentoobrowse.filetypes ft -WHERE (STRING_TO_ARRAY(fl.filename, '/')) ~ ft.spec diff --git a/gentoobrowse-api/service/sql/maintenance/gitListCreateIdx.sql b/gentoobrowse-api/service/sql/maintenance/gitListCreateIdx.sql deleted file mode 100644 index 1b3a6ca..0000000 --- a/gentoobrowse-api/service/sql/maintenance/gitListCreateIdx.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE INDEX idx_filelist_mode ON filelist(phase, status) diff --git a/gentoobrowse-api/service/sql/maintenance/gitListCreateRaw.sql b/gentoobrowse-api/service/sql/maintenance/gitListCreateRaw.sql deleted file mode 100644 index 3007cd8..0000000 --- a/gentoobrowse-api/service/sql/maintenance/gitListCreateRaw.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE TEMPORARY TABLE filelistraw ( - repoid INT NOT NULL, - filename TEXT NOT NULL, - status CHARACTER(1) NOT NULL) diff --git a/gentoobrowse-api/service/sql/maintenance/updatesMergeFilelistEntries.sql b/gentoobrowse-api/service/sql/maintenance/updatesMergeFilelistEntries.sql deleted file mode 100644 index 68cdbee..0000000 --- a/gentoobrowse-api/service/sql/maintenance/updatesMergeFilelistEntries.sql +++ /dev/null @@ -1,9 +0,0 @@ -UPDATE filelist SET - status = CASE WHEN status = 'A' THEN 'M' END -WHERE pathparts IN ( - SELECT pathparts - FROM filelist - WHERE filetypeid = 1 - AND status IN ('A', 'D') - GROUP BY pathparts - HAVING COUNT(*) = 2) diff --git a/gentoobrowse-api/service/sql/maintenance/updatesMoveFilelist.sql b/gentoobrowse-api/service/sql/maintenance/updatesMoveFilelist.sql deleted file mode 100644 index 0a34c82..0000000 --- a/gentoobrowse-api/service/sql/maintenance/updatesMoveFilelist.sql +++ /dev/null @@ -1,6 +0,0 @@ -UPDATE filelist SET - pathparts[3] = ?, - pathparts[4] = REGEXP_REPLACE(pathparts[4], CONCAT('^', REPLACE(?, '+', '\+'), '(-[0-9])'), CONCAT(?::text, '\1')) -WHERE pathparts[3] = ? -AND pathparts[4] ~ CONCAT('^', replace(?, '+', '\+'), '-[0-9]') -AND filetypeId = 1 diff --git a/gentoobrowse-api/unittests/Jamfile.jam b/gentoobrowse-api/unittests/Jamfile.jam index 74a67df..e6a7560 100644 --- a/gentoobrowse-api/unittests/Jamfile.jam +++ b/gentoobrowse-api/unittests/Jamfile.jam @@ -80,9 +80,6 @@ run run testMaintenance.cpp - ../service/sql/maintenance/gitListCreate.sql - ../service/sql/maintenance/gitListCreateRaw.sql - ../service/sql/maintenance/gitListCreateIdx.sql : : : <dependency>../db/schema.sql <define>BOOST_TEST_DYN_LINK @@ -90,9 +87,6 @@ run <library>git2 <library>../service//icetray <implicit-dependency>testCommon - <icetray.sql.namespace>Gentoo::Service - <icetray.sql.connector>postgresql - <icetray.sql.basedir>../service : testMaintenance ; run diff --git a/gentoobrowse-api/unittests/basedata.sql b/gentoobrowse-api/unittests/basedata.sql deleted file mode 100644 index bcaf63d..0000000 --- a/gentoobrowse-api/unittests/basedata.sql +++ /dev/null @@ -1,9 +0,0 @@ -SET statement_timeout = 0; -SET lock_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SET check_function_bodies = false; -SET client_min_messages = warning; -SET search_path = gentoobrowse, pg_catalog; - -COPY gentoobrowse.filetypes (filetypeid, description, spec, phase, updateOrder) FROM '$SCRIPTDIR/fixtures/filetypes.dat'; diff --git a/gentoobrowse-api/unittests/data.sql b/gentoobrowse-api/unittests/data.sql index 4120934..074b3d3 100644 --- a/gentoobrowse-api/unittests/data.sql +++ b/gentoobrowse-api/unittests/data.sql @@ -17,7 +17,6 @@ COPY gentoobrowse.masksets (setno, person, email, dateadded, message, n, atomspe COPY gentoobrowse.ebuild_masks (setno, ebuildid) FROM '$SCRIPTDIR/fixtures/ebuild_masks.dat'; COPY gentoobrowse.ebuild_rdeps (ebuildid, packageid, versionspec, flags, op, slot) FROM '$SCRIPTDIR/fixtures/ebuild_rdeps.dat'; COPY gentoobrowse.ebuild_uses (ebuildid, use) FROM '$SCRIPTDIR/fixtures/ebuild_uses.dat'; -COPY gentoobrowse.filetypes (filetypeid, description, spec, phase, updateOrder) FROM '$SCRIPTDIR/fixtures/filetypes.dat'; COPY gentoobrowse.license (name, legalbumph) FROM '$SCRIPTDIR/fixtures/license.dat'; COPY gentoobrowse.news (newsid, title, posted, authorname, authoremail, atomspec, body, urls) FROM '$SCRIPTDIR/fixtures/news.dat'; COPY gentoobrowse.package_urls (packageid, url) FROM '$SCRIPTDIR/fixtures/package_urls.dat'; diff --git a/gentoobrowse-api/unittests/fixtures/files.dat b/gentoobrowse-api/unittests/fixtures/files.dat deleted file mode 100644 index e69de29..0000000 --- a/gentoobrowse-api/unittests/fixtures/files.dat +++ /dev/null diff --git a/gentoobrowse-api/unittests/fixtures/filetypes.dat b/gentoobrowse-api/unittests/fixtures/filetypes.dat deleted file mode 100644 index 34c95ee..0000000 --- a/gentoobrowse-api/unittests/fixtures/filetypes.dat +++ /dev/null @@ -1,2 +0,0 @@ -4 package metadata.xml {"(3,metadata.xml)"} 1 \N -11 news {"(1,metadata)","(2,news)","(4,%.txt)"} 1 \N diff --git a/gentoobrowse-api/unittests/mockDefs.cpp b/gentoobrowse-api/unittests/mockDefs.cpp index a067b01..a865c39 100644 --- a/gentoobrowse-api/unittests/mockDefs.cpp +++ b/gentoobrowse-api/unittests/mockDefs.cpp @@ -29,8 +29,7 @@ Maintenance::Maintenance() : "--GentooBrowseAPI.BugRoot=file://" + (rootDir / "fixtures" / "bugs").string() }), PQ::Mock("user=postgres dbname=postgres", "GentooBrowseAPI", { - rootDir.parent_path() / "db" / "schema.sql", - rootDir / "basedata.sql" }) + rootDir.parent_path() / "db" / "schema.sql" }) { replace("mailserver", new MockMailServerImpl()); } diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp index 1e3ae8e..0b451ca 100644 --- a/gentoobrowse-api/unittests/testMaintenance.cpp +++ b/gentoobrowse-api/unittests/testMaintenance.cpp @@ -8,9 +8,6 @@ #include <git2.h> #include <fstream> #include <maintenanceimpl.h> -#include <sql/maintenance/gitListCreateRaw.sql.h> -#include <sql/maintenance/gitListCreate.sql.h> -#include <sql/maintenance/gitListCreateIdx.sql.h> class MaintenanceClientCombined : public Maintenance, public TestClient { }; @@ -75,31 +72,23 @@ class M2 : public Gentoo::Service::Maintenance { M2(IceTray::DatabasePoolPtr d) : Gentoo::Service::Maintenance(d) { - auto dbc = db->get(); - DB::TransactionScope tx(dbc.get()); - updateFileTypes(dbc.get()); } - void applyDiffOfFolders(int64_t repoId, const boost::filesystem::path & from, const boost::filesystem::path & to) const + void applyDiffOfFolders(const boost::filesystem::path & from, const boost::filesystem::path & to) const { auto dbc = db->get(); auto fromFiles = fileSet(from); auto toFiles = fileSet(to); dbc->execute("SET search_path = gentoobrowse, pg_catalog"); - Gentoo::Service::sql::maintenance::gitListCreateRaw.modify(dbc.get())->execute(); - auto ins = dbc->modify("INSERT INTO filelistraw(repoid, status, filename) VALUES(?, ?, ?)"); - ins->bindParamI(0, repoId); - newFiles(ins, fromFiles, toFiles); - removedFiles(ins, fromFiles, toFiles); - changedFiles(ins, fromFiles, toFiles); - Gentoo::Service::sql::maintenance::gitListCreate.modify(dbc.get())->execute(); - Gentoo::Service::sql::maintenance::gitListCreateIdx.modify(dbc.get())->execute(); + DB::TransactionScope tx(dbc.get()); + Gentoo::Service::ChangeSet changes; + newFiles(changes, fromFiles, toFiles); + removedFiles(changes, fromFiles, toFiles); + changedFiles(changes, fromFiles, toFiles); boost::filesystem::remove(treeDir); boost::filesystem::create_symlink(to, treeDir); - applyFileChanges(dbc.get(), "/"); + applyFileChanges(dbc.get(), 1, "/", to, changes); boost::filesystem::remove(treeDir); - dbc->execute("DROP TABLE filelist"); - dbc->execute("DROP TABLE filelistraw"); dbc->execute("SET search_path = public, pg_catalog"); } @@ -113,34 +102,38 @@ class M2 : public Gentoo::Service::Maintenance { } return found; } - static void newFiles(DB::ModifyCommandPtr ins, const FileMap & from, const FileMap & to) + + static Gentoo::Service::PathParts split(const boost::filesystem::path & path) + { + Gentoo::Service::PathParts pp; + for (const auto & p : path) { + pp.push_back(p.string()); + } + return pp; + } + + static void newFiles(Gentoo::Service::ChangeSet & changes, const FileMap & from, const FileMap & to) { - ins->bindParamS(1, "A"); for(const auto & f : to) { if (from.find(f.first) == from.end()) { - ins->bindParamS(2, f.first.string()); - ins->execute(); + changes[f.first] = { split(f.first), GIT_DELTA_ADDED }; } } } - static void removedFiles(DB::ModifyCommandPtr ins, const FileMap & from, const FileMap & to) + static void removedFiles(Gentoo::Service::ChangeSet & changes, const FileMap & from, const FileMap & to) { - ins->bindParamS(1, "D"); for(const auto & f : from) { if (to.find(f.first) == to.end()) { - ins->bindParamS(2, f.first.string()); - ins->execute(); + changes[f.first] = { split(f.first), GIT_DELTA_DELETED }; } } } - static void changedFiles(DB::ModifyCommandPtr ins, const FileMap & from, const FileMap & to) + static void changedFiles(Gentoo::Service::ChangeSet & changes, const FileMap & from, const FileMap & to) { - ins->bindParamS(1, "M"); for(const auto & f : to) { auto i = from.find(f.first); if (i != from.end() && i->second != f.second) { - ins->bindParamS(2, f.first.string()); - ins->execute(); + changes[f.first] = { split(f.first), GIT_DELTA_MODIFIED }; } } } @@ -172,19 +165,19 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree ) M2 m2(p); BOOST_TEST_CONTEXT("4156eb45cf3b0ce1d7125b84efd8688c2d6e831d") { - m2.applyDiffOfFolders(1, binDir / "empty", rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d"); + m2.applyDiffOfFolders(binDir / "empty", rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d"); doRefreshPackageTree(db, 5, 1, 482, 981, 3626, 4593, 501, 393, 238, 50, 1573, 2008, 1543, 81, 152); } BOOST_TEST_CONTEXT("756569aa764177340726dd3d40b41d89b11b20c7") { - m2.applyDiffOfFolders(1, rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d", rootDir / "fixtures" / "756569aa764177340726dd3d40b41d89b11b20c7"); + m2.applyDiffOfFolders(rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d", rootDir / "fixtures" / "756569aa764177340726dd3d40b41d89b11b20c7"); doRefreshPackageTree(db, 5, 1, 483, 982, 3638, 4599, 502, 393, 238, 50, 1573, 2009, 1546, 79, 152); } BOOST_TEST_CONTEXT("empty") { - m2.applyDiffOfFolders(1, rootDir / "fixtures" / "756569aa764177340726dd3d40b41d89b11b20c7", binDir / "empty"); + m2.applyDiffOfFolders(rootDir / "fixtures" / "756569aa764177340726dd3d40b41d89b11b20c7", binDir / "empty"); doRefreshPackageTree(db, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); } |