From b709345af30e7e7e32935907da6cf2f42ba049cb Mon Sep 17 00:00:00 2001 From: Dan Goodliffe Date: Sat, 23 Apr 2016 17:19:02 +0100 Subject: Build the package tree the ebuild cache --- .../service/maintenance/categoryMetaProcessor.cpp | 13 ++--- .../service/maintenance/ebuildMetaProcessor.cpp | 64 ++++++++++++++++++---- .../maintenance/packageManifestProcessor.cpp | 15 ++--- gentoobrowse-api/service/maintenanceimpl.cpp | 2 +- .../service/sql/maintenance/categoryDelete.sql | 2 - .../service/sql/maintenance/categoryInsert.sql | 5 +- .../service/sql/maintenance/categoryMetaUpdate.sql | 3 + .../service/sql/maintenance/categoryPrune.sql | 5 ++ .../service/sql/maintenance/categoryUpdate.sql | 3 - .../service/sql/maintenance/ebuildDelete.sql | 3 +- .../service/sql/maintenance/packageDelete.sql | 4 -- .../service/sql/maintenance/packageInsert.sql | 3 + .../service/sql/maintenance/packagePrune.sql | 7 +++ gentoobrowse-api/unittests/fixtures/filetypes.dat | 8 +-- gentoobrowse-api/unittests/testMaintenance.cpp | 12 ++-- 15 files changed, 95 insertions(+), 54 deletions(-) delete mode 100644 gentoobrowse-api/service/sql/maintenance/categoryDelete.sql create mode 100644 gentoobrowse-api/service/sql/maintenance/categoryMetaUpdate.sql create mode 100644 gentoobrowse-api/service/sql/maintenance/categoryPrune.sql delete mode 100644 gentoobrowse-api/service/sql/maintenance/categoryUpdate.sql delete mode 100644 gentoobrowse-api/service/sql/maintenance/packageDelete.sql create mode 100644 gentoobrowse-api/service/sql/maintenance/packagePrune.sql diff --git a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp index cd4cdfe..72c41c9 100644 --- a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp @@ -3,9 +3,7 @@ #include "fileUtils.h" #include "xmlUtils.h" #include "dbUtils.h" -#include -#include -#include +#include namespace U = Gentoo::Utils; using namespace Gentoo::Utils::File; @@ -17,7 +15,7 @@ namespace Gentoo { void CategoryMetaProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const { - upsert(dbc->modify(sql::maintenance::categoryInsert::sql), fn, path); + upsert(dbc->modify(sql::maintenance::categoryMetaUpdate::sql), fn, path); } void @@ -35,14 +33,15 @@ namespace Gentoo { void CategoryMetaProcessor::modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const { - upsert(dbc->modify(sql::maintenance::categoryUpdate::sql), fn, path); + upsert(dbc->modify(sql::maintenance::categoryMetaUpdate::sql), fn, path); } void CategoryMetaProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const { - auto m = dbc->modify(sql::maintenance::categoryDelete::sql); - m->bindParamS(0, (fn / 1).string()); + auto m = dbc->modify(sql::maintenance::categoryMetaUpdate::sql); + m->bindNull(0); + m->bindParamS(1, (fn / 1).string()); m->execute(); } } diff --git a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp index 1085b70..638d525 100644 --- a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp @@ -5,6 +5,10 @@ #include "fileUtils.h" #include "xmlUtils.h" #include "dbUtils.h" +#include +#include +#include +#include #include #include #include @@ -58,17 +62,35 @@ namespace Gentoo { Glib::MatchInfo matches; const Glib::ustring pv = (fn / 4).string(); if (packageVersion->match(pv, matches)) { - auto m = dbc->modify(sql::maintenance::ebuildInsert::sql); EbuildCacheParser ecp(path); + const std::string repoName = (fn / 0).string(); + const std::string categoryName = (fn / 3).string(); + const std::string packageName = matches.fetch(1); + const std::string ebuildVersion = matches.fetch(2); + + // Maybe create a category + auto ci = dbc->modify(sql::maintenance::categoryInsert::sql); + ci->bindParamS(0, categoryName); + ci->bindParamS(1, categoryName); + ci->execute(); + // Maybe create a package + auto pi = dbc->modify(sql::maintenance::packageInsert::sql); + pi->bindParamS(0, packageName); + Utils::Database::bindOptionalsS(pi, 1, { ecp.get("DESCRIPTION") }); + pi->bindParamS(2, packageName); + pi->bindParamS(3, categoryName); + pi->execute(); + // Create an ebuild + auto m = dbc->modify(sql::maintenance::ebuildInsert::sql); m->bindParamS(0, matches.fetch(2)); m->bindParamS(1, matches.fetch(2)); Utils::Database::bindOptionalsS(m, 2, { ecp.get("SLOT") }); Utils::Database::bindOptionalsS(m, 3, { ecp.get("LICENSE") }); m->bindParamT(4, boost::posix_time::from_time_t(ecp.getStat().st_mtim.tv_sec)); - m->bindParamS(5, (fn / 0).string()); // repo - m->bindParamS(6, (fn / 3).string()); // category - m->bindParamS(7, matches.fetch(1)); // package - m->execute(); + m->bindParamS(5, repoName); + m->bindParamS(6, categoryName); + m->bindParamS(7, packageName); + m->execute(false); } } @@ -87,18 +109,38 @@ namespace Gentoo { m->bindParamS(4, (fn / 3).string()); // category m->bindParamS(5, matches.fetch(1)); // package m->bindParamS(6, matches.fetch(2)); // version - m->execute(); + m->execute(false); } } void EbuildMetaProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const { - auto m = dbc->modify(sql::maintenance::ebuildDelete::sql); - m->bindParamS(0, (fn / 0).string()); - m->bindParamS(1, (fn / 3).string()); - m->bindParamS(2, (fn / 4).string()); - m->execute(); + Glib::MatchInfo matches; + const Glib::ustring pv = (fn / 4).string(); + if (packageVersion->match(pv, matches)) { + const std::string repoName = (fn / 0).string(); + const std::string categoryName = (fn / 3).string(); + const std::string packageName = matches.fetch(1); + const std::string ebuildVersion = matches.fetch(2); + + fprintf(stderr, "r = %s, c = %s, p = %s, v = %s\n", + repoName.c_str(), categoryName.c_str(), packageName.c_str(), ebuildVersion.c_str()); + + // Delete ebuild + auto m = dbc->modify(sql::maintenance::ebuildDelete::sql); + m->bindParamS(0, repoName); + m->bindParamS(1, categoryName); + m->bindParamS(2, packageName); + m->bindParamS(3, ebuildVersion); + m->execute(false); + // Prune packages + auto pp = dbc->modify(sql::maintenance::packagePrune::sql); + pp->bindParamS(0, categoryName); + pp->execute(); + // Prune categories + dbc->execute(sql::maintenance::categoryPrune::sql); + } } } } diff --git a/gentoobrowse-api/service/maintenance/packageManifestProcessor.cpp b/gentoobrowse-api/service/maintenance/packageManifestProcessor.cpp index 8d6147c..a84f4a4 100644 --- a/gentoobrowse-api/service/maintenance/packageManifestProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/packageManifestProcessor.cpp @@ -1,8 +1,6 @@ #include "packageManifestProcessor.h" #include #include "fileUtils.h" -#include -#include using namespace Gentoo::Utils::File; @@ -13,20 +11,15 @@ namespace Gentoo { void PackageManifestProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path &) const { - auto m = dbc->modify(sql::maintenance::packageInsert::sql); - m->bindParamS(0, (fn / 2).string()); - m->bindParamS(1, fn.parent_path().string()); - m->bindParamS(2, (fn / 1).string()); - m->execute(); + (void)dbc; + (void)fn; } void PackageManifestProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const { - auto m = dbc->modify(sql::maintenance::packageDelete::sql); - m->bindParamS(0, (fn / 2).string()); - m->bindParamS(1, (fn / 1).string()); - m->execute(); + (void)dbc; + (void)fn; } } } diff --git a/gentoobrowse-api/service/maintenanceimpl.cpp b/gentoobrowse-api/service/maintenanceimpl.cpp index 18df8a1..5ea6b8e 100644 --- a/gentoobrowse-api/service/maintenanceimpl.cpp +++ b/gentoobrowse-api/service/maintenanceimpl.cpp @@ -14,9 +14,9 @@ /* 10 category metadata.xml {"(2,metadata.xml)"} 4 package metadata.xml {"(3,metadata.xml)"} -8 package manifests {"(3,Manifest)"} 1 ebuild metadata {"(1,metadata)","(2,md5-cache)"} +8 package manifests {"(3,Manifest)"} 2 changelog {"(3,ChangeLog)"} 3 masks {"(1,profiles)","(2,package.mask)"} 5 use_global {"(1,profiles)","(2,use.desc)"} diff --git a/gentoobrowse-api/service/sql/maintenance/categoryDelete.sql b/gentoobrowse-api/service/sql/maintenance/categoryDelete.sql deleted file mode 100644 index ad961c8..0000000 --- a/gentoobrowse-api/service/sql/maintenance/categoryDelete.sql +++ /dev/null @@ -1,2 +0,0 @@ -DELETE FROM gentoobrowse.categories -WHERE name = ? diff --git a/gentoobrowse-api/service/sql/maintenance/categoryInsert.sql b/gentoobrowse-api/service/sql/maintenance/categoryInsert.sql index fa8a1ca..27321a0 100644 --- a/gentoobrowse-api/service/sql/maintenance/categoryInsert.sql +++ b/gentoobrowse-api/service/sql/maintenance/categoryInsert.sql @@ -1,2 +1,3 @@ -INSERT INTO gentoobrowse.categories(summary, name) -VALUES(gentoobrowse.textclean(?), ?) +INSERT INTO gentoobrowse.categories(name) +SELECT ? +WHERE (NOT EXISTS (SELECT name FROM gentoobrowse.categories WHERE name = ?)) diff --git a/gentoobrowse-api/service/sql/maintenance/categoryMetaUpdate.sql b/gentoobrowse-api/service/sql/maintenance/categoryMetaUpdate.sql new file mode 100644 index 0000000..28c45b1 --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/categoryMetaUpdate.sql @@ -0,0 +1,3 @@ +UPDATE gentoobrowse.categories SET + summary = gentoobrowse.textclean(?) +WHERE name = ? diff --git a/gentoobrowse-api/service/sql/maintenance/categoryPrune.sql b/gentoobrowse-api/service/sql/maintenance/categoryPrune.sql new file mode 100644 index 0000000..3b75683 --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/categoryPrune.sql @@ -0,0 +1,5 @@ +DELETE FROM gentoobrowse.categories c +WHERE NOT EXISTS ( + SELECT FROM gentoobrowse.packages p + WHERE p.categoryid = c.categoryid) + diff --git a/gentoobrowse-api/service/sql/maintenance/categoryUpdate.sql b/gentoobrowse-api/service/sql/maintenance/categoryUpdate.sql deleted file mode 100644 index 28c45b1..0000000 --- a/gentoobrowse-api/service/sql/maintenance/categoryUpdate.sql +++ /dev/null @@ -1,3 +0,0 @@ -UPDATE gentoobrowse.categories SET - summary = gentoobrowse.textclean(?) -WHERE name = ? diff --git a/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql b/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql index 2c33008..bd46bcd 100644 --- a/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql +++ b/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql @@ -5,4 +5,5 @@ AND c.categoryid = p.categoryid AND e.repoid = r.repoid AND r.name = ? AND c.name = ? -AND CONCAT(p.name, '-', e.version) = ? +AND p.name = ? +AND e.version = ? diff --git a/gentoobrowse-api/service/sql/maintenance/packageDelete.sql b/gentoobrowse-api/service/sql/maintenance/packageDelete.sql deleted file mode 100644 index 5a2a0f1..0000000 --- a/gentoobrowse-api/service/sql/maintenance/packageDelete.sql +++ /dev/null @@ -1,4 +0,0 @@ -DELETE FROM gentoobrowse.packages p -USING gentoobrowse.categories c -WHERE p.name = ? -AND c.name = ? diff --git a/gentoobrowse-api/service/sql/maintenance/packageInsert.sql b/gentoobrowse-api/service/sql/maintenance/packageInsert.sql index 223cdbe..8e910f8 100644 --- a/gentoobrowse-api/service/sql/maintenance/packageInsert.sql +++ b/gentoobrowse-api/service/sql/maintenance/packageInsert.sql @@ -1,4 +1,7 @@ INSERT INTO gentoobrowse.packages(categoryid, name, description) SELECT c.categoryid, ?, ? FROM gentoobrowse.categories c + LEFT OUTER JOIN gentoobrowse.packages p + ON c.categoryid = p.categoryid AND p.name = ? WHERE c.name = ? +AND p.packageid IS NULL diff --git a/gentoobrowse-api/service/sql/maintenance/packagePrune.sql b/gentoobrowse-api/service/sql/maintenance/packagePrune.sql new file mode 100644 index 0000000..510ed75 --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/packagePrune.sql @@ -0,0 +1,7 @@ +DELETE FROM gentoobrowse.packages p +USING gentoobrowse.categories c +WHERE p.categoryid = c.categoryid +AND c.name = ? +AND NOT EXISTS ( + SELECT FROM gentoobrowse.ebuilds e + WHERE p.packageid = e.packageid) diff --git a/gentoobrowse-api/unittests/fixtures/filetypes.dat b/gentoobrowse-api/unittests/fixtures/filetypes.dat index 7f3d1dd..5d8ed02 100644 --- a/gentoobrowse-api/unittests/fixtures/filetypes.dat +++ b/gentoobrowse-api/unittests/fixtures/filetypes.dat @@ -1,11 +1,11 @@ 2 changelog {"(3,ChangeLog)"} \N 3 masks {"(1,profiles)","(2,package.mask)"} \N 7 licenses {"(1,licenses)"} \N -8 manifests {"(3,Manifest)"} 2 +8 manifests {"(3,Manifest)"} \N 9 use_grouped {"(1,profiles)","(2,desc)","(3,%.desc)"} \N 5 use_global {"(1,profiles)","(2,use.desc)"} \N 6 use_local {"(1,profiles)","(2,use.local.desc)"} \N -1 ebuild metadata {"(1,metadata)","(2,md5-cache)"} 3 -10 category metadata {"(2,metadata.xml)"} 1 -4 package metadata.xml {"(3,metadata.xml)"} \N +1 ebuild metadata {"(1,metadata)","(2,md5-cache)"} 1 +10 category metadata.xml {"(2,metadata.xml)"} 2 +4 package metadata.xml {"(3,metadata.xml)"} 3 11 news {"(1,metadata)","(2,news)","(4,%.txt)"} \N diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp index f597f07..45819e0 100644 --- a/gentoobrowse-api/unittests/testMaintenance.cpp +++ b/gentoobrowse-api/unittests/testMaintenance.cpp @@ -58,10 +58,8 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree ) SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.files", int64_t, 2084); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, 5); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories WHERE name = 'dev-vcs'", int64_t, 1); - SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 478); - // Missing packages - // SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 981); - SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 977); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 482); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 981); db->execute("COPY gentoobrowse.categories TO '/tmp/categories1.tsv'"); db->execute("COPY gentoobrowse.packages TO '/tmp/packages1.tsv'"); db->execute("COPY gentoobrowse.ebuilds TO '/tmp/ebuilds1.tsv'"); @@ -73,10 +71,8 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree ) SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.files", int64_t, 2034); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, 5); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories WHERE name = 'dev-vcs'", int64_t, 1); - SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 480); - // Missing packages - // SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 982); - SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 978); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 484); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 982); db->execute("COPY gentoobrowse.categories TO '/tmp/categories2.tsv'"); db->execute("COPY gentoobrowse.packages TO '/tmp/packages2.tsv'"); db->execute("COPY gentoobrowse.ebuilds TO '/tmp/ebuilds2.tsv'"); -- cgit v1.2.3