diff options
14 files changed, 92 insertions, 51 deletions
diff --git a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp index cd4cdfe..72c41c9 100644 --- a/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/categoryMetaProcessor.cpp @@ -3,9 +3,7 @@ #include "fileUtils.h" #include "xmlUtils.h" #include "dbUtils.h" -#include <sql/maintenance/categoryInsert.sql.h> -#include <sql/maintenance/categoryUpdate.sql.h> -#include <sql/maintenance/categoryDelete.sql.h> +#include <sql/maintenance/categoryMetaUpdate.sql.h> namespace U = Gentoo::Utils; using namespace Gentoo::Utils::File; @@ -17,7 +15,7 @@ namespace Gentoo { void CategoryMetaProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const { - upsert(dbc->modify(sql::maintenance::categoryInsert::sql), fn, path); + upsert(dbc->modify(sql::maintenance::categoryMetaUpdate::sql), fn, path); } void @@ -35,14 +33,15 @@ namespace Gentoo { void CategoryMetaProcessor::modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) const { - upsert(dbc->modify(sql::maintenance::categoryUpdate::sql), fn, path); + upsert(dbc->modify(sql::maintenance::categoryMetaUpdate::sql), fn, path); } void CategoryMetaProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const { - auto m = dbc->modify(sql::maintenance::categoryDelete::sql); - m->bindParamS(0, (fn / 1).string()); + auto m = dbc->modify(sql::maintenance::categoryMetaUpdate::sql); + m->bindNull(0); + m->bindParamS(1, (fn / 1).string()); m->execute(); } } diff --git a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp index 1085b70..638d525 100644 --- a/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/ebuildMetaProcessor.cpp @@ -5,6 +5,10 @@ #include "fileUtils.h" #include "xmlUtils.h" #include "dbUtils.h" +#include <sql/maintenance/categoryInsert.sql.h> +#include <sql/maintenance/packageInsert.sql.h> +#include <sql/maintenance/packagePrune.sql.h> +#include <sql/maintenance/categoryPrune.sql.h> #include <sql/maintenance/ebuildInsert.sql.h> #include <sql/maintenance/ebuildUpdate.sql.h> #include <sql/maintenance/ebuildDelete.sql.h> @@ -58,17 +62,35 @@ namespace Gentoo { Glib::MatchInfo matches; const Glib::ustring pv = (fn / 4).string(); if (packageVersion->match(pv, matches)) { - auto m = dbc->modify(sql::maintenance::ebuildInsert::sql); EbuildCacheParser ecp(path); + const std::string repoName = (fn / 0).string(); + const std::string categoryName = (fn / 3).string(); + const std::string packageName = matches.fetch(1); + const std::string ebuildVersion = matches.fetch(2); + + // Maybe create a category + auto ci = dbc->modify(sql::maintenance::categoryInsert::sql); + ci->bindParamS(0, categoryName); + ci->bindParamS(1, categoryName); + ci->execute(); + // Maybe create a package + auto pi = dbc->modify(sql::maintenance::packageInsert::sql); + pi->bindParamS(0, packageName); + Utils::Database::bindOptionalsS(pi, 1, { ecp.get("DESCRIPTION") }); + pi->bindParamS(2, packageName); + pi->bindParamS(3, categoryName); + pi->execute(); + // Create an ebuild + auto m = dbc->modify(sql::maintenance::ebuildInsert::sql); m->bindParamS(0, matches.fetch(2)); m->bindParamS(1, matches.fetch(2)); Utils::Database::bindOptionalsS(m, 2, { ecp.get("SLOT") }); Utils::Database::bindOptionalsS(m, 3, { ecp.get("LICENSE") }); m->bindParamT(4, boost::posix_time::from_time_t(ecp.getStat().st_mtim.tv_sec)); - m->bindParamS(5, (fn / 0).string()); // repo - m->bindParamS(6, (fn / 3).string()); // category - m->bindParamS(7, matches.fetch(1)); // package - m->execute(); + m->bindParamS(5, repoName); + m->bindParamS(6, categoryName); + m->bindParamS(7, packageName); + m->execute(false); } } @@ -87,18 +109,38 @@ namespace Gentoo { m->bindParamS(4, (fn / 3).string()); // category m->bindParamS(5, matches.fetch(1)); // package m->bindParamS(6, matches.fetch(2)); // version - m->execute(); + m->execute(false); } } void EbuildMetaProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const { - auto m = dbc->modify(sql::maintenance::ebuildDelete::sql); - m->bindParamS(0, (fn / 0).string()); - m->bindParamS(1, (fn / 3).string()); - m->bindParamS(2, (fn / 4).string()); - m->execute(); + Glib::MatchInfo matches; + const Glib::ustring pv = (fn / 4).string(); + if (packageVersion->match(pv, matches)) { + const std::string repoName = (fn / 0).string(); + const std::string categoryName = (fn / 3).string(); + const std::string packageName = matches.fetch(1); + const std::string ebuildVersion = matches.fetch(2); + + fprintf(stderr, "r = %s, c = %s, p = %s, v = %s\n", + repoName.c_str(), categoryName.c_str(), packageName.c_str(), ebuildVersion.c_str()); + + // Delete ebuild + auto m = dbc->modify(sql::maintenance::ebuildDelete::sql); + m->bindParamS(0, repoName); + m->bindParamS(1, categoryName); + m->bindParamS(2, packageName); + m->bindParamS(3, ebuildVersion); + m->execute(false); + // Prune packages + auto pp = dbc->modify(sql::maintenance::packagePrune::sql); + pp->bindParamS(0, categoryName); + pp->execute(); + // Prune categories + dbc->execute(sql::maintenance::categoryPrune::sql); + } } } } diff --git a/gentoobrowse-api/service/maintenance/packageManifestProcessor.cpp b/gentoobrowse-api/service/maintenance/packageManifestProcessor.cpp index 8d6147c..a84f4a4 100644 --- a/gentoobrowse-api/service/maintenance/packageManifestProcessor.cpp +++ b/gentoobrowse-api/service/maintenance/packageManifestProcessor.cpp @@ -1,8 +1,6 @@ #include "packageManifestProcessor.h" #include <modifycommand.h> #include "fileUtils.h" -#include <sql/maintenance/packageInsert.sql.h> -#include <sql/maintenance/packageDelete.sql.h> using namespace Gentoo::Utils::File; @@ -13,20 +11,15 @@ namespace Gentoo { void PackageManifestProcessor::created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path &) const { - auto m = dbc->modify(sql::maintenance::packageInsert::sql); - m->bindParamS(0, (fn / 2).string()); - m->bindParamS(1, fn.parent_path().string()); - m->bindParamS(2, (fn / 1).string()); - m->execute(); + (void)dbc; + (void)fn; } void PackageManifestProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) const { - auto m = dbc->modify(sql::maintenance::packageDelete::sql); - m->bindParamS(0, (fn / 2).string()); - m->bindParamS(1, (fn / 1).string()); - m->execute(); + (void)dbc; + (void)fn; } } } diff --git a/gentoobrowse-api/service/maintenanceimpl.cpp b/gentoobrowse-api/service/maintenanceimpl.cpp index 18df8a1..5ea6b8e 100644 --- a/gentoobrowse-api/service/maintenanceimpl.cpp +++ b/gentoobrowse-api/service/maintenanceimpl.cpp @@ -14,9 +14,9 @@ /* 10 category metadata.xml {"(2,metadata.xml)"} 4 package metadata.xml {"(3,metadata.xml)"} -8 package manifests {"(3,Manifest)"} 1 ebuild metadata {"(1,metadata)","(2,md5-cache)"} +8 package manifests {"(3,Manifest)"} 2 changelog {"(3,ChangeLog)"} 3 masks {"(1,profiles)","(2,package.mask)"} 5 use_global {"(1,profiles)","(2,use.desc)"} diff --git a/gentoobrowse-api/service/sql/maintenance/categoryDelete.sql b/gentoobrowse-api/service/sql/maintenance/categoryDelete.sql deleted file mode 100644 index ad961c8..0000000 --- a/gentoobrowse-api/service/sql/maintenance/categoryDelete.sql +++ /dev/null @@ -1,2 +0,0 @@ -DELETE FROM gentoobrowse.categories -WHERE name = ? diff --git a/gentoobrowse-api/service/sql/maintenance/categoryInsert.sql b/gentoobrowse-api/service/sql/maintenance/categoryInsert.sql index fa8a1ca..27321a0 100644 --- a/gentoobrowse-api/service/sql/maintenance/categoryInsert.sql +++ b/gentoobrowse-api/service/sql/maintenance/categoryInsert.sql @@ -1,2 +1,3 @@ -INSERT INTO gentoobrowse.categories(summary, name) -VALUES(gentoobrowse.textclean(?), ?) +INSERT INTO gentoobrowse.categories(name) +SELECT ? +WHERE (NOT EXISTS (SELECT name FROM gentoobrowse.categories WHERE name = ?)) diff --git a/gentoobrowse-api/service/sql/maintenance/categoryUpdate.sql b/gentoobrowse-api/service/sql/maintenance/categoryMetaUpdate.sql index 28c45b1..28c45b1 100644 --- a/gentoobrowse-api/service/sql/maintenance/categoryUpdate.sql +++ b/gentoobrowse-api/service/sql/maintenance/categoryMetaUpdate.sql diff --git a/gentoobrowse-api/service/sql/maintenance/categoryPrune.sql b/gentoobrowse-api/service/sql/maintenance/categoryPrune.sql new file mode 100644 index 0000000..3b75683 --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/categoryPrune.sql @@ -0,0 +1,5 @@ +DELETE FROM gentoobrowse.categories c +WHERE NOT EXISTS ( + SELECT FROM gentoobrowse.packages p + WHERE p.categoryid = c.categoryid) + diff --git a/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql b/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql index 2c33008..bd46bcd 100644 --- a/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql +++ b/gentoobrowse-api/service/sql/maintenance/ebuildDelete.sql @@ -5,4 +5,5 @@ AND c.categoryid = p.categoryid AND e.repoid = r.repoid AND r.name = ? AND c.name = ? -AND CONCAT(p.name, '-', e.version) = ? +AND p.name = ? +AND e.version = ? diff --git a/gentoobrowse-api/service/sql/maintenance/packageDelete.sql b/gentoobrowse-api/service/sql/maintenance/packageDelete.sql deleted file mode 100644 index 5a2a0f1..0000000 --- a/gentoobrowse-api/service/sql/maintenance/packageDelete.sql +++ /dev/null @@ -1,4 +0,0 @@ -DELETE FROM gentoobrowse.packages p -USING gentoobrowse.categories c -WHERE p.name = ? -AND c.name = ? diff --git a/gentoobrowse-api/service/sql/maintenance/packageInsert.sql b/gentoobrowse-api/service/sql/maintenance/packageInsert.sql index 223cdbe..8e910f8 100644 --- a/gentoobrowse-api/service/sql/maintenance/packageInsert.sql +++ b/gentoobrowse-api/service/sql/maintenance/packageInsert.sql @@ -1,4 +1,7 @@ INSERT INTO gentoobrowse.packages(categoryid, name, description) SELECT c.categoryid, ?, ? FROM gentoobrowse.categories c + LEFT OUTER JOIN gentoobrowse.packages p + ON c.categoryid = p.categoryid AND p.name = ? WHERE c.name = ? +AND p.packageid IS NULL diff --git a/gentoobrowse-api/service/sql/maintenance/packagePrune.sql b/gentoobrowse-api/service/sql/maintenance/packagePrune.sql new file mode 100644 index 0000000..510ed75 --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/packagePrune.sql @@ -0,0 +1,7 @@ +DELETE FROM gentoobrowse.packages p +USING gentoobrowse.categories c +WHERE p.categoryid = c.categoryid +AND c.name = ? +AND NOT EXISTS ( + SELECT FROM gentoobrowse.ebuilds e + WHERE p.packageid = e.packageid) diff --git a/gentoobrowse-api/unittests/fixtures/filetypes.dat b/gentoobrowse-api/unittests/fixtures/filetypes.dat index 7f3d1dd..5d8ed02 100644 --- a/gentoobrowse-api/unittests/fixtures/filetypes.dat +++ b/gentoobrowse-api/unittests/fixtures/filetypes.dat @@ -1,11 +1,11 @@ 2 changelog {"(3,ChangeLog)"} \N 3 masks {"(1,profiles)","(2,package.mask)"} \N 7 licenses {"(1,licenses)"} \N -8 manifests {"(3,Manifest)"} 2 +8 manifests {"(3,Manifest)"} \N 9 use_grouped {"(1,profiles)","(2,desc)","(3,%.desc)"} \N 5 use_global {"(1,profiles)","(2,use.desc)"} \N 6 use_local {"(1,profiles)","(2,use.local.desc)"} \N -1 ebuild metadata {"(1,metadata)","(2,md5-cache)"} 3 -10 category metadata {"(2,metadata.xml)"} 1 -4 package metadata.xml {"(3,metadata.xml)"} \N +1 ebuild metadata {"(1,metadata)","(2,md5-cache)"} 1 +10 category metadata.xml {"(2,metadata.xml)"} 2 +4 package metadata.xml {"(3,metadata.xml)"} 3 11 news {"(1,metadata)","(2,news)","(4,%.txt)"} \N diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp index f597f07..45819e0 100644 --- a/gentoobrowse-api/unittests/testMaintenance.cpp +++ b/gentoobrowse-api/unittests/testMaintenance.cpp @@ -58,10 +58,8 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree ) SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.files", int64_t, 2084); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, 5); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories WHERE name = 'dev-vcs'", int64_t, 1); - SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 478); - // Missing packages - // SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 981); - SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 977); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 482); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 981); db->execute("COPY gentoobrowse.categories TO '/tmp/categories1.tsv'"); db->execute("COPY gentoobrowse.packages TO '/tmp/packages1.tsv'"); db->execute("COPY gentoobrowse.ebuilds TO '/tmp/ebuilds1.tsv'"); @@ -73,10 +71,8 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree ) SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.files", int64_t, 2034); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories", int64_t, 5); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.categories WHERE name = 'dev-vcs'", int64_t, 1); - SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 480); - // Missing packages - // SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 982); - SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 978); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.packages", int64_t, 484); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuilds", int64_t, 982); db->execute("COPY gentoobrowse.categories TO '/tmp/categories2.tsv'"); db->execute("COPY gentoobrowse.packages TO '/tmp/packages2.tsv'"); db->execute("COPY gentoobrowse.ebuilds TO '/tmp/ebuilds2.tsv'"); |