From 856d92e4d6a36f2944906cce8d9290e83f857927 Mon Sep 17 00:00:00 2001 From: Dan Goodliffe Date: Sun, 29 May 2016 00:09:11 +0100 Subject: Import bug list htmls --- gentoobrowse-api/api/maintenance.ice | 1 + gentoobrowse-api/service/maintenanceBugs.cpp | 98 ++++++++++++++++++++++ gentoobrowse-api/service/maintenanceCommon.cpp | 14 ++++ gentoobrowse-api/service/maintenanceimpl.h | 4 + gentoobrowse-api/unittests/Jamfile.jam | 8 ++ .../unittests/fixtures/bugs/buglist-CONFIRMED.html | 13 +++ .../fixtures/bugs/buglist-IN_PROGRESS.html | 16 ++++ .../fixtures/bugs/buglist-UNCONFIRMED.html | 17 ++++ gentoobrowse-api/unittests/mockDefs.cpp | 1 + gentoobrowse-api/unittests/mockDefs.h | 4 + gentoobrowse-api/unittests/testBugs.cpp | 20 +++++ gentoobrowse-api/unittests/testMaintenance.cpp | 4 - 12 files changed, 196 insertions(+), 4 deletions(-) create mode 100644 gentoobrowse-api/service/maintenanceBugs.cpp create mode 100644 gentoobrowse-api/service/maintenanceCommon.cpp create mode 100644 gentoobrowse-api/unittests/fixtures/bugs/buglist-CONFIRMED.html create mode 100644 gentoobrowse-api/unittests/fixtures/bugs/buglist-IN_PROGRESS.html create mode 100644 gentoobrowse-api/unittests/fixtures/bugs/buglist-UNCONFIRMED.html create mode 100644 gentoobrowse-api/unittests/testBugs.cpp diff --git a/gentoobrowse-api/api/maintenance.ice b/gentoobrowse-api/api/maintenance.ice index 55389b2..0e80fed 100644 --- a/gentoobrowse-api/api/maintenance.ice +++ b/gentoobrowse-api/api/maintenance.ice @@ -1,6 +1,7 @@ module Gentoo { interface Maintenance { idempotent void refreshPackageTree(); + idempotent void refreshBugs(); }; }; diff --git a/gentoobrowse-api/service/maintenanceBugs.cpp b/gentoobrowse-api/service/maintenanceBugs.cpp new file mode 100644 index 0000000..0917f97 --- /dev/null +++ b/gentoobrowse-api/service/maintenanceBugs.cpp @@ -0,0 +1,98 @@ +#include "maintenanceimpl.h" +#include +#include +#include +#include +#include +#include +#include +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include +#include +#pragma GCC diagnostic pop +#include +#include + +namespace Gentoo { + namespace Service { + Utils::Lexer::PatternPtr bugLink = Utils::Lexer::regex( + "Bug:(\\d+) - \"\" status:(\\w*) resolution:(\\w*) severity:(\\w*).*", G_REGEX_OPTIMIZE); + + class BugListParser : public xmlpp::SaxParser, Utils::Lexer { + public: + BugListParser(DB::ModifyCommandPtr i) : + ins(i) + { + rules.push_back({ { InitialState }, bugLink, [this](auto es) { + ins->bindParamI(0, boost::lexical_cast(*es->pattern->match(1))); + ins->bindParamS(2, *es->pattern->match(1)); + ins->bindParamS(1, *es->pattern->match(3)); + }}); + } + + protected: + void on_start_element(const Glib::ustring & e, const xmlpp::SaxParser::AttributeList &) override + { + stk.push(e); + } + + void on_characters(const Glib::ustring & t) override + { + if (stk.top() == "a") { + attributes += t; + } + else if (stk.top() == "em") { + summary += t; + } + } + + void on_end_element(const Glib::ustring & e) override + { + stk.pop(); + if (e == "li") { + fprintf(stderr, "attributes = '%s', summary = '%s'\n", attributes.c_str(), summary.c_str()); + extract(attributes.c_str(), attributes.length()); + ins->bindParamS(3, summary); + ins->execute(); + attributes.clear(); + summary.clear(); + } + } + + private: + DB::ModifyCommandPtr ins; + std::stack stk; + + Glib::ustring attributes; + Glib::ustring summary; + }; + + void + Maintenance::refreshBugs(const Ice::Current & c) + { + boost::filesystem::path root = properties(c)->getPropertyWithDefault( + "GentooBrowseAPI.BugRoot", "https://bugs.gentoo.org/data/cached"); + + auto dbc = db->get(); + DB::TransactionScope tx(dbc.get()); + DB::TablePatch tp; + tp.pk = { "bugId" }; + tp.cols = { "bugId", "severity", "status", "summary" }; + tp.dest = "gentoobrowse.bugs"; + tp.src = Utils::Database::emptyClone(dbc.get(), "gentoobrowse.bugs"); + auto ins = Utils::Database::tablePatchInserter(dbc.get(), tp); + BugListParser blp(ins); + for(const auto & bl : { + "buglist-CONFIRMED.html", + "buglist-UNCONFIRMED.html", + "buglist-IN_PROGRESS.html" }) { + AdHoc::Net::CurlStreamSource css((root / bl).string()); + css.setopt(CURLOPT_ENCODING, "deflate, gzip"); + AdHoc::Net::CurlStream cs(css); + blp.parse_stream(cs); + } + dbc->patchTable(&tp); + } + } +} + diff --git a/gentoobrowse-api/service/maintenanceCommon.cpp b/gentoobrowse-api/service/maintenanceCommon.cpp new file mode 100644 index 0000000..c05132b --- /dev/null +++ b/gentoobrowse-api/service/maintenanceCommon.cpp @@ -0,0 +1,14 @@ +#include "maintenanceimpl.h" +#include +#include + +namespace Gentoo { + namespace Service { + Ice::PropertiesPtr + Maintenance::properties(const Ice::Current & c) + { + return c.adapter->getCommunicator()->getProperties(); + } + } +} + diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h index 6331672..7da0d55 100644 --- a/gentoobrowse-api/service/maintenanceimpl.h +++ b/gentoobrowse-api/service/maintenanceimpl.h @@ -1,6 +1,7 @@ #ifndef MAINTENANCEIMPL_H #define MAINTENANCEIMPL_H +#include #include #include #include @@ -33,6 +34,7 @@ namespace Gentoo { ~Maintenance(); void refreshPackageTree(const Ice::Current &) override; + void refreshBugs(const Ice::Current &) override; private: static void createTempFileList(DB::Connection *, const boost::filesystem::path &); @@ -42,6 +44,8 @@ namespace Gentoo { void fileCreated(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path & tmp, DB::SelectCommandPtr s); void fileHandle(int64_t filetypeId, const FileProcessors *, const FileHandleFunc &) const; + static Ice::PropertiesPtr properties(const Ice::Current &); + template static FileProcessorPtr createFileProessor(); FileProcessorFactories fpfs; diff --git a/gentoobrowse-api/unittests/Jamfile.jam b/gentoobrowse-api/unittests/Jamfile.jam index 71aa67c..4f38f9d 100644 --- a/gentoobrowse-api/unittests/Jamfile.jam +++ b/gentoobrowse-api/unittests/Jamfile.jam @@ -65,6 +65,14 @@ run testCommon : testPortage ; +run + testBugs.cpp + : : : + ../db/schema.sql + BOOST_TEST_DYN_LINK + testCommon + : testBugs ; + explicit testPerf ; run testPerf.cpp : : : diff --git a/gentoobrowse-api/unittests/fixtures/bugs/buglist-CONFIRMED.html b/gentoobrowse-api/unittests/fixtures/bugs/buglist-CONFIRMED.html new file mode 100644 index 0000000..a1c20ba --- /dev/null +++ b/gentoobrowse-api/unittests/fixtures/bugs/buglist-CONFIRMED.html @@ -0,0 +1,13 @@ + +Bug listing with status CONFIRMED as at 2016/05/27 18:48:05 +

Bug listing with status CONFIRMED as at 2016/05/27 18:48:05

+
Done. Count=12520
diff --git a/gentoobrowse-api/unittests/fixtures/bugs/buglist-IN_PROGRESS.html b/gentoobrowse-api/unittests/fixtures/bugs/buglist-IN_PROGRESS.html new file mode 100644 index 0000000..efa195a --- /dev/null +++ b/gentoobrowse-api/unittests/fixtures/bugs/buglist-IN_PROGRESS.html @@ -0,0 +1,16 @@ + +Bug listing with status IN_PROGRESS as at 2016/05/27 18:48:07 +

Bug listing with status IN_PROGRESS as at 2016/05/27 18:48:07

+
Done. Count=1397
diff --git a/gentoobrowse-api/unittests/fixtures/bugs/buglist-UNCONFIRMED.html b/gentoobrowse-api/unittests/fixtures/bugs/buglist-UNCONFIRMED.html new file mode 100644 index 0000000..b174bea --- /dev/null +++ b/gentoobrowse-api/unittests/fixtures/bugs/buglist-UNCONFIRMED.html @@ -0,0 +1,17 @@ + +Bug listing with status UNCONFIRMED as at 2016/05/27 18:48:04 +

Bug listing with status UNCONFIRMED as at 2016/05/27 18:48:04

+
Done. Count=6753
diff --git a/gentoobrowse-api/unittests/mockDefs.cpp b/gentoobrowse-api/unittests/mockDefs.cpp index 3221909..2d89fab 100644 --- a/gentoobrowse-api/unittests/mockDefs.cpp +++ b/gentoobrowse-api/unittests/mockDefs.cpp @@ -9,6 +9,7 @@ Service::Service() : } Maintenance::Maintenance() : + IceTray::DryIce({ "--GentooBrowseAPI.BugRoot=file://" + (rootDir / "fixtures" / "bugs").string() }), PQ::Mock("user=postgres dbname=postgres", "GentooBrowseAPI", { rootDir.parent_path() / "db" / "schema.sql", rootDir / "basedata.sql" }) diff --git a/gentoobrowse-api/unittests/mockDefs.h b/gentoobrowse-api/unittests/mockDefs.h index 3b3b013..792f875 100644 --- a/gentoobrowse-api/unittests/mockDefs.h +++ b/gentoobrowse-api/unittests/mockDefs.h @@ -6,6 +6,7 @@ #include #include #include +#include class DLL_PUBLIC Service : public IceTray::DryIce, PQ::Mock { public: @@ -25,5 +26,8 @@ class DLL_PUBLIC TestClient : public IceTray::DryIceClient { Gentoo::PortagePrx p; }; +#define SQL_REQUIRE_EQUAL(sql, type, expected) \ + db->select(sql)->forEachRow([&expected](const auto & n) { BOOST_REQUIRE_EQUAL(expected, n); }); + #endif diff --git a/gentoobrowse-api/unittests/testBugs.cpp b/gentoobrowse-api/unittests/testBugs.cpp new file mode 100644 index 0000000..2121d5d --- /dev/null +++ b/gentoobrowse-api/unittests/testBugs.cpp @@ -0,0 +1,20 @@ +#define BOOST_TEST_MODULE TestBugs +#include + +#include "mockDefs.h" + +BOOST_GLOBAL_FIXTURE( Maintenance ); + +BOOST_FIXTURE_TEST_SUITE(tp, TestClient) + +BOOST_AUTO_TEST_CASE( importBugHtml ) +{ + const int64_t expectedBugCount = 34; + m->refreshBugs(); + + auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("GentooBrowseAPI")); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.bugs", int64_t, expectedBugCount); +} + +BOOST_AUTO_TEST_SUITE_END(); + diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp index be045a5..c39b9af 100644 --- a/gentoobrowse-api/unittests/testMaintenance.cpp +++ b/gentoobrowse-api/unittests/testMaintenance.cpp @@ -1,6 +1,5 @@ #define BOOST_TEST_MODULE TestMaintenance #include -#include #include "mockDefs.h" #include @@ -37,9 +36,6 @@ class SampleData { } }; -#define SQL_REQUIRE_EQUAL(sql, type, expected) \ - db->select(sql)->forEachRow([&expected](const auto & n) { BOOST_REQUIRE_EQUAL(expected, n); }); - BOOST_FIXTURE_TEST_SUITE(tp, TestClient) void -- cgit v1.2.3