summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--gentoobrowse-api/api/maintenance.ice1
-rw-r--r--gentoobrowse-api/service/maintenanceBugs.cpp98
-rw-r--r--gentoobrowse-api/service/maintenanceCommon.cpp14
-rw-r--r--gentoobrowse-api/service/maintenanceimpl.h4
-rw-r--r--gentoobrowse-api/unittests/Jamfile.jam8
-rw-r--r--gentoobrowse-api/unittests/fixtures/bugs/buglist-CONFIRMED.html13
-rw-r--r--gentoobrowse-api/unittests/fixtures/bugs/buglist-IN_PROGRESS.html16
-rw-r--r--gentoobrowse-api/unittests/fixtures/bugs/buglist-UNCONFIRMED.html17
-rw-r--r--gentoobrowse-api/unittests/mockDefs.cpp1
-rw-r--r--gentoobrowse-api/unittests/mockDefs.h4
-rw-r--r--gentoobrowse-api/unittests/testBugs.cpp20
-rw-r--r--gentoobrowse-api/unittests/testMaintenance.cpp4
12 files changed, 196 insertions, 4 deletions
diff --git a/gentoobrowse-api/api/maintenance.ice b/gentoobrowse-api/api/maintenance.ice
index 55389b2..0e80fed 100644
--- a/gentoobrowse-api/api/maintenance.ice
+++ b/gentoobrowse-api/api/maintenance.ice
@@ -1,6 +1,7 @@
module Gentoo {
interface Maintenance {
idempotent void refreshPackageTree();
+ idempotent void refreshBugs();
};
};
diff --git a/gentoobrowse-api/service/maintenanceBugs.cpp b/gentoobrowse-api/service/maintenanceBugs.cpp
new file mode 100644
index 0000000..0917f97
--- /dev/null
+++ b/gentoobrowse-api/service/maintenanceBugs.cpp
@@ -0,0 +1,98 @@
+#include "maintenanceimpl.h"
+#include <boost/lexical_cast.hpp>
+#include <selectcommandUtil.impl.h>
+#include <tablepatch.h>
+#include <sqlWriter.h>
+#include <buffer.h>
+#include <scopeExit.h>
+#include <boost/filesystem/operations.hpp>
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#include <curlStream.h>
+#include <libxml++/parsers/saxparser.h>
+#pragma GCC diagnostic pop
+#include <utils/lexer.h>
+#include <utils/dbUtils.h>
+
+namespace Gentoo {
+ namespace Service {
+ Utils::Lexer::PatternPtr bugLink = Utils::Lexer::regex(
+ "Bug:(\\d+) - \"\" status:(\\w*) resolution:(\\w*) severity:(\\w*).*", G_REGEX_OPTIMIZE);
+
+ class BugListParser : public xmlpp::SaxParser, Utils::Lexer {
+ public:
+ BugListParser(DB::ModifyCommandPtr i) :
+ ins(i)
+ {
+ rules.push_back({ { InitialState }, bugLink, [this](auto es) {
+ ins->bindParamI(0, boost::lexical_cast<int64_t>(*es->pattern->match(1)));
+ ins->bindParamS(2, *es->pattern->match(1));
+ ins->bindParamS(1, *es->pattern->match(3));
+ }});
+ }
+
+ protected:
+ void on_start_element(const Glib::ustring & e, const xmlpp::SaxParser::AttributeList &) override
+ {
+ stk.push(e);
+ }
+
+ void on_characters(const Glib::ustring & t) override
+ {
+ if (stk.top() == "a") {
+ attributes += t;
+ }
+ else if (stk.top() == "em") {
+ summary += t;
+ }
+ }
+
+ void on_end_element(const Glib::ustring & e) override
+ {
+ stk.pop();
+ if (e == "li") {
+ fprintf(stderr, "attributes = '%s', summary = '%s'\n", attributes.c_str(), summary.c_str());
+ extract(attributes.c_str(), attributes.length());
+ ins->bindParamS(3, summary);
+ ins->execute();
+ attributes.clear();
+ summary.clear();
+ }
+ }
+
+ private:
+ DB::ModifyCommandPtr ins;
+ std::stack<Glib::ustring> stk;
+
+ Glib::ustring attributes;
+ Glib::ustring summary;
+ };
+
+ void
+ Maintenance::refreshBugs(const Ice::Current & c)
+ {
+ boost::filesystem::path root = properties(c)->getPropertyWithDefault(
+ "GentooBrowseAPI.BugRoot", "https://bugs.gentoo.org/data/cached");
+
+ auto dbc = db->get();
+ DB::TransactionScope tx(dbc.get());
+ DB::TablePatch tp;
+ tp.pk = { "bugId" };
+ tp.cols = { "bugId", "severity", "status", "summary" };
+ tp.dest = "gentoobrowse.bugs";
+ tp.src = Utils::Database::emptyClone(dbc.get(), "gentoobrowse.bugs");
+ auto ins = Utils::Database::tablePatchInserter(dbc.get(), tp);
+ BugListParser blp(ins);
+ for(const auto & bl : {
+ "buglist-CONFIRMED.html",
+ "buglist-UNCONFIRMED.html",
+ "buglist-IN_PROGRESS.html" }) {
+ AdHoc::Net::CurlStreamSource css((root / bl).string());
+ css.setopt(CURLOPT_ENCODING, "deflate, gzip");
+ AdHoc::Net::CurlStream cs(css);
+ blp.parse_stream(cs);
+ }
+ dbc->patchTable(&tp);
+ }
+ }
+}
+
diff --git a/gentoobrowse-api/service/maintenanceCommon.cpp b/gentoobrowse-api/service/maintenanceCommon.cpp
new file mode 100644
index 0000000..c05132b
--- /dev/null
+++ b/gentoobrowse-api/service/maintenanceCommon.cpp
@@ -0,0 +1,14 @@
+#include "maintenanceimpl.h"
+#include <Ice/ObjectAdapter.h>
+#include <Ice/Communicator.h>
+
+namespace Gentoo {
+ namespace Service {
+ Ice::PropertiesPtr
+ Maintenance::properties(const Ice::Current & c)
+ {
+ return c.adapter->getCommunicator()->getProperties();
+ }
+ }
+}
+
diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h
index 6331672..7da0d55 100644
--- a/gentoobrowse-api/service/maintenanceimpl.h
+++ b/gentoobrowse-api/service/maintenanceimpl.h
@@ -1,6 +1,7 @@
#ifndef MAINTENANCEIMPL_H
#define MAINTENANCEIMPL_H
+#include <Ice/Properties.h>
#include <maintenance.h>
#include <visibility.h>
#include <abstractDatabaseClient.h>
@@ -33,6 +34,7 @@ namespace Gentoo {
~Maintenance();
void refreshPackageTree(const Ice::Current &) override;
+ void refreshBugs(const Ice::Current &) override;
private:
static void createTempFileList(DB::Connection *, const boost::filesystem::path &);
@@ -42,6 +44,8 @@ namespace Gentoo {
void fileCreated(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path & tmp, DB::SelectCommandPtr s);
void fileHandle(int64_t filetypeId, const FileProcessors *, const FileHandleFunc &) const;
+ static Ice::PropertiesPtr properties(const Ice::Current &);
+
template <typename T>
static FileProcessorPtr createFileProessor();
FileProcessorFactories fpfs;
diff --git a/gentoobrowse-api/unittests/Jamfile.jam b/gentoobrowse-api/unittests/Jamfile.jam
index 71aa67c..4f38f9d 100644
--- a/gentoobrowse-api/unittests/Jamfile.jam
+++ b/gentoobrowse-api/unittests/Jamfile.jam
@@ -65,6 +65,14 @@ run
<library>testCommon
: testPortage ;
+run
+ testBugs.cpp
+ : : :
+ <dependency>../db/schema.sql
+ <define>BOOST_TEST_DYN_LINK
+ <library>testCommon
+ : testBugs ;
+
explicit testPerf ;
run testPerf.cpp
: : :
diff --git a/gentoobrowse-api/unittests/fixtures/bugs/buglist-CONFIRMED.html b/gentoobrowse-api/unittests/fixtures/bugs/buglist-CONFIRMED.html
new file mode 100644
index 0000000..a1c20ba
--- /dev/null
+++ b/gentoobrowse-api/unittests/fixtures/bugs/buglist-CONFIRMED.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"><html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+<head><title>Bug listing with status CONFIRMED as at 2016/05/27 18:48:05</title></head>
+<body><h1>Bug listing with status CONFIRMED as at 2016/05/27 18:48:05</h1>
+<div><ul><li><a href='https://bugs.gentoo.org/2905'>Bug:2905 - "<em>[IDEA] Support searching for files installed by all available packages, even those that aren't currently installed.</em>" status:CONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/10735'>Bug:10735 - "<em>sys-apps/portage or app-portage/gentoolkit - Feature request: Show dependencies in detail; clarifying whether they are DEPEND, RDEPEND or PDEPEND as well as whether they are optional.</em>" status:CONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/17040'>Bug:17040 - "<em>tmispell-0.5.0.ebuild (new ebuild)</em>" status:CONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/18587'>Bug:18587 - "<em>libcwd-0.99.28.ebuild (New Package)</em>" status:CONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/19109'>Bug:19109 - "<em>Some PERL module ebuilds named in strange format</em>" status:CONFIRMED resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/21509'>Bug:21509 - "<em>[PATCH] [IDEA] Ability to cleanly stop emerging multiple packages after the current jobs complete.</em>" status:CONFIRMED resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/21527'>Bug:21527 - "<em>net-misc/ntp: ntp-client init.d: default to `sntp` instead of `ntpdate`</em>" status:CONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/21862'>Bug:21862 - "<em>Retire: Ned Ludd (solar)</em>" status:CONFIRMED resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/22715'>Bug:22715 - "<em>Packages reassignment: Donnie Berkholz (spyderous) / dberkholz</em>" status:CONFIRMED resolution: severity:normal</a></li>
+</ul>Done. Count=12520</div></body></html>
diff --git a/gentoobrowse-api/unittests/fixtures/bugs/buglist-IN_PROGRESS.html b/gentoobrowse-api/unittests/fixtures/bugs/buglist-IN_PROGRESS.html
new file mode 100644
index 0000000..efa195a
--- /dev/null
+++ b/gentoobrowse-api/unittests/fixtures/bugs/buglist-IN_PROGRESS.html
@@ -0,0 +1,16 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"><html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+<head><title>Bug listing with status IN_PROGRESS as at 2016/05/27 18:48:07</title></head>
+<body><h1>Bug listing with status IN_PROGRESS as at 2016/05/27 18:48:07</h1>
+<div><ul><li><a href='https://bugs.gentoo.org/473'>Bug:473 - "<em>Gentoo Tinderbox</em>" status:IN_PROGRESS resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/23851'>Bug:23851 - "<em>[Tracker] Portage should validate ability to install before installing</em>" status:IN_PROGRESS resolution: severity:major</a></li>
+<li><a href='https://bugs.gentoo.org/33740'>Bug:33740 - "<em>Reply address and fax-phone number missing</em>" status:IN_PROGRESS resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/45285'>Bug:45285 - "<em>Livecd: add chrp support</em>" status:IN_PROGRESS resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/52500'>Bug:52500 - "<em>GLSA's reference incorrect ebuilds after renames</em>" status:IN_PROGRESS resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/68339'>Bug:68339 - "<em>bad keyboard selection mojo on livecd</em>" status:IN_PROGRESS resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/83769'>Bug:83769 - "<em>xemacs hangs with UTF-8 locale</em>" status:IN_PROGRESS resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/87242'>Bug:87242 - "<em>linux-info.eclass: unable to find kernel version - SUBLEVEL and EXTRAVERSION not present when using KBUILD_OUTPUT</em>" status:IN_PROGRESS resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/88596'>Bug:88596 - "<em>sys-devel/libtool stores gcc-path internally. sometimes a bad thing due to gcc update</em>" status:IN_PROGRESS resolution: severity:major</a></li>
+<li><a href='https://bugs.gentoo.org/92982'>Bug:92982 - "<em>app-misc/livecd-tools-1.0.20: Incorporate rebuildfstab into autoconfig</em>" status:IN_PROGRESS resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/99446'>Bug:99446 - "<em>sys-power/acpi-support-0.50 (New Package)</em>" status:IN_PROGRESS resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/106815'>Bug:106815 - "<em>ufed enhancements to filter list of flags</em>" status:IN_PROGRESS resolution: severity:enhancement</a></li>
+</ul>Done. Count=1397</div></body></html>
diff --git a/gentoobrowse-api/unittests/fixtures/bugs/buglist-UNCONFIRMED.html b/gentoobrowse-api/unittests/fixtures/bugs/buglist-UNCONFIRMED.html
new file mode 100644
index 0000000..b174bea
--- /dev/null
+++ b/gentoobrowse-api/unittests/fixtures/bugs/buglist-UNCONFIRMED.html
@@ -0,0 +1,17 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"><html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+<head><title>Bug listing with status UNCONFIRMED as at 2016/05/27 18:48:04</title></head>
+<body><h1>Bug listing with status UNCONFIRMED as at 2016/05/27 18:48:04</h1>
+<div><ul><li><a href='https://bugs.gentoo.org/19910'>Bug:19910 - "<em>app-office/moneydance - easy to use personal finance software that is loaded with all the features you need: online banking and bill payment, account management, budgeting and investment tracking</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/52076'>Bug:52076 - "<em>Write a proper implementation using a config variable and write a GLEP to change policy to allow the user to change the default maildir location.</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/81424'>Bug:81424 - "<em>gnome-translate-0.99.ebuild (New Package)</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/105191'>Bug:105191 - "<em>media-?/fig2ps (new package)</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/109031'>Bug:109031 - "<em>ccscript3-0.8.1.ebuild (New Package)</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/109374'>Bug:109374 - "<em>net-analyzer/netmonitor: console application that monitors network intefaces</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/128538'>Bug:128538 - "<em>/bin/hostname should be installed from coreutils not net-tools</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/142049'>Bug:142049 - "<em>ebuild req.: Aptana</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/146951'>Bug:146951 - "<em>sys-fs/quota - value too large for defined data type on Alpha systems</em>" status:UNCONFIRMED resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/161003'>Bug:161003 - "<em>package.provided needs to catch virtual/ usage</em>" status:UNCONFIRMED resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/171500'>Bug:171500 - "<em>arcload on mips octane r12k doesn't load</em>" status:UNCONFIRMED resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/176236'>Bug:176236 - "<em>media-video/xanim: doesn't include VCR1 codec from upstream FTP</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/255644'>Bug:255644 - "<em>app-emulation/basiliskII-jit: a lot improved ebuild</em>" status:UNCONFIRMED resolution: severity:trivial</a></li>
+</ul>Done. Count=6753</div></body></html>
diff --git a/gentoobrowse-api/unittests/mockDefs.cpp b/gentoobrowse-api/unittests/mockDefs.cpp
index 3221909..2d89fab 100644
--- a/gentoobrowse-api/unittests/mockDefs.cpp
+++ b/gentoobrowse-api/unittests/mockDefs.cpp
@@ -9,6 +9,7 @@ Service::Service() :
}
Maintenance::Maintenance() :
+ IceTray::DryIce({ "--GentooBrowseAPI.BugRoot=file://" + (rootDir / "fixtures" / "bugs").string() }),
PQ::Mock("user=postgres dbname=postgres", "GentooBrowseAPI", {
rootDir.parent_path() / "db" / "schema.sql",
rootDir / "basedata.sql" })
diff --git a/gentoobrowse-api/unittests/mockDefs.h b/gentoobrowse-api/unittests/mockDefs.h
index 3b3b013..792f875 100644
--- a/gentoobrowse-api/unittests/mockDefs.h
+++ b/gentoobrowse-api/unittests/mockDefs.h
@@ -6,6 +6,7 @@
#include <dryice.h>
#include <portage.h>
#include <maintenance.h>
+#include <selectcommandUtil.impl.h>
class DLL_PUBLIC Service : public IceTray::DryIce, PQ::Mock {
public:
@@ -25,5 +26,8 @@ class DLL_PUBLIC TestClient : public IceTray::DryIceClient {
Gentoo::PortagePrx p;
};
+#define SQL_REQUIRE_EQUAL(sql, type, expected) \
+ db->select(sql)->forEachRow<type>([&expected](const auto & n) { BOOST_REQUIRE_EQUAL(expected, n); });
+
#endif
diff --git a/gentoobrowse-api/unittests/testBugs.cpp b/gentoobrowse-api/unittests/testBugs.cpp
new file mode 100644
index 0000000..2121d5d
--- /dev/null
+++ b/gentoobrowse-api/unittests/testBugs.cpp
@@ -0,0 +1,20 @@
+#define BOOST_TEST_MODULE TestBugs
+#include <boost/test/unit_test.hpp>
+
+#include "mockDefs.h"
+
+BOOST_GLOBAL_FIXTURE( Maintenance );
+
+BOOST_FIXTURE_TEST_SUITE(tp, TestClient)
+
+BOOST_AUTO_TEST_CASE( importBugHtml )
+{
+ const int64_t expectedBugCount = 34;
+ m->refreshBugs();
+
+ auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("GentooBrowseAPI"));
+ SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.bugs", int64_t, expectedBugCount);
+}
+
+BOOST_AUTO_TEST_SUITE_END();
+
diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp
index be045a5..c39b9af 100644
--- a/gentoobrowse-api/unittests/testMaintenance.cpp
+++ b/gentoobrowse-api/unittests/testMaintenance.cpp
@@ -1,6 +1,5 @@
#define BOOST_TEST_MODULE TestMaintenance
#include <boost/test/unit_test.hpp>
-#include <selectcommandUtil.impl.h>
#include "mockDefs.h"
#include <definedDirs.h>
@@ -37,9 +36,6 @@ class SampleData {
}
};
-#define SQL_REQUIRE_EQUAL(sql, type, expected) \
- db->select(sql)->forEachRow<type>([&expected](const auto & n) { BOOST_REQUIRE_EQUAL(expected, n); });
-
BOOST_FIXTURE_TEST_SUITE(tp, TestClient)
void
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 17:08:37 +0000