Import bug list htmls

12 files changed, 196 insertions, 4 deletions

diff --git a/gentoobrowse-api/api/maintenance.ice b/gentoobrowse-api/api/maintenance.ice
index 55389b2..0e80fed 100644
--- a/gentoobrowse-api/api/maintenance.ice
+++ b/gentoobrowse-api/api/maintenance.ice
@@ -1,6 +1,7 @@
 module Gentoo {
 	interface Maintenance {
 		idempotent void refreshPackageTree();
+		idempotent void refreshBugs();
 	};
 };
 
diff --git a/gentoobrowse-api/service/maintenanceBugs.cpp b/gentoobrowse-api/service/maintenanceBugs.cpp
new file mode 100644
index 0000000..0917f97
--- /dev/null
+++ b/gentoobrowse-api/service/maintenanceBugs.cpp
@@ -0,0 +1,98 @@
+#include "maintenanceimpl.h"
+#include <boost/lexical_cast.hpp>
+#include <selectcommandUtil.impl.h>
+#include <tablepatch.h>
+#include <sqlWriter.h>
+#include <buffer.h>
+#include <scopeExit.h>
+#include <boost/filesystem/operations.hpp>
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#include <curlStream.h>
+#include <libxml++/parsers/saxparser.h>
+#pragma GCC diagnostic pop
+#include <utils/lexer.h>
+#include <utils/dbUtils.h>
+
+namespace Gentoo {
+	namespace Service {
+		Utils::Lexer::PatternPtr bugLink = Utils::Lexer::regex(
+				"Bug:(\\d+) - \"\" status:(\\w*) resolution:(\\w*) severity:(\\w*).*", G_REGEX_OPTIMIZE);
+
+		class BugListParser : public xmlpp::SaxParser, Utils::Lexer {
+			public:
+				BugListParser(DB::ModifyCommandPtr i) :
+					ins(i)
+				{
+					rules.push_back({ { InitialState }, bugLink, [this](auto es) {
+							ins->bindParamI(0, boost::lexical_cast<int64_t>(*es->pattern->match(1)));
+							ins->bindParamS(2, *es->pattern->match(1));
+							ins->bindParamS(1, *es->pattern->match(3));
+						}});
+				}
+
+			protected:
+				void on_start_element(const Glib::ustring & e, const xmlpp::SaxParser::AttributeList &) override
+				{
+					stk.push(e);
+				}
+
+				void on_characters(const Glib::ustring & t) override
+				{
+					if (stk.top() == "a") {
+						attributes += t;
+					}
+					else if (stk.top() == "em") {
+						summary += t;
+					}
+				}
+
+				void on_end_element(const Glib::ustring & e) override
+				{
+					stk.pop();
+					if (e == "li") {
+						fprintf(stderr, "attributes = '%s', summary = '%s'\n", attributes.c_str(), summary.c_str());
+						extract(attributes.c_str(), attributes.length());
+						ins->bindParamS(3, summary);
+						ins->execute();
+						attributes.clear();
+						summary.clear();
+					}
+				}
+
+			private:
+				DB::ModifyCommandPtr ins;
+				std::stack<Glib::ustring> stk;
+
+				Glib::ustring attributes;
+				Glib::ustring summary;
+		};
+
+		void
+		Maintenance::refreshBugs(const Ice::Current & c)
+		{
+			boost::filesystem::path root = properties(c)->getPropertyWithDefault(
+					"GentooBrowseAPI.BugRoot", "https://bugs.gentoo.org/data/cached");
+
+			auto dbc = db->get();
+			DB::TransactionScope tx(dbc.get());
+			DB::TablePatch tp;
+			tp.pk = { "bugId" };
+			tp.cols = { "bugId", "severity", "status", "summary" };
+			tp.dest = "gentoobrowse.bugs";
+			tp.src = Utils::Database::emptyClone(dbc.get(), "gentoobrowse.bugs");
+			auto ins = Utils::Database::tablePatchInserter(dbc.get(), tp);
+			BugListParser blp(ins);
+			for(const auto & bl : {
+					"buglist-CONFIRMED.html",
+					"buglist-UNCONFIRMED.html",
+					"buglist-IN_PROGRESS.html" }) {
+				AdHoc::Net::CurlStreamSource css((root / bl).string());
+				css.setopt(CURLOPT_ENCODING, "deflate, gzip");
+				AdHoc::Net::CurlStream cs(css);
+				blp.parse_stream(cs);
+			}
+			dbc->patchTable(&tp);
+		}
+	}
+}
+
diff --git a/gentoobrowse-api/service/maintenanceCommon.cpp b/gentoobrowse-api/service/maintenanceCommon.cpp
new file mode 100644
index 0000000..c05132b
--- /dev/null
+++ b/gentoobrowse-api/service/maintenanceCommon.cpp
@@ -0,0 +1,14 @@
+#include "maintenanceimpl.h"
+#include <Ice/ObjectAdapter.h>
+#include <Ice/Communicator.h>
+
+namespace Gentoo {
+	namespace Service {
+		Ice::PropertiesPtr
+		Maintenance::properties(const Ice::Current & c)
+		{
+			return c.adapter->getCommunicator()->getProperties();
+		}
+	}
+}
+
diff --git a/gentoobrowse-api/service/maintenanceimpl.h b/gentoobrowse-api/service/maintenanceimpl.h
index 6331672..7da0d55 100644
--- a/gentoobrowse-api/service/maintenanceimpl.h
+++ b/gentoobrowse-api/service/maintenanceimpl.h
@@ -1,6 +1,7 @@
 #ifndef MAINTENANCEIMPL_H
 #define MAINTENANCEIMPL_H
 
+#include <Ice/Properties.h>
 #include <maintenance.h>
 #include <visibility.h>
 #include <abstractDatabaseClient.h>
@@ -33,6 +34,7 @@ namespace Gentoo {
 				~Maintenance();
 
 				void refreshPackageTree(const Ice::Current &) override;
+				void refreshBugs(const Ice::Current &) override;
 
 			private:
 				static void createTempFileList(DB::Connection *, const boost::filesystem::path &);
@@ -42,6 +44,8 @@ namespace Gentoo {
 				void fileCreated(DB::Connection * dbc, const FileProcessors *, const boost::filesystem::path & tmp, DB::SelectCommandPtr s);
 				void fileHandle(int64_t filetypeId, const FileProcessors *, const FileHandleFunc &) const;
 
+				static Ice::PropertiesPtr properties(const Ice::Current &);
+
 				template <typename T>
 				static FileProcessorPtr createFileProessor();
 				FileProcessorFactories fpfs;
diff --git a/gentoobrowse-api/unittests/Jamfile.jam b/gentoobrowse-api/unittests/Jamfile.jam
index 71aa67c..4f38f9d 100644
--- a/gentoobrowse-api/unittests/Jamfile.jam
+++ b/gentoobrowse-api/unittests/Jamfile.jam
@@ -65,6 +65,14 @@ run
 	<library>testCommon
 	: testPortage ;
 
+run
+	testBugs.cpp
+	: : :
+	<dependency>../db/schema.sql
+	<define>BOOST_TEST_DYN_LINK
+	<library>testCommon
+	: testBugs ;
+
 explicit testPerf ;
 run testPerf.cpp
 	: : :
diff --git a/gentoobrowse-api/unittests/fixtures/bugs/buglist-CONFIRMED.html b/gentoobrowse-api/unittests/fixtures/bugs/buglist-CONFIRMED.html
new file mode 100644
index 0000000..a1c20ba
--- /dev/null
+++ b/gentoobrowse-api/unittests/fixtures/bugs/buglist-CONFIRMED.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"><html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+<head><title>Bug listing with status CONFIRMED   as at 2016/05/27 18:48:05</title></head>
+<body><h1>Bug listing with status CONFIRMED   as at 2016/05/27 18:48:05</h1>
+<div><ul><li><a href='https://bugs.gentoo.org/2905'>Bug:2905 - "<em>[IDEA] Support searching for files installed by all available packages, even those that aren't currently installed.</em>" status:CONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/10735'>Bug:10735 - "<em>sys-apps/portage or app-portage/gentoolkit - Feature request: Show dependencies in detail; clarifying whether they are DEPEND, RDEPEND or PDEPEND as well as whether they are optional.</em>" status:CONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/17040'>Bug:17040 - "<em>tmispell-0.5.0.ebuild (new ebuild)</em>" status:CONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/18587'>Bug:18587 - "<em>libcwd-0.99.28.ebuild (New Package)</em>" status:CONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/19109'>Bug:19109 - "<em>Some PERL module ebuilds named in strange format</em>" status:CONFIRMED resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/21509'>Bug:21509 - "<em>[PATCH] [IDEA] Ability to cleanly stop emerging multiple packages after the current jobs complete.</em>" status:CONFIRMED resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/21527'>Bug:21527 - "<em>net-misc/ntp: ntp-client init.d: default to `sntp` instead of `ntpdate`</em>" status:CONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/21862'>Bug:21862 - "<em>Retire: Ned Ludd (solar)</em>" status:CONFIRMED resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/22715'>Bug:22715 - "<em>Packages reassignment: Donnie Berkholz (spyderous) / dberkholz</em>" status:CONFIRMED resolution: severity:normal</a></li>
+</ul>Done. Count=12520</div></body></html>
diff --git a/gentoobrowse-api/unittests/fixtures/bugs/buglist-IN_PROGRESS.html b/gentoobrowse-api/unittests/fixtures/bugs/buglist-IN_PROGRESS.html
new file mode 100644
index 0000000..efa195a
--- /dev/null
+++ b/gentoobrowse-api/unittests/fixtures/bugs/buglist-IN_PROGRESS.html
@@ -0,0 +1,16 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"><html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+<head><title>Bug listing with status IN_PROGRESS   as at 2016/05/27 18:48:07</title></head>
+<body><h1>Bug listing with status IN_PROGRESS   as at 2016/05/27 18:48:07</h1>
+<div><ul><li><a href='https://bugs.gentoo.org/473'>Bug:473 - "<em>Gentoo Tinderbox</em>" status:IN_PROGRESS resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/23851'>Bug:23851 - "<em>[Tracker] Portage should validate ability to install before installing</em>" status:IN_PROGRESS resolution: severity:major</a></li>
+<li><a href='https://bugs.gentoo.org/33740'>Bug:33740 - "<em>Reply address and fax-phone number missing</em>" status:IN_PROGRESS resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/45285'>Bug:45285 - "<em>Livecd: add chrp support</em>" status:IN_PROGRESS resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/52500'>Bug:52500 - "<em>GLSA's reference incorrect ebuilds after renames</em>" status:IN_PROGRESS resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/68339'>Bug:68339 - "<em>bad keyboard selection mojo on livecd</em>" status:IN_PROGRESS resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/83769'>Bug:83769 - "<em>xemacs  hangs with UTF-8 locale</em>" status:IN_PROGRESS resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/87242'>Bug:87242 - "<em>linux-info.eclass: unable to find kernel version - SUBLEVEL and EXTRAVERSION not present when using KBUILD_OUTPUT</em>" status:IN_PROGRESS resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/88596'>Bug:88596 - "<em>sys-devel/libtool stores gcc-path internally.  sometimes a bad thing due to gcc update</em>" status:IN_PROGRESS resolution: severity:major</a></li>
+<li><a href='https://bugs.gentoo.org/92982'>Bug:92982 - "<em>app-misc/livecd-tools-1.0.20: Incorporate rebuildfstab into autoconfig</em>" status:IN_PROGRESS resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/99446'>Bug:99446 - "<em>sys-power/acpi-support-0.50 (New Package)</em>" status:IN_PROGRESS resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/106815'>Bug:106815 - "<em>ufed enhancements to filter list of flags</em>" status:IN_PROGRESS resolution: severity:enhancement</a></li>
+</ul>Done. Count=1397</div></body></html>
diff --git a/gentoobrowse-api/unittests/fixtures/bugs/buglist-UNCONFIRMED.html b/gentoobrowse-api/unittests/fixtures/bugs/buglist-UNCONFIRMED.html
new file mode 100644
index 0000000..b174bea
--- /dev/null
+++ b/gentoobrowse-api/unittests/fixtures/bugs/buglist-UNCONFIRMED.html
@@ -0,0 +1,17 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"><html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+<head><title>Bug listing with status UNCONFIRMED   as at 2016/05/27 18:48:04</title></head>
+<body><h1>Bug listing with status UNCONFIRMED   as at 2016/05/27 18:48:04</h1>
+<div><ul><li><a href='https://bugs.gentoo.org/19910'>Bug:19910 - "<em>app-office/moneydance - easy to use personal finance software that is loaded with all the features you need: online banking and bill payment, account management, budgeting and investment tracking</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/52076'>Bug:52076 - "<em>Write a proper implementation using a config variable and write a GLEP to change policy to allow the user to change the default maildir location.</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/81424'>Bug:81424 - "<em>gnome-translate-0.99.ebuild (New Package)</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/105191'>Bug:105191 - "<em>media-?/fig2ps (new package)</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/109031'>Bug:109031 - "<em>ccscript3-0.8.1.ebuild (New Package)</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/109374'>Bug:109374 - "<em>net-analyzer/netmonitor: console application that monitors network intefaces</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/128538'>Bug:128538 - "<em>/bin/hostname should be installed from coreutils not net-tools</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/142049'>Bug:142049 - "<em>ebuild req.: Aptana</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/146951'>Bug:146951 - "<em>sys-fs/quota - value too large for defined data type on Alpha systems</em>" status:UNCONFIRMED resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/161003'>Bug:161003 - "<em>package.provided needs to catch virtual/ usage</em>" status:UNCONFIRMED resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/171500'>Bug:171500 - "<em>arcload on mips octane r12k doesn't load</em>" status:UNCONFIRMED resolution: severity:normal</a></li>
+<li><a href='https://bugs.gentoo.org/176236'>Bug:176236 - "<em>media-video/xanim: doesn't include VCR1 codec from upstream FTP</em>" status:UNCONFIRMED resolution: severity:enhancement</a></li>
+<li><a href='https://bugs.gentoo.org/255644'>Bug:255644 - "<em>app-emulation/basiliskII-jit: a lot improved ebuild</em>" status:UNCONFIRMED resolution: severity:trivial</a></li>
+</ul>Done. Count=6753</div></body></html>
diff --git a/gentoobrowse-api/unittests/mockDefs.cpp b/gentoobrowse-api/unittests/mockDefs.cpp
index 3221909..2d89fab 100644
--- a/gentoobrowse-api/unittests/mockDefs.cpp
+++ b/gentoobrowse-api/unittests/mockDefs.cpp
@@ -9,6 +9,7 @@ Service::Service() :
 }
 
 Maintenance::Maintenance() :
+	IceTray::DryIce({ "--GentooBrowseAPI.BugRoot=file://" + (rootDir / "fixtures" / "bugs").string() }),
 	PQ::Mock("user=postgres dbname=postgres", "GentooBrowseAPI", {
 			rootDir.parent_path() / "db" / "schema.sql",
 			rootDir / "basedata.sql" })
diff --git a/gentoobrowse-api/unittests/mockDefs.h b/gentoobrowse-api/unittests/mockDefs.h
index 3b3b013..792f875 100644
--- a/gentoobrowse-api/unittests/mockDefs.h
+++ b/gentoobrowse-api/unittests/mockDefs.h
@@ -6,6 +6,7 @@
 #include <dryice.h>
 #include <portage.h>
 #include <maintenance.h>
+#include <selectcommandUtil.impl.h>
 
 class DLL_PUBLIC Service : public IceTray::DryIce, PQ::Mock {
 	public:
@@ -25,5 +26,8 @@ class DLL_PUBLIC TestClient : public IceTray::DryIceClient {
 		Gentoo::PortagePrx p;
 };
 
+#define SQL_REQUIRE_EQUAL(sql, type, expected) \
+	db->select(sql)->forEachRow<type>([&expected](const auto & n) { BOOST_REQUIRE_EQUAL(expected, n); });
+
 #endif
 
diff --git a/gentoobrowse-api/unittests/testBugs.cpp b/gentoobrowse-api/unittests/testBugs.cpp
new file mode 100644
index 0000000..2121d5d
--- /dev/null
+++ b/gentoobrowse-api/unittests/testBugs.cpp
@@ -0,0 +1,20 @@
+#define BOOST_TEST_MODULE TestBugs
+#include <boost/test/unit_test.hpp>
+
+#include "mockDefs.h"
+
+BOOST_GLOBAL_FIXTURE( Maintenance );
+
+BOOST_FIXTURE_TEST_SUITE(tp, TestClient)
+
+BOOST_AUTO_TEST_CASE( importBugHtml )
+{
+	const int64_t expectedBugCount = 34;
+	m->refreshBugs();
+
+	auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("GentooBrowseAPI"));
+	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.bugs", int64_t, expectedBugCount);
+}
+
+BOOST_AUTO_TEST_SUITE_END();
+
diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp
index be045a5..c39b9af 100644
--- a/gentoobrowse-api/unittests/testMaintenance.cpp
+++ b/gentoobrowse-api/unittests/testMaintenance.cpp
@@ -1,6 +1,5 @@
 #define BOOST_TEST_MODULE TestMaintenance
 #include <boost/test/unit_test.hpp>
-#include <selectcommandUtil.impl.h>
 
 #include "mockDefs.h"
 #include <definedDirs.h>
@@ -37,9 +36,6 @@ class SampleData {
 		}
 };
 
-#define SQL_REQUIRE_EQUAL(sql, type, expected) \
-	db->select(sql)->forEachRow<type>([&expected](const auto & n) { BOOST_REQUIRE_EQUAL(expected, n); });
-
 BOOST_FIXTURE_TEST_SUITE(tp, TestClient)
 
 void