Import package tree news

9 files changed, 259 insertions, 5 deletions

diff --git a/gentoobrowse-api/service/maintenance/newsProcessor.cpp b/gentoobrowse-api/service/maintenance/newsProcessor.cpp
new file mode 100644
index 0000000..0b5b920
--- /dev/null
+++ b/gentoobrowse-api/service/maintenance/newsProcessor.cpp
@@ -0,0 +1,50 @@
+#include "newsProcessor.h"
+#include "utils/fileUtils.h"
+#include "utils/xmlUtils.h"
+#include "utils/dbUtils.h"
+#include "news.h"
+#include <modifycommand.h>
+#include <slicer/slicer.h>
+#include <db/sqlInsertSerializer.h>
+#include <db/sqlUpdateSerializer.h>
+#include <sql/maintenance/newsDelete.sql.h>
+
+namespace U = Gentoo::Utils;
+using namespace Gentoo::Utils::File;
+
+namespace Gentoo {
+	namespace Service {
+		const int NewsProcessor::FILETYPEID = 11;
+
+		void
+		NewsProcessor::created(DB::Connection * dbc, const boost::filesystem::path &, const boost::filesystem::path & path)
+		{
+			importNews<Slicer::SqlInsertSerializer>(dbc, path);
+		}
+
+		void
+		NewsProcessor::modified(DB::Connection * dbc, const boost::filesystem::path &, const boost::filesystem::path & path)
+		{
+			importNews<Slicer::SqlUpdateSerializer>(dbc, path);
+		}
+
+		template <typename Serializer>
+		void
+		NewsProcessor::importNews(DB::Connection * dbc, const boost::filesystem::path & path)
+		{
+			Gentoo::Utils::MemMap m(path);
+			auto news = Portage::Utils::News::parse(reinterpret_cast<const gchar *>(m.data), m.getStat().st_size);
+			news->newsid = path.parent_path().leaf().string();
+			Slicer::SerializeAny<Serializer>(news, dbc, "gentoobrowse.news");
+		}
+
+		void
+		NewsProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn)
+		{
+			auto del = dbc->modify(sql::maintenance::newsDelete.getSql());
+			del->bindParamS(0, fn.parent_path().leaf().string());
+			del->execute();
+		}
+	}
+}
+
diff --git a/gentoobrowse-api/service/maintenance/newsProcessor.h b/gentoobrowse-api/service/maintenance/newsProcessor.h
new file mode 100644
index 0000000..e499ed9
--- /dev/null
+++ b/gentoobrowse-api/service/maintenance/newsProcessor.h
@@ -0,0 +1,27 @@
+#ifndef GENTOOBROWSE_API_SERVICE_MAINTENANCE_NEWSPROC_H
+#define GENTOOBROWSE_API_SERVICE_MAINTENANCE_NEWSPROC_H
+
+#include "../maintenanceimpl.h"
+#include <connection.h>
+#include <boost/filesystem/path.hpp>
+#include <portage-models.h>
+
+namespace Gentoo {
+	namespace Service {
+		class NewsProcessor : public Maintenance::FileProcessor {
+			public:
+				static const int FILETYPEID;
+
+				void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override;
+				void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override;
+				void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) override;
+
+			private:
+				template <typename Serializer>
+				static void importNews(DB::Connection * dbc, const boost::filesystem::path & path);
+		};
+	}
+}
+
+#endif
+
diff --git a/gentoobrowse-api/service/maintenancePackageTree.cpp b/gentoobrowse-api/service/maintenancePackageTree.cpp
index f5a06de..03f8fe5 100644
--- a/gentoobrowse-api/service/maintenancePackageTree.cpp
+++ b/gentoobrowse-api/service/maintenancePackageTree.cpp
@@ -17,6 +17,7 @@
 #include "maintenance/useLocalProcessor.h"
 #include "maintenance/useGroupProcessor.h"
 #include "maintenance/masksProcessor.h"
+#include "maintenance/newsProcessor.h"
 
 /*
 10	category metadata.xml	{"(2,metadata.xml)"}
@@ -26,11 +27,11 @@
 6	use_local	{"(1,profiles)","(2,use.local.desc)"}
 9	use_grouped	{"(1,profiles)","(2,desc)","(3,%.desc)"}
 3	masks	{"(1,profiles)","(2,package.mask)"}
+11	news	{"(1,metadata)","(2,news)","(4,%.txt)"}
 
 8	package manifests	{"(3,Manifest)"}
 2	changelog	{"(3,ChangeLog)"}
 7	licenses	{"(1,licenses)"}
-11	news	{"(1,metadata)","(2,news)","(4,%.txt)"}
 */
 
 namespace Gentoo {
@@ -87,6 +88,7 @@ namespace Gentoo {
 			fpfs[UseLocalProcessor::FILETYPEID] = &createFileProessor<UseLocalProcessor>;
 			fpfs[UseGroupProcessor::FILETYPEID] = &createFileProessor<UseGroupProcessor>;
 			fpfs[MasksProcessor::FILETYPEID] = &createFileProessor<MasksProcessor>;
+			fpfs[NewsProcessor::FILETYPEID] = &createFileProessor<NewsProcessor>;
 		}
 
 		Maintenance::~Maintenance()
diff --git a/gentoobrowse-api/service/news.cpp b/gentoobrowse-api/service/news.cpp
new file mode 100644
index 0000000..726f9b4
--- /dev/null
+++ b/gentoobrowse-api/service/news.cpp
@@ -0,0 +1,92 @@
+#include "news.h"
+#include <boost/algorithm/string/split.hpp>
+
+Gentoo::Utils::Lexer::PatternPtr Title(Gentoo::Utils::Lexer::regex("^Title: (.+)$\\s", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE)));
+Gentoo::Utils::Lexer::PatternPtr DisplayIfInstalled(Gentoo::Utils::Lexer::regex("^Display-If-Installed: (.+)$\\s", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE)));
+Gentoo::Utils::Lexer::PatternPtr Author(Gentoo::Utils::Lexer::regex("^Author: (.+) <([^>]+)>$\\s", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE)));
+Gentoo::Utils::Lexer::PatternPtr Posted(Gentoo::Utils::Lexer::regex("^Posted: ([0-9]{4}-[0-9]{2}-[0-9]{2})$\\s", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE)));
+Gentoo::Utils::Lexer::PatternPtr IgnoredHeader(Gentoo::Utils::Lexer::regex("^[^ :]+: .+$\\s", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE)));
+Gentoo::Utils::Lexer::PatternPtr BlankLine(Gentoo::Utils::Lexer::regex("^$\\s", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE)));
+Gentoo::Utils::Lexer::PatternPtr BodyText(Gentoo::Utils::Lexer::regex("^(.*)$\\s?", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE)));
+Gentoo::Utils::Lexer::PatternPtr Link(Gentoo::Utils::Lexer::regex("^\\[[[:digit:]]+\\] ([[:alpha:]]+://.*)$\\s?", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE)));
+
+const std::string Body("Body");
+
+namespace Portage {
+	namespace Utils {
+		template<typename T, typename Y> IceUtil::Optional<T> iuo(const boost::optional<Y> & x)
+		{
+			if (x) {
+				return *x;
+			}
+			return IceUtil::Optional<T>();
+		}
+
+		template<typename T, typename Y> std::vector<T> split(const boost::optional<Y> & x)
+		{
+			std::vector<T> rtn;
+			if (x) {
+				boost::algorithm::split(rtn, *x, [](auto c) { return c == ','; });
+				std::sort(rtn.begin(), rtn.end());
+			}
+			return rtn;
+		}
+
+		News::News()
+		{
+			// title: words
+			rules.push_back({ { InitialState }, Title, [this](auto es) {
+					news->title = *es->pattern->match(1);
+				} });
+			// posted: date
+			rules.push_back({ { InitialState }, Posted, [this](auto es) {
+					news->posted = *es->pattern->match(1);
+				} });
+			// display-if-installed: atomspec
+			rules.push_back({ { InitialState }, DisplayIfInstalled, [this](auto es) {
+					news->atomspec.push_back(*es->pattern->match(1));
+				} });
+			// author: name <email>
+			rules.push_back({ { InitialState }, Author, [this](auto es) {
+					news->authorname = *es->pattern->match(1);
+					news->authoremail = *es->pattern->match(2);
+				} });
+			// other headers
+			rules.push_back({ { InitialState }, IgnoredHeader, [this](auto) {
+					} });
+			// blank
+			rules.push_back({ { InitialState }, BlankLine, [this](auto es) {
+					es->setState(Body);
+					news->body.push_back(std::string());
+				} });
+			// body blank
+			rules.push_back({ { Body }, BlankLine, [this](auto) {
+					news->body.push_back(std::string());
+				} });
+			// link
+			rules.push_back({ { Body }, Link, [this](auto es) {
+					news->urls.push_back(*es->pattern->match(1));
+				} });
+			// body text
+			rules.push_back({ { Body }, BodyText, [this](auto es) {
+					if (!news->body.back().empty()) {
+						news->body.back().append(" ");
+					}
+					news->body.back().append(*es->pattern->match(1));
+				} });
+			news = new Gentoo::NewsItem();
+		}
+
+		Gentoo::NewsItemPtr
+		News::parse(const gchar * str, size_t len)
+		{
+			News d;
+			d.extract(str, len);
+			d.news->body.erase(
+				std::remove_if(d.news->body.begin(), d.news->body.end(), [](const auto & s) { return s.empty(); }),
+				d.news->body.end());
+			return d.news;
+		}
+	}
+}
+
diff --git a/gentoobrowse-api/service/news.h b/gentoobrowse-api/service/news.h
new file mode 100644
index 0000000..ae82bb7
--- /dev/null
+++ b/gentoobrowse-api/service/news.h
@@ -0,0 +1,24 @@
+#ifndef GENTOOBROWSE_SERVICE_DEPEND_H
+#define GENTOOBROWSE_SERVICE_DEPEND_H
+
+#include <visibility.h>
+#include <istream>
+#include <vector>
+#include <portage-models.h>
+#include "utils/lexer.h"
+
+namespace Portage {
+	namespace Utils {
+		class News : Gentoo::Utils::Lexer {
+			private:
+				News();
+
+			public:
+				Gentoo::NewsItemPtr news;
+				DLL_PUBLIC static Gentoo::NewsItemPtr parse(const gchar * string, size_t length);
+		};
+	}
+}
+
+#endif
+
diff --git a/gentoobrowse-api/service/sql/maintenance/newsDelete.sql b/gentoobrowse-api/service/sql/maintenance/newsDelete.sql
new file mode 100644
index 0000000..bbd5e6f
--- /dev/null
+++ b/gentoobrowse-api/service/sql/maintenance/newsDelete.sql
@@ -0,0 +1 @@
+DELETE FROM gentoobrowse.news WHERE newsId = ?
diff --git a/gentoobrowse-api/unittests/Jamfile.jam b/gentoobrowse-api/unittests/Jamfile.jam
index 4f38f9d..c8a440f 100644
--- a/gentoobrowse-api/unittests/Jamfile.jam
+++ b/gentoobrowse-api/unittests/Jamfile.jam
@@ -50,6 +50,14 @@ run
 	: testDepend ;
 
 run
+	testNews.cpp ../service/utils/fileUtils.cpp
+	: : :
+	<define>BOOST_TEST_DYN_LINK
+	<library>testCommon
+	<library>..//boost_date_time
+	: testNews ;
+
+run
 	testMaintenance.cpp
 	: : :
 	<dependency>../db/schema.sql
diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp
index c39b9af..dc4c92b 100644
--- a/gentoobrowse-api/unittests/testMaintenance.cpp
+++ b/gentoobrowse-api/unittests/testMaintenance.cpp
@@ -41,7 +41,7 @@ BOOST_FIXTURE_TEST_SUITE(tp, TestClient)
 void
 doRefreshPackageTree(SampleData & sd, DB::ConnectionPtr db, const std::string & archive, const std::string & dir,
 		Gentoo::MaintenancePrx m, int64_t files, int64_t cats, int64_t devvcs, int64_t pkgs, int64_t ebs, int64_t ebus,
-		int64_t ebas, int64_t pus, int64_t ug, int64_t ul, int64_t ugs, int64_t ugds, int64_t deps, int64_t rdeps)
+		int64_t ebas, int64_t pus, int64_t ug, int64_t ul, int64_t ugs, int64_t ugds, int64_t deps, int64_t rdeps, int64_t news)
 {
 	if (!archive.empty()) {
 		sd.extract(archive, dir);
@@ -63,6 +63,7 @@ doRefreshPackageTree(SampleData & sd, DB::ConnectionPtr db, const std::string &
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.use_group", int64_t, ugds);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuild_deps", int64_t, deps);
 	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuild_rdeps", int64_t, rdeps);
+	SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.news", int64_t, news);
 }
 
 BOOST_AUTO_TEST_CASE( refreshPackageTree )
@@ -75,7 +76,7 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree )
 	insRepo->execute();
 
 	doRefreshPackageTree(sd, db, "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d", "gentoo",
-			m, 2084, 5, 1, 482, 981, 3626, 4593, 501, 393, 238, 50, 1573, 2008, 1543);
+			m, 2084, 5, 1, 482, 981, 3626, 4593, 501, 393, 238, 50, 1573, 2008, 1543, 81);
 
 	db->execute("COPY gentoobrowse.files TO '/tmp/files1.tsv'");
 	db->execute("COPY gentoobrowse.categories TO '/tmp/categories1.tsv'");
@@ -90,9 +91,10 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree )
 	db->execute("COPY gentoobrowse.use_group TO '/tmp/use_group1.tsv'");
 	db->execute("COPY gentoobrowse.masksets TO '/tmp/masksets1.tsv'");
 	db->execute("COPY gentoobrowse.ebuild_masks TO '/tmp/ebuild_masks1.tsv'");
+	db->execute("COPY gentoobrowse.news TO '/tmp/news1.tsv'");
 
 	doRefreshPackageTree(sd, db, "756569aa764177340726dd3d40b41d89b11b20c7", "gentoo",
-			m, 2087, 5, 1, 484, 982, 3638, 4599, 503, 393, 238, 50, 1573, 2009, 1546);
+			m, 2087, 5, 1, 484, 982, 3638, 4599, 503, 393, 238, 50, 1573, 2009, 1546, 79);
 
 	db->execute("COPY gentoobrowse.categories TO '/tmp/categories2.tsv'");
 	db->execute("COPY gentoobrowse.packages TO '/tmp/packages2.tsv'");
@@ -106,9 +108,10 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree )
 	db->execute("COPY gentoobrowse.use_group TO '/tmp/use_group2.tsv'");
 	db->execute("COPY gentoobrowse.masksets TO '/tmp/masksets2.tsv'");
 	db->execute("COPY gentoobrowse.ebuild_masks TO '/tmp/ebuild_masks2.tsv'");
+	db->execute("COPY gentoobrowse.news TO '/tmp/news2.tsv'");
 
 	doRefreshPackageTree(sd, db, "", "gentoo",
-			m, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+			m, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 	m->refreshPackageTree();
 }
 
diff --git a/gentoobrowse-api/unittests/testNews.cpp b/gentoobrowse-api/unittests/testNews.cpp
new file mode 100644
index 0000000..734a91e
--- /dev/null
+++ b/gentoobrowse-api/unittests/testNews.cpp
@@ -0,0 +1,47 @@
+#define BOOST_TEST_MODULE TestNews
+#include <boost/test/unit_test.hpp>
+
+#include <news.h>
+#include <utils/fileUtils.h>
+#include <definedDirs.h>
+
+BOOST_AUTO_TEST_CASE( news_2016_04_07_kde_plasma5_stable )
+{
+	Gentoo::Utils::MemMap m(rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d" / "metadata" / "news" / "2016-04-07-kde-plasma5-stable" / "2016-04-07-kde-plasma5-stable.en.txt");
+	auto news = Portage::Utils::News::parse(reinterpret_cast<const gchar *>(m.data), m.getStat().st_size);
+	BOOST_REQUIRE_EQUAL(news->title, "KDE Plasma 5 Upgrade");
+	BOOST_REQUIRE(news->authorname);
+	BOOST_REQUIRE_EQUAL(*news->authorname, "Michael Palimaka");
+	BOOST_REQUIRE(news->authoremail);
+	BOOST_REQUIRE_EQUAL(*news->authoremail, "kensington@gentoo.org");
+	BOOST_REQUIRE_EQUAL(news->posted, "2016-04-02");
+	for(const auto & s : news->body) {
+		BOOST_TEST_INFO(s);
+	}
+	BOOST_REQUIRE_EQUAL(news->body.size(), 6);
+	BOOST_REQUIRE_EQUAL(news->body[0], "KDE Workspaces 4.11 has reached end of life and is no longer supported by upstream. It is therefore recommended for all users to upgrade to KDE Plasma 5.");
+	BOOST_REQUIRE_EQUAL(news->urls.size(), 2);
+	BOOST_REQUIRE_EQUAL(news->urls[0], "https://wiki.gentoo.org/wiki/KDE/Plasma_5_upgrade");
+	BOOST_REQUIRE_EQUAL(news->atomspec.size(), 1);
+	BOOST_REQUIRE_EQUAL(news->atomspec[0], "kde-base/plasma-workspace");
+}
+
+BOOST_AUTO_TEST_CASE( news_2010_03_23_new_subprofiles )
+{
+	Gentoo::Utils::MemMap m(rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d" / "metadata" / "news" / "2010-03-23-new-subprofiles" / "2010-03-23-new-subprofiles.en.txt");
+	auto news = Portage::Utils::News::parse(reinterpret_cast<const gchar *>(m.data), m.getStat().st_size);
+	BOOST_REQUIRE_EQUAL(news->title, "New desktop subprofiles for GNOME and KDE");
+	BOOST_REQUIRE(news->authorname);
+	BOOST_REQUIRE_EQUAL(*news->authorname, "Theo Chatzimichos");
+	BOOST_REQUIRE(news->authoremail);
+	BOOST_REQUIRE_EQUAL(*news->authoremail, "tampakrap@gentoo.org");
+	BOOST_REQUIRE_EQUAL(news->posted, "2010-03-23");
+	for(const auto & s : news->body) {
+		BOOST_TEST_INFO(s);
+	}
+	BOOST_REQUIRE_EQUAL(news->body.size(), 3);
+	BOOST_REQUIRE_EQUAL(news->body[2], "(I'll commit the change on Friday, 26 Mar 2010)");
+	BOOST_REQUIRE_EQUAL(news->urls.size(), 0);
+	BOOST_REQUIRE_EQUAL(news->atomspec.size(), 0);
+}
+