From 4d739dc0b97fe6d117ca3329e7492c2022b685f3 Mon Sep 17 00:00:00 2001 From: Dan Goodliffe Date: Wed, 1 Jun 2016 13:31:31 +0100 Subject: Import package tree news --- .../service/maintenance/newsProcessor.cpp | 50 ++++++++++++ .../service/maintenance/newsProcessor.h | 27 +++++++ .../service/maintenancePackageTree.cpp | 4 +- gentoobrowse-api/service/news.cpp | 92 ++++++++++++++++++++++ gentoobrowse-api/service/news.h | 24 ++++++ .../service/sql/maintenance/newsDelete.sql | 1 + gentoobrowse-api/unittests/Jamfile.jam | 8 ++ gentoobrowse-api/unittests/testMaintenance.cpp | 11 ++- gentoobrowse-api/unittests/testNews.cpp | 47 +++++++++++ 9 files changed, 259 insertions(+), 5 deletions(-) create mode 100644 gentoobrowse-api/service/maintenance/newsProcessor.cpp create mode 100644 gentoobrowse-api/service/maintenance/newsProcessor.h create mode 100644 gentoobrowse-api/service/news.cpp create mode 100644 gentoobrowse-api/service/news.h create mode 100644 gentoobrowse-api/service/sql/maintenance/newsDelete.sql create mode 100644 gentoobrowse-api/unittests/testNews.cpp diff --git a/gentoobrowse-api/service/maintenance/newsProcessor.cpp b/gentoobrowse-api/service/maintenance/newsProcessor.cpp new file mode 100644 index 0000000..0b5b920 --- /dev/null +++ b/gentoobrowse-api/service/maintenance/newsProcessor.cpp @@ -0,0 +1,50 @@ +#include "newsProcessor.h" +#include "utils/fileUtils.h" +#include "utils/xmlUtils.h" +#include "utils/dbUtils.h" +#include "news.h" +#include +#include +#include +#include +#include + +namespace U = Gentoo::Utils; +using namespace Gentoo::Utils::File; + +namespace Gentoo { + namespace Service { + const int NewsProcessor::FILETYPEID = 11; + + void + NewsProcessor::created(DB::Connection * dbc, const boost::filesystem::path &, const boost::filesystem::path & path) + { + importNews(dbc, path); + } + + void + NewsProcessor::modified(DB::Connection * dbc, const boost::filesystem::path &, const boost::filesystem::path & path) + { + importNews(dbc, path); + } + + template + void + NewsProcessor::importNews(DB::Connection * dbc, const boost::filesystem::path & path) + { + Gentoo::Utils::MemMap m(path); + auto news = Portage::Utils::News::parse(reinterpret_cast(m.data), m.getStat().st_size); + news->newsid = path.parent_path().leaf().string(); + Slicer::SerializeAny(news, dbc, "gentoobrowse.news"); + } + + void + NewsProcessor::deleted(DB::Connection * dbc, const boost::filesystem::path & fn) + { + auto del = dbc->modify(sql::maintenance::newsDelete.getSql()); + del->bindParamS(0, fn.parent_path().leaf().string()); + del->execute(); + } + } +} + diff --git a/gentoobrowse-api/service/maintenance/newsProcessor.h b/gentoobrowse-api/service/maintenance/newsProcessor.h new file mode 100644 index 0000000..e499ed9 --- /dev/null +++ b/gentoobrowse-api/service/maintenance/newsProcessor.h @@ -0,0 +1,27 @@ +#ifndef GENTOOBROWSE_API_SERVICE_MAINTENANCE_NEWSPROC_H +#define GENTOOBROWSE_API_SERVICE_MAINTENANCE_NEWSPROC_H + +#include "../maintenanceimpl.h" +#include +#include +#include + +namespace Gentoo { + namespace Service { + class NewsProcessor : public Maintenance::FileProcessor { + public: + static const int FILETYPEID; + + void created(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; + void modified(DB::Connection * dbc, const boost::filesystem::path & fn, const boost::filesystem::path & path) override; + void deleted(DB::Connection * dbc, const boost::filesystem::path & fn) override; + + private: + template + static void importNews(DB::Connection * dbc, const boost::filesystem::path & path); + }; + } +} + +#endif + diff --git a/gentoobrowse-api/service/maintenancePackageTree.cpp b/gentoobrowse-api/service/maintenancePackageTree.cpp index f5a06de..03f8fe5 100644 --- a/gentoobrowse-api/service/maintenancePackageTree.cpp +++ b/gentoobrowse-api/service/maintenancePackageTree.cpp @@ -17,6 +17,7 @@ #include "maintenance/useLocalProcessor.h" #include "maintenance/useGroupProcessor.h" #include "maintenance/masksProcessor.h" +#include "maintenance/newsProcessor.h" /* 10 category metadata.xml {"(2,metadata.xml)"} @@ -26,11 +27,11 @@ 6 use_local {"(1,profiles)","(2,use.local.desc)"} 9 use_grouped {"(1,profiles)","(2,desc)","(3,%.desc)"} 3 masks {"(1,profiles)","(2,package.mask)"} +11 news {"(1,metadata)","(2,news)","(4,%.txt)"} 8 package manifests {"(3,Manifest)"} 2 changelog {"(3,ChangeLog)"} 7 licenses {"(1,licenses)"} -11 news {"(1,metadata)","(2,news)","(4,%.txt)"} */ namespace Gentoo { @@ -87,6 +88,7 @@ namespace Gentoo { fpfs[UseLocalProcessor::FILETYPEID] = &createFileProessor; fpfs[UseGroupProcessor::FILETYPEID] = &createFileProessor; fpfs[MasksProcessor::FILETYPEID] = &createFileProessor; + fpfs[NewsProcessor::FILETYPEID] = &createFileProessor; } Maintenance::~Maintenance() diff --git a/gentoobrowse-api/service/news.cpp b/gentoobrowse-api/service/news.cpp new file mode 100644 index 0000000..726f9b4 --- /dev/null +++ b/gentoobrowse-api/service/news.cpp @@ -0,0 +1,92 @@ +#include "news.h" +#include + +Gentoo::Utils::Lexer::PatternPtr Title(Gentoo::Utils::Lexer::regex("^Title: (.+)$\\s", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE))); +Gentoo::Utils::Lexer::PatternPtr DisplayIfInstalled(Gentoo::Utils::Lexer::regex("^Display-If-Installed: (.+)$\\s", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE))); +Gentoo::Utils::Lexer::PatternPtr Author(Gentoo::Utils::Lexer::regex("^Author: (.+) <([^>]+)>$\\s", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE))); +Gentoo::Utils::Lexer::PatternPtr Posted(Gentoo::Utils::Lexer::regex("^Posted: ([0-9]{4}-[0-9]{2}-[0-9]{2})$\\s", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE))); +Gentoo::Utils::Lexer::PatternPtr IgnoredHeader(Gentoo::Utils::Lexer::regex("^[^ :]+: .+$\\s", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE))); +Gentoo::Utils::Lexer::PatternPtr BlankLine(Gentoo::Utils::Lexer::regex("^$\\s", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE))); +Gentoo::Utils::Lexer::PatternPtr BodyText(Gentoo::Utils::Lexer::regex("^(.*)$\\s?", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE))); +Gentoo::Utils::Lexer::PatternPtr Link(Gentoo::Utils::Lexer::regex("^\\[[[:digit:]]+\\] ([[:alpha:]]+://.*)$\\s?", (GRegexCompileFlags)(G_REGEX_OPTIMIZE | G_REGEX_MULTILINE))); + +const std::string Body("Body"); + +namespace Portage { + namespace Utils { + template IceUtil::Optional iuo(const boost::optional & x) + { + if (x) { + return *x; + } + return IceUtil::Optional(); + } + + template std::vector split(const boost::optional & x) + { + std::vector rtn; + if (x) { + boost::algorithm::split(rtn, *x, [](auto c) { return c == ','; }); + std::sort(rtn.begin(), rtn.end()); + } + return rtn; + } + + News::News() + { + // title: words + rules.push_back({ { InitialState }, Title, [this](auto es) { + news->title = *es->pattern->match(1); + } }); + // posted: date + rules.push_back({ { InitialState }, Posted, [this](auto es) { + news->posted = *es->pattern->match(1); + } }); + // display-if-installed: atomspec + rules.push_back({ { InitialState }, DisplayIfInstalled, [this](auto es) { + news->atomspec.push_back(*es->pattern->match(1)); + } }); + // author: name + rules.push_back({ { InitialState }, Author, [this](auto es) { + news->authorname = *es->pattern->match(1); + news->authoremail = *es->pattern->match(2); + } }); + // other headers + rules.push_back({ { InitialState }, IgnoredHeader, [this](auto) { + } }); + // blank + rules.push_back({ { InitialState }, BlankLine, [this](auto es) { + es->setState(Body); + news->body.push_back(std::string()); + } }); + // body blank + rules.push_back({ { Body }, BlankLine, [this](auto) { + news->body.push_back(std::string()); + } }); + // link + rules.push_back({ { Body }, Link, [this](auto es) { + news->urls.push_back(*es->pattern->match(1)); + } }); + // body text + rules.push_back({ { Body }, BodyText, [this](auto es) { + if (!news->body.back().empty()) { + news->body.back().append(" "); + } + news->body.back().append(*es->pattern->match(1)); + } }); + news = new Gentoo::NewsItem(); + } + + Gentoo::NewsItemPtr + News::parse(const gchar * str, size_t len) + { + News d; + d.extract(str, len); + d.news->body.erase( + std::remove_if(d.news->body.begin(), d.news->body.end(), [](const auto & s) { return s.empty(); }), + d.news->body.end()); + return d.news; + } + } +} + diff --git a/gentoobrowse-api/service/news.h b/gentoobrowse-api/service/news.h new file mode 100644 index 0000000..ae82bb7 --- /dev/null +++ b/gentoobrowse-api/service/news.h @@ -0,0 +1,24 @@ +#ifndef GENTOOBROWSE_SERVICE_DEPEND_H +#define GENTOOBROWSE_SERVICE_DEPEND_H + +#include +#include +#include +#include +#include "utils/lexer.h" + +namespace Portage { + namespace Utils { + class News : Gentoo::Utils::Lexer { + private: + News(); + + public: + Gentoo::NewsItemPtr news; + DLL_PUBLIC static Gentoo::NewsItemPtr parse(const gchar * string, size_t length); + }; + } +} + +#endif + diff --git a/gentoobrowse-api/service/sql/maintenance/newsDelete.sql b/gentoobrowse-api/service/sql/maintenance/newsDelete.sql new file mode 100644 index 0000000..bbd5e6f --- /dev/null +++ b/gentoobrowse-api/service/sql/maintenance/newsDelete.sql @@ -0,0 +1 @@ +DELETE FROM gentoobrowse.news WHERE newsId = ? diff --git a/gentoobrowse-api/unittests/Jamfile.jam b/gentoobrowse-api/unittests/Jamfile.jam index 4f38f9d..c8a440f 100644 --- a/gentoobrowse-api/unittests/Jamfile.jam +++ b/gentoobrowse-api/unittests/Jamfile.jam @@ -49,6 +49,14 @@ run testCommon : testDepend ; +run + testNews.cpp ../service/utils/fileUtils.cpp + : : : + BOOST_TEST_DYN_LINK + testCommon + ..//boost_date_time + : testNews ; + run testMaintenance.cpp : : : diff --git a/gentoobrowse-api/unittests/testMaintenance.cpp b/gentoobrowse-api/unittests/testMaintenance.cpp index c39b9af..dc4c92b 100644 --- a/gentoobrowse-api/unittests/testMaintenance.cpp +++ b/gentoobrowse-api/unittests/testMaintenance.cpp @@ -41,7 +41,7 @@ BOOST_FIXTURE_TEST_SUITE(tp, TestClient) void doRefreshPackageTree(SampleData & sd, DB::ConnectionPtr db, const std::string & archive, const std::string & dir, Gentoo::MaintenancePrx m, int64_t files, int64_t cats, int64_t devvcs, int64_t pkgs, int64_t ebs, int64_t ebus, - int64_t ebas, int64_t pus, int64_t ug, int64_t ul, int64_t ugs, int64_t ugds, int64_t deps, int64_t rdeps) + int64_t ebas, int64_t pus, int64_t ug, int64_t ul, int64_t ugs, int64_t ugds, int64_t deps, int64_t rdeps, int64_t news) { if (!archive.empty()) { sd.extract(archive, dir); @@ -63,6 +63,7 @@ doRefreshPackageTree(SampleData & sd, DB::ConnectionPtr db, const std::string & SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.use_group", int64_t, ugds); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuild_deps", int64_t, deps); SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.ebuild_rdeps", int64_t, rdeps); + SQL_REQUIRE_EQUAL("SELECT COUNT(*) FROM gentoobrowse.news", int64_t, news); } BOOST_AUTO_TEST_CASE( refreshPackageTree ) @@ -75,7 +76,7 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree ) insRepo->execute(); doRefreshPackageTree(sd, db, "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d", "gentoo", - m, 2084, 5, 1, 482, 981, 3626, 4593, 501, 393, 238, 50, 1573, 2008, 1543); + m, 2084, 5, 1, 482, 981, 3626, 4593, 501, 393, 238, 50, 1573, 2008, 1543, 81); db->execute("COPY gentoobrowse.files TO '/tmp/files1.tsv'"); db->execute("COPY gentoobrowse.categories TO '/tmp/categories1.tsv'"); @@ -90,9 +91,10 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree ) db->execute("COPY gentoobrowse.use_group TO '/tmp/use_group1.tsv'"); db->execute("COPY gentoobrowse.masksets TO '/tmp/masksets1.tsv'"); db->execute("COPY gentoobrowse.ebuild_masks TO '/tmp/ebuild_masks1.tsv'"); + db->execute("COPY gentoobrowse.news TO '/tmp/news1.tsv'"); doRefreshPackageTree(sd, db, "756569aa764177340726dd3d40b41d89b11b20c7", "gentoo", - m, 2087, 5, 1, 484, 982, 3638, 4599, 503, 393, 238, 50, 1573, 2009, 1546); + m, 2087, 5, 1, 484, 982, 3638, 4599, 503, 393, 238, 50, 1573, 2009, 1546, 79); db->execute("COPY gentoobrowse.categories TO '/tmp/categories2.tsv'"); db->execute("COPY gentoobrowse.packages TO '/tmp/packages2.tsv'"); @@ -106,9 +108,10 @@ BOOST_AUTO_TEST_CASE( refreshPackageTree ) db->execute("COPY gentoobrowse.use_group TO '/tmp/use_group2.tsv'"); db->execute("COPY gentoobrowse.masksets TO '/tmp/masksets2.tsv'"); db->execute("COPY gentoobrowse.ebuild_masks TO '/tmp/ebuild_masks2.tsv'"); + db->execute("COPY gentoobrowse.news TO '/tmp/news2.tsv'"); doRefreshPackageTree(sd, db, "", "gentoo", - m, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + m, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); m->refreshPackageTree(); } diff --git a/gentoobrowse-api/unittests/testNews.cpp b/gentoobrowse-api/unittests/testNews.cpp new file mode 100644 index 0000000..734a91e --- /dev/null +++ b/gentoobrowse-api/unittests/testNews.cpp @@ -0,0 +1,47 @@ +#define BOOST_TEST_MODULE TestNews +#include + +#include +#include +#include + +BOOST_AUTO_TEST_CASE( news_2016_04_07_kde_plasma5_stable ) +{ + Gentoo::Utils::MemMap m(rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d" / "metadata" / "news" / "2016-04-07-kde-plasma5-stable" / "2016-04-07-kde-plasma5-stable.en.txt"); + auto news = Portage::Utils::News::parse(reinterpret_cast(m.data), m.getStat().st_size); + BOOST_REQUIRE_EQUAL(news->title, "KDE Plasma 5 Upgrade"); + BOOST_REQUIRE(news->authorname); + BOOST_REQUIRE_EQUAL(*news->authorname, "Michael Palimaka"); + BOOST_REQUIRE(news->authoremail); + BOOST_REQUIRE_EQUAL(*news->authoremail, "kensington@gentoo.org"); + BOOST_REQUIRE_EQUAL(news->posted, "2016-04-02"); + for(const auto & s : news->body) { + BOOST_TEST_INFO(s); + } + BOOST_REQUIRE_EQUAL(news->body.size(), 6); + BOOST_REQUIRE_EQUAL(news->body[0], "KDE Workspaces 4.11 has reached end of life and is no longer supported by upstream. It is therefore recommended for all users to upgrade to KDE Plasma 5."); + BOOST_REQUIRE_EQUAL(news->urls.size(), 2); + BOOST_REQUIRE_EQUAL(news->urls[0], "https://wiki.gentoo.org/wiki/KDE/Plasma_5_upgrade"); + BOOST_REQUIRE_EQUAL(news->atomspec.size(), 1); + BOOST_REQUIRE_EQUAL(news->atomspec[0], "kde-base/plasma-workspace"); +} + +BOOST_AUTO_TEST_CASE( news_2010_03_23_new_subprofiles ) +{ + Gentoo::Utils::MemMap m(rootDir / "fixtures" / "4156eb45cf3b0ce1d7125b84efd8688c2d6e831d" / "metadata" / "news" / "2010-03-23-new-subprofiles" / "2010-03-23-new-subprofiles.en.txt"); + auto news = Portage::Utils::News::parse(reinterpret_cast(m.data), m.getStat().st_size); + BOOST_REQUIRE_EQUAL(news->title, "New desktop subprofiles for GNOME and KDE"); + BOOST_REQUIRE(news->authorname); + BOOST_REQUIRE_EQUAL(*news->authorname, "Theo Chatzimichos"); + BOOST_REQUIRE(news->authoremail); + BOOST_REQUIRE_EQUAL(*news->authoremail, "tampakrap@gentoo.org"); + BOOST_REQUIRE_EQUAL(news->posted, "2010-03-23"); + for(const auto & s : news->body) { + BOOST_TEST_INFO(s); + } + BOOST_REQUIRE_EQUAL(news->body.size(), 3); + BOOST_REQUIRE_EQUAL(news->body[2], "(I'll commit the change on Friday, 26 Mar 2010)"); + BOOST_REQUIRE_EQUAL(news->urls.size(), 0); + BOOST_REQUIRE_EQUAL(news->atomspec.size(), 0); +} + -- cgit v1.2.3