1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
|
#include "maintenanceimpl.h"
#include <stack>
#include <boost/lexical_cast.hpp>
#include <selectcommandUtil.impl.h>
#include <tablepatch.h>
#include <sqlWriter.h>
#include <buffer.h>
#include <scopeExit.h>
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#include <curlStream.h>
#include <libxml++/parsers/saxparser.h>
#pragma GCC diagnostic pop
#include <lexer.h>
#include <lexer-regex.h>
#include <utils/dbUtils.h>
#include <compileTimeFormatter.h>
#include <logger.h>
namespace Gentoo {
namespace Service {
AdHoc::Lexer::PatternPtr bugLink = AdHoc::LexerMatchers::regex(
R"R(Bug:(\d+) - "" status:(\w*) resolution:(\w*) severity:(\w*).*)R", G_REGEX_OPTIMIZE);
class BugListParser : public xmlpp::SaxParser, AdHoc::Lexer {
public:
BugListParser(DB::ModifyCommandPtr i) :
ins(i)
{
rules.push_back({ { InitialState }, bugLink, [this](auto es) {
ins->bindParamI(0, boost::lexical_cast<int64_t>(*es->pattern()->match(1)));
ins->bindParamS(2, *es->pattern()->match(2));
ins->bindParamS(1, *es->pattern()->match(4));
}});
}
protected:
void on_start_element(const Glib::ustring & e, const xmlpp::SaxParser::AttributeList &) override
{
stk.push(e);
}
void on_characters(const Glib::ustring & t) override
{
if (stk.top() == "a") {
attributes += t;
}
else if (stk.top() == "em") {
summary += t;
}
}
void on_end_element(const Glib::ustring & e) override
{
stk.pop();
if (e == "li") {
extract(attributes.c_str(), attributes.length());
ins->bindParamS(3, summary);
ins->execute();
attributes.clear();
summary.clear();
}
}
private:
DB::ModifyCommandPtr ins;
std::stack<Glib::ustring> stk;
Glib::ustring attributes;
Glib::ustring summary;
};
AdHocFormatter(UpdatingBugs, "Refreshing bug list from %?");
AdHocFormatter(UpdatingBugUrl, " ... %?");
AdHocFormatter(UpdatingBugsResult, "Refreshed bug list: %? added, %? updated, %? removed");
void
Maintenance::refreshBugs(const Ice::Current & c)
{
auto log = LOGMANAGER()->getLogger(__FUNCTION__);
std::filesystem::path root = properties(c)->getPropertyWithDefault(
"GentooBrowseAPI.BugRoot", "https://bugs.gentoo.org/data/cached");
log->messagectf<UpdatingBugs>(IceTray::Logging::LogLevel::INFO, root);
auto dbc = db->get();
DB::TransactionScope tx(*dbc.get());
DB::TablePatch tp;
tp.pk = { "bugId" };
tp.cols = { "bugId", "severity", "status", "summary" };
tp.dest = "gentoobrowse.bugs";
tp.src = Utils::Database::emptyClone(dbc.get(), "gentoobrowse.bugs");
auto ins = Utils::Database::tablePatchInserter(dbc.get(), tp);
BugListParser blp(ins);
for(const auto & bl : {
"buglist-CONFIRMED.html",
"buglist-UNCONFIRMED.html",
"buglist-IN_PROGRESS.html" }) {
log->messagectf<UpdatingBugUrl>(IceTray::Logging::LogLevel::DEBUG, bl);
AdHoc::Net::CurlStreamSource css(root / bl);
css.setopt(CURLOPT_ENCODING, "deflate, gzip");
css.setopt(CURLOPT_TIMEOUT, 20L);
AdHoc::Net::CurlStream cs(css);
blp.parse_stream(cs);
}
auto result = dbc->patchTable(&tp);
log->messagectf<UpdatingBugsResult>(IceTray::Logging::LogLevel::INFO, result.inserts, result.updates, result.deletes);
Utils::Database::drop(dbc.get(), tp.src);
}
}
}
|
