diff options
Diffstat (limited to 'gentoobrowse-api/service/utils')
-rw-r--r-- | gentoobrowse-api/service/utils/dbUtils.cpp | 106 | ||||
-rw-r--r-- | gentoobrowse-api/service/utils/dbUtils.h | 28 | ||||
-rw-r--r-- | gentoobrowse-api/service/utils/ebuildCacheParser.cpp | 34 | ||||
-rw-r--r-- | gentoobrowse-api/service/utils/ebuildCacheParser.h | 28 | ||||
-rw-r--r-- | gentoobrowse-api/service/utils/entityWhereFilter.cpp | 47 | ||||
-rw-r--r-- | gentoobrowse-api/service/utils/entityWhereFilter.h | 27 | ||||
-rw-r--r-- | gentoobrowse-api/service/utils/fileUtils.cpp | 60 | ||||
-rw-r--r-- | gentoobrowse-api/service/utils/fileUtils.h | 45 | ||||
-rw-r--r-- | gentoobrowse-api/service/utils/lexer.cpp | 137 | ||||
-rw-r--r-- | gentoobrowse-api/service/utils/lexer.h | 62 | ||||
-rw-r--r-- | gentoobrowse-api/service/utils/splitEbuildProps.cpp | 29 | ||||
-rw-r--r-- | gentoobrowse-api/service/utils/splitEbuildProps.h | 25 | ||||
-rw-r--r-- | gentoobrowse-api/service/utils/xmlUtils.cpp | 26 | ||||
-rw-r--r-- | gentoobrowse-api/service/utils/xmlUtils.h | 20 |
14 files changed, 674 insertions, 0 deletions
diff --git a/gentoobrowse-api/service/utils/dbUtils.cpp b/gentoobrowse-api/service/utils/dbUtils.cpp new file mode 100644 index 0000000..ae07ff4 --- /dev/null +++ b/gentoobrowse-api/service/utils/dbUtils.cpp @@ -0,0 +1,106 @@ +#include "dbUtils.h" +#include <boost/lexical_cast.hpp> +#include <boost/algorithm/string/join.hpp> +#include <tablepatch.h> +#include <buffer.h> +#include <atomic> + +namespace Gentoo { + namespace Utils { + namespace Database { + bool + bindOptionalsS(DB::Command * db, unsigned int c, const std::vector<boost::optional<Glib::ustring> > & vs) + { + for(const auto & v : vs) { + if (v) { + db->bindParamS(c, *v); + return true; + } + } + db->bindNull(c); + return false; + } + + void + bindOptionalS(DB::Command * db, unsigned int c, const IceUtil::Optional<std::string> & v) + { + if (v) { + db->bindParamS(c, *v); + } + else { + db->bindNull(c); + } + } + + std::atomic<uint16_t> tempTableNumber; + std::string + tempTableName() + { + return stringbf("_tmpTable_%x", tempTableNumber++); + } + + std::string + createTempWith(DB::Connection * db, const std::string & sql, const std::set<std::string> & keys) + { + auto tempTable = tempTableName(); + db->execute("CREATE TEMPORARY TABLE " + tempTable + " AS " + sql); + db->execute("ALTER TABLE " + tempTable + " ADD CONSTRAINT pk" + tempTable + " PRIMARY KEY (" + boost::algorithm::join(keys, ",") + ")"); + return tempTable; + } + + std::string + emptyClone(DB::Connection * db, const std::string & orig) + { + auto tempTable = orig; + auto dot = tempTable.rfind('.'); + if (dot != std::string::npos) { + tempTable = tempTable.substr(dot + 1); + } + tempTable += "_clone_" + boost::lexical_cast<std::string>(db); + db->execute("CREATE TEMPORARY TABLE " + tempTable + " AS SELECT * FROM " + orig + " WHERE false"); + return tempTable; + } + + std::pair<std::string, DB::ModifyCommandPtr> + customTemp(DB::Connection * db, const std::map<std::string, const std::string> & cols) + { + return namedTemp(db, "tmp_" + boost::lexical_cast<std::string>(db), cols); + } + + std::pair<std::string, DB::ModifyCommandPtr> + namedTemp(DB::Connection * db, const std::string & tempTable, const std::map<std::string, const std::string> & cols) + { + std::set<std::string> keys; + std::set<std::string> defs; + for (auto c : cols) { + keys.insert(c.first); + defs.insert(c.first + " " + c.second); + } + db->execute("CREATE TEMPORARY TABLE " + tempTable + "(" + + boost::join(defs, ",") + ")"); + return { tempTable, tablePatchInserter(db, tempTable, keys) }; + } + + void + drop(DB::Connection * db, const std::string & table) + { + db->execute("DROP TABLE " + table); + } + + DB::ModifyCommandPtr + tablePatchInserter(DB::Connection * dbc, const DB::TablePatch & p) + { + return tablePatchInserter(dbc, p.src, p.cols); + } + DB::ModifyCommandPtr + tablePatchInserter(DB::Connection * dbc, const std::string & t, const std::set<std::string> & c) + { + return dbc->modify( + "INSERT INTO " + t + + "(" + boost::algorithm::join(c, ", ") + + ") VALUES(" + boost::algorithm::join(std::vector<std::string>(c.size(), "?"), ", ") + ")"); + } + } + } +} + diff --git a/gentoobrowse-api/service/utils/dbUtils.h b/gentoobrowse-api/service/utils/dbUtils.h new file mode 100644 index 0000000..aa46c33 --- /dev/null +++ b/gentoobrowse-api/service/utils/dbUtils.h @@ -0,0 +1,28 @@ +#ifndef GENTOOBROWSE_API_SERVICE_DBUTILS_H +#define GENTOOBROWSE_API_SERVICE_DBUTILS_H + +#include <command.h> +#include <modifycommand.h> +#include <connection.h> +#include <IceUtil/Exception.h> +#include <IceUtil/Optional.h> + +namespace Gentoo { + namespace Utils { + namespace Database { + bool bindOptionalsS(DB::Command * db, unsigned int c, const std::vector<boost::optional<Glib::ustring> > & vs); + void bindOptionalS(DB::Command * db, unsigned int c, const IceUtil::Optional<std::string> & v); + + std::string createTempWith(DB::Connection *, const std::string &, const std::set<std::string> & keys = std::set<std::string>()); + std::string emptyClone(DB::Connection *, const std::string &); + std::pair<std::string, DB::ModifyCommandPtr> customTemp(DB::Connection *, const std::map<std::string, const std::string> & cols); + std::pair<std::string, DB::ModifyCommandPtr> namedTemp(DB::Connection *, const std::string &, const std::map<std::string, const std::string> & cols); + void drop(DB::Connection *, const std::string &); + DB::ModifyCommandPtr tablePatchInserter(DB::Connection *, const DB::TablePatch &); + DB::ModifyCommandPtr tablePatchInserter(DB::Connection *, const std::string &, const std::set<std::string> &); + } + } +} + +#endif + diff --git a/gentoobrowse-api/service/utils/ebuildCacheParser.cpp b/gentoobrowse-api/service/utils/ebuildCacheParser.cpp new file mode 100644 index 0000000..ad894db --- /dev/null +++ b/gentoobrowse-api/service/utils/ebuildCacheParser.cpp @@ -0,0 +1,34 @@ +#include "ebuildCacheParser.h" + +namespace U = Gentoo::Utils; + +namespace Gentoo { + namespace Utils { + EbuildCacheParser::EbuildCacheParser(const boost::filesystem::path & p) : + U::MemMap(p) + { + const char * chardata = (const char *)this->data; + while (const char * eq = strchr(chardata, '=')) { + if (const char * nl = strchr(eq + 1, '\n')) { + kvs.insert({ std::string(chardata, eq), { eq + 1, nl } }); + chardata = nl + 1; + } + else { + kvs.insert({ std::string(chardata, eq), { eq + 1, (const char *)this->data + st.st_size } }); + return; + } + } + } + + boost::optional<Glib::ustring> + EbuildCacheParser::get(const std::string & key) const + { + auto kvi = kvs.find(key); + if (kvi == kvs.end()) { + return boost::optional<Glib::ustring>(); + } + return Glib::ustring(kvi->second.first, kvi->second.second); + } + } +} + diff --git a/gentoobrowse-api/service/utils/ebuildCacheParser.h b/gentoobrowse-api/service/utils/ebuildCacheParser.h new file mode 100644 index 0000000..e9f1dfc --- /dev/null +++ b/gentoobrowse-api/service/utils/ebuildCacheParser.h @@ -0,0 +1,28 @@ +#ifndef GENTOOBROWSE_API_SERVICE_MAINTENANCE_EBUILDCACHEPARSER_H +#define GENTOOBROWSE_API_SERVICE_MAINTENANCE_EBUILDCACHEPARSER_H + +#include "fileUtils.h" +#include <map> +#include <boost/optional.hpp> +#include <boost/filesystem/path.hpp> +#include <glibmm/ustring.h> + +namespace Gentoo { + namespace Utils { + class EbuildCacheParser : public Gentoo::Utils::MemMap { + public: + typedef std::pair<const char *, const char *> Range; + typedef std::map<std::string, const Range> KVs; + + EbuildCacheParser(const boost::filesystem::path & p); + + boost::optional<Glib::ustring> get(const std::string & key) const; + + private: + KVs kvs; + }; + } +} + +#endif + diff --git a/gentoobrowse-api/service/utils/entityWhereFilter.cpp b/gentoobrowse-api/service/utils/entityWhereFilter.cpp new file mode 100644 index 0000000..be80971 --- /dev/null +++ b/gentoobrowse-api/service/utils/entityWhereFilter.cpp @@ -0,0 +1,47 @@ +#include "entityWhereFilter.h" +#include <command.h> + +namespace Gentoo { + namespace Utils { + EntityWhereFilter::EntityWhereFilter(const std::string & en) : + entityColName(en) + { + } + + EntityWhereFilter::EntityWhereFilter(const std::string & en, int64_t e) : + entityColName(en) + { + entityIds.insert(e); + } + + EntityWhereFilter::EntityWhereFilter(const std::string & en, const EntityIds & e) : + entityIds(e), + entityColName(en) + { + } + + void + EntityWhereFilter::writeSql(AdHoc::Buffer & sql) + { + sql.appendbf("a.%s IN (", entityColName); + for (EntityIds::const_iterator ei = entityIds.begin(); ei != entityIds.end(); ++ei) { + if (ei != entityIds.begin()) { + sql.append(", ?"); + } + else { + sql.append("?"); + } + } + sql.append(")"); + } + + void + EntityWhereFilter::bindParams(DB::Command * c, unsigned int & offset) + { + for (const auto & entityId : entityIds) { + c->bindParamI(offset++, entityId); + } + } + } +} + diff --git a/gentoobrowse-api/service/utils/entityWhereFilter.h b/gentoobrowse-api/service/utils/entityWhereFilter.h new file mode 100644 index 0000000..a637288 --- /dev/null +++ b/gentoobrowse-api/service/utils/entityWhereFilter.h @@ -0,0 +1,27 @@ +#ifndef GENTOOBROWSE_API_SERVICE_MAINTENANCE_ENTITYWHEREFILTER_H +#define GENTOOBROWSE_API_SERVICE_MAINTENANCE_ENTITYWHEREFILTER_H + +#include <sqlWriter.h> +#include <set> + +namespace Gentoo { + namespace Utils { + class EntityWhereFilter : public DB::SqlWriter { + public: + typedef std::set<int64_t> EntityIds; + + EntityWhereFilter(const std::string & en); + EntityWhereFilter(const std::string & en, int64_t e); + EntityWhereFilter(const std::string & en, const EntityIds & e); + + void writeSql(AdHoc::Buffer & sql) override; + void bindParams(DB::Command * c, unsigned int & offset) override; + + EntityIds entityIds; + const std::string entityColName; + }; + } +} + +#endif + diff --git a/gentoobrowse-api/service/utils/fileUtils.cpp b/gentoobrowse-api/service/utils/fileUtils.cpp new file mode 100644 index 0000000..1446b9f --- /dev/null +++ b/gentoobrowse-api/service/utils/fileUtils.cpp @@ -0,0 +1,60 @@ +#include "fileUtils.h" +#include <fcntl.h> +#include <unistd.h> +#include <sys/mman.h> + +namespace Gentoo { + namespace Utils { + namespace File { + boost::filesystem::path operator/(const boost::filesystem::path & p, unsigned int n) + { + auto pp = p.begin(); + while (n--) ++pp; + return *pp; + } + } + + FileHandle::FileHandle(const boost::filesystem::path & path) : + fh(open(path.c_str(), O_RDONLY)) + { + if (fh < 0) { + throw std::runtime_error("Failed to open " + path.string()); + } + } + + FileHandle::~FileHandle() + { + close(fh); + } + + FileHandleStat::FileHandleStat(const boost::filesystem::path & path) : + FileHandle(path) + { + if (fstat(fh, &st)) { + throw std::runtime_error("Failed to stat " + path.string()); + } + } + + const struct stat & + FileHandleStat::getStat() const + { + return st; + } + + MemMap::MemMap(const boost::filesystem::path & path) : + FileHandleStat(path), + data(mmap(0, st.st_size, PROT_READ, MAP_SHARED, fh, 0)) + { + if (data == (void*)-1) { + throw std::runtime_error("Failed to mmap " + path.string()); + } + } + + MemMap::~MemMap() + { + munmap(data, st.st_size); + } + + } +} + diff --git a/gentoobrowse-api/service/utils/fileUtils.h b/gentoobrowse-api/service/utils/fileUtils.h new file mode 100644 index 0000000..d73b1ef --- /dev/null +++ b/gentoobrowse-api/service/utils/fileUtils.h @@ -0,0 +1,45 @@ +#ifndef GENTOOBROWSE_API_SERVICE_FILEUTILS_H +#define GENTOOBROWSE_API_SERVICE_FILEUTILS_H + +#include <boost/filesystem/path.hpp> +#include <sys/stat.h> + +namespace Gentoo { + namespace Utils { + namespace File { + boost::filesystem::path operator/(const boost::filesystem::path & p, unsigned int n); + } + + class FileHandle { + public: + FileHandle(const boost::filesystem::path & path); + virtual ~FileHandle(); + + protected: + const int fh; + }; + + class FileHandleStat : public FileHandle { + public: + FileHandleStat(const boost::filesystem::path & path); + + const struct stat & getStat() const; + + protected: + struct stat st; + }; + + class MemMap : public FileHandleStat { + public: + MemMap(const boost::filesystem::path & path); + ~MemMap(); + + void * const data; + }; + + } +} + +#endif + + diff --git a/gentoobrowse-api/service/utils/lexer.cpp b/gentoobrowse-api/service/utils/lexer.cpp new file mode 100644 index 0000000..8ed83b8 --- /dev/null +++ b/gentoobrowse-api/service/utils/lexer.cpp @@ -0,0 +1,137 @@ +#include "lexer.h" + +namespace Gentoo { + namespace Utils { + const Lexer::State Lexer::InitialState = ""; + + class Regex : public Lexer::Pattern { + public: + Regex(const Glib::ustring & pattern, GRegexCompileFlags compile, GRegexMatchFlags match) : + err(nullptr), + regex(g_regex_new(pattern.c_str(), compile, match, &err)), + info(nullptr) + { + if (!regex) { + std::runtime_error e(std::string("Failed to create GRegex: ") + err->message); + g_error_free(err); + throw e; + } + } + + ~Regex() + { + if (err) { + g_error_free(err); + } + if (info) { + g_match_info_free(info); + } + g_regex_unref(regex); + } + + bool matches(const gchar * string, size_t length, size_t position) const override + { + if (info) { + g_match_info_free(info); + } + g_regex_match_full(regex, string, length, position, G_REGEX_MATCH_ANCHORED, &info, &err); + if (err) { + std::runtime_error e(std::string("Failed to execute regex: ") + err->message); + g_error_free(err); + throw e; + } + str = string; + return g_match_info_matches(info); + } + + size_t matchedLength() const override + { + gint start, end; + g_match_info_fetch_pos(info, 0, &start, &end); + return end - start; + } + + boost::optional<Glib::ustring> match(int n) const override + { + gint start, end; + if (g_match_info_fetch_pos(info, n, &start, &end)) { + if (start == -1 && end == -1) { + return boost::optional<Glib::ustring>(); + } + return Glib::ustring(str + start, end - start); + } + return boost::optional<Glib::ustring>(); + } + + private: + mutable GError * err; + GRegex * regex; + mutable GMatchInfo * info; + mutable const gchar * str; + }; + + Lexer::PatternPtr + Lexer::regex(const Glib::ustring & pattern, GRegexCompileFlags compile, GRegexMatchFlags match) + { + return PatternPtr(new Regex(pattern, compile, match)); + } + + void + Lexer::extract(const gchar * string, size_t length) const + { + ExecuteState es; + while (es.position < length) { + const Rule * selected = nullptr; + for (const auto & r : rules) { + const auto & s = boost::get<0>(r); + if (s.find(es.getState()) == s.end()) { + continue; + } + const auto & p = boost::get<1>(r); + if (p->matches(string, length, es.position)) { + selected = &r; + break; + } + } + if (!selected) { + throw std::runtime_error(std::string("Unexpected input at ") + (string + es.position)); + } + es.pattern = boost::get<1>(*selected); + const auto & h = boost::get<2>(*selected); + h(&es); + es.position += es.pattern->matchedLength(); + } + + } + + Lexer::ExecuteState::ExecuteState() : + position(0) + { + stateStack.push_back(InitialState); + } + + void + Lexer::ExecuteState::setState(const State & s) + { + stateStack.back() = s; + } + + void + Lexer::ExecuteState::pushState(const State & s) + { + stateStack.push_back(s); + } + + void + Lexer::ExecuteState::popState() + { + stateStack.pop_back(); + } + + const Lexer::State & + Lexer::ExecuteState::getState() const + { + return stateStack.back(); + } + } +} diff --git a/gentoobrowse-api/service/utils/lexer.h b/gentoobrowse-api/service/utils/lexer.h new file mode 100644 index 0000000..44d3d57 --- /dev/null +++ b/gentoobrowse-api/service/utils/lexer.h @@ -0,0 +1,62 @@ +#ifndef GENTOOBROWSE_SERVICE_UTILS_LEXER_H +#define GENTOOBROWSE_SERVICE_UTILS_LEXER_H + +#include <vector> +#include <glibmm/ustring.h> +#include <set> +#include <boost/tuple/tuple.hpp> +#include <boost/function.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/optional.hpp> + +namespace Gentoo { + namespace Utils { + class Lexer { + public: + class Pattern { + public: + virtual ~Pattern() = default; + + virtual bool matches(const gchar *, size_t, size_t) const = 0; + virtual size_t matchedLength() const = 0; + virtual boost::optional<Glib::ustring> match(int) const = 0; + }; + typedef boost::shared_ptr<Pattern> PatternPtr; + + typedef std::string State; + typedef std::set<State> States; + + class ExecuteState { + public: + ExecuteState(); + + void pushState(const State &); + void popState(); + void setState(const State &); + const State & getState() const; + + size_t position; + PatternPtr pattern; + + private: + std::vector<State> stateStack; + }; + + typedef boost::function<void(ExecuteState *)> Handler; + typedef boost::tuple<States, PatternPtr, Handler> Rule; + typedef std::vector<Rule> Rules; + + static const State InitialState; + Rules rules; + + static PatternPtr regex(const Glib::ustring &, GRegexCompileFlags compile = (GRegexCompileFlags)0, GRegexMatchFlags match = (GRegexMatchFlags)0); + + public: + void extract(const gchar * string, size_t length) const; + }; + + } +} + +#endif + diff --git a/gentoobrowse-api/service/utils/splitEbuildProps.cpp b/gentoobrowse-api/service/utils/splitEbuildProps.cpp new file mode 100644 index 0000000..8116742 --- /dev/null +++ b/gentoobrowse-api/service/utils/splitEbuildProps.cpp @@ -0,0 +1,29 @@ +#include "splitEbuildProps.h" +#include <command.h> +#include "dbUtils.h" + +namespace Gentoo { + namespace Utils { + SplitEbuildProps::SplitEbuildProps(const std::string & ce, int e, const std::string & cp, const boost::optional<Glib::ustring> & p) : + entityId(e), + colEntityName(ce), + colPropName(cp), + props(p) + { + } + + void + SplitEbuildProps::writeSql(AdHoc::Buffer & sql) + { + sql.appendbf("(SELECT DISTINCT ?::int %s, trim(regexp_split_to_table(?, '\\s+'), '+') %s)", colEntityName, colPropName); + } + + void + SplitEbuildProps::bindParams(DB::Command * c, unsigned int & offset) + { + c->bindParamI(offset++, entityId); + Gentoo::Utils::Database::bindOptionalsS(c, offset++, { props }); + } + } +} + diff --git a/gentoobrowse-api/service/utils/splitEbuildProps.h b/gentoobrowse-api/service/utils/splitEbuildProps.h new file mode 100644 index 0000000..4739b7e --- /dev/null +++ b/gentoobrowse-api/service/utils/splitEbuildProps.h @@ -0,0 +1,25 @@ +#ifndef GENTOOBROWSE_API_SERVICE_MAINTENANCE_SPLITEBUILDPROPS_H +#define GENTOOBROWSE_API_SERVICE_MAINTENANCE_SPLITEBUILDPROPS_H + +#include <sqlWriter.h> +#include <glibmm/ustring.h> +#include <boost/optional.hpp> + +namespace Gentoo { + namespace Utils { + class SplitEbuildProps : public DB::SqlWriter { + public: + SplitEbuildProps(const std::string & ce, int e, const std::string & cp, const boost::optional<Glib::ustring> & p); + + void writeSql(AdHoc::Buffer & sql) override; + void bindParams(DB::Command * c, unsigned int & offset) override; + + const int entityId; + const std::string colEntityName, colPropName; + const boost::optional<Glib::ustring> props; + }; + } +} + +#endif + diff --git a/gentoobrowse-api/service/utils/xmlUtils.cpp b/gentoobrowse-api/service/utils/xmlUtils.cpp new file mode 100644 index 0000000..3fc3b44 --- /dev/null +++ b/gentoobrowse-api/service/utils/xmlUtils.cpp @@ -0,0 +1,26 @@ +#include "xmlUtils.h" + +namespace Gentoo { + namespace Utils { + XmlDoc::XmlDoc(const boost::filesystem::path & path) : + xmlpp::DomParser(path.string()) + { + } + + boost::optional<Glib::ustring> + XmlDoc::getXPathValue(const Glib::ustring & xp) + { + auto ns = get_document()->get_root_node()->find(xp); + if (ns.size() >= 1) { + if (auto cn = dynamic_cast<const xmlpp::ContentNode *>(ns.front())) { + return cn->get_content(); + } + } + else if (ns.size() > 1) { + throw std::logic_error("Ambiguous xpath " + xp); + } + return boost::optional<Glib::ustring>(); + } + } +} + diff --git a/gentoobrowse-api/service/utils/xmlUtils.h b/gentoobrowse-api/service/utils/xmlUtils.h new file mode 100644 index 0000000..158e134 --- /dev/null +++ b/gentoobrowse-api/service/utils/xmlUtils.h @@ -0,0 +1,20 @@ +#ifndef GENTOOBROWSE_API_SERVICE_XMLUTILS_H +#define GENTOOBROWSE_API_SERVICE_XMLUTILS_H + +#include <libxml++/parsers/domparser.h> +#include <boost/filesystem/path.hpp> +#include <boost/optional.hpp> + +namespace Gentoo { + namespace Utils { + class XmlDoc : public xmlpp::DomParser { + public: + XmlDoc(const boost::filesystem::path &); + + boost::optional<Glib::ustring> getXPathValue(const Glib::ustring &); + }; + } +} + +#endif + |