FileSearch - xtrans-1.3.5.tar.bz2

From afb5ebabf55c5c806e221ad15fca2fa0d5ef6d2b Mon Sep 17 00:00:00 2001 From: Dan Goodliffe Date: Tue, 6 Mar 2018 08:30:08 +0000 Subject: Initial commit, WIP --- .gitignore | 1 + Jamroot.jam | 56 +++++ service/Jamfile.jam | 64 ++++++ service/api.ice | 19 ++ service/apiImpl.cpp | 91 ++++++++ service/apiImpl.h | 25 ++ service/data.sql | 3 + .../filesearching/xtrans-1.3.5.tar.bz2.html | 251 +++++++++++++++++++++ .../fixtures/filesearching/zstd-1.3.3.tar.gz.html | 243 ++++++++++++++++++++ service/main.cpp | 18 ++ service/models.ice | 25 ++ service/schema.sql | 8 + service/sql/getServices.sql | 3 + service/test.cpp | 77 +++++++ slice.jam | 58 +++++ 15 files changed, 942 insertions(+) create mode 100644 .gitignore create mode 100644 Jamroot.jam create mode 100644 service/Jamfile.jam create mode 100644 service/api.ice create mode 100644 service/apiImpl.cpp create mode 100644 service/apiImpl.h create mode 100644 service/data.sql create mode 100644 service/fixtures/filesearching/xtrans-1.3.5.tar.bz2.html create mode 100644 service/fixtures/filesearching/zstd-1.3.3.tar.gz.html create mode 100644 service/main.cpp create mode 100644 service/models.ice create mode 100644 service/schema.sql create mode 100644 service/sql/getServices.sql create mode 100644 service/test.cpp create mode 100644 slice.jam diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ba077a4 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +bin diff --git a/Jamroot.jam b/Jamroot.jam new file mode 100644 index 0000000..f035381 --- /dev/null +++ b/Jamroot.jam @@ -0,0 +1,56 @@ +import os ; +import slice ; + +using gcc : : [ os.environ CXX ] ; + +variant coverage : debug ; + +project + : requirements + "-std=c++17 -fvisibility=hidden" + "-Wl,-z,defs,--warn-once,--gc-sections" + release:"-fvisibility-inlines-hidden -flto=2" + release:"-flto=2" + debug:"-W -Wall -Wextra -Werror -Wwrite-strings" + coverage:"--coverage" + coverage:"--coverage" + ; + +build-project service ; + +# Some useful aliases + +lib adhocutil : : : : /usr/include/adhocutil ; +lib slicer : : : : /usr/include/slicer ; +lib slicer-db : : : : /usr/include/slicer ; +lib netfs-api : : : : /usr/include/netfs ; +lib icetray : : : : /usr/include/icetray ; +lib dbppcore : : : : /usr/include/dbpp ; +lib Ice ; +lib IceUtil ; +lib pthread ; +lib IceBox ; +lib boost_filesystem ; +lib boost_system ; +lib boost_thread ; +lib boost_date_time ; + +lib xml2 : : : : /usr/include/libxml2 ; +lib glibmm-2.4 ; +lib gobject-2.0 ; +lib glib-2.0 ; +lib sigc-2.0 ; + +alias glibmm : : : : + /usr/include/glibmm-2.4 + /usr/lib/glibmm-2.4/include + /usr/include/glib-2.0 + /usr/lib/glib-2.0/include + /usr/include/sigc++-2.0 + /usr/lib/sigc++-2.0/include + glibmm-2.4 + gobject-2.0 + glib-2.0 + sigc-2.0 + ; + diff --git a/service/Jamfile.jam b/service/Jamfile.jam new file mode 100644 index 0000000..e7bfed9 --- /dev/null +++ b/service/Jamfile.jam @@ -0,0 +1,64 @@ +import icetray ; +import package ; +import testing ; + +lib boost_utf : : boost_unit_test_framework ; +lib dbpp-postgresql : : : : /usr/include/dbpp-postgresql ; +lib dryice : : : : /usr/include/icetray ; + +lib mirrorsearch : + [ glob *.cpp *.ice sql/*.sql : test.cpp ] + : + yes + ..//adhocutil + ..//dbppcore + ..//boost_system + ..//boost_filesystem + ..//boost_date_time + ..//Ice + ..//IceBox + ..//IceUtil + ..//pthread + ..//icetray + ..//slicer + ..//slicer-db + ..//glibmm + ..//xml2 + MirrorSearch + . + . + : : + . + ..//icetray + ; + +path-constant me : . ; + +run test.cpp + : : + data.sql + schema.sql + : + BOOST_TEST_DYN_LINK + ..//boost_system + ..//boost_filesystem + boost_utf + ROOT=\"$(me)\" + dbpp-postgresql + ..//dbppcore + ..//adhocutil + ..//boost_system + ..//boost_filesystem + ..//netfs-api + ..//IceUtil + ..//Ice + ..//IceBox + ..//pthread + dryice + mirrorsearch + mirrorsearch +; + +package.install install : : : mirrorsearch ; + + diff --git a/service/api.ice b/service/api.ice new file mode 100644 index 0000000..c207c3d --- /dev/null +++ b/service/api.ice @@ -0,0 +1,19 @@ +#ifndef MIRRORSEARCH_API +#define MIRRORSEARCH_API + +#include + +module MirrorSearch { + exception XmlError { + string msg; + }; + + interface Search { + idempotent SearchServices getServices(); + idempotent SearchHits getMatches(string filename); + idempotent optional(0) string feelingLucky(string filename); + }; +}; + +#endif + diff --git a/service/apiImpl.cpp b/service/apiImpl.cpp new file mode 100644 index 0000000..588d360 --- /dev/null +++ b/service/apiImpl.cpp @@ -0,0 +1,91 @@ +#include "apiImpl.h" + +#include +#include +#include + +#include +#include +#include +#include + +namespace MirrorSearch { + SearchImpl::SearchImpl(IceTray::DatabasePoolPtr db) : + IceTray::AbstractDatabaseClient(db), + log(LOGMANAGER()->getLogger()) + { + } + + SearchServices SearchImpl::getServices(const ::Ice::Current&) + { + return fetch(sql::getServices); + } + + typedef std::shared_ptr xmlDocSPtr; + typedef std::shared_ptr xmlXPathContextSPtr; + typedef std::shared_ptr xmlXPathObjectSPtr; + + static auto getDoc(const ::std::string & url, int flags) + { + if (auto doc = xmlDocSPtr(htmlReadFile(url.c_str(), NULL, flags), xmlFreeDoc)) { + return doc; + } + throw XmlError("Failed to open " + url); + } + + static auto getXPathCxt(const xmlDocSPtr & doc) + { + if (auto xpathCtx = xmlXPathContextSPtr(xmlXPathNewContext(doc.get()), xmlXPathFreeContext)) { + return xpathCtx; + } + throw XmlError("Failed to create xpath context"); + } + + static auto getXPathObj(const ::std::string & xpath, const xmlXPathContextSPtr & ctx, xmlXPathObjectType type) + { + if (auto xpathObj = xmlXPathObjectSPtr(xmlXPathEvalExpression(BAD_CAST xpath.c_str(), ctx.get()), xmlXPathFreeObject)) { + if (xpathObj->type != type) { + throw XmlError("Xpath evaluates to wrong type " + xpath); + } + return xpathObj; + } + throw XmlError("Failed to evaluate xpath " + xpath); + } + + void SearchImpl::callService(const ::std::string & fn, const SearchServicePtr & s, SearchHits & sh) const + { + auto fmt = AdHoc::Buffer::getFormat(s->baseurl); + auto url = (*fmt % fn).str(); + auto doc = getDoc(url, + HTML_PARSE_RECOVER | HTML_PARSE_NODEFDTD | HTML_PARSE_NOIMPLIED | + HTML_PARSE_NOWARNING | HTML_PARSE_NOERROR); + auto xpathCtx = getXPathCxt(doc); + auto xpathObj = getXPathObj(s->listxpath, xpathCtx, xmlXPathObjectType::XPATH_NODESET); + log->messagebf(LOG::INFO, "%d nodes matched %s", xpathObj->nodesetval->nodeNr, s->listxpath); + for (int row = 0; row < xpathObj->nodesetval->nodeNr; row += 1) { + xpathCtx->node = xpathObj->nodesetval->nodeTab[row]; + auto xpathObjI = getXPathObj(s->urlxpath, xpathCtx, xmlXPathObjectType::XPATH_STRING); + if (xpathObjI->stringval && *xpathObjI->stringval) { + sh.push_back(new SearchHit(0, s->id, (const char *) xpathObjI->stringval)); + } + } + } + + SearchHits SearchImpl::getMatches(const ::std::string & fn, const ::Ice::Current & c) + { + SearchHits sh; + for (const auto & s : getServices(c)) { + callService(fn, s, sh); + } + return sh; + } + + ::IceUtil::Optional<::std::string> SearchImpl::feelingLucky(const ::std::string & fn, const ::Ice::Current & c) + { + const auto ms = getMatches(fn, c); + if (ms.empty()) + return IceUtil::None; + return ms.front()->url; + } +} + diff --git a/service/apiImpl.h b/service/apiImpl.h new file mode 100644 index 0000000..e56368c --- /dev/null +++ b/service/apiImpl.h @@ -0,0 +1,25 @@ +#ifndef MIRRORSEARCH_APIIMPL_H +#define MIRRORSEARCH_APIIMPL_H + +#include +#include +#include + +namespace MirrorSearch { + class SearchImpl : public Search, public IceTray::AbstractDatabaseClient { + public: + SearchImpl(IceTray::DatabasePoolPtr); + + virtual SearchServices getServices(const ::Ice::Current& = ::Ice::Current()) override; + virtual SearchHits getMatches(const ::std::string&, const ::Ice::Current& = ::Ice::Current()) override; + virtual ::IceUtil::Optional<::std::string> feelingLucky(const ::std::string&, const ::Ice::Current& = ::Ice::Current()) override; + + private: + void callService(const ::std::string & fn, const SearchServicePtr & s, SearchHits & sh) const; + + LOG::LoggerPtr log; + }; +} + +#endif + diff --git a/service/data.sql b/service/data.sql new file mode 100644 index 0000000..e14e721 --- /dev/null +++ b/service/data.sql @@ -0,0 +1,3 @@ +INSERT INTO searchservices(name, baseurl, listxpath, urlxpath) + VALUES('file searching mock', 'file://$SCRIPTDIR/fixtures/filesearching/%s.html', '//pre[@class=''list'']/a[@class=''lf'']', 'string(@href)') + ; diff --git a/service/fixtures/filesearching/xtrans-1.3.5.tar.bz2.html b/service/fixtures/filesearching/xtrans-1.3.5.tar.bz2.html new file mode 100644 index 0000000..27b627a --- /dev/null +++ b/service/fixtures/filesearching/xtrans-1.3.5.tar.bz2.html @@ -0,0 +1,251 @@ + + +FileSearch - xtrans-1.3.5.tar.bz2 + + + + + + + + + + +

+ + + + +

+ +

+   1   182.5K ftp.cs.mun.ca/pub/mirror/gentoo/distfiles/xtrans-1.3.5.tar.bz2
+   2   182.5K ftp.dvo.ru/pub/Gentoo/distfiles/xtrans-1.3.5.tar.bz2
+   3   182.5K ftp.dvo.ru/pub/distfiles/xtrans-1.3.5.tar.bz2
+   4   182.5K ftp.fu-berlin.de/unix/X11/FTP.X.ORG/pub/individual/lib/xtrans-1.3.5.tar.bz2
+   5      536 ftp.fu-berlin.de/unix/X11/FTP.X.ORG/pub/individual/lib/xtrans-1.3.5.tar.bz2.sig
+   6   182.5K ftp.gr.debian.org/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2
+   7      536 ftp.gr.debian.org/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2.sig
+   8   182.5K ftp.linux.org.tr/gentoo/distfiles/xtrans-1.3.5.tar.bz2
+   9   182.5K ftp.ntua.gr/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2
+  10      536 ftp.ntua.gr/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2.sig
+  11   182.5K ftp.rediris.es/sites/gentoo.org/distfiles/xtrans-1.3.5.tar.bz2
+  12   182.5K ftp.tr.debian.org/gentoo/distfiles/xtrans-1.3.5.tar.bz2
+

+ + + + + +

+ + + + +

+ + +

+

+

+ + + + + + + diff --git a/service/fixtures/filesearching/zstd-1.3.3.tar.gz.html b/service/fixtures/filesearching/zstd-1.3.3.tar.gz.html new file mode 100644 index 0000000..74e70e2 --- /dev/null +++ b/service/fixtures/filesearching/zstd-1.3.3.tar.gz.html @@ -0,0 +1,243 @@ + + +FileSearch - zstd-1.3.3.tar.gz + + + + + + + + + + +

+ + + + +

+ +

+ + + + + +

+zstd-1.3.3.tar.gz not found
+Try to use search forms to refine the parameters of your request. +

+ + + + + +

+ + +

+

+

+ + + + + + + diff --git a/service/main.cpp b/service/main.cpp new file mode 100644 index 0000000..97bc89e --- /dev/null +++ b/service/main.cpp @@ -0,0 +1,18 @@ +#include +#include +#include +#include "apiImpl.h" + +namespace MirrorSearch { + class Api : public IceTray::Service { + public: + void addObjects(const std::string &, const Ice::CommunicatorPtr & ic, const Ice::StringSeq &, const Ice::ObjectAdapterPtr & adp) override + { + auto dbpool = getConnectionPool(ic, "postgresql", "MirrorSearch"); + adp->add(new SearchImpl(dbpool), ic->stringToIdentity("Search")); + } + }; + + NAMEDFACTORY("default", MirrorSearch::Api, IceTray::ServiceFactory); +} + diff --git a/service/models.ice b/service/models.ice new file mode 100644 index 0000000..cb48459 --- /dev/null +++ b/service/models.ice @@ -0,0 +1,25 @@ +#ifndef MIRRORSEARCH_MODELS +#define MIRRORSEARCH_MODELS + +module MirrorSearch { + class SearchService { + ["slicer:db:pkey"] + int id; + string name; + string baseurl; + string listxpath; + string urlxpath; + }; + class SearchHit { + ["slicer:db:pkey"] + int id; + int serviceid; + string url; + }; + + sequence SearchServices; + sequence SearchHits; +}; + +#endif + diff --git a/service/schema.sql b/service/schema.sql new file mode 100644 index 0000000..b529172 --- /dev/null +++ b/service/schema.sql @@ -0,0 +1,8 @@ +CREATE TABLE searchservices( + id int GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + name text not null, + baseurl text not null, + listxpath text not null, + urlxpath text not null +); + diff --git a/service/sql/getServices.sql b/service/sql/getServices.sql new file mode 100644 index 0000000..40382f2 --- /dev/null +++ b/service/sql/getServices.sql @@ -0,0 +1,3 @@ +SELECT id, name, baseurl, listxpath, urlxpath +FROM searchservices +ORDER BY id diff --git a/service/test.cpp b/service/test.cpp new file mode 100644 index 0000000..c115712 --- /dev/null +++ b/service/test.cpp @@ -0,0 +1,77 @@ +#define BOOST_TEST_MODULE MirrorSearch +#include + +#include +#include +#include +#include + +class Service : PQ::Mock, public IceTray::DryIce { + public: + Service() : PQ::Mock("user=postgres", "MirrorSearch", { + rootDir / "schema.sql", + rootDir / "data.sql" + }) { } + +}; + +class TestClient : public IceTray::DryIceClient { + public: + TestClient() : + s(getProxy("Search")) + { + } + + MirrorSearch::SearchPrx s; +}; + +BOOST_TEST_GLOBAL_FIXTURE(Service); + +BOOST_FIXTURE_TEST_SUITE(tc, TestClient); + +BOOST_AUTO_TEST_CASE(sanity) +{ + BOOST_REQUIRE(s); + s->ice_ping(); +} + +BOOST_AUTO_TEST_CASE(getServices) +{ + auto ss = s->getServices(); + BOOST_REQUIRE_EQUAL(ss.size(), 1); + BOOST_CHECK_EQUAL(ss.front()->id, 1); + BOOST_CHECK_EQUAL(ss.front()->name, "file searching mock"); + BOOST_CHECK_NE(ss.front()->baseurl, "file://$SCRIPTDIR/fixtures/filesearching/%s.html"); + BOOST_CHECK_EQUAL(ss.front()->baseurl.substr(0, 8), "file:///"); + BOOST_CHECK(!ss.front()->listxpath.empty()); + BOOST_CHECK(!ss.front()->urlxpath.empty()); +} + +BOOST_AUTO_TEST_CASE(getMatches_zstd_notfound) +{ + auto ms = s->getMatches("zstd-1.3.3.tar.gz"); + BOOST_REQUIRE(ms.empty()); +} + +BOOST_AUTO_TEST_CASE(getMatches_xtrans) +{ + auto ms = s->getMatches("xtrans-1.3.5.tar.bz2"); + BOOST_REQUIRE_EQUAL(ms.size(), 12); + BOOST_REQUIRE_EQUAL(ms.front()->url, "ftp://ftp.cs.mun.ca/pub/mirror/gentoo/distfiles/xtrans-1.3.5.tar.bz2"); + BOOST_REQUIRE_EQUAL(ms.back()->url, "ftp://ftp.tr.debian.org/gentoo/distfiles/xtrans-1.3.5.tar.bz2"); +} + +BOOST_AUTO_TEST_CASE(getMatches_zstd_notfound_lucky) +{ + BOOST_REQUIRE(!s->feelingLucky("zstd-1.3.3.tar.gz")); +} + +BOOST_AUTO_TEST_CASE(getMatches_xtrans_lucky) +{ + auto fl = s->feelingLucky("xtrans-1.3.5.tar.bz2"); + BOOST_REQUIRE(fl); + BOOST_REQUIRE_EQUAL(*fl, "ftp://ftp.cs.mun.ca/pub/mirror/gentoo/distfiles/xtrans-1.3.5.tar.bz2"); +} + +BOOST_AUTO_TEST_SUITE_END(); + diff --git a/slice.jam b/slice.jam new file mode 100644 index 0000000..434e562 --- /dev/null +++ b/slice.jam @@ -0,0 +1,58 @@ +import type : register ; +import generators : register-standard ; +import type ; +import feature : feature ; +import scanner ; +import toolset ; + +type.register SLICE : ice ; + +feature slicer : no yes pure ; +feature allow-ice : no yes ; +feature ice-visibility : public hidden ; + +class slice-scanner : common-scanner +{ + rule pattern ( ) + { + return "^[ \t]*#[ \t]*include[ ]*[<\"]([^>\"]*)[>\"]" ; + } +} + +scanner.register slice-scanner : include ; + +type.set-scanner SLICE : slice-scanner ; + +generators.register-standard slice.slice2cpp : SLICE : CPP H : no ; +generators.register-standard slice.slicer : SLICE : CPP CPP(slicer-%) H : yes ; +generators.register-standard slice.slicer.pure : SLICE : CPP(slicer-%) : pure ; + +toolset.flags slice.slice2cpp INCLUDES ; +toolset.flags slice.slice2cpp DLLEXPORT public : --dll-export JAM_DLL_PUBLIC ; +toolset.flags slice.slice2cpp ALLOWICE yes : --ice ; +toolset.flags slice.slicer INCLUDES ; +toolset.flags slice.slicer DLLEXPORT public : --dll-export JAM_DLL_PUBLIC ; +toolset.flags slice.slicer ALLOWICE yes : --ice ; +toolset.flags slice.slicer.pure INCLUDES ; +toolset.flags slice.slicer.pure ALLOWICE yes : --ice ; + +actions slice.slice2cpp +{ + slice2cpp -I"$(INCLUDES)" --checksum --output-dir $(1[1]:D) $(2) $(DLLEXPORT) $(ALLOWICE[1]) +} + +actions slice.slicer +{ + slice2cpp -I"$(INCLUDES)" --checksum --output-dir $(1[1]:D) $(2) $(DLLEXPORT) $(ALLOWICE[1]) + slicer -I"$(INCLUDES)" $(2) $(1[2]) $(ALLOWICE[1]) +} + +actions slice.slicer.pure +{ + slicer -I"$(INCLUDES)" $(2) $(1[1]) $(ALLOWICE[1]) +} + +IMPORT $(__name__) : slice.slice2cpp : : slice.slice2cpp ; +IMPORT $(__name__) : slice.slicer : : slice.slicer ; +IMPORT $(__name__) : slice.slicer.pure : : slice.slicer.pure ; + -- cgit v1.2.3