diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Jamroot.jam | 56 | ||||
| -rw-r--r-- | service/Jamfile.jam | 64 | ||||
| -rw-r--r-- | service/api.ice | 19 | ||||
| -rw-r--r-- | service/apiImpl.cpp | 91 | ||||
| -rw-r--r-- | service/apiImpl.h | 25 | ||||
| -rw-r--r-- | service/data.sql | 3 | ||||
| -rw-r--r-- | service/fixtures/filesearching/xtrans-1.3.5.tar.bz2.html | 251 | ||||
| -rw-r--r-- | service/fixtures/filesearching/zstd-1.3.3.tar.gz.html | 243 | ||||
| -rw-r--r-- | service/main.cpp | 18 | ||||
| -rw-r--r-- | service/models.ice | 25 | ||||
| -rw-r--r-- | service/schema.sql | 8 | ||||
| -rw-r--r-- | service/sql/getServices.sql | 3 | ||||
| -rw-r--r-- | service/test.cpp | 77 | ||||
| -rw-r--r-- | slice.jam | 58 |
15 files changed, 942 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ba077a4 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +bin diff --git a/Jamroot.jam b/Jamroot.jam new file mode 100644 index 0000000..f035381 --- /dev/null +++ b/Jamroot.jam @@ -0,0 +1,56 @@ +import os ; +import slice ; + +using gcc : : [ os.environ CXX ] ; + +variant coverage : debug ; + +project + : requirements + <cxxflags>"-std=c++17 -fvisibility=hidden" + <linkflags>"-Wl,-z,defs,--warn-once,--gc-sections" + <variant>release:<cxxflags>"-fvisibility-inlines-hidden -flto=2" + <variant>release:<linkflags>"-flto=2" + <variant>debug:<cxxflags>"-W -Wall -Wextra -Werror -Wwrite-strings" + <variant>coverage:<cxxflags>"--coverage" + <variant>coverage:<linkflags>"--coverage" + ; + +build-project service ; + +# Some useful aliases + +lib adhocutil : : : : <include>/usr/include/adhocutil ; +lib slicer : : : : <include>/usr/include/slicer ; +lib slicer-db : : : : <include>/usr/include/slicer ; +lib netfs-api : : : : <include>/usr/include/netfs ; +lib icetray : : : : <include>/usr/include/icetray ; +lib dbppcore : : : : <include>/usr/include/dbpp ; +lib Ice ; +lib IceUtil ; +lib pthread ; +lib IceBox ; +lib boost_filesystem ; +lib boost_system ; +lib boost_thread ; +lib boost_date_time ; + +lib xml2 : : : : <include>/usr/include/libxml2 ; +lib glibmm-2.4 ; +lib gobject-2.0 ; +lib glib-2.0 ; +lib sigc-2.0 ; + +alias glibmm : : : : + <include>/usr/include/glibmm-2.4 + <include>/usr/lib/glibmm-2.4/include + <include>/usr/include/glib-2.0 + <include>/usr/lib/glib-2.0/include + <include>/usr/include/sigc++-2.0 + <include>/usr/lib/sigc++-2.0/include + <library>glibmm-2.4 + <library>gobject-2.0 + <library>glib-2.0 + <library>sigc-2.0 + ; + diff --git a/service/Jamfile.jam b/service/Jamfile.jam new file mode 100644 index 0000000..e7bfed9 --- /dev/null +++ b/service/Jamfile.jam @@ -0,0 +1,64 @@ +import icetray ; +import package ; +import testing ; + +lib boost_utf : : <name>boost_unit_test_framework ; +lib dbpp-postgresql : : : : <include>/usr/include/dbpp-postgresql ; +lib dryice : : : : <include>/usr/include/icetray ; + +lib mirrorsearch : + [ glob *.cpp *.ice sql/*.sql : test.cpp ] + : + <slicer>yes + <library>..//adhocutil + <library>..//dbppcore + <library>..//boost_system + <library>..//boost_filesystem + <library>..//boost_date_time + <library>..//Ice + <library>..//IceBox + <library>..//IceUtil + <library>..//pthread + <library>..//icetray + <library>..//slicer + <library>..//slicer-db + <library>..//glibmm + <library>..//xml2 + <icetray.sql.namespace>MirrorSearch + <icetray.sql.basedir>. + <include>. + : : + <include>. + <library>..//icetray + ; + +path-constant me : . ; + +run test.cpp + : : + data.sql + schema.sql + : + <define>BOOST_TEST_DYN_LINK + <library>..//boost_system + <library>..//boost_filesystem + <library>boost_utf + <define>ROOT=\"$(me)\" + <library>dbpp-postgresql + <library>..//dbppcore + <library>..//adhocutil + <library>..//boost_system + <library>..//boost_filesystem + <library>..//netfs-api + <library>..//IceUtil + <library>..//Ice + <library>..//IceBox + <library>..//pthread + <library>dryice + <library>mirrorsearch + <implicit-dependency>mirrorsearch +; + +package.install install : : : mirrorsearch ; + + diff --git a/service/api.ice b/service/api.ice new file mode 100644 index 0000000..c207c3d --- /dev/null +++ b/service/api.ice @@ -0,0 +1,19 @@ +#ifndef MIRRORSEARCH_API +#define MIRRORSEARCH_API + +#include <models.ice> + +module MirrorSearch { + exception XmlError { + string msg; + }; + + interface Search { + idempotent SearchServices getServices(); + idempotent SearchHits getMatches(string filename); + idempotent optional(0) string feelingLucky(string filename); + }; +}; + +#endif + diff --git a/service/apiImpl.cpp b/service/apiImpl.cpp new file mode 100644 index 0000000..588d360 --- /dev/null +++ b/service/apiImpl.cpp @@ -0,0 +1,91 @@ +#include "apiImpl.h" + +#include <sql/getServices.sql.h> +#include <buffer.h> +#include <memory> + +#include <libxml/xpath.h> +#include <libxml/xpathInternals.h> +#include <libxml/HTMLparser.h> +#include <libxml/HTMLtree.h> + +namespace MirrorSearch { + SearchImpl::SearchImpl(IceTray::DatabasePoolPtr db) : + IceTray::AbstractDatabaseClient(db), + log(LOGMANAGER()->getLogger<SearchImpl>()) + { + } + + SearchServices SearchImpl::getServices(const ::Ice::Current&) + { + return fetch<SearchServices>(sql::getServices); + } + + typedef std::shared_ptr<xmlDoc> xmlDocSPtr; + typedef std::shared_ptr<xmlXPathContext> xmlXPathContextSPtr; + typedef std::shared_ptr<xmlXPathObject> xmlXPathObjectSPtr; + + static auto getDoc(const ::std::string & url, int flags) + { + if (auto doc = xmlDocSPtr(htmlReadFile(url.c_str(), NULL, flags), xmlFreeDoc)) { + return doc; + } + throw XmlError("Failed to open " + url); + } + + static auto getXPathCxt(const xmlDocSPtr & doc) + { + if (auto xpathCtx = xmlXPathContextSPtr(xmlXPathNewContext(doc.get()), xmlXPathFreeContext)) { + return xpathCtx; + } + throw XmlError("Failed to create xpath context"); + } + + static auto getXPathObj(const ::std::string & xpath, const xmlXPathContextSPtr & ctx, xmlXPathObjectType type) + { + if (auto xpathObj = xmlXPathObjectSPtr(xmlXPathEvalExpression(BAD_CAST xpath.c_str(), ctx.get()), xmlXPathFreeObject)) { + if (xpathObj->type != type) { + throw XmlError("Xpath evaluates to wrong type " + xpath); + } + return xpathObj; + } + throw XmlError("Failed to evaluate xpath " + xpath); + } + + void SearchImpl::callService(const ::std::string & fn, const SearchServicePtr & s, SearchHits & sh) const + { + auto fmt = AdHoc::Buffer::getFormat(s->baseurl); + auto url = (*fmt % fn).str(); + auto doc = getDoc(url, + HTML_PARSE_RECOVER | HTML_PARSE_NODEFDTD | HTML_PARSE_NOIMPLIED | + HTML_PARSE_NOWARNING | HTML_PARSE_NOERROR); + auto xpathCtx = getXPathCxt(doc); + auto xpathObj = getXPathObj(s->listxpath, xpathCtx, xmlXPathObjectType::XPATH_NODESET); + log->messagebf(LOG::INFO, "%d nodes matched %s", xpathObj->nodesetval->nodeNr, s->listxpath); + for (int row = 0; row < xpathObj->nodesetval->nodeNr; row += 1) { + xpathCtx->node = xpathObj->nodesetval->nodeTab[row]; + auto xpathObjI = getXPathObj(s->urlxpath, xpathCtx, xmlXPathObjectType::XPATH_STRING); + if (xpathObjI->stringval && *xpathObjI->stringval) { + sh.push_back(new SearchHit(0, s->id, (const char *) xpathObjI->stringval)); + } + } + } + + SearchHits SearchImpl::getMatches(const ::std::string & fn, const ::Ice::Current & c) + { + SearchHits sh; + for (const auto & s : getServices(c)) { + callService(fn, s, sh); + } + return sh; + } + + ::IceUtil::Optional<::std::string> SearchImpl::feelingLucky(const ::std::string & fn, const ::Ice::Current & c) + { + const auto ms = getMatches(fn, c); + if (ms.empty()) + return IceUtil::None; + return ms.front()->url; + } +} + diff --git a/service/apiImpl.h b/service/apiImpl.h new file mode 100644 index 0000000..e56368c --- /dev/null +++ b/service/apiImpl.h @@ -0,0 +1,25 @@ +#ifndef MIRRORSEARCH_APIIMPL_H +#define MIRRORSEARCH_APIIMPL_H + +#include <api.h> +#include <abstractDatabaseClient.h> +#include <logger.h> + +namespace MirrorSearch { + class SearchImpl : public Search, public IceTray::AbstractDatabaseClient { + public: + SearchImpl(IceTray::DatabasePoolPtr); + + virtual SearchServices getServices(const ::Ice::Current& = ::Ice::Current()) override; + virtual SearchHits getMatches(const ::std::string&, const ::Ice::Current& = ::Ice::Current()) override; + virtual ::IceUtil::Optional<::std::string> feelingLucky(const ::std::string&, const ::Ice::Current& = ::Ice::Current()) override; + + private: + void callService(const ::std::string & fn, const SearchServicePtr & s, SearchHits & sh) const; + + LOG::LoggerPtr log; + }; +} + +#endif + diff --git a/service/data.sql b/service/data.sql new file mode 100644 index 0000000..e14e721 --- /dev/null +++ b/service/data.sql @@ -0,0 +1,3 @@ +INSERT INTO searchservices(name, baseurl, listxpath, urlxpath) + VALUES('file searching mock', 'file://$SCRIPTDIR/fixtures/filesearching/%s.html', '//pre[@class=''list'']/a[@class=''lf'']', 'string(@href)') + ; diff --git a/service/fixtures/filesearching/xtrans-1.3.5.tar.bz2.html b/service/fixtures/filesearching/xtrans-1.3.5.tar.bz2.html new file mode 100644 index 0000000..27b627a --- /dev/null +++ b/service/fixtures/filesearching/xtrans-1.3.5.tar.bz2.html @@ -0,0 +1,251 @@ +<HTML> +<HEAD> +<TITLE>FileSearch - xtrans-1.3.5.tar.bz2</TITLE> +<link rel=stylesheet href=/styles/fs.css> +</HEAD> +<BODY bgcolor=#FFFFFF leftmargin=0 topmargin=0 marginwidth=0 marginheight=0> +<FORM ACTION=/cgi-bin/s METHOD=GET> +<TABLE width=100% border=0 cellspacing=0 cellpadding=0> + <TR bgcolor=#CCA782 valign=bottom align=left> + <TD colspan=2 height=87> + <TABLE width=100% border=0 cellspacing=0 cellpadding=0> + <TR> + <TD width=160 align=right valign=bottom><a href=/><IMG src=/img/b.gif width=160 height=10 border=0><img src=/img/filesearchen.gif width=142 height=77 border=0 valign=bottom></a></TD> + <TD valign=bottom width=100%> + <TABLE width=100% border=0 cellspacing=0 cellpadding=0> + <TR> + <td bgcolor=#CCA782 valign=bottom class=text align=center><img src=/img/b.gif height=1 width=10></td> + <td rowspan=2 background=/img/poisklght.gif bgcolor=#CCA782 valign=bottom class=taplight align=center><img src=/img/b.gif height=1 width=73><br>search</td> + <td rowspan=2 background=/img/advpoiskdrk.gif bgcolor=#CCA782 valign=bottom align=center width=135 class=taplight nowrap><img src=/img/b.gif height=1 width=135><br><a href=/advanced/ class=tapdark>advanced search</a></td> + <td align=right class=text width=100%>...</td> + <td align=right><img src=/img/b.gif width=1 height=21></td> + </TR> + <TR> + <TD bgcolor=#000000 valign=bottom class=text align=center><img src=/img/b.gif height=1 width=10></TD> + <TD align=right bgcolor=#000000 colspan=2><IMG src=/img/b.gif width=274 height=1></TD> + </TR> + </TABLE> + <TABLE border=0 cellpadding=0 cellspacing=0 width=100%> + <TR> + <td width=1 bgcolor=#000000 ><img src=/img/b.gif width=1 height=48></td> + <td bgcolor=#FFE7CE valign=middle nowrap> + <INPUT type=text name=q value="xtrans-1.3.5.tar.bz2" SIZE=20> + <select name=t> + <option VALUE=f selected>file/directory + <option VALUE=m>music (mp3) + <option VALUE=p>images + <option VALUE=v>video + <option VALUE=s>server + </select> + <select name=d> +<option value="">all sites +<option value="com">.com +<option value="net">.net +<option value="edu">.edu +<option value="org">.org +<option value="au">.au +<option value="be">.be +<option value="ca">.ca +<option value="ch">.ch +<option value="de">.de +<option value="dk">.dk +<option value="es">.es +<option value="fr">.fr +<option value="it">.it +<option value="jp">.jp +<option value="lv">.lv +<option value="nl">.nl +<option value="no">.no +<option value="ru,su">.ru +<option value="se">.se +<option value="ua">.ua +<option value="uk">.uk + + </select> + <img src=/img/b.gif height=1 width=5></td> + <td bgcolor=#FFE7CE valign=middle width=630> + <input type=image src=/img/findengl.gif border=0> + </td> + <td width=1 bgcolor=#000000 ><img src=/img/b.gif width=1 height=46></td> + </TR> + <TR> + <TD rowspan=1 colspan=4 bgcolor=#000000><IMG src=/img/b.gif width=3 height=1 border=0></TD> + </TR> + </TABLE> + <img src=/img/b.gif width=10 height=6></TD> + <TD valign=bottom width=20><IMG src=/img/b.gif width=20 height=8></TD> + </TR> + </TABLE> + </TD> + </TR> + <TR valign=center align=center> + <TD colspan=2 background=/img/inside-2x1.gif> +<!--ENGLISH TOP--><div id="RTBDIV_50381">
+ <div id="RTBPL_50381">
+ <a href="//rtbsystem.com/ru/advertiser/request">Добавить рекламное обьявление</a>
+ </div>
+</div>
+<script type="text/javascript">
+ if (document.getElementById('RTBDIV_50381')) {
+ document.write('<scr'+'ipt type="text/javascript" async '
+ +'src="//code.rtbsystem.com/50381.js?t='+ new Date().getTime() + '" charset="utf-8" ></scr'+'ipt>');
+ }
+</script><!-- Start Counter --> +<!-- SpyLOG v2 f:0211 --> +<script language="javascript"> +u="u311.79.spylog.com";d=document;nv=navigator;na=nv.appName;t="";p=1; +sz=" width=1 height=1 "; +hl=history.length;d.cookie="b=b";c=0; +bv=Math.round(parseFloat(nv.appVersion)*100); +if (d.cookie) c=1;n=(na.substring(0,2)=="Mi")?0:1; +if((n==0)||(bv >= 300)){rn=Math.random();t=(new Date()).getTimezoneOffset();} else {rn=0;} +z="p="+p+"&rn="+rn+"&t="+t+"&c="+c+"&hl="+hl; +if (self != top) { fr=1;} else { fr=0;} +r=escape(d.referrer);r1=""; +sl="1.0";h=0; +</script> +<script language="javascript1.1"> +pl="";sl="1.1"; +if((n==1) && (bv >= 300)) +{ for(var i = 0; i < nv.plugins.length; i++) +pl += nv.plugins[i].name+":"; } +j = (navigator.javaEnabled() ? "Y" : "N"); +</script> +<script language=javascript1.2> +sl="1.2";s=screen;wh=s.width+'x'+s.height; +px=(n==0)?screen.colorDepth:screen.pixelDepth;z+="&wh="+wh+"&px="+px; +</script> +<script language=javascript1.3> +sl="1.3" +</script> +<script language="javascript"> +y=""; +y+="<a href='http://"+u+"/cnt?f=3&p="+p+"&rn="+rn+"' target=_blank>"; +y+="<img src='http://"+u+"/cnt?"; +y+=z+"&j="+j+"&sl="+sl+"&r="+r+"&r1="+r1+"&fr="+fr+"&pg="+escape(window.location.href)+"&pl="+escape(pl); +y+="' border=0 "+sz+" alt='SpyLOG' align=right>"; +y+="</a>"; +d.write(y); +</script> +<script language="javascript1.2"><!-- +if (n == 0) { d.write("<");d.write("!--"); } +//--></script> +<noscript> +<a href="http://u311.79.spylog.com/cnt?f=3&p=1" target=_blank> +<img src="http://u311.79.spylog.com/cnt?p=1" alt='SpyLOG' border='0' width=1 height=1 align=right> +</a> +</noscript> +<script language="javascript1.2"><!-- +if (n == 0) { d.write("--");d.write(">"); } +//--></script> +<!-- SpyLOG --> +<!-- End Counter --> +</TD> +</TR> +</table> +<input type=hidden name=l value=en> +</form> + + +<table border=0> +<tr> +<td valign=top > + +<script type="text/javascript"><!-- +google_ad_client = "pub-7005770717406130"; +google_ad_width = 160; +google_ad_height = 600; +google_ad_format = "160x600_as"; +google_ad_type = "text_image"; +//2006-12-05: filesearch_left_160x600 +google_ad_channel = "8508766014"; +google_color_border = "FFFFFF"; +google_color_bg = "FFFFFF"; +google_color_link = "000000"; +google_color_text = "000000"; +google_color_url = "000000"; +//--></script> +<script type="text/javascript" + src="http://pagead2.googlesyndication.com/pagead/show_ads.js"> +</script> + +</td> + +<td valign=top > + +<pre class=list> + 1 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.cs.mun.ca/ class=ls>ftp.cs.mun.ca</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.cs.mun.ca/pub/mirror/gentoo/distfiles class=lg>/pub/mirror/gentoo/distfiles/</a><a href=ftp://ftp.cs.mun.ca/pub/mirror/gentoo/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a> + 2 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.dvo.ru/ class=ls>ftp.dvo.ru</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.dvo.ru/pub/Gentoo/distfiles class=lg>/pub/Gentoo/distfiles/</a><a href=ftp://ftp.dvo.ru/pub/Gentoo/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a> + 3 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.dvo.ru/ class=ls>ftp.dvo.ru</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.dvo.ru/pub/distfiles class=lg>/pub/distfiles/</a><a href=ftp://ftp.dvo.ru/pub/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a> + 4 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.fu-berlin.de/ class=ls>ftp.fu-berlin.de</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.fu-berlin.de/unix/X11/FTP.X.ORG/pub/individual/lib class=lg>/unix/X11/FTP.X.ORG/pub/individual/lib/</a><a href=ftp://ftp.fu-berlin.de/unix/X11/FTP.X.ORG/pub/individual/lib/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a> + 5 <img src=/img/iconany.gif width=16 height=16> <b> 536</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.fu-berlin.de/ class=ls>ftp.fu-berlin.de</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.fu-berlin.de/unix/X11/FTP.X.ORG/pub/individual/lib class=lg>/unix/X11/FTP.X.ORG/pub/individual/lib/</a><a href=ftp://ftp.fu-berlin.de/unix/X11/FTP.X.ORG/pub/individual/lib/xtrans-1.3.5.tar.bz2.sig class=lf>xtrans-1.3.5.tar.bz2.sig</a> + 6 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.gr.debian.org/ class=ls>ftp.gr.debian.org</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.gr.debian.org/pub/X11/X.org/individual/lib class=lg>/pub/X11/X.org/individual/lib/</a><a href=ftp://ftp.gr.debian.org/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a> + 7 <img src=/img/iconany.gif width=16 height=16> <b> 536</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.gr.debian.org/ class=ls>ftp.gr.debian.org</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.gr.debian.org/pub/X11/X.org/individual/lib class=lg>/pub/X11/X.org/individual/lib/</a><a href=ftp://ftp.gr.debian.org/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2.sig class=lf>xtrans-1.3.5.tar.bz2.sig</a> + 8 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.linux.org.tr/ class=ls>ftp.linux.org.tr</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.linux.org.tr/gentoo/distfiles class=lg>/gentoo/distfiles/</a><a href=ftp://ftp.linux.org.tr/gentoo/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a> + 9 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.ntua.gr/ class=ls>ftp.ntua.gr</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.ntua.gr/pub/X11/X.org/individual/lib class=lg>/pub/X11/X.org/individual/lib/</a><a href=ftp://ftp.ntua.gr/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a> + 10 <img src=/img/iconany.gif width=16 height=16> <b> 536</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.ntua.gr/ class=ls>ftp.ntua.gr</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.ntua.gr/pub/X11/X.org/individual/lib class=lg>/pub/X11/X.org/individual/lib/</a><a href=ftp://ftp.ntua.gr/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2.sig class=lf>xtrans-1.3.5.tar.bz2.sig</a> + 11 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.rediris.es/ class=ls>ftp.rediris.es</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.rediris.es/sites/gentoo.org/distfiles class=lg>/sites/gentoo.org/distfiles/</a><a href=ftp://ftp.rediris.es/sites/gentoo.org/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a> + 12 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.tr.debian.org/ class=ls>ftp.tr.debian.org</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.tr.debian.org/gentoo/distfiles class=lg>/gentoo/distfiles/</a><a href=ftp://ftp.tr.debian.org/gentoo/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a> +</pre> + + +<script type="text/javascript"><!-- +google_ad_client = "pub-7005770717406130"; +/* filesearch_down_728x90 */ +google_ad_slot = "6424137562"; +google_ad_width = 728; +google_ad_height = 90; +//--> +</script> +<script type="text/javascript" +src="http://pagead2.googlesyndication.com/pagead/show_ads.js"> +</script> + + </td></tr></table> +<table width=100% border=0 cellspacing=0 cellpadding=0 bgcolor=#FAE6CA> +<tr bgcolor=#000000><td><img src=/img/b.gif height=1 width=1></td></tr> +<tr> +<td background=/img/inside-4x1-1x6_1.gif><img src=/img/b.gif height=20 width=20></td> +</tr></table> +<table width=100% border=0 cellspacing=0 cellpadding=0 bgcolor=#FAE6CA> + <tr><td align=center> +<!--ENGLISH BOT--><br><br> +</td></tr> +</table> + +<!--Kavanga START--> + +<script type="text/javascript"><!-- +if (Math.ceil(Math.random()*10) ) { + <!--Kavanga START--> + <!--Сайт: filesearch.ru--> + <!--Категория: Компьютеры и ПО--> + if (typeof(pr) == 'undefined') { var pr = Math.floor(Math.random() * 1000000); } + if (typeof(document.referrer) != 'undefined') { + if (typeof(afReferrer) == 'undefined') { + afReferrer = escape(document.referrer); + } + } else { + afReferrer = ''; + } + var addate = new Date(); + + if (Math.ceil(Math.random()*10) ) { + <!--Тип баннера: Pop Under--> + document.write('<scr' + 'ipt type="text/javascript" src="http://a.kavanga.ru/3604/prepareCode?ph=b&p1=our&p2=o&pucn=a&pfc=a&pfb=a&plp=a&pli=a&pop=a&pr=' + pr + '&pt=b&pd=' + addate.getDate() + '&pw=' + addate.getDay() + '&pv=' + addate.getHours() + '&prr=' + afReferrer + '"><\/scr' + 'ipt>'); + } else { + <!--Тип баннера: Rich Media--> + document.write('<scr' + 'ipt type="text/javascript" src="http://a.kavanga.ru/3604/prepareCode?p1=oip&p2=p&pucn=a&pfc=a&pfb=a&plp=a&pli=a&pop=a&pr=' + pr +'&pt=b&pd=' + addate.getDate() + '&pw=' + addate.getDay() + '&pv=' + addate.getHours() + '&py=a&prr=' + afReferrer + '"><\/scr' + 'ipt>'); + } + <!--Kavanga END--> +} else { + <!-- AdMedia START --> + var RndNum4NoCash = Math.round(Math.random() * 1000000000); + document.write('<script language="JavaScript" src="http://ad.adriver.ru/cgi-bin/erle.cgi?sid=36667&target=top&bt=16&pz=0&rnd=' + RndNum4NoCash + '"><\/script>'); + <!-- AdMedia END --> +} +// --> +</script> + +<!--Kavanga END--> +</BODY></HTML> diff --git a/service/fixtures/filesearching/zstd-1.3.3.tar.gz.html b/service/fixtures/filesearching/zstd-1.3.3.tar.gz.html new file mode 100644 index 0000000..74e70e2 --- /dev/null +++ b/service/fixtures/filesearching/zstd-1.3.3.tar.gz.html @@ -0,0 +1,243 @@ +<HTML> +<HEAD> +<TITLE>FileSearch - zstd-1.3.3.tar.gz</TITLE> +<link rel=stylesheet href=/styles/fs.css> +</HEAD> +<BODY bgcolor=#FFFFFF leftmargin=0 topmargin=0 marginwidth=0 marginheight=0> +<FORM ACTION=/cgi-bin/s METHOD=GET> +<TABLE width=100% border=0 cellspacing=0 cellpadding=0> + <TR bgcolor=#CCA782 valign=bottom align=left> + <TD colspan=2 height=87> + <TABLE width=100% border=0 cellspacing=0 cellpadding=0> + <TR> + <TD width=160 align=right valign=bottom><a href=/><IMG src=/img/b.gif width=160 height=10 border=0><img src=/img/filesearchen.gif width=142 height=77 border=0 valign=bottom></a></TD> + <TD valign=bottom width=100%> + <TABLE width=100% border=0 cellspacing=0 cellpadding=0> + <TR> + <td bgcolor=#CCA782 valign=bottom class=text align=center><img src=/img/b.gif height=1 width=10></td> + <td rowspan=2 background=/img/poisklght.gif bgcolor=#CCA782 valign=bottom class=taplight align=center><img src=/img/b.gif height=1 width=73><br>search</td> + <td rowspan=2 background=/img/advpoiskdrk.gif bgcolor=#CCA782 valign=bottom align=center width=135 class=taplight nowrap><img src=/img/b.gif height=1 width=135><br><a href=/advanced/ class=tapdark>advanced search</a></td> + <td align=right class=text width=100%>...</td> + <td align=right><img src=/img/b.gif width=1 height=21></td> + </TR> + <TR> + <TD bgcolor=#000000 valign=bottom class=text align=center><img src=/img/b.gif height=1 width=10></TD> + <TD align=right bgcolor=#000000 colspan=2><IMG src=/img/b.gif width=274 height=1></TD> + </TR> + </TABLE> + <TABLE border=0 cellpadding=0 cellspacing=0 width=100%> + <TR> + <td width=1 bgcolor=#000000 ><img src=/img/b.gif width=1 height=48></td> + <td bgcolor=#FFE7CE valign=middle nowrap> + <INPUT type=text name=q value="zstd-1.3.3.tar.gz" SIZE=20> + <select name=t> + <option VALUE=f selected>file/directory + <option VALUE=m>music (mp3) + <option VALUE=p>images + <option VALUE=v>video + <option VALUE=s>server + </select> + <select name=d> +<option value="">all sites +<option value="com">.com +<option value="net">.net +<option value="edu">.edu +<option value="org">.org +<option value="au">.au +<option value="be">.be +<option value="ca">.ca +<option value="ch">.ch +<option value="de">.de +<option value="dk">.dk +<option value="es">.es +<option value="fr">.fr +<option value="it">.it +<option value="jp">.jp +<option value="lv">.lv +<option value="nl">.nl +<option value="no">.no +<option value="ru,su">.ru +<option value="se">.se +<option value="ua">.ua +<option value="uk">.uk + + </select> + <img src=/img/b.gif height=1 width=5></td> + <td bgcolor=#FFE7CE valign=middle width=630> + <input type=image src=/img/findengl.gif border=0> + </td> + <td width=1 bgcolor=#000000 ><img src=/img/b.gif width=1 height=46></td> + </TR> + <TR> + <TD rowspan=1 colspan=4 bgcolor=#000000><IMG src=/img/b.gif width=3 height=1 border=0></TD> + </TR> + </TABLE> + <img src=/img/b.gif width=10 height=6></TD> + <TD valign=bottom width=20><IMG src=/img/b.gif width=20 height=8></TD> + </TR> + </TABLE> + </TD> + </TR> + <TR valign=center align=center> + <TD colspan=2 background=/img/inside-2x1.gif> +<!--ENGLISH TOP--><div id="RTBDIV_50381">
+ <div id="RTBPL_50381">
+ <a href="//rtbsystem.com/ru/advertiser/request">Добавить рекламное обьявление</a>
+ </div>
+</div>
+<script type="text/javascript">
+ if (document.getElementById('RTBDIV_50381')) {
+ document.write('<scr'+'ipt type="text/javascript" async '
+ +'src="//code.rtbsystem.com/50381.js?t='+ new Date().getTime() + '" charset="utf-8" ></scr'+'ipt>');
+ }
+</script><!-- Start Counter --> +<!-- SpyLOG v2 f:0211 --> +<script language="javascript"> +u="u311.79.spylog.com";d=document;nv=navigator;na=nv.appName;t="";p=1; +sz=" width=1 height=1 "; +hl=history.length;d.cookie="b=b";c=0; +bv=Math.round(parseFloat(nv.appVersion)*100); +if (d.cookie) c=1;n=(na.substring(0,2)=="Mi")?0:1; +if((n==0)||(bv >= 300)){rn=Math.random();t=(new Date()).getTimezoneOffset();} else {rn=0;} +z="p="+p+"&rn="+rn+"&t="+t+"&c="+c+"&hl="+hl; +if (self != top) { fr=1;} else { fr=0;} +r=escape(d.referrer);r1=""; +sl="1.0";h=0; +</script> +<script language="javascript1.1"> +pl="";sl="1.1"; +if((n==1) && (bv >= 300)) +{ for(var i = 0; i < nv.plugins.length; i++) +pl += nv.plugins[i].name+":"; } +j = (navigator.javaEnabled() ? "Y" : "N"); +</script> +<script language=javascript1.2> +sl="1.2";s=screen;wh=s.width+'x'+s.height; +px=(n==0)?screen.colorDepth:screen.pixelDepth;z+="&wh="+wh+"&px="+px; +</script> +<script language=javascript1.3> +sl="1.3" +</script> +<script language="javascript"> +y=""; +y+="<a href='http://"+u+"/cnt?f=3&p="+p+"&rn="+rn+"' target=_blank>"; +y+="<img src='http://"+u+"/cnt?"; +y+=z+"&j="+j+"&sl="+sl+"&r="+r+"&r1="+r1+"&fr="+fr+"&pg="+escape(window.location.href)+"&pl="+escape(pl); +y+="' border=0 "+sz+" alt='SpyLOG' align=right>"; +y+="</a>"; +d.write(y); +</script> +<script language="javascript1.2"><!-- +if (n == 0) { d.write("<");d.write("!--"); } +//--></script> +<noscript> +<a href="http://u311.79.spylog.com/cnt?f=3&p=1" target=_blank> +<img src="http://u311.79.spylog.com/cnt?p=1" alt='SpyLOG' border='0' width=1 height=1 align=right> +</a> +</noscript> +<script language="javascript1.2"><!-- +if (n == 0) { d.write("--");d.write(">"); } +//--></script> +<!-- SpyLOG --> +<!-- End Counter --> +</TD> +</TR> +</table> +<input type=hidden name=l value=en> +</form> + + +<table border=0> +<tr> +<td valign=top > + +<script type="text/javascript"><!-- +google_ad_client = "pub-7005770717406130"; +google_ad_width = 160; +google_ad_height = 600; +google_ad_format = "160x600_as"; +google_ad_type = "text_image"; +//2006-12-05: filesearch_left_160x600 +google_ad_channel = "8508766014"; +google_color_border = "FFFFFF"; +google_color_bg = "FFFFFF"; +google_color_link = "000000"; +google_color_text = "000000"; +google_color_url = "000000"; +//--></script> +<script type="text/javascript" + src="http://pagead2.googlesyndication.com/pagead/show_ads.js"> +</script> + +</td> + +<td valign=top > + +<pre class=list> +</pre> + + +<script type="text/javascript"><!-- +google_ad_client = "pub-7005770717406130"; +/* filesearch_down_728x90 */ +google_ad_slot = "6424137562"; +google_ad_width = 728; +google_ad_height = 90; +//--> +</script> +<script type="text/javascript" +src="http://pagead2.googlesyndication.com/pagead/show_ads.js"> +</script> + + </td></tr></table> <blockquote> +<b>zstd-1.3.3.tar.gz</b> not found<br> +Try to use <a href=http://www.filesearching.com/><b>search forms</b></a> to refine the parameters of your request. +<br></blockquote> + +<table width=100% border=0 cellspacing=0 cellpadding=0 bgcolor=#FAE6CA> +<tr bgcolor=#000000><td><img src=/img/b.gif height=1 width=1></td></tr> +<tr> +<td background=/img/inside-4x1-1x6_1.gif><img src=/img/b.gif height=20 width=20></td> +</tr></table> +<table width=100% border=0 cellspacing=0 cellpadding=0 bgcolor=#FAE6CA> + <tr><td align=center> +<!--ENGLISH BOT--><br><br> +</td></tr> +</table> + +<!--Kavanga START--> + +<script type="text/javascript"><!-- +if (Math.ceil(Math.random()*10) ) { + <!--Kavanga START--> + <!--Сайт: filesearch.ru--> + <!--Категория: Компьютеры и ПО--> + if (typeof(pr) == 'undefined') { var pr = Math.floor(Math.random() * 1000000); } + if (typeof(document.referrer) != 'undefined') { + if (typeof(afReferrer) == 'undefined') { + afReferrer = escape(document.referrer); + } + } else { + afReferrer = ''; + } + var addate = new Date(); + + if (Math.ceil(Math.random()*10) ) { + <!--Тип баннера: Pop Under--> + document.write('<scr' + 'ipt type="text/javascript" src="http://a.kavanga.ru/3604/prepareCode?ph=b&p1=our&p2=o&pucn=a&pfc=a&pfb=a&plp=a&pli=a&pop=a&pr=' + pr + '&pt=b&pd=' + addate.getDate() + '&pw=' + addate.getDay() + '&pv=' + addate.getHours() + '&prr=' + afReferrer + '"><\/scr' + 'ipt>'); + } else { + <!--Тип баннера: Rich Media--> + document.write('<scr' + 'ipt type="text/javascript" src="http://a.kavanga.ru/3604/prepareCode?p1=oip&p2=p&pucn=a&pfc=a&pfb=a&plp=a&pli=a&pop=a&pr=' + pr +'&pt=b&pd=' + addate.getDate() + '&pw=' + addate.getDay() + '&pv=' + addate.getHours() + '&py=a&prr=' + afReferrer + '"><\/scr' + 'ipt>'); + } + <!--Kavanga END--> +} else { + <!-- AdMedia START --> + var RndNum4NoCash = Math.round(Math.random() * 1000000000); + document.write('<script language="JavaScript" src="http://ad.adriver.ru/cgi-bin/erle.cgi?sid=36667&target=top&bt=16&pz=0&rnd=' + RndNum4NoCash + '"><\/script>'); + <!-- AdMedia END --> +} +// --> +</script> + +<!--Kavanga END--> +</BODY></HTML> diff --git a/service/main.cpp b/service/main.cpp new file mode 100644 index 0000000..97bc89e --- /dev/null +++ b/service/main.cpp @@ -0,0 +1,18 @@ +#include <Ice/Communicator.h> +#include <Ice/ObjectAdapter.h> +#include <icetrayService.h> +#include "apiImpl.h" + +namespace MirrorSearch { + class Api : public IceTray::Service { + public: + void addObjects(const std::string &, const Ice::CommunicatorPtr & ic, const Ice::StringSeq &, const Ice::ObjectAdapterPtr & adp) override + { + auto dbpool = getConnectionPool(ic, "postgresql", "MirrorSearch"); + adp->add(new SearchImpl(dbpool), ic->stringToIdentity("Search")); + } + }; + + NAMEDFACTORY("default", MirrorSearch::Api, IceTray::ServiceFactory); +} + diff --git a/service/models.ice b/service/models.ice new file mode 100644 index 0000000..cb48459 --- /dev/null +++ b/service/models.ice @@ -0,0 +1,25 @@ +#ifndef MIRRORSEARCH_MODELS +#define MIRRORSEARCH_MODELS + +module MirrorSearch { + class SearchService { + ["slicer:db:pkey"] + int id; + string name; + string baseurl; + string listxpath; + string urlxpath; + }; + class SearchHit { + ["slicer:db:pkey"] + int id; + int serviceid; + string url; + }; + + sequence<SearchService> SearchServices; + sequence<SearchHit> SearchHits; +}; + +#endif + diff --git a/service/schema.sql b/service/schema.sql new file mode 100644 index 0000000..b529172 --- /dev/null +++ b/service/schema.sql @@ -0,0 +1,8 @@ +CREATE TABLE searchservices( + id int GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + name text not null, + baseurl text not null, + listxpath text not null, + urlxpath text not null +); + diff --git a/service/sql/getServices.sql b/service/sql/getServices.sql new file mode 100644 index 0000000..40382f2 --- /dev/null +++ b/service/sql/getServices.sql @@ -0,0 +1,3 @@ +SELECT id, name, baseurl, listxpath, urlxpath +FROM searchservices +ORDER BY id diff --git a/service/test.cpp b/service/test.cpp new file mode 100644 index 0000000..c115712 --- /dev/null +++ b/service/test.cpp @@ -0,0 +1,77 @@ +#define BOOST_TEST_MODULE MirrorSearch +#include <boost/test/unit_test.hpp> + +#include <pq-mock.h> +#include <dryice.h> +#include <definedDirs.h> +#include <api.h> + +class Service : PQ::Mock, public IceTray::DryIce { + public: + Service() : PQ::Mock("user=postgres", "MirrorSearch", { + rootDir / "schema.sql", + rootDir / "data.sql" + }) { } + +}; + +class TestClient : public IceTray::DryIceClient { + public: + TestClient() : + s(getProxy<MirrorSearch::SearchPrx>("Search")) + { + } + + MirrorSearch::SearchPrx s; +}; + +BOOST_TEST_GLOBAL_FIXTURE(Service); + +BOOST_FIXTURE_TEST_SUITE(tc, TestClient); + +BOOST_AUTO_TEST_CASE(sanity) +{ + BOOST_REQUIRE(s); + s->ice_ping(); +} + +BOOST_AUTO_TEST_CASE(getServices) +{ + auto ss = s->getServices(); + BOOST_REQUIRE_EQUAL(ss.size(), 1); + BOOST_CHECK_EQUAL(ss.front()->id, 1); + BOOST_CHECK_EQUAL(ss.front()->name, "file searching mock"); + BOOST_CHECK_NE(ss.front()->baseurl, "file://$SCRIPTDIR/fixtures/filesearching/%s.html"); + BOOST_CHECK_EQUAL(ss.front()->baseurl.substr(0, 8), "file:///"); + BOOST_CHECK(!ss.front()->listxpath.empty()); + BOOST_CHECK(!ss.front()->urlxpath.empty()); +} + +BOOST_AUTO_TEST_CASE(getMatches_zstd_notfound) +{ + auto ms = s->getMatches("zstd-1.3.3.tar.gz"); + BOOST_REQUIRE(ms.empty()); +} + +BOOST_AUTO_TEST_CASE(getMatches_xtrans) +{ + auto ms = s->getMatches("xtrans-1.3.5.tar.bz2"); + BOOST_REQUIRE_EQUAL(ms.size(), 12); + BOOST_REQUIRE_EQUAL(ms.front()->url, "ftp://ftp.cs.mun.ca/pub/mirror/gentoo/distfiles/xtrans-1.3.5.tar.bz2"); + BOOST_REQUIRE_EQUAL(ms.back()->url, "ftp://ftp.tr.debian.org/gentoo/distfiles/xtrans-1.3.5.tar.bz2"); +} + +BOOST_AUTO_TEST_CASE(getMatches_zstd_notfound_lucky) +{ + BOOST_REQUIRE(!s->feelingLucky("zstd-1.3.3.tar.gz")); +} + +BOOST_AUTO_TEST_CASE(getMatches_xtrans_lucky) +{ + auto fl = s->feelingLucky("xtrans-1.3.5.tar.bz2"); + BOOST_REQUIRE(fl); + BOOST_REQUIRE_EQUAL(*fl, "ftp://ftp.cs.mun.ca/pub/mirror/gentoo/distfiles/xtrans-1.3.5.tar.bz2"); +} + +BOOST_AUTO_TEST_SUITE_END(); + diff --git a/slice.jam b/slice.jam new file mode 100644 index 0000000..434e562 --- /dev/null +++ b/slice.jam @@ -0,0 +1,58 @@ +import type : register ; +import generators : register-standard ; +import type ; +import feature : feature ; +import scanner ; +import toolset ; + +type.register SLICE : ice ; + +feature slicer : no yes pure ; +feature allow-ice : no yes ; +feature ice-visibility : public hidden ; + +class slice-scanner : common-scanner +{ + rule pattern ( ) + { + return "^[ \t]*#[ \t]*include[ ]*[<\"]([^>\"]*)[>\"]" ; + } +} + +scanner.register slice-scanner : include ; + +type.set-scanner SLICE : slice-scanner ; + +generators.register-standard slice.slice2cpp : SLICE : CPP H : <slicer>no ; +generators.register-standard slice.slicer : SLICE : CPP CPP(slicer-%) H : <slicer>yes ; +generators.register-standard slice.slicer.pure : SLICE : CPP(slicer-%) : <slicer>pure ; + +toolset.flags slice.slice2cpp INCLUDES <include> ; +toolset.flags slice.slice2cpp DLLEXPORT <ice-visibility>public : --dll-export JAM_DLL_PUBLIC ; +toolset.flags slice.slice2cpp ALLOWICE <allow-ice>yes : --ice ; +toolset.flags slice.slicer INCLUDES <include> ; +toolset.flags slice.slicer DLLEXPORT <ice-visibility>public : --dll-export JAM_DLL_PUBLIC ; +toolset.flags slice.slicer ALLOWICE <allow-ice>yes : --ice ; +toolset.flags slice.slicer.pure INCLUDES <include> ; +toolset.flags slice.slicer.pure ALLOWICE <allow-ice>yes : --ice ; + +actions slice.slice2cpp +{ + slice2cpp -I"$(INCLUDES)" --checksum --output-dir $(1[1]:D) $(2) $(DLLEXPORT) $(ALLOWICE[1]) +} + +actions slice.slicer +{ + slice2cpp -I"$(INCLUDES)" --checksum --output-dir $(1[1]:D) $(2) $(DLLEXPORT) $(ALLOWICE[1]) + slicer -I"$(INCLUDES)" $(2) $(1[2]) $(ALLOWICE[1]) +} + +actions slice.slicer.pure +{ + slicer -I"$(INCLUDES)" $(2) $(1[1]) $(ALLOWICE[1]) +} + +IMPORT $(__name__) : slice.slice2cpp : : slice.slice2cpp ; +IMPORT $(__name__) : slice.slicer : : slice.slicer ; +IMPORT $(__name__) : slice.slicer.pure : : slice.slicer.pure ; + |