summaryrefslogtreecommitdiff
path: root/service
diff options
context:
space:
mode:
Diffstat (limited to 'service')
-rw-r--r--service/Jamfile.jam64
-rw-r--r--service/api.ice19
-rw-r--r--service/apiImpl.cpp91
-rw-r--r--service/apiImpl.h25
-rw-r--r--service/data.sql3
-rw-r--r--service/fixtures/filesearching/xtrans-1.3.5.tar.bz2.html251
-rw-r--r--service/fixtures/filesearching/zstd-1.3.3.tar.gz.html243
-rw-r--r--service/main.cpp18
-rw-r--r--service/models.ice25
-rw-r--r--service/schema.sql8
-rw-r--r--service/sql/getServices.sql3
-rw-r--r--service/test.cpp77
12 files changed, 827 insertions, 0 deletions
diff --git a/service/Jamfile.jam b/service/Jamfile.jam
new file mode 100644
index 0000000..e7bfed9
--- /dev/null
+++ b/service/Jamfile.jam
@@ -0,0 +1,64 @@
+import icetray ;
+import package ;
+import testing ;
+
+lib boost_utf : : <name>boost_unit_test_framework ;
+lib dbpp-postgresql : : : : <include>/usr/include/dbpp-postgresql ;
+lib dryice : : : : <include>/usr/include/icetray ;
+
+lib mirrorsearch :
+ [ glob *.cpp *.ice sql/*.sql : test.cpp ]
+ :
+ <slicer>yes
+ <library>..//adhocutil
+ <library>..//dbppcore
+ <library>..//boost_system
+ <library>..//boost_filesystem
+ <library>..//boost_date_time
+ <library>..//Ice
+ <library>..//IceBox
+ <library>..//IceUtil
+ <library>..//pthread
+ <library>..//icetray
+ <library>..//slicer
+ <library>..//slicer-db
+ <library>..//glibmm
+ <library>..//xml2
+ <icetray.sql.namespace>MirrorSearch
+ <icetray.sql.basedir>.
+ <include>.
+ : :
+ <include>.
+ <library>..//icetray
+ ;
+
+path-constant me : . ;
+
+run test.cpp
+ : :
+ data.sql
+ schema.sql
+ :
+ <define>BOOST_TEST_DYN_LINK
+ <library>..//boost_system
+ <library>..//boost_filesystem
+ <library>boost_utf
+ <define>ROOT=\"$(me)\"
+ <library>dbpp-postgresql
+ <library>..//dbppcore
+ <library>..//adhocutil
+ <library>..//boost_system
+ <library>..//boost_filesystem
+ <library>..//netfs-api
+ <library>..//IceUtil
+ <library>..//Ice
+ <library>..//IceBox
+ <library>..//pthread
+ <library>dryice
+ <library>mirrorsearch
+ <implicit-dependency>mirrorsearch
+;
+
+package.install install : : : mirrorsearch ;
+
+
diff --git a/service/api.ice b/service/api.ice
new file mode 100644
index 0000000..c207c3d
--- /dev/null
+++ b/service/api.ice
@@ -0,0 +1,19 @@
+#ifndef MIRRORSEARCH_API
+#define MIRRORSEARCH_API
+
+#include <models.ice>
+
+module MirrorSearch {
+ exception XmlError {
+ string msg;
+ };
+
+ interface Search {
+ idempotent SearchServices getServices();
+ idempotent SearchHits getMatches(string filename);
+ idempotent optional(0) string feelingLucky(string filename);
+ };
+};
+
+#endif
+
diff --git a/service/apiImpl.cpp b/service/apiImpl.cpp
new file mode 100644
index 0000000..588d360
--- /dev/null
+++ b/service/apiImpl.cpp
@@ -0,0 +1,91 @@
+#include "apiImpl.h"
+
+#include <sql/getServices.sql.h>
+#include <buffer.h>
+#include <memory>
+
+#include <libxml/xpath.h>
+#include <libxml/xpathInternals.h>
+#include <libxml/HTMLparser.h>
+#include <libxml/HTMLtree.h>
+
+namespace MirrorSearch {
+ SearchImpl::SearchImpl(IceTray::DatabasePoolPtr db) :
+ IceTray::AbstractDatabaseClient(db),
+ log(LOGMANAGER()->getLogger<SearchImpl>())
+ {
+ }
+
+ SearchServices SearchImpl::getServices(const ::Ice::Current&)
+ {
+ return fetch<SearchServices>(sql::getServices);
+ }
+
+ typedef std::shared_ptr<xmlDoc> xmlDocSPtr;
+ typedef std::shared_ptr<xmlXPathContext> xmlXPathContextSPtr;
+ typedef std::shared_ptr<xmlXPathObject> xmlXPathObjectSPtr;
+
+ static auto getDoc(const ::std::string & url, int flags)
+ {
+ if (auto doc = xmlDocSPtr(htmlReadFile(url.c_str(), NULL, flags), xmlFreeDoc)) {
+ return doc;
+ }
+ throw XmlError("Failed to open " + url);
+ }
+
+ static auto getXPathCxt(const xmlDocSPtr & doc)
+ {
+ if (auto xpathCtx = xmlXPathContextSPtr(xmlXPathNewContext(doc.get()), xmlXPathFreeContext)) {
+ return xpathCtx;
+ }
+ throw XmlError("Failed to create xpath context");
+ }
+
+ static auto getXPathObj(const ::std::string & xpath, const xmlXPathContextSPtr & ctx, xmlXPathObjectType type)
+ {
+ if (auto xpathObj = xmlXPathObjectSPtr(xmlXPathEvalExpression(BAD_CAST xpath.c_str(), ctx.get()), xmlXPathFreeObject)) {
+ if (xpathObj->type != type) {
+ throw XmlError("Xpath evaluates to wrong type " + xpath);
+ }
+ return xpathObj;
+ }
+ throw XmlError("Failed to evaluate xpath " + xpath);
+ }
+
+ void SearchImpl::callService(const ::std::string & fn, const SearchServicePtr & s, SearchHits & sh) const
+ {
+ auto fmt = AdHoc::Buffer::getFormat(s->baseurl);
+ auto url = (*fmt % fn).str();
+ auto doc = getDoc(url,
+ HTML_PARSE_RECOVER | HTML_PARSE_NODEFDTD | HTML_PARSE_NOIMPLIED |
+ HTML_PARSE_NOWARNING | HTML_PARSE_NOERROR);
+ auto xpathCtx = getXPathCxt(doc);
+ auto xpathObj = getXPathObj(s->listxpath, xpathCtx, xmlXPathObjectType::XPATH_NODESET);
+ log->messagebf(LOG::INFO, "%d nodes matched %s", xpathObj->nodesetval->nodeNr, s->listxpath);
+ for (int row = 0; row < xpathObj->nodesetval->nodeNr; row += 1) {
+ xpathCtx->node = xpathObj->nodesetval->nodeTab[row];
+ auto xpathObjI = getXPathObj(s->urlxpath, xpathCtx, xmlXPathObjectType::XPATH_STRING);
+ if (xpathObjI->stringval && *xpathObjI->stringval) {
+ sh.push_back(new SearchHit(0, s->id, (const char *) xpathObjI->stringval));
+ }
+ }
+ }
+
+ SearchHits SearchImpl::getMatches(const ::std::string & fn, const ::Ice::Current & c)
+ {
+ SearchHits sh;
+ for (const auto & s : getServices(c)) {
+ callService(fn, s, sh);
+ }
+ return sh;
+ }
+
+ ::IceUtil::Optional<::std::string> SearchImpl::feelingLucky(const ::std::string & fn, const ::Ice::Current & c)
+ {
+ const auto ms = getMatches(fn, c);
+ if (ms.empty())
+ return IceUtil::None;
+ return ms.front()->url;
+ }
+}
+
diff --git a/service/apiImpl.h b/service/apiImpl.h
new file mode 100644
index 0000000..e56368c
--- /dev/null
+++ b/service/apiImpl.h
@@ -0,0 +1,25 @@
+#ifndef MIRRORSEARCH_APIIMPL_H
+#define MIRRORSEARCH_APIIMPL_H
+
+#include <api.h>
+#include <abstractDatabaseClient.h>
+#include <logger.h>
+
+namespace MirrorSearch {
+ class SearchImpl : public Search, public IceTray::AbstractDatabaseClient {
+ public:
+ SearchImpl(IceTray::DatabasePoolPtr);
+
+ virtual SearchServices getServices(const ::Ice::Current& = ::Ice::Current()) override;
+ virtual SearchHits getMatches(const ::std::string&, const ::Ice::Current& = ::Ice::Current()) override;
+ virtual ::IceUtil::Optional<::std::string> feelingLucky(const ::std::string&, const ::Ice::Current& = ::Ice::Current()) override;
+
+ private:
+ void callService(const ::std::string & fn, const SearchServicePtr & s, SearchHits & sh) const;
+
+ LOG::LoggerPtr log;
+ };
+}
+
+#endif
+
diff --git a/service/data.sql b/service/data.sql
new file mode 100644
index 0000000..e14e721
--- /dev/null
+++ b/service/data.sql
@@ -0,0 +1,3 @@
+INSERT INTO searchservices(name, baseurl, listxpath, urlxpath)
+ VALUES('file searching mock', 'file://$SCRIPTDIR/fixtures/filesearching/%s.html', '//pre[@class=''list'']/a[@class=''lf'']', 'string(@href)')
+ ;
diff --git a/service/fixtures/filesearching/xtrans-1.3.5.tar.bz2.html b/service/fixtures/filesearching/xtrans-1.3.5.tar.bz2.html
new file mode 100644
index 0000000..27b627a
--- /dev/null
+++ b/service/fixtures/filesearching/xtrans-1.3.5.tar.bz2.html
@@ -0,0 +1,251 @@
+<HTML>
+<HEAD>
+<TITLE>FileSearch - xtrans-1.3.5.tar.bz2</TITLE>
+<link rel=stylesheet href=/styles/fs.css>
+</HEAD>
+<BODY bgcolor=#FFFFFF leftmargin=0 topmargin=0 marginwidth=0 marginheight=0>
+<FORM ACTION=/cgi-bin/s METHOD=GET>
+<TABLE width=100% border=0 cellspacing=0 cellpadding=0>
+ <TR bgcolor=#CCA782 valign=bottom align=left>
+ <TD colspan=2 height=87>
+ <TABLE width=100% border=0 cellspacing=0 cellpadding=0>
+ <TR>
+ <TD width=160 align=right valign=bottom><a href=/><IMG src=/img/b.gif width=160 height=10 border=0><img src=/img/filesearchen.gif width=142 height=77 border=0 valign=bottom></a></TD>
+ <TD valign=bottom width=100%>
+ <TABLE width=100% border=0 cellspacing=0 cellpadding=0>
+ <TR>
+ <td bgcolor=#CCA782 valign=bottom class=text align=center><img src=/img/b.gif height=1 width=10></td>
+ <td rowspan=2 background=/img/poisklght.gif bgcolor=#CCA782 valign=bottom class=taplight align=center><img src=/img/b.gif height=1 width=73><br>search</td>
+ <td rowspan=2 background=/img/advpoiskdrk.gif bgcolor=#CCA782 valign=bottom align=center width=135 class=taplight nowrap><img src=/img/b.gif height=1 width=135><br><a href=/advanced/ class=tapdark>advanced search</a></td>
+ <td align=right class=text width=100%>...</td>
+ <td align=right><img src=/img/b.gif width=1 height=21></td>
+ </TR>
+ <TR>
+ <TD bgcolor=#000000 valign=bottom class=text align=center><img src=/img/b.gif height=1 width=10></TD>
+ <TD align=right bgcolor=#000000 colspan=2><IMG src=/img/b.gif width=274 height=1></TD>
+ </TR>
+ </TABLE>
+ <TABLE border=0 cellpadding=0 cellspacing=0 width=100%>
+ <TR>
+ <td width=1 bgcolor=#000000 ><img src=/img/b.gif width=1 height=48></td>
+ <td bgcolor=#FFE7CE valign=middle nowrap>&nbsp;
+ <INPUT type=text name=q value="xtrans-1.3.5.tar.bz2" SIZE=20>&nbsp;
+ <select name=t>
+ <option VALUE=f selected>file/directory
+ <option VALUE=m>music (mp3)
+ <option VALUE=p>images
+ <option VALUE=v>video
+ <option VALUE=s>server
+ </select>
+ <select name=d>
+<option value="">all sites
+<option value="com">.com
+<option value="net">.net
+<option value="edu">.edu
+<option value="org">.org
+<option value="au">.au
+<option value="be">.be
+<option value="ca">.ca
+<option value="ch">.ch
+<option value="de">.de
+<option value="dk">.dk
+<option value="es">.es
+<option value="fr">.fr
+<option value="it">.it
+<option value="jp">.jp
+<option value="lv">.lv
+<option value="nl">.nl
+<option value="no">.no
+<option value="ru,su">.ru
+<option value="se">.se
+<option value="ua">.ua
+<option value="uk">.uk
+
+ </select>&nbsp;
+ <img src=/img/b.gif height=1 width=5></td>
+ <td bgcolor=#FFE7CE valign=middle width=630>
+ <input type=image src=/img/findengl.gif border=0>
+ </td>
+ <td width=1 bgcolor=#000000 ><img src=/img/b.gif width=1 height=46></td>
+ </TR>
+ <TR>
+ <TD rowspan=1 colspan=4 bgcolor=#000000><IMG src=/img/b.gif width=3 height=1 border=0></TD>
+ </TR>
+ </TABLE>
+ <img src=/img/b.gif width=10 height=6></TD>
+ <TD valign=bottom width=20><IMG src=/img/b.gif width=20 height=8></TD>
+ </TR>
+ </TABLE>
+ </TD>
+ </TR>
+ <TR valign=center align=center>
+ <TD colspan=2 background=/img/inside-2x1.gif>
+<!--ENGLISH TOP--><div id="RTBDIV_50381">
+ <div id="RTBPL_50381">
+ <a href="//rtbsystem.com/ru/advertiser/request">Добавить рекламное обьявление</a>
+ </div>
+</div>
+<script type="text/javascript">
+ if (document.getElementById('RTBDIV_50381')) {
+ document.write('<scr'+'ipt type="text/javascript" async '
+ +'src="//code.rtbsystem.com/50381.js?t='+ new Date().getTime() + '" charset="utf-8" ></scr'+'ipt>');
+ }
+</script><!-- Start Counter -->
+<!-- SpyLOG v2 f:0211 -->
+<script language="javascript">
+u="u311.79.spylog.com";d=document;nv=navigator;na=nv.appName;t="";p=1;
+sz=" width=1 height=1 ";
+hl=history.length;d.cookie="b=b";c=0;
+bv=Math.round(parseFloat(nv.appVersion)*100);
+if (d.cookie) c=1;n=(na.substring(0,2)=="Mi")?0:1;
+if((n==0)||(bv >= 300)){rn=Math.random();t=(new Date()).getTimezoneOffset();} else {rn=0;}
+z="p="+p+"&rn="+rn+"&t="+t+"&c="+c+"&hl="+hl;
+if (self != top) { fr=1;} else { fr=0;}
+r=escape(d.referrer);r1="";
+sl="1.0";h=0;
+</script>
+<script language="javascript1.1">
+pl="";sl="1.1";
+if((n==1) && (bv >= 300))
+{ for(var i = 0; i < nv.plugins.length; i++)
+pl += nv.plugins[i].name+":"; }
+j = (navigator.javaEnabled() ? "Y" : "N");
+</script>
+<script language=javascript1.2>
+sl="1.2";s=screen;wh=s.width+'x'+s.height;
+px=(n==0)?screen.colorDepth:screen.pixelDepth;z+="&wh="+wh+"&px="+px;
+</script>
+<script language=javascript1.3>
+sl="1.3"
+</script>
+<script language="javascript">
+y="";
+y+="<a href='http://"+u+"/cnt?f=3&p="+p+"&rn="+rn+"' target=_blank>";
+y+="<img src='http://"+u+"/cnt?";
+y+=z+"&j="+j+"&sl="+sl+"&r="+r+"&r1="+r1+"&fr="+fr+"&pg="+escape(window.location.href)+"&pl="+escape(pl);
+y+="' border=0 "+sz+" alt='SpyLOG' align=right>";
+y+="</a>";
+d.write(y);
+</script>
+<script language="javascript1.2"><!--
+if (n == 0) { d.write("<");d.write("!--"); }
+//--></script>
+<noscript>
+<a href="http://u311.79.spylog.com/cnt?f=3&p=1" target=_blank>
+<img src="http://u311.79.spylog.com/cnt?p=1" alt='SpyLOG' border='0' width=1 height=1 align=right>
+</a>
+</noscript>
+<script language="javascript1.2"><!--
+if (n == 0) { d.write("--");d.write(">"); }
+//--></script>
+<!-- SpyLOG -->
+<!-- End Counter -->
+</TD>
+</TR>
+</table>
+<input type=hidden name=l value=en>
+</form>
+
+
+<table border=0>
+<tr>
+<td valign=top >
+
+<script type="text/javascript"><!--
+google_ad_client = "pub-7005770717406130";
+google_ad_width = 160;
+google_ad_height = 600;
+google_ad_format = "160x600_as";
+google_ad_type = "text_image";
+//2006-12-05: filesearch_left_160x600
+google_ad_channel = "8508766014";
+google_color_border = "FFFFFF";
+google_color_bg = "FFFFFF";
+google_color_link = "000000";
+google_color_text = "000000";
+google_color_url = "000000";
+//--></script>
+<script type="text/javascript"
+ src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+</script>
+
+</td>
+
+<td valign=top >
+
+<pre class=list>
+ 1 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.cs.mun.ca/ class=ls>ftp.cs.mun.ca</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.cs.mun.ca/pub/mirror/gentoo/distfiles class=lg>/pub/mirror/gentoo/distfiles/</a><a href=ftp://ftp.cs.mun.ca/pub/mirror/gentoo/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+ 2 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.dvo.ru/ class=ls>ftp.dvo.ru</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.dvo.ru/pub/Gentoo/distfiles class=lg>/pub/Gentoo/distfiles/</a><a href=ftp://ftp.dvo.ru/pub/Gentoo/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+ 3 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.dvo.ru/ class=ls>ftp.dvo.ru</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.dvo.ru/pub/distfiles class=lg>/pub/distfiles/</a><a href=ftp://ftp.dvo.ru/pub/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+ 4 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.fu-berlin.de/ class=ls>ftp.fu-berlin.de</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.fu-berlin.de/unix/X11/FTP.X.ORG/pub/individual/lib class=lg>/unix/X11/FTP.X.ORG/pub/individual/lib/</a><a href=ftp://ftp.fu-berlin.de/unix/X11/FTP.X.ORG/pub/individual/lib/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+ 5 <img src=/img/iconany.gif width=16 height=16> <b> 536</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.fu-berlin.de/ class=ls>ftp.fu-berlin.de</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.fu-berlin.de/unix/X11/FTP.X.ORG/pub/individual/lib class=lg>/unix/X11/FTP.X.ORG/pub/individual/lib/</a><a href=ftp://ftp.fu-berlin.de/unix/X11/FTP.X.ORG/pub/individual/lib/xtrans-1.3.5.tar.bz2.sig class=lf>xtrans-1.3.5.tar.bz2.sig</a>
+ 6 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.gr.debian.org/ class=ls>ftp.gr.debian.org</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.gr.debian.org/pub/X11/X.org/individual/lib class=lg>/pub/X11/X.org/individual/lib/</a><a href=ftp://ftp.gr.debian.org/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+ 7 <img src=/img/iconany.gif width=16 height=16> <b> 536</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.gr.debian.org/ class=ls>ftp.gr.debian.org</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.gr.debian.org/pub/X11/X.org/individual/lib class=lg>/pub/X11/X.org/individual/lib/</a><a href=ftp://ftp.gr.debian.org/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2.sig class=lf>xtrans-1.3.5.tar.bz2.sig</a>
+ 8 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.linux.org.tr/ class=ls>ftp.linux.org.tr</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.linux.org.tr/gentoo/distfiles class=lg>/gentoo/distfiles/</a><a href=ftp://ftp.linux.org.tr/gentoo/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+ 9 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.ntua.gr/ class=ls>ftp.ntua.gr</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.ntua.gr/pub/X11/X.org/individual/lib class=lg>/pub/X11/X.org/individual/lib/</a><a href=ftp://ftp.ntua.gr/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+ 10 <img src=/img/iconany.gif width=16 height=16> <b> 536</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.ntua.gr/ class=ls>ftp.ntua.gr</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.ntua.gr/pub/X11/X.org/individual/lib class=lg>/pub/X11/X.org/individual/lib/</a><a href=ftp://ftp.ntua.gr/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2.sig class=lf>xtrans-1.3.5.tar.bz2.sig</a>
+ 11 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.rediris.es/ class=ls>ftp.rediris.es</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.rediris.es/sites/gentoo.org/distfiles class=lg>/sites/gentoo.org/distfiles/</a><a href=ftp://ftp.rediris.es/sites/gentoo.org/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+ 12 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.tr.debian.org/ class=ls>ftp.tr.debian.org</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.tr.debian.org/gentoo/distfiles class=lg>/gentoo/distfiles/</a><a href=ftp://ftp.tr.debian.org/gentoo/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+</pre>
+
+
+<script type="text/javascript"><!--
+google_ad_client = "pub-7005770717406130";
+/* filesearch_down_728x90 */
+google_ad_slot = "6424137562";
+google_ad_width = 728;
+google_ad_height = 90;
+//-->
+</script>
+<script type="text/javascript"
+src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+</script>
+
+ </td></tr></table>
+<table width=100% border=0 cellspacing=0 cellpadding=0 bgcolor=#FAE6CA>
+<tr bgcolor=#000000><td><img src=/img/b.gif height=1 width=1></td></tr>
+<tr>
+<td background=/img/inside-4x1-1x6_1.gif><img src=/img/b.gif height=20 width=20></td>
+</tr></table>
+<table width=100% border=0 cellspacing=0 cellpadding=0 bgcolor=#FAE6CA>
+ <tr><td align=center>
+<!--ENGLISH BOT--><br><br>
+</td></tr>
+</table>
+
+<!--Kavanga START-->
+
+<script type="text/javascript"><!--
+if (Math.ceil(Math.random()*10) ) {
+ <!--Kavanga START-->
+ <!--Сайт: filesearch.ru-->
+ <!--Категория: Компьютеры и ПО-->
+ if (typeof(pr) == 'undefined') { var pr = Math.floor(Math.random() * 1000000); }
+ if (typeof(document.referrer) != 'undefined') {
+ if (typeof(afReferrer) == 'undefined') {
+ afReferrer = escape(document.referrer);
+ }
+ } else {
+ afReferrer = '';
+ }
+ var addate = new Date();
+
+ if (Math.ceil(Math.random()*10) ) {
+ <!--Тип баннера: Pop Under-->
+ document.write('<scr' + 'ipt type="text/javascript" src="http://a.kavanga.ru/3604/prepareCode?ph=b&amp;p1=our&amp;p2=o&amp;pucn=a&amp;pfc=a&amp;pfb=a&amp;plp=a&amp;pli=a&amp;pop=a&amp;pr=' + pr + '&amp;pt=b&amp;pd=' + addate.getDate() + '&amp;pw=' + addate.getDay() + '&amp;pv=' + addate.getHours() + '&amp;prr=' + afReferrer + '"><\/scr' + 'ipt>');
+ } else {
+ <!--Тип баннера: Rich Media-->
+ document.write('<scr' + 'ipt type="text/javascript" src="http://a.kavanga.ru/3604/prepareCode?p1=oip&amp;p2=p&amp;pucn=a&amp;pfc=a&amp;pfb=a&amp;plp=a&amp;pli=a&amp;pop=a&amp;pr=' + pr +'&amp;pt=b&amp;pd=' + addate.getDate() + '&amp;pw=' + addate.getDay() + '&amp;pv=' + addate.getHours() + '&amp;py=a&amp;prr=' + afReferrer + '"><\/scr' + 'ipt>');
+ }
+ <!--Kavanga END-->
+} else {
+ <!-- AdMedia START -->
+ var RndNum4NoCash = Math.round(Math.random() * 1000000000);
+ document.write('<script language="JavaScript" src="http://ad.adriver.ru/cgi-bin/erle.cgi?sid=36667&target=top&bt=16&pz=0&rnd=' + RndNum4NoCash + '"><\/script>');
+ <!-- AdMedia END -->
+}
+// -->
+</script>
+
+<!--Kavanga END-->
+</BODY></HTML>
diff --git a/service/fixtures/filesearching/zstd-1.3.3.tar.gz.html b/service/fixtures/filesearching/zstd-1.3.3.tar.gz.html
new file mode 100644
index 0000000..74e70e2
--- /dev/null
+++ b/service/fixtures/filesearching/zstd-1.3.3.tar.gz.html
@@ -0,0 +1,243 @@
+<HTML>
+<HEAD>
+<TITLE>FileSearch - zstd-1.3.3.tar.gz</TITLE>
+<link rel=stylesheet href=/styles/fs.css>
+</HEAD>
+<BODY bgcolor=#FFFFFF leftmargin=0 topmargin=0 marginwidth=0 marginheight=0>
+<FORM ACTION=/cgi-bin/s METHOD=GET>
+<TABLE width=100% border=0 cellspacing=0 cellpadding=0>
+ <TR bgcolor=#CCA782 valign=bottom align=left>
+ <TD colspan=2 height=87>
+ <TABLE width=100% border=0 cellspacing=0 cellpadding=0>
+ <TR>
+ <TD width=160 align=right valign=bottom><a href=/><IMG src=/img/b.gif width=160 height=10 border=0><img src=/img/filesearchen.gif width=142 height=77 border=0 valign=bottom></a></TD>
+ <TD valign=bottom width=100%>
+ <TABLE width=100% border=0 cellspacing=0 cellpadding=0>
+ <TR>
+ <td bgcolor=#CCA782 valign=bottom class=text align=center><img src=/img/b.gif height=1 width=10></td>
+ <td rowspan=2 background=/img/poisklght.gif bgcolor=#CCA782 valign=bottom class=taplight align=center><img src=/img/b.gif height=1 width=73><br>search</td>
+ <td rowspan=2 background=/img/advpoiskdrk.gif bgcolor=#CCA782 valign=bottom align=center width=135 class=taplight nowrap><img src=/img/b.gif height=1 width=135><br><a href=/advanced/ class=tapdark>advanced search</a></td>
+ <td align=right class=text width=100%>...</td>
+ <td align=right><img src=/img/b.gif width=1 height=21></td>
+ </TR>
+ <TR>
+ <TD bgcolor=#000000 valign=bottom class=text align=center><img src=/img/b.gif height=1 width=10></TD>
+ <TD align=right bgcolor=#000000 colspan=2><IMG src=/img/b.gif width=274 height=1></TD>
+ </TR>
+ </TABLE>
+ <TABLE border=0 cellpadding=0 cellspacing=0 width=100%>
+ <TR>
+ <td width=1 bgcolor=#000000 ><img src=/img/b.gif width=1 height=48></td>
+ <td bgcolor=#FFE7CE valign=middle nowrap>&nbsp;
+ <INPUT type=text name=q value="zstd-1.3.3.tar.gz" SIZE=20>&nbsp;
+ <select name=t>
+ <option VALUE=f selected>file/directory
+ <option VALUE=m>music (mp3)
+ <option VALUE=p>images
+ <option VALUE=v>video
+ <option VALUE=s>server
+ </select>
+ <select name=d>
+<option value="">all sites
+<option value="com">.com
+<option value="net">.net
+<option value="edu">.edu
+<option value="org">.org
+<option value="au">.au
+<option value="be">.be
+<option value="ca">.ca
+<option value="ch">.ch
+<option value="de">.de
+<option value="dk">.dk
+<option value="es">.es
+<option value="fr">.fr
+<option value="it">.it
+<option value="jp">.jp
+<option value="lv">.lv
+<option value="nl">.nl
+<option value="no">.no
+<option value="ru,su">.ru
+<option value="se">.se
+<option value="ua">.ua
+<option value="uk">.uk
+
+ </select>&nbsp;
+ <img src=/img/b.gif height=1 width=5></td>
+ <td bgcolor=#FFE7CE valign=middle width=630>
+ <input type=image src=/img/findengl.gif border=0>
+ </td>
+ <td width=1 bgcolor=#000000 ><img src=/img/b.gif width=1 height=46></td>
+ </TR>
+ <TR>
+ <TD rowspan=1 colspan=4 bgcolor=#000000><IMG src=/img/b.gif width=3 height=1 border=0></TD>
+ </TR>
+ </TABLE>
+ <img src=/img/b.gif width=10 height=6></TD>
+ <TD valign=bottom width=20><IMG src=/img/b.gif width=20 height=8></TD>
+ </TR>
+ </TABLE>
+ </TD>
+ </TR>
+ <TR valign=center align=center>
+ <TD colspan=2 background=/img/inside-2x1.gif>
+<!--ENGLISH TOP--><div id="RTBDIV_50381">
+ <div id="RTBPL_50381">
+ <a href="//rtbsystem.com/ru/advertiser/request">Добавить рекламное обьявление</a>
+ </div>
+</div>
+<script type="text/javascript">
+ if (document.getElementById('RTBDIV_50381')) {
+ document.write('<scr'+'ipt type="text/javascript" async '
+ +'src="//code.rtbsystem.com/50381.js?t='+ new Date().getTime() + '" charset="utf-8" ></scr'+'ipt>');
+ }
+</script><!-- Start Counter -->
+<!-- SpyLOG v2 f:0211 -->
+<script language="javascript">
+u="u311.79.spylog.com";d=document;nv=navigator;na=nv.appName;t="";p=1;
+sz=" width=1 height=1 ";
+hl=history.length;d.cookie="b=b";c=0;
+bv=Math.round(parseFloat(nv.appVersion)*100);
+if (d.cookie) c=1;n=(na.substring(0,2)=="Mi")?0:1;
+if((n==0)||(bv >= 300)){rn=Math.random();t=(new Date()).getTimezoneOffset();} else {rn=0;}
+z="p="+p+"&rn="+rn+"&t="+t+"&c="+c+"&hl="+hl;
+if (self != top) { fr=1;} else { fr=0;}
+r=escape(d.referrer);r1="";
+sl="1.0";h=0;
+</script>
+<script language="javascript1.1">
+pl="";sl="1.1";
+if((n==1) && (bv >= 300))
+{ for(var i = 0; i < nv.plugins.length; i++)
+pl += nv.plugins[i].name+":"; }
+j = (navigator.javaEnabled() ? "Y" : "N");
+</script>
+<script language=javascript1.2>
+sl="1.2";s=screen;wh=s.width+'x'+s.height;
+px=(n==0)?screen.colorDepth:screen.pixelDepth;z+="&wh="+wh+"&px="+px;
+</script>
+<script language=javascript1.3>
+sl="1.3"
+</script>
+<script language="javascript">
+y="";
+y+="<a href='http://"+u+"/cnt?f=3&p="+p+"&rn="+rn+"' target=_blank>";
+y+="<img src='http://"+u+"/cnt?";
+y+=z+"&j="+j+"&sl="+sl+"&r="+r+"&r1="+r1+"&fr="+fr+"&pg="+escape(window.location.href)+"&pl="+escape(pl);
+y+="' border=0 "+sz+" alt='SpyLOG' align=right>";
+y+="</a>";
+d.write(y);
+</script>
+<script language="javascript1.2"><!--
+if (n == 0) { d.write("<");d.write("!--"); }
+//--></script>
+<noscript>
+<a href="http://u311.79.spylog.com/cnt?f=3&p=1" target=_blank>
+<img src="http://u311.79.spylog.com/cnt?p=1" alt='SpyLOG' border='0' width=1 height=1 align=right>
+</a>
+</noscript>
+<script language="javascript1.2"><!--
+if (n == 0) { d.write("--");d.write(">"); }
+//--></script>
+<!-- SpyLOG -->
+<!-- End Counter -->
+</TD>
+</TR>
+</table>
+<input type=hidden name=l value=en>
+</form>
+
+
+<table border=0>
+<tr>
+<td valign=top >
+
+<script type="text/javascript"><!--
+google_ad_client = "pub-7005770717406130";
+google_ad_width = 160;
+google_ad_height = 600;
+google_ad_format = "160x600_as";
+google_ad_type = "text_image";
+//2006-12-05: filesearch_left_160x600
+google_ad_channel = "8508766014";
+google_color_border = "FFFFFF";
+google_color_bg = "FFFFFF";
+google_color_link = "000000";
+google_color_text = "000000";
+google_color_url = "000000";
+//--></script>
+<script type="text/javascript"
+ src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+</script>
+
+</td>
+
+<td valign=top >
+
+<pre class=list>
+</pre>
+
+
+<script type="text/javascript"><!--
+google_ad_client = "pub-7005770717406130";
+/* filesearch_down_728x90 */
+google_ad_slot = "6424137562";
+google_ad_width = 728;
+google_ad_height = 90;
+//-->
+</script>
+<script type="text/javascript"
+src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+</script>
+
+ </td></tr></table> <blockquote>
+<b>zstd-1.3.3.tar.gz</b> not found<br>
+Try to use <a href=http://www.filesearching.com/><b>search forms</b></a> to refine the parameters of your request.
+<br></blockquote>
+
+<table width=100% border=0 cellspacing=0 cellpadding=0 bgcolor=#FAE6CA>
+<tr bgcolor=#000000><td><img src=/img/b.gif height=1 width=1></td></tr>
+<tr>
+<td background=/img/inside-4x1-1x6_1.gif><img src=/img/b.gif height=20 width=20></td>
+</tr></table>
+<table width=100% border=0 cellspacing=0 cellpadding=0 bgcolor=#FAE6CA>
+ <tr><td align=center>
+<!--ENGLISH BOT--><br><br>
+</td></tr>
+</table>
+
+<!--Kavanga START-->
+
+<script type="text/javascript"><!--
+if (Math.ceil(Math.random()*10) ) {
+ <!--Kavanga START-->
+ <!--Сайт: filesearch.ru-->
+ <!--Категория: Компьютеры и ПО-->
+ if (typeof(pr) == 'undefined') { var pr = Math.floor(Math.random() * 1000000); }
+ if (typeof(document.referrer) != 'undefined') {
+ if (typeof(afReferrer) == 'undefined') {
+ afReferrer = escape(document.referrer);
+ }
+ } else {
+ afReferrer = '';
+ }
+ var addate = new Date();
+
+ if (Math.ceil(Math.random()*10) ) {
+ <!--Тип баннера: Pop Under-->
+ document.write('<scr' + 'ipt type="text/javascript" src="http://a.kavanga.ru/3604/prepareCode?ph=b&amp;p1=our&amp;p2=o&amp;pucn=a&amp;pfc=a&amp;pfb=a&amp;plp=a&amp;pli=a&amp;pop=a&amp;pr=' + pr + '&amp;pt=b&amp;pd=' + addate.getDate() + '&amp;pw=' + addate.getDay() + '&amp;pv=' + addate.getHours() + '&amp;prr=' + afReferrer + '"><\/scr' + 'ipt>');
+ } else {
+ <!--Тип баннера: Rich Media-->
+ document.write('<scr' + 'ipt type="text/javascript" src="http://a.kavanga.ru/3604/prepareCode?p1=oip&amp;p2=p&amp;pucn=a&amp;pfc=a&amp;pfb=a&amp;plp=a&amp;pli=a&amp;pop=a&amp;pr=' + pr +'&amp;pt=b&amp;pd=' + addate.getDate() + '&amp;pw=' + addate.getDay() + '&amp;pv=' + addate.getHours() + '&amp;py=a&amp;prr=' + afReferrer + '"><\/scr' + 'ipt>');
+ }
+ <!--Kavanga END-->
+} else {
+ <!-- AdMedia START -->
+ var RndNum4NoCash = Math.round(Math.random() * 1000000000);
+ document.write('<script language="JavaScript" src="http://ad.adriver.ru/cgi-bin/erle.cgi?sid=36667&target=top&bt=16&pz=0&rnd=' + RndNum4NoCash + '"><\/script>');
+ <!-- AdMedia END -->
+}
+// -->
+</script>
+
+<!--Kavanga END-->
+</BODY></HTML>
diff --git a/service/main.cpp b/service/main.cpp
new file mode 100644
index 0000000..97bc89e
--- /dev/null
+++ b/service/main.cpp
@@ -0,0 +1,18 @@
+#include <Ice/Communicator.h>
+#include <Ice/ObjectAdapter.h>
+#include <icetrayService.h>
+#include "apiImpl.h"
+
+namespace MirrorSearch {
+ class Api : public IceTray::Service {
+ public:
+ void addObjects(const std::string &, const Ice::CommunicatorPtr & ic, const Ice::StringSeq &, const Ice::ObjectAdapterPtr & adp) override
+ {
+ auto dbpool = getConnectionPool(ic, "postgresql", "MirrorSearch");
+ adp->add(new SearchImpl(dbpool), ic->stringToIdentity("Search"));
+ }
+ };
+
+ NAMEDFACTORY("default", MirrorSearch::Api, IceTray::ServiceFactory);
+}
+
diff --git a/service/models.ice b/service/models.ice
new file mode 100644
index 0000000..cb48459
--- /dev/null
+++ b/service/models.ice
@@ -0,0 +1,25 @@
+#ifndef MIRRORSEARCH_MODELS
+#define MIRRORSEARCH_MODELS
+
+module MirrorSearch {
+ class SearchService {
+ ["slicer:db:pkey"]
+ int id;
+ string name;
+ string baseurl;
+ string listxpath;
+ string urlxpath;
+ };
+ class SearchHit {
+ ["slicer:db:pkey"]
+ int id;
+ int serviceid;
+ string url;
+ };
+
+ sequence<SearchService> SearchServices;
+ sequence<SearchHit> SearchHits;
+};
+
+#endif
+
diff --git a/service/schema.sql b/service/schema.sql
new file mode 100644
index 0000000..b529172
--- /dev/null
+++ b/service/schema.sql
@@ -0,0 +1,8 @@
+CREATE TABLE searchservices(
+ id int GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+ name text not null,
+ baseurl text not null,
+ listxpath text not null,
+ urlxpath text not null
+);
+
diff --git a/service/sql/getServices.sql b/service/sql/getServices.sql
new file mode 100644
index 0000000..40382f2
--- /dev/null
+++ b/service/sql/getServices.sql
@@ -0,0 +1,3 @@
+SELECT id, name, baseurl, listxpath, urlxpath
+FROM searchservices
+ORDER BY id
diff --git a/service/test.cpp b/service/test.cpp
new file mode 100644
index 0000000..c115712
--- /dev/null
+++ b/service/test.cpp
@@ -0,0 +1,77 @@
+#define BOOST_TEST_MODULE MirrorSearch
+#include <boost/test/unit_test.hpp>
+
+#include <pq-mock.h>
+#include <dryice.h>
+#include <definedDirs.h>
+#include <api.h>
+
+class Service : PQ::Mock, public IceTray::DryIce {
+ public:
+ Service() : PQ::Mock("user=postgres", "MirrorSearch", {
+ rootDir / "schema.sql",
+ rootDir / "data.sql"
+ }) { }
+
+};
+
+class TestClient : public IceTray::DryIceClient {
+ public:
+ TestClient() :
+ s(getProxy<MirrorSearch::SearchPrx>("Search"))
+ {
+ }
+
+ MirrorSearch::SearchPrx s;
+};
+
+BOOST_TEST_GLOBAL_FIXTURE(Service);
+
+BOOST_FIXTURE_TEST_SUITE(tc, TestClient);
+
+BOOST_AUTO_TEST_CASE(sanity)
+{
+ BOOST_REQUIRE(s);
+ s->ice_ping();
+}
+
+BOOST_AUTO_TEST_CASE(getServices)
+{
+ auto ss = s->getServices();
+ BOOST_REQUIRE_EQUAL(ss.size(), 1);
+ BOOST_CHECK_EQUAL(ss.front()->id, 1);
+ BOOST_CHECK_EQUAL(ss.front()->name, "file searching mock");
+ BOOST_CHECK_NE(ss.front()->baseurl, "file://$SCRIPTDIR/fixtures/filesearching/%s.html");
+ BOOST_CHECK_EQUAL(ss.front()->baseurl.substr(0, 8), "file:///");
+ BOOST_CHECK(!ss.front()->listxpath.empty());
+ BOOST_CHECK(!ss.front()->urlxpath.empty());
+}
+
+BOOST_AUTO_TEST_CASE(getMatches_zstd_notfound)
+{
+ auto ms = s->getMatches("zstd-1.3.3.tar.gz");
+ BOOST_REQUIRE(ms.empty());
+}
+
+BOOST_AUTO_TEST_CASE(getMatches_xtrans)
+{
+ auto ms = s->getMatches("xtrans-1.3.5.tar.bz2");
+ BOOST_REQUIRE_EQUAL(ms.size(), 12);
+ BOOST_REQUIRE_EQUAL(ms.front()->url, "ftp://ftp.cs.mun.ca/pub/mirror/gentoo/distfiles/xtrans-1.3.5.tar.bz2");
+ BOOST_REQUIRE_EQUAL(ms.back()->url, "ftp://ftp.tr.debian.org/gentoo/distfiles/xtrans-1.3.5.tar.bz2");
+}
+
+BOOST_AUTO_TEST_CASE(getMatches_zstd_notfound_lucky)
+{
+ BOOST_REQUIRE(!s->feelingLucky("zstd-1.3.3.tar.gz"));
+}
+
+BOOST_AUTO_TEST_CASE(getMatches_xtrans_lucky)
+{
+ auto fl = s->feelingLucky("xtrans-1.3.5.tar.bz2");
+ BOOST_REQUIRE(fl);
+ BOOST_REQUIRE_EQUAL(*fl, "ftp://ftp.cs.mun.ca/pub/mirror/gentoo/distfiles/xtrans-1.3.5.tar.bz2");
+}
+
+BOOST_AUTO_TEST_SUITE_END();
+
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 07:41:17 +0000