Initial commit, WIP

12 files changed, 827 insertions, 0 deletions

diff --git a/service/Jamfile.jam b/service/Jamfile.jam
new file mode 100644
index 0000000..e7bfed9
--- /dev/null
+++ b/service/Jamfile.jam
@@ -0,0 +1,64 @@
+import icetray ;
+import package ;
+import testing ;
+
+lib boost_utf : : <name>boost_unit_test_framework ;
+lib dbpp-postgresql : : : : <include>/usr/include/dbpp-postgresql ;
+lib dryice : : : : <include>/usr/include/icetray ;
+
+lib mirrorsearch :
+	[ glob *.cpp *.ice sql/*.sql : test.cpp ]
+	:
+	<slicer>yes
+	<library>..//adhocutil
+	<library>..//dbppcore
+	<library>..//boost_system
+	<library>..//boost_filesystem
+	<library>..//boost_date_time
+	<library>..//Ice
+	<library>..//IceBox
+	<library>..//IceUtil
+	<library>..//pthread
+	<library>..//icetray
+	<library>..//slicer
+	<library>..//slicer-db
+	<library>..//glibmm
+	<library>..//xml2
+	<icetray.sql.namespace>MirrorSearch
+	<icetray.sql.basedir>.
+	<include>.
+	: :
+	<include>.
+	<library>..//icetray
+	;
+
+path-constant me : . ;
+
+run test.cpp
+	: :
+	data.sql
+	schema.sql
+	:
+	<define>BOOST_TEST_DYN_LINK
+	<library>..//boost_system
+	<library>..//boost_filesystem
+	<library>boost_utf
+	<define>ROOT=\"$(me)\"
+	<library>dbpp-postgresql
+	<library>..//dbppcore
+	<library>..//adhocutil
+	<library>..//boost_system
+	<library>..//boost_filesystem
+	<library>..//netfs-api
+	<library>..//IceUtil
+	<library>..//Ice
+	<library>..//IceBox
+	<library>..//pthread
+	<library>dryice
+	<library>mirrorsearch
+	<implicit-dependency>mirrorsearch
+;
+
+package.install install : : : mirrorsearch ;
+
+
diff --git a/service/api.ice b/service/api.ice
new file mode 100644
index 0000000..c207c3d
--- /dev/null
+++ b/service/api.ice
@@ -0,0 +1,19 @@
+#ifndef MIRRORSEARCH_API
+#define MIRRORSEARCH_API
+
+#include <models.ice>
+
+module MirrorSearch {
+	exception XmlError {
+		string msg;
+	};
+
+	interface Search {
+		idempotent SearchServices getServices();
+		idempotent SearchHits getMatches(string filename);
+		idempotent optional(0) string feelingLucky(string filename);
+	};
+};
+
+#endif
+
diff --git a/service/apiImpl.cpp b/service/apiImpl.cpp
new file mode 100644
index 0000000..588d360
--- /dev/null
+++ b/service/apiImpl.cpp
@@ -0,0 +1,91 @@
+#include "apiImpl.h"
+
+#include <sql/getServices.sql.h>
+#include <buffer.h>
+#include <memory>
+
+#include <libxml/xpath.h>
+#include <libxml/xpathInternals.h>
+#include <libxml/HTMLparser.h>
+#include <libxml/HTMLtree.h>
+
+namespace MirrorSearch {
+	SearchImpl::SearchImpl(IceTray::DatabasePoolPtr db) :
+		IceTray::AbstractDatabaseClient(db),
+		log(LOGMANAGER()->getLogger<SearchImpl>())
+	{
+	}
+
+	SearchServices SearchImpl::getServices(const ::Ice::Current&)
+	{
+		return fetch<SearchServices>(sql::getServices);
+	}
+
+	typedef std::shared_ptr<xmlDoc> xmlDocSPtr;
+	typedef std::shared_ptr<xmlXPathContext> xmlXPathContextSPtr;
+	typedef std::shared_ptr<xmlXPathObject> xmlXPathObjectSPtr;
+
+	static auto getDoc(const ::std::string & url, int flags)
+	{
+		if (auto doc = xmlDocSPtr(htmlReadFile(url.c_str(), NULL, flags), xmlFreeDoc)) {
+			return doc;
+		}
+		throw XmlError("Failed to open " + url);
+	}
+
+	static auto getXPathCxt(const xmlDocSPtr & doc)
+	{
+		if (auto xpathCtx = xmlXPathContextSPtr(xmlXPathNewContext(doc.get()), xmlXPathFreeContext)) {
+			return xpathCtx;
+		}
+		throw XmlError("Failed to create xpath context");
+	}
+
+	static auto getXPathObj(const ::std::string & xpath, const xmlXPathContextSPtr & ctx, xmlXPathObjectType type)
+	{
+		if (auto xpathObj = xmlXPathObjectSPtr(xmlXPathEvalExpression(BAD_CAST xpath.c_str(), ctx.get()), xmlXPathFreeObject)) {
+			if (xpathObj->type != type) {
+				throw XmlError("Xpath evaluates to wrong type " + xpath);
+			}
+			return xpathObj;
+		}
+		throw XmlError("Failed to evaluate xpath " + xpath);
+	}
+
+	void SearchImpl::callService(const ::std::string & fn, const SearchServicePtr & s, SearchHits & sh) const
+	{
+		auto fmt = AdHoc::Buffer::getFormat(s->baseurl);
+		auto url = (*fmt % fn).str();
+		auto doc = getDoc(url,
+				HTML_PARSE_RECOVER | HTML_PARSE_NODEFDTD | HTML_PARSE_NOIMPLIED |
+				HTML_PARSE_NOWARNING | HTML_PARSE_NOERROR);
+		auto xpathCtx = getXPathCxt(doc);
+		auto xpathObj = getXPathObj(s->listxpath, xpathCtx, xmlXPathObjectType::XPATH_NODESET);
+		log->messagebf(LOG::INFO, "%d nodes matched %s", xpathObj->nodesetval->nodeNr, s->listxpath);
+		for (int row = 0; row < xpathObj->nodesetval->nodeNr; row += 1) {
+			xpathCtx->node = xpathObj->nodesetval->nodeTab[row];
+			auto xpathObjI = getXPathObj(s->urlxpath, xpathCtx, xmlXPathObjectType::XPATH_STRING);
+			if (xpathObjI->stringval && *xpathObjI->stringval) {
+				sh.push_back(new SearchHit(0, s->id, (const char *) xpathObjI->stringval));
+			}
+		}
+	}
+
+	SearchHits SearchImpl::getMatches(const ::std::string & fn, const ::Ice::Current & c)
+	{
+		SearchHits sh;
+		for (const auto & s : getServices(c)) {
+			callService(fn, s, sh);
+		}
+		return sh;
+	}
+
+	::IceUtil::Optional<::std::string> SearchImpl::feelingLucky(const ::std::string & fn, const ::Ice::Current & c)
+	{
+		const auto ms = getMatches(fn, c);
+		if (ms.empty())
+			return IceUtil::None;
+		return ms.front()->url;
+	}
+}
+
diff --git a/service/apiImpl.h b/service/apiImpl.h
new file mode 100644
index 0000000..e56368c
--- /dev/null
+++ b/service/apiImpl.h
@@ -0,0 +1,25 @@
+#ifndef MIRRORSEARCH_APIIMPL_H
+#define MIRRORSEARCH_APIIMPL_H
+
+#include <api.h>
+#include <abstractDatabaseClient.h>
+#include <logger.h>
+
+namespace MirrorSearch {
+	class SearchImpl : public Search, public IceTray::AbstractDatabaseClient {
+		public:
+			SearchImpl(IceTray::DatabasePoolPtr);
+
+			virtual SearchServices getServices(const ::Ice::Current& = ::Ice::Current()) override;
+			virtual SearchHits getMatches(const ::std::string&, const ::Ice::Current& = ::Ice::Current()) override;
+			virtual ::IceUtil::Optional<::std::string> feelingLucky(const ::std::string&, const ::Ice::Current& = ::Ice::Current()) override;
+
+		private:
+			void callService(const ::std::string & fn, const SearchServicePtr & s, SearchHits & sh) const;
+
+			LOG::LoggerPtr log;
+	};
+}
+
+#endif
+
diff --git a/service/data.sql b/service/data.sql
new file mode 100644
index 0000000..e14e721
--- /dev/null
+++ b/service/data.sql
@@ -0,0 +1,3 @@
+INSERT INTO searchservices(name, baseurl, listxpath, urlxpath)
+  VALUES('file searching mock', 'file://$SCRIPTDIR/fixtures/filesearching/%s.html', '//pre[@class=''list'']/a[@class=''lf'']', 'string(@href)')
+  ;
diff --git a/service/fixtures/filesearching/xtrans-1.3.5.tar.bz2.html b/service/fixtures/filesearching/xtrans-1.3.5.tar.bz2.html
new file mode 100644
index 0000000..27b627a
--- /dev/null
+++ b/service/fixtures/filesearching/xtrans-1.3.5.tar.bz2.html
@@ -0,0 +1,251 @@
+<HTML>
+<HEAD>
+<TITLE>FileSearch - xtrans-1.3.5.tar.bz2</TITLE>
+<link rel=stylesheet href=/styles/fs.css>
+</HEAD>
+<BODY bgcolor=#FFFFFF leftmargin=0 topmargin=0 marginwidth=0 marginheight=0>
+<FORM ACTION=/cgi-bin/s METHOD=GET>
+<TABLE width=100% border=0 cellspacing=0 cellpadding=0>
+ <TR bgcolor=#CCA782 valign=bottom align=left>
+ <TD colspan=2 height=87>
+ <TABLE width=100% border=0 cellspacing=0 cellpadding=0>
+  <TR>
+  <TD width=160 align=right valign=bottom><a href=/><IMG src=/img/b.gif width=160 height=10 border=0><img src=/img/filesearchen.gif width=142 height=77 border=0 valign=bottom></a></TD>
+  <TD valign=bottom width=100%>
+  <TABLE width=100% border=0 cellspacing=0 cellpadding=0>
+  <TR>
+  <td bgcolor=#CCA782 valign=bottom class=text align=center><img src=/img/b.gif height=1 width=10></td>
+  <td rowspan=2 background=/img/poisklght.gif bgcolor=#CCA782 valign=bottom class=taplight align=center><img src=/img/b.gif height=1 width=73><br>search</td>
+  <td rowspan=2 background=/img/advpoiskdrk.gif bgcolor=#CCA782 valign=bottom align=center width=135 class=taplight nowrap><img src=/img/b.gif height=1 width=135><br><a href=/advanced/ class=tapdark>advanced search</a></td>
+  <td align=right class=text width=100%>...</td>
+  <td align=right><img src=/img/b.gif width=1 height=21></td>
+  </TR>
+  <TR>
+   <TD bgcolor=#000000 valign=bottom class=text align=center><img src=/img/b.gif height=1 width=10></TD>
+   <TD align=right bgcolor=#000000 colspan=2><IMG src=/img/b.gif width=274 height=1></TD>
+  </TR>
+  </TABLE>
+  <TABLE border=0 cellpadding=0 cellspacing=0 width=100%>
+  <TR>
+  <td width=1 bgcolor=#000000 ><img src=/img/b.gif width=1 height=48></td>
+  <td bgcolor=#FFE7CE valign=middle nowrap>&nbsp;
+  <INPUT type=text name=q value="xtrans-1.3.5.tar.bz2" SIZE=20>&nbsp;
+  <select name=t>
+  <option VALUE=f selected>file/directory
+  <option VALUE=m>music (mp3)
+  <option VALUE=p>images
+  <option VALUE=v>video
+  <option VALUE=s>server
+  </select>
+  <select name=d>
+<option value="">all sites
+<option value="com">.com
+<option value="net">.net
+<option value="edu">.edu
+<option value="org">.org
+<option value="au">.au
+<option value="be">.be
+<option value="ca">.ca
+<option value="ch">.ch
+<option value="de">.de
+<option value="dk">.dk
+<option value="es">.es
+<option value="fr">.fr
+<option value="it">.it
+<option value="jp">.jp
+<option value="lv">.lv
+<option value="nl">.nl
+<option value="no">.no
+<option value="ru,su">.ru
+<option value="se">.se
+<option value="ua">.ua
+<option value="uk">.uk
+
+  </select>&nbsp;
+  <img src=/img/b.gif height=1 width=5></td>
+  <td bgcolor=#FFE7CE valign=middle width=630>
+  <input type=image src=/img/findengl.gif border=0>
+  </td>
+  <td width=1 bgcolor=#000000 ><img src=/img/b.gif width=1 height=46></td>
+  </TR>
+  <TR>
+   <TD rowspan=1 colspan=4 bgcolor=#000000><IMG src=/img/b.gif width=3 height=1 border=0></TD>
+  </TR>
+  </TABLE>
+  <img src=/img/b.gif width=10 height=6></TD>
+  <TD valign=bottom width=20><IMG src=/img/b.gif width=20 height=8></TD>
+  </TR>
+ </TABLE>
+ </TD>
+ </TR>
+ <TR valign=center align=center>
+ <TD colspan=2 background=/img/inside-2x1.gif>
+<!--ENGLISH TOP--><div id="RTBDIV_50381">

+        <div id="RTBPL_50381">

+            <a href="//rtbsystem.com/ru/advertiser/request">Добавить рекламное обьявление</a>

+        </div>

+</div>

+<script type="text/javascript">

+    if (document.getElementById('RTBDIV_50381')) {

+        document.write('<scr'+'ipt type="text/javascript" async '

+            +'src="//code.rtbsystem.com/50381.js?t='+ new Date().getTime() + '" charset="utf-8" ></scr'+'ipt>');

+    }

+</script><!-- Start Counter -->
+<!-- SpyLOG v2 f:0211 -->
+<script language="javascript">
+u="u311.79.spylog.com";d=document;nv=navigator;na=nv.appName;t="";p=1;
+sz=" width=1 height=1 ";
+hl=history.length;d.cookie="b=b";c=0;
+bv=Math.round(parseFloat(nv.appVersion)*100);
+if (d.cookie) c=1;n=(na.substring(0,2)=="Mi")?0:1;
+if((n==0)||(bv >= 300)){rn=Math.random();t=(new Date()).getTimezoneOffset();} else {rn=0;}
+z="p="+p+"&rn="+rn+"&t="+t+"&c="+c+"&hl="+hl;
+if (self != top) { fr=1;} else { fr=0;}
+r=escape(d.referrer);r1="";
+sl="1.0";h=0;
+</script>
+<script language="javascript1.1">
+pl="";sl="1.1";
+if((n==1) && (bv >= 300))
+{ for(var i = 0; i < nv.plugins.length; i++)
+pl += nv.plugins[i].name+":"; }
+j = (navigator.javaEnabled() ? "Y" : "N");
+</script>
+<script language=javascript1.2>
+sl="1.2";s=screen;wh=s.width+'x'+s.height;
+px=(n==0)?screen.colorDepth:screen.pixelDepth;z+="&wh="+wh+"&px="+px;
+</script>
+<script language=javascript1.3>
+sl="1.3"
+</script>
+<script language="javascript">
+y="";
+y+="<a href='http://"+u+"/cnt?f=3&p="+p+"&rn="+rn+"' target=_blank>";
+y+="<img src='http://"+u+"/cnt?";
+y+=z+"&j="+j+"&sl="+sl+"&r="+r+"&r1="+r1+"&fr="+fr+"&pg="+escape(window.location.href)+"&pl="+escape(pl);
+y+="' border=0 "+sz+" alt='SpyLOG' align=right>";
+y+="</a>";
+d.write(y);
+</script>
+<script language="javascript1.2"><!--
+if (n == 0) { d.write("<");d.write("!--"); }
+//--></script>
+<noscript>
+<a href="http://u311.79.spylog.com/cnt?f=3&p=1" target=_blank>
+<img src="http://u311.79.spylog.com/cnt?p=1" alt='SpyLOG' border='0' width=1 height=1 align=right>
+</a>
+</noscript>
+<script language="javascript1.2"><!--
+if (n == 0) { d.write("--");d.write(">"); }
+//--></script>
+<!-- SpyLOG -->
+<!-- End Counter -->
+</TD>
+</TR>
+</table>
+<input type=hidden name=l value=en>
+</form>
+
+
+<table border=0>
+<tr>
+<td valign=top >
+
+<script type="text/javascript"><!--
+google_ad_client = "pub-7005770717406130";
+google_ad_width = 160;
+google_ad_height = 600;
+google_ad_format = "160x600_as";
+google_ad_type = "text_image";
+//2006-12-05: filesearch_left_160x600
+google_ad_channel = "8508766014";
+google_color_border = "FFFFFF";
+google_color_bg = "FFFFFF";
+google_color_link = "000000";
+google_color_text = "000000";
+google_color_url = "000000";
+//--></script>
+<script type="text/javascript"
+ src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+</script>
+
+</td>
+
+<td valign=top >
+
+<pre class=list>
+   1 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.cs.mun.ca/ class=ls>ftp.cs.mun.ca</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.cs.mun.ca/pub/mirror/gentoo/distfiles class=lg>/pub/mirror/gentoo/distfiles/</a><a href=ftp://ftp.cs.mun.ca/pub/mirror/gentoo/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+   2 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.dvo.ru/ class=ls>ftp.dvo.ru</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.dvo.ru/pub/Gentoo/distfiles class=lg>/pub/Gentoo/distfiles/</a><a href=ftp://ftp.dvo.ru/pub/Gentoo/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+   3 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.dvo.ru/ class=ls>ftp.dvo.ru</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.dvo.ru/pub/distfiles class=lg>/pub/distfiles/</a><a href=ftp://ftp.dvo.ru/pub/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+   4 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.fu-berlin.de/ class=ls>ftp.fu-berlin.de</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.fu-berlin.de/unix/X11/FTP.X.ORG/pub/individual/lib class=lg>/unix/X11/FTP.X.ORG/pub/individual/lib/</a><a href=ftp://ftp.fu-berlin.de/unix/X11/FTP.X.ORG/pub/individual/lib/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+   5 <img src=/img/iconany.gif width=16 height=16> <b>    536</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.fu-berlin.de/ class=ls>ftp.fu-berlin.de</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.fu-berlin.de/unix/X11/FTP.X.ORG/pub/individual/lib class=lg>/unix/X11/FTP.X.ORG/pub/individual/lib/</a><a href=ftp://ftp.fu-berlin.de/unix/X11/FTP.X.ORG/pub/individual/lib/xtrans-1.3.5.tar.bz2.sig class=lf>xtrans-1.3.5.tar.bz2.sig</a>
+   6 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.gr.debian.org/ class=ls>ftp.gr.debian.org</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.gr.debian.org/pub/X11/X.org/individual/lib class=lg>/pub/X11/X.org/individual/lib/</a><a href=ftp://ftp.gr.debian.org/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+   7 <img src=/img/iconany.gif width=16 height=16> <b>    536</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.gr.debian.org/ class=ls>ftp.gr.debian.org</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.gr.debian.org/pub/X11/X.org/individual/lib class=lg>/pub/X11/X.org/individual/lib/</a><a href=ftp://ftp.gr.debian.org/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2.sig class=lf>xtrans-1.3.5.tar.bz2.sig</a>
+   8 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.linux.org.tr/ class=ls>ftp.linux.org.tr</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.linux.org.tr/gentoo/distfiles class=lg>/gentoo/distfiles/</a><a href=ftp://ftp.linux.org.tr/gentoo/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+   9 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.ntua.gr/ class=ls>ftp.ntua.gr</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.ntua.gr/pub/X11/X.org/individual/lib class=lg>/pub/X11/X.org/individual/lib/</a><a href=ftp://ftp.ntua.gr/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+  10 <img src=/img/iconany.gif width=16 height=16> <b>    536</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.ntua.gr/ class=ls>ftp.ntua.gr</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.ntua.gr/pub/X11/X.org/individual/lib class=lg>/pub/X11/X.org/individual/lib/</a><a href=ftp://ftp.ntua.gr/pub/X11/X.org/individual/lib/xtrans-1.3.5.tar.bz2.sig class=lf>xtrans-1.3.5.tar.bz2.sig</a>
+  11 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.rediris.es/ class=ls>ftp.rediris.es</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.rediris.es/sites/gentoo.org/distfiles class=lg>/sites/gentoo.org/distfiles/</a><a href=ftp://ftp.rediris.es/sites/gentoo.org/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+  12 <img src=/img/iconany.gif width=16 height=16> <b> 182.5K</b> <a href=/cgi-bin/s?t=n&l=en&q=ftp.tr.debian.org/ class=ls>ftp.tr.debian.org</a><a href=/cgi-bin/s?t=n&l=en&q=ftp.tr.debian.org/gentoo/distfiles class=lg>/gentoo/distfiles/</a><a href=ftp://ftp.tr.debian.org/gentoo/distfiles/xtrans-1.3.5.tar.bz2 class=lf>xtrans-1.3.5.tar.bz2</a>
+</pre>
+
+
+<script type="text/javascript"><!--
+google_ad_client = "pub-7005770717406130";
+/* filesearch_down_728x90 */
+google_ad_slot = "6424137562";
+google_ad_width = 728;
+google_ad_height = 90;
+//-->
+</script>
+<script type="text/javascript"
+src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+</script>
+
+	 </td></tr></table> 
+<table width=100% border=0 cellspacing=0 cellpadding=0 bgcolor=#FAE6CA>
+<tr bgcolor=#000000><td><img src=/img/b.gif height=1 width=1></td></tr>
+<tr>
+<td background=/img/inside-4x1-1x6_1.gif><img src=/img/b.gif height=20 width=20></td>
+</tr></table>
+<table width=100% border=0 cellspacing=0 cellpadding=0 bgcolor=#FAE6CA>
+ <tr><td align=center>
+<!--ENGLISH BOT--><br><br>
+</td></tr>
+</table>
+
+<!--Kavanga START-->
+
+<script type="text/javascript"><!--
+if (Math.ceil(Math.random()*10)  ) {
+    <!--Kavanga START-->
+    <!--Сайт: filesearch.ru-->
+    <!--Категория: Компьютеры и ПО-->
+    if (typeof(pr) == 'undefined') { var pr = Math.floor(Math.random() * 1000000); }
+    if (typeof(document.referrer) != 'undefined') {
+      if (typeof(afReferrer) == 'undefined') {
+        afReferrer = escape(document.referrer);
+      }
+    } else {
+      afReferrer = '';
+    }
+    var addate = new Date();
+
+    if (Math.ceil(Math.random()*10)  ) {
+        <!--Тип баннера: Pop Under-->
+        document.write('<scr' + 'ipt type="text/javascript" src="http://a.kavanga.ru/3604/prepareCode?ph=b&amp;p1=our&amp;p2=o&amp;pucn=a&amp;pfc=a&amp;pfb=a&amp;plp=a&amp;pli=a&amp;pop=a&amp;pr=' + pr + '&amp;pt=b&amp;pd=' + addate.getDate() + '&amp;pw=' + addate.getDay() + '&amp;pv=' + addate.getHours() + '&amp;prr=' + afReferrer + '"><\/scr' + 'ipt>');
+    } else {
+        <!--Тип баннера: Rich Media-->
+        document.write('<scr' + 'ipt type="text/javascript" src="http://a.kavanga.ru/3604/prepareCode?p1=oip&amp;p2=p&amp;pucn=a&amp;pfc=a&amp;pfb=a&amp;plp=a&amp;pli=a&amp;pop=a&amp;pr=' + pr +'&amp;pt=b&amp;pd=' + addate.getDate() + '&amp;pw=' + addate.getDay() + '&amp;pv=' + addate.getHours() + '&amp;py=a&amp;prr=' + afReferrer + '"><\/scr' + 'ipt>');
+    }
+    <!--Kavanga END-->
+} else {
+    <!-- AdMedia START -->
+    var RndNum4NoCash = Math.round(Math.random() * 1000000000);
+    document.write('<script language="JavaScript" src="http://ad.adriver.ru/cgi-bin/erle.cgi?sid=36667&target=top&bt=16&pz=0&rnd=' + RndNum4NoCash + '"><\/script>');
+    <!-- AdMedia END -->
+}
+// -->
+</script>
+
+<!--Kavanga END-->
+</BODY></HTML>
diff --git a/service/fixtures/filesearching/zstd-1.3.3.tar.gz.html b/service/fixtures/filesearching/zstd-1.3.3.tar.gz.html
new file mode 100644
index 0000000..74e70e2
--- /dev/null
+++ b/service/fixtures/filesearching/zstd-1.3.3.tar.gz.html
@@ -0,0 +1,243 @@
+<HTML>
+<HEAD>
+<TITLE>FileSearch - zstd-1.3.3.tar.gz</TITLE>
+<link rel=stylesheet href=/styles/fs.css>
+</HEAD>
+<BODY bgcolor=#FFFFFF leftmargin=0 topmargin=0 marginwidth=0 marginheight=0>
+<FORM ACTION=/cgi-bin/s METHOD=GET>
+<TABLE width=100% border=0 cellspacing=0 cellpadding=0>
+ <TR bgcolor=#CCA782 valign=bottom align=left>
+ <TD colspan=2 height=87>
+ <TABLE width=100% border=0 cellspacing=0 cellpadding=0>
+  <TR>
+  <TD width=160 align=right valign=bottom><a href=/><IMG src=/img/b.gif width=160 height=10 border=0><img src=/img/filesearchen.gif width=142 height=77 border=0 valign=bottom></a></TD>
+  <TD valign=bottom width=100%>
+  <TABLE width=100% border=0 cellspacing=0 cellpadding=0>
+  <TR>
+  <td bgcolor=#CCA782 valign=bottom class=text align=center><img src=/img/b.gif height=1 width=10></td>
+  <td rowspan=2 background=/img/poisklght.gif bgcolor=#CCA782 valign=bottom class=taplight align=center><img src=/img/b.gif height=1 width=73><br>search</td>
+  <td rowspan=2 background=/img/advpoiskdrk.gif bgcolor=#CCA782 valign=bottom align=center width=135 class=taplight nowrap><img src=/img/b.gif height=1 width=135><br><a href=/advanced/ class=tapdark>advanced search</a></td>
+  <td align=right class=text width=100%>...</td>
+  <td align=right><img src=/img/b.gif width=1 height=21></td>
+  </TR>
+  <TR>
+   <TD bgcolor=#000000 valign=bottom class=text align=center><img src=/img/b.gif height=1 width=10></TD>
+   <TD align=right bgcolor=#000000 colspan=2><IMG src=/img/b.gif width=274 height=1></TD>
+  </TR>
+  </TABLE>
+  <TABLE border=0 cellpadding=0 cellspacing=0 width=100%>
+  <TR>
+  <td width=1 bgcolor=#000000 ><img src=/img/b.gif width=1 height=48></td>
+  <td bgcolor=#FFE7CE valign=middle nowrap>&nbsp;
+  <INPUT type=text name=q value="zstd-1.3.3.tar.gz" SIZE=20>&nbsp;
+  <select name=t>
+  <option VALUE=f selected>file/directory
+  <option VALUE=m>music (mp3)
+  <option VALUE=p>images
+  <option VALUE=v>video
+  <option VALUE=s>server
+  </select>
+  <select name=d>
+<option value="">all sites
+<option value="com">.com
+<option value="net">.net
+<option value="edu">.edu
+<option value="org">.org
+<option value="au">.au
+<option value="be">.be
+<option value="ca">.ca
+<option value="ch">.ch
+<option value="de">.de
+<option value="dk">.dk
+<option value="es">.es
+<option value="fr">.fr
+<option value="it">.it
+<option value="jp">.jp
+<option value="lv">.lv
+<option value="nl">.nl
+<option value="no">.no
+<option value="ru,su">.ru
+<option value="se">.se
+<option value="ua">.ua
+<option value="uk">.uk
+
+  </select>&nbsp;
+  <img src=/img/b.gif height=1 width=5></td>
+  <td bgcolor=#FFE7CE valign=middle width=630>
+  <input type=image src=/img/findengl.gif border=0>
+  </td>
+  <td width=1 bgcolor=#000000 ><img src=/img/b.gif width=1 height=46></td>
+  </TR>
+  <TR>
+   <TD rowspan=1 colspan=4 bgcolor=#000000><IMG src=/img/b.gif width=3 height=1 border=0></TD>
+  </TR>
+  </TABLE>
+  <img src=/img/b.gif width=10 height=6></TD>
+  <TD valign=bottom width=20><IMG src=/img/b.gif width=20 height=8></TD>
+  </TR>
+ </TABLE>
+ </TD>
+ </TR>
+ <TR valign=center align=center>
+ <TD colspan=2 background=/img/inside-2x1.gif>
+<!--ENGLISH TOP--><div id="RTBDIV_50381">

+        <div id="RTBPL_50381">

+            <a href="//rtbsystem.com/ru/advertiser/request">Добавить рекламное обьявление</a>

+        </div>

+</div>

+<script type="text/javascript">

+    if (document.getElementById('RTBDIV_50381')) {

+        document.write('<scr'+'ipt type="text/javascript" async '

+            +'src="//code.rtbsystem.com/50381.js?t='+ new Date().getTime() + '" charset="utf-8" ></scr'+'ipt>');

+    }

+</script><!-- Start Counter -->
+<!-- SpyLOG v2 f:0211 -->
+<script language="javascript">
+u="u311.79.spylog.com";d=document;nv=navigator;na=nv.appName;t="";p=1;
+sz=" width=1 height=1 ";
+hl=history.length;d.cookie="b=b";c=0;
+bv=Math.round(parseFloat(nv.appVersion)*100);
+if (d.cookie) c=1;n=(na.substring(0,2)=="Mi")?0:1;
+if((n==0)||(bv >= 300)){rn=Math.random();t=(new Date()).getTimezoneOffset();} else {rn=0;}
+z="p="+p+"&rn="+rn+"&t="+t+"&c="+c+"&hl="+hl;
+if (self != top) { fr=1;} else { fr=0;}
+r=escape(d.referrer);r1="";
+sl="1.0";h=0;
+</script>
+<script language="javascript1.1">
+pl="";sl="1.1";
+if((n==1) && (bv >= 300))
+{ for(var i = 0; i < nv.plugins.length; i++)
+pl += nv.plugins[i].name+":"; }
+j = (navigator.javaEnabled() ? "Y" : "N");
+</script>
+<script language=javascript1.2>
+sl="1.2";s=screen;wh=s.width+'x'+s.height;
+px=(n==0)?screen.colorDepth:screen.pixelDepth;z+="&wh="+wh+"&px="+px;
+</script>
+<script language=javascript1.3>
+sl="1.3"
+</script>
+<script language="javascript">
+y="";
+y+="<a href='http://"+u+"/cnt?f=3&p="+p+"&rn="+rn+"' target=_blank>";
+y+="<img src='http://"+u+"/cnt?";
+y+=z+"&j="+j+"&sl="+sl+"&r="+r+"&r1="+r1+"&fr="+fr+"&pg="+escape(window.location.href)+"&pl="+escape(pl);
+y+="' border=0 "+sz+" alt='SpyLOG' align=right>";
+y+="</a>";
+d.write(y);
+</script>
+<script language="javascript1.2"><!--
+if (n == 0) { d.write("<");d.write("!--"); }
+//--></script>
+<noscript>
+<a href="http://u311.79.spylog.com/cnt?f=3&p=1" target=_blank>
+<img src="http://u311.79.spylog.com/cnt?p=1" alt='SpyLOG' border='0' width=1 height=1 align=right>
+</a>
+</noscript>
+<script language="javascript1.2"><!--
+if (n == 0) { d.write("--");d.write(">"); }
+//--></script>
+<!-- SpyLOG -->
+<!-- End Counter -->
+</TD>
+</TR>
+</table>
+<input type=hidden name=l value=en>
+</form>
+
+
+<table border=0>
+<tr>
+<td valign=top >
+
+<script type="text/javascript"><!--
+google_ad_client = "pub-7005770717406130";
+google_ad_width = 160;
+google_ad_height = 600;
+google_ad_format = "160x600_as";
+google_ad_type = "text_image";
+//2006-12-05: filesearch_left_160x600
+google_ad_channel = "8508766014";
+google_color_border = "FFFFFF";
+google_color_bg = "FFFFFF";
+google_color_link = "000000";
+google_color_text = "000000";
+google_color_url = "000000";
+//--></script>
+<script type="text/javascript"
+ src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+</script>
+
+</td>
+
+<td valign=top >
+
+<pre class=list>
+</pre>
+
+
+<script type="text/javascript"><!--
+google_ad_client = "pub-7005770717406130";
+/* filesearch_down_728x90 */
+google_ad_slot = "6424137562";
+google_ad_width = 728;
+google_ad_height = 90;
+//-->
+</script>
+<script type="text/javascript"
+src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+</script>
+
+	 </td></tr></table> <blockquote>
+<b>zstd-1.3.3.tar.gz</b> not found<br>
+Try to use <a href=http://www.filesearching.com/><b>search forms</b></a> to refine the parameters of your request.
+<br></blockquote>
+
+<table width=100% border=0 cellspacing=0 cellpadding=0 bgcolor=#FAE6CA>
+<tr bgcolor=#000000><td><img src=/img/b.gif height=1 width=1></td></tr>
+<tr>
+<td background=/img/inside-4x1-1x6_1.gif><img src=/img/b.gif height=20 width=20></td>
+</tr></table>
+<table width=100% border=0 cellspacing=0 cellpadding=0 bgcolor=#FAE6CA>
+ <tr><td align=center>
+<!--ENGLISH BOT--><br><br>
+</td></tr>
+</table>
+
+<!--Kavanga START-->
+
+<script type="text/javascript"><!--
+if (Math.ceil(Math.random()*10)  ) {
+    <!--Kavanga START-->
+    <!--Сайт: filesearch.ru-->
+    <!--Категория: Компьютеры и ПО-->
+    if (typeof(pr) == 'undefined') { var pr = Math.floor(Math.random() * 1000000); }
+    if (typeof(document.referrer) != 'undefined') {
+      if (typeof(afReferrer) == 'undefined') {
+        afReferrer = escape(document.referrer);
+      }
+    } else {
+      afReferrer = '';
+    }
+    var addate = new Date();
+
+    if (Math.ceil(Math.random()*10)  ) {
+        <!--Тип баннера: Pop Under-->
+        document.write('<scr' + 'ipt type="text/javascript" src="http://a.kavanga.ru/3604/prepareCode?ph=b&amp;p1=our&amp;p2=o&amp;pucn=a&amp;pfc=a&amp;pfb=a&amp;plp=a&amp;pli=a&amp;pop=a&amp;pr=' + pr + '&amp;pt=b&amp;pd=' + addate.getDate() + '&amp;pw=' + addate.getDay() + '&amp;pv=' + addate.getHours() + '&amp;prr=' + afReferrer + '"><\/scr' + 'ipt>');
+    } else {
+        <!--Тип баннера: Rich Media-->
+        document.write('<scr' + 'ipt type="text/javascript" src="http://a.kavanga.ru/3604/prepareCode?p1=oip&amp;p2=p&amp;pucn=a&amp;pfc=a&amp;pfb=a&amp;plp=a&amp;pli=a&amp;pop=a&amp;pr=' + pr +'&amp;pt=b&amp;pd=' + addate.getDate() + '&amp;pw=' + addate.getDay() + '&amp;pv=' + addate.getHours() + '&amp;py=a&amp;prr=' + afReferrer + '"><\/scr' + 'ipt>');
+    }
+    <!--Kavanga END-->
+} else {
+    <!-- AdMedia START -->
+    var RndNum4NoCash = Math.round(Math.random() * 1000000000);
+    document.write('<script language="JavaScript" src="http://ad.adriver.ru/cgi-bin/erle.cgi?sid=36667&target=top&bt=16&pz=0&rnd=' + RndNum4NoCash + '"><\/script>');
+    <!-- AdMedia END -->
+}
+// -->
+</script>
+
+<!--Kavanga END-->
+</BODY></HTML>
diff --git a/service/main.cpp b/service/main.cpp
new file mode 100644
index 0000000..97bc89e
--- /dev/null
+++ b/service/main.cpp
@@ -0,0 +1,18 @@
+#include <Ice/Communicator.h>
+#include <Ice/ObjectAdapter.h>
+#include <icetrayService.h>
+#include "apiImpl.h"
+
+namespace MirrorSearch {
+	class Api : public IceTray::Service {
+		public:
+			void addObjects(const std::string &, const Ice::CommunicatorPtr & ic, const Ice::StringSeq &, const Ice::ObjectAdapterPtr & adp) override
+			{
+				auto dbpool = getConnectionPool(ic, "postgresql", "MirrorSearch");
+				adp->add(new SearchImpl(dbpool), ic->stringToIdentity("Search"));
+			}
+	};
+
+	NAMEDFACTORY("default", MirrorSearch::Api, IceTray::ServiceFactory);
+}
+
diff --git a/service/models.ice b/service/models.ice
new file mode 100644
index 0000000..cb48459
--- /dev/null
+++ b/service/models.ice
@@ -0,0 +1,25 @@
+#ifndef MIRRORSEARCH_MODELS
+#define MIRRORSEARCH_MODELS
+
+module MirrorSearch {
+	class SearchService {
+		["slicer:db:pkey"]
+		int id;
+		string name;
+		string baseurl;
+		string listxpath;
+		string urlxpath;
+	};
+	class SearchHit {
+		["slicer:db:pkey"]
+		int id;
+		int serviceid;
+		string url;
+	};
+
+	sequence<SearchService> SearchServices;
+	sequence<SearchHit> SearchHits;
+};
+
+#endif
+
diff --git a/service/schema.sql b/service/schema.sql
new file mode 100644
index 0000000..b529172
--- /dev/null
+++ b/service/schema.sql
@@ -0,0 +1,8 @@
+CREATE TABLE searchservices(
+    id int GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+		name text not null,
+		baseurl text not null,
+		listxpath text not null,
+		urlxpath text not null
+);
+
diff --git a/service/sql/getServices.sql b/service/sql/getServices.sql
new file mode 100644
index 0000000..40382f2
--- /dev/null
+++ b/service/sql/getServices.sql
@@ -0,0 +1,3 @@
+SELECT id, name, baseurl, listxpath, urlxpath
+FROM searchservices
+ORDER BY id
diff --git a/service/test.cpp b/service/test.cpp
new file mode 100644
index 0000000..c115712
--- /dev/null
+++ b/service/test.cpp
@@ -0,0 +1,77 @@
+#define BOOST_TEST_MODULE MirrorSearch
+#include <boost/test/unit_test.hpp>
+
+#include <pq-mock.h>
+#include <dryice.h>
+#include <definedDirs.h>
+#include <api.h>
+
+class Service : PQ::Mock, public IceTray::DryIce {
+	public:
+		Service() : PQ::Mock("user=postgres", "MirrorSearch", {
+			rootDir / "schema.sql",
+			rootDir / "data.sql"
+		}) { }
+
+};
+
+class TestClient : public IceTray::DryIceClient {
+	public:
+		TestClient() :
+			s(getProxy<MirrorSearch::SearchPrx>("Search"))
+		{
+		}
+
+		MirrorSearch::SearchPrx s;
+};
+
+BOOST_TEST_GLOBAL_FIXTURE(Service);
+
+BOOST_FIXTURE_TEST_SUITE(tc, TestClient);
+
+BOOST_AUTO_TEST_CASE(sanity)
+{
+	BOOST_REQUIRE(s);
+	s->ice_ping();
+}
+
+BOOST_AUTO_TEST_CASE(getServices)
+{
+	auto ss = s->getServices();
+	BOOST_REQUIRE_EQUAL(ss.size(), 1);
+	BOOST_CHECK_EQUAL(ss.front()->id, 1);
+	BOOST_CHECK_EQUAL(ss.front()->name, "file searching mock");
+	BOOST_CHECK_NE(ss.front()->baseurl, "file://$SCRIPTDIR/fixtures/filesearching/%s.html");
+	BOOST_CHECK_EQUAL(ss.front()->baseurl.substr(0, 8), "file:///");
+	BOOST_CHECK(!ss.front()->listxpath.empty());
+	BOOST_CHECK(!ss.front()->urlxpath.empty());
+}
+
+BOOST_AUTO_TEST_CASE(getMatches_zstd_notfound)
+{
+	auto ms = s->getMatches("zstd-1.3.3.tar.gz");
+	BOOST_REQUIRE(ms.empty());
+}
+
+BOOST_AUTO_TEST_CASE(getMatches_xtrans)
+{
+	auto ms = s->getMatches("xtrans-1.3.5.tar.bz2");
+	BOOST_REQUIRE_EQUAL(ms.size(), 12);
+	BOOST_REQUIRE_EQUAL(ms.front()->url, "ftp://ftp.cs.mun.ca/pub/mirror/gentoo/distfiles/xtrans-1.3.5.tar.bz2");
+	BOOST_REQUIRE_EQUAL(ms.back()->url, "ftp://ftp.tr.debian.org/gentoo/distfiles/xtrans-1.3.5.tar.bz2");
+}
+
+BOOST_AUTO_TEST_CASE(getMatches_zstd_notfound_lucky)
+{
+	BOOST_REQUIRE(!s->feelingLucky("zstd-1.3.3.tar.gz"));
+}
+
+BOOST_AUTO_TEST_CASE(getMatches_xtrans_lucky)
+{
+	auto fl = s->feelingLucky("xtrans-1.3.5.tar.bz2");
+	BOOST_REQUIRE(fl);
+	BOOST_REQUIRE_EQUAL(*fl, "ftp://ftp.cs.mun.ca/pub/mirror/gentoo/distfiles/xtrans-1.3.5.tar.bz2");
+}
+
+BOOST_AUTO_TEST_SUITE_END();
+