Use curl instead of libxml2's networking as we need more control over HTTP requests

3 files changed, 70 insertions, 6 deletions

diff --git a/service/Jamfile.jam b/service/Jamfile.jam
index e7bfed9..8f15002 100644
--- a/service/Jamfile.jam
+++ b/service/Jamfile.jam
@@ -5,11 +5,13 @@ import testing ;
 lib boost_utf : : <name>boost_unit_test_framework ;
 lib dbpp-postgresql : : : : <include>/usr/include/dbpp-postgresql ;
 lib dryice : : : : <include>/usr/include/icetray ;
+lib curl ;
 
 lib mirrorsearch :
 	[ glob *.cpp *.ice sql/*.sql : test.cpp ]
 	:
 	<slicer>yes
+	<library>curl
 	<library>..//adhocutil
 	<library>..//dbppcore
 	<library>..//boost_system
diff --git a/service/apiImpl.cpp b/service/apiImpl.cpp
index 28d88b4..ab86d98 100644
--- a/service/apiImpl.cpp
+++ b/service/apiImpl.cpp
@@ -9,6 +9,10 @@
 #include <libxml/xpathInternals.h>
 #include <libxml/HTMLparser.h>
 #include <libxml/HTMLtree.h>
+#include <curl/curl.h>
+
+#define CESSO(curl, opt, expr) \
+	BOOST_VERIFY_MSG(CURLE_OK == curl_easy_setopt(curl.get(), opt, expr), "Failed setting option " #opt);
 
 namespace MirrorSearch {
 	SearchImpl::SearchImpl(IceTray::DatabasePoolPtr db) :
@@ -34,14 +38,62 @@ namespace MirrorSearch {
 		return std::bind(&libxmlErrorHandler<Fmt, P...>, std::placeholders::_1, p...);
 	}
 
+	template<typename Fmt, typename ... P>
+	void
+	curlErrorHandler(const std::string & fn, const char * errbuf, const P & ... p)
+	{
+		throw XmlError(Fmt::get(fn, errbuf, p...));
+	}
+	template<typename Fmt, typename ... P>
+	auto cEHB(const char * errbuf, const P & ... p)
+	{
+		return std::bind(&curlErrorHandler<Fmt, P...>, std::placeholders::_1, errbuf, p...);
+	}
+
 	typedef UPtr<xmlDoc> xmlDocSPtr;
 	typedef UPtr<xmlXPathContext> xmlXPathContextSPtr;
 	typedef UPtr<xmlXPathObject> xmlXPathObjectSPtr;
+	typedef UPtr<xmlParserCtxt> xmlParserCtxtSPtr;
+
+	typedef std::function<size_t(const char *, size_t)> CurlWriteCallback;
 
-	AdHocFormatter(Read, "Failed to read in %? (%?) [%?, %?]");
-	static auto getDoc(const ::std::string & url, int flags)
+	static size_t write_callback(char * ptr, size_t size, size_t nmemb, void * userdata)
 	{
-		return make_unique(htmlReadFile, xmlFreeDoc, lEHB<Read>(url, flags), url.c_str(), (const char*)NULL, flags);
+		return (*(MirrorSearch::CurlWriteCallback *)(userdata))(ptr, size * nmemb);
+	}
+
+	AdHocFormatter(Read, "Failed to read in %? (%?) [%?]");
+	UPtr<xmlDoc> getDoc(const SearchServicePtr & ss, const std::string & fn) {
+		auto fmt = AdHoc::Buffer::getFormat(ss->baseurl);
+		auto url = (*fmt % fn).str();
+		char errbuf[CURL_ERROR_SIZE] = "";
+
+		xmlParserCtxtSPtr ctx { nullptr, nullptr };
+
+		auto curl = make_unique(curl_easy_init, curl_easy_cleanup, cEHB<Read>(errbuf, url));
+		BOOST_ASSERT(curl);
+		CESSO(curl, CURLOPT_URL, url.c_str());
+		CESSO(curl, CURLOPT_WRITEFUNCTION, write_callback);
+		CurlWriteCallback cb = [&ctx, &url, &ss](auto data, auto size) {
+			if (!ctx) {
+				ctx = make_unique(htmlCreatePushParserCtxt, htmlFreeParserCtxt, lEHB<Read>(url),
+						(xmlSAXHandlerPtr)NULL, (void*)NULL, data, size, url.c_str(), XML_CHAR_ENCODING_NONE);
+				htmlCtxtUseOptions(ctx.get(), ss->parserflags);
+			}
+			else {
+				htmlParseChunk(ctx.get(), data, size, 0);
+			}
+			return size;
+		};
+		CESSO(curl, CURLOPT_WRITEDATA, &cb);
+		if (curl_easy_perform(curl.get()) != CURLE_OK) {
+			curlErrorHandler<Read>(failingFunction((void*)&curl_easy_perform), errbuf, url);
+		}
+		BOOST_VERIFY_MSG(ctx, "No ctx and no previous error should never happen.");
+		htmlParseChunk(ctx.get(), "", 0, 1);
+
+		UPtr<xmlDoc> doc = { ctx->myDoc, xmlFreeDoc };
+		return doc;
 	}
 
 	AdHocFormatter(XPathCtx, "Failed to create xpath context in %? (%?)");
@@ -62,9 +114,7 @@ namespace MirrorSearch {
 
 	void SearchImpl::callService(const ::std::string & fn, const SearchServicePtr & s, SearchHits & sh) const
 	{
-		auto fmt = AdHoc::Buffer::getFormat(s->baseurl);
-		auto url = (*fmt % fn).str();
-		auto doc = getDoc(url, s->parserflags);
+		auto doc = getDoc(s, fn);
 		auto xpathCtx = getXPathCxt(doc);
 		auto xpathObj = getXPathObj(s->listxpath, xpathCtx, xmlXPathObjectType::XPATH_NODESET);
 		log->messagebf(LOG::INFO, "%d nodes matched %s", xpathObj->nodesetval->nodeNr, s->listxpath);
diff --git a/service/main.cpp b/service/main.cpp
index 97bc89e..00563f4 100644
--- a/service/main.cpp
+++ b/service/main.cpp
@@ -2,10 +2,22 @@
 #include <Ice/ObjectAdapter.h>
 #include <icetrayService.h>
 #include "apiImpl.h"
+#include <curl/curl.h>
+#include <libxml/parser.h>
 
 namespace MirrorSearch {
 	class Api : public IceTray::Service {
 		public:
+			Api()
+			{
+				xmlInitParser();
+				curl_global_init(0);
+			}
+			~Api()
+			{
+				xmlCleanupParser();
+				curl_global_cleanup();
+			}
 			void addObjects(const std::string &, const Ice::CommunicatorPtr & ic, const Ice::StringSeq &, const Ice::ObjectAdapterPtr & adp) override
 			{
 				auto dbpool = getConnectionPool(ic, "postgresql", "MirrorSearch");