diff options
| author | Dan Goodliffe <dan@randomdan.homeip.net> | 2018-03-08 21:19:21 +0000 |
|---|---|---|
| committer | Dan Goodliffe <dan@randomdan.homeip.net> | 2018-03-08 22:41:03 +0000 |
| commit | c3d93684368c82c0d7f969997264c121cbaafabd (patch) | |
| tree | 85b37e5b9f82ac70b660857b0828fdf4f9cca685 | |
| parent | Split the builds into the usual domain, api and service components (diff) | |
| download | mirrorsearch-c3d93684368c82c0d7f969997264c121cbaafabd.tar.bz2 mirrorsearch-c3d93684368c82c0d7f969997264c121cbaafabd.tar.xz mirrorsearch-c3d93684368c82c0d7f969997264c121cbaafabd.zip | |
Allow configuring referrer and useragent, also set a collection of other handy headers
| -rw-r--r-- | service/apiImpl.cpp | 11 | ||||
| -rw-r--r-- | service/models.ice | 2 | ||||
| -rw-r--r-- | service/schema.sql | 2 | ||||
| -rw-r--r-- | service/sql/getServices.sql | 2 |
4 files changed, 16 insertions, 1 deletions
diff --git a/service/apiImpl.cpp b/service/apiImpl.cpp index ab86d98..9795769 100644 --- a/service/apiImpl.cpp +++ b/service/apiImpl.cpp @@ -86,6 +86,17 @@ namespace MirrorSearch { return size; }; CESSO(curl, CURLOPT_WRITEDATA, &cb); + if (ss->referrer) { + CESSO(curl, CURLOPT_REFERER, ss->referrer->c_str()); + } + if (ss->useragent) { + CESSO(curl, CURLOPT_USERAGENT, ss->useragent->c_str()); + } + CESSO(curl, CURLOPT_TIMEOUT, 5L); + // Enables compression + CESSO(curl, CURLOPT_ACCEPT_ENCODING, ""); + CESSO(curl, CURLOPT_HTTP_CONTENT_DECODING, 1L); + CESSO(curl, CURLOPT_TCP_FASTOPEN, 1L); if (curl_easy_perform(curl.get()) != CURLE_OK) { curlErrorHandler<Read>(failingFunction((void*)&curl_easy_perform), errbuf, url); } diff --git a/service/models.ice b/service/models.ice index 971e7b9..7d83829 100644 --- a/service/models.ice +++ b/service/models.ice @@ -7,6 +7,8 @@ module MirrorSearch { int id; string name; string baseurl; + optional(0) string referrer; + optional(1) string useragent; string listxpath; string urlxpath; int parserflags; diff --git a/service/schema.sql b/service/schema.sql index 93a87d3..c439d73 100644 --- a/service/schema.sql +++ b/service/schema.sql @@ -2,6 +2,8 @@ CREATE TABLE searchservices( id int GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, name text not null, baseurl text not null, + referrer text, + useragent text, parserflags int not null, listxpath text not null, urlxpath text not null diff --git a/service/sql/getServices.sql b/service/sql/getServices.sql index be2fba6..677262b 100644 --- a/service/sql/getServices.sql +++ b/service/sql/getServices.sql @@ -1,3 +1,3 @@ -SELECT id, name, baseurl, parserflags, listxpath, urlxpath +SELECT id, name, baseurl, parserflags, listxpath, urlxpath, referrer, useragent FROM searchservices ORDER BY id |