summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/curlOp.cpp41
-rw-r--r--src/curlOp.hpp40
-rw-r--r--src/ingestor.cpp21
-rw-r--r--src/ingestor.hpp2
-rw-r--r--src/sql.cpp4
-rw-r--r--src/sql.hpp1
-rw-r--r--src/sql/entityUpdateDetail.sql1
-rw-r--r--src/uaLookup.cpp42
-rw-r--r--src/uaLookup.hpp31
9 files changed, 127 insertions, 56 deletions
diff --git a/src/curlOp.cpp b/src/curlOp.cpp
new file mode 100644
index 0000000..1a4ad11
--- /dev/null
+++ b/src/curlOp.cpp
@@ -0,0 +1,41 @@
+#include "curlOp.hpp"
+#include <string_view>
+
+namespace WebStat {
+ namespace {
+ size_t
+ stringAppend(const char * ptr, size_t size, size_t nmemb, std::string * result)
+ {
+ result->append(ptr, nmemb * size);
+ return nmemb * size;
+ }
+ }
+
+ CurlOperation::CurlOperation() : hnd {curl_easy_init()}, err {}
+ {
+ curl_easy_setopt(hnd.get(), CURLOPT_NOPROGRESS, 1L);
+ curl_easy_setopt(hnd.get(), CURLOPT_MAXREDIRS, 50L);
+ curl_easy_setopt(hnd.get(), CURLOPT_TCP_KEEPALIVE, 1L);
+ curl_easy_setopt(hnd.get(), CURLOPT_FAILONERROR, 1L);
+ curl_easy_setopt(hnd.get(), CURLOPT_ERRORBUFFER, err.data());
+ curl_easy_setopt(hnd.get(), CURLOPT_WRITEDATA, &result);
+ curl_easy_setopt(hnd.get(), CURLOPT_WRITEFUNCTION, &stringAppend);
+ }
+
+ void
+ CurlOperation::addForm(const char * name, const std::string_view data)
+ {
+ if (!mime) {
+ mime.reset(curl_mime_init(hnd.get()));
+ curl_easy_setopt(hnd.get(), CURLOPT_MIMEPOST, mime.get());
+ }
+ auto part = curl_mime_addpart(mime.get());
+ curl_mime_data(part, data.data(), data.length());
+ curl_mime_name(part, name);
+ }
+
+ void
+ CurlOperation::onError(DB::Connection *) const
+ {
+ }
+}
diff --git a/src/curlOp.hpp b/src/curlOp.hpp
new file mode 100644
index 0000000..ce42fb4
--- /dev/null
+++ b/src/curlOp.hpp
@@ -0,0 +1,40 @@
+#pragma once
+
+#include "util.hpp"
+#include <connection_fwd.h>
+#include <curl/curl.h>
+#include <memory>
+#include <stdexcept>
+#include <string>
+
+namespace WebStat {
+ class CurlError : public std::runtime_error {
+ public:
+ explicit CurlError(CURLcode code, const char * msg) : std::runtime_error {msg}, code(code) { }
+
+ CURLcode code;
+ };
+
+ using CurlPtr = std::unique_ptr<CURL, DeleteWith<&curl_easy_cleanup>>;
+ using CurlMimePtr = std::unique_ptr<curl_mime, DeleteWith<&curl_mime_free>>;
+ using CurlErrorBuf = std::array<char, CURL_ERROR_SIZE>;
+ using CurlMultiPtr = std::unique_ptr<CURLM, DeleteWith<&curl_multi_cleanup>>;
+
+ class CurlOperation {
+ public:
+ CurlOperation();
+ virtual ~CurlOperation() = default;
+
+ SPECIAL_MEMBERS_DEFAULT_MOVE_NO_COPY(CurlOperation);
+
+ void addForm(const char * name, std::string_view data);
+
+ virtual void whenComplete(DB::Connection *) const = 0;
+ virtual void onError(DB::Connection *) const;
+
+ CurlPtr hnd;
+ CurlMimePtr mime;
+ CurlErrorBuf err;
+ std::string result;
+ };
+}
diff --git a/src/ingestor.cpp b/src/ingestor.cpp
index 26bb289..c54ca53 100644
--- a/src/ingestor.cpp
+++ b/src/ingestor.cpp
@@ -1,5 +1,6 @@
#include "ingestor.hpp"
#include "sql.hpp"
+#include "uaLookup.hpp"
#include "util.hpp"
#include <connection.h>
#include <dbTypes.h>
@@ -94,7 +95,12 @@ namespace WebStat {
while (auto msg = curl_multi_info_read(curl.get(), &remaining)) {
if (msg->msg == CURLMSG_DONE) {
if (auto operationItr = curlOperations.find(msg->easy_handle); operationItr != curlOperations.end()) {
- if (msg->data.result == CURLE_OK) { }
+ if (msg->data.result == CURLE_OK) {
+ operationItr->second->whenComplete(dbpool->get().get());
+ }
+ else {
+ operationItr->second->onError(dbpool->get().get());
+ }
curl_multi_remove_handle(curl.get(), msg->easy_handle);
curlOperations.erase(operationItr);
}
@@ -186,7 +192,18 @@ namespace WebStat {
values | std::views::take_while(&std::optional<Entity>::has_value), [this, &insert](auto && entity) {
const auto & [entityId, type, value] = *entity;
bindMany(insert, 0, entityId, ENTITY_TYPE_VALUES[std::to_underlying(type)], value);
- insert->execute();
+ if (insert->execute() > 0) {
+ switch (type) {
+ case EntityType::UserAgent: {
+ auto curlOp = curlGetUserAgentDetail(entityId, value, userAgentAPI.c_str());
+ auto added = curlOperations.emplace(curlOp->hnd.get(), std::move(curlOp));
+ curl_multi_add_handle(curl.get(), added.first->first);
+ break;
+ }
+ default:
+ break;
+ }
+ }
existingEntities.emplace(std::get<0>(*entity));
});
}
diff --git a/src/ingestor.hpp b/src/ingestor.hpp
index b55886e..a0dd1b9 100644
--- a/src/ingestor.hpp
+++ b/src/ingestor.hpp
@@ -1,7 +1,7 @@
#pragma once
+#include "curlOp.hpp"
#include "logTypes.hpp"
-#include "uaLookup.hpp"
#include <c++11Helpers.h>
#include <connectionPool.h>
#include <connection_fwd.h>
diff --git a/src/sql.cpp b/src/sql.cpp
index fbfadd3..9c0d992 100644
--- a/src/sql.cpp
+++ b/src/sql.cpp
@@ -12,6 +12,9 @@ namespace WebStat::SQL {
const std::string ENTITY_INSERT {
#embed "sql/entityInsert.sql"
};
+ const std::string ENTITY_UPDATE_DETAIL {
+#embed "sql/entityUpdateDetail.sql"
+ };
const std::string HOST_UPSERT {
#embed "sql/hostUpsert.sql"
};
@@ -19,6 +22,7 @@ namespace WebStat::SQL {
const DB::CommandOptionsPtr VAR##_OPTS = std::make_shared<DB::CommandOptions>(std::hash<std::string> {}(VAR))
HASH_OPTS(ACCESS_LOG_INSERT);
HASH_OPTS(ENTITY_INSERT);
+ HASH_OPTS(ENTITY_UPDATE_DETAIL);
HASH_OPTS(HOST_UPSERT);
#undef HASH_OPTS
}
diff --git a/src/sql.hpp b/src/sql.hpp
index 8183513..f0dfb05 100644
--- a/src/sql.hpp
+++ b/src/sql.hpp
@@ -10,6 +10,7 @@ namespace WebStat::SQL {
EMBED_DECLARE(ACCESS_LOG_INSERT);
EMBED_DECLARE(ENTITY_INSERT);
+ EMBED_DECLARE(ENTITY_UPDATE_DETAIL);
EMBED_DECLARE(HOST_UPSERT);
#undef EMBED_DECLARE
}
diff --git a/src/sql/entityUpdateDetail.sql b/src/sql/entityUpdateDetail.sql
new file mode 100644
index 0000000..9e5ea82
--- /dev/null
+++ b/src/sql/entityUpdateDetail.sql
@@ -0,0 +1 @@
+UPDATE entities SET detail = $2 WHERE id = $1
diff --git a/src/uaLookup.cpp b/src/uaLookup.cpp
index 5b31065..dbef015 100644
--- a/src/uaLookup.cpp
+++ b/src/uaLookup.cpp
@@ -1,44 +1,24 @@
#include "uaLookup.hpp"
-#include <memory>
-#include <string_view>
+#include "sql.hpp"
+#include <connection.h>
+#include <dbTypes.h>
+#include <modifycommand.h>
namespace WebStat {
- namespace {
- size_t
- stringAppend(const char * ptr, size_t size, size_t nmemb, std::string * result)
- {
- result->append(ptr, nmemb * size);
- return nmemb * size;
- }
- }
-
- CurlOperation::CurlOperation() : hnd {curl_easy_init()}, err {}
- {
- curl_easy_setopt(hnd.get(), CURLOPT_NOPROGRESS, 1L);
- curl_easy_setopt(hnd.get(), CURLOPT_MAXREDIRS, 50L);
- curl_easy_setopt(hnd.get(), CURLOPT_TCP_KEEPALIVE, 1L);
- curl_easy_setopt(hnd.get(), CURLOPT_FAILONERROR, 1L);
- curl_easy_setopt(hnd.get(), CURLOPT_ERRORBUFFER, err.data());
- curl_easy_setopt(hnd.get(), CURLOPT_WRITEDATA, &result);
- curl_easy_setopt(hnd.get(), CURLOPT_WRITEFUNCTION, &stringAppend);
- }
+ UserAgentLookupOperation::UserAgentLookupOperation(Crc32Value entityId) : entityId {entityId} { }
void
- CurlOperation::addForm(const char * name, const std::string_view data)
+ UserAgentLookupOperation::whenComplete(DB::Connection * dbconn) const
{
- if (!mime) {
- mime.reset(curl_mime_init(hnd.get()));
- curl_easy_setopt(hnd.get(), CURLOPT_MIMEPOST, mime.get());
- }
- auto part = curl_mime_addpart(mime.get());
- curl_mime_data(part, data.data(), data.length());
- curl_mime_name(part, name);
+ auto upd = dbconn->modify(SQL::ENTITY_UPDATE_DETAIL, SQL::ENTITY_UPDATE_DETAIL_OPTS);
+ bindMany(upd, 0, entityId, result);
+ upd->execute();
}
std::unique_ptr<CurlOperation>
- curlGetUserAgentDetail(const std::string_view uas, const char * baseUrl)
+ curlGetUserAgentDetail(Crc32Value entityId, const std::string_view uas, const char * baseUrl)
{
- auto request = std::make_unique<CurlOperation>();
+ auto request = std::make_unique<UserAgentLookupOperation>(entityId);
curl_easy_setopt(request->hnd.get(), CURLOPT_URL, baseUrl);
curl_easy_setopt(
diff --git a/src/uaLookup.hpp b/src/uaLookup.hpp
index 5529a40..9714253 100644
--- a/src/uaLookup.hpp
+++ b/src/uaLookup.hpp
@@ -1,34 +1,21 @@
#pragma once
-#include "util.hpp"
+#include "curlOp.hpp"
+#include "logTypes.hpp"
+#include <connection_fwd.h>
#include <curl/curl.h>
#include <memory>
-#include <stdexcept>
-#include <string>
namespace WebStat {
- class CurlError : public std::runtime_error {
+ class UserAgentLookupOperation : public CurlOperation {
public:
- explicit CurlError(CURLcode code, const char * msg) : std::runtime_error {msg}, code(code) { }
+ UserAgentLookupOperation(Crc32Value entityId);
- CURLcode code;
- };
-
- using CurlPtr = std::unique_ptr<CURL, DeleteWith<&curl_easy_cleanup>>;
- using CurlMimePtr = std::unique_ptr<curl_mime, DeleteWith<&curl_mime_free>>;
- using CurlErrorBuf = std::array<char, CURL_ERROR_SIZE>;
- using CurlMultiPtr = std::unique_ptr<CURLM, DeleteWith<&curl_multi_cleanup>>;
-
- class CurlOperation {
- public:
- CurlOperation();
- void addForm(const char * name, std::string_view data);
+ void whenComplete(DB::Connection *) const override;
- CurlPtr hnd;
- CurlMimePtr mime;
- CurlErrorBuf err;
- std::string result;
+ Crc32Value entityId;
};
- std::unique_ptr<CurlOperation> curlGetUserAgentDetail(std::string_view uas, const char * baseUrl);
+ std::unique_ptr<CurlOperation> curlGetUserAgentDetail(
+ Crc32Value entityId, std::string_view uas, const char * baseUrl);
}
generated by cgit v1.2.3 (git 2.46.0) at 2025-10-13 21:16:16 +0000