From 55439681e7a45489e5a77c2d6169f4b722525c96 Mon Sep 17 00:00:00 2001
From: Dan Goodliffe <dan@randomdan.homeip.net>
Date: Fri, 12 Sep 2025 00:13:56 +0100
Subject: Perform background curl operations when not processing log input

---
 src/ingestor.cpp | 26 ++++++++++++++++++++++++++
 src/ingestor.hpp |  3 +++
 2 files changed, 29 insertions(+)

(limited to 'src')

diff --git a/src/ingestor.cpp b/src/ingestor.cpp
index cfdd87f..26bb289 100644
--- a/src/ingestor.cpp
+++ b/src/ingestor.cpp
@@ -86,6 +86,26 @@ namespace WebStat {
 				>(input, R"({} {} {} {:[A-Z]} {} {} {} {} {} {} {} {})");
 	}
 
+	void
+	Ingestor::handleCurlOperations()
+	{
+		int remaining {};
+		curl_multi_perform(curl.get(), nullptr);
+		while (auto msg = curl_multi_info_read(curl.get(), &remaining)) {
+			if (msg->msg == CURLMSG_DONE) {
+				if (auto operationItr = curlOperations.find(msg->easy_handle); operationItr != curlOperations.end()) {
+					if (msg->data.result == CURLE_OK) { }
+					curl_multi_remove_handle(curl.get(), msg->easy_handle);
+					curlOperations.erase(operationItr);
+				}
+				else {
+					curlOperations.erase(msg->easy_handle);
+					std::println(std::cerr, "Failed to lookup CurlOperation");
+				}
+			}
+		}
+	}
+
 	void
 	Ingestor::ingestLog(std::FILE * input)
 	{
@@ -101,6 +121,12 @@ namespace WebStat {
 					break;
 				}
 			}
+			else if (!curlOperations.empty()) {
+				handleCurlOperations();
+			}
+		}
+		while (!curlOperations.empty() && curl_multi_poll(curl.get(), nullptr, 0, INT_MAX, nullptr) == CURLM_OK) {
+			handleCurlOperations();
 		}
 	}
 
diff --git a/src/ingestor.hpp b/src/ingestor.hpp
index f181ea3..b55886e 100644
--- a/src/ingestor.hpp
+++ b/src/ingestor.hpp
@@ -43,10 +43,13 @@ namespace WebStat {
 		void storeEntities(DB::Connection *, std::span<const std::optional<Entity>>) const;
 		using NewEntities = std::array<std::optional<Entity>, MAX_NEW_ENTITIES>;
 		template<typename... T> NewEntities newEntities(const std::tuple<T...> &) const;
+		void handleCurlOperations();
 
+		using CurlOperations = std::map<CURL *, std::unique_ptr<CurlOperation>>;
 		mutable std::flat_set<Crc32Value> existingEntities;
 		uint32_t hostnameId;
 		DB::ConnectionPoolPtr dbpool;
 		CurlMultiPtr curl;
+		mutable CurlOperations curlOperations;
 	};
 }
-- 
cgit v1.2.3