From af422b45c7fb406dae35dc3f811e50d92e854f58 Mon Sep 17 00:00:00 2001
From: Dan Goodliffe <dan@randomdan.homeip.net>
Date: Sat, 2 May 2026 23:13:08 +0100
Subject: Start curl operations from any thread

Ingest is now background only, so don't limit where they're started
from. Adds some unfortunate locking around the curl maps.
---
 src/ingestor.cpp      | 13 ++++++++-----
 src/ingestor.hpp      |  2 +-
 test/testing-util.cpp |  2 +-
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/ingestor.cpp b/src/ingestor.cpp
index 4af2f2d..971cc29 100644
--- a/src/ingestor.cpp
+++ b/src/ingestor.cpp
@@ -120,7 +120,7 @@ namespace WebStat {
 		storeQueueLines {&Ingestor::jobStoreQueuedLines},
 		hostnameId {insert(dbpool->get(), SQL::HOST_UPSERT, SQL::HOST_UPSERT_OPTS, host.nodename, host.sysname,
 				host.release, host.version, host.machine, host.domainname)},
-		curl {curl_multi_init()}, mainThread {std::this_thread::get_id()}
+		curl {curl_multi_init()}
 	{
 		assert(!currentIngestor);
 		currentIngestor = this;
@@ -239,7 +239,7 @@ namespace WebStat {
 			if (expiredThenSet(lastCheckedJobs, settings.checkJobsAfter)) {
 				runJobsAsNeeded();
 			}
-			if (!curlOperations.empty()) {
+			if (std::lock_guard curlOperationsLock {curlOperationsMutex}; !curlOperations.empty()) {
 				handleCurlOperations();
 			}
 		}
@@ -547,7 +547,7 @@ namespace WebStat {
 		assert(!entity.id);
 		const auto & [typeName, onInsert] = ENTITY_TYPE_VALUES[std::to_underlying(entity.type)];
 		entity.id = insert(dbconn, SQL::ENTITY_INSERT, SQL::ENTITY_INSERT_OPTS, entity.value, typeName);
-		if (onInsert && std::this_thread::get_id() == mainThread) {
+		if (onInsert) {
 			std::invoke(onInsert, this, entity);
 		}
 		stats.entitiesInserted += 1;
@@ -558,8 +558,11 @@ namespace WebStat {
 	{
 		const auto & [entityHash, entityId, type, value] = entity;
 		auto curlOp = curlGetUserAgentDetail(*entityId, value, settings.userAgentAPI.c_str());
-		auto added = curlOperations.emplace(curlOp->hnd.get(), std::move(curlOp));
-		curl_multi_add_handle(curl.get(), added.first->first);
+		{
+			std::lock_guard curlOperationsLock {curlOperationsMutex};
+			auto added = curlOperations.emplace(curlOp->hnd.get(), std::move(curlOp));
+			curl_multi_add_handle(curl.get(), added.first->first);
+		}
 	}
 
 	template<typename... T>
diff --git a/src/ingestor.hpp b/src/ingestor.hpp
index 64b3357..81041ff 100644
--- a/src/ingestor.hpp
+++ b/src/ingestor.hpp
@@ -131,6 +131,6 @@ namespace WebStat {
 		EntityId hostnameId;
 		CurlMultiPtr curl;
 		mutable CurlOperations curlOperations;
-		std::thread::id mainThread;
+		mutable std::mutex curlOperationsMutex;
 	};
 }
diff --git a/test/testing-util.cpp b/test/testing-util.cpp
index 9e5a6af..141940a 100644
--- a/test/testing-util.cpp
+++ b/test/testing-util.cpp
@@ -6,7 +6,7 @@ namespace WebStat {
 	MockDB::MockDB() :
 		DB::PluginMock<PQ::Mock>("webstat", {SRC_DIR / "schema.sql"}, "user=postgres dbname=postgres") { }
 
-	MockDBPool::MockDBPool(std::string poolName) : DB::BasicConnectionPool(1, 1), name {std::move(poolName)} { }
+	MockDBPool::MockDBPool(std::string poolName) : DB::BasicConnectionPool(2, 1), name {std::move(poolName)} { }
 
 	DB::ConnectionPtr
 	MockDBPool::createResource() const
-- 
cgit v1.3