Perform background curl operations when not processing log input

2 files changed, 29 insertions, 0 deletions

diff --git a/src/ingestor.cpp b/src/ingestor.cpp
index cfdd87f..26bb289 100644
--- a/src/ingestor.cpp
+++ b/src/ingestor.cpp
@@ -87,6 +87,26 @@ namespace WebStat {
 	}
 
 	void
+	Ingestor::handleCurlOperations()
+	{
+		int remaining {};
+		curl_multi_perform(curl.get(), nullptr);
+		while (auto msg = curl_multi_info_read(curl.get(), &remaining)) {
+			if (msg->msg == CURLMSG_DONE) {
+				if (auto operationItr = curlOperations.find(msg->easy_handle); operationItr != curlOperations.end()) {
+					if (msg->data.result == CURLE_OK) { }
+					curl_multi_remove_handle(curl.get(), msg->easy_handle);
+					curlOperations.erase(operationItr);
+				}
+				else {
+					curlOperations.erase(msg->easy_handle);
+					std::println(std::cerr, "Failed to lookup CurlOperation");
+				}
+			}
+		}
+	}
+
+	void
 	Ingestor::ingestLog(std::FILE * input)
 	{
 		curl_waitfd logIn {.fd = fileno(input), .events = CURL_WAIT_POLLIN, .revents = 0};
@@ -101,6 +121,12 @@ namespace WebStat {
 					break;
 				}
 			}
+			else if (!curlOperations.empty()) {
+				handleCurlOperations();
+			}
+		}
+		while (!curlOperations.empty() && curl_multi_poll(curl.get(), nullptr, 0, INT_MAX, nullptr) == CURLM_OK) {
+			handleCurlOperations();
 		}
 	}
 
diff --git a/src/ingestor.hpp b/src/ingestor.hpp
index f181ea3..b55886e 100644
--- a/src/ingestor.hpp
+++ b/src/ingestor.hpp
@@ -43,10 +43,13 @@ namespace WebStat {
 		void storeEntities(DB::Connection *, std::span<const std::optional<Entity>>) const;
 		using NewEntities = std::array<std::optional<Entity>, MAX_NEW_ENTITIES>;
 		template<typename... T> NewEntities newEntities(const std::tuple<T...> &) const;
+		void handleCurlOperations();
 
+		using CurlOperations = std::map<CURL *, std::unique_ptr<CurlOperation>>;
 		mutable std::flat_set<Crc32Value> existingEntities;
 		uint32_t hostnameId;
 		DB::ConnectionPoolPtr dbpool;
 		CurlMultiPtr curl;
+		mutable CurlOperations curlOperations;
 	};
 }