From 21a9cb6866afe3354bee60d852ebffc7ef3a8355 Mon Sep 17 00:00:00 2001
From: Dan Goodliffe <dan@randomdan.homeip.net>
Date: Mon, 25 Aug 2025 13:06:07 +0100
Subject: Insert new entities and log entry in a transaction

If no new entities are required, no transaction is created.
---
 src/ingestor.cpp | 53 +++++++++++++++++++++++++++++++++++------------------
 src/ingestor.hpp |  6 +++++-
 src/util.hpp     |  6 +++---
 3 files changed, 43 insertions(+), 22 deletions(-)

diff --git a/src/ingestor.cpp b/src/ingestor.cpp
index 1b54a64..f824c80 100644
--- a/src/ingestor.cpp
+++ b/src/ingestor.cpp
@@ -4,6 +4,7 @@
 #include <connection.h>
 #include <dbTypes.h>
 #include <modifycommand.h>
+#include <ranges>
 #include <scn/scan.h>
 #include <syslog.h>
 #include <utility>
@@ -55,7 +56,9 @@ namespace WebStat {
 	Ingestor::Ingestor(const std::string_view hostname, DB::ConnectionPtr dbconn) :
 		hostnameId {crc32(hostname)}, dbconn {std::move(dbconn)}
 	{
-		storeEntities(std::make_tuple(std::make_pair(hostnameId, hostname)));
+		storeEntities({
+				std::make_pair(hostnameId, hostname),
+		});
 	}
 
 	Ingestor::ScanResult
@@ -92,7 +95,11 @@ namespace WebStat {
 		if (auto result = scanLogLine(line)) {
 			linesParsed++;
 			const auto values = crc32ScanValues(result->values());
-			storeEntities(values);
+			std::optional<DB::TransactionScope> dbtx;
+			if (const auto newEnts = newEntities(values); newEnts.front()) {
+				dbtx.emplace(*dbconn);
+				storeEntities(newEnts);
+			}
 			storeLogLine(values);
 		}
 		else {
@@ -102,32 +109,42 @@ namespace WebStat {
 	}
 
 	template<typename... T>
-	size_t
-	Ingestor::storeEntities(const std::tuple<T...> & values) const
+	Ingestor::NewEntities
+	Ingestor::newEntities(const std::tuple<T...> & values) const
 	{
-		return visitSum(
-				[this]<typename X>(const X & entity) -> size_t {
-					auto insertIfReqd = [this](auto && entity) -> size_t {
-						if (existingEntities.contains(entity.first)) {
-							return 0;
+		Ingestor::NewEntities rtn;
+		auto next = rtn.begin();
+		visit(
+				[this, &next]<typename X>(const X & entity) {
+					auto addNewIfReqd = [&next, this](auto && entity) mutable {
+						if (!existingEntities.contains(entity.first)) {
+							*next++ = entity;
 						}
-						auto insert = dbconn->modify(SQL::ENTITY_INSERT, SQL::ENTITY_INSERT_OPTS);
-						insert->bindParamI(0, entity.first);
-						insert->bindParamS(1, entity.second);
-						insert->execute();
-						existingEntities.emplace(entity.first);
-						return 1;
+						return 0;
 					};
 
 					if constexpr (std::is_same_v<X, Entity>) {
-						return insertIfReqd(entity);
+						addNewIfReqd(entity);
 					}
 					else if constexpr (std::is_same_v<X, std::optional<Entity>>) {
-						entity.transform(insertIfReqd).value_or(0);
+						entity.transform(addNewIfReqd);
 					}
-					return 0;
 				},
 				values);
+		return rtn;
+	}
+
+	void
+	Ingestor::storeEntities(const std::span<const std::optional<Entity>> values) const
+	{
+		std::ranges::for_each(
+				values | std::views::take_while(&std::optional<Entity>::has_value), [this](auto && entity) {
+					auto insert = dbconn->modify(SQL::ENTITY_INSERT, SQL::ENTITY_INSERT_OPTS);
+					insert->bindParamI(0, entity->first);
+					insert->bindParamS(1, entity->second);
+					insert->execute();
+					existingEntities.emplace(entity->first);
+				});
 	}
 
 	template<typename... T>
diff --git a/src/ingestor.hpp b/src/ingestor.hpp
index 7141515..040331f 100644
--- a/src/ingestor.hpp
+++ b/src/ingestor.hpp
@@ -6,6 +6,7 @@
 #include <cstdio>
 #include <flat_set>
 #include <scn/scan.h>
+#include <span>
 
 namespace WebStat {
 	class Ingestor {
@@ -33,7 +34,10 @@ namespace WebStat {
 		size_t linesDiscarded = 0;
 
 	private:
-		template<typename... T> size_t storeEntities(const std::tuple<T...> &) const;
+		static constexpr size_t MAX_NEW_ENTITIES = 6;
+		void storeEntities(std::span<const std::optional<Entity>>) const;
+		using NewEntities = std::array<std::optional<Entity>, MAX_NEW_ENTITIES>;
+		template<typename... T> NewEntities newEntities(const std::tuple<T...> &) const;
 
 		mutable std::flat_set<Crc32Value> existingEntities;
 		uint32_t hostnameId;
diff --git a/src/util.hpp b/src/util.hpp
index 0ed260c..dcdc654 100644
--- a/src/util.hpp
+++ b/src/util.hpp
@@ -5,11 +5,11 @@
 namespace WebStat {
 	template<typename... T>
 	auto
-	visitSum(auto && visitor, const std::tuple<T...> & values)
+	visit(auto && visitor, const std::tuple<T...> & values)
 	{
-		return std::apply(
+		std::apply(
 				[&](auto &&... value) {
-					return (visitor(value) + ...);
+					(visitor(value), ...);
 				},
 				values);
 	}
-- 
cgit v1.2.3