From b9b25dc52ad73f1be70f42d649b7b52bc4807208 Mon Sep 17 00:00:00 2001
From: Dan Goodliffe <dan@randomdan.homeip.net>
Date: Sat, 23 Aug 2025 17:24:48 +0100
Subject: Store the entities for each log line

---
 src/ingestor.cpp | 34 ++++++++++++++++++++++++++--------
 src/ingestor.hpp |  3 +++
 2 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/src/ingestor.cpp b/src/ingestor.cpp
index 4084d2e..a3c49e7 100644
--- a/src/ingestor.cpp
+++ b/src/ingestor.cpp
@@ -71,17 +71,35 @@ namespace WebStat {
 	{
 		while (auto line = scn::scan<std::string>(input, "{:[^\n]}\n")) {
 			linesRead++;
-			if (auto result = scanLogLine(line->value())) {
-				linesParsed++;
-				std::ignore = result->values();
-			}
-			else {
-				syslog(LOG_WARNING, "Discarded line: [%s]", line->value().c_str());
-				linesDiscarded++;
-			}
+			ingestLogLine(line->value());
 		}
 	}
 
+	void
+	Ingestor::ingestLogLine(const std::string_view line)
+	{
+		if (auto result = scanLogLine(line)) {
+			linesParsed++;
+			const auto values = crc32ScanValues(result->values());
+			storeEntities(values);
+		}
+		else {
+			syslog(LOG_WARNING, "Discarded line: [%.*s]", static_cast<int>(line.length()), line.data());
+			linesDiscarded++;
+		}
+	}
+
+	template<typename... T>
+	void
+	Ingestor::storeEntities(const std::tuple<T...> & values) const
+	{
+		std::apply(
+				[this](auto &&... value) {
+					(this->storeEntity(value), ...);
+				},
+				values);
+	}
+
 	template<typename T>
 	void
 	Ingestor::storeEntity(const T &) const
diff --git a/src/ingestor.hpp b/src/ingestor.hpp
index 892cc9c..899f179 100644
--- a/src/ingestor.hpp
+++ b/src/ingestor.hpp
@@ -22,6 +22,7 @@ namespace WebStat {
 		[[nodiscard]] static ScanResult scanLogLine(std::string_view);
 
 		void ingestLog(std::FILE *);
+		void ingestLogLine(std::string_view);
 
 		template<typename T> void storeEntity(const T &) const;
 		void storeEntity(Entity) const;
@@ -33,6 +34,8 @@ namespace WebStat {
 		size_t linesDiscarded = 0;
 
 	private:
+		template<typename... T> void storeEntities(const std::tuple<T...> &) const;
+
 		uint32_t hostnameId;
 		DB::ConnectionPtr dbconn;
 	};
-- 
cgit v1.2.3