From 34051da2f27ffa40d0b6d20ae891a497fe73bfe5 Mon Sep 17 00:00:00 2001
From: Dan Goodliffe <dan@randomdan.homeip.net>
Date: Sat, 17 Jan 2026 19:40:47 +0000
Subject: Attempt to save uninsertable log lines to the entities table

If that fails, we still park them as before, such as when the DB is
unavailable. Those which are saved as entities require investigation why
they couldn't be saved, much like UnparsableLines.
---
 src/ingestor.cpp | 28 ++++++++++++++++++++--------
 1 file changed, 20 insertions(+), 8 deletions(-)

(limited to 'src/ingestor.cpp')

diff --git a/src/ingestor.cpp b/src/ingestor.cpp
index 44107ce..81642be 100644
--- a/src/ingestor.cpp
+++ b/src/ingestor.cpp
@@ -180,15 +180,26 @@ namespace WebStat {
 			linesParsed++;
 			const auto values = crc32ScanValues(result->values());
 			NewEntityIds ids;
-			{
-				std::optional<DB::TransactionScope> dbtx;
-				if (const auto newEnts = newEntities(values); newEnts.front()) {
-					dbtx.emplace(*dbconn);
-					ids = storeEntities(dbconn, newEnts);
+			try {
+				{
+					std::optional<DB::TransactionScope> dbtx;
+					if (const auto newEnts = newEntities(values); newEnts.front()) {
+						dbtx.emplace(*dbconn);
+						ids = storeEntities(dbconn, newEnts);
+					}
+					storeLogLine(dbconn, values);
+				}
+				rememberNewEntityIds(ids);
+			}
+			catch (const DB::Error & originalError) {
+				try {
+					const auto uninsertableLine = ToEntity<EntityType::UninsertableLine> {}(line);
+					rememberNewEntityIds(storeEntities(dbconn, {uninsertableLine}));
+				}
+				catch (const std::exception &) {
+					throw originalError;
 				}
-				storeLogLine(dbconn, values);
 			}
-			rememberNewEntityIds(ids);
 		}
 		else {
 			linesDiscarded++;
@@ -310,7 +321,7 @@ namespace WebStat {
 	Ingestor::NewEntityIds
 	Ingestor::storeEntities(DB::Connection * dbconn, const std::span<const std::optional<Entity>> values) const
 	{
-		static constexpr std::array<std::pair<std::string_view, void (Ingestor::*)(const Entity &) const>, 7>
+		static constexpr std::array<std::pair<std::string_view, void (Ingestor::*)(const Entity &) const>, 8>
 				ENTITY_TYPE_VALUES {{
 						{"host", nullptr},
 						{"virtual_host", nullptr},
@@ -319,6 +330,7 @@ namespace WebStat {
 						{"referrer", nullptr},
 						{"user_agent", &Ingestor::onNewUserAgent},
 						{"unparsable_line", nullptr},
+						{"uninsertable_line", nullptr},
 				}};
 
 		auto insert = dbconn->modify(SQL::ENTITY_INSERT, SQL::ENTITY_INSERT_OPTS);
-- 
cgit v1.2.3