From 71803b97f1c9e31f2027da48bb742353f9c43e62 Mon Sep 17 00:00:00 2001
From: Dan Goodliffe <dan.goodliffe@octal.co.uk>
Date: Tue, 23 Sep 2025 13:31:26 +0100
Subject: Write unparsable lines to the entity table

Diagnostics and the ability to ingest later.
---
 src/ingestor.cpp | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

(limited to 'src/ingestor.cpp')

diff --git a/src/ingestor.cpp b/src/ingestor.cpp
index e1cc7c6..db5f317 100644
--- a/src/ingestor.cpp
+++ b/src/ingestor.cpp
@@ -121,7 +121,7 @@ namespace WebStat {
 			if (logIn.revents) {
 				if (auto line = scn::scan<std::string>(input, "{:[^\n]}\n")) {
 					linesRead++;
-					ingestLogLine(dbpool->get().get(), line->value());
+					ingestLogLine(line->value());
 				}
 				else {
 					break;
@@ -136,6 +136,12 @@ namespace WebStat {
 		}
 	}
 
+	void
+	Ingestor::ingestLogLine(const std::string_view line)
+	{
+		ingestLogLine(dbpool->get().get(), line);
+	}
+
 	void
 	Ingestor::ingestLogLine(DB::Connection * dbconn, const std::string_view line)
 	{
@@ -150,8 +156,9 @@ namespace WebStat {
 			storeLogLine(dbconn, values);
 		}
 		else {
-			syslog(LOG_WARNING, "Discarded line: [%.*s]", static_cast<int>(line.length()), line.data());
 			linesDiscarded++;
+			const auto unparsableLine = toEntity(line, EntityType::UnparsableLine);
+			storeEntities(dbconn, {unparsableLine});
 		}
 	}
 
@@ -185,7 +192,7 @@ namespace WebStat {
 	Ingestor::storeEntities(DB::Connection * dbconn, const std::span<const std::optional<Entity>> values) const
 	{
 		static constexpr std::array ENTITY_TYPE_VALUES {
-				"host", "virtual_host", "path", "query_string", "referrer", "user_agent"};
+				"host", "virtual_host", "path", "query_string", "referrer", "user_agent", "unparsable_line"};
 
 		auto insert = dbconn->modify(SQL::ENTITY_INSERT, SQL::ENTITY_INSERT_OPTS);
 		std::ranges::for_each(
-- 
cgit v1.2.3