From 61e106fe6a15f76b73125682e91d5c17ee875be2 Mon Sep 17 00:00:00 2001 From: Dan Goodliffe Date: Sat, 4 Jul 2026 12:11:38 +0100 Subject: Include extra details when storing unparsable and uninsertable lines. Now includes the hostnameId, timestamp and optionally error message. Updates idx_entities_retryinsert to account for details not being NULL. Also fixes the issue where the entities inserted and rollback would still be recorded in the entity cache if the access log insert failed. This would have lead to PK violations later on. --- src/ingestor.cpp | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) (limited to 'src/ingestor.cpp') diff --git a/src/ingestor.cpp b/src/ingestor.cpp index ff5cd92..077f7d8 100644 --- a/src/ingestor.cpp +++ b/src/ingestor.cpp @@ -420,17 +420,16 @@ namespace WebStat { try { DB::TransactionScope lineTx {*dbconn}; storeNewEntities(dbconn, valuesEntities); - existingEntities()->insert_range(valuesEntities | ENTITY_IDS); storeLogLine(dbconn, values); + existingEntities()->insert_range(valuesEntities | ENTITY_IDS); } catch (const DB::Error & originalError) { try { DB::TransactionScope lineTx {*dbconn}; - auto uninsertableLine = ToEntity {}(line); - storeNewEntity(dbconn, uninsertableLine); + const auto uninsertableLineId = storeUninsertableLine(dbconn, line, originalError); log(LOG_NOTICE, "Failed to store parsed line and/or associated entties, but did store raw line, %u:%s", - *uninsertableLine.id, line.c_str()); + uninsertableLineId, line.c_str()); } catch (const std::exception & excp) { log(LOG_NOTICE, "Failed to store line in any form, DB connection lost? %s", excp.what()); @@ -440,9 +439,8 @@ namespace WebStat { } else { stats.linesParseFailed++; - auto unparsableLine = ToEntity {}(line); - storeNewEntity(dbconn, unparsableLine); - log(LOG_NOTICE, "Failed to parse line, this is a bug: %u:%s", *unparsableLine.id, line.c_str()); + const auto unparsableLineId = storeUnparsableLine(dbconn, line); + log(LOG_NOTICE, "Failed to parse line, this is a bug: %u:%s", unparsableLineId, line.c_str()); } } } @@ -652,6 +650,20 @@ namespace WebStat { stats.entitiesInserted += 1; } + EntityId + Ingestor::storeUnparsableLine(DB::Connection * dbconn, const std::string_view line) const + { + return insert(dbconn, SQL::UNPARSABLE_INSERT, SQL::UNPARSABLE_INSERT_OPTS, line, hostnameId); + } + + EntityId + Ingestor::storeUninsertableLine( + DB::Connection * dbconn, const std::string_view line, const std::exception & excp) const + { + return insert( + dbconn, SQL::UNINSERTABLE_INSERT, SQL::UNINSERTABLE_INSERT_OPTS, line, hostnameId, excp.what()); + } + void Ingestor::onNewUserAgent(const Entity & entity) const { -- cgit v1.3