Include extra details when storing unparsable and uninsertable lines.

Now includes the hostnameId, timestamp and optionally error message.
Updates idx_entities_retryinsert to account for details not being NULL.
Also fixes the issue where the entities inserted and rollback would
still be recorded in the entity cache if the access log insert failed.
This would have lead to PK violations later on.

1 files changed, 19 insertions, 7 deletions

diff --git a/src/ingestor.cpp b/src/ingestor.cpp
index ff5cd92..077f7d8 100644
--- a/src/ingestor.cpp
+++ b/src/ingestor.cpp
@@ -420,17 +420,16 @@ namespace WebStat {
 				try {
 					DB::TransactionScope lineTx {*dbconn};
 					storeNewEntities(dbconn, valuesEntities);
-					existingEntities()->insert_range(valuesEntities | ENTITY_IDS);
 					storeLogLine(dbconn, values);
+					existingEntities()->insert_range(valuesEntities | ENTITY_IDS);
 				}
 				catch (const DB::Error & originalError) {
 					try {
 						DB::TransactionScope lineTx {*dbconn};
-						auto uninsertableLine = ToEntity<EntityType::UninsertableLine> {}(line);
-						storeNewEntity(dbconn, uninsertableLine);
+						const auto uninsertableLineId = storeUninsertableLine(dbconn, line, originalError);
 						log(LOG_NOTICE,
 								"Failed to store parsed line and/or associated entties, but did store raw line, %u:%s",
-								*uninsertableLine.id, line.c_str());
+								uninsertableLineId, line.c_str());
 					}
 					catch (const std::exception & excp) {
 						log(LOG_NOTICE, "Failed to store line in any form, DB connection lost? %s", excp.what());
@@ -440,9 +439,8 @@ namespace WebStat {
 			}
 			else {
 				stats.linesParseFailed++;
-				auto unparsableLine = ToEntity<EntityType::UnparsableLine> {}(line);
-				storeNewEntity(dbconn, unparsableLine);
-				log(LOG_NOTICE, "Failed to parse line, this is a bug: %u:%s", *unparsableLine.id, line.c_str());
+				const auto unparsableLineId = storeUnparsableLine(dbconn, line);
+				log(LOG_NOTICE, "Failed to parse line, this is a bug: %u:%s", unparsableLineId, line.c_str());
 			}
 		}
 	}
@@ -652,6 +650,20 @@ namespace WebStat {
 		stats.entitiesInserted += 1;
 	}
 
+	EntityId
+	Ingestor::storeUnparsableLine(DB::Connection * dbconn, const std::string_view line) const
+	{
+		return insert<EntityId>(dbconn, SQL::UNPARSABLE_INSERT, SQL::UNPARSABLE_INSERT_OPTS, line, hostnameId);
+	}
+
+	EntityId
+	Ingestor::storeUninsertableLine(
+			DB::Connection * dbconn, const std::string_view line, const std::exception & excp) const
+	{
+		return insert<EntityId>(
+				dbconn, SQL::UNINSERTABLE_INSERT, SQL::UNINSERTABLE_INSERT_OPTS, line, hostnameId, excp.what());
+	}
+
 	void
 	Ingestor::onNewUserAgent(const Entity & entity) const
 	{