From 61e106fe6a15f76b73125682e91d5c17ee875be2 Mon Sep 17 00:00:00 2001 From: Dan Goodliffe Date: Sat, 4 Jul 2026 12:11:38 +0100 Subject: Include extra details when storing unparsable and uninsertable lines. Now includes the hostnameId, timestamp and optionally error message. Updates idx_entities_retryinsert to account for details not being NULL. Also fixes the issue where the entities inserted and rollback would still be recorded in the entity cache if the access log insert failed. This would have lead to PK violations later on. --- src/sql/markEntityRetried.sql | 2 +- src/sql/selectUninsertableLines.sql | 2 +- src/sql/uninsertableInsert.sql | 5 +++++ src/sql/unparsableInsert.sql | 4 ++++ 4 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 src/sql/uninsertableInsert.sql create mode 100644 src/sql/unparsableInsert.sql (limited to 'src/sql') diff --git a/src/sql/markEntityRetried.sql b/src/sql/markEntityRetried.sql index 6ec2263..a52a15c 100644 --- a/src/sql/markEntityRetried.sql +++ b/src/sql/markEntityRetried.sql @@ -1,6 +1,6 @@ UPDATE entities SET - detail = jsonb_build_object('retriedAt', CURRENT_TIMESTAMP at time zone 'utc', 'error', ?::text) + detail = detail || jsonb_build_object('retriedAt', CURRENT_TIMESTAMP at time zone 'utc', 'error', ?::text) WHERE id = ? diff --git a/src/sql/selectUninsertableLines.sql b/src/sql/selectUninsertableLines.sql index 894ab67..060e800 100644 --- a/src/sql/selectUninsertableLines.sql +++ b/src/sql/selectUninsertableLines.sql @@ -5,7 +5,7 @@ FROM entities WHERE type = 'uninsertable_line' - AND detail IS NULL + AND detail ->> 'retriedAt' IS NULL ORDER BY id LIMIT ? diff --git a/src/sql/uninsertableInsert.sql b/src/sql/uninsertableInsert.sql new file mode 100644 index 0000000..5b09393 --- /dev/null +++ b/src/sql/uninsertableInsert.sql @@ -0,0 +1,5 @@ +INSERT INTO entities(type, value, detail) + VALUES ('uninsertable_line', $1, jsonb_build_object('hostnameId', $2::int, 'error', $3::text, + 'timestamp', CURRENT_TIMESTAMP)) +RETURNING + id diff --git a/src/sql/unparsableInsert.sql b/src/sql/unparsableInsert.sql new file mode 100644 index 0000000..6441f93 --- /dev/null +++ b/src/sql/unparsableInsert.sql @@ -0,0 +1,4 @@ +INSERT INTO entities(type, value, detail) + VALUES ('unparsable_line', $1, jsonb_build_object('hostnameId', $2::int, 'timestamp', CURRENT_TIMESTAMP)) +RETURNING + id -- cgit v1.3