From 1c929e46d8e7b14a74f80a21b5f30c9abbee410f Mon Sep 17 00:00:00 2001
From: Dan Goodliffe <dan.goodliffe@octal.co.uk>
Date: Fri, 1 May 2026 10:32:48 +0100
Subject: Limit the number lines stored at once

Limits the number lines inserted per transactions, and the number of
transactions before returning to reading input.
Prevents long running transactions in the case when queued lines has
grown in size.
---
 src/ingestor.hpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'src/ingestor.hpp')

diff --git a/src/ingestor.hpp b/src/ingestor.hpp
index 195f325..916f200 100644
--- a/src/ingestor.hpp
+++ b/src/ingestor.hpp
@@ -26,6 +26,7 @@ namespace WebStat {
 		unsigned int dbMax = 4;
 		unsigned int dbKeep = 2;
 		size_t maxBatchSize = 1;
+		size_t maxBatches = 5;
 		minutes checkJobsAfter = 1min;
 		minutes freqIngestParkedLines = 30min;
 		minutes freqPurgeOldLogs = 6h;
@@ -39,6 +40,7 @@ namespace WebStat {
 	class Ingestor {
 	public:
 		using LineBatch = std::vector<std::string>;
+		using LinesView = std::span<const std::string>;
 		Ingestor(const utsname &, IngestorSettings);
 		Ingestor(const utsname &, DB::ConnectionPoolPtr, IngestorSettings);
 
@@ -54,7 +56,7 @@ namespace WebStat {
 
 		void ingestLog(std::FILE *);
 		void tryIngestQueuedLogLines();
-		void ingestLogLines(DB::Connection *, const LineBatch & lines);
+		void ingestLogLines(DB::Connection *, LinesView lines);
 		std::expected<std::filesystem::path, int> parkQueuedLogLines();
 		void runJobsAsNeeded();
 
-- 
cgit v1.3