From 1c929e46d8e7b14a74f80a21b5f30c9abbee410f Mon Sep 17 00:00:00 2001 From: Dan Goodliffe Date: Fri, 1 May 2026 10:32:48 +0100 Subject: Limit the number lines stored at once Limits the number lines inserted per transactions, and the number of transactions before returning to reading input. Prevents long running transactions in the case when queued lines has grown in size. --- src/ingestor.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'src/ingestor.cpp') diff --git a/src/ingestor.cpp b/src/ingestor.cpp index c41454f..04216f8 100644 --- a/src/ingestor.cpp +++ b/src/ingestor.cpp @@ -253,14 +253,19 @@ namespace WebStat { void Ingestor::tryIngestQueuedLogLines() { + auto storedEnd = queuedLines.begin(); try { - ingestLogLines(dbpool->get().get(), queuedLines); - queuedLines.clear(); + for (auto batch : + queuedLines | std::views::chunk(settings.maxBatchSize) | std::views::take(settings.maxBatches)) { + ingestLogLines(dbpool->get().get(), batch); + storedEnd = batch.end(); + } } catch (const std::exception & excp) { log(LOG_ERR, "Unhandled exception: %s, clearing known entity list", excp.what()); existingEntities.clear(); } + queuedLines.erase(queuedLines.begin(), storedEnd); } template @@ -285,7 +290,7 @@ namespace WebStat { } void - Ingestor::ingestLogLines(DB::Connection * dbconn, const LineBatch & lines) + Ingestor::ingestLogLines(DB::Connection * dbconn, const LinesView lines) { auto entityIds = std::views::transform([](auto && value) { return std::make_pair(value->hash, *value->id); -- cgit v1.3