From 3e99d080b2a3a9b6eae85ae9e3224534744ad7b9 Mon Sep 17 00:00:00 2001
From: Dan Goodliffe <dan@randomdan.homeip.net>
Date: Tue, 30 Sep 2025 00:50:29 +0100
Subject: Write log lines to files on error

We call this parking, later we can reattempt ingestion after whatever
caused the failure has been fixed.
---
 src/ingestor.cpp            | 15 ++++++++++++++-
 src/ingestor.hpp            |  3 +++
 src/webstat_logger_main.cpp |  2 ++
 test/test-ingest.cpp        | 28 ++++++++++++++++++++++++++++
 4 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/src/ingestor.cpp b/src/ingestor.cpp
index db11799..639eed0 100644
--- a/src/ingestor.cpp
+++ b/src/ingestor.cpp
@@ -4,6 +4,7 @@
 #include "util.hpp"
 #include <connection.h>
 #include <dbTypes.h>
+#include <fstream>
 #include <modifycommand.h>
 #include <ranges>
 #include <scn/scan.h>
@@ -148,7 +149,12 @@ namespace WebStat {
 	void
 	Ingestor::ingestLogLine(const std::string_view line)
 	{
-		ingestLogLine(dbpool->get().get(), line);
+		try {
+			ingestLogLine(dbpool->get().get(), line);
+		}
+		catch (const std::exception &) {
+			parkLogLine(line);
+		}
 	}
 
 	void
@@ -171,6 +177,13 @@ namespace WebStat {
 		}
 	}
 
+	void
+	Ingestor::parkLogLine(std::string_view line)
+	{
+		std::ofstream {settings.fallbackDir / std::format("parked-{}.log", crc32(line))} << line;
+		linesParked++;
+	}
+
 	template<typename... T>
 	Ingestor::NewEntities
 	Ingestor::newEntities(const std::tuple<T...> & values) const
diff --git a/src/ingestor.hpp b/src/ingestor.hpp
index 719b65b..42c6699 100644
--- a/src/ingestor.hpp
+++ b/src/ingestor.hpp
@@ -16,6 +16,7 @@ namespace WebStat {
 	struct IngestorSettings : Settings {
 		std::string dbConnStr = "dbname=webstat user=webstat";
 		std::string userAgentAPI = "https://useragentstring.com";
+		std::filesystem::path fallbackDir = "/var/log/webstat";
 		unsigned int dbMax = 4;
 		unsigned int dbKeep = 2;
 	};
@@ -38,6 +39,7 @@ namespace WebStat {
 		void ingestLog(std::FILE *);
 		void ingestLogLine(std::string_view);
 		void ingestLogLine(DB::Connection *, std::string_view);
+		void parkLogLine(std::string_view);
 
 		template<typename... T> void storeLogLine(DB::Connection *, const std::tuple<T...> &) const;
 
@@ -49,6 +51,7 @@ namespace WebStat {
 		size_t linesRead = 0;
 		size_t linesParsed = 0;
 		size_t linesDiscarded = 0;
+		size_t linesParked = 0;
 
 	private:
 		static constexpr size_t MAX_NEW_ENTITIES = 6;
diff --git a/src/webstat_logger_main.cpp b/src/webstat_logger_main.cpp
index eb8a30d..9ce4e73 100644
--- a/src/webstat_logger_main.cpp
+++ b/src/webstat_logger_main.cpp
@@ -38,6 +38,8 @@ main(int argc, char ** argv)
 		 "Maximum number of concurrent write/read write DB connections")
 		("db.wr.keep", po::value(&settings.dbKeep)->default_value(settings.dbKeep),
 		 "Number of write/read write DB connections to keep open")
+		("fallback.dir", po::value(&settings.fallbackDir)->default_value(settings.fallbackDir),
+		 "Path to write access logs to when the database is unavailable")
 		;
 	// clang-format on
 	po::variables_map optVars;
diff --git a/test/test-ingest.cpp b/test/test-ingest.cpp
index 83bac48..f86dd25 100644
--- a/test/test-ingest.cpp
+++ b/test/test-ingest.cpp
@@ -198,9 +198,18 @@ public:
 		WebStat::Ingestor {WebStat::getTestUtsName("test-hostname"), std::make_shared<MockDBPool>("webstat"),
 				{
 						.userAgentAPI = FIXTURE_URL_BASE + "/userAgent.json",
+						.fallbackDir = std::format("/tmp/webstat-{}", getpid()),
 				}}
 	{
+		std::filesystem::create_directories(settings.fallbackDir);
 	}
+
+	~TestIngestor() override
+	{
+		std::filesystem::remove_all(settings.fallbackDir);
+	}
+
+	SPECIAL_MEMBERS_DELETE(TestIngestor);
 };
 
 BOOST_FIXTURE_TEST_SUITE(I, TestIngestor);
@@ -231,6 +240,25 @@ BOOST_AUTO_TEST_CASE(StoreLog, *boost::unit_test::depends_on("I/StoreLogLine"))
 	BOOST_CHECK_EQUAL(linesDiscarded, 0);
 }
 
+BOOST_AUTO_TEST_CASE(ParkLogLine)
+{
+	parkLogLine(LOGLINE1);
+	BOOST_CHECK_EQUAL(linesParked, 1);
+	const auto path = settings.fallbackDir / "parked-3377916038.log";
+	BOOST_TEST_INFO(path);
+	BOOST_REQUIRE(std::filesystem::exists(path));
+	BOOST_CHECK_EQUAL(std::filesystem::file_size(path), LOGLINE1.length());
+}
+
+BOOST_TEST_DECORATOR(*boost::unit_test::depends_on("I/ParkLogLine"))
+
+BOOST_AUTO_TEST_CASE(ParkLogLineOnError)
+{
+	BOOST_REQUIRE_NO_THROW(dbpool->get()->execute("SET search_path = ''"));
+	BOOST_REQUIRE_NO_THROW(ingestLogLine(LOGLINE1));
+	BOOST_CHECK_EQUAL(linesParked, 1);
+}
+
 BOOST_AUTO_TEST_CASE(FetchMockUserAgentDetail)
 {
 	const auto uaDetailReq = WebStat::curlGetUserAgentDetail(0,
-- 
cgit v1.2.3