Add logging :-o

Adds virtual log function, real implementation writes to syslog.
Test implementation writes to BOOST_TEST_MESSAGE, perf implementation
discards.
Replaces existing prints to stderr and adds logs to all key points.

5 files changed, 59 insertions, 10 deletions

diff --git a/src/ingestor.cpp b/src/ingestor.cpp
index 1de9ab9..fb592d7 100644
--- a/src/ingestor.cpp
+++ b/src/ingestor.cpp
@@ -109,8 +109,9 @@ namespace WebStat {
 	}
 
 	void
-	Ingestor::terminate(int)
+	Ingestor::terminate(int sigNo)
 	{
+		log(LOG_NOTICE, "Caught sig %d, terminating", sigNo);
 		terminated = true;
 		curl_multi_wakeup(curl.get());
 	}
@@ -154,7 +155,7 @@ namespace WebStat {
 				}
 				else {
 					curlOperations.erase(msg->easy_handle);
-					std::println(std::cerr, "Failed to lookup CurlOperation");
+					log(LOG_WARNING, "Failed to lookup CurlOperation");
 				}
 			}
 		}
@@ -204,7 +205,8 @@ namespace WebStat {
 			ingestLogLines(dbpool->get().get(), queuedLines);
 			queuedLines.clear();
 		}
-		catch (const std::exception &) {
+		catch (const std::exception & excp) {
+			log(LOG_ERR, "Unhandled exception: %s, clearing known entity list", excp.what());
 			existingEntities.clear();
 		}
 	}
@@ -234,8 +236,12 @@ namespace WebStat {
 						DB::TransactionScope dbtx {*dbconn};
 						const auto uninsertableLine = ToEntity<EntityType::UninsertableLine> {}(line);
 						storeEntities(dbconn, {uninsertableLine});
+						log(LOG_NOTICE,
+								"Failed to store parsed line and/or associated entties, but did store raw line, %u:%s",
+								std::get<Crc32Value>(uninsertableLine), line.c_str());
 					}
-					catch (const std::exception &) {
+					catch (const std::exception & excp) {
+						log(LOG_NOTICE, "Failed to store line in any form, DB connection lost? %s", excp.what());
 						throw originalError;
 					}
 				}
@@ -243,6 +249,8 @@ namespace WebStat {
 			else {
 				linesDiscarded++;
 				const auto unparsableLine = ToEntity<EntityType::UnparsableLine> {}(line);
+				log(LOG_NOTICE, "Failed to parse line, this is a bug: %u:%s", std::get<Crc32Value>(unparsableLine),
+						line.c_str());
 				storeEntities(dbconn, {unparsableLine});
 			}
 		}
@@ -263,10 +271,13 @@ namespace WebStat {
 			if (fflush(parked.get()) == 0) {
 				linesParked += queuedLines.size();
 				queuedLines.clear();
+				return;
 			}
-			else {
-				std::filesystem::remove(path);
-			}
+			std::filesystem::remove(path);
+		}
+		log(LOG_ERR, "Failed to park %zu queued lines:", queuedLines.size());
+		for (const auto & line : queuedLines) {
+			log(LOG_ERR, "\t%.*s", static_cast<int>(line.length()), line.data());
 		}
 	}
 
@@ -280,7 +291,8 @@ namespace WebStat {
 						job.currentRun->get();
 						job.lastRun = now;
 					}
-					catch (const std::exception &) {
+					catch (const std::exception & excp) {
+						log(LOG_ERR, "Job run failed: %s", excp.what());
 						// Error, retry in half the frequency
 						job.lastRun = now - (freq / 2);
 					}
diff --git a/src/ingestor.hpp b/src/ingestor.hpp
index 94e0d5c..e890bcf 100644
--- a/src/ingestor.hpp
+++ b/src/ingestor.hpp
@@ -106,6 +106,7 @@ namespace WebStat {
 
 		static void sigtermHandler(int);
 		void terminate(int);
+		[[gnu::format(printf, 3, 4)]] virtual void log(int level, const char * msgfmt, ...) const = 0;
 
 		using CurlOperations = std::map<CURL *, std::unique_ptr<CurlOperation>>;
 		uint32_t hostnameId;
diff --git a/src/webstat_logger_main.cpp b/src/webstat_logger_main.cpp
index 39d794a..3eaa9f0 100644
--- a/src/webstat_logger_main.cpp
+++ b/src/webstat_logger_main.cpp
@@ -5,6 +5,7 @@
 #include <iostream>
 #include <pq-connection.h>
 #include <sys/utsname.h>
+#include <syslog.h>
 
 namespace {
 	[[nodiscard]]
@@ -17,6 +18,19 @@ namespace {
 		}
 		return uts;
 	}
+
+	class MainIngestor : public WebStat::Ingestor {
+		using Ingestor::Ingestor;
+
+		void
+		log(int level, const char * msgfmt, ...) const override
+		{
+			va_list args;
+			va_start(args, msgfmt);
+			vsyslog(level, msgfmt, args);
+			va_end(args);
+		}
+	};
 }
 
 #define LEXICAL_CAST_DURATION(UNIT) \
@@ -87,7 +101,7 @@ main(int argc, char ** argv)
 	po::notify(optVars);
 
 	try {
-		WebStat::Ingestor {getHostDetail(), std::move(settings)}.ingestLog(stdin);
+		MainIngestor {getHostDetail(), std::move(settings)}.ingestLog(stdin);
 		return EXIT_SUCCESS;
 	}
 	catch (const std::exception & excp) {
diff --git a/test/perf-ingest.cpp b/test/perf-ingest.cpp
index c403349..cf0be0d 100644
--- a/test/perf-ingest.cpp
+++ b/test/perf-ingest.cpp
@@ -18,10 +18,19 @@ namespace {
 		static const WebStat::MockDB mockdb;
 	}
 
+	class PerfIngestor : public WebStat::Ingestor {
+		using Ingestor::Ingestor;
+
+		void
+		log(int, const char *, ...) const override
+		{
+		}
+	};
+
 	void
 	doIngestFile(benchmark::State & state)
 	{
-		WebStat::Ingestor ingestor {WebStat::getTestUtsName("perf-hostname"),
+		PerfIngestor ingestor {WebStat::getTestUtsName("perf-hostname"),
 				std::make_shared<WebStat::MockDBPool>("webstat"),
 				{
 						.userAgentAPI = {},
diff --git a/test/test-ingest.cpp b/test/test-ingest.cpp
index a1dc5e9..9ce48be 100644
--- a/test/test-ingest.cpp
+++ b/test/test-ingest.cpp
@@ -3,6 +3,7 @@
 #include <boost/test/unit_test.hpp>
 
 #include "testing-util.hpp"
+#include <cstdio>
 #include <selectcommandUtil.impl.h>
 
 #include <ingestor.hpp>
@@ -217,6 +218,18 @@ public:
 	}
 
 	SPECIAL_MEMBERS_DELETE(TestIngestor);
+
+	[[gnu::format(printf, 3, 4)]] void
+	log(int, const char * msgfmt, ...) const override
+	{
+		va_list args;
+		va_start(args, msgfmt);
+		std::unique_ptr<char, DeleteWith<&free>> msg;
+		BOOST_REQUIRE_GE(vasprintf(std::out_ptr(msg), msgfmt, args), 0);
+		va_end(args);
+		BOOST_REQUIRE(msg);
+		BOOST_TEST_MESSAGE(msg.get());
+	}
 };
 
 BOOST_FIXTURE_TEST_SUITE(I, TestIngestor);