diff options
| -rw-r--r-- | src/ingestor.cpp | 15 | ||||
| -rw-r--r-- | src/ingestor.hpp | 3 | ||||
| -rw-r--r-- | src/webstat_logger_main.cpp | 2 | ||||
| -rw-r--r-- | test/test-ingest.cpp | 28 |
4 files changed, 47 insertions, 1 deletions
diff --git a/src/ingestor.cpp b/src/ingestor.cpp index db11799..639eed0 100644 --- a/src/ingestor.cpp +++ b/src/ingestor.cpp @@ -4,6 +4,7 @@ #include "util.hpp" #include <connection.h> #include <dbTypes.h> +#include <fstream> #include <modifycommand.h> #include <ranges> #include <scn/scan.h> @@ -148,7 +149,12 @@ namespace WebStat { void Ingestor::ingestLogLine(const std::string_view line) { - ingestLogLine(dbpool->get().get(), line); + try { + ingestLogLine(dbpool->get().get(), line); + } + catch (const std::exception &) { + parkLogLine(line); + } } void @@ -171,6 +177,13 @@ namespace WebStat { } } + void + Ingestor::parkLogLine(std::string_view line) + { + std::ofstream {settings.fallbackDir / std::format("parked-{}.log", crc32(line))} << line; + linesParked++; + } + template<typename... T> Ingestor::NewEntities Ingestor::newEntities(const std::tuple<T...> & values) const diff --git a/src/ingestor.hpp b/src/ingestor.hpp index 719b65b..42c6699 100644 --- a/src/ingestor.hpp +++ b/src/ingestor.hpp @@ -16,6 +16,7 @@ namespace WebStat { struct IngestorSettings : Settings { std::string dbConnStr = "dbname=webstat user=webstat"; std::string userAgentAPI = "https://useragentstring.com"; + std::filesystem::path fallbackDir = "/var/log/webstat"; unsigned int dbMax = 4; unsigned int dbKeep = 2; }; @@ -38,6 +39,7 @@ namespace WebStat { void ingestLog(std::FILE *); void ingestLogLine(std::string_view); void ingestLogLine(DB::Connection *, std::string_view); + void parkLogLine(std::string_view); template<typename... T> void storeLogLine(DB::Connection *, const std::tuple<T...> &) const; @@ -49,6 +51,7 @@ namespace WebStat { size_t linesRead = 0; size_t linesParsed = 0; size_t linesDiscarded = 0; + size_t linesParked = 0; private: static constexpr size_t MAX_NEW_ENTITIES = 6; diff --git a/src/webstat_logger_main.cpp b/src/webstat_logger_main.cpp index eb8a30d..9ce4e73 100644 --- a/src/webstat_logger_main.cpp +++ b/src/webstat_logger_main.cpp @@ -38,6 +38,8 @@ main(int argc, char ** argv) "Maximum number of concurrent write/read write DB connections") ("db.wr.keep", po::value(&settings.dbKeep)->default_value(settings.dbKeep), "Number of write/read write DB connections to keep open") + ("fallback.dir", po::value(&settings.fallbackDir)->default_value(settings.fallbackDir), + "Path to write access logs to when the database is unavailable") ; // clang-format on po::variables_map optVars; diff --git a/test/test-ingest.cpp b/test/test-ingest.cpp index 83bac48..f86dd25 100644 --- a/test/test-ingest.cpp +++ b/test/test-ingest.cpp @@ -198,9 +198,18 @@ public: WebStat::Ingestor {WebStat::getTestUtsName("test-hostname"), std::make_shared<MockDBPool>("webstat"), { .userAgentAPI = FIXTURE_URL_BASE + "/userAgent.json", + .fallbackDir = std::format("/tmp/webstat-{}", getpid()), }} { + std::filesystem::create_directories(settings.fallbackDir); } + + ~TestIngestor() override + { + std::filesystem::remove_all(settings.fallbackDir); + } + + SPECIAL_MEMBERS_DELETE(TestIngestor); }; BOOST_FIXTURE_TEST_SUITE(I, TestIngestor); @@ -231,6 +240,25 @@ BOOST_AUTO_TEST_CASE(StoreLog, *boost::unit_test::depends_on("I/StoreLogLine")) BOOST_CHECK_EQUAL(linesDiscarded, 0); } +BOOST_AUTO_TEST_CASE(ParkLogLine) +{ + parkLogLine(LOGLINE1); + BOOST_CHECK_EQUAL(linesParked, 1); + const auto path = settings.fallbackDir / "parked-3377916038.log"; + BOOST_TEST_INFO(path); + BOOST_REQUIRE(std::filesystem::exists(path)); + BOOST_CHECK_EQUAL(std::filesystem::file_size(path), LOGLINE1.length()); +} + +BOOST_TEST_DECORATOR(*boost::unit_test::depends_on("I/ParkLogLine")) + +BOOST_AUTO_TEST_CASE(ParkLogLineOnError) +{ + BOOST_REQUIRE_NO_THROW(dbpool->get()->execute("SET search_path = ''")); + BOOST_REQUIRE_NO_THROW(ingestLogLine(LOGLINE1)); + BOOST_CHECK_EQUAL(linesParked, 1); +} + BOOST_AUTO_TEST_CASE(FetchMockUserAgentDetail) { const auto uaDetailReq = WebStat::curlGetUserAgentDetail(0, |