diff options
| author | Dan Goodliffe <dan.goodliffe@octal.co.uk> | 2025-09-23 13:31:26 +0100 |
|---|---|---|
| committer | Dan Goodliffe <dan.goodliffe@octal.co.uk> | 2025-09-23 13:31:26 +0100 |
| commit | 71803b97f1c9e31f2027da48bb742353f9c43e62 (patch) | |
| tree | 99f418aa8cf4d4dd9edd2de1985295c48b4ba904 /src/ingestor.cpp | |
| parent | 4785b4d8c5bac4e03ed8dd2a4c01ec098da60d7b (diff) | |
| download | webstat-71803b97f1c9e31f2027da48bb742353f9c43e62.tar.bz2 webstat-71803b97f1c9e31f2027da48bb742353f9c43e62.tar.xz webstat-71803b97f1c9e31f2027da48bb742353f9c43e62.zip | |
Write unparsable lines to the entity table
Diagnostics and the ability to ingest later.
Diffstat (limited to 'src/ingestor.cpp')
| -rw-r--r-- | src/ingestor.cpp | 13 |
1 files changed, 10 insertions, 3 deletions
diff --git a/src/ingestor.cpp b/src/ingestor.cpp index e1cc7c6..db5f317 100644 --- a/src/ingestor.cpp +++ b/src/ingestor.cpp @@ -121,7 +121,7 @@ namespace WebStat { if (logIn.revents) { if (auto line = scn::scan<std::string>(input, "{:[^\n]}\n")) { linesRead++; - ingestLogLine(dbpool->get().get(), line->value()); + ingestLogLine(line->value()); } else { break; @@ -137,6 +137,12 @@ namespace WebStat { } void + Ingestor::ingestLogLine(const std::string_view line) + { + ingestLogLine(dbpool->get().get(), line); + } + + void Ingestor::ingestLogLine(DB::Connection * dbconn, const std::string_view line) { if (auto result = scanLogLine(line)) { @@ -150,8 +156,9 @@ namespace WebStat { storeLogLine(dbconn, values); } else { - syslog(LOG_WARNING, "Discarded line: [%.*s]", static_cast<int>(line.length()), line.data()); linesDiscarded++; + const auto unparsableLine = toEntity(line, EntityType::UnparsableLine); + storeEntities(dbconn, {unparsableLine}); } } @@ -185,7 +192,7 @@ namespace WebStat { Ingestor::storeEntities(DB::Connection * dbconn, const std::span<const std::optional<Entity>> values) const { static constexpr std::array ENTITY_TYPE_VALUES { - "host", "virtual_host", "path", "query_string", "referrer", "user_agent"}; + "host", "virtual_host", "path", "query_string", "referrer", "user_agent", "unparsable_line"}; auto insert = dbconn->modify(SQL::ENTITY_INSERT, SQL::ENTITY_INSERT_OPTS); std::ranges::for_each( |