diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/ingestor.cpp | 28 | ||||
| -rw-r--r-- | src/logTypes.hpp | 1 | ||||
| -rw-r--r-- | src/schema.sql | 3 |
3 files changed, 23 insertions, 9 deletions
diff --git a/src/ingestor.cpp b/src/ingestor.cpp index 44107ce..81642be 100644 --- a/src/ingestor.cpp +++ b/src/ingestor.cpp @@ -180,15 +180,26 @@ namespace WebStat { linesParsed++; const auto values = crc32ScanValues(result->values()); NewEntityIds ids; - { - std::optional<DB::TransactionScope> dbtx; - if (const auto newEnts = newEntities(values); newEnts.front()) { - dbtx.emplace(*dbconn); - ids = storeEntities(dbconn, newEnts); + try { + { + std::optional<DB::TransactionScope> dbtx; + if (const auto newEnts = newEntities(values); newEnts.front()) { + dbtx.emplace(*dbconn); + ids = storeEntities(dbconn, newEnts); + } + storeLogLine(dbconn, values); + } + rememberNewEntityIds(ids); + } + catch (const DB::Error & originalError) { + try { + const auto uninsertableLine = ToEntity<EntityType::UninsertableLine> {}(line); + rememberNewEntityIds(storeEntities(dbconn, {uninsertableLine})); + } + catch (const std::exception &) { + throw originalError; } - storeLogLine(dbconn, values); } - rememberNewEntityIds(ids); } else { linesDiscarded++; @@ -310,7 +321,7 @@ namespace WebStat { Ingestor::NewEntityIds Ingestor::storeEntities(DB::Connection * dbconn, const std::span<const std::optional<Entity>> values) const { - static constexpr std::array<std::pair<std::string_view, void (Ingestor::*)(const Entity &) const>, 7> + static constexpr std::array<std::pair<std::string_view, void (Ingestor::*)(const Entity &) const>, 8> ENTITY_TYPE_VALUES {{ {"host", nullptr}, {"virtual_host", nullptr}, @@ -319,6 +330,7 @@ namespace WebStat { {"referrer", nullptr}, {"user_agent", &Ingestor::onNewUserAgent}, {"unparsable_line", nullptr}, + {"uninsertable_line", nullptr}, }}; auto insert = dbconn->modify(SQL::ENTITY_INSERT, SQL::ENTITY_INSERT_OPTS); diff --git a/src/logTypes.hpp b/src/logTypes.hpp index 0262060..f9395d1 100644 --- a/src/logTypes.hpp +++ b/src/logTypes.hpp @@ -30,6 +30,7 @@ namespace WebStat { Referrer, UserAgent, UnparsableLine, + UninsertableLine, }; using Crc32Value = uint32_t; diff --git a/src/schema.sql b/src/schema.sql index d04535e..7648b79 100644 --- a/src/schema.sql +++ b/src/schema.sql @@ -27,7 +27,8 @@ CREATE TYPE entity AS ENUM( 'query_string', 'referrer', 'user_agent', - 'unparsable_line' + 'unparsable_line', + 'uninsertable_line' ); CREATE TABLE entities( |