diff options
| -rw-r--r-- | src/ingestor.cpp | 53 | ||||
| -rw-r--r-- | src/ingestor.hpp | 6 | ||||
| -rw-r--r-- | src/util.hpp | 6 |
3 files changed, 43 insertions, 22 deletions
diff --git a/src/ingestor.cpp b/src/ingestor.cpp index 1b54a64..f824c80 100644 --- a/src/ingestor.cpp +++ b/src/ingestor.cpp @@ -4,6 +4,7 @@ #include <connection.h> #include <dbTypes.h> #include <modifycommand.h> +#include <ranges> #include <scn/scan.h> #include <syslog.h> #include <utility> @@ -55,7 +56,9 @@ namespace WebStat { Ingestor::Ingestor(const std::string_view hostname, DB::ConnectionPtr dbconn) : hostnameId {crc32(hostname)}, dbconn {std::move(dbconn)} { - storeEntities(std::make_tuple(std::make_pair(hostnameId, hostname))); + storeEntities({ + std::make_pair(hostnameId, hostname), + }); } Ingestor::ScanResult @@ -92,7 +95,11 @@ namespace WebStat { if (auto result = scanLogLine(line)) { linesParsed++; const auto values = crc32ScanValues(result->values()); - storeEntities(values); + std::optional<DB::TransactionScope> dbtx; + if (const auto newEnts = newEntities(values); newEnts.front()) { + dbtx.emplace(*dbconn); + storeEntities(newEnts); + } storeLogLine(values); } else { @@ -102,32 +109,42 @@ namespace WebStat { } template<typename... T> - size_t - Ingestor::storeEntities(const std::tuple<T...> & values) const + Ingestor::NewEntities + Ingestor::newEntities(const std::tuple<T...> & values) const { - return visitSum( - [this]<typename X>(const X & entity) -> size_t { - auto insertIfReqd = [this](auto && entity) -> size_t { - if (existingEntities.contains(entity.first)) { - return 0; + Ingestor::NewEntities rtn; + auto next = rtn.begin(); + visit( + [this, &next]<typename X>(const X & entity) { + auto addNewIfReqd = [&next, this](auto && entity) mutable { + if (!existingEntities.contains(entity.first)) { + *next++ = entity; } - auto insert = dbconn->modify(SQL::ENTITY_INSERT, SQL::ENTITY_INSERT_OPTS); - insert->bindParamI(0, entity.first); - insert->bindParamS(1, entity.second); - insert->execute(); - existingEntities.emplace(entity.first); - return 1; + return 0; }; if constexpr (std::is_same_v<X, Entity>) { - return insertIfReqd(entity); + addNewIfReqd(entity); } else if constexpr (std::is_same_v<X, std::optional<Entity>>) { - entity.transform(insertIfReqd).value_or(0); + entity.transform(addNewIfReqd); } - return 0; }, values); + return rtn; + } + + void + Ingestor::storeEntities(const std::span<const std::optional<Entity>> values) const + { + std::ranges::for_each( + values | std::views::take_while(&std::optional<Entity>::has_value), [this](auto && entity) { + auto insert = dbconn->modify(SQL::ENTITY_INSERT, SQL::ENTITY_INSERT_OPTS); + insert->bindParamI(0, entity->first); + insert->bindParamS(1, entity->second); + insert->execute(); + existingEntities.emplace(entity->first); + }); } template<typename... T> diff --git a/src/ingestor.hpp b/src/ingestor.hpp index 7141515..040331f 100644 --- a/src/ingestor.hpp +++ b/src/ingestor.hpp @@ -6,6 +6,7 @@ #include <cstdio> #include <flat_set> #include <scn/scan.h> +#include <span> namespace WebStat { class Ingestor { @@ -33,7 +34,10 @@ namespace WebStat { size_t linesDiscarded = 0; private: - template<typename... T> size_t storeEntities(const std::tuple<T...> &) const; + static constexpr size_t MAX_NEW_ENTITIES = 6; + void storeEntities(std::span<const std::optional<Entity>>) const; + using NewEntities = std::array<std::optional<Entity>, MAX_NEW_ENTITIES>; + template<typename... T> NewEntities newEntities(const std::tuple<T...> &) const; mutable std::flat_set<Crc32Value> existingEntities; uint32_t hostnameId; diff --git a/src/util.hpp b/src/util.hpp index 0ed260c..dcdc654 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -5,11 +5,11 @@ namespace WebStat { template<typename... T> auto - visitSum(auto && visitor, const std::tuple<T...> & values) + visit(auto && visitor, const std::tuple<T...> & values) { - return std::apply( + std::apply( [&](auto &&... value) { - return (visitor(value) + ...); + (visitor(value), ...); }, values); } |