diff options
| author | Dan Goodliffe <dan@randomdan.homeip.net> | 2026-04-18 01:00:42 +0100 |
|---|---|---|
| committer | Dan Goodliffe <dan@randomdan.homeip.net> | 2026-04-18 01:00:42 +0100 |
| commit | 7b148411b127ebf6fdefdb1b0decd2886cdfc17b (patch) | |
| tree | 56385615fe86dbb5ab7ce4a90d696d97ddf21d7e /src/ingestor.hpp | |
| parent | 1e551e618a63c869fde6a4b327566b38696a5f45 (diff) | |
| parent | fa6074eaf52be4254c17b74f20193aa96c940df8 (diff) | |
| download | webstat-7b148411b127ebf6fdefdb1b0decd2886cdfc17b.tar.bz2 webstat-7b148411b127ebf6fdefdb1b0decd2886cdfc17b.tar.xz webstat-7b148411b127ebf6fdefdb1b0decd2886cdfc17b.zip | |
Merge remote-tracking branch 'origin/no-crc32'HEADwebstat-0.4main
Diffstat (limited to 'src/ingestor.hpp')
| -rw-r--r-- | src/ingestor.hpp | 18 |
1 files changed, 9 insertions, 9 deletions
diff --git a/src/ingestor.hpp b/src/ingestor.hpp index fcddc92..195f325 100644 --- a/src/ingestor.hpp +++ b/src/ingestor.hpp @@ -7,7 +7,8 @@ #include <connectionPool.h> #include <connection_fwd.h> #include <cstdio> -#include <flat_set> +#include <expected> +#include <flat_map> #include <future> #include <scn/scan.h> #include <span> @@ -54,7 +55,7 @@ namespace WebStat { void ingestLog(std::FILE *); void tryIngestQueuedLogLines(); void ingestLogLines(DB::Connection *, const LineBatch & lines); - void parkQueuedLogLines(); + std::expected<std::filesystem::path, int> parkQueuedLogLines(); void runJobsAsNeeded(); unsigned int jobIngestParkedLines(); @@ -78,7 +79,7 @@ namespace WebStat { DB::ConnectionPoolPtr dbpool; mutable Stats stats {}; - std::flat_set<Crc32Value> existingEntities; + std::flat_map<EntityHash, EntityId> existingEntities; LineBatch queuedLines; bool terminated = false; @@ -99,11 +100,10 @@ namespace WebStat { Job purgeOldLogs; private: - static constexpr size_t MAX_NEW_ENTITIES = 6; - using NewEntityIds = std::array<std::optional<Crc32Value>, MAX_NEW_ENTITIES>; - NewEntityIds storeEntities(DB::Connection *, std::span<const std::optional<Entity>>) const; - using NewEntities = std::array<std::optional<Entity>, MAX_NEW_ENTITIES>; - template<typename... T> NewEntities newEntities(const std::tuple<T...> &) const; + template<typename... T> static std::vector<Entity *> entities(std::tuple<T...> &); + void fillKnownEntities(std::span<Entity *>) const; + void storeNewEntities(DB::Connection *, std::span<Entity *>) const; + void storeNewEntity(DB::Connection *, Entity &) const; void onNewUserAgent(const Entity &) const; void handleCurlOperations(); void logStats() const; @@ -119,7 +119,7 @@ namespace WebStat { [[gnu::format(printf, 3, 4)]] virtual void log(int level, const char * msgfmt, ...) const = 0; using CurlOperations = std::map<CURL *, std::unique_ptr<CurlOperation>>; - uint32_t hostnameId; + EntityId hostnameId; CurlMultiPtr curl; mutable CurlOperations curlOperations; std::thread::id mainThread; |