diff options
| author | Dan Goodliffe <dan.goodliffe@octal.co.uk> | 2026-05-19 12:12:28 +0100 |
|---|---|---|
| committer | Dan Goodliffe <dan.goodliffe@octal.co.uk> | 2026-05-19 12:12:28 +0100 |
| commit | a3d868ca57591a6d977da8a7e9ec5c14976e73ec (patch) | |
| tree | 803085077c0a4ad148359738b985e117dff75c42 /src/ingestor.hpp | |
| parent | 29f458117184af5b1507cac01b48b41bfbad568a (diff) | |
| parent | cabd894779c2d2e453c8ed7771fdbd17c881bc2c (diff) | |
| download | webstat-a3d868ca57591a6d977da8a7e9ec5c14976e73ec.tar.bz2 webstat-a3d868ca57591a6d977da8a7e9ec5c14976e73ec.tar.xz webstat-a3d868ca57591a6d977da8a7e9ec5c14976e73ec.zip | |
Merge remote-tracking branch 'origin/retry-store'
Diffstat (limited to 'src/ingestor.hpp')
| -rw-r--r-- | src/ingestor.hpp | 6 |
1 files changed, 4 insertions, 2 deletions
diff --git a/src/ingestor.hpp b/src/ingestor.hpp index 738357b..2050b7c 100644 --- a/src/ingestor.hpp +++ b/src/ingestor.hpp @@ -28,6 +28,7 @@ namespace WebStat { size_t maxBatchSize = 1; minutes checkJobsAfter = 1min; minutes freqIngestParkedLines = 30min; + minutes freqRetryUninsertableLines = 4h; minutes freqPurgeOldLogs = 6h; unsigned int purgeDaysToKeep = 61; // ~2 months unsigned int purgeDeleteMax = 10'000; @@ -78,6 +79,7 @@ namespace WebStat { Job::Result jobReadParkedLines(); Job::Result jobPurgeOldLogs(); Job::Result jobStoreQueuedLines(); + Job::Result jobRetryUninsertableLines(); template<typename... T> void storeLogLine(DB::Connection *, const std::tuple<T...> &) const; @@ -99,7 +101,7 @@ namespace WebStat { DB::ConnectionPoolPtr dbpool; mutable Stats stats {}; - std::map<EntityHash, EntityId> existingEntities; + ThreadSafeT<std::map<EntityHash, EntityId>> existingEntities; LineBatch queuedLines, processingLines; bool terminated = false; @@ -109,8 +111,8 @@ namespace WebStat { Job ingestParkedLines; Job purgeOldLogs; Job storeQueueLines; + Job retryUninsertableLines; - private: template<typename... T> static std::vector<Entity *> entities(std::tuple<T...> &); void fillKnownEntities(std::span<Entity *>) const; void storeNewEntities(DB::Connection *, std::span<Entity *>) const; |