diff options
Diffstat (limited to 'src/ingestor.hpp')
| -rw-r--r-- | src/ingestor.hpp | 10 |
1 files changed, 10 insertions, 0 deletions
diff --git a/src/ingestor.hpp b/src/ingestor.hpp index afd4a0f..a20071e 100644 --- a/src/ingestor.hpp +++ b/src/ingestor.hpp @@ -23,6 +23,7 @@ namespace WebStat { unsigned int dbMax = 4; unsigned int dbKeep = 2; int idleJobsAfter = duration_cast<milliseconds>(1min).count(); + minutes freqIngestParkedLines = 30min; }; class Ingestor { @@ -44,6 +45,9 @@ namespace WebStat { void ingestLogLine(std::string_view); void ingestLogLine(DB::Connection *, std::string_view); void parkLogLine(std::string_view); + void runJobsIdle(); + + void jobIngestParkedLines(); template<typename... T> void storeLogLine(DB::Connection *, const std::tuple<T...> &) const; @@ -57,6 +61,9 @@ namespace WebStat { size_t linesDiscarded = 0; size_t linesParked = 0; + using JobLastRunTime = std::chrono::system_clock::time_point; + JobLastRunTime lastRunIngestParkedLines; + private: static constexpr size_t MAX_NEW_ENTITIES = 6; void storeEntities(DB::Connection *, std::span<const std::optional<Entity>>) const; @@ -64,6 +71,9 @@ namespace WebStat { template<typename... T> NewEntities newEntities(const std::tuple<T...> &) const; void handleCurlOperations(); + void jobIngestParkedLine(const std::filesystem::directory_iterator &); + void jobIngestParkedLine(const std::filesystem::path &, uintmax_t size); + using CurlOperations = std::map<CURL *, std::unique_ptr<CurlOperation>>; mutable std::flat_set<Crc32Value> existingEntities; uint32_t hostnameId; |