Add parked line import jobwebstat-0.2

Periodically, on idle, scan for and import previously parked lines.

1 files changed, 10 insertions, 0 deletions

diff --git a/src/ingestor.hpp b/src/ingestor.hpp
index afd4a0f..a20071e 100644
--- a/src/ingestor.hpp
+++ b/src/ingestor.hpp
@@ -23,6 +23,7 @@ namespace WebStat {
 		unsigned int dbMax = 4;
 		unsigned int dbKeep = 2;
 		int idleJobsAfter = duration_cast<milliseconds>(1min).count();
+		minutes freqIngestParkedLines = 30min;
 	};
 
 	class Ingestor {
@@ -44,6 +45,9 @@ namespace WebStat {
 		void ingestLogLine(std::string_view);
 		void ingestLogLine(DB::Connection *, std::string_view);
 		void parkLogLine(std::string_view);
+		void runJobsIdle();
+
+		void jobIngestParkedLines();
 
 		template<typename... T> void storeLogLine(DB::Connection *, const std::tuple<T...> &) const;
 
@@ -57,6 +61,9 @@ namespace WebStat {
 		size_t linesDiscarded = 0;
 		size_t linesParked = 0;
 
+		using JobLastRunTime = std::chrono::system_clock::time_point;
+		JobLastRunTime lastRunIngestParkedLines;
+
 	private:
 		static constexpr size_t MAX_NEW_ENTITIES = 6;
 		void storeEntities(DB::Connection *, std::span<const std::optional<Entity>>) const;
@@ -64,6 +71,9 @@ namespace WebStat {
 		template<typename... T> NewEntities newEntities(const std::tuple<T...> &) const;
 		void handleCurlOperations();
 
+		void jobIngestParkedLine(const std::filesystem::directory_iterator &);
+		void jobIngestParkedLine(const std::filesystem::path &, uintmax_t size);
+
 		using CurlOperations = std::map<CURL *, std::unique_ptr<CurlOperation>>;
 		mutable std::flat_set<Crc32Value> existingEntities;
 		uint32_t hostnameId;