Track existing entities, don't attempt reinsert

This is an ever growing set, but in theory should be finite, and is only
4bytes per entity.

2 files changed, 6 insertions, 0 deletions

diff --git a/src/ingestor.cpp b/src/ingestor.cpp
index e316e84..8547745 100644
--- a/src/ingestor.cpp
+++ b/src/ingestor.cpp
@@ -121,10 +121,14 @@ namespace WebStat {
 	size_t
 	Ingestor::storeEntity(const Entity entity) const
 	{
+		if (existingEntities.contains(entity.first)) {
+			return 0;
+		}
 		auto insert = dbconn->modify(SQL::ENTITY_INSERT, SQL::ENTITY_INSERT_OPTS);
 		insert->bindParamI(0, entity.first);
 		insert->bindParamS(1, entity.second);
 		insert->execute();
+		existingEntities.emplace(entity.first);
 		return 1;
 	}
 
diff --git a/src/ingestor.hpp b/src/ingestor.hpp
index 53ba81f..458b941 100644
--- a/src/ingestor.hpp
+++ b/src/ingestor.hpp
@@ -4,6 +4,7 @@
 #include <c++11Helpers.h>
 #include <connection_fwd.h>
 #include <cstdio>
+#include <flat_set>
 #include <scn/scan.h>
 
 namespace WebStat {
@@ -39,6 +40,7 @@ namespace WebStat {
 	private:
 		template<typename... T> size_t storeEntities(const std::tuple<T...> &) const;
 
+		mutable std::flat_set<Crc32Value> existingEntities;
 		uint32_t hostnameId;
 		DB::ConnectionPtr dbconn;
 	};