diff options
-rw-r--r-- | src/ingestor.cpp | 28 | ||||
-rw-r--r-- | src/ingestor.hpp | 1 | ||||
-rw-r--r-- | src/sql.cpp | 4 | ||||
-rw-r--r-- | src/sql.hpp | 1 | ||||
-rw-r--r-- | src/sql/accessLogInsert.sql | 4 | ||||
-rw-r--r-- | test/test-ingest.cpp | 12 |
6 files changed, 50 insertions, 0 deletions
diff --git a/src/ingestor.cpp b/src/ingestor.cpp index a3c49e7..c1206a2 100644 --- a/src/ingestor.cpp +++ b/src/ingestor.cpp @@ -1,12 +1,22 @@ #include "ingestor.hpp" #include "sql.hpp" #include <connection.h> +#include <dbTypes.h> #include <modifycommand.h> #include <scn/scan.h> #include <syslog.h> #include <utility> #include <zlib.h> +namespace DB { + template<> + void + DB::Command::bindParam(unsigned int idx, const WebStat::Entity & entity) + { + bindParamI(idx, entity.first); + } +} + namespace WebStat { namespace { Crc32Value @@ -82,6 +92,7 @@ namespace WebStat { linesParsed++; const auto values = crc32ScanValues(result->values()); storeEntities(values); + storeLogLine(values); } else { syslog(LOG_WARNING, "Discarded line: [%.*s]", static_cast<int>(line.length()), line.data()); @@ -122,4 +133,21 @@ namespace WebStat { storeEntity(*entity); } } + + template<typename... T> + void + Ingestor::storeLogLine(const std::tuple<T...> & values) const + { + auto insert = dbconn->modify(SQL::ACCESS_LOG_INSERT, SQL::ACCESS_LOG_INSERT_OPTS); + + insert->bindParam(0, hostnameId); + std::apply( + [&insert](auto &&... value) { + unsigned int param = 1; + (insert->bindParam(param++, value), ...); + }, + values); + + insert->execute(); + } } diff --git a/src/ingestor.hpp b/src/ingestor.hpp index 899f179..07a9a7d 100644 --- a/src/ingestor.hpp +++ b/src/ingestor.hpp @@ -27,6 +27,7 @@ namespace WebStat { template<typename T> void storeEntity(const T &) const; void storeEntity(Entity) const; void storeEntity(std::optional<Entity>) const; + template<typename... T> void storeLogLine(const std::tuple<T...> &) const; protected: size_t linesRead = 0; diff --git a/src/sql.cpp b/src/sql.cpp index 8512657..52e9520 100644 --- a/src/sql.cpp +++ b/src/sql.cpp @@ -6,11 +6,15 @@ namespace WebStat::SQL { // https://github.com/ccache/ccache/issues/1540 // ccache:disable + const std::string ACCESS_LOG_INSERT { +#embed "sql/accessLogInsert.sql" + }; const std::string ENTITY_INSERT { #embed "sql/entityInsert.sql" }; #define HASH_OPTS(VAR) \ const DB::CommandOptionsPtr VAR##_OPTS = std::make_shared<DB::CommandOptions>(std::hash<std::string> {}(VAR)) + HASH_OPTS(ACCESS_LOG_INSERT); HASH_OPTS(ENTITY_INSERT); #undef HASH_OPTS } diff --git a/src/sql.hpp b/src/sql.hpp index 4b598e7..1426384 100644 --- a/src/sql.hpp +++ b/src/sql.hpp @@ -8,6 +8,7 @@ namespace WebStat::SQL { extern const std::string Name; \ extern const DB::CommandOptionsPtr Name##_OPTS + EMBED_DECLARE(ACCESS_LOG_INSERT); EMBED_DECLARE(ENTITY_INSERT); #undef EMBED_DECLARE } diff --git a/src/sql/accessLogInsert.sql b/src/sql/accessLogInsert.sql new file mode 100644 index 0000000..e0c39b8 --- /dev/null +++ b/src/sql/accessLogInsert.sql @@ -0,0 +1,4 @@ +INSERT INTO +access_log(hostname, virtual_host, remoteip, request_time, method, path, query_string, protocol, status, size, duration, + referrer, user_agent) +VALUES(?, ?, ?, TO_TIMESTAMP(? / 1000000.0) at time zone 'utc', ?, ?, ?, ?, ?, ?, ? * '1us'::interval, ?, ?) diff --git a/test/test-ingest.cpp b/test/test-ingest.cpp index af063f3..965f502 100644 --- a/test/test-ingest.cpp +++ b/test/test-ingest.cpp @@ -186,3 +186,15 @@ BOOST_DATA_TEST_CASE(ExtractFields, BOOST_REQUIRE(result); BOOST_CHECK_EQUAL(result->values(), expected); } + +BOOST_TEST_DECORATOR(*boost::unit_test::depends_on("ExtractFields")) + +BOOST_DATA_TEST_CASE(StoreLogLine, + boost::unit_test::data::make({ + LOGLINE1, + LOGLINE2, + }), + line) +{ + WebStat::Ingestor {"test-hostname", DB::MockDatabase::openConnectionTo("webstat")}.ingestLogLine(line); +} |