diff options
Diffstat (limited to 'test/test-ingest.cpp')
| -rw-r--r-- | test/test-ingest.cpp | 49 |
1 files changed, 37 insertions, 12 deletions
diff --git a/test/test-ingest.cpp b/test/test-ingest.cpp index 9ce48be..8c82764 100644 --- a/test/test-ingest.cpp +++ b/test/test-ingest.cpp @@ -58,6 +58,8 @@ namespace DB { } } +BOOST_TEST_DONT_PRINT_LOG_VALUE(Ingestor::Stats); + BOOST_DATA_TEST_CASE(QuotedStringsGood, boost::unit_test::data::make<WebStat::ParseData<WebStat::QuotedString>>({ {R"("")", ""}, @@ -229,7 +231,10 @@ public: va_end(args); BOOST_REQUIRE(msg); BOOST_TEST_MESSAGE(msg.get()); + ++logsWritten; } + + mutable size_t logsWritten = 0; }; BOOST_FIXTURE_TEST_SUITE(I, TestIngestor); @@ -244,9 +249,11 @@ BOOST_DATA_TEST_CASE(StoreLogLine, line) { ingestLogLines(DB::MockDatabase::openConnectionTo("webstat").get(), {std::string {line}}); - BOOST_CHECK_EQUAL(linesRead, 0); - BOOST_CHECK_EQUAL(linesParsed, 1); - BOOST_CHECK_EQUAL(linesDiscarded, 0); + BOOST_CHECK_EQUAL(stats.linesRead, 0); + BOOST_CHECK_EQUAL(stats.linesParsed, 1); + BOOST_CHECK_EQUAL(stats.linesParseFailed, 0); + BOOST_CHECK_EQUAL(stats.logsInserted, 1); + BOOST_CHECK_EQUAL(stats.entitiesInserted, 5); BOOST_CHECK_EQUAL(existingEntities.size(), 5); } @@ -256,9 +263,11 @@ BOOST_AUTO_TEST_CASE(StoreLog, *boost::unit_test::depends_on("I/StoreLogLine")) WebStat::FilePtr input {fopen(log.path.c_str(), "r")}; BOOST_REQUIRE(input); ingestLog(input.get()); - BOOST_CHECK_EQUAL(linesRead, 10); - BOOST_CHECK_EQUAL(linesParsed, 10); - BOOST_CHECK_EQUAL(linesDiscarded, 0); + BOOST_CHECK_EQUAL(stats.linesRead, 10); + BOOST_CHECK_EQUAL(stats.linesParsed, 10); + BOOST_CHECK_EQUAL(stats.linesParseFailed, 0); + BOOST_CHECK_GE(stats.entitiesInserted, 1); + BOOST_CHECK_EQUAL(stats.entitiesInserted, existingEntities.size()); } BOOST_AUTO_TEST_CASE(TerminateHandler, *boost::unit_test::timeout(5)) @@ -269,9 +278,9 @@ BOOST_AUTO_TEST_CASE(TerminateHandler, *boost::unit_test::timeout(5)) raise(SIGTERM); BOOST_REQUIRE(terminated); ingestLog(input.get()); - BOOST_CHECK_EQUAL(linesRead, 0); - BOOST_CHECK_EQUAL(linesParsed, 0); - BOOST_CHECK_EQUAL(linesDiscarded, 0); + BOOST_CHECK_EQUAL(stats.linesRead, 0); + BOOST_CHECK_EQUAL(stats.linesParsed, 0); + BOOST_CHECK_EQUAL(stats.linesParseFailed, 0); } BOOST_AUTO_TEST_CASE(ParkLogLine) @@ -289,10 +298,9 @@ BOOST_AUTO_TEST_CASE(ParkLogLineOnError, *boost::unit_test::depends_on("I/ParkLo { BOOST_REQUIRE(existingEntities.empty()); constexpr std::string_view LOGLINE_BAD_VERB - = R"LOG(git.randomdan.homeip.net 98.82.40.168 1755561576768318 CAUSEPARK "/repo/gentoobrowse-api/commit/gentoobrowse-api/unittests/fixtures/756569aa764177340726dd3d40b41d89b11b20c7/app-crypt/pdfcrack/Manifest" "?h=gentoobrowse-api-0.9.1&id=a2ed3fd30333721accd4b697bfcb6cc4165c7714" HTTP/1.1 200 1884 107791 "-" "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot) Chrome/119.0.6045.214 Safari/537.36")LOG"; + = R"LOG(git.randomdan.homeip.net 98.82.40.168 1755561576768318 CAUSEPARSEFAIL "/repo/gentoobrowse-api/commit/gentoobrowse-api/unittests/fixtures/756569aa764177340726dd3d40b41d89b11b20c7/app-crypt/pdfcrack/Manifest" "?h=gentoobrowse-api-0.9.1&id=a2ed3fd30333721accd4b697bfcb6cc4165c7714" HTTP/1.1 200 1884 107791 "-" "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot) Chrome/119.0.6045.214 Safari/537.36")LOG"; BOOST_REQUIRE_NO_THROW(ingestLogLines(dbpool->get().get(), {std::string {LOGLINE_BAD_VERB}})); - BOOST_CHECK_EQUAL(linesParked, 0); - BOOST_CHECK_EQUAL(linesDiscarded, 1); + BOOST_CHECK_EQUAL(stats.linesParseFailed, 1); } BOOST_AUTO_TEST_CASE(IngestParked, *boost::unit_test::depends_on("I/ParkLogLine")) @@ -390,6 +398,9 @@ BOOST_AUTO_TEST_CASE(DiscardUnparsable) }}; auto rows = select->as<Crc32Value, std::string_view>(); BOOST_CHECK_EQUAL_COLLECTIONS(rows.begin(), rows.end(), EXPECTED.begin(), EXPECTED.end()); + BOOST_CHECK_EQUAL(stats.linesParseFailed, 1); + BOOST_CHECK_EQUAL(stats.entitiesInserted, 1); + BOOST_CHECK(existingEntities.empty()); // Don't clutter existing entities with junk logs } BOOST_AUTO_TEST_CASE(PurgeOldJob) @@ -397,6 +408,20 @@ BOOST_AUTO_TEST_CASE(PurgeOldJob) BOOST_CHECK_EQUAL(2, jobPurgeOldLogs()); } +BOOST_AUTO_TEST_CASE(LogStatsSignal) +{ + BOOST_REQUIRE_EQUAL(logsWritten, 0); + raise(SIGUSR1); + BOOST_CHECK_EQUAL(logsWritten, 1); +} + +BOOST_AUTO_TEST_CASE(LogResetSignal) +{ + stats = {1, 2, 3, 4, 5}; + raise(SIGUSR2); + BOOST_CHECK_EQUAL(stats, Stats {}); +} + BOOST_AUTO_TEST_SUITE_END(); BOOST_AUTO_TEST_CASE(FetchRealUserAgentDetail, *boost::unit_test::disabled()) |