blob: 714151592ffd1a356c5355b97aa0bcf2049fc05c (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
|
#pragma once
#include "logTypes.hpp"
#include <c++11Helpers.h>
#include <connection_fwd.h>
#include <cstdio>
#include <flat_set>
#include <scn/scan.h>
namespace WebStat {
class Ingestor {
public:
Ingestor(std::string_view hostname, DB::ConnectionPtr dbconn);
virtual ~Ingestor() = default;
SPECIAL_MEMBERS_DEFAULT_MOVE_NO_COPY(Ingestor);
using ScanResult = decltype(scn::scan<std::string_view, std::string_view, uint64_t, std::string_view,
QuotedString, QueryString, std::string_view, unsigned short, unsigned int, unsigned int, CLFString,
CLFString>(std::declval<std::string_view>(), ""));
using ScanValues = std::remove_cvref_t<decltype(std::declval<WebStat::Ingestor::ScanResult>()->values())>;
[[nodiscard]] static ScanResult scanLogLine(std::string_view);
void ingestLog(std::FILE *);
void ingestLogLine(std::string_view);
template<typename... T> void storeLogLine(const std::tuple<T...> &) const;
protected:
size_t linesRead = 0;
size_t linesParsed = 0;
size_t linesDiscarded = 0;
private:
template<typename... T> size_t storeEntities(const std::tuple<T...> &) const;
mutable std::flat_set<Crc32Value> existingEntities;
uint32_t hostnameId;
DB::ConnectionPtr dbconn;
};
}
|
