Process new field, content-type, in input streamHEADmain

5 files changed, 21 insertions, 11 deletions

diff --git a/src/ingestor.cpp b/src/ingestor.cpp
index 81642be..da39c59 100644
--- a/src/ingestor.cpp
+++ b/src/ingestor.cpp
@@ -54,7 +54,8 @@ namespace WebStat {
 		{
 			static constexpr std::tuple<ToEntity<EntityType::VirtualHost>, std::identity, std::identity, std::identity,
 					ToEntity<EntityType::Path>, ToEntity<EntityType::QueryString>, std::identity, std::identity,
-					std::identity, std::identity, ToEntity<EntityType::Referrer>, ToEntity<EntityType::UserAgent>>
+					std::identity, std::identity, ToEntity<EntityType::Referrer>, ToEntity<EntityType::UserAgent>,
+					ToEntity<EntityType::ContentType>>
 					ENTITY_TYPE_MAP;
 			static constexpr size_t VALUE_COUNT = std::tuple_size_v<Ingestor::ScanValues>;
 			static_assert(VALUE_COUNT == std::tuple_size_v<decltype(ENTITY_TYPE_MAP)>);
@@ -99,8 +100,9 @@ namespace WebStat {
 				unsigned int, // size : %B : 1234
 				unsigned int, // duration : %D : 1234
 				CLFString, // referrer : "%{Referer}i" : "https://google.com/whatever" or "-"
-				CLFString // user_agent : "%{User-agent}i" : "Chromium v123.4" or "-"
-				>(input, R"({} {} {} {:[A-Z]} {} {} {} {} {} {} {} {})");
+				CLFString, // user_agent : "%{User-agent}i" : "Chromium v123.4" or "-"
+				CLFString // content_type : "%{Content-type}o" : "test/plain" or "-"
+				>(input, R"({} {} {} {:[A-Z]} {} {} {} {} {} {} {} {} {})");
 	}
 
 	void
@@ -321,7 +323,7 @@ namespace WebStat {
 	Ingestor::NewEntityIds
 	Ingestor::storeEntities(DB::Connection * dbconn, const std::span<const std::optional<Entity>> values) const
 	{
-		static constexpr std::array<std::pair<std::string_view, void (Ingestor::*)(const Entity &) const>, 8>
+		static constexpr std::array<std::pair<std::string_view, void (Ingestor::*)(const Entity &) const>, 9>
 				ENTITY_TYPE_VALUES {{
 						{"host", nullptr},
 						{"virtual_host", nullptr},
@@ -331,6 +333,7 @@ namespace WebStat {
 						{"user_agent", &Ingestor::onNewUserAgent},
 						{"unparsable_line", nullptr},
 						{"uninsertable_line", nullptr},
+						{"content_type", nullptr},
 				}};
 
 		auto insert = dbconn->modify(SQL::ENTITY_INSERT, SQL::ENTITY_INSERT_OPTS);
diff --git a/src/ingestor.hpp b/src/ingestor.hpp
index a19c8ec..67a7a15 100644
--- a/src/ingestor.hpp
+++ b/src/ingestor.hpp
@@ -43,7 +43,7 @@ namespace WebStat {
 
 		using ScanResult = decltype(scn::scan<std::string_view, std::string_view, uint64_t, std::string_view,
 				QuotedString, QueryString, std::string_view, unsigned short, unsigned int, unsigned int, CLFString,
-				CLFString>(std::declval<std::string_view>(), ""));
+				CLFString, CLFString>(std::declval<std::string_view>(), ""));
 		using ScanValues = std::remove_cvref_t<decltype(std::declval<WebStat::Ingestor::ScanResult>()->values())>;
 
 		[[nodiscard]] static ScanResult scanLogLine(std::string_view);
diff --git a/src/logTypes.hpp b/src/logTypes.hpp
index f9395d1..71393b2 100644
--- a/src/logTypes.hpp
+++ b/src/logTypes.hpp
@@ -31,6 +31,7 @@ namespace WebStat {
 		UserAgent,
 		UnparsableLine,
 		UninsertableLine,
+		ContentType,
 	};
 
 	using Crc32Value = uint32_t;
diff --git a/src/schema.sql b/src/schema.sql
index 7648b79..8008b3c 100644
--- a/src/schema.sql
+++ b/src/schema.sql
@@ -28,7 +28,8 @@ CREATE TYPE entity AS ENUM(
 	'referrer',
 	'user_agent',
 	'unparsable_line',
-	'uninsertable_line'
+	'uninsertable_line',
+	'content_type'
 );
 
 CREATE TABLE entities(
@@ -55,13 +56,15 @@ CREATE TABLE access_log(
 	duration interval second(6) NOT NULL,
 	referrer oid,
 	user_agent oid,
+	content_type oid,
 	CONSTRAINT pk_access_log PRIMARY KEY (id),
 	CONSTRAINT fk_access_log_hostname FOREIGN KEY (hostname) REFERENCES entities(id),
 	CONSTRAINT fk_access_log_virtualhost FOREIGN KEY (virtual_host) REFERENCES entities(id),
 	CONSTRAINT fk_access_log_path FOREIGN KEY (path) REFERENCES entities(id),
 	CONSTRAINT fk_access_log_query_string FOREIGN KEY (query_string) REFERENCES entities(id),
 	CONSTRAINT fk_access_log_referrer FOREIGN KEY (referrer) REFERENCES entities(id),
-	CONSTRAINT fk_access_log_user_agent FOREIGN KEY (user_agent) REFERENCES entities(id)
+	CONSTRAINT fk_access_log_user_agent FOREIGN KEY (user_agent) REFERENCES entities(id),
+	CONSTRAINT fk_access_log_content_type FOREIGN KEY (content_type) REFERENCES entities(id)
 );
 
 CREATE OR REPLACE VIEW access_log_view AS
@@ -85,7 +88,9 @@ SELECT
 	r.id referrer_id,
 	r.value referrer,
 	u.id user_agent_id,
-	u.value user_agent
+	u.value user_agent,
+	c.id content_type_id,
+	c.value content_type
 FROM
 	access_log l
 	LEFT OUTER JOIN entities h ON l.hostname = h.id
@@ -93,4 +98,5 @@ FROM
 	LEFT OUTER JOIN entities p ON l.path = p.id
 	LEFT OUTER JOIN entities q ON l.query_string = q.id
 	LEFT OUTER JOIN entities r ON l.referrer = r.id
-	LEFT OUTER JOIN entities u ON l.user_agent = u.id;
+	LEFT OUTER JOIN entities u ON l.user_agent = u.id
+	LEFT OUTER JOIN entities c ON l.user_agent = c.id;
diff --git a/src/sql/accessLogInsert.sql b/src/sql/accessLogInsert.sql
index 42f809b..518045e 100644
--- a/src/sql/accessLogInsert.sql
+++ b/src/sql/accessLogInsert.sql
@@ -1,3 +1,3 @@
 INSERT INTO access_log(hostname, virtual_host, remoteip, request_time, method, path, query_string, protocol, status,
- size, duration, referrer, user_agent)
-	VALUES (?, ?, ?, TO_TIMESTAMP(? / 1000000.0) at time zone 'utc', ?, ?, ?, ?, ?, ?, ? * '1us'::interval, ?, ?)
+ size, duration, referrer, user_agent, content_type)
+	VALUES (?, ?, ?, TO_TIMESTAMP(? / 1000000.0) at time zone 'utc', ?, ?, ?, ?, ?, ?, ? * '1us'::interval, ?, ?, ?)