Import the masks

Fix filecache updating
Make things seperate files

3 files changed, 103 insertions, 56 deletions

diff --git a/gentoobrowse/console/fileimport.xml b/gentoobrowse/console/fileimport.xml
new file mode 100644
index 0000000..5d326e9
--- /dev/null
+++ b/gentoobrowse/console/fileimport.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0"?>
+<packageimport name="bugassociate" xmlns:xi="http://www.w3.org/2001/XInclude"
+		xmlns:project2="http://project2.randomdan.homeip.net">
+	<xi:include href="../datasources/postgres.xml" />
+	<project2:sqlmerge name="files" datasource="postgres" targettable="files">
+		<project2:sqliterate name="filetypes" datasource="postgres">
+			<sql>
+				SELECT filetypeid, searchroot, LENGTH(searchroot) + 1 || '-' AS rootlen, searchspec
+				FROM filetypes
+				ORDER BY filetypeid DESC
+			</sql>
+			<project2:prociterate name="findCacheFiles" newline="&#10;" encoding="ascii" fieldSep="|" quoteChar=""
+					path="*find ^searchroot ^searchspec -printf &quot;%p|%t\n&quot; | cut -b ^rootlen | sed &apos;s#^# ^filetypeid |#&apos;" >
+				<columns>
+					<column>filetypeid</column>
+					<column>filename</column>
+					<column>moddate</column>
+				</columns>
+			</project2:prociterate>
+		</project2:sqliterate>
+		<columns>
+			<column key="true">filename</column>
+			<column>moddate</column>
+			<column>filetypeid</column>
+		</columns>
+	</project2:sqlmerge>
+
+	<project2:sqlmerge name="loadCacheContent" datasource="postgres" targettable="filecache"
+			updatewhere="*a.fileid IN (SELECT o.fileid FROM files o WHERE o.cachedat IS NULL OR o.cachedat != o.moddate)">
+		<project2:sqliterate name="outDatadFiles" datasource="postgres">
+			<sql>
+				SELECT f.fileid, ft.searchroot || f.filename AS filename, f.cachedat, f.moddate
+				FROM files f, filetypes ft
+				WHERE (f.cachedat IS NULL OR f.cachedat != f.moddate)
+				AND f.filetypeid = ft.filetypeid
+				ORDER BY f.fileid;
+			</sql>
+			<project2:prociterate name="readFileContent" newline="&#10;" encoding="utf-8" fieldSep="|" quoteChar=""
+					path="*grep -n . ^filename | sed -e &apos;s#:#|#&apos; -e &apos;s#^# ^fileid |#&apos;" >
+				<columns>
+					<column>fileid</column>
+					<column>line</column>
+					<column>content</column>
+				</columns>
+			</project2:prociterate>
+		</project2:sqliterate>
+		<columns>
+			<column key="true">fileid</column>
+			<column key="true">line</column>
+			<column>content</column>
+		</columns>
+	</project2:sqlmerge>
+	<project2:sqltask name="setCachedTime" datasource="postgres">
+		<sql>UPDATE files SET cachedat = moddate WHERE cachedat != moddate OR cachedat IS NULL</sql>
+	</project2:sqltask>
+</packageimport>
diff --git a/gentoobrowse/console/packagefileimport.xml b/gentoobrowse/console/packagefileimport.xml
index 422655f..99cd60a 100644
--- a/gentoobrowse/console/packagefileimport.xml
+++ b/gentoobrowse/console/packagefileimport.xml
@@ -2,61 +2,7 @@
 <packageimport name="bugassociate" xmlns:xi="http://www.w3.org/2001/XInclude"
 		xmlns:project2="http://project2.randomdan.homeip.net">
 	<xi:include href="../datasources/postgres.xml" />
-
-	<project2:sqlmerge name="files" datasource="postgres" targettable="files">
-		<project2:sqliterate name="filetypes" datasource="postgres">
-			<sql>
-				SELECT filetypeid, searchroot, LENGTH(searchroot) + 1 || '-' AS rootlen, searchspec
-				FROM filetypes
-				ORDER BY filetypeid DESC
-			</sql>
-			<project2:prociterate name="findCacheFiles" newline="&#10;" encoding="ascii" fieldSep="|" quoteChar=""
-					path="*find ^searchroot ^searchspec -printf &quot;%p|%t\n&quot; | cut -b ^rootlen | sed &apos;s#^# ^filetypeid |#&apos;" >
-				<columns>
-					<column>filetypeid</column>
-					<column>filename</column>
-					<column>moddate</column>
-				</columns>
-			</project2:prociterate>
-		</project2:sqliterate>
-		<columns>
-			<column key="true">filename</column>
-			<column>moddate</column>
-			<column>filetypeid</column>
-		</columns>
-	</project2:sqlmerge>
-
-	<project2:sqlmerge name="loadCacheContent" datasource="postgres" targettable="filecache"
-			updatewhere="*a.fileid IN (SELECT o.fileid FROM files o WHERE o.cachedat IS NULL OR o.cachedat != o.moddate)">
-		<project2:sqliterate name="outDatadFiles" datasource="postgres">
-			<sql>
-				SELECT f.fileid, ft.searchroot || f.filename AS filename, f.cachedat, f.moddate
-				FROM files f, filetypes ft
-				WHERE (f.cachedat IS NULL OR f.cachedat != f.moddate)
-				AND f.filetypeid = ft.filetypeid
-				ORDER BY f.fileid;
-			</sql>
-			<project2:prociterate name="readFileContent" newline="&#10;" encoding="utf-8" fieldSep="|" quoteChar=""
-					path="*grep -n . ^filename | sed -e &apos;s#:#|#&apos; -e &apos;s#^# ^fileid |#&apos;" >
-				<columns>
-					<column>fileid</column>
-					<column>line</column>
-					<column>content</column>
-				</columns>
-			</project2:prociterate>
-			<project2:sqltask name="setCachedTime" datasource="postgres">
-				<sql>UPDATE files SET cachedat = moddate WHERE fileid = ?</sql>
-				<parameters>
-					<param bind="0" value="^fileid" />
-				</parameters>
-			</project2:sqltask>
-		</project2:sqliterate>
-		<columns>
-			<column key="true">fileid</column>
-			<column key="true">line</column>
-			<column>content</column>
-		</columns>
-	</project2:sqlmerge>
-
+	<xi:include href="fileimport.xml" />
 	<xi:include href="packageimport.xml" />
+	<xi:include href="packagemasks.xml" />
 </packageimport>
diff --git a/gentoobrowse/console/packagemasks.xml b/gentoobrowse/console/packagemasks.xml
new file mode 100644
index 0000000..20fd8fd
--- /dev/null
+++ b/gentoobrowse/console/packagemasks.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0"?>
+<packagemasks name="packagemasks" xmlns:xi="http://www.w3.org/2001/XInclude"
+		xmlns:project2="http://project2.randomdan.homeip.net">
+	<xi:include href="../datasources/postgres.xml" />
+	<project2:sqlmerge name="masksets" datasource="postgres" targettable="masksets" useview="yes">
+		<sql>
+			SELECT d.setno,
+				MIN(field[1]) AS person,
+				MIN(field[2]) AS email,
+				CAST((REGEXP_MATCHES(MIN(field[3]), '(\\d+ *(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec) \\d+)', 'i'))[1] AS DATE) AS dateadded,
+				TRIM(STRING_AGG(REGEXP_REPLACE(content, '^#+', ''), ' ' ORDER BY m.line)) AS message
+			FROM (SELECT setno, line AS line, REGEXP_MATCHES(content, '^# *(.+) +&lt; *(.+) *> \\( *(.+) *\\)$') AS field
+				FROM file_masks_withset
+				WHERE line IN(SELECT min(line)
+					FROM file_masks_withset
+					GROUP BY setno)) AS d,
+			(SELECT c.line, c.content, c.setno
+			 FROM file_masks_withset c
+			 WHERE c.content LIKE '#%') AS m
+			WHERE m.setno = d.setno
+			AND m.line > d.line
+			GROUP BY d.setno
+		</sql>
+		<columns>
+			<column key="true">setno</column>
+			<column>person</column>
+			<column>email</column>
+			<column>dateadded</column>
+			<column>message</column>
+		</columns>
+	</project2:sqlmerge>
+	<project2:sqlmerge name="masks" datasource="postgres" targettable="ebuild_masks" useview="yes">
+		<sql>
+			SELECT line, setno, ebuildfilter(content) AS ebuildid
+			FROM file_masks_withset
+			WHERE content NOT LIKE '#%'
+		</sql>
+		<columns>
+			<column key="true">line</column>
+			<column key="true">setno</column>
+			<column key="true">ebuildid</column>
+		</columns>
+	</project2:sqlmerge>
+</packagemasks>
+