Import and deal with the existance of changelogs

4 files changed, 57 insertions, 28 deletions

diff --git a/gentoobrowse/console/packagefileimport.xml b/gentoobrowse/console/packagefileimport.xml
index 45a41a8..422655f 100644
--- a/gentoobrowse/console/packagefileimport.xml
+++ b/gentoobrowse/console/packagefileimport.xml
@@ -4,16 +4,25 @@
 	<xi:include href="../datasources/postgres.xml" />
 
 	<project2:sqlmerge name="files" datasource="postgres" targettable="files">
-		<project2:prociterate name="findCacheFiles" newline="&#10;" encoding="ascii" fieldSep="|" quoteChar=""
-				path="find /usr/portage/metadata/cache/ -type f -printf &quot;%p|%t\n&quot; | sed &apos;s#/usr/portage/metadata/cache/##&apos;" >
-			<columns>
-				<column>filename</column>
-				<column>moddate</column>
-			</columns>
-		</project2:prociterate>
+		<project2:sqliterate name="filetypes" datasource="postgres">
+			<sql>
+				SELECT filetypeid, searchroot, LENGTH(searchroot) + 1 || '-' AS rootlen, searchspec
+				FROM filetypes
+				ORDER BY filetypeid DESC
+			</sql>
+			<project2:prociterate name="findCacheFiles" newline="&#10;" encoding="ascii" fieldSep="|" quoteChar=""
+					path="*find ^searchroot ^searchspec -printf &quot;%p|%t\n&quot; | cut -b ^rootlen | sed &apos;s#^# ^filetypeid |#&apos;" >
+				<columns>
+					<column>filetypeid</column>
+					<column>filename</column>
+					<column>moddate</column>
+				</columns>
+			</project2:prociterate>
+		</project2:sqliterate>
 		<columns>
 			<column key="true">filename</column>
 			<column>moddate</column>
+			<column>filetypeid</column>
 		</columns>
 	</project2:sqlmerge>
 
@@ -21,12 +30,13 @@
 			updatewhere="*a.fileid IN (SELECT o.fileid FROM files o WHERE o.cachedat IS NULL OR o.cachedat != o.moddate)">
 		<project2:sqliterate name="outDatadFiles" datasource="postgres">
 			<sql>
-				SELECT fileid, '/usr/portage/metadata/cache/' || filename AS filename
-				FROM files
-				WHERE cachedat IS NULL OR cachedat != moddate
-				ORDER BY fileid
+				SELECT f.fileid, ft.searchroot || f.filename AS filename, f.cachedat, f.moddate
+				FROM files f, filetypes ft
+				WHERE (f.cachedat IS NULL OR f.cachedat != f.moddate)
+				AND f.filetypeid = ft.filetypeid
+				ORDER BY f.fileid;
 			</sql>
-			<project2:prociterate name="readFileContent" newline="&#10;" encoding="ascii" fieldSep="|" quoteChar=""
+			<project2:prociterate name="readFileContent" newline="&#10;" encoding="utf-8" fieldSep="|" quoteChar=""
 					path="*grep -n . ^filename | sed -e &apos;s#:#|#&apos; -e &apos;s#^# ^fileid |#&apos;" >
 				<columns>
 					<column>fileid</column>
diff --git a/gentoobrowse/console/packageimport.xml b/gentoobrowse/console/packageimport.xml
index 6f8d39d..82ab447 100644
--- a/gentoobrowse/console/packageimport.xml
+++ b/gentoobrowse/console/packageimport.xml
@@ -4,7 +4,7 @@
 	<xi:include href="../datasources/postgres.xml" />
 
 	<project2:sqlmerge name="categories" datasource="postgres" targettable="categories">
-		<sql>SELECT DISTINCT (REGEXP_MATCHES(filename, '^[^/]+'))[1] AS name FROM files</sql>
+		<sql>SELECT DISTINCT (REGEXP_MATCHES(filename, '^[^/]+'))[1] AS name FROM files WHERE filetypeid = 1</sql>
 		<columns>
 			<column key="true">name</column>
 		</columns>
@@ -36,7 +36,8 @@
 				(REGEXP_MATCHES(filename, '^[^/]+'))[1] AS cat,
 				(REGEXP_MATCHES(filename, '/(.+)-[0-9]'))[1] AS name,
 				(REGEXP_MATCHES(filename, '/.+-([0-9].*)$'))[1] AS version
-				FROM files f) f, categories c, packages p
+				FROM files f
+				WHERE filetypeid = 1) f, categories c, packages p
 			WHERE f.cat = c.name
 			AND c.categoryid = p.categoryid
 			AND p.name = f.name
@@ -51,9 +52,11 @@
 
 	<project2:sqlmerge name="archs" datasource="postgres" targettable="ebuild_archs">
 		<sql>
-			SELECT DISTINCT f.fileid AS ebuildid, REGEXP_SPLIT_TO_TABLE(f.content, ' ') AS arch
-			FROM filecache f
-			WHERE f.line = 9
+			SELECT DISTINCT f.fileid AS ebuildid, REGEXP_SPLIT_TO_TABLE(fc.content, ' ') AS arch
+			FROM filecache fc, files f
+			WHERE fc.line = 9
+			AND fc.fileid = f.fileid
+			AND filetypeid = 1
 		</sql>
 		<columns>
 			<column key="true">ebuildid</column>
@@ -63,9 +66,11 @@
 
 	<project2:sqlmerge name="uses" datasource="postgres" targettable="ebuild_uses">
 		<sql>
-			SELECT DISTINCT f.fileid AS ebuildid, REGEXP_SPLIT_TO_TABLE(f.content, ' ') AS use
-			FROM filecache f
-			WHERE f.line = 11
+			SELECT DISTINCT fc.fileid AS ebuildid, REGEXP_SPLIT_TO_TABLE(fc.content, ' ') AS use
+			FROM filecache fc, files f
+			WHERE fc.line = 11
+			AND fc.fileid = f.fileid
+			AND filetypeid = 1
 		</sql>
 		<columns>
 			<column key="true">ebuildid</column>
@@ -75,10 +80,12 @@
 
 	<project2:sqlmerge name="urls" datasource="postgres" targettable="package_urls">
 		<sql>
-			SELECT DISTINCT e.packageid, REGEXP_SPLIT_TO_TABLE(f.content, ' ') AS url
-			FROM filecache f, ebuilds e
-			WHERE f.line = 6
-			AND f.fileid = e.ebuildid
+			SELECT DISTINCT e.packageid, REGEXP_SPLIT_TO_TABLE(fc.content, ' ') AS url
+			FROM filecache fc, ebuilds e, files f
+			WHERE fc.line = 6
+			AND fc.fileid = e.ebuildid
+			AND fc.fileid = f.fileid
+			AND filetypeid = 1
 		</sql>
 		<columns>
 			<column key="true">packageid</column>
@@ -97,9 +104,11 @@
 					SELECT fileid, REGEXP_MATCHES(dep,
 						'([^a-z0-9]*)([a-z0-9-]+)/([a-z0-9-]+?)(-([0-9][^:]*))?(:([0-9.]+))?(\\[([^]]+)\\])?') as field
 					FROM (
-						SELECT f.fileid, REGEXP_SPLIT_TO_TABLE(f.content, ' ') AS dep
-						FROM filecache f
+						SELECT fc.fileid, REGEXP_SPLIT_TO_TABLE(fc.content, ' ') AS dep
+						FROM filecache fc, files f
 						WHERE line = 1
+						AND fc.fileid = f.fileid
+						AND filetypeid = 1
 					) src
 				) AS src2
 			) AS src3, categories c, packages p
diff --git a/gentoobrowse/views/newebuilds.xml b/gentoobrowse/views/newebuilds.xml
index a95fa3c..8f1b544 100644
--- a/gentoobrowse/views/newebuilds.xml
+++ b/gentoobrowse/views/newebuilds.xml
@@ -11,7 +11,12 @@
 			WHERE e.ebuildid = f.fileid
 			AND e.packageid = p.packageid
 			AND p.categoryid = c.categoryid
-			AND f.firstseen IN (SELECT DISTINCT f.firstseen FROM files f ORDER BY f.firstseen DESC LIMIT 3)
+			AND f.firstseen IN (
+					SELECT DISTINCT f.firstseen
+					FROM files f
+					WHERE f.filetypeid = 1
+					ORDER BY f.firstseen
+					DESC LIMIT 3)
 			ORDER BY tracked DESC, c.name, p.name, e.versioninst
 		</sql>
 		<parameters>
@@ -20,7 +25,11 @@
 	</project2:sqlview>
 	<project2:sqlview name="syncs" recordname="sync" datasource="postgres">
 		<sql>
-			SELECT DISTINCT f.firstseen AS synctime FROM files f ORDER BY f.firstseen DESC LIMIT 3
+			SELECT DISTINCT f.firstseen AS synctime
+			FROM files f
+			WHERE filetypeid = 1
+			ORDER BY f.firstseen
+			DESC LIMIT 3
 		</sql>
 	</project2:sqlview>
 </block>
diff --git a/gentoobrowse/views/newebuildsatom.xml b/gentoobrowse/views/newebuildsatom.xml
index fba9e2b..93ae207 100644
--- a/gentoobrowse/views/newebuildsatom.xml
+++ b/gentoobrowse/views/newebuildsatom.xml
@@ -12,6 +12,7 @@
 			AND u.username = ?
 			AND up.packageid = p.packageid
 			AND f.firstseen >= up.trackedsince
+			AND f.filetypeid = 1
 			ORDER BY f.firstseen DESC, c.name, p.name, e.versioninst
 			LIMIT 50
 		</sql>