Add missing execute on custom SQL merge inserter

Remove no longer needed second constructor on variables
Tidy up variable parent construction and use of attributes
Allow row set implementors to extend and add attributes of their own
Have attributes return a variant type
Add the filesystem rowset for finding files and directories

8 files changed, 452 insertions, 31 deletions

diff --git a/project2/Jamfile.jam b/project2/Jamfile.jam
index 011c5dd..640fa39 100644
--- a/project2/Jamfile.jam
+++ b/project2/Jamfile.jam
@@ -43,10 +43,12 @@ lib p2processes :
 	;
 
 lib p2files :
+	fsRows.cpp
 	fileRows.cpp
 	:
 	<library>../libmisc//misc
 	<library>libxmlpp
+	<library>boost_filesystem
 	;
 
 lib p2regex :
diff --git a/project2/fsRows.cpp b/project2/fsRows.cpp
new file mode 100644
index 0000000..f316e44
--- /dev/null
+++ b/project2/fsRows.cpp
@@ -0,0 +1,334 @@
+#include "fsRows.h"
+#include "xmlObjectLoader.h"
+#include "rowProcessor.h"
+#include "exceptions.h"
+#include <boost/filesystem/operations.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/algorithm/string/split.hpp>
+#include <boost/bind.hpp>
+#include <boost/lexical_cast.hpp>
+#include <syslog.h>
+#include <sys/types.h>
+#include <pwd.h>
+#include <grp.h>
+#include <stdio.h>
+
+typedef boost::filesystem::basic_directory_iterator<FsRows::Path> DirEnt;
+
+ElementLoaderImpl<FsRows> fsrowsLoader("fsrows");
+
+const Glib::ustring field_absPath("absPath");
+const Glib::ustring field_relPath("relPath");
+const Glib::ustring field_size("size");
+const Glib::ustring field_modDate("modifiedDate");
+const Glib::ustring field_createDate("createdDate");
+const Glib::ustring field_user("owningUser");
+const Glib::ustring field_group("owningGroup");
+const Glib::ustring field_mode("mode");
+const Glib::ustring field_perms("perms");
+const Glib::ustring field_type("type");
+
+bool FsRows::SpecBase::recurse(const FsRows *) const { return true; }
+bool FsRows::SpecBase::matches(const FsRows *) const { return true; }
+const boost::filesystem::path & FsRows::SpecBase::curPath(const FsRows * fs) const { return fs->curPath; }
+unsigned int FsRows::SpecBase::depth(const FsRows * fs) const { return fs->depth; }
+struct stat & FsRows::SpecBase::curStat(const FsRows * fs) const { return fs->curStat; }
+
+class FsRowSpecName : public FsRows::SpecBase {
+	public:
+		FsRowSpecName(const Glib::ustring & v) : pattern(v) { }
+		bool matches(const FsRows * fs) const {
+			// Based on code written by Jack Handy - jakkhandy@hotmail.com
+			// from http://www.codeproject.com/KB/string/wildcmp.aspx
+			Glib::ustring::const_iterator wild = pattern.begin();
+			Glib::ustring leaf(curPath(fs).leaf());
+			Glib::ustring::const_iterator string = leaf.begin();
+
+			while ((string != leaf.end()) && (*wild != '*')) {
+				if ((*wild != *string) && (*wild != '?')) {
+					return false;
+				}
+				wild++;
+				string++;
+			}
+
+			Glib::ustring::const_iterator cp, mp;
+			while (string != leaf.end()) {
+				if (*wild == '*') {
+					if (!*++wild) {
+						return true;
+					}
+					mp = wild;
+					cp = string;
+					cp++;
+				} else if ((*wild == *string) || (*wild == '?')) {
+					wild++;
+					string++;
+				} else {
+					wild = mp;
+					string = cp++;
+				}
+			}
+
+			while (*wild == '*') {
+				wild++;
+			}
+			return wild == pattern.end();
+		}
+		const Glib::ustring pattern;
+};
+class FsRowSpecType : public FsRows::SpecBase {
+	public:
+		FsRowSpecType(const Glib::ustring & v) : types(v) { }
+		bool matches(const FsRows * fs) const {
+			if (S_ISREG(curStat(fs).st_mode)) {
+				return types.find('f') != Glib::ustring::npos;
+			}
+			if (S_ISDIR(curStat(fs).st_mode)) {
+				return types.find('d') != Glib::ustring::npos;
+			}
+			if (S_ISCHR(curStat(fs).st_mode)) {
+				return types.find('c') != Glib::ustring::npos;
+			}
+			if (S_ISBLK(curStat(fs).st_mode)) {
+				return types.find('b') != Glib::ustring::npos;
+			}
+			if (S_ISFIFO(curStat(fs).st_mode)) {
+				return types.find('p') != Glib::ustring::npos;
+			}
+			if (S_ISLNK(curStat(fs).st_mode)) {
+				return types.find('l') != Glib::ustring::npos;
+			}
+			if (S_ISSOCK(curStat(fs).st_mode)) {
+				return types.find('s') != Glib::ustring::npos;
+			}
+			return false;
+		}
+		const Glib::ustring types;
+};
+class FsRowSpecMaxDepth : public FsRows::SpecBase {
+	public:
+		FsRowSpecMaxDepth(const Glib::ustring & v) : maxDepth(boost::lexical_cast<unsigned int>(v)) { }
+		bool recurse(const FsRows * fs) const {
+			return (depth(fs) < maxDepth);
+		}
+		const unsigned int maxDepth;
+};
+
+FsRows::FsRows(const xmlpp::Element * p) :
+	SourceObject(p),
+	RowSet(p)
+{
+}
+
+FsRows::~FsRows()
+{
+}
+
+void
+FsRows::loadComplete(const CommonObjects *)
+{
+}
+
+void
+FsRows::setFilter(const Glib::ustring &)
+{
+}
+
+void
+FsRows::execute(const RowProcessor * rp) const
+{
+	rowNum = 0;
+	depth = 0;
+	fsRoot = rp->getParameter("root");
+	// Ensure there is a trailing /
+	fsRoot = (fsRoot / ".").remove_filename();
+	specs.clear();
+	SpecSpec s;
+	boost::algorithm::split(s, rp->getParameter("spec"), Glib::Unicode::isspace, boost::algorithm::token_compress_on);
+	for (SpecSpec::const_iterator sf = s.begin(); sf != s.end(); ) {
+		const Glib::ustring & name = (*sf++);
+		if (name == "-name") {
+			specs.insert(new FsRowSpecName(*sf++));
+		}
+		else if (name == "-type") {
+			specs.insert(new FsRowSpecType(*sf++));
+		}
+		else if (name == "-maxdepth") {
+			specs.insert(new FsRowSpecMaxDepth(*sf++));
+		}
+		else {
+			throw NotSupported(name);
+		}
+	}
+	execute(fsRoot, rp);
+}
+
+void
+FsRows::execute(const Path & dir, const RowProcessor * rp) const
+{
+	depth += 1;
+	try {
+		for (DirEnt itr(dir); itr != DirEnt(); ++itr) {
+			curPathStr = itr->path().string();
+			curPath = itr->path();
+			stat(curPathStr.c_str(), &curStat);
+
+			if (boost::algorithm::all(specs, boost::bind(&SpecBase::matches, _1, this))) {
+				rp->rowReady();
+				rowNum += 1;
+			}
+
+			if (S_ISDIR(curStat.st_mode) && boost::algorithm::all(specs, boost::bind(&SpecBase::recurse, _1, this))) {
+				execute(*itr, rp);
+			}
+		}
+	}
+	catch (const boost::filesystem::basic_filesystem_error<Path> &) {
+	}
+	depth -= 1;
+}
+
+unsigned int
+FsRows::columnCount() const
+{
+	return 1;
+}
+
+bool
+FsRows::isNull(unsigned int) const
+{
+	return false;
+}
+
+bool
+FsRows::isNull(const Glib::ustring &) const
+{
+	return false;
+}
+
+const Glib::ustring &
+FsRows::getCurrentValue(const Glib::ustring &) const
+{
+	return curPathStr;
+}
+
+const Glib::ustring &
+FsRows::getCurrentValue(unsigned int) const
+{
+	return curPathStr;
+}
+
+const Glib::ustring &
+FsRows::getColumnName(unsigned int) const
+{
+	return field_absPath;
+}
+
+RowSet::RowAttribute
+FsRows::resolveAttr(const Glib::ustring & a) const
+{
+	if (a == field_relPath) {
+		return boost::bind(&FsRows::fileRelPath, this);
+	}
+	if (a == field_size) {
+		return boost::bind(&FsRows::fileSize, this);
+	}
+	if (a == field_modDate) {
+		return boost::bind(&FsRows::fileModDate, this);
+	}
+	if (a == field_createDate) {
+		return boost::bind(&FsRows::fileCreateDate, this);
+	}
+	if (a == field_user) {
+		return boost::bind(&FsRows::fileUser, this);
+	}
+	if (a == field_group) {
+		return boost::bind(&FsRows::fileGroup, this);
+	}
+	if (a == field_mode) {
+		return boost::bind(&FsRows::fileMode, this);
+	}
+	if (a == field_perms) {
+		return boost::bind(&FsRows::filePerms, this);
+	}
+	if (a == field_type) {
+		return boost::bind(&FsRows::fileType, this);
+	}
+	return RowSet::resolveAttr(a);
+}
+
+VariableType
+FsRows::fileRelPath() const
+{
+	return curPathStr.substr(fsRoot.native_directory_string().length());
+}
+
+VariableType
+FsRows::fileSize() const
+{
+	return curStat.st_size;
+}
+
+VariableType
+FsRows::fileModDate() const
+{
+	char buf[40];
+	struct tm tm;
+	gmtime_r(&curStat.st_mtime, &tm);
+	strftime(buf, sizeof(buf), "%F %T", &tm);
+	return Glib::ustring(buf);
+}
+
+VariableType
+FsRows::fileCreateDate() const
+{
+	char buf[40];
+	struct tm tm;
+	gmtime_r(&curStat.st_ctime, &tm);
+	strftime(buf, sizeof(buf), "%F %T", &tm);
+	return Glib::ustring(buf);
+}
+
+VariableType
+FsRows::fileUser() const
+{
+	struct passwd * p = getpwuid(curStat.st_uid);
+	if (p) {
+		return p->pw_name;
+	}
+	else {
+		return curStat.st_uid;
+	}
+}
+
+VariableType
+FsRows::fileGroup() const
+{
+	struct group * g = getgrgid(curStat.st_gid);
+	if (g) {
+		return g->gr_name;
+	}
+	else {
+		return curStat.st_gid;
+	}
+}
+
+VariableType
+FsRows::fileMode() const
+{
+	throw NotSupported(__PRETTY_FUNCTION__);
+}
+
+VariableType
+FsRows::filePerms() const
+{
+	throw NotSupported(__PRETTY_FUNCTION__);
+}
+
+VariableType
+FsRows::fileType() const
+{
+	throw NotSupported(__PRETTY_FUNCTION__);
+}
+
diff --git a/project2/fsRows.h b/project2/fsRows.h
new file mode 100644
index 0000000..08b17c5
--- /dev/null
+++ b/project2/fsRows.h
@@ -0,0 +1,68 @@
+#ifndef FSROWS_H
+#define FSROWS_H
+
+#include <libxml++/nodes/element.h>
+#include <boost/intrusive_ptr.hpp>
+#include <boost/filesystem/path.hpp>
+#include <sys/stat.h>
+#include "variables.h"
+#include "rowSet.h"
+
+class CommonObjects;
+
+class FsRows : public RowSet {
+	public:
+		class SpecBase : public virtual IntrusivePtrBase {
+			public:
+				virtual bool recurse(const FsRows * fs) const;
+				virtual bool matches(const FsRows * fs) const;
+			protected:
+				const boost::filesystem::path & curPath(const FsRows * fs) const;
+				unsigned int depth(const FsRows * fs) const;
+				struct stat & curStat(const FsRows * fs) const;
+		};
+		typedef boost::intrusive_ptr<SpecBase> SpecBasePtr;
+		typedef std::set<SpecBasePtr> SpecBases;
+		typedef std::list<Glib::ustring> SpecSpec;
+		typedef boost::filesystem::basic_path<std::string, boost::filesystem::path_traits> Path;
+
+		FsRows(const xmlpp::Element * p);
+		~FsRows();
+
+		void execute(const RowProcessor *) const;
+		virtual void loadComplete(const CommonObjects *);
+		virtual void setFilter(const Glib::ustring &);
+		unsigned int columnCount() const;
+		const Glib::ustring & getColumnName(unsigned int col) const;
+		const Glib::ustring & getCurrentValue(const Glib::ustring & id) const;
+		const Glib::ustring & getCurrentValue(unsigned int col) const;
+		bool isNull(unsigned int col) const;
+		bool isNull(const Glib::ustring & id) const;
+		virtual RowAttribute resolveAttr(const Glib::ustring & attrName) const;
+
+		VariableType fileRelPath() const;
+		VariableType fileSize() const;
+		VariableType fileModDate() const;
+		VariableType fileCreateDate() const;
+		VariableType fileUser() const;
+		VariableType fileGroup() const;
+		VariableType fileMode() const;
+		VariableType filePerms() const;
+		VariableType fileType() const;
+
+	protected:
+		void execute(const Path & dir, const RowProcessor *) const;
+		mutable SpecBases specs;
+		mutable boost::filesystem::path fsRoot;
+		mutable boost::filesystem::path curPath;
+		mutable Glib::ustring curPathStr;
+		mutable unsigned int depth;
+		mutable struct stat curStat;
+
+		friend class SpecBase;
+};
+
+#endif
+
+
+
diff --git a/project2/rowSet.cpp b/project2/rowSet.cpp
index 0869b58..318407a 100644
--- a/project2/rowSet.cpp
+++ b/project2/rowSet.cpp
@@ -2,6 +2,7 @@
 #include "rowUser.h"
 #include "commonObjects.h"
 #include <boost/foreach.hpp>
+#include <boost/bind.hpp>
 #include <syslog.h>
 
 RowSet::RowValuesStack RowSet::stack;
@@ -52,9 +53,18 @@ RowSet::rowChanged() const
 	}
 }
 
-unsigned int
+VariableType
 RowSet::getRowNum() const
 {
 	return rowNum;
 }
 
+RowSet::RowAttribute
+RowSet::resolveAttr(const Glib::ustring & attrName) const
+{
+	if (attrName == "rownum") {
+		return boost::bind(&RowSet::getRowNum, this);
+	}
+	throw FieldDoesNotExist();
+}
+
diff --git a/project2/rowSet.h b/project2/rowSet.h
index 612a0b4..f25f937 100644
--- a/project2/rowSet.h
+++ b/project2/rowSet.h
@@ -8,6 +8,8 @@
 #include <boost/intrusive_ptr.hpp>
 #include "sourceObject.h"
 #include "iHaveParameters.h"
+#include "variables.h"
+#include <boost/function.hpp>
 
 class RowProcessor;
 class RowSet;
@@ -16,6 +18,7 @@ typedef std::map<std::string, RowSetPtr> RowSets;
 
 class RowSet : public virtual SourceObject {
 	public:
+		typedef boost::function0<VariableType> RowAttribute;
 		class ParentOutOfRange : public std::exception { };
 		class FieldDoesNotExist : public std::exception { };
 		typedef std::vector<const RowSet *> RowValuesStack;
@@ -29,10 +32,11 @@ class RowSet : public virtual SourceObject {
 		virtual const Glib::ustring & getCurrentValue(unsigned int col) const = 0;
 		virtual bool isNull(unsigned int col) const = 0;
 		virtual bool isNull(const Glib::ustring & id) const = 0;
-		unsigned int getRowNum() const;
+		VariableType getRowNum() const;
 		virtual void execute(const RowProcessor *) const = 0;
 		void use(const RowUser * r) const;
 		void finish(const RowUser * r) const;
+		virtual RowAttribute resolveAttr(const Glib::ustring & attrName) const;
 
 		static const RowValuesStack & Stack() { return stack; }
 		static void beginRow(const RowSet * r);
diff --git a/project2/sqlMergeTask.cpp b/project2/sqlMergeTask.cpp
index 93e91db..156956a 100644
--- a/project2/sqlMergeTask.cpp
+++ b/project2/sqlMergeTask.cpp
@@ -26,6 +26,7 @@ class SqlMergeInsert : public IHaveParameters, public virtual SourceObject, publ
 			BOOST_FOREACH(const Parameters::value_type & v, parameters) {
 				insert->bindParamS(col++, v.second->value);
 			}
+			insert->execute();
 		}
 	private:
 		friend void attach(SqlMergeInsertPtr i, ModifyCommand * insert);
diff --git a/project2/variables.cpp b/project2/variables.cpp
index b0aecea..2267e8b 100644
--- a/project2/variables.cpp
+++ b/project2/variables.cpp
@@ -5,6 +5,7 @@
 #include "rowUser.h"
 #include <libxml++/nodes/textnode.h>
 #include <stdexcept>
+#include <syslog.h>
 #include <boost/tokenizer.hpp>
 #include <boost/foreach.hpp>
 #include <boost/algorithm/string/predicate.hpp>
@@ -22,7 +23,6 @@ class VariableLiteral : public VariableImpl {
 class VariableImplDyn : public VariableImpl {
 	public:
 		VariableImplDyn(const Glib::ustring & src) : VariableImpl(src), cacheValid(false) { }
-		VariableImplDyn(const Glib::ustring & src, boost::optional<Glib::ustring> def) : VariableImpl(src, def), cacheValid(false) { }
 		virtual const Glib::ustring & value() const = 0;
 
 	protected:
@@ -90,23 +90,38 @@ class VariableUri : public VariableImplDyn {
 		}
 };
 
+template <class T>
+static void assignHelper(Glib::ustring & dest, const T & src) {
+	const Glib::ustring * srcp = boost::get<const Glib::ustring>(&src);
+	if (srcp) {
+		dest = *srcp;
+	}
+	else {
+		dest = boost::lexical_cast<Glib::ustring>(src);
+	}
+}
+template <>
+void assignHelper(Glib::ustring & dest, const Glib::ustring & src) {
+	dest = src;
+}
+
 class VariableParent : public VariableImplDyn, public RowUser {
 	public:
 		VariableParent(const Glib::ustring & src, const RowUser * d) :
-			VariableImplDyn(Glib::ustring(std::find_if(src.begin(), src.end(), isalpha), src.end())),
+			VariableImplDyn(src),
 			row(NULL),
-			depth(src.length() - source.length()),
+			depth(src.find_first_not_of('^')),
+			attr(src[depth] == '!'),
 			dep(d)
 		{
-			bind();
 		}
 		VariableParent(const Glib::ustring & name, unsigned int d) :
-			VariableImplDyn(name, NULL),
+			VariableImplDyn(name),
 			row(NULL),
 			depth(d),
+			attr(name[0] == '!'),
 			dep(NULL)
 		{
-			bind();
 		}
 		~VariableParent()
 		{
@@ -132,6 +147,7 @@ class VariableParent : public VariableImplDyn, public RowUser {
 						if (dep) {
 							row->use(dep);
 						}
+						bind();
 					}
 					getValue(cache, row);
 				}
@@ -156,31 +172,22 @@ class VariableParent : public VariableImplDyn, public RowUser {
 			cacheValid = false;
 		}
 	protected:
-		static void assignHelper(Glib::ustring & dest, const Glib::ustring & src) {
-			dest = src;
-		}
-		void bind()
+		void bind() const
 		{
-			if (name[0] == '!') {
-				if (name == "!rownum") {
-					getValue = boost::bind(&assignHelper, _1,
-							boost::bind(&boost::lexical_cast<Glib::ustring, unsigned int>,
-								boost::bind(&RowSet::getRowNum, _2)));
-				}
-				else {
-					throw RowSet::FieldDoesNotExist();
-				}
+			if (attr) {
+				getValue = boost::bind(&assignHelper<VariableType>, _1, boost::bind(row->resolveAttr(name)));
 			}
 			else {
 				typedef const Glib::ustring & (RowSet::*gCV)(const Glib::ustring &) const;
-				getValue = boost::bind(&assignHelper, _1,
+				getValue = boost::bind(&assignHelper<Glib::ustring>, _1,
 						boost::bind((gCV)&RowSet::getCurrentValue, _2, name));
 			}
 		}
 		mutable const RowSet * row;
 		const size_t depth;
+		const bool attr;
 		const RowUser * dep;
-		boost::function2<void, Glib::ustring &, const RowSet *> getValue;
+		mutable boost::function2<void, Glib::ustring &, const RowSet *> getValue;
 };
 
 class VariableParse : public VariableImplDyn, public RowUser {
@@ -270,13 +277,6 @@ Variable::makeParent(const Glib::ustring & n, unsigned int d)
 	return Variable(new VariableParent(n, d));
 }
 
-VariableImpl::VariableImpl(const Glib::ustring & src, boost::optional<Glib::ustring> def) :
-	source(src),
-	name(src),
-	defaultValue(def)
-{
-}
-
 VariableImpl::VariableImpl(const Glib::ustring & src) :
 	source(src)
 {
diff --git a/project2/variables.h b/project2/variables.h
index c728284..8b3f7a4 100644
--- a/project2/variables.h
+++ b/project2/variables.h
@@ -3,12 +3,15 @@
 
 #include <boost/intrusive_ptr.hpp>
 #include <boost/optional.hpp>
+#include <stdint.h>
 #include <glibmm/ustring.h>
 #include <libxml++/nodes/element.h>
 #include <libxml++/attribute.h>
 #include "intrusivePtrBase.h"
+#include <boost/variant.hpp>
 
 class RowUser;
+typedef boost::variant<Glib::ustring, unsigned int, long int> VariableType;
 
 class VariableImpl : public virtual IntrusivePtrBase {
 	public:
@@ -16,7 +19,6 @@ class VariableImpl : public virtual IntrusivePtrBase {
 
 	protected:
 		VariableImpl(const Glib::ustring & src);
-		VariableImpl(const Glib::ustring & src, boost::optional<Glib::ustring> def);
 		virtual ~VariableImpl() = 0;
 
 		const Glib::ustring source;