Move text reader into it's own class

4 files changed, 106 insertions, 58 deletions

diff --git a/project2/common/stream.h b/project2/common/stream.h
index 14293f0..eebefab 100644
--- a/project2/common/stream.h
+++ b/project2/common/stream.h
@@ -7,7 +7,8 @@
 class Stream : public SourceObject {
 	public:
 		template<typename... X> Stream(const X &... x) : SourceObject(x...) { }
-		typedef boost::function2<size_t, const char *, size_t> Sink;
+
+		typedef boost::function<size_t(const char *, size_t)> Sink;
 		virtual void runStream(const Sink &) const = 0;
 };
 typedef boost::intrusive_ptr<Stream> StreamPtr;
diff --git a/project2/streams/streamRows.cpp b/project2/streams/streamRows.cpp
index f807e4f..78f6963 100644
--- a/project2/streams/streamRows.cpp
+++ b/project2/streams/streamRows.cpp
@@ -4,9 +4,9 @@
 #include "stream.h"
 #include "definedColumns.h"
 #include "rowProcessor.h"
+#include "textReader.h"
 #include <boost/algorithm/string/predicate.hpp>
 
-SimpleSysCallException(ReallocFail);
 class RowProcessor;
 
 /// Base class for Project2 components that create a row set based on the contents of a byte stream
@@ -52,8 +52,7 @@ class StreamRows : public DefinedColumns, public RowSet {
 			countBlankRows(p->value("countBlankRows", false)),
 			newline(p->value("newline", "\n").as<Glib::ustring>()),
 			encoding(p->value("encoding", "utf-8").as<std::string>()),
-			skipheader(p->value("skipheader", 0).as<int64_t>()),
-			convertRequired(encoding != "utf-8")
+			skipheader(p->value("skipheader", 0).as<int64_t>())
 		{
 			p->script->loader.addLoadTarget(p, Storer::into<ElementLoader>(&stream));
 		}
@@ -61,60 +60,9 @@ class StreamRows : public DefinedColumns, public RowSet {
 		void execute(const Glib::ustring &, const RowProcessor * rp) const
 		{
 			ParseState ps(this, rp);
-			char * buf = NULL;
-			size_t bufLen = 0;
-			ScopeObject tidy([&]{ free(buf); });
-			stream->runStream([&](const char * bytes, size_t bytesLen) -> size_t {
-					size_t used = 0, len = 0;
-					const char * src;
-					size_t srcLen;
-					if (bufLen) {
-						char * nbuf = (char*)realloc(buf, bufLen + bytesLen);
-						if (!nbuf) {
-							free(buf);
-							throw ReallocFail(errno);
-						}
-						src = buf = nbuf;
-						memcpy(buf + bufLen, bytes, bytesLen);
-						srcLen = bufLen += bytesLen;
-					}
-					else {
-						src = bytes;
-						srcLen = bytesLen;
-					}
-					if (convertRequired) {
-						gchar * utf8 = g_convert(src, srcLen, "utf-8", encoding.c_str(), &used, &len, NULL);
-						for (const gchar * iter = utf8; *iter; iter = g_utf8_next_char(iter)) {
-							this->pushChar(g_utf8_get_char(iter), ps);
-						}
-						free(utf8);
-					}
-					else {
-						const gchar * firstInvalid;
-						g_utf8_validate(src, srcLen, &firstInvalid);
-						for (const gchar * iter = src; iter < firstInvalid && *iter ; iter = g_utf8_next_char(iter)) {
-							this->pushChar(g_utf8_get_char(iter), ps);
-						}
-						used = firstInvalid - src;
-					}
-					size_t newBuf = srcLen - used;
-					if (newBuf) {
-						if (bufLen < newBuf) {
-							char * nbuf = (char*)realloc(buf, newBuf);
-							if (!nbuf) {
-								free(buf);
-								throw ReallocFail(errno);
-							}
-							buf = nbuf;
-						}
-						bufLen = newBuf;
-						memcpy(buf, src + used, bufLen);
-					}
-					else {
-						bufLen = 0;
-					}
-					return bytesLen;
-					});
+			TextReader::CharSink cs = boost::bind(&StreamRows::pushChar, this, _1, boost::ref(ps));
+			TextReader t(encoding.c_str());
+			stream->runStream(boost::bind(&TextReader::read, &t, _1, _2, cs));
 		}
 
 		void pushChar(gunichar c, ParseState & ps) const
diff --git a/project2/streams/textReader.cpp b/project2/streams/textReader.cpp
new file mode 100644
index 0000000..8a10d4f
--- /dev/null
+++ b/project2/streams/textReader.cpp
@@ -0,0 +1,80 @@
+#include <pch.hpp>
+#include "textReader.h"
+
+SimpleSysCallException(ReallocFail);
+
+TextReader::TextReader() :
+	convertRequired(false),
+	encoding(NULL),
+	buf(NULL),
+	bufLen(0)
+{
+}
+
+TextReader::TextReader(const gchar * enc) :
+	convertRequired(strcmp(enc, "utf-8")),
+	encoding(enc),
+	buf(NULL),
+	bufLen(0)
+{
+}
+
+TextReader::~TextReader()
+{
+	free(buf);
+}
+
+size_t
+TextReader::read(const char * bytes, size_t bytesLen, const CharSink & pushChar)
+{
+	size_t used = 0, len = 0;
+	const char * src;
+	size_t srcLen;
+	if (bufLen) {
+		char * nbuf = (char*)realloc(buf, bufLen + bytesLen);
+		if (!nbuf) {
+			free(buf);
+			throw ReallocFail(errno);
+		}
+		src = buf = nbuf;
+		memcpy(buf + bufLen, bytes, bytesLen);
+		srcLen = bufLen += bytesLen;
+	}
+	else {
+		src = bytes;
+		srcLen = bytesLen;
+	}
+	if (convertRequired) {
+		gchar * utf8 = g_convert(src, srcLen, "utf-8", encoding, &used, &len, NULL);
+		for (const gchar * iter = utf8; *iter; iter = g_utf8_next_char(iter)) {
+			pushChar(g_utf8_get_char(iter));
+		}
+		free(utf8);
+	}
+	else {
+		const gchar * firstInvalid;
+		g_utf8_validate(src, srcLen, &firstInvalid);
+		for (const gchar * iter = src; iter < firstInvalid && *iter ; iter = g_utf8_next_char(iter)) {
+			pushChar(g_utf8_get_char(iter));
+		}
+		used = firstInvalid - src;
+	}
+	size_t newBuf = srcLen - used;
+	if (newBuf) {
+		if (bufLen < newBuf) {
+			char * nbuf = (char*)realloc(buf, newBuf);
+			if (!nbuf) {
+				free(buf);
+				throw ReallocFail(errno);
+			}
+			buf = nbuf;
+		}
+		bufLen = newBuf;
+		memcpy(buf, src + used, bufLen);
+	}
+	else {
+		bufLen = 0;
+	}
+	return bytesLen;
+}
+
diff --git a/project2/streams/textReader.h b/project2/streams/textReader.h
new file mode 100644
index 0000000..898d4ed
--- /dev/null
+++ b/project2/streams/textReader.h
@@ -0,0 +1,19 @@
+
+
+class TextReader {
+	public:
+		typedef boost::function<void(gunichar)> CharSink;
+
+		TextReader();
+		TextReader(const gchar * encoding);
+		~TextReader();
+
+		size_t read(const char * bytes, size_t bytesLen, const CharSink &);
+
+	private:
+		const bool convertRequired;
+		const char * encoding;
+		char * buf;
+		size_t bufLen;
+};
+