Use a flex based parser for SQL instead of a noddy upto ; based one

Tested with PQ, comments, multi-line comments and strings with '' and ; in

7 files changed, 147 insertions, 8 deletions

diff --git a/project2/sql/Jamfile.jam b/project2/sql/Jamfile.jam
index 0af9f5a..8a3661b 100644
--- a/project2/sql/Jamfile.jam
+++ b/project2/sql/Jamfile.jam
@@ -1,8 +1,11 @@
+import lex ;
+
 alias glibmm : : : :
 	<cflags>"`pkg-config --cflags glibmm-2.4`"
 	<linkflags>"`pkg-config --libs glibmm-2.4`"
 	;
 lib boost_filesystem ;
+lib fl ;
 
 build-project unittests ;
 
@@ -64,6 +67,7 @@ cpp-pch pch : pch.hpp :
 
 lib p2sql :
 	pch
+	sql.ll
 	[ glob *.cpp : sql-mod*.cpp ]
 	../../libdbpp//dbpp
 	:
@@ -73,6 +77,7 @@ lib p2sql :
 	<library>../common//p2common
 	<library>../lib//p2lib
 	<include>../../libmisc
+	<library>fl
 	: :
 	<include>.
 	;
diff --git a/project2/sql/mockDatabase.cpp b/project2/sql/mockDatabase.cpp
index 2f26282..47fe99f 100644
--- a/project2/sql/mockDatabase.cpp
+++ b/project2/sql/mockDatabase.cpp
@@ -4,6 +4,8 @@
 #include <logger.h>
 #include <fstream>
 #include <modifycommand.h>
+#include <FlexLexer.h>
+#include "sqlFlexLexer.h"
 
 MockDatabase::MockDatabase(const std::string & name) :
 	mockName(name)
@@ -47,12 +49,8 @@ MockDatabase::PlaySchemaScript(DB::Connection * conn, const boost::filesystem::p
 	Logger()->messagebf(LOG_DEBUG, "%s << %s", mockName, s);
 	std::ifstream f;
 	f.open(s.string());
-	while (!f.eof()) {
-		char buf[BUFSIZ];
-		f.getline(buf, BUFSIZ, ';');
-		if (!f.eof())
-			conn->execute(buf);
-	}
+	auto lexer = boost::shared_ptr<FlexLexer>(new sqlFlexLexer(f, conn));
+	while(lexer->yylex() != 0) ;
 	f.close();
 }
 
diff --git a/project2/sql/sql.ll b/project2/sql/sql.ll
new file mode 100644
index 0000000..2127e3f
--- /dev/null
+++ b/project2/sql/sql.ll
@@ -0,0 +1,82 @@
+%option batch
+%option c++
+%option noyywrap
+%option yyclass="sqlFlexLexer"
+
+%{
+#include <stdexcept>
+#include "sqlFlexLexer.h"
+std::string comment;
+std::string statement;
+%}
+
+non_newline [^\r\n]
+mcomment_start "/*"
+mcomment_stop "*/"
+comment			("--"{non_newline}*)
+other .
+quote '
+quote_apos ''
+
+%x COMMENT
+%x STATEMENT
+%x QUOTE
+
+%%
+{mcomment_start} {
+	comment += YYText();
+	yy_push_state(COMMENT);
+}
+
+<COMMENT>{mcomment_stop} {
+	comment += YYText();
+	Comment(comment);
+	comment.clear();
+	yy_pop_state();
+}
+
+<COMMENT>(.|\n) {
+	comment += YYText();
+}
+
+<COMMENT><<EOF>> {
+	throw std::runtime_error("Unterminated comment");
+}
+
+{comment} { 
+	Comment(YYText());
+}
+
+{other} {
+	statement += YYText();
+	yy_push_state(STATEMENT);
+}
+
+<STATEMENT>{quote} {
+	statement += YYText();
+	yy_push_state(QUOTE);
+}
+
+<QUOTE>{quote} {
+	statement += YYText();
+	yy_pop_state();
+}
+
+<QUOTE>{quote_apos} {
+	statement += YYText();
+}
+
+<QUOTE>. {
+	statement += YYText();
+}
+
+<STATEMENT>; {
+	Statement(statement);
+	statement.clear();
+	yy_pop_state();
+}
+
+<STATEMENT>. {
+	statement += YYText();
+}
+
diff --git a/project2/sql/sqlFlexLexer.cpp b/project2/sql/sqlFlexLexer.cpp
new file mode 100644
index 0000000..2b26f20
--- /dev/null
+++ b/project2/sql/sqlFlexLexer.cpp
@@ -0,0 +1,28 @@
+#include <FlexLexer.h>
+#include "sqlFlexLexer.h"
+#include <logger.h>
+
+sqlFlexLexer::sqlFlexLexer(std::istream & f, DB::Connection * c) :
+	yyFlexLexer(&f, NULL),
+	conn(c)
+{
+}
+
+void
+sqlFlexLexer::LexerOutput(const char *, int)
+{
+}
+
+void
+sqlFlexLexer::Comment(const std::string & text)
+{
+	Logger()->messagebf(LOG_DEBUG, "Got comment: %s", text);
+}
+
+void
+sqlFlexLexer::Statement(const std::string & text)
+{
+	Logger()->messagebf(LOG_DEBUG, "Got statement: %s", text);
+	conn->execute(text);
+}
+
diff --git a/project2/sql/sqlFlexLexer.h b/project2/sql/sqlFlexLexer.h
new file mode 100644
index 0000000..09c3eae
--- /dev/null
+++ b/project2/sql/sqlFlexLexer.h
@@ -0,0 +1,18 @@
+#include <istream>
+#include <connection.h>
+
+class sqlFlexLexer : public yyFlexLexer {
+	public:
+		sqlFlexLexer(std::istream &, DB::Connection *);
+		int yylex();
+
+		void Comment(const std::string &);
+		void Statement(const std::string &);
+
+	protected:
+		void LexerOutput(const char *, int) override;
+
+	private:
+		DB::Connection * conn;
+};
+
diff --git a/project2/sql/unittests/pqschema.sql b/project2/sql/unittests/pqschema.sql
index 96848eb..506365b 100644
--- a/project2/sql/unittests/pqschema.sql
+++ b/project2/sql/unittests/pqschema.sql
@@ -1,3 +1,11 @@
+--
+-- pg_dump style comment
+-- Table: test; owner: comment: ;
+--
+/*
+	 This is
+	 a
+	 multiline comment */
 CREATE TABLE test(
 		id int,
 		fl numeric(5,2),
@@ -5,4 +13,4 @@ CREATE TABLE test(
 		boolean bool,
 		dt timestamp without time zone,
 		ts interval);
-INSERT INTO test VALUES(4, 123.45, 'some text', true, '2015-04-27 23:06:03', '1 day 14:13:12');
+INSERT INTO test VALUES(4, 123.45, 'some text with a ; in it and a '' too', true, '2015-04-27 23:06:03', '1 day 14:13:12');
diff --git a/project2/sql/unittests/testpq.cpp b/project2/sql/unittests/testpq.cpp
index 1d38e8c..38b07a2 100644
--- a/project2/sql/unittests/testpq.cpp
+++ b/project2/sql/unittests/testpq.cpp
@@ -103,7 +103,7 @@ BOOST_AUTO_TEST_CASE( bindAndSelectOther )
 	while (select->fetch()) {
 		assertColumnValueHelper(*select, 0, 4);
 		assertColumnValueHelper(*select, 1, 123.45);
-		assertColumnValueHelper(*select, 2, std::string("some text"));
+		assertColumnValueHelper(*select, 2, std::string("some text with a ; in it and a ' too"));
 		assertColumnValueHelper(*select, 3, true);
 		assertColumnValueHelper(*select, 4, boost::posix_time::ptime_from_tm({ 3, 6, 23, 27, 3, 115, 0, 0, 0, 0, 0}));
 		assertColumnValueHelper(*select, 5, boost::posix_time::time_duration(38, 13, 12));