From 0fe2ea539bfff5b78cd50b0c23ab55ad9bbc96b5 Mon Sep 17 00:00:00 2001 From: Dan Goodliffe Date: Mon, 25 May 2015 19:32:47 +0100 Subject: Use a flex based parser for SQL instead of a noddy upto ; based one Tested with PQ, comments, multi-line comments and strings with '' and ; in --- project2/sql/Jamfile.jam | 5 +++ project2/sql/mockDatabase.cpp | 10 ++--- project2/sql/sql.ll | 82 +++++++++++++++++++++++++++++++++++++ project2/sql/sqlFlexLexer.cpp | 28 +++++++++++++ project2/sql/sqlFlexLexer.h | 18 ++++++++ project2/sql/unittests/pqschema.sql | 10 ++++- project2/sql/unittests/testpq.cpp | 2 +- 7 files changed, 147 insertions(+), 8 deletions(-) create mode 100644 project2/sql/sql.ll create mode 100644 project2/sql/sqlFlexLexer.cpp create mode 100644 project2/sql/sqlFlexLexer.h diff --git a/project2/sql/Jamfile.jam b/project2/sql/Jamfile.jam index 0af9f5a..8a3661b 100644 --- a/project2/sql/Jamfile.jam +++ b/project2/sql/Jamfile.jam @@ -1,8 +1,11 @@ +import lex ; + alias glibmm : : : : "`pkg-config --cflags glibmm-2.4`" "`pkg-config --libs glibmm-2.4`" ; lib boost_filesystem ; +lib fl ; build-project unittests ; @@ -64,6 +67,7 @@ cpp-pch pch : pch.hpp : lib p2sql : pch + sql.ll [ glob *.cpp : sql-mod*.cpp ] ../../libdbpp//dbpp : @@ -73,6 +77,7 @@ lib p2sql : ../common//p2common ../lib//p2lib ../../libmisc + fl : : . ; diff --git a/project2/sql/mockDatabase.cpp b/project2/sql/mockDatabase.cpp index 2f26282..47fe99f 100644 --- a/project2/sql/mockDatabase.cpp +++ b/project2/sql/mockDatabase.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include "sqlFlexLexer.h" MockDatabase::MockDatabase(const std::string & name) : mockName(name) @@ -47,12 +49,8 @@ MockDatabase::PlaySchemaScript(DB::Connection * conn, const boost::filesystem::p Logger()->messagebf(LOG_DEBUG, "%s << %s", mockName, s); std::ifstream f; f.open(s.string()); - while (!f.eof()) { - char buf[BUFSIZ]; - f.getline(buf, BUFSIZ, ';'); - if (!f.eof()) - conn->execute(buf); - } + auto lexer = boost::shared_ptr(new sqlFlexLexer(f, conn)); + while(lexer->yylex() != 0) ; f.close(); } diff --git a/project2/sql/sql.ll b/project2/sql/sql.ll new file mode 100644 index 0000000..2127e3f --- /dev/null +++ b/project2/sql/sql.ll @@ -0,0 +1,82 @@ +%option batch +%option c++ +%option noyywrap +%option yyclass="sqlFlexLexer" + +%{ +#include +#include "sqlFlexLexer.h" +std::string comment; +std::string statement; +%} + +non_newline [^\r\n] +mcomment_start "/*" +mcomment_stop "*/" +comment ("--"{non_newline}*) +other . +quote ' +quote_apos '' + +%x COMMENT +%x STATEMENT +%x QUOTE + +%% +{mcomment_start} { + comment += YYText(); + yy_push_state(COMMENT); +} + +{mcomment_stop} { + comment += YYText(); + Comment(comment); + comment.clear(); + yy_pop_state(); +} + +(.|\n) { + comment += YYText(); +} + +<> { + throw std::runtime_error("Unterminated comment"); +} + +{comment} { + Comment(YYText()); +} + +{other} { + statement += YYText(); + yy_push_state(STATEMENT); +} + +{quote} { + statement += YYText(); + yy_push_state(QUOTE); +} + +{quote} { + statement += YYText(); + yy_pop_state(); +} + +{quote_apos} { + statement += YYText(); +} + +. { + statement += YYText(); +} + +; { + Statement(statement); + statement.clear(); + yy_pop_state(); +} + +. { + statement += YYText(); +} + diff --git a/project2/sql/sqlFlexLexer.cpp b/project2/sql/sqlFlexLexer.cpp new file mode 100644 index 0000000..2b26f20 --- /dev/null +++ b/project2/sql/sqlFlexLexer.cpp @@ -0,0 +1,28 @@ +#include +#include "sqlFlexLexer.h" +#include + +sqlFlexLexer::sqlFlexLexer(std::istream & f, DB::Connection * c) : + yyFlexLexer(&f, NULL), + conn(c) +{ +} + +void +sqlFlexLexer::LexerOutput(const char *, int) +{ +} + +void +sqlFlexLexer::Comment(const std::string & text) +{ + Logger()->messagebf(LOG_DEBUG, "Got comment: %s", text); +} + +void +sqlFlexLexer::Statement(const std::string & text) +{ + Logger()->messagebf(LOG_DEBUG, "Got statement: %s", text); + conn->execute(text); +} + diff --git a/project2/sql/sqlFlexLexer.h b/project2/sql/sqlFlexLexer.h new file mode 100644 index 0000000..09c3eae --- /dev/null +++ b/project2/sql/sqlFlexLexer.h @@ -0,0 +1,18 @@ +#include +#include + +class sqlFlexLexer : public yyFlexLexer { + public: + sqlFlexLexer(std::istream &, DB::Connection *); + int yylex(); + + void Comment(const std::string &); + void Statement(const std::string &); + + protected: + void LexerOutput(const char *, int) override; + + private: + DB::Connection * conn; +}; + diff --git a/project2/sql/unittests/pqschema.sql b/project2/sql/unittests/pqschema.sql index 96848eb..506365b 100644 --- a/project2/sql/unittests/pqschema.sql +++ b/project2/sql/unittests/pqschema.sql @@ -1,3 +1,11 @@ +-- +-- pg_dump style comment +-- Table: test; owner: comment: ; +-- +/* + This is + a + multiline comment */ CREATE TABLE test( id int, fl numeric(5,2), @@ -5,4 +13,4 @@ CREATE TABLE test( boolean bool, dt timestamp without time zone, ts interval); -INSERT INTO test VALUES(4, 123.45, 'some text', true, '2015-04-27 23:06:03', '1 day 14:13:12'); +INSERT INTO test VALUES(4, 123.45, 'some text with a ; in it and a '' too', true, '2015-04-27 23:06:03', '1 day 14:13:12'); diff --git a/project2/sql/unittests/testpq.cpp b/project2/sql/unittests/testpq.cpp index 1d38e8c..38b07a2 100644 --- a/project2/sql/unittests/testpq.cpp +++ b/project2/sql/unittests/testpq.cpp @@ -103,7 +103,7 @@ BOOST_AUTO_TEST_CASE( bindAndSelectOther ) while (select->fetch()) { assertColumnValueHelper(*select, 0, 4); assertColumnValueHelper(*select, 1, 123.45); - assertColumnValueHelper(*select, 2, std::string("some text")); + assertColumnValueHelper(*select, 2, std::string("some text with a ; in it and a ' too")); assertColumnValueHelper(*select, 3, true); assertColumnValueHelper(*select, 4, boost::posix_time::ptime_from_tm({ 3, 6, 23, 27, 3, 115, 0, 0, 0, 0, 0})); assertColumnValueHelper(*select, 5, boost::posix_time::time_duration(38, 13, 12)); -- cgit v1.2.3