From 15c1e5566f7ad8448985ca6e52805ceb4ec389a8 Mon Sep 17 00:00:00 2001 From: Dan Goodliffe Date: Mon, 24 May 2021 01:01:30 +0100 Subject: Basic support for queries with bound parameters Bit of a reshuffle to make the types not DB specific, add some basic tests over query execution, no selects yet and no validation anything is actually right. --- lib/Jamfile.jam | 2 ++ lib/dbConn.h | 14 ++++++++ lib/dbTypes.h | 31 ++++++++++++++++++ lib/helpers.h | 19 +++++++++++ lib/input/mysqlConn.cpp | 78 +++++++++++++++++++++++++++++++++++++++++++++ lib/input/mysqlConn.h | 8 +++-- lib/mysql_types.h | 40 +++++++++++------------ lib/output/pq/pqConn.cpp | 70 ++++++++++++++++++++++++++++++++++++++++ lib/output/pq/pqConn.h | 8 ++++- lib/row.cpp | 1 + lib/row.h | 4 +-- lib/streamSupport.cpp | 10 +++--- lib/streamSupport.h | 12 ++----- test/Jamfile.jam | 2 ++ test/test-mysql.cpp | 25 +++++++++++++++ test/test-postgresql.cpp | 28 ++++++++++++++++ test/test-rawDataReader.cpp | 6 ++-- test/test-streams.cpp | 4 +-- 18 files changed, 318 insertions(+), 44 deletions(-) create mode 100644 lib/dbConn.h create mode 100644 lib/dbTypes.h create mode 100644 test/test-mysql.cpp create mode 100644 test/test-postgresql.cpp diff --git a/lib/Jamfile.jam b/lib/Jamfile.jam index 05699a8..c54dabc 100644 --- a/lib/Jamfile.jam +++ b/lib/Jamfile.jam @@ -4,7 +4,9 @@ lib mygrate : static . ..//libmariadb + ..//libpq : : . ..//libmariadb + ..//libpq ; diff --git a/lib/dbConn.h b/lib/dbConn.h new file mode 100644 index 0000000..26b9a64 --- /dev/null +++ b/lib/dbConn.h @@ -0,0 +1,14 @@ +#ifndef MYGRATE_DBCONN_H +#define MYGRATE_DBCONN_H + +#include +#include + +namespace MyGrate { + class DbConn { + virtual void query(const char * const) = 0; + virtual void query(const char * const, const std::initializer_list &) = 0; + }; +} + +#endif diff --git a/lib/dbTypes.h b/lib/dbTypes.h new file mode 100644 index 0000000..ba0cd70 --- /dev/null +++ b/lib/dbTypes.h @@ -0,0 +1,31 @@ +#ifndef MYGRATE_DBTYPES_H +#define MYGRATE_DBTYPES_H + +#include "bitset.h" +#include +#include +#include +#include + +struct timespec; + +namespace MyGrate { + struct Date { + uint16_t year; + uint8_t month; + uint8_t day; + }; + struct Time { + uint8_t hour; + uint8_t minute; + uint8_t second; + }; + struct DateTime : public Date, public Time { + }; + using Blob = std::span; + + using DbValue = std::variant; +} + +#endif diff --git a/lib/helpers.h b/lib/helpers.h index 3a94ead..a8a630e 100644 --- a/lib/helpers.h +++ b/lib/helpers.h @@ -3,6 +3,7 @@ #include #include +#include #include namespace MyGrate { @@ -30,6 +31,24 @@ namespace MyGrate { i /= 100; return r; } + + template + concept Stringable = requires(T a) + { + { + std::to_string(a) + } -> std::same_as; + }; + template + concept Viewable = requires(T a) + { + { + a.data() + } -> std::convertible_to; + { + a.size() + } -> std::integral; + }; } #endif diff --git a/lib/input/mysqlConn.cpp b/lib/input/mysqlConn.cpp index 3a2c9f4..179f9d5 100644 --- a/lib/input/mysqlConn.cpp +++ b/lib/input/mysqlConn.cpp @@ -1,9 +1,18 @@ #include "mysqlConn.h" #include "helpers.h" +#include +#include +#include +#include #include +#include #include +#include +#include namespace MyGrate::Input { + using StmtPtr = std::unique_ptr; + MySQLConn::MySQLConn( const char * const host, const char * const user, const char * const pass, unsigned short port) : st_mysql {} @@ -26,4 +35,73 @@ namespace MyGrate::Input { { verify(!mysql_query(this, q), q); } + + struct Bindings { + // NOLINTNEXTLINE(hicpp-explicit-conversions) + explicit Bindings(const std::initializer_list & vs) + { + binds.reserve(vs.size()); + extras.reserve(vs.size()); + for (const auto & v : vs) { + std::visit(*this, v); + } + } + template + void + operator()(const T & v) + { + auto & b = binds.emplace_back(); + b.buffer_type = MySQL::CType::type; + b.buffer = const_cast(&v); + b.is_unsigned = std::unsigned_integral; + } + template + void + operator()(const T & v) + { + auto & b = binds.emplace_back(); + b.buffer_type = MySQL::CType::type; + b.buffer = const_cast(&v); + } + template + void + operator()(const T & v) + { + auto & b = binds.emplace_back(); + b.buffer_type = MySQL::CType::type; + b.buffer = const_cast(v.data()); + b.length = &extras.emplace_back(v.size(), 0).len; + } + void + operator()(const std::nullptr_t &) + { + auto & b = binds.emplace_back(); + b.buffer = nullptr; + b.is_null = &extras.emplace_back(0, 1).null; + } + template + void + operator()(const T &) + { + throw std::runtime_error("Not implemented"); + } + struct extra { + explicit extra(unsigned long l, my_bool n = 0) : len {l}, null {n} { } + unsigned long len; + my_bool null; + }; + std::vector binds; + std::vector extras; + }; + + void + MySQLConn::query(const char * const q, const std::initializer_list & vs) + { + StmtPtr stmt {mysql_stmt_init(this), &mysql_stmt_close}; + verify(!mysql_stmt_prepare(stmt.get(), q, strlen(q)), q); + verify(mysql_stmt_param_count(stmt.get()) == vs.size(), "Param count mismatch"); + Bindings b {vs}; + verify(!mysql_stmt_bind_param(stmt.get(), b.binds.data()), "Param count mismatch"); + verify(!mysql_stmt_execute(stmt.get()), q); + } } diff --git a/lib/input/mysqlConn.h b/lib/input/mysqlConn.h index 2f57f33..9e4ec25 100644 --- a/lib/input/mysqlConn.h +++ b/lib/input/mysqlConn.h @@ -1,15 +1,19 @@ #ifndef MYGRATE_INPUT_MYSQLCONN_H #define MYGRATE_INPUT_MYSQLCONN_H +#include +#include +#include #include namespace MyGrate::Input { - class MySQLConn : public MYSQL { + class MySQLConn : public MYSQL, public DbConn { public: MySQLConn(const char * const host, const char * const user, const char * const pass, unsigned short port); ~MySQLConn(); - void query(const char * const); + void query(const char * const) override; + void query(const char * const q, const std::initializer_list &) override; }; } diff --git a/lib/mysql_types.h b/lib/mysql_types.h index af45b4f..4f5355c 100644 --- a/lib/mysql_types.h +++ b/lib/mysql_types.h @@ -2,34 +2,19 @@ #define MYGRATE_MYSQL_TYPES_H #include "bitset.h" +#include "dbTypes.h" #include #include #include #include -#include #include -#include - -struct timespec; namespace MyGrate { class RawDataReader; namespace MySQL { - struct Date { - uint16_t year; - uint8_t month; - uint8_t day; - }; - struct Time { - uint8_t hour; - uint8_t minute; - uint8_t second; - }; - struct DateTime : public Date, public Time { - }; - using Blob = std::span; template struct Type; + template struct CType; #define DEFINE_BASE_TYPE(ET, CT, MDS, SIGN) \ template<> struct Type { \ @@ -37,6 +22,11 @@ namespace MyGrate { constexpr static size_t md_bytes {MDS}; \ static C read(RawDataReader & md, RawDataReader & data); \ } +#define DEFINE_CTYPE(ET, CT) \ + template<> struct CType { \ + using C = CT; \ + static constexpr enum_field_types type {ET}; \ + } #define DEFINE_TYPE(ET, CT, MDS) DEFINE_BASE_TYPE(ET, CT, MDS, false) #define DEFINE_ITYPE(ET, BCT, MDS) \ DEFINE_BASE_TYPE(ET, BCT, MDS, false); \ @@ -74,12 +64,22 @@ namespace MyGrate { DEFINE_TYPE(MYSQL_TYPE_JSON, std::string_view, 2); DEFINE_TYPE(MYSQL_TYPE_GEOMETRY, Blob, 1); + DEFINE_CTYPE(MYSQL_TYPE_DOUBLE, double); + DEFINE_CTYPE(MYSQL_TYPE_FLOAT, float); + DEFINE_CTYPE(MYSQL_TYPE_SHORT, int16_t); + DEFINE_CTYPE(MYSQL_TYPE_LONG, int32_t); + DEFINE_CTYPE(MYSQL_TYPE_LONGLONG, int64_t); + DEFINE_CTYPE(MYSQL_TYPE_TINY, int8_t); + DEFINE_CTYPE(MYSQL_TYPE_SHORT, uint16_t); + DEFINE_CTYPE(MYSQL_TYPE_LONG, uint32_t); + DEFINE_CTYPE(MYSQL_TYPE_LONGLONG, uint64_t); + DEFINE_CTYPE(MYSQL_TYPE_TINY, uint8_t); + DEFINE_CTYPE(MYSQL_TYPE_STRING, std::string_view); + DEFINE_CTYPE(MYSQL_TYPE_BLOB, Blob); + #undef DEFINE_ITYPE #undef DEFINE_USTYPE #undef DEFINE_TYPE - - using FieldValue = std::variant; } } diff --git a/lib/output/pq/pqConn.cpp b/lib/output/pq/pqConn.cpp index 78cdc06..4f55ba8 100644 --- a/lib/output/pq/pqConn.cpp +++ b/lib/output/pq/pqConn.cpp @@ -1,8 +1,17 @@ #include "pqConn.h" +#include #include +#include +#include +#include #include +#include +#include +#include namespace MyGrate::Output::Pq { + using ResPtr = std::unique_ptr; + PqConn::PqConn(const char * const str) : conn {PQconnectdb(str)} { verify(PQstatus(conn) == CONNECTION_OK, "Connection failure"); @@ -14,6 +23,67 @@ namespace MyGrate::Output::Pq { PQfinish(conn); } + void + PqConn::query(const char * const q) + { + ResPtr res {PQexec(conn, q), &PQclear}; + verify(PQresultStatus(res.get()) == PGRES_COMMAND_OK, q); + } + + struct Bindings { + // NOLINTNEXTLINE(hicpp-explicit-conversions) + explicit Bindings(const std::initializer_list & vs) + { + bufs.reserve(vs.size()); + values.reserve(vs.size()); + lengths.reserve(vs.size()); + for (const auto & v : vs) { + std::visit(*this, v); + } + } + template + void + operator()(const T & v) + { + bufs.emplace_back(std::to_string(v)); + const auto & vw {bufs.back()}; + values.emplace_back(vw.data()); + lengths.emplace_back(vw.length()); + } + template + void + operator()(const T & v) + { + values.emplace_back(v.data()); + lengths.emplace_back(v.size()); + } + template + void + operator()(const T &) + { + throw std::runtime_error("Not implemented"); + } + void + operator()(const std::nullptr_t &) + { + values.emplace_back(nullptr); + lengths.emplace_back(0); + } + + std::vector bufs; + std::vector values; + std::vector lengths; + }; + + void + PqConn::query(const char * const q, const std::initializer_list & vs) + { + Bindings b {vs}; + ResPtr res {PQexecParams(conn, q, (int)vs.size(), nullptr, b.values.data(), b.lengths.data(), nullptr, 0), + &PQclear}; + verify(PQresultStatus(res.get()) == PGRES_COMMAND_OK, q); + } + void PqConn::notice_processor(void * p, const char * n) { diff --git a/lib/output/pq/pqConn.h b/lib/output/pq/pqConn.h index bcf30b6..613af6f 100644 --- a/lib/output/pq/pqConn.h +++ b/lib/output/pq/pqConn.h @@ -1,14 +1,20 @@ #ifndef MYGRATE_OUTPUT_PQ_PQCONN_H #define MYGRATE_OUTPUT_PQ_PQCONN_H +#include +#include +#include #include namespace MyGrate::Output::Pq { - class PqConn { + class PqConn : public DbConn { public: explicit PqConn(const char * const str); virtual ~PqConn(); + void query(const char * const) override; + void query(const char * const, const std::initializer_list &) override; + private: static void notice_processor(void *, const char *); virtual void notice_processor(const char *) const; diff --git a/lib/row.cpp b/lib/row.cpp index 65ac591..9d81906 100644 --- a/lib/row.cpp +++ b/lib/row.cpp @@ -1,6 +1,7 @@ #include "row.h" #include "bitset.h" #include "mariadb_repl.h" +#include "mysql_types.h" #include "rawDataReader.h" #include #include diff --git a/lib/row.h b/lib/row.h index a40b1b2..35d8bcd 100644 --- a/lib/row.h +++ b/lib/row.h @@ -1,7 +1,7 @@ #ifndef MYGRATE_ROW_H #define MYGRATE_ROW_H -#include "mysql_types.h" +#include "dbTypes.h" #include #include namespace MyGrate { @@ -11,7 +11,7 @@ struct st_mariadb_rpl_rows_event; struct st_mariadb_rpl_table_map_event; namespace MyGrate { - class Row : public std::vector { + class Row : public std::vector { public: Row(const st_mariadb_rpl_rows_event &, const st_mariadb_rpl_table_map_event &); diff --git a/lib/streamSupport.cpp b/lib/streamSupport.cpp index acf6f7e..32c1fce 100644 --- a/lib/streamSupport.cpp +++ b/lib/streamSupport.cpp @@ -1,7 +1,7 @@ #include "streamSupport.h" #include "bitset.h" #include "compileTimeFormatter.h" -#include "mysql_types.h" +#include "dbTypes.h" #include #include #include @@ -35,21 +35,21 @@ namespace std { } std::ostream & - operator<<(std::ostream & s, const MyGrate::MySQL::Date & d) + operator<<(std::ostream & s, const MyGrate::Date & d) { return AdHoc::scprintf<"%04d-%02d-%02d">(s, d.year, d.month, d.day); } std::ostream & - operator<<(std::ostream & s, const MyGrate::MySQL::Time & t) + operator<<(std::ostream & s, const MyGrate::Time & t) { return AdHoc::scprintf<"%02d:%02d:%02d">(s, t.hour, t.minute, t.second); } std::ostream & - operator<<(std::ostream & s, const MyGrate::MySQL::DateTime & dt) + operator<<(std::ostream & s, const MyGrate::DateTime & dt) { - return AdHoc::scprintf<"%? %?">(s, (const MyGrate::MySQL::Date)dt, (const MyGrate::MySQL::Time)dt); + return AdHoc::scprintf<"%? %?">(s, (const MyGrate::Date)dt, (const MyGrate::Time)dt); } std::ostream & diff --git a/lib/streamSupport.h b/lib/streamSupport.h index 60e68d3..cab62e6 100644 --- a/lib/streamSupport.h +++ b/lib/streamSupport.h @@ -11,14 +11,8 @@ #include namespace MyGrate { class BitSet; -} -namespace MyGrate::MySQL { struct Date; -} -namespace MyGrate::MySQL { struct DateTime; -} -namespace MyGrate::MySQL { struct Time; } struct timespec; @@ -33,11 +27,11 @@ namespace std { std::ostream & operator<<(std::ostream & s, const timespec & ts); - std::ostream & operator<<(std::ostream & s, const MyGrate::MySQL::Date & d); + std::ostream & operator<<(std::ostream & s, const MyGrate::Date & d); - std::ostream & operator<<(std::ostream & s, const MyGrate::MySQL::Time & t); + std::ostream & operator<<(std::ostream & s, const MyGrate::Time & t); - std::ostream & operator<<(std::ostream & s, const MyGrate::MySQL::DateTime & dt); + std::ostream & operator<<(std::ostream & s, const MyGrate::DateTime & dt); std::ostream & operator<<(std::ostream & s, const MyGrate::BitSet & bs); diff --git a/test/Jamfile.jam b/test/Jamfile.jam index 3c479b9..2cdf94c 100644 --- a/test/Jamfile.jam +++ b/test/Jamfile.jam @@ -10,3 +10,5 @@ run test-rawDataReader.cpp ; run test-bitset.cpp ; run test-streams.cpp ; run test-misc.cpp ; +run test-mysql.cpp ; +run test-postgresql.cpp ; diff --git a/test/test-mysql.cpp b/test/test-mysql.cpp new file mode 100644 index 0000000..5a22567 --- /dev/null +++ b/test/test-mysql.cpp @@ -0,0 +1,25 @@ +#define BOOST_TEST_MODULE MySQL +#include + +#include +#include +#include + +BOOST_AUTO_TEST_CASE(simple) +{ + BOOST_CHECK_THROW(([]() { + MyGrate::Input::MySQLConn {"192.168.1.38", "repl", "repl", 3306}; + }()), + std::runtime_error); + MyGrate::Input::MySQLConn c {"192.168.1.38", "repl", "r3pl", 3306}; + BOOST_CHECK_NO_THROW(c.query("SET @var = ''")); + BOOST_CHECK_NO_THROW(c.query("SET @var = 'something'")); + BOOST_CHECK_THROW(c.query("SET @var = "), std::runtime_error); + BOOST_CHECK_THROW(c.query("SET @var = ?", {}), std::logic_error); + BOOST_CHECK_NO_THROW(c.query("SET @var = ''", {})); + BOOST_CHECK_NO_THROW(c.query("SET @var = ?", {1})); + BOOST_CHECK_NO_THROW(c.query("SET @var = ?", {"string_view"})); + BOOST_CHECK_NO_THROW(c.query("SET @var = ?", {nullptr})); + BOOST_CHECK_NO_THROW(c.query("SET @var = ?", {1.2})); + BOOST_CHECK_THROW(c.query("SET @var = ?", {MyGrate::Time {}}), std::runtime_error); +} diff --git a/test/test-postgresql.cpp b/test/test-postgresql.cpp new file mode 100644 index 0000000..5940b38 --- /dev/null +++ b/test/test-postgresql.cpp @@ -0,0 +1,28 @@ +#define BOOST_TEST_MODULE PostgreSQL +#include + +#include +#include +#include + +BOOST_AUTO_TEST_CASE(simple) +{ + BOOST_CHECK_THROW(([]() { + MyGrate::Output::Pq::PqConn {"nonsense"}; + }()), + std::runtime_error); + MyGrate::Output::Pq::PqConn c {"user=postgres"}; + BOOST_CHECK_NO_THROW(c.query("SET application_name = ''")); + BOOST_CHECK_NO_THROW(c.query("SET application_name = 'something'")); + BOOST_CHECK_THROW(c.query("SET application_name = "), std::runtime_error); + // BOOST_CHECK_THROW(c.query("SET application_name = $1", {}), std::logic_error); + BOOST_CHECK_NO_THROW(c.query("SET application_name = 'something'", {})); + c.query("DROP TABLE IF EXISTS test"); + c.query("CREATE TABLE test(c text)"); + BOOST_CHECK_NO_THROW(c.query("INSERT INTO test VALUES($1)", {1})); + BOOST_CHECK_NO_THROW(c.query("INSERT INTO test VALUES($1)", {"string_view"})); + BOOST_CHECK_NO_THROW(c.query("INSERT INTO test VALUES($1)", {nullptr})); + BOOST_CHECK_NO_THROW(c.query("INSERT INTO test VALUES($1)", {1.2})); + BOOST_CHECK_THROW(c.query("INSERT INTO test VALUES($1)", {MyGrate::Time {}}), std::runtime_error); + c.query("DROP TABLE test"); +} diff --git a/test/test-rawDataReader.cpp b/test/test-rawDataReader.cpp index b211fe3..3d34bff 100644 --- a/test/test-rawDataReader.cpp +++ b/test/test-rawDataReader.cpp @@ -4,12 +4,12 @@ #include #include "bitset.h" +#include "dbTypes.h" #include "helpers.h" #include "mariadb_repl.h" #include #include #include -#include #include #include #include @@ -125,7 +125,7 @@ BOOST_DATA_TEST_CASE(read_bytes, bytes) { RawDataReader rdr {bytes.data(), bytes.size()}; - const auto out {rdr.viewValue(bytes.size())}; + const auto out {rdr.viewValue(bytes.size())}; BOOST_CHECK_EQUAL_COLLECTIONS(out.begin(), out.end(), bytes.begin(), bytes.end()); } @@ -140,7 +140,7 @@ BOOST_DATA_TEST_CASE(read_bytes_overflow, bytes) { RawDataReader rdr {bytes.data(), bytes.size()}; - BOOST_CHECK_THROW(rdr.viewValue(bytes.size() + 1), std::range_error); + BOOST_CHECK_THROW(rdr.viewValue(bytes.size() + 1), std::range_error); } BOOST_DATA_TEST_CASE(read_field_type, diff --git a/test/test-streams.cpp b/test/test-streams.cpp index 3958e36..a4ff1a5 100644 --- a/test/test-streams.cpp +++ b/test/test-streams.cpp @@ -3,9 +3,9 @@ #include #include "bitset.h" +#include "dbTypes.h" #include "helpers.h" #include "mariadb_repl.h" -#include "mysql_types.h" #include #include #include @@ -46,7 +46,7 @@ BOOST_DATA_TEST_CASE(tms, } BOOST_DATA_TEST_CASE(rts, - boost::unit_test::data::make>({ + boost::unit_test::data::make>({ {{{2016, 1, 4}, {12, 13, 14}}, "2016-01-04 12:13:14"}, {{{2016, 12, 31}, {0, 0, 1}}, "2016-12-31 00:00:01"}, }), -- cgit v1.2.3