From fcdca58617caf6a8c034a91588d6abb399be6b57 Mon Sep 17 00:00:00 2001 From: Dan Goodliffe Date: Tue, 18 May 2021 00:06:37 +0100 Subject: Initial commit, still lots to do! --- .gitignore | 1 + Jamroot.jam | 38 ++++ iwyu.json | 58 ++++++ lib/Jamfile.jam | 9 + lib/bitset.cpp | 66 ++++++ lib/bitset.h | 45 ++++ lib/compileTimeFormatter.h | 495 ++++++++++++++++++++++++++++++++++++++++++++ lib/helpers.h | 35 ++++ lib/mysql_types.cpp | 212 +++++++++++++++++++ lib/mysql_types.h | 87 ++++++++ lib/rawDataReader.cpp | 47 +++++ lib/rawDataReader.h | 76 +++++++ lib/row.cpp | 101 +++++++++ lib/row.h | 21 ++ lib/streamSupport.cpp | 53 +++++ lib/streamSupport.h | 68 ++++++ main/Jamfile.jam | 2 + main/main.cpp | 195 +++++++++++++++++ test/Jamfile.jam | 11 + test/helpers.h | 44 ++++ test/test-bitset.cpp | 31 +++ test/test-rawDataReader.cpp | 163 +++++++++++++++ test/test-streams.cpp | 92 ++++++++ 23 files changed, 1950 insertions(+) create mode 100644 .gitignore create mode 100644 Jamroot.jam create mode 100644 iwyu.json create mode 100644 lib/Jamfile.jam create mode 100644 lib/bitset.cpp create mode 100644 lib/bitset.h create mode 100644 lib/compileTimeFormatter.h create mode 100644 lib/helpers.h create mode 100644 lib/mysql_types.cpp create mode 100644 lib/mysql_types.h create mode 100644 lib/rawDataReader.cpp create mode 100644 lib/rawDataReader.h create mode 100644 lib/row.cpp create mode 100644 lib/row.h create mode 100644 lib/streamSupport.cpp create mode 100644 lib/streamSupport.h create mode 100644 main/Jamfile.jam create mode 100644 main/main.cpp create mode 100644 test/Jamfile.jam create mode 100644 test/helpers.h create mode 100644 test/test-bitset.cpp create mode 100644 test/test-rawDataReader.cpp create mode 100644 test/test-streams.cpp diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ba077a4 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +bin diff --git a/Jamroot.jam b/Jamroot.jam new file mode 100644 index 0000000..7a18a1e --- /dev/null +++ b/Jamroot.jam @@ -0,0 +1,38 @@ +using gcc ; + +using pkg-config ; +import pkg-config ; +import testing ; + +pkg-config.import libmariadb ; +pkg-config.import libpq ; + +variant coverage : debug ; + +project : requirements + 20 + hidden + "-Wl,-z,defs,--warn-once,--gc-sections" + release:on + debug:extra + debug:on + coverage:on + tidy:boost-* + tidy:bugprone-* + tidy:bugprone-macro-parentheses + tidy:clang-* + tidy:misc-* + tidy:misc-non-private-member-variables-in-classes + tidy:modernize-* + tidy:modernize-use-trailing-return-type + tidy:hicpp-* + tidy:hicpp-vararg + tidy:hicpp-signed-bitwise + tidy:hicpp-named-parameter + tidy:hicpp-no-array-decay + tidy:performance-* + tidy:iwyu.json + ; + +build-project main ; +build-project test ; diff --git a/iwyu.json b/iwyu.json new file mode 100644 index 0000000..5785628 --- /dev/null +++ b/iwyu.json @@ -0,0 +1,58 @@ +[ + { + "symbol": [ + "tm", + "private", + "", + "public" + ] + }, + { + "include": [ + "", + "private", + "", + "public" + ] + }, + { + "include": [ + "", + "private", + "", + "public" + ] + }, + { + "include": [ + "@", + "private", + "", + "public" + ] + }, + { + "include": [ + "@", + "private", + "", + "public" + ] + }, + { + "include": [ + "@", + "private", + "", + "public" + ] + }, + { + "include": [ + "@", + "private", + "", + "public" + ] + } +] diff --git a/lib/Jamfile.jam b/lib/Jamfile.jam new file mode 100644 index 0000000..6a31bec --- /dev/null +++ b/lib/Jamfile.jam @@ -0,0 +1,9 @@ +lib mygrate : + [ glob *.cpp ] + : + static + ..//libmariadb + : : + . + ..//libmariadb + ; diff --git a/lib/bitset.cpp b/lib/bitset.cpp new file mode 100644 index 0000000..bbd802d --- /dev/null +++ b/lib/bitset.cpp @@ -0,0 +1,66 @@ +#include "bitset.h" + +namespace MyGrate { + static_assert(std::forward_iterator); + + BitSet::Iterator::Iterator(const std::byte * bb) : b {bb} { } + + bool + BitSet::Iterator::operator*() const + { + return (*b & m) != std::byte {}; + } + + bool + BitSet::Iterator::operator==(const BitSet::Iterator & o) const + { + return b == o.b && m == o.m; + } + + bool + BitSet::Iterator::operator!=(const BitSet::Iterator & o) const + { + return b != o.b || m != o.m; + } + + BitSet::Iterator & + BitSet::Iterator::operator++() + { + if (m == std::byte {128}) { + b++; + m = std::byte {1}; + } + else { + m <<= 1; + } + return *this; + } + + BitSet::Iterator + BitSet::Iterator::operator++(int) + { + auto pre {*this}; + ++*this; + return pre; + } + + BitSet::BitSet(const std::span bs) : bytes {bs} { } + + std::size_t + BitSet::size() const + { + return bytes.size() * 8; + } + + BitSet::Iterator + BitSet::begin() const + { + return BitSet::Iterator {bytes.data()}; + } + + BitSet::Iterator + BitSet::end() const + { + return BitSet::Iterator {bytes.data() + bytes.size()}; + } +} diff --git a/lib/bitset.h b/lib/bitset.h new file mode 100644 index 0000000..40ee3ed --- /dev/null +++ b/lib/bitset.h @@ -0,0 +1,45 @@ +#ifndef MYGRATE_BITSET_H +#define MYGRATE_BITSET_H + +#include +#include +#include + +namespace MyGrate { + class BitSet { + public: + class Iterator { + public: + using iterator_category = std::forward_iterator_tag; + using difference_type = int; + using value_type = bool; + using pointer = bool *; + using reference = bool; + + explicit Iterator(); + explicit Iterator(const std::byte *); + + bool operator*() const; + bool operator!=(const Iterator &) const; + bool operator==(const Iterator &) const; + + Iterator & operator++(); + Iterator operator++(int); + + private: + const std::byte * b; + std::byte m {1}; + }; + + explicit BitSet(const std::span b); + + std::size_t size() const; + Iterator begin() const; + Iterator end() const; + + private: + const std::span bytes; + }; +} + +#endif diff --git a/lib/compileTimeFormatter.h b/lib/compileTimeFormatter.h new file mode 100644 index 0000000..6e8c813 --- /dev/null +++ b/lib/compileTimeFormatter.h @@ -0,0 +1,495 @@ +#ifndef ADHOCUTIL_COMPILE_TIME_FORMATTER_H +#define ADHOCUTIL_COMPILE_TIME_FORMATTER_H + +#include +#include +#include +#include +#include +#include // IWYU pragma: export + +namespace AdHoc { + // Template char utils + template + constexpr bool + isdigit(const char_type & ch) + { + return (ch >= '0' && ch <= '9'); + } + + template + constexpr bool + ispositivedigit(const char_type & ch) + { + return (ch >= '1' && ch <= '9'); + } + + // Template string utils + template + static constexpr auto + strlen() + { + auto off = 0U; + while (S[off]) { + ++off; + } + return off; + } + + template + static constexpr auto + strlen(const char_type * S) + { + auto off = 0U; + while (S[off]) { + ++off; + } + return off; + } + + template()> + static constexpr std::optional + strchr() + { + static_assert(start <= L); + decltype(start) off = start; + while (off < L && S[off] != n) { + ++off; + } + if (off == L) { + return {}; + } + return off; + } + + template()> + static constexpr decltype(L) + strchrnul() + { + decltype(start) off = start; + while (off < L && S[off] != n) { + ++off; + } + return off; + } + + template class FormatterDetail; + + /// Template used to apply parameters to a stream. + template struct StreamWriter { + /// Write parameters to stream. + template + static void + write(stream &, const Pn &...) + { + static_assert(!L, "invalid format string/arguments"); + } + }; + + /// Helper to simplify implementations of StreamWriter. + template struct StreamWriterBase { + /// Continue processing parameters. + template + static inline void + next(stream & s, const Pn &... pn) + { + FormatterDetail::template Parser::run(s, pn...); + } + }; + +#define StreamWriterT(C...) \ + template \ + struct StreamWriter : \ + public StreamWriterBase + +#define StreamWriterTP(P, C...) \ + template \ + struct StreamWriter : \ + public StreamWriterBase + + // Default stream writer formatter + StreamWriterT('?') { + template + static inline void + write(stream & s, const P & p, const Pn &... pn) + { + s << p; + StreamWriter::next(s, pn...); + } + }; + + // Escaped % stream writer formatter + StreamWriterT('%') { + template + static inline void + write(stream & s, const Pn &... pn) + { + s << '%'; + StreamWriter::next(s, pn...); + } + }; + + template + static inline void + appendStream(stream & s, const char_type * p, size_t n) + { + s.write(p, n); + } + + template + static inline auto + streamLength(stream & s) + { + return s.tellp(); + } + + /** + * Compile time string formatter. + * @param S the format string. + */ + template class FormatterDetail { + private: + using strlen_t = decltype(strlen()); + template friend struct StreamWriterBase; + + public: + /// The derived charater type of the format string. + using char_type = typename std::decay::type; + /** + * Get a string containing the result of formatting. + * @param pn the format arguments. + * @return the formatted string. + */ + template + static inline auto + get(const Pn &... pn) + { + std::basic_stringstream s; + return write(s, pn...).str(); + } + /** + * Get a string containing the result of formatting. + * @param pn the format arguments. + * @return the formatted string. + */ + template + inline auto + operator()(const Pn &... pn) const + { + return get(pn...); + } + + /** + * Write the result of formatting to the given stream. + * @param s the stream to write to. + * @param pn the format arguments. + * @return the stream. + */ + template + static inline stream & + write(stream & s, const Pn &... pn) + { + return Parser::run(s, pn...); + } + /** + * Write the result of formatting to the given stream. + * @param s the stream to write to. + * @param pn the format arguments. + * @return the stream. + */ + template + inline typename std::enable_if, stream>, stream>::type & + operator()(stream & s, const Pn &... pn) const + { + return write(s, pn...); + } + + private: + template struct Parser { + static inline stream & + run(stream & s, const Pn &... pn) + { + if (pos != L) { + constexpr auto ph = strchrnul(); + if constexpr (ph != pos) { + appendStream(s, &S[pos], ph - pos); + } + if constexpr (ph != L) { + packAndWrite(s, pn...); + } + } + return s; + } + template + static inline void + packAndWrite(stream & s, const Pn &... pn) + { + if constexpr (ph + off == L || sizeof...(Pck) == 32) { + StreamWriter::write(s, pn...); + } + else if constexpr (ph + off < L) { + packAndWrite(s, pn...); + } + } + }; + }; + + // New C++20 implementation + namespace support { + template class basic_fixed_string : public std::array { + public: + // cppcheck-suppress noExplicitConstructor + constexpr basic_fixed_string(const CharT (&str)[N + 1]) + { + for (decltype(N) x = 0; x < N; x++) { + this->at(x) = str[x]; + } + } + constexpr basic_fixed_string(const CharT * str, decltype(N) len) + { + for (decltype(N) x = 0; x < len; x++) { + this->at(x) = str[x]; + } + } + }; + + template + basic_fixed_string(const CharT (&str)[N]) -> basic_fixed_string; + } + + template class LiteralFormatter : public FormatterDetail { + }; + + template + class Formatter : + public FormatterDetail::type, L>(S, L), L> { + }; + +#define AdHocFormatter(name, str) using name = ::AdHoc::LiteralFormatter + + template + inline auto + scprintf(const Pn &... pn) + { + return FormatterDetail::get(pn...); + } + + template + inline auto & + scprintf(stream & strm, const Pn &... pn) + { + return FormatterDetail::write(strm, pn...); + } + + template + inline auto & + cprintf(const Pn &... pn) + { + return scprintf(std::cout, pn...); + } +} + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AdHoc { +#define BASICCONV(PARAMTYPE, OP, ...) \ + StreamWriterT(__VA_ARGS__) { \ + template \ + static inline void \ + write(stream & s, const PARAMTYPE & p, const Pn &... pn) \ + { \ + OP; \ + s.copyfmt(std::ios(nullptr)); \ + StreamWriter::next(s, pn...); \ + } \ + } + + // Integers (d, i, o, u, x, X) +#define INTCONV(BASE, OP, CONV) \ + BASICCONV(BASE, OP, CONV); \ + BASICCONV(short BASE, OP, 'h', CONV); \ + BASICCONV(long BASE, OP, 'l', CONV); \ + BASICCONV(long long BASE, OP, 'l', 'l', CONV); + INTCONV(int, s << std::dec << p, 'i'); + INTCONV(int, s << std::dec << p, 'd'); + INTCONV(unsigned int, s << std::oct << p, 'o'); + INTCONV(unsigned int, s << std::dec << p, 'u'); + INTCONV(unsigned int, s << std::nouppercase << std::hex << p, 'x'); + INTCONV(unsigned int, s << std::uppercase << std::hex << p, 'X'); +#undef INTCONV + + BASICCONV(intmax_t, s << std::dec << p, 'j', 'd'); + BASICCONV(uintmax_t, s << std::dec << p, 'j', 'u'); + BASICCONV(ssize_t, s << std::dec << p, 'z', 'd'); + BASICCONV(size_t, s << std::dec << p, 'z', 'u'); + BASICCONV(short int, s << std::dec << p, 'h', 'h', 'i'); // char + BASICCONV(short int, s << std::dec << p, 'h', 'h', 'd'); // char + BASICCONV(unsigned char, s << std::dec << p, 'h', 'h', 'u'); + BASICCONV(unsigned char, s << std::oct << p, 'h', 'h', 'o'); + BASICCONV(unsigned char, s << std::nouppercase << std::hex << p, 'h', 'h', 'x'); + BASICCONV(unsigned char, s << std::uppercase << std::hex << p, 'h', 'h', 'X'); + + // Floating point (a, A, e, E, f, F, g, G) +#define FPCONV(BASE, OP, CONV) \ + BASICCONV(BASE, OP, CONV); \ + BASICCONV(long BASE, OP, 'L', CONV); + FPCONV(double, s << std::nouppercase << std::hexfloat << p, 'a'); + FPCONV(double, s << std::uppercase << std::hexfloat << p, 'A'); + FPCONV(double, s << std::nouppercase << std::scientific << p, 'e'); + FPCONV(double, s << std::uppercase << std::scientific << p, 'E'); + FPCONV(double, s << std::nouppercase << std::fixed << p, 'f'); + FPCONV(double, s << std::uppercase << std::fixed << p, 'F'); + FPCONV(double, s << std::nouppercase << std::defaultfloat << p, 'g'); + FPCONV(double, s << std::uppercase << std::defaultfloat << p, 'G'); +#undef FPCONV + + BASICCONV(std::string_view, s << p, 's'); + BASICCONV(std::wstring_view, s << p, 'l', 's'); + BASICCONV(char, s << p, 'c'); + BASICCONV(wchar_t, s << p, 'l', 'c'); +#undef BASICCONV + StreamWriterT('p') { + template + static inline void + write(stream & s, Obj * const ptr, const Pn &... pn) + { + s << std::showbase << std::hex << (long unsigned int)ptr; + s.copyfmt(std::ios(nullptr)); + StreamWriter::next(s, pn...); + } + template + static inline void + write(stream & s, const Ptr & ptr, const Pn &... pn) + { + write(s, ptr.get(), pn...); + } + }; + + StreamWriterT('m') { + template + static inline void + write(stream & s, const Pn &... pn) + { + s << strerror(errno); + s.copyfmt(std::ios(nullptr)); + StreamWriter::next(s, pn...); + } + }; + StreamWriterT('n') { + template + static inline void + write(stream & s, int * n, const Pn &... pn) + { + BOOST_ASSERT_MSG(n, "%n conversion requires non-null parameter"); + *n = streamLength(s); + s.copyfmt(std::ios(nullptr)); + StreamWriter::next(s, pn...); + } + }; + + //// + // Width/precision embedded in format string + template + constexpr auto + decdigits() + { + static_assert((isdigit(chs) && ... && true)); + int n = 0; + ( + [&n](auto ch) { + n = (n * 10) + (ch - '0'); + }(chs), + ...); + return n; + } +#define AUTON(z, n, data) \ + BOOST_PP_COMMA_IF(n) \ + auto BOOST_PP_CAT(data, n) +#define NS(z, n, data) \ + BOOST_PP_COMMA_IF(n) \ + BOOST_PP_CAT(data, n) +#define ISDIGIT(z, n, data) &&isdigit(BOOST_PP_CAT(data, BOOST_PP_ADD(n, 1))) +#define FMTWIDTH(unused, d, data) \ + template \ + struct StreamWriter::type, '%', \ + BOOST_PP_REPEAT(BOOST_PP_ADD(d, 1), NS, n), nn, sn...> { \ + template \ + static inline void \ + write(stream & s, const Pn &... pn) \ + { \ + constexpr auto p = decdigits(); \ + s << std::setw(p); \ + StreamWriter::write(s, pn...); \ + } \ + }; + BOOST_PP_REPEAT(6, FMTWIDTH, void); +#define FMTPRECISION(unused, d, data) \ + template \ + struct StreamWriter::type, '%', '.', \ + BOOST_PP_REPEAT(BOOST_PP_ADD(d, 1), NS, n), nn, sn...> { \ + template \ + static inline void \ + write(stream & s, const Pn &... pn) \ + { \ + constexpr auto p = decdigits(); \ + s << std::setprecision(p); \ + StreamWriter::write(s, pn...); \ + } \ + }; + BOOST_PP_REPEAT(6, FMTPRECISION, void); +#undef AUTON +#undef NS +#undef ISDIGIT +#undef FMTWIDTH + + StreamWriterT('.', '*') { + template + static inline void + write(stream & s, int l, const Pn &... pn) + { + s << std::setw(l); + StreamWriter::write(s, pn...); + } + }; + StreamWriterT('.', '*', 's') { + template + static inline void + write(stream & s, int l, const std::string_view & p, const Pn &... pn) + { + s << p.substr(0, l); + s.copyfmt(std::ios(nullptr)); + StreamWriter::next(s, pn...); + } + }; + + // Flags +#define FLAGCONV(OP, ...) \ + StreamWriterT(__VA_ARGS__) { \ + template \ + static inline void \ + write(stream & s, const Pn &... pn) \ + { \ + OP; \ + StreamWriter::write(s, pn...); \ + } \ + }; + FLAGCONV(s << std::showbase, '#'); + FLAGCONV(s << std::setfill('0'), '0'); + FLAGCONV(s << std::left, '-'); + FLAGCONV(s << std::showpos, '+'); + FLAGCONV(s << std::setfill(' '), ' '); +#undef FLAGCONV +} + +#endif diff --git a/lib/helpers.h b/lib/helpers.h new file mode 100644 index 0000000..3a94ead --- /dev/null +++ b/lib/helpers.h @@ -0,0 +1,35 @@ +#ifndef MYGRATE_HELPERS_H +#define MYGRATE_HELPERS_H + +#include +#include +#include + +namespace MyGrate { + template + constexpr inline auto + bitslice(const I i, uint8_t offset, uint8_t size) + { + return (i >> offset) & ((1U << size) - 1U); + } + + template + constexpr inline void + verify(bool expr, P &&... p) + { + if (!expr) { + throw X(std::forward(p)...); + } + } + + template + constexpr inline auto + mod100_extract(T & i) + { + const auto r {i % 100}; + i /= 100; + return r; + } +} + +#endif diff --git a/lib/mysql_types.cpp b/lib/mysql_types.cpp new file mode 100644 index 0000000..fa4698d --- /dev/null +++ b/lib/mysql_types.cpp @@ -0,0 +1,212 @@ +#include "mysql_types.h" +#include "helpers.h" +#include "rawDataReader.h" +#include +#include + +namespace MyGrate::MySQL { + typename Type::C + Type::read(RawDataReader &, RawDataReader &) + { + return nullptr; + } + +#define INTEGER_TYPE(ET) \ + typename Type::C Type::read(RawDataReader &, RawDataReader & data) \ + { \ + return data.readValue::C>(); \ + } \ + typename Type::C Type::read(RawDataReader &, RawDataReader & data) \ + { \ + return data.readValue::C>(); \ + } + INTEGER_TYPE(MYSQL_TYPE_TINY); + INTEGER_TYPE(MYSQL_TYPE_SHORT); + INTEGER_TYPE(MYSQL_TYPE_LONG); + INTEGER_TYPE(MYSQL_TYPE_LONGLONG); +#undef INTEGER_TYPE + +#define FLOAT_TYPE(ET) \ + typename Type::C Type::read(RawDataReader & md, RawDataReader & data) \ + { \ + verify(sizeof(typename Type::C) == md.readValue(), "Invalid " #ET " size"); \ + return data.readValue::C>(); \ + } + FLOAT_TYPE(MYSQL_TYPE_FLOAT); + FLOAT_TYPE(MYSQL_TYPE_DOUBLE); +#undef FLOAT_TYPE + + typename Type::C + Type::read(RawDataReader &, RawDataReader &) + { + throw std::logic_error("Not implemented"); + } + + typename Type::C + Type::read(RawDataReader &, RawDataReader &) + { + throw std::logic_error("Not implemented"); + } + +#define INTEGER_TYPE(ET, s, l) \ + typename Type::C Type::read(RawDataReader &, RawDataReader & data) \ + { \ + return data.readValue::C, l>(); \ + } + INTEGER_TYPE(MYSQL_TYPE_INT24, false, 3); + INTEGER_TYPE(MYSQL_TYPE_INT24, true, 3); + INTEGER_TYPE(MYSQL_TYPE_YEAR, false, 2); +#undef INTEGER24_TYPE + + static Blob + readBlob(RawDataReader & md, RawDataReader & data) + { + const auto lenlen {md.readValue()}; + const auto len {data.readValue(lenlen)}; + return data.viewValue(len); + } +#define BLOB_TYPE(ET) \ + typename Type::C Type::read(RawDataReader & md, RawDataReader & data) \ + { \ + return readBlob(md, data); \ + } + BLOB_TYPE(MYSQL_TYPE_TINY_BLOB); + BLOB_TYPE(MYSQL_TYPE_MEDIUM_BLOB); + BLOB_TYPE(MYSQL_TYPE_LONG_BLOB); + BLOB_TYPE(MYSQL_TYPE_GEOMETRY); // Ummm, pass this to the target to handle? +#undef BLOB_TYPE + + typename Type::C + Type::read(RawDataReader & md, RawDataReader & data) + { + const auto lenlen {md.readValue()}; + const auto len {data.readValue(lenlen)}; + return data.viewValue(len); + } + + typename Type::C + Type::read(RawDataReader &, RawDataReader &) + { + throw std::logic_error("Not implemented"); + } + +#define STRING_TYPE(ET) \ + typename Type::C Type::read(RawDataReader & md, RawDataReader & data) \ + { \ + const auto realtype {md.readValue()}; \ + const auto lenlen {md.readValue()}; \ + switch (realtype) { \ + case MYSQL_TYPE_ENUM: { \ + return data.viewValue(lenlen); \ + break; \ + } \ + default: \ + throw std::logic_error("Not implemented: sub-type: " + std::to_string(realtype)); \ + break; \ + } \ + throw std::logic_error("Didn't return a value: " + std::to_string(realtype)); \ + } + STRING_TYPE(MYSQL_TYPE_STRING); + STRING_TYPE(MYSQL_TYPE_JSON); + STRING_TYPE(MYSQL_TYPE_SET); + STRING_TYPE(MYSQL_TYPE_ENUM); +#undef STRING_TYPE + + typename Type::C + Type::read(RawDataReader & md, RawDataReader & data) + { + const auto fieldlen {md.readValue()}; + const auto len {data.readValue(fieldlen > 255 ? 2 : 1)}; + return data.viewValue(len); + } + + typename Type::C + Type::read(RawDataReader & md, RawDataReader & data) + { + const auto fieldlen {md.readValue()}; + const auto len {md.readValue(fieldlen > 255 ? 2 : 1)}; + return data.viewValue(len); + } + + typename Type::C + Type::read(RawDataReader &, RawDataReader & data) + { + auto dtint {data.readValue()}; + DateTime dt; + dt.second = mod100_extract(dtint); + dt.minute = mod100_extract(dtint); + dt.hour = mod100_extract(dtint); + dt.day = mod100_extract(dtint); + dt.month = mod100_extract(dtint); + dt.year = dtint; + return dt; + } + + typename Type::C + Type::read(RawDataReader &, RawDataReader & data) + { + auto tint {data.readValue()}; + Time t; + t.second = mod100_extract(tint); + t.minute = mod100_extract(tint); + t.hour = tint; + return t; + } + + typename Type::C + Type::read(RawDataReader &, RawDataReader &) + { + throw std::logic_error("Not implemented"); + } + + typename Type::C + Type::read(RawDataReader &, RawDataReader & data) + { + auto dint {data.readValue()}; + Date d; + d.day = bitslice(dint, 0, 6); + d.month = bitslice(dint, 6, 4); + d.day = bitslice(dint, 10, 14); + return d; + } + + typename Type::C + Type::read(RawDataReader &, RawDataReader &) + { + throw std::logic_error("Not implemented"); + } + + typename Type::C + Type::read(RawDataReader &, RawDataReader & data) + { + return {data.readValue(), 0}; + } + + typename Type::C + Type::read(RawDataReader & md, RawDataReader & data) + { + const auto decimals {md.readValue()}; + return {data.readValue(), data.readValue(decimals)}; + } + + typename Type::C + Type::read(RawDataReader & md, RawDataReader & data) + { + md.discard(1); + union { + uint64_t i; + std::array b; + } r {}; + r.b = data.readValue(); + std::reverse(r.b.begin(), r.b.end()); + + DateTime dt; + dt.year = (bitslice(r.i, 22, 17) / 13); + dt.month = (bitslice(r.i, 22, 17) % 13); + dt.day = bitslice(r.i, 17, 5); + dt.hour = bitslice(r.i, 12, 5); + dt.minute = bitslice(r.i, 6, 6); + dt.second = bitslice(r.i, 0, 6); + return dt; + } +} diff --git a/lib/mysql_types.h b/lib/mysql_types.h new file mode 100644 index 0000000..ec578f7 --- /dev/null +++ b/lib/mysql_types.h @@ -0,0 +1,87 @@ +#ifndef MYGRATE_MYSQL_TYPES_H +#define MYGRATE_MYSQL_TYPES_H + +#include "bitset.h" +#include +#include +#include +#include +#include // IWYU pragma: keep +#include +#include +#include + +#include + +namespace MyGrate { + class RawDataReader; + namespace MySQL { + struct Date { + uint16_t year; + uint8_t month; + uint8_t day; + }; + struct Time { + uint8_t hour; + uint8_t minute; + uint8_t second; + }; + struct DateTime : public Date, public Time { + }; + using Blob = std::span; + + template struct Type; + +#define DEFINE_BASE_TYPE(ET, CT, MDS, SIGN) \ + template<> struct Type { \ + using C = CT; \ + constexpr static size_t md_bytes {MDS}; \ + static C read(RawDataReader & md, RawDataReader & data); \ + } +#define DEFINE_TYPE(ET, CT, MDS) DEFINE_BASE_TYPE(ET, CT, MDS, false) +#define DEFINE_ITYPE(ET, BCT, MDS) \ + DEFINE_BASE_TYPE(ET, BCT, MDS, false); \ + DEFINE_BASE_TYPE(ET, u##BCT, MDS, true) + + DEFINE_TYPE(MYSQL_TYPE_DECIMAL, double, 2); + DEFINE_ITYPE(MYSQL_TYPE_TINY, int8_t, 0); + DEFINE_ITYPE(MYSQL_TYPE_SHORT, int16_t, 0); + DEFINE_ITYPE(MYSQL_TYPE_LONG, int32_t, 0); + DEFINE_TYPE(MYSQL_TYPE_FLOAT, float, 1); + DEFINE_TYPE(MYSQL_TYPE_DOUBLE, double, 1); + DEFINE_TYPE(MYSQL_TYPE_NULL, std::nullptr_t, 0); + DEFINE_TYPE(MYSQL_TYPE_TIMESTAMP, timespec, 0); + DEFINE_TYPE(MYSQL_TYPE_TIMESTAMP2, timespec, 1); + DEFINE_ITYPE(MYSQL_TYPE_LONGLONG, int64_t, 0); + DEFINE_ITYPE(MYSQL_TYPE_INT24, int32_t, 0); + DEFINE_TYPE(MYSQL_TYPE_DATE, Date, 0); + DEFINE_TYPE(MYSQL_TYPE_TIME, Time, 0); + DEFINE_TYPE(MYSQL_TYPE_TIME2, Time, 0); + DEFINE_TYPE(MYSQL_TYPE_DATETIME, DateTime, 0); + DEFINE_TYPE(MYSQL_TYPE_DATETIME2, DateTime, 1); + DEFINE_TYPE(MYSQL_TYPE_YEAR, int16_t, 0); + DEFINE_TYPE(MYSQL_TYPE_NEWDATE, Date, 0); + DEFINE_TYPE(MYSQL_TYPE_VARCHAR, std::string_view, 2); + DEFINE_TYPE(MYSQL_TYPE_BIT, BitSet, 2); + DEFINE_TYPE(MYSQL_TYPE_NEWDECIMAL, double, 2); + DEFINE_TYPE(MYSQL_TYPE_ENUM, std::string_view, 0); + DEFINE_TYPE(MYSQL_TYPE_SET, std::string_view, 0); + DEFINE_TYPE(MYSQL_TYPE_TINY_BLOB, Blob, 0); + DEFINE_TYPE(MYSQL_TYPE_MEDIUM_BLOB, Blob, 0); + DEFINE_TYPE(MYSQL_TYPE_LONG_BLOB, Blob, 0); + DEFINE_TYPE(MYSQL_TYPE_BLOB, std::string_view, 1); + DEFINE_TYPE(MYSQL_TYPE_VAR_STRING, std::string_view, 2); + DEFINE_TYPE(MYSQL_TYPE_STRING, std::string_view, 2); + DEFINE_TYPE(MYSQL_TYPE_JSON, std::string_view, 2); + DEFINE_TYPE(MYSQL_TYPE_GEOMETRY, Blob, 1); + +#undef DEFINE_ITYPE +#undef DEFINE_USTYPE +#undef DEFINE_TYPE + + using FieldValue = std::variant; + } +} + +#endif diff --git a/lib/rawDataReader.cpp b/lib/rawDataReader.cpp new file mode 100644 index 0000000..2dd87a9 --- /dev/null +++ b/lib/rawDataReader.cpp @@ -0,0 +1,47 @@ +#include "rawDataReader.h" +#include "helpers.h" +#include "streamSupport.h" +#include +#include +#include + +namespace MyGrate { + RawDataReader::RawDataReader(const void * const d, std::size_t l) : + rawdata {static_cast(d)}, len {l} + { + } + + RawDataReader::RawDataReader(const MARIADB_STRING & str) : RawDataReader {str.str, str.length} { } + + void + RawDataReader::offsetSizeCheck(size_t l) const + { + if (offset + l > len) { + throw std::range_error("Read beyond end of data"); + } + } + + void + RawDataReader::discard(size_t d) + { + offset += d; + } + + template<> + uint64_t + RawDataReader::readValue() + { + switch (const auto byte1 {readValue()}) { + case 0 ... 250: // The value as-is + return byte1; + case 252: // 2 bytes + return readValue(); + case 253: // 3 bytes + return readValue(); + case 254: // 8 bytes + return readValue(); + default:; + } + throw std::domain_error("Invalid first byte of packed integer"); + } +} diff --git a/lib/rawDataReader.h b/lib/rawDataReader.h new file mode 100644 index 0000000..0ed2847 --- /dev/null +++ b/lib/rawDataReader.h @@ -0,0 +1,76 @@ +#ifndef MYGRATE_RAW_DATA_READER_H +#define MYGRATE_RAW_DATA_READER_H + +#include "helpers.h" +#include +#include +#include +#include +#include +#include // IWYU pragma: keep +#include +#include +#include + +#include + +namespace MyGrate { + struct PackedInteger { + }; + template struct type_map { + using target = T; + }; + template<> struct type_map { + using target = uint64_t; + }; + + class RawDataReader { + public: + explicit RawDataReader(const void * const d, std::size_t len); + explicit RawDataReader(const MARIADB_STRING & str); + + template + typename type_map::target + readValue() + { + static_assert(L > 0 && L <= sizeof(T), "Read exceeds target size"); + return readValue(L); + } + + template + typename type_map::target + readValue(size_t L) + { + verify(L > 0 && L <= sizeof(T), "Read exceeds target size"); + offsetSizeCheck(L); + T v {}; + memcpy(&v, rawdata + offset, L); + offset += L; + return v; + } + + template + T + viewValue(size_t L) + { + static_assert(sizeof(typename T::value_type) == sizeof(std::byte)); + offsetSizeCheck(L); + T v {reinterpret_cast(rawdata + offset), L}; + offset += L; + return v; + } + + void discard(size_t); + + private: + void offsetSizeCheck(size_t) const; + + const std::byte * const rawdata; + const std::size_t len; + std::size_t offset {0}; + }; + + template<> uint64_t RawDataReader::readValue(); +} + +#endif diff --git a/lib/row.cpp b/lib/row.cpp new file mode 100644 index 0000000..d59cdba --- /dev/null +++ b/lib/row.cpp @@ -0,0 +1,101 @@ +#include "row.h" +#include +#include + +namespace MyGrate { + Row::Row(const st_mariadb_rpl_rows_event & r, const st_mariadb_rpl_table_map_event & t) : row {r}, tm {t} { } + + void + Row::forEachField(const std::function & cb) + { + MyGrate::RawDataReader md {tm.metadata}; + MyGrate::RawDataReader data {row.row_data, row.row_data_size}; + const size_t flagBytes {(tm.column_count + 7) / 8}; + MyGrate::BitSet nullFlags {data.viewValue>(flagBytes)}; + MyGrate::BitSet columnFlags {{reinterpret_cast(row.column_bitmap), flagBytes}}; + auto nullIter {nullFlags.begin()}; + auto colIter {columnFlags.begin()}; + for (auto c {0U}; c < tm.column_count; c++) { + if (*colIter) { + const enum_field_types type {(enum_field_types)(unsigned char)tm.column_types.str[c]}; + const auto null {*nullIter}; + if (null) { + switch (type) { + case MYSQL_TYPE_BIT: + case MYSQL_TYPE_ENUM: + case MYSQL_TYPE_SET: + case MYSQL_TYPE_NEWDECIMAL: + case MYSQL_TYPE_DECIMAL: + case MYSQL_TYPE_VARCHAR: + case MYSQL_TYPE_VAR_STRING: + case MYSQL_TYPE_STRING: + md.discard(2); + break; + case MYSQL_TYPE_TINY_BLOB: + case MYSQL_TYPE_MEDIUM_BLOB: + case MYSQL_TYPE_LONG_BLOB: + case MYSQL_TYPE_BLOB: + case MYSQL_TYPE_FLOAT: + case MYSQL_TYPE_DOUBLE: + case MYSQL_TYPE_TIMESTAMP2: + case MYSQL_TYPE_DATETIME2: + case MYSQL_TYPE_TIME2: + md.discard(1); + break; + default:; + } + cb(nullptr); + } + else { + switch (type) { +#define TYPE_CALLBACK(T) \ + case T: \ + cb(MySQL::Type::read(md, data)); \ + break + TYPE_CALLBACK(MYSQL_TYPE_DECIMAL); + TYPE_CALLBACK(MYSQL_TYPE_TINY); + TYPE_CALLBACK(MYSQL_TYPE_SHORT); + TYPE_CALLBACK(MYSQL_TYPE_LONG); + TYPE_CALLBACK(MYSQL_TYPE_FLOAT); + TYPE_CALLBACK(MYSQL_TYPE_DOUBLE); + TYPE_CALLBACK(MYSQL_TYPE_NULL); + TYPE_CALLBACK(MYSQL_TYPE_TIMESTAMP); + TYPE_CALLBACK(MYSQL_TYPE_LONGLONG); + TYPE_CALLBACK(MYSQL_TYPE_INT24); + TYPE_CALLBACK(MYSQL_TYPE_DATE); + TYPE_CALLBACK(MYSQL_TYPE_TIME); + TYPE_CALLBACK(MYSQL_TYPE_DATETIME); + TYPE_CALLBACK(MYSQL_TYPE_YEAR); + TYPE_CALLBACK(MYSQL_TYPE_NEWDATE); + TYPE_CALLBACK(MYSQL_TYPE_BIT); + TYPE_CALLBACK(MYSQL_TYPE_TIMESTAMP2); + TYPE_CALLBACK(MYSQL_TYPE_TIME2); + TYPE_CALLBACK(MYSQL_TYPE_JSON); + TYPE_CALLBACK(MYSQL_TYPE_NEWDECIMAL); + TYPE_CALLBACK(MYSQL_TYPE_ENUM); + TYPE_CALLBACK(MYSQL_TYPE_SET); + TYPE_CALLBACK(MYSQL_TYPE_TINY_BLOB); + TYPE_CALLBACK(MYSQL_TYPE_MEDIUM_BLOB); + TYPE_CALLBACK(MYSQL_TYPE_LONG_BLOB); + TYPE_CALLBACK(MYSQL_TYPE_BLOB); + TYPE_CALLBACK(MYSQL_TYPE_GEOMETRY); + TYPE_CALLBACK(MYSQL_TYPE_VAR_STRING); + TYPE_CALLBACK(MYSQL_TYPE_VARCHAR); + TYPE_CALLBACK(MYSQL_TYPE_DATETIME2); + TYPE_CALLBACK(MYSQL_TYPE_STRING); +#undef TYPE_CALLBACK + case MAX_NO_FIELD_TYPES: + // This is why you don't the end of the enum as member! +#ifdef NDEBUG + [[fallthrough]]; + default: +#endif + throw std::logic_error("Unknown type: " + std::to_string(type)); + } + } + ++nullIter; + } + ++colIter; + } + } +} diff --git a/lib/row.h b/lib/row.h new file mode 100644 index 0000000..ac2c135 --- /dev/null +++ b/lib/row.h @@ -0,0 +1,21 @@ +#ifndef MYGRATE_ROW_H +#define MYGRATE_ROW_H + +#include "mysql_types.h" +#include "rawDataReader.h" +#include + +namespace MyGrate { + class Row { + public: + Row(const st_mariadb_rpl_rows_event &, const st_mariadb_rpl_table_map_event &); + + void forEachField(const std::function & callback); + + private: + const st_mariadb_rpl_rows_event & row; + const st_mariadb_rpl_table_map_event & tm; + }; +} + +#endif diff --git a/lib/streamSupport.cpp b/lib/streamSupport.cpp new file mode 100644 index 0000000..541b191 --- /dev/null +++ b/lib/streamSupport.cpp @@ -0,0 +1,53 @@ +#include "streamSupport.h" +#include "compileTimeFormatter.h" + +namespace std { + std::ostream & + operator<<(std::ostream & strm, const std::byte byt) + { + return AdHoc::scprintf<"%#02x">(strm, std::to_integer(byt)); + } + + std::ostream & + operator<<(std::ostream & s, const MARIADB_STRING & str) + { + return s.write(str.str, str.length); + } + + std::ostream & + operator<<(std::ostream & s, const tm & tm) + { + return AdHoc::scprintf<"%04d-%02d-%02d %02d:%02d:%02d">( + s, tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + } + + std::ostream & + operator<<(std::ostream & s, const timespec & ts) + { + return AdHoc::scprintf<"%d.%09d">(s, ts.tv_sec, ts.tv_nsec); + } + + std::ostream & + operator<<(std::ostream & s, const MyGrate::MySQL::Date & d) + { + return AdHoc::scprintf<"%04d-%02d-%02d">(s, d.year, d.month, d.day); + } + + std::ostream & + operator<<(std::ostream & s, const MyGrate::MySQL::Time & t) + { + return AdHoc::scprintf<"%02d:%02d:%02d">(s, t.hour, t.minute, t.second); + } + + std::ostream & + operator<<(std::ostream & s, const MyGrate::MySQL::DateTime & dt) + { + return AdHoc::scprintf<"%? %?">(s, (const MyGrate::MySQL::Date)dt, (const MyGrate::MySQL::Time)dt); + } + + std::ostream & + operator<<(std::ostream & s, const MyGrate::BitSet & bs) + { + return s << std::make_pair(bs.begin(), bs.end()); + } +} diff --git a/lib/streamSupport.h b/lib/streamSupport.h new file mode 100644 index 0000000..0723faf --- /dev/null +++ b/lib/streamSupport.h @@ -0,0 +1,68 @@ +#ifndef MYGRATE_STREAM_SUPPORT_H +#define MYGRATE_STREAM_SUPPORT_H + +#include +#include +#include +#include +#include +#include +#include + +#include "mysql_types.h" +#include + +namespace std { + std::ostream & operator<<(std::ostream & strm, const std::byte byt); + + std::ostream & operator<<(std::ostream & s, const MARIADB_STRING & str); + + std::ostream & operator<<(std::ostream & s, const tm & tm); + + std::ostream & operator<<(std::ostream & s, const timespec & ts); + + std::ostream & operator<<(std::ostream & s, const MyGrate::MySQL::Date & d); + + std::ostream & operator<<(std::ostream & s, const MyGrate::MySQL::Time & t); + + std::ostream & operator<<(std::ostream & s, const MyGrate::MySQL::DateTime & dt); + + std::ostream & operator<<(std::ostream & s, const MyGrate::BitSet & bs); + + template + inline std::ostream & + operator<<(std::ostream & strm, const std::pair range) + { + strm << '['; + for (auto i {range.first}; i != range.second; i++) { + if (i != range.first) { + strm << ','; + } + strm << *i; + } + return strm << ']'; + } + + template + std::ostream & + operator<<(std::ostream & strm, const std::span spn) + { + return strm << std::make_pair(spn.begin(), spn.end()); + } + + template + std::ostream & + operator<<(std::ostream & strm, const std::vector & v) + { + return strm << std::span(v.data(), v.size()); + } + + template + std::ostream & + operator<<(std::ostream & strm, const std::array & a) + { + return strm << std::span(a.begin(), a.end()); + } +} + +#endif diff --git a/main/Jamfile.jam b/main/Jamfile.jam new file mode 100644 index 0000000..d03d005 --- /dev/null +++ b/main/Jamfile.jam @@ -0,0 +1,2 @@ +exe mygrate : main.cpp : ../lib//mygrate ; +#run main.cpp : : : ../lib//mygrate ; diff --git a/main/main.cpp b/main/main.cpp new file mode 100644 index 0000000..d976a0a --- /dev/null +++ b/main/main.cpp @@ -0,0 +1,195 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace AdHoc; +using MariaDB_Rpl_Ptr = std::unique_ptr; +using MariaDB_Event_Ptr = std::unique_ptr; + +struct EventHandler { + std::string_view name; + void (*func)(MariaDB_Event_Ptr); +}; +using EventHandlers = std::array; + +using TableId = decltype(st_mariadb_rpl_table_map_event::table_id); +using TableMaps = std::map; +TableMaps tableMaps; + +struct write { + template + void + operator()(const T & v) const + { + scprintf<"\t\t%?\n">(std::cout, v); + } + void + operator()(const uint8_t & v) const + { + scprintf<"\t\t%d\n">(std::cout, v); + } + void + operator()(const int8_t & v) const + { + scprintf<"\t\t%d\n">(std::cout, v); + } +}; +static void +dumpRowData(const st_mariadb_rpl_rows_event & row) +{ + MyGrate::Row {row, tableMaps.at(row.table_id)->event.table_map}.forEachField([](auto && fv) { + std::visit(write {}, fv); + }); +} + +static void +doTableMap(MariaDB_Event_Ptr event) +{ + const auto & tm = event->event.table_map; + AdHoc::scprintf<"Table map %?.%? -> %?\n">(std::cout, tm.database, tm.table, tm.table_id); + // for (auto c = 0U; c < tm.column_types.length; c++) { + // AdHoc::scprintf<"\t%#02hx\n">(std::cout, tm.column_types.str[c]); + //} + tableMaps.insert_or_assign(tm.table_id, std::move(event)); +} + +static void +doWrite(MariaDB_Event_Ptr event) +{ + const auto & rs = event->event.rows; + AdHoc::scprintf<"Insert into %?\n">(std::cout, rs.table_id); + dumpRowData(event->event.rows); +} + +static void +doUpdate(MariaDB_Event_Ptr event) +{ + const auto & rs = event->event.rows; + AdHoc::scprintf<"Update %?\n">(std::cout, rs.table_id); + dumpRowData(event->event.rows); +} + +static void +doDelete(MariaDB_Event_Ptr event) +{ + const auto & rs = event->event.rows; + AdHoc::scprintf<"Delete from %?\n">(std::cout, rs.table_id); + dumpRowData(event->event.rows); +} + +constexpr EventHandlers eventHandlers {[]() { + EventHandlers eh {}; + eh[UNKNOWN_EVENT] = {"UNKNOWN_EVENT", nullptr}; + eh[START_EVENT_V3] = {"START_EVENT_V3", nullptr}; + eh[QUERY_EVENT] = {"QUERY_EVENT", nullptr}; + eh[STOP_EVENT] = {"STOP_EVENT", nullptr}; + eh[ROTATE_EVENT] = {"ROTATE_EVENT", nullptr}; + eh[INTVAR_EVENT] = {"INTVAR_EVENT", nullptr}; + eh[LOAD_EVENT] = {"LOAD_EVENT", nullptr}; + eh[SLAVE_EVENT] = {"SLAVE_EVENT", nullptr}; + eh[CREATE_FILE_EVENT] = {"CREATE_FILE_EVENT", nullptr}; + eh[APPEND_BLOCK_EVENT] = {"APPEND_BLOCK_EVENT", nullptr}; + eh[EXEC_LOAD_EVENT] = {"EXEC_LOAD_EVENT", nullptr}; + eh[DELETE_FILE_EVENT] = {"DELETE_FILE_EVENT", nullptr}; + eh[NEW_LOAD_EVENT] = {"NEW_LOAD_EVENT", nullptr}; + eh[RAND_EVENT] = {"RAND_EVENT", nullptr}; + eh[USER_VAR_EVENT] = {"USER_VAR_EVENT", nullptr}; + eh[FORMAT_DESCRIPTION_EVENT] = {"FORMAT_DESCRIPTION_EVENT", nullptr}; + eh[XID_EVENT] = {"XID_EVENT", nullptr}; + eh[BEGIN_LOAD_QUERY_EVENT] = {"BEGIN_LOAD_QUERY_EVENT", nullptr}; + eh[EXECUTE_LOAD_QUERY_EVENT] = {"EXECUTE_LOAD_QUERY_EVENT", nullptr}; + eh[TABLE_MAP_EVENT] = {"TABLE_MAP_EVENT", doTableMap}; + eh[PRE_GA_WRITE_ROWS_EVENT] = {"PRE_GA_WRITE_ROWS_EVENT", nullptr}; + eh[PRE_GA_UPDATE_ROWS_EVENT] = {"PRE_GA_UPDATE_ROWS_EVENT", nullptr}; + eh[PRE_GA_DELETE_ROWS_EVENT] = {"PRE_GA_DELETE_ROWS_EVENT", nullptr}; + eh[WRITE_ROWS_EVENT_V1] = {"WRITE_ROWS_EVENT_V1", doWrite}; + eh[UPDATE_ROWS_EVENT_V1] = {"UPDATE_ROWS_EVENT_V1", doUpdate}; + eh[DELETE_ROWS_EVENT_V1] = {"DELETE_ROWS_EVENT_V1", doDelete}; + eh[INCIDENT_EVENT] = {"INCIDENT_EVENT", nullptr}; + eh[HEARTBEAT_LOG_EVENT] = {"HEARTBEAT_LOG_EVENT", nullptr}; + eh[IGNORABLE_LOG_EVENT] = {"IGNORABLE_LOG_EVENT", nullptr}; + eh[ROWS_QUERY_LOG_EVENT] = {"ROWS_QUERY_LOG_EVENT", nullptr}; + eh[WRITE_ROWS_EVENT] = {"WRITE_ROWS_EVENT", nullptr}; + eh[UPDATE_ROWS_EVENT] = {"UPDATE_ROWS_EVENT", nullptr}; + eh[DELETE_ROWS_EVENT] = {"DELETE_ROWS_EVENT", nullptr}; + eh[GTID_LOG_EVENT] = {"GTID_LOG_EVENT", nullptr}; + eh[ANONYMOUS_GTID_LOG_EVENT] = {"ANONYMOUS_GTID_LOG_EVENT", nullptr}; + eh[PREVIOUS_GTIDS_LOG_EVENT] = {"PREVIOUS_GTIDS_LOG_EVENT", nullptr}; + eh[TRANSACTION_CONTEXT_EVENT] = {"TRANSACTION_CONTEXT_EVENT", nullptr}; + eh[VIEW_CHANGE_EVENT] = {"VIEW_CHANGE_EVENT", nullptr}; + eh[XA_PREPARE_LOG_EVENT] = {"XA_PREPARE_LOG_EVENT", nullptr}; + eh[MARIA_EVENTS_BEGIN] = {"MARIA_EVENTS_BEGIN", nullptr}; + eh[ANNOTATE_ROWS_EVENT] = {"ANNOTATE_ROWS_EVENT", nullptr}; + eh[BINLOG_CHECKPOINT_EVENT] = {"BINLOG_CHECKPOINT_EVENT", nullptr}; + eh[GTID_EVENT] = {"GTID_EVENT", nullptr}; + eh[GTID_LIST_EVENT] = {"GTID_LIST_EVENT", nullptr}; + eh[START_ENCRYPTION_EVENT] = {"START_ENCRYPTION_EVENT", nullptr}; + eh[QUERY_COMPRESSED_EVENT] = {"QUERY_COMPRESSED_EVENT", nullptr}; + eh[WRITE_ROWS_COMPRESSED_EVENT_V1] = {"WRITE_ROWS_COMPRESSED_EVENT_V1", nullptr}; + eh[UPDATE_ROWS_COMPRESSED_EVENT_V1] = {"UPDATE_ROWS_COMPRESSED_EVENT_V1", nullptr}; + eh[DELETE_ROWS_COMPRESSED_EVENT_V1] = {"DELETE_ROWS_COMPRESSED_EVENT_V1", nullptr}; + eh[WRITE_ROWS_COMPRESSED_EVENT] = {"WRITE_ROWS_COMPRESSED_EVENT", nullptr}; + eh[UPDATE_ROWS_COMPRESSED_EVENT] = {"UPDATE_ROWS_COMPRESSED_EVENT", nullptr}; + eh[DELETE_ROWS_COMPRESSED_EVENT] = {"DELETE_ROWS_COMPRESSED_EVENT", nullptr}; + return eh; +}()}; + +static void +read_events(MYSQL * mysql) +{ + auto rpl = MariaDB_Rpl_Ptr {mariadb_rpl_init(mysql), &mariadb_rpl_close}; + + mysql_query(mysql, "SET @mariadb_slave_capability=4"); + // mysql_query(mysql, "SET @slave_gtid_strict_mode=1"); + // mysql_query(mysql, "SET @slave_gtid_ignore_duplicates=1"); + mysql_query(mysql, "SET NAMES utf8"); + mysql_query(mysql, "SET @master_binlog_checksum= @@global.binlog_checksum"); + + mariadb_rpl_optionsv(rpl.get(), MARIADB_RPL_SERVER_ID, 12); + mariadb_rpl_optionsv(rpl.get(), MARIADB_RPL_FILENAME, "mariadb-bin.000242"); + mariadb_rpl_optionsv(rpl.get(), MARIADB_RPL_START, 4); + mariadb_rpl_optionsv(rpl.get(), MARIADB_RPL_FLAGS, MARIADB_RPL_BINLOG_SEND_ANNOTATE_ROWS); + + if (mariadb_rpl_open(rpl.get())) { + throw std::runtime_error("Failed to mariadb_rpl_open"); + } + + while (MariaDB_Event_Ptr event {mariadb_rpl_fetch(rpl.get(), nullptr), &mariadb_free_rpl_event}) { + const auto & h = eventHandlers.at(event->event_type); + if (h.func) { + h.func(std::move(event)); + } + else { + std::cout << h.name << "\n"; + } + } +} + +int +main(int, char **) +{ + std::cout << std::boolalpha; + MYSQL conn; + mysql_init(&conn); + if (!mysql_real_connect(&conn, "192.168.1.38", "repl", "r3pl", "", + // NOLINTNEXTLINE(hicpp-signed-bitwise) + 3306, nullptr, CLIENT_LOCAL_FILES | CLIENT_MULTI_STATEMENTS)) { + throw std::runtime_error("ConnectionError"); + } + read_events(&conn); + mysql_close(&conn); +} diff --git a/test/Jamfile.jam b/test/Jamfile.jam new file mode 100644 index 0000000..e6a833a --- /dev/null +++ b/test/Jamfile.jam @@ -0,0 +1,11 @@ +lib boost_unit_test_framework ; + +project : requirements + ../lib//mygrate + boost_unit_test_framework + BOOST_TEST_DYN_LINK + ; + +run test-rawDataReader.cpp ; +run test-bitset.cpp ; +run test-streams.cpp ; diff --git a/test/helpers.h b/test/helpers.h new file mode 100644 index 0000000..107eda5 --- /dev/null +++ b/test/helpers.h @@ -0,0 +1,44 @@ +#ifndef MYGRATE_TEST_HELPERS_H +#define MYGRATE_TEST_HELPERS_H + +#include +#include +#include + +inline constexpr std::byte operator""_b(const unsigned long long hex) +{ + return std::byte(hex); +} + +inline constexpr bool +operator==(const struct tm & a, const struct tm & b) +{ + return (a.tm_year == b.tm_year) && (a.tm_mon == b.tm_mon) && (a.tm_mday == b.tm_mday) && (a.tm_hour == b.tm_hour) + && (a.tm_min == b.tm_min) && (a.tm_sec == b.tm_sec); +} + +namespace std { + template + inline constexpr bool + operator!=(const byte b, const T i) + { + return to_integer(b) != i; + } +} + +inline struct tm +make_tm(int year, int mon, int day, int hr, int min, int sec) +{ + struct tm tm { + }; + tm.tm_year = year - 1900; + tm.tm_mon = mon - 1; + tm.tm_mday = day; + tm.tm_hour = hr; + tm.tm_min = min; + tm.tm_sec = sec; + mktime(&tm); + return tm; +} + +#endif diff --git a/test/test-bitset.cpp b/test/test-bitset.cpp new file mode 100644 index 0000000..83e136c --- /dev/null +++ b/test/test-bitset.cpp @@ -0,0 +1,31 @@ +#define BOOST_TEST_MODULE BitSet +#include +#include + +#include "helpers.h" +#include +#include + +using namespace MyGrate; + +BOOST_TEST_DONT_PRINT_LOG_VALUE(BitSet::Iterator); + +BOOST_AUTO_TEST_CASE(bitset) +{ + std::byte bytes[3] {0x00_b, 0xFF_b, 0xa1_b}; + BitSet bs {bytes}; + + BOOST_REQUIRE_EQUAL(bs.size(), 24); + + std::bitset<24> exp {0xa1ff00}; + + auto iter {bs.begin()}; + + for (auto x {0U}; x < 24 && iter != bs.end(); x++) { + BOOST_TEST_INFO(x); + BOOST_CHECK_EQUAL(*iter, exp.test(x)); + iter++; + } + + BOOST_CHECK_EQUAL(iter, bs.end()); +} diff --git a/test/test-rawDataReader.cpp b/test/test-rawDataReader.cpp new file mode 100644 index 0000000..4ac0a8a --- /dev/null +++ b/test/test-rawDataReader.cpp @@ -0,0 +1,163 @@ +#define BOOST_TEST_MODULE RawDataReader +#include +#include +#include + +#include "helpers.h" +#include +#include +#include + +using namespace MyGrate; + +using Bytes = std::vector; +template using BytesTo = std::tuple; + +BOOST_DATA_TEST_CASE(read_packedinteger, + boost::unit_test::data::make>({ + {{0x00, 0xb2, 0x2, 0}, 0}, + {{0x01, 0xb2, 0x2, 0}, 1}, + {{0xfa, 0xb2, 0x2, 0}, 0xfa}, + {{0xfc, 0xb2, 0x2, 0}, 0x02b2}, + {{0xfd, 0xb2, 0x2, 0}, 690}, + {{0xfe, 0xb2, 0x2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 690}, + }), + bytes, out) +{ + RawDataReader rdr {bytes.data(), bytes.size()}; + BOOST_CHECK_EQUAL((rdr.readValue()), out); +} + +BOOST_DATA_TEST_CASE(invalid_packed_ints, + boost::unit_test::data::make({ + {0xFF, 0x78, 0x38, 0x1a, 0x0b}, + {0xFB, 0x78, 0x38, 0x1a, 0x0b}, + }), + bytes) +{ + RawDataReader rdr {bytes.data(), bytes.size()}; + BOOST_CHECK_THROW(rdr.readValue(), std::domain_error); +} + +BOOST_DATA_TEST_CASE(read_overflow, + boost::unit_test::data::make({ + {0x00, 0xFB, 0x12, 0x00}, + }), + bytes) +{ + RawDataReader rdr {bytes.data(), bytes.size()}; + rdr.discard(1); + BOOST_CHECK_EQUAL(rdr.readValue(), 0x12FB); + BOOST_CHECK_THROW(rdr.readValue(), std::range_error); +} + +/* +BOOST_DATA_TEST_CASE(read_overflow_string, + boost::unit_test::data::make({ + {0x04, 'a', 'b', 'c'}, + }), + bytes) +{ + RawDataReader rdr {bytes.data(), bytes.size()}; + // BOOST_CHECK_THROW(rdr.readValue>(), std::range_error); +} + +BOOST_DATA_TEST_CASE(read_datetime2, + boost::unit_test::data::make>({ + {{0x99, 0x78, 0x38, 0x1a, 0x0b}, make_tm(2006, 2, 28, 1, 40, 11)}, + }), + bytes, exp) +{ + RawDataReader rdr {bytes.data(), bytes.size()}; + (void)exp; + // BOOST_CHECK_EQUAL((rdr.readValue()), out); +} + +BOOST_DATA_TEST_CASE(read_varchars1, + boost::unit_test::data::make>({ + {{0x00}, ""}, + {{0x01, 'A'}, "A"}, + {{0x03, 'a', 'b', 'c'}, "abc"}, + {{0x10, '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}, + "0123456789abcdef"}, + }), + bytes, exp) +{ + RawDataReader rdr {bytes.data(), bytes.size()}; + (void)exp; + // BOOST_CHECK_EQUAL((rdr.readValue>()), exp); +} + +BOOST_DATA_TEST_CASE(read_varchars2, + boost::unit_test::data::make>({ + {{0x00, 0x00}, ""}, + {{0x01, 0x00, 'A'}, "A"}, + {{0x03, 0x00, 'a', 'b', 'c'}, "abc"}, + {{0x10, 0x00, '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}, + "0123456789abcdef"}, + }), + bytes, exp) +{ + RawDataReader rdr {bytes.data(), bytes.size()}; + (void)exp; + // BOOST_CHECK_EQUAL((rdr.readValue>()), exp); +} +*/ + +BOOST_DATA_TEST_CASE(read_bytes, + boost::unit_test::data::make({ + {}, + {0x01}, + {0x00, 0x01, 0x02}, + {0xFF, 0xFE, 0xFD}, + {0xFF, 0xFE, 0xFD, 0x00, 0x01, 0x02}, + }), + bytes) +{ + RawDataReader rdr {bytes.data(), bytes.size()}; + const auto out {rdr.viewValue(bytes.size())}; + BOOST_CHECK_EQUAL_COLLECTIONS(out.begin(), out.end(), bytes.begin(), bytes.end()); +} + +BOOST_DATA_TEST_CASE(read_bytes_overflow, + boost::unit_test::data::make({ + {}, + {0x01}, + {0x00, 0x01, 0x02}, + {0xFF, 0xFE, 0xFD}, + {0xFF, 0xFE, 0xFD, 0x00, 0x01, 0x02}, + }), + bytes) +{ + RawDataReader rdr {bytes.data(), bytes.size()}; + BOOST_CHECK_THROW(rdr.viewValue(bytes.size() + 1), std::range_error); +} + +BOOST_DATA_TEST_CASE(read_field_type, + boost::unit_test::data::make>({ + {{0x00}, MYSQL_TYPE_DECIMAL}, + {{0x04}, MYSQL_TYPE_FLOAT}, + {{0xFF}, MYSQL_TYPE_GEOMETRY}, + }), + bytes, exp) +{ + RawDataReader rdr {bytes.data(), bytes.size()}; + BOOST_CHECK_EQUAL(rdr.readValue(1), exp); +} + +BOOST_AUTO_TEST_CASE(rdr_from_MARIADB_STRING) +{ + char buf[5] = "test"; + MARIADB_STRING str {buf, strlen(buf)}; + RawDataReader rdr {str}; + BOOST_CHECK_EQUAL(rdr.viewValue(4), "test"); +} + +using SimpleTypes = boost::mpl::list; + +BOOST_AUTO_TEST_CASE_TEMPLATE(rdr_read_simple, T, SimpleTypes) +{ + RawDataReader rdr {"don't care, some bytes", 20}; + rdr.readValue(); +} diff --git a/test/test-streams.cpp b/test/test-streams.cpp new file mode 100644 index 0000000..9e79bd6 --- /dev/null +++ b/test/test-streams.cpp @@ -0,0 +1,92 @@ +#define BOOST_TEST_MODULE StreamSupport +#include +#include + +#include "helpers.h" +#include + +BOOST_FIXTURE_TEST_SUITE(stream, std::stringstream); + +template using ToStream = std::tuple; +BOOST_DATA_TEST_CASE(bytes, + boost::unit_test::data::make>({ + //{0x00_b, "0x00"}, + {0x10_b, "0x10"}, + {0xFE_b, "0xfe"}, + {0xff_b, "0xff"}, + }), + in, exp) +{ + *this << in; + BOOST_CHECK_EQUAL(this->str(), exp); +} + +BOOST_DATA_TEST_CASE(tms, + boost::unit_test::data::make>({ + {make_tm(2016, 1, 4, 12, 13, 14), "2016-01-04 12:13:14"}, + {make_tm(2016, 12, 31, 0, 0, 1), "2016-12-31 00:00:01"}, + }), + in, exp) +{ + *this << in; + BOOST_CHECK_EQUAL(this->str(), exp); +} + +BOOST_DATA_TEST_CASE(rts, + boost::unit_test::data::make>({ + {{{2016, 1, 4}, {12, 13, 14}}, "2016-01-04 12:13:14"}, + {{{2016, 12, 31}, {0, 0, 1}}, "2016-12-31 00:00:01"}, + }), + in, exp) +{ + *this << in; + BOOST_CHECK_EQUAL(this->str(), exp); +} + +BOOST_DATA_TEST_CASE(tss, + boost::unit_test::data::make>({ + {{0, 0}, "0.000000000"}, + {{0, 1}, "0.000000001"}, + {{1, 0}, "1.000000000"}, + {{123, 0}, "123.000000000"}, + {{123, 999999999}, "123.999999999"}, + }), + in, exp) +{ + *this << in; + BOOST_CHECK_EQUAL(this->str(), exp); +} +constexpr std::byte somebits[3] {0x00_b, 0xFF_b, 0xa1_b}; +BOOST_DATA_TEST_CASE(bss, + boost::unit_test::data::make>({ + {{MyGrate::BitSet {somebits}}, "[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,0,0,0,0,1,0,1]"}, + }), + in, exp) +{ + *this << in; + BOOST_CHECK_EQUAL(this->str(), exp); +} + +BOOST_AUTO_TEST_CASE(mariadb_string) +{ + char buf[5] = "test"; + MARIADB_STRING str {buf, strlen(buf)}; + *this << str; + BOOST_CHECK_EQUAL(this->str(), buf); +} + +BOOST_AUTO_TEST_CASE(array) +{ + std::array arr {1, 123456, -78910}; + *this << arr; + BOOST_CHECK_EQUAL(this->str(), "[1,123456,-78910]"); +} + +BOOST_AUTO_TEST_CASE(vector) +{ + std::vector arr {1, 123456, -78910}; + *this << arr; + BOOST_CHECK_EQUAL(this->str(), "[1,123456,-78910]"); +} + +BOOST_AUTO_TEST_SUITE_END(); -- cgit v1.2.3