From 7f53ebf97a5d030b72cfadd16cdb786c28037e11 Mon Sep 17 00:00:00 2001 From: Dan Goodliffe Date: Fri, 7 Oct 2016 18:08:17 +0100 Subject: Increased flexibility for reading post body content and support for deserializing x-www-form-urlencoded payloads --- icespider/common/routes.ice | 1 + icespider/compile/routeCompiler.cpp | 53 +++++++++++--- icespider/core/ihttpRequest.h | 17 ++++- icespider/fcgi/Jamfile.jam | 2 +- icespider/fcgi/hextable.c | 13 ++++ icespider/fcgi/xwwwFormUrlEncoded.cpp | 126 ++++++++++++++++++++++++++++++++++ icespider/unittests/Jamfile.jam | 8 +++ icespider/unittests/test-fcgi.ice | 10 +++ icespider/unittests/testApp.cpp | 4 +- icespider/unittests/testCompile.cpp | 6 +- icespider/unittests/testFcgi.cpp | 83 ++++++++++++++++++++++ icespider/unittests/testRoutes.json | 11 +++ 12 files changed, 319 insertions(+), 15 deletions(-) create mode 100644 icespider/fcgi/hextable.c create mode 100644 icespider/fcgi/xwwwFormUrlEncoded.cpp create mode 100644 icespider/unittests/test-fcgi.ice diff --git a/icespider/common/routes.ice b/icespider/common/routes.ice index 3f9eda4..37939c1 100644 --- a/icespider/common/routes.ice +++ b/icespider/common/routes.ice @@ -12,6 +12,7 @@ module IceSpider { bool isOptional = false; ["slicer:name:default"] optional(1) string defaultExpr; + optional(2) string type; ["slicer:ignore"] bool hasUserSource = true; diff --git a/icespider/compile/routeCompiler.cpp b/icespider/compile/routeCompiler.cpp index b8b89db..d6ad73a 100644 --- a/icespider/compile/routeCompiler.cpp +++ b/icespider/compile/routeCompiler.cpp @@ -123,10 +123,12 @@ namespace IceSpider { for (const auto & p : ps) { auto defined = r.second->params.find(p.first); if (defined != r.second->params.end()) { - if (!defined->second->key) defined->second->key = defined->first; + if (!defined->second->key && defined->second->source != ParameterSource::Body) { + defined->second->key = defined->first; + } } else { - defined = r.second->params.insert({ p.first, new Parameter(ParameterSource::URL, p.first, false, IceUtil::Optional(), false) }).first; + defined = r.second->params.insert({ p.first, new Parameter(ParameterSource::URL, p.first, false, IceUtil::Optional(), IceUtil::Optional(), false) }).first; } auto d = defined->second; if (d->source == ParameterSource::URL) { @@ -296,8 +298,8 @@ namespace IceSpider { auto proxies = initializeProxies(output, r.second); for (const auto & p : r.second->params) { if (p.second->hasUserSource) { - fprintf(output, ",\n"); if (p.second->source == ParameterSource::URL) { + fprintf(output, ",\n"); Path path(r.second->path); unsigned int idx = -1; for (const auto & pp : path.parts) { @@ -310,7 +312,10 @@ namespace IceSpider { fprintbf(4, output, "_pi_%s(%d)", p.first, idx); } else { - fprintbf(4, output, "_pn_%s(\"%s\")", p.first, *p.second->key); + if (p.second->key) { + fprintf(output, ",\n"); + fprintbf(4, output, "_pn_%s(\"%s\")", p.first, *p.second->key); + } } } if (p.second->defaultExpr) { @@ -330,18 +335,48 @@ namespace IceSpider { fprintbf(3, output, "void execute(IceSpider::IHttpRequest * request) const\n"); fprintbf(3, output, "{\n"); auto ps = findParameters(r.second, units); + bool doneBody = false; for (const auto & p : r.second->params) { if (p.second->hasUserSource) { auto ip = ps.find(p.first)->second; - fprintbf(4, output, "auto _p_%s(request->get%sParam<%s>(_p%c_%s)", - p.first, getEnumString(p.second->source), Slice::typeToString(ip->type()), - p.second->source == ParameterSource::URL ? 'i' : 'n', - p.first); + if (p.second->source == ParameterSource::Body) { + if (p.second->key) { + if (!doneBody) { + if (p.second->type) { + fprintbf(4, output, "auto _pbody(request->getBody<%s>());\n", + *p.second->type); + } + else { + fprintbf(4, output, "auto _pbody(request->getBody());\n"); + } + doneBody = true; + } + if (p.second->type) { + fprintbf(4, output, "auto _p_%s(_pbody->%s", + p.first, p.first); + } + else { + fprintbf(4, output, "auto _p_%s(request->getBodyParam<%s>(_pbody, _pn_%s)", + p.first, Slice::typeToString(ip->type()), + p.first); + } + } + else { + fprintbf(4, output, "auto _p_%s(request->getBody<%s>()", + p.first, Slice::typeToString(ip->type())); + } + } + else { + fprintbf(4, output, "auto _p_%s(request->get%sParam<%s>(_p%c_%s)", + p.first, getEnumString(p.second->source), Slice::typeToString(ip->type()), + p.second->source == ParameterSource::URL ? 'i' : 'n', + p.first); + } if (!p.second->isOptional && p.second->source != ParameterSource::URL) { fprintbf(0, output, " /\n"); if (p.second->defaultExpr) { fprintbf(5, output, " [this]() { return _pd_%s; }", - p.first); + p.first); } else { fprintbf(5, output, " [this]() { return requiredParameterNotFound<%s>(\"%s\", _pn_%s); }", diff --git a/icespider/core/ihttpRequest.h b/icespider/core/ihttpRequest.h index 38598e8..6f6d300 100644 --- a/icespider/core/ihttpRequest.h +++ b/icespider/core/ihttpRequest.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace IceSpider { class Core; @@ -37,11 +38,25 @@ namespace IceSpider { template T getURLParam(unsigned int) const; template - IceUtil::Optional getBodyParam(const std::string &) const + IceUtil::Optional getBody() const { return Slicer::DeserializeAnyWith(getDeserializer()); } template + IceUtil::Optional getBodyParam(const IceUtil::Optional & map, const std::string & key) const + { + if (!map) { + return IceUtil::Optional(); + } + auto i = map->find(key); + if (i == map->end()) { + return IceUtil::Optional(); + } + else { + return boost::lexical_cast(i->second); + } + } + template IceUtil::Optional getQueryStringParam(const std::string & key) const; template IceUtil::Optional getHeaderParam(const std::string & key) const; diff --git a/icespider/fcgi/Jamfile.jam b/icespider/fcgi/Jamfile.jam index c607ed1..c6c4ca0 100644 --- a/icespider/fcgi/Jamfile.jam +++ b/icespider/fcgi/Jamfile.jam @@ -2,7 +2,7 @@ lib fcgi : : fcgi ; lib fcgi++ : : fcgi++ ; lib icespider-fcgi : - [ glob-tree *.cpp : bin ] + [ glob-tree *.c *.cpp : bin ] : fcgi fcgi++ diff --git a/icespider/fcgi/hextable.c b/icespider/fcgi/hextable.c new file mode 100644 index 0000000..3086d49 --- /dev/null +++ b/icespider/fcgi/hextable.c @@ -0,0 +1,13 @@ +// GCC doesn't support this sort of initialization in C++, only plain C. +// https://gcc.gnu.org/onlinedocs/gcc-5.4.0/gcc/Designated-Inits.html + +const long hextable[] = { + [0 ... '0' - 1] = -1, + ['9' + 1 ... 'A' - 1] = -1, + ['G' ... 'a' - 1] = -1, + ['g' ... 255] = -1, + ['0'] = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + ['A'] = 10, 11, 12, 13, 14, 15, + ['a'] = 10, 11, 12, 13, 14, 15 +}; + diff --git a/icespider/fcgi/xwwwFormUrlEncoded.cpp b/icespider/fcgi/xwwwFormUrlEncoded.cpp new file mode 100644 index 0000000..a2ca9ee --- /dev/null +++ b/icespider/fcgi/xwwwFormUrlEncoded.cpp @@ -0,0 +1,126 @@ +#include +#include +#include +#include +#include + +namespace ba = boost::algorithm; + +extern long hextable[]; + +namespace IceSpider { + + class XWwwFormUrlEncoded : public Slicer::Deserializer { + public: + XWwwFormUrlEncoded(std::istream & in) : + input(std::istreambuf_iterator(in), std::istreambuf_iterator()) + { + } + + void Deserialize(Slicer::ModelPartForRootPtr mp) override + { + mp->Create(); + mp->OnEachChild([this](auto, auto mp, auto) { + switch (mp->GetType()) { + case Slicer::mpt_Simple: + this->DeserializeSimple(mp); + break; + case Slicer::mpt_Complex: + this->DeserializeComplex(mp); + break; + case Slicer::mpt_Dictionary: + this->DeserializeDictionary(mp); + break; + default: + throw IceSpider::Http400_BadRequest(); + break; + } + }); + mp->Complete(); + } + + private: + class SetFromString : public Slicer::ValueSource { + public: + SetFromString(const std::string & v) : s(v) + { + } + + void set(bool & t) const override + { + if (s == "true") t = true; + else if (s == "false") t = false; + else throw Http400_BadRequest(); + } + + void set(std::string & t) const override + { + t.reserve(s.size()); + for (auto i = s.begin(); i != s.end(); i++) { + if (*i == '+') t += ' '; + else if (*i == '%') { + t += (16 * hextable[(int)*++i]) + hextable[(int)*++i]; + } + else t += *i; + } + } + +#define SET(T) \ + void set(T & t) const override \ + { \ + t = boost::lexical_cast(s); \ + } + + SET(Ice::Byte); + SET(Ice::Short); + SET(Ice::Int); + SET(Ice::Long); + SET(Ice::Float); + SET(Ice::Double); + const std::string & s; + }; + + typedef boost::function KVh; + void iterateVars(const KVh & h) + { + for (auto pi = ba::make_split_iterator(input, ba::first_finder("&", ba::is_equal())); pi != decltype(pi)(); ++pi) { + auto eq = std::find(pi->begin(), pi->end(), '='); + if (eq == pi->end()) { + h(std::string(pi->begin(), pi->end()), std::string()); + } + else { + h(std::string(pi->begin(), eq), std::string(eq + 1, pi->end())); + } + } + } + void DeserializeSimple(Slicer::ModelPartPtr mp) + { + iterateVars([mp](auto, auto v) { + mp->SetValue(new SetFromString(v)); + }); + } + void DeserializeComplex(Slicer::ModelPartPtr mp) + { + mp->Create(); + iterateVars([mp](auto k, auto v) { + if (auto m = mp->GetChild(k)) { + m->SetValue(new SetFromString(v)); + } + }); + mp->Complete(); + } + void DeserializeDictionary(Slicer::ModelPartPtr mp) + { + iterateVars([mp](auto k, auto v) { + auto p = mp->GetAnonChild(); + p->GetChild("key")->SetValue(new SetFromString(k)); + p->GetChild("value")->SetValue(new SetFromString(v)); + p->Complete(); + }); + } + const std::string input; + }; +} + +NAMEDFACTORY("application/x-www-form-urlencoded", IceSpider::XWwwFormUrlEncoded, Slicer::StreamDeserializerFactory); + diff --git a/icespider/unittests/Jamfile.jam b/icespider/unittests/Jamfile.jam index f5e8f2f..4ee2bd8 100644 --- a/icespider/unittests/Jamfile.jam +++ b/icespider/unittests/Jamfile.jam @@ -23,6 +23,7 @@ lib slicer-json : : : : /usr/include/slicer ; lib slicer-xml : : : : /usr/include/slicer ; lib boost_utf : : boost_unit_test_framework ; lib boost_system ; +lib boost_filesystem ; lib dl ; path-constant me : . ; @@ -81,14 +82,21 @@ run run testFcgi.cpp + test-fcgi.ice ../fcgi/cgiRequestBase.cpp + ../fcgi/xwwwFormUrlEncoded.cpp + ../fcgi/hextable.c : : : + yes BOOST_TEST_DYN_LINK testCommon ../common//icespider-common ../core//icespider-core boost_system + boost_filesystem slicer + slicer-json + adhocutil ../fcgi : testFcgi ; diff --git a/icespider/unittests/test-fcgi.ice b/icespider/unittests/test-fcgi.ice new file mode 100644 index 0000000..6be9086 --- /dev/null +++ b/icespider/unittests/test-fcgi.ice @@ -0,0 +1,10 @@ +module TestFcgi { + class Complex { + string alpha; + double number; + bool boolean; + string spaces; + string empty; + }; +}; + diff --git a/icespider/unittests/testApp.cpp b/icespider/unittests/testApp.cpp index fc7d66a..5f37f62 100644 --- a/icespider/unittests/testApp.cpp +++ b/icespider/unittests/testApp.cpp @@ -27,7 +27,7 @@ void forceEarlyChangeDir() BOOST_AUTO_TEST_CASE( testLoadConfiguration ) { - BOOST_REQUIRE_EQUAL(10, AdHoc::PluginManager::getDefault()->getAll().size()); + BOOST_REQUIRE_EQUAL(11, AdHoc::PluginManager::getDefault()->getAll().size()); } class TestRequest : public IHttpRequest { @@ -96,7 +96,7 @@ BOOST_AUTO_TEST_CASE( testCoreSettings ) { BOOST_REQUIRE_EQUAL(5, routes.size()); BOOST_REQUIRE_EQUAL(1, routes[0].size()); - BOOST_REQUIRE_EQUAL(4, routes[1].size()); + BOOST_REQUIRE_EQUAL(5, routes[1].size()); BOOST_REQUIRE_EQUAL(1, routes[2].size()); BOOST_REQUIRE_EQUAL(2, routes[3].size()); BOOST_REQUIRE_EQUAL(2, routes[4].size()); diff --git a/icespider/unittests/testCompile.cpp b/icespider/unittests/testCompile.cpp index 4fb7827..8d1cefa 100644 --- a/icespider/unittests/testCompile.cpp +++ b/icespider/unittests/testCompile.cpp @@ -36,7 +36,7 @@ BOOST_AUTO_TEST_CASE( testLoadConfiguration ) rc.applyDefaults(cfg, u); BOOST_REQUIRE_EQUAL("common", cfg->name); - BOOST_REQUIRE_EQUAL(10, cfg->routes.size()); + BOOST_REQUIRE_EQUAL(11, cfg->routes.size()); BOOST_REQUIRE_EQUAL("/", cfg->routes["index"]->path); BOOST_REQUIRE_EQUAL(HttpMethod::GET, cfg->routes["index"]->method); @@ -106,6 +106,7 @@ BOOST_AUTO_TEST_CASE( testCompile ) BOOST_AUTO_TEST_CASE( testLink ) { auto outputo = binDir / "testRoutes.o"; + BOOST_REQUIRE(boost::filesystem::exists(outputo)); auto outputso = binDir / "testRoutes.so"; auto linkCommand = boost::algorithm::join>({ @@ -121,12 +122,13 @@ BOOST_AUTO_TEST_CASE( testLink ) BOOST_AUTO_TEST_CASE( testLoad ) { auto outputso = binDir / "testRoutes.so"; + BOOST_REQUIRE(boost::filesystem::exists(outputso)); auto lib = dlopen(outputso.c_str(), RTLD_NOW); BOOST_TEST_INFO(dlerror()); BOOST_REQUIRE(lib); - BOOST_REQUIRE_EQUAL(10, AdHoc::PluginManager::getDefault()->getAll().size()); + BOOST_REQUIRE_EQUAL(11, AdHoc::PluginManager::getDefault()->getAll().size()); // smoke test (block ensure dlclose dones't cause segfault) { auto route = AdHoc::PluginManager::getDefault()->get("common::index"); diff --git a/icespider/unittests/testFcgi.cpp b/icespider/unittests/testFcgi.cpp index f2a58a4..94ce092 100644 --- a/icespider/unittests/testFcgi.cpp +++ b/icespider/unittests/testFcgi.cpp @@ -2,7 +2,9 @@ #include #include +#include #include +#include class TestRequest : public IceSpider::CgiRequestBase { public: @@ -24,6 +26,24 @@ class TestRequest : public IceSpider::CgiRequestBase { // LCOV_EXCL_STOP }; +class TestPayloadRequest : public TestRequest { + public: + TestPayloadRequest(IceSpider::Core * c, char ** env, std::istream & s) : + TestRequest(c, env), + in(s) + { + initialize(); + } + + std::istream & getInputStream() const override + { + return in; + } + + private: + std::istream & in; +}; + class CharPtrPtrArray : public std::vector { public: CharPtrPtrArray() @@ -172,5 +192,68 @@ BOOST_AUTO_TEST_CASE( requestmethod_bad ) BOOST_REQUIRE_THROW(r.getRequestMethod(), IceSpider::Http405_MethodNotAllowed); } +BOOST_AUTO_TEST_CASE( postxwwwformurlencoded_simple ) +{ + CharPtrPtrArray env ({ "SCRIPT_NAME=/", "REQUEST_METHOD=No", "CONTENT_TYPE=application/x-www-form-urlencoded" }); + std::stringstream f("value=314"); + TestPayloadRequest r(this, env, f); + auto n = r.getBody(); + BOOST_REQUIRE_EQUAL(314, n); +} + +BOOST_AUTO_TEST_CASE( postxwwwformurlencoded_dictionary ) +{ + CharPtrPtrArray env ({ "SCRIPT_NAME=/", "REQUEST_METHOD=No", "CONTENT_TYPE=application/x-www-form-urlencoded" }); + std::stringstream f("alpha=abcde&number=3.14&boolean=true&spaces=This+is+a%20string.&empty="); + TestPayloadRequest r(this, env, f); + auto n = *r.getBody(); + BOOST_REQUIRE_EQUAL(5, n.size()); + BOOST_REQUIRE_EQUAL("abcde", n["alpha"]); + BOOST_REQUIRE_EQUAL("3.14", n["number"]); + BOOST_REQUIRE_EQUAL("true", n["boolean"]); + BOOST_REQUIRE_EQUAL("This is a string.", n["spaces"]); + BOOST_REQUIRE_EQUAL("", n["empty"]); +} + +BOOST_AUTO_TEST_CASE( postxwwwformurlencoded_complex ) +{ + CharPtrPtrArray env ({ "SCRIPT_NAME=/", "REQUEST_METHOD=No", "CONTENT_TYPE=application/x-www-form-urlencoded" }); + std::stringstream f("alpha=abcde&number=3.14&boolean=true&empty=&spaces=This+is+a%20string."); + TestPayloadRequest r(this, env, f); + auto n = *r.getBody(); + BOOST_REQUIRE_EQUAL("abcde", n->alpha); + BOOST_REQUIRE_EQUAL(3.14, n->number); + BOOST_REQUIRE_EQUAL(true, n->boolean); + BOOST_REQUIRE_EQUAL("This is a string.", n->spaces); + BOOST_REQUIRE_EQUAL("", n->empty); +} + +BOOST_AUTO_TEST_CASE( postjson_complex ) +{ + CharPtrPtrArray env ({ "SCRIPT_NAME=/", "REQUEST_METHOD=No", "CONTENT_TYPE=application/json" }); + std::stringstream f("{\"alpha\":\"abcde\",\"number\":3.14,\"boolean\":true,\"empty\":\"\",\"spaces\":\"This is a string.\"}"); + TestPayloadRequest r(this, env, f); + auto n = *r.getBody(); + BOOST_REQUIRE_EQUAL("abcde", n->alpha); + BOOST_REQUIRE_EQUAL(3.14, n->number); + BOOST_REQUIRE_EQUAL(true, n->boolean); + BOOST_REQUIRE_EQUAL("This is a string.", n->spaces); + BOOST_REQUIRE_EQUAL("", n->empty); +} + +BOOST_AUTO_TEST_CASE( postjson_dictionary ) +{ + CharPtrPtrArray env ({ "SCRIPT_NAME=/", "REQUEST_METHOD=No", "CONTENT_TYPE=application/json" }); + std::stringstream f("{\"alpha\":\"abcde\",\"number\":\"3.14\",\"boolean\":\"true\",\"empty\":\"\",\"spaces\":\"This is a string.\"}"); + TestPayloadRequest r(this, env, f); + auto n = *r.getBody(); + BOOST_REQUIRE_EQUAL(5, n.size()); + BOOST_REQUIRE_EQUAL("abcde", n["alpha"]); + BOOST_REQUIRE_EQUAL("3.14", n["number"]); + BOOST_REQUIRE_EQUAL("true", n["boolean"]); + BOOST_REQUIRE_EQUAL("This is a string.", n["spaces"]); + BOOST_REQUIRE_EQUAL("", n["empty"]); +} + BOOST_AUTO_TEST_SUITE_END(); diff --git a/icespider/unittests/testRoutes.json b/icespider/unittests/testRoutes.json index 13a6806..5bf002f 100644 --- a/icespider/unittests/testRoutes.json +++ b/icespider/unittests/testRoutes.json @@ -23,6 +23,17 @@ "method": "DELETE", "operation": "TestIceSpider.TestApi.returnNothing" }, + "del2": { + "path": "/{s}", + "method": "DELETE", + "operation": "TestIceSpider.TestApi.returnNothing", + "params": { + "s": { + "source": "Body", + "key": "value" + } + } + }, "update": { "path": "/{id}", "method": "POST", -- cgit v1.2.3