Increased flexibility for reading post body content and support for deserializing x-www-form-urlencoded payloads

12 files changed, 319 insertions, 15 deletions

diff --git a/icespider/common/routes.ice b/icespider/common/routes.ice
index 3f9eda4..37939c1 100644
--- a/icespider/common/routes.ice
+++ b/icespider/common/routes.ice
@@ -12,6 +12,7 @@ module IceSpider {
 		bool isOptional = false;
 		["slicer:name:default"]
 		optional(1) string defaultExpr;
+		optional(2) string type;
 
 		["slicer:ignore"]
 		bool hasUserSource = true;
diff --git a/icespider/compile/routeCompiler.cpp b/icespider/compile/routeCompiler.cpp
index b8b89db..d6ad73a 100644
--- a/icespider/compile/routeCompiler.cpp
+++ b/icespider/compile/routeCompiler.cpp
@@ -123,10 +123,12 @@ namespace IceSpider {
 				for (const auto & p : ps) {
 					auto defined = r.second->params.find(p.first);
 					if (defined != r.second->params.end()) {
-						if (!defined->second->key) defined->second->key = defined->first;
+						if (!defined->second->key && defined->second->source != ParameterSource::Body) {
+							defined->second->key = defined->first;
+						}
 					}
 					else {
-						defined = r.second->params.insert({ p.first, new Parameter(ParameterSource::URL, p.first, false, IceUtil::Optional<std::string>(), false) }).first;
+						defined = r.second->params.insert({ p.first, new Parameter(ParameterSource::URL, p.first, false, IceUtil::Optional<std::string>(), IceUtil::Optional<std::string>(), false) }).first;
 					}
 					auto d = defined->second;
 					if (d->source == ParameterSource::URL) {
@@ -296,8 +298,8 @@ namespace IceSpider {
 				auto proxies = initializeProxies(output, r.second);
 				for (const auto & p : r.second->params) {
 					if (p.second->hasUserSource) {
-						fprintf(output, ",\n");
 						if (p.second->source == ParameterSource::URL) {
+							fprintf(output, ",\n");
 							Path path(r.second->path);
 							unsigned int idx = -1;
 							for (const auto & pp : path.parts) {
@@ -310,7 +312,10 @@ namespace IceSpider {
 							fprintbf(4, output, "_pi_%s(%d)", p.first, idx);
 						}
 						else {
-							fprintbf(4, output, "_pn_%s(\"%s\")", p.first, *p.second->key);
+							if (p.second->key) {
+								fprintf(output, ",\n");
+								fprintbf(4, output, "_pn_%s(\"%s\")", p.first, *p.second->key);
+							}
 						}
 					}
 					if (p.second->defaultExpr) {
@@ -330,18 +335,48 @@ namespace IceSpider {
 				fprintbf(3, output, "void execute(IceSpider::IHttpRequest * request) const\n");
 				fprintbf(3, output, "{\n");
 				auto ps = findParameters(r.second, units);
+				bool doneBody = false;
 				for (const auto & p : r.second->params) {
 					if (p.second->hasUserSource) {
 						auto ip = ps.find(p.first)->second;
-						fprintbf(4, output, "auto _p_%s(request->get%sParam<%s>(_p%c_%s)",
-										 p.first, getEnumString(p.second->source), Slice::typeToString(ip->type()),
-										 p.second->source == ParameterSource::URL ? 'i' : 'n',
-										 p.first);
+						if (p.second->source == ParameterSource::Body) {
+							if (p.second->key) {
+								if (!doneBody) {
+									if (p.second->type) {
+										fprintbf(4, output, "auto _pbody(request->getBody<%s>());\n",
+												*p.second->type);
+									}
+									else {
+										fprintbf(4, output, "auto _pbody(request->getBody<IceSpider::StringMap>());\n");
+									}
+									doneBody = true;
+								}
+								if (p.second->type) {
+									fprintbf(4, output, "auto _p_%s(_pbody->%s",
+											p.first, p.first);
+								}
+								else {
+									fprintbf(4, output, "auto _p_%s(request->getBodyParam<%s>(_pbody, _pn_%s)",
+											p.first, Slice::typeToString(ip->type()),
+											p.first);
+								}
+							}
+							else {
+								fprintbf(4, output, "auto _p_%s(request->getBody<%s>()",
+										p.first, Slice::typeToString(ip->type()));
+							}
+						}
+						else {
+							fprintbf(4, output, "auto _p_%s(request->get%sParam<%s>(_p%c_%s)",
+									p.first, getEnumString(p.second->source), Slice::typeToString(ip->type()),
+									p.second->source == ParameterSource::URL ? 'i' : 'n',
+									p.first);
+						}
 						if (!p.second->isOptional && p.second->source != ParameterSource::URL) {
 							fprintbf(0, output, " /\n");
 							if (p.second->defaultExpr) {
 								fprintbf(5, output, " [this]() { return _pd_%s; }",
-												 p.first);
+										p.first);
 							}
 							else {
 								fprintbf(5, output, " [this]() { return requiredParameterNotFound<%s>(\"%s\", _pn_%s); }",
diff --git a/icespider/core/ihttpRequest.h b/icespider/core/ihttpRequest.h
index 38598e8..6f6d300 100644
--- a/icespider/core/ihttpRequest.h
+++ b/icespider/core/ihttpRequest.h
@@ -8,6 +8,7 @@
 #include <routes.h>
 #include <slicer/slicer.h>
 #include <IceUtil/Optional.h>
+#include <boost/lexical_cast.hpp>
 
 namespace IceSpider {
 	class Core;
@@ -37,11 +38,25 @@ namespace IceSpider {
 			template<typename T>
 			T getURLParam(unsigned int) const;
 			template<typename T>
-			IceUtil::Optional<T> getBodyParam(const std::string &) const
+			IceUtil::Optional<T> getBody() const
 			{
 				return Slicer::DeserializeAnyWith<T>(getDeserializer());
 			}
 			template<typename T>
+			IceUtil::Optional<T> getBodyParam(const IceUtil::Optional<IceSpider::StringMap> & map, const std::string & key) const
+			{
+				if (!map) {
+					return IceUtil::Optional<T>();
+				}
+				auto i = map->find(key);
+				if (i == map->end()) {
+					return IceUtil::Optional<T>();
+				}
+				else {
+					return boost::lexical_cast<T>(i->second);
+				}
+			}
+			template<typename T>
 			IceUtil::Optional<T> getQueryStringParam(const std::string & key) const;
 			template<typename T>
 			IceUtil::Optional<T> getHeaderParam(const std::string & key) const;
diff --git a/icespider/fcgi/Jamfile.jam b/icespider/fcgi/Jamfile.jam
index c607ed1..c6c4ca0 100644
--- a/icespider/fcgi/Jamfile.jam
+++ b/icespider/fcgi/Jamfile.jam
@@ -2,7 +2,7 @@ lib fcgi : : <name>fcgi ;
 lib fcgi++ : : <name>fcgi++ ;
 
 lib icespider-fcgi :
-	[ glob-tree *.cpp : bin ]
+	[ glob-tree *.c *.cpp : bin ]
 	:
 	<library>fcgi
 	<library>fcgi++
diff --git a/icespider/fcgi/hextable.c b/icespider/fcgi/hextable.c
new file mode 100644
index 0000000..3086d49
--- /dev/null
+++ b/icespider/fcgi/hextable.c
@@ -0,0 +1,13 @@
+// GCC doesn't support this sort of initialization in C++, only plain C.
+// https://gcc.gnu.org/onlinedocs/gcc-5.4.0/gcc/Designated-Inits.html
+
+const long hextable[] = {
+	[0 ... '0' - 1] = -1,
+	['9' + 1 ... 'A' - 1] = -1,
+	['G' ... 'a' - 1] = -1,
+	['g' ... 255] = -1,
+	['0'] = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
+	['A'] = 10, 11, 12, 13, 14, 15,
+	['a'] = 10, 11, 12, 13, 14, 15
+};
+
diff --git a/icespider/fcgi/xwwwFormUrlEncoded.cpp b/icespider/fcgi/xwwwFormUrlEncoded.cpp
new file mode 100644
index 0000000..a2ca9ee
--- /dev/null
+++ b/icespider/fcgi/xwwwFormUrlEncoded.cpp
@@ -0,0 +1,126 @@
+#include <slicer/slicer.h>
+#include <exceptions.h>
+#include <boost/algorithm/string/split.hpp>
+#include <boost/lexical_cast.hpp>
+#include <Ice/BuiltinSequences.h>
+
+namespace ba = boost::algorithm;
+
+extern long hextable[];
+
+namespace IceSpider {
+
+	class XWwwFormUrlEncoded : public Slicer::Deserializer {
+		public:
+			XWwwFormUrlEncoded(std::istream & in) :
+				input(std::istreambuf_iterator<char>(in), std::istreambuf_iterator<char>())
+			{
+			}
+
+			void Deserialize(Slicer::ModelPartForRootPtr mp) override
+			{
+				mp->Create();
+				mp->OnEachChild([this](auto, auto mp, auto) {
+					switch (mp->GetType()) {
+						case Slicer::mpt_Simple:
+							this->DeserializeSimple(mp);
+							break;
+						case Slicer::mpt_Complex:
+							this->DeserializeComplex(mp);
+							break;
+						case Slicer::mpt_Dictionary:
+							this->DeserializeDictionary(mp);
+							break;
+						default:
+							throw IceSpider::Http400_BadRequest();
+							break;
+					}
+				});
+				mp->Complete();
+			}
+
+		private:
+			class SetFromString : public Slicer::ValueSource {
+				public:
+					SetFromString(const std::string & v) : s(v)
+					{
+					}
+
+					void set(bool & t) const override
+					{
+						if (s == "true") t = true;
+						else if (s == "false") t = false;
+						else throw Http400_BadRequest();
+					}
+
+					void set(std::string & t) const override
+					{
+						t.reserve(s.size());
+						for (auto i = s.begin(); i != s.end(); i++) {
+							if (*i == '+') t += ' ';
+							else if (*i == '%') {
+								t += (16 * hextable[(int)*++i]) + hextable[(int)*++i];
+							}
+							else t += *i;
+						}
+					}
+
+#define SET(T) \
+					void set(T & t) const override \
+					{ \
+						t = boost::lexical_cast<T>(s); \
+					}
+
+					SET(Ice::Byte);
+					SET(Ice::Short);
+					SET(Ice::Int);
+					SET(Ice::Long);
+					SET(Ice::Float);
+					SET(Ice::Double);
+					const std::string & s;
+			};
+
+			typedef boost::function<void(const std::string &, const std::string &)> KVh;
+			void iterateVars(const KVh & h)
+			{
+				for (auto pi = ba::make_split_iterator(input, ba::first_finder("&", ba::is_equal())); pi != decltype(pi)(); ++pi) {
+					auto eq = std::find(pi->begin(), pi->end(), '=');
+					if (eq == pi->end()) {
+						h(std::string(pi->begin(), pi->end()), std::string());
+					}
+					else {
+						h(std::string(pi->begin(), eq), std::string(eq + 1, pi->end()));
+					}
+				}
+			}
+			void DeserializeSimple(Slicer::ModelPartPtr mp)
+			{
+				iterateVars([mp](auto, auto v) {
+					mp->SetValue(new SetFromString(v));
+				});
+			}
+			void DeserializeComplex(Slicer::ModelPartPtr mp)
+			{
+				mp->Create();
+				iterateVars([mp](auto k, auto v) {
+					if (auto m = mp->GetChild(k)) {
+						m->SetValue(new SetFromString(v));
+					}
+				});
+				mp->Complete();
+			}
+			void DeserializeDictionary(Slicer::ModelPartPtr mp)
+			{
+				iterateVars([mp](auto k, auto v) {
+					auto p = mp->GetAnonChild();
+					p->GetChild("key")->SetValue(new SetFromString(k));
+					p->GetChild("value")->SetValue(new SetFromString(v));
+					p->Complete();
+				});
+			}
+			const std::string input;
+	};
+}
+
+NAMEDFACTORY("application/x-www-form-urlencoded", IceSpider::XWwwFormUrlEncoded, Slicer::StreamDeserializerFactory);
+
diff --git a/icespider/unittests/Jamfile.jam b/icespider/unittests/Jamfile.jam
index f5e8f2f..4ee2bd8 100644
--- a/icespider/unittests/Jamfile.jam
+++ b/icespider/unittests/Jamfile.jam
@@ -23,6 +23,7 @@ lib slicer-json : : : : <include>/usr/include/slicer ;
 lib slicer-xml : : : : <include>/usr/include/slicer ;
 lib boost_utf : : <name>boost_unit_test_framework ;
 lib boost_system ;
+lib boost_filesystem ;
 lib dl ;
 path-constant me : . ;
 
@@ -81,14 +82,21 @@ run
 
 run
 	testFcgi.cpp
+	test-fcgi.ice
 	../fcgi/cgiRequestBase.cpp
+	../fcgi/xwwwFormUrlEncoded.cpp
+	../fcgi/hextable.c
 	: : :
+	<slicer>yes
 	<define>BOOST_TEST_DYN_LINK
 	<library>testCommon
 	<library>../common//icespider-common
 	<library>../core//icespider-core
 	<library>boost_system
+	<library>boost_filesystem
 	<library>slicer
+	<library>slicer-json
+	<library>adhocutil
 	<include>../fcgi
 	: testFcgi ;
 
diff --git a/icespider/unittests/test-fcgi.ice b/icespider/unittests/test-fcgi.ice
new file mode 100644
index 0000000..6be9086
--- /dev/null
+++ b/icespider/unittests/test-fcgi.ice
@@ -0,0 +1,10 @@
+module TestFcgi {
+	class Complex {
+		string alpha;
+		double number;
+		bool boolean;
+		string spaces;
+		string empty;
+	};
+};
+
diff --git a/icespider/unittests/testApp.cpp b/icespider/unittests/testApp.cpp
index fc7d66a..5f37f62 100644
--- a/icespider/unittests/testApp.cpp
+++ b/icespider/unittests/testApp.cpp
@@ -27,7 +27,7 @@ void forceEarlyChangeDir()
 
 BOOST_AUTO_TEST_CASE( testLoadConfiguration )
 {
-	BOOST_REQUIRE_EQUAL(10, AdHoc::PluginManager::getDefault()->getAll<IceSpider::RouteHandlerFactory>().size());
+	BOOST_REQUIRE_EQUAL(11, AdHoc::PluginManager::getDefault()->getAll<IceSpider::RouteHandlerFactory>().size());
 }
 
 class TestRequest : public IHttpRequest {
@@ -96,7 +96,7 @@ BOOST_AUTO_TEST_CASE( testCoreSettings )
 {
 	BOOST_REQUIRE_EQUAL(5, routes.size());
 	BOOST_REQUIRE_EQUAL(1, routes[0].size());
-	BOOST_REQUIRE_EQUAL(4, routes[1].size());
+	BOOST_REQUIRE_EQUAL(5, routes[1].size());
 	BOOST_REQUIRE_EQUAL(1, routes[2].size());
 	BOOST_REQUIRE_EQUAL(2, routes[3].size());
 	BOOST_REQUIRE_EQUAL(2, routes[4].size());
diff --git a/icespider/unittests/testCompile.cpp b/icespider/unittests/testCompile.cpp
index 4fb7827..8d1cefa 100644
--- a/icespider/unittests/testCompile.cpp
+++ b/icespider/unittests/testCompile.cpp
@@ -36,7 +36,7 @@ BOOST_AUTO_TEST_CASE( testLoadConfiguration )
 	rc.applyDefaults(cfg, u);
 
 	BOOST_REQUIRE_EQUAL("common", cfg->name);
-	BOOST_REQUIRE_EQUAL(10, cfg->routes.size());
+	BOOST_REQUIRE_EQUAL(11, cfg->routes.size());
 
 	BOOST_REQUIRE_EQUAL("/", cfg->routes["index"]->path);
 	BOOST_REQUIRE_EQUAL(HttpMethod::GET, cfg->routes["index"]->method);
@@ -106,6 +106,7 @@ BOOST_AUTO_TEST_CASE( testCompile )
 BOOST_AUTO_TEST_CASE( testLink )
 {
 	auto outputo = binDir / "testRoutes.o";
+	BOOST_REQUIRE(boost::filesystem::exists(outputo));
 	auto outputso = binDir / "testRoutes.so";
 
 	auto linkCommand = boost::algorithm::join<std::vector<std::string>>({
@@ -121,12 +122,13 @@ BOOST_AUTO_TEST_CASE( testLink )
 BOOST_AUTO_TEST_CASE( testLoad )
 {
 	auto outputso = binDir / "testRoutes.so";
+	BOOST_REQUIRE(boost::filesystem::exists(outputso));
 
 	auto lib = dlopen(outputso.c_str(), RTLD_NOW);
 	BOOST_TEST_INFO(dlerror());
 	BOOST_REQUIRE(lib);
 
-	BOOST_REQUIRE_EQUAL(10, AdHoc::PluginManager::getDefault()->getAll<IceSpider::RouteHandlerFactory>().size());
+	BOOST_REQUIRE_EQUAL(11, AdHoc::PluginManager::getDefault()->getAll<IceSpider::RouteHandlerFactory>().size());
 	// smoke test (block ensure dlclose dones't cause segfault)
 	{
 		auto route = AdHoc::PluginManager::getDefault()->get<IceSpider::RouteHandlerFactory>("common::index");
diff --git a/icespider/unittests/testFcgi.cpp b/icespider/unittests/testFcgi.cpp
index f2a58a4..94ce092 100644
--- a/icespider/unittests/testFcgi.cpp
+++ b/icespider/unittests/testFcgi.cpp
@@ -2,7 +2,9 @@
 #include <boost/test/unit_test.hpp>
 
 #include <core.h>
+#include <definedDirs.h>
 #include <cgiRequestBase.h>
+#include <test-fcgi.h>
 
 class TestRequest : public IceSpider::CgiRequestBase {
 	public:
@@ -24,6 +26,24 @@ class TestRequest : public IceSpider::CgiRequestBase {
 		// LCOV_EXCL_STOP
 };
 
+class TestPayloadRequest : public TestRequest {
+	public:
+		TestPayloadRequest(IceSpider::Core * c, char ** env, std::istream & s) :
+			TestRequest(c, env),
+			in(s)
+		{
+			initialize();
+		}
+
+		std::istream & getInputStream() const override
+		{
+			return in;
+		}
+
+	private:
+		std::istream & in;
+};
+
 class CharPtrPtrArray : public std::vector<char *> {
 	public:
 		CharPtrPtrArray()
@@ -172,5 +192,68 @@ BOOST_AUTO_TEST_CASE( requestmethod_bad )
 	BOOST_REQUIRE_THROW(r.getRequestMethod(), IceSpider::Http405_MethodNotAllowed);
 }
 
+BOOST_AUTO_TEST_CASE( postxwwwformurlencoded_simple )
+{
+	CharPtrPtrArray env ({ "SCRIPT_NAME=/", "REQUEST_METHOD=No", "CONTENT_TYPE=application/x-www-form-urlencoded" });
+	std::stringstream f("value=314");
+	TestPayloadRequest r(this, env, f);
+	auto n = r.getBody<int>();
+	BOOST_REQUIRE_EQUAL(314, n);
+}
+
+BOOST_AUTO_TEST_CASE( postxwwwformurlencoded_dictionary )
+{
+	CharPtrPtrArray env ({ "SCRIPT_NAME=/", "REQUEST_METHOD=No", "CONTENT_TYPE=application/x-www-form-urlencoded" });
+	std::stringstream f("alpha=abcde&number=3.14&boolean=true&spaces=This+is+a%20string.&empty=");
+	TestPayloadRequest r(this, env, f);
+	auto n = *r.getBody<IceSpider::StringMap>();
+	BOOST_REQUIRE_EQUAL(5, n.size());
+	BOOST_REQUIRE_EQUAL("abcde", n["alpha"]);
+	BOOST_REQUIRE_EQUAL("3.14", n["number"]);
+	BOOST_REQUIRE_EQUAL("true", n["boolean"]);
+	BOOST_REQUIRE_EQUAL("This is a string.", n["spaces"]);
+	BOOST_REQUIRE_EQUAL("", n["empty"]);
+}
+
+BOOST_AUTO_TEST_CASE( postxwwwformurlencoded_complex )
+{
+	CharPtrPtrArray env ({ "SCRIPT_NAME=/", "REQUEST_METHOD=No", "CONTENT_TYPE=application/x-www-form-urlencoded" });
+	std::stringstream f("alpha=abcde&number=3.14&boolean=true&empty=&spaces=This+is+a%20string.");
+	TestPayloadRequest r(this, env, f);
+	auto n = *r.getBody<TestFcgi::ComplexPtr>();
+	BOOST_REQUIRE_EQUAL("abcde", n->alpha);
+	BOOST_REQUIRE_EQUAL(3.14, n->number);
+	BOOST_REQUIRE_EQUAL(true, n->boolean);
+	BOOST_REQUIRE_EQUAL("This is a string.", n->spaces);
+	BOOST_REQUIRE_EQUAL("", n->empty);
+}
+
+BOOST_AUTO_TEST_CASE( postjson_complex )
+{
+	CharPtrPtrArray env ({ "SCRIPT_NAME=/", "REQUEST_METHOD=No", "CONTENT_TYPE=application/json" });
+	std::stringstream f("{\"alpha\":\"abcde\",\"number\":3.14,\"boolean\":true,\"empty\":\"\",\"spaces\":\"This is a string.\"}");
+	TestPayloadRequest r(this, env, f);
+	auto n = *r.getBody<TestFcgi::ComplexPtr>();
+	BOOST_REQUIRE_EQUAL("abcde", n->alpha);
+	BOOST_REQUIRE_EQUAL(3.14, n->number);
+	BOOST_REQUIRE_EQUAL(true, n->boolean);
+	BOOST_REQUIRE_EQUAL("This is a string.", n->spaces);
+	BOOST_REQUIRE_EQUAL("", n->empty);
+}
+
+BOOST_AUTO_TEST_CASE( postjson_dictionary )
+{
+	CharPtrPtrArray env ({ "SCRIPT_NAME=/", "REQUEST_METHOD=No", "CONTENT_TYPE=application/json" });
+	std::stringstream f("{\"alpha\":\"abcde\",\"number\":\"3.14\",\"boolean\":\"true\",\"empty\":\"\",\"spaces\":\"This is a string.\"}");
+	TestPayloadRequest r(this, env, f);
+	auto n = *r.getBody<IceSpider::StringMap>();
+	BOOST_REQUIRE_EQUAL(5, n.size());
+	BOOST_REQUIRE_EQUAL("abcde", n["alpha"]);
+	BOOST_REQUIRE_EQUAL("3.14", n["number"]);
+	BOOST_REQUIRE_EQUAL("true", n["boolean"]);
+	BOOST_REQUIRE_EQUAL("This is a string.", n["spaces"]);
+	BOOST_REQUIRE_EQUAL("", n["empty"]);
+}
+
 BOOST_AUTO_TEST_SUITE_END();
 
diff --git a/icespider/unittests/testRoutes.json b/icespider/unittests/testRoutes.json
index 13a6806..5bf002f 100644
--- a/icespider/unittests/testRoutes.json
+++ b/icespider/unittests/testRoutes.json
@@ -23,6 +23,17 @@
 			"method": "DELETE",
 			"operation": "TestIceSpider.TestApi.returnNothing"
 		},
+		"del2": {
+			"path": "/{s}",
+			"method": "DELETE",
+			"operation": "TestIceSpider.TestApi.returnNothing",
+			"params": {
+				"s": {
+					"source": "Body",
+					"key": "value"
+				}
+			}
+		},
 		"update": {
 			"path": "/{id}",
 			"method": "POST",