From 5b36659096afb86daa81496560a496f33f413d9c Mon Sep 17 00:00:00 2001
From: Dan Goodliffe <dan@randomdan.homeip.net>
Date: Sat, 7 May 2022 02:30:20 +0100
Subject: Fix and test performing large writes

Merges sync and async tests, variety sizes of random data, validated with direct file map
---
 netfs/fuse/fuseFiles.cpp          | 62 +++++++++++++++++++++++++++++----------
 netfs/unittests/testEdgeCases.cpp | 36 +++++++----------------
 2 files changed, 58 insertions(+), 40 deletions(-)

diff --git a/netfs/fuse/fuseFiles.cpp b/netfs/fuse/fuseFiles.cpp
index a44f56e..9baf51f 100644
--- a/netfs/fuse/fuseFiles.cpp
+++ b/netfs/fuse/fuseFiles.cpp
@@ -184,24 +184,56 @@ namespace NetFS {
 			auto remote = of->remote;
 			auto bytes = reinterpret_cast<const ::Ice::Byte *>(buf);
 			if (fcr->Async) {
-				waitOnWriteRangeAndThen<void>(s, o, of, [o, s, bytes, &of, remote](const auto & key) {
-					auto p = std::make_shared<OpenFile::WriteState>();
-					of->bg.insert({key, p});
-					remote->writeAsync(
-							o, safe {s}, std::make_pair(bytes, bytes + s),
-							[p, of, key]() {
-								p->promise.set_value();
-								ScopeLock(of->mutex) {
-									of->bg.erase(key);
-								}
-							},
-							[p, of](auto e) {
-								p->promise.set_exception(e);
+				auto sendSome = [&bytes, &o, &remote, &of](safe<size_t> blockSize) {
+					waitOnWriteRangeAndThen<void>(
+							blockSize, o, of, [o, blockSize, bytes, &of, remote](const auto & key) {
+								auto p = std::make_shared<OpenFile::WriteState>();
+								of->bg.insert({key, p});
+								remote->writeAsync(
+										o, blockSize, std::make_pair(bytes, bytes + blockSize),
+										[p, of, key]() {
+											p->promise.set_value();
+											ScopeLock(of->mutex) {
+												of->bg.erase(key);
+											}
+										},
+										[p, of](auto e) {
+											p->promise.set_exception(e);
+										});
 							});
-				});
+					bytes += blockSize;
+					o += blockSize;
+				};
+				auto bs = of->blockSizes(s);
+				if (bs.size1) {
+					sendSome(bs.size1);
+				}
+				while (bs.N--) {
+					sendSome(bs.sizeN);
+				}
 			}
 			else {
-				remote->write(o, safe {s}, std::make_pair(bytes, bytes + s));
+				if (of->bodyMaxSize < s) {
+					std::vector<std::future<void>> ops;
+					auto sendSome = [&ops, &bytes, &o, &remote](safe<size_t> blockSize) {
+						ops.emplace_back(remote->writeAsync(o, blockSize, std::make_pair(bytes, bytes + blockSize)));
+						bytes += blockSize;
+						o += blockSize;
+					};
+					auto bs = of->blockSizes(s);
+					if (bs.size1) {
+						sendSome(bs.size1);
+					}
+					while (bs.N--) {
+						sendSome(bs.sizeN);
+					}
+					std::for_each(ops.begin(), ops.end(), [](auto && op) {
+						op.get();
+					});
+				}
+				else {
+					remote->write(o, safe {s}, std::make_pair(bytes, bytes + s));
+				}
 			}
 			return safe {s};
 		}
diff --git a/netfs/unittests/testEdgeCases.cpp b/netfs/unittests/testEdgeCases.cpp
index f47ecd4..d587b13 100644
--- a/netfs/unittests/testEdgeCases.cpp
+++ b/netfs/unittests/testEdgeCases.cpp
@@ -6,7 +6,7 @@
 #include <boost/test/data/test_case.hpp>
 #include <boost/test/unit_test.hpp>
 #include <definedDirs.h>
-#include <random>
+#include <fileUtils.h>
 
 const std::string testEndpoint("tcp -h localhost -p 12014");
 
@@ -189,38 +189,24 @@ BOOST_DATA_TEST_CASE(blockSizes_scatter,
 	BOOST_CHECK_NE(bs.size1, bs.sizeN);
 }
 
-BOOST_AUTO_TEST_CASE(bigWritesAsync)
+BOOST_DATA_TEST_CASE(bigWrites,
+		boost::unit_test::data::make({"defaultFuseNoAsync.xml", "defaultFuse.xml"})
+				* boost::unit_test::data::xrange<size_t>(1024 * 100, 4096 * 1024, 512UL * 1024),
+		config, size)
 {
 	MockDaemonHost daemon(testEndpoint, {"--NetFSD.ConfigPath=" + (rootDir / "defaultDaemon.xml").string()});
-	FuseMockHost fuse(testEndpoint, {(rootDir / "defaultFuse.xml").string() + ":testvol", (rootDir / "test").string()});
+	FuseMockHost fuse(testEndpoint, {(rootDir / config).string() + ":testvol", (rootDir / "test").string()});
 
-	std::vector<char> buf(1 * 1024 * 1024);
+	const auto buf = FuseMock::genRandomData(size);
 	fuse_file_info fi {};
 	fi.flags = O_RDWR;
 	auto fd = fuse.fuse->create("/big", 0600, &fi);
 	BOOST_REQUIRE_GE(fd, 0);
 	BOOST_REQUIRE_NE(fi.fh, 0);
-	// async success
 	BOOST_CHECK_EQUAL(buf.size(), fuse.fuse->write("/big", buf.data(), buf.size(), 0, &fi));
-	// failure on flush/close
-	BOOST_REQUIRE_EQUAL(-EIO, fuse.fuse->release("/big", &fi));
-	BOOST_CHECK_EQUAL(0, std::filesystem::file_size(daemon.TestExportRoot / "big"));
-}
-
-BOOST_AUTO_TEST_CASE(bigWritesNoAsync)
-{
-	MockDaemonHost daemon(testEndpoint, {"--NetFSD.ConfigPath=" + (rootDir / "defaultDaemon.xml").string()});
-	FuseMockHost fuse(
-			testEndpoint, {(rootDir / "defaultFuseNoAsync.xml").string() + ":testvol", (rootDir / "test").string()});
-
-	std::vector<char> buf(1 * 1024 * 1024);
-	fuse_file_info fi {};
-	fi.flags = O_RDWR;
-	auto fd = fuse.fuse->create("/big", 0600, &fi);
-	BOOST_REQUIRE_GE(fd, 0);
-	BOOST_REQUIRE_NE(fi.fh, 0);
-	// sync fail
-	BOOST_CHECK_EQUAL(-EIO, fuse.fuse->write("/big", buf.data(), buf.size(), 0, &fi));
 	BOOST_REQUIRE_EQUAL(0, fuse.fuse->release("/big", &fi));
-	BOOST_CHECK_EQUAL(0, std::filesystem::file_size(daemon.TestExportRoot / "big"));
+
+	AdHoc::FileUtils::MemMap mm(daemon.TestExportRoot / "big");
+	const auto file {mm.sv()};
+	BOOST_CHECK_EQUAL_COLLECTIONS(buf.begin(), buf.end(), file.begin(), file.end());
 }
-- 
cgit v1.2.3