Fix up behaviour of table based SI parsers to stop only when they have received all on a required set of tables. Required updated sample SI packets which were actually complete and revisions to test cases. Allows for removal of hacky HasPending workaround.

16 files changed, 276 insertions, 84 deletions

diff --git a/p2pvr/devices/Jamfile.jam b/p2pvr/devices/Jamfile.jam
index 92573d5..bb9f70e 100644
--- a/p2pvr/devices/Jamfile.jam
+++ b/p2pvr/devices/Jamfile.jam
@@ -1,5 +1,6 @@
 import type ;
 import generators ;
+import testing ;
 
 lib boost_system ;
 lib boost_thread ;
@@ -8,7 +9,7 @@ lib boost_program_options ;
 lib lzma ;
 
 lib p2pvrdevices :
-	[ glob-tree *.cpp : mockTuner.cpp ]
+	[ glob-tree *.cpp : mock*.cpp ]
 	:
 	<library>boost_system
 	<library>boost_filesystem
@@ -40,9 +41,8 @@ IMPORT $(__name__) : datxz.embed.asm : : datxz.embed.asm ;
 
 lib p2pvrMockTuner :
 	mockTuner.cpp
-	[ glob-tree *.datxz ]
 	:
-	<library>lzma
+	<library>mockdata
 	<library>boost_system
 	<library>boost_thread
 	<library>boost_filesystem
@@ -60,3 +60,25 @@ lib p2pvrMockTuner :
 	<include>.
 	;
 
+lib mockdata :
+	mockData.cpp
+	[ glob-tree *.datxz ]
+	:
+	<library>lzma
+	<library>../ice//p2pvrice
+	<implicit-dependency>../ice//p2pvrice
+	;
+
+run
+	mockDataValidate.cpp
+	: : :
+	<library>../dvb//p2pvrdvb
+	<library>../ice//p2pvrice
+	<library>..//adhocutil
+	<library>..//icetray
+	<library>mockdata
+	<library>boost_filesystem
+	<library>boost_system
+	<library>..//boost_utf
+	<define>BOOST_TEST_DYN_LINK
+	;
diff --git a/p2pvr/devices/mockData.cpp b/p2pvr/devices/mockData.cpp
new file mode 100644
index 0000000..31a7100
--- /dev/null
+++ b/p2pvr/devices/mockData.cpp
@@ -0,0 +1,75 @@
+#include "mockData.h"
+#include <lzma.h>
+#include <Ice/Stream.h>
+#include <Ice/Initialize.h>
+#include <Ice/Communicator.h>
+
+#define ResourceFile(resource) \
+extern "C" { \
+	extern char resource##_start, resource##_end;\
+	extern unsigned int resource##_len; \
+}
+
+namespace P2PVR {
+	namespace DVB {
+		namespace Testing {
+			ResourceFile(network);
+			ResourceFile(services);
+			ResourceFile(events1);
+			ResourceFile(events2);
+			ResourceFile(pat);
+			ResourceFile(pmt);
+			ResourceFile(vid);
+
+			void LZMA_ASSERT(int ret_xz)
+			{
+				if (ret_xz != LZMA_OK) {
+					throw std::runtime_error("LZMA Decompressor error");
+				}
+			}
+
+			MockData::MockData(Ice::CommunicatorPtr c) : ic(c) { }
+
+			::P2PVR::Data MockData::decompress(const char * front, size_t len) const
+			{
+				lzma_stream strm = LZMA_STREAM_INIT;
+				const uint32_t flags = LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED;
+				const uint64_t memory_limit = UINT64_MAX;
+				LZMA_ASSERT(lzma_stream_decoder(&strm, memory_limit, flags));
+				strm.next_in = (unsigned char *)front;
+				strm.avail_in = len;
+				uint8_t buf[BUFSIZ];
+
+				::P2PVR::Data data;
+				data.reserve(len * 20);
+				do {
+					strm.next_out = buf;
+					strm.avail_out = BUFSIZ;
+					LZMA_ASSERT(lzma_code(&strm, LZMA_RUN));
+					for (auto idx = 0u; idx < BUFSIZ - strm.avail_out; idx += 1) {
+						data.push_back(buf[idx]);
+					}
+				} while (strm.avail_out == 0);
+				data.shrink_to_fit();
+				return data;
+			}
+
+			MockData::PacketList MockData::decompressAndRead(const char * front, size_t len) const
+			{
+				PacketList packets;
+				auto istrm = Ice::createInputStream(ic, decompress(front, len));
+				istrm->read(packets);
+				return packets;
+			}
+
+			MockData::PacketList MockData::getNetwork() const { return decompressAndRead(&network_start, network_len); }
+			MockData::PacketList MockData::getServices() const { return decompressAndRead(&services_start, services_len); }
+			MockData::PacketList MockData::getEvents1() const { return decompressAndRead(&events1_start, events1_len); }
+			MockData::PacketList MockData::getEvents2() const { return decompressAndRead(&events2_start, events2_len); }
+			MockData::PacketList MockData::getPAT() const { return decompressAndRead(&pat_start, pat_len); }
+			MockData::PacketList MockData::getPMT() const { return decompressAndRead(&pmt_start, pmt_len); }
+			MockData::PacketList MockData::getVideo() const { return decompressAndRead(&vid_start, vid_len); }
+		}
+	}
+}
+
diff --git a/p2pvr/devices/mockData.h b/p2pvr/devices/mockData.h
new file mode 100644
index 0000000..b86807d
--- /dev/null
+++ b/p2pvr/devices/mockData.h
@@ -0,0 +1,35 @@
+#ifndef P2PVR_DEVICES_MOCKDATA_H
+#define P2PVR_DEVICES_MOCKDATA_H
+
+#include <dvb.h>
+#include <list>
+#include <visibility.h>
+
+namespace P2PVR {
+	namespace DVB {
+		namespace Testing {
+			class DLL_PUBLIC MockData {
+				public:
+					typedef std::list<::P2PVR::Data> PacketList;
+
+					MockData(Ice::CommunicatorPtr);
+
+					PacketList getNetwork() const;
+					PacketList getServices() const;
+					PacketList getEvents1() const;
+					PacketList getEvents2() const;
+					PacketList getPMT() const;
+					PacketList getPAT() const;
+					PacketList getVideo() const;
+
+				protected:
+					DLL_PRIVATE ::P2PVR::Data decompress(const char * front, size_t len) const;
+					DLL_PRIVATE PacketList decompressAndRead(const char * front, size_t len) const;
+
+					Ice::CommunicatorPtr ic;
+			};
+		}
+	}
+}
+
+#endif
diff --git a/p2pvr/devices/mockDataValidate.cpp b/p2pvr/devices/mockDataValidate.cpp
new file mode 100644
index 0000000..6bd6b5b
--- /dev/null
+++ b/p2pvr/devices/mockDataValidate.cpp
@@ -0,0 +1,126 @@
+#define BOOST_TEST_MODULE ValidateTestData
+#include <boost/test/unit_test.hpp>
+#include <Ice/Communicator.h>
+#include <Ice/Initialize.h>
+#include <siParsers/event.h>
+#include <siParsers/network.h>
+#include <siParsers/programAssociation.h>
+#include <siParsers/programMap.h>
+#include <siParsers/service.h>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/bind.hpp>
+#include "mockData.h"
+
+BOOST_TEST_DONT_PRINT_LOG_VALUE(P2PVR::DVB::Testing::MockData::PacketList::const_iterator);
+
+template <typename SIObject, typename Base>
+class DiscardSI : public Base, public P2PVR::DVB::Testing::MockData {
+	public:
+		DiscardSI() :
+			MockData(Ice::initialize({}))
+		{
+		}
+		DiscardSI(const P2PVR::DVBSI::SiTableParserBase::RequiredContentIds & ids) :
+			Base(ids),
+			MockData(Ice::initialize({}))
+		{
+		}
+		virtual ~DiscardSI() { ic->destroy(); }
+
+		bool HandleTable(const SIObject & o) override
+		{
+			objects.push_back(o);
+			return false;
+		}
+
+		std::list<SIObject> validate(const P2PVR::DVB::Testing::MockData::PacketList & packets, unsigned int count, unsigned int objectCount)
+		{
+			BOOST_REQUIRE_EQUAL(count, packets.size());
+			auto ends = std::count_if(packets.begin(), packets.end(), [this](const auto & p) { return this->NewData(p, current); });
+			for (const auto & x : Base::parents) {
+				BOOST_TEST_INFO(x);
+			}
+			BOOST_REQUIRE_GT(ends, 0);
+			BOOST_REQUIRE_EQUAL(objectCount, objects.size());
+			return objects;
+		}
+
+	protected:
+		std::list<SIObject> objects;
+		Ice::Current current;
+};
+
+::DVBSI::NetworkPtr network;
+std::list<::DVBSI::TransportStreamPtr> streams;
+class NetworkTester : public DiscardSI<::DVBSI::NetworkPtr, ::P2PVR::DVBSI::SiNetworkInformationParser> {
+	public:
+};
+class ServiceTester : public DiscardSI<::DVBSI::TransportStreamPtr, ::P2PVR::DVBSI::SiServicesParser> {
+	public:
+		ServiceTester() : DiscardSI<::DVBSI::TransportStreamPtr, ::P2PVR::DVBSI::SiServicesParser>(getIds())
+		{
+		}
+		static P2PVR::DVBSI::SiTableParserBase::RequiredContentIds getIds()
+		{
+			P2PVR::DVBSI::SiTableParserBase::RequiredContentIds ids;
+			for (const auto & t : network->TransportStreams) {
+				if (t->Satellite || t->Cable || t->Terrestrial) {
+					ids.insert(t->TransportStreamId);
+				}
+			}
+			return ids;
+		}
+};
+class Events1Tester : public DiscardSI<::DVBSI::EventPtr, ::P2PVR::DVBSI::SiEpgParser> {
+	public:
+		Events1Tester() : DiscardSI<::DVBSI::EventPtr, ::P2PVR::DVBSI::SiEpgParser>(getIds())
+		{
+		}
+		static P2PVR::DVBSI::SiTableParserBase::RequiredContentIds getIds()
+		{
+			P2PVR::DVBSI::SiTableParserBase::RequiredContentIds ids;
+			for (const auto & st : streams) {
+				for (const auto & s : st->Services) {
+					if (s->EitSchedule) {
+						ids.insert(s->ServiceId);
+					}
+				}
+			}
+			return ids;
+		}
+};
+typedef DiscardSI<::DVBSI::ProgramMapPtr, ::P2PVR::DVBSI::SiProgramMapParser> PMTester;
+typedef DiscardSI<::P2PVR::DVBSI::ProgramAssociationMapPtr, ::P2PVR::DVBSI::SiProgramAssociationParser> PATester;
+
+BOOST_FIXTURE_TEST_SUITE(ValidateTestDataNetwork, NetworkTester);
+
+BOOST_AUTO_TEST_CASE( networkPackets )
+{
+	network = validate(getNetwork(), 3, 1).front();
+}
+
+BOOST_AUTO_TEST_SUITE_END();
+
+BOOST_FIXTURE_TEST_SUITE(ValidateTestDataService, ServiceTester);
+
+BOOST_AUTO_TEST_CASE( servicePackets )
+{
+	streams = validate(getServices(), 25, 8);
+}
+
+BOOST_AUTO_TEST_SUITE_END();
+
+BOOST_FIXTURE_TEST_SUITE(ValidateTestDataEvents1, Events1Tester);
+
+BOOST_AUTO_TEST_CASE( events1Packets )
+{
+	validate(getEvents1(), 64266, 23132);
+}
+
+BOOST_AUTO_TEST_CASE( events2Packets )
+{
+	validate(getEvents2(), 64847, 23126);
+}
+
+BOOST_AUTO_TEST_SUITE_END();
+
diff --git a/p2pvr/devices/mockTuner.cpp b/p2pvr/devices/mockTuner.cpp
index 8e772c6..706bbd9 100644
--- a/p2pvr/devices/mockTuner.cpp
+++ b/p2pvr/devices/mockTuner.cpp
@@ -12,24 +12,8 @@ extern "C" { \
 static const Ice::ByteSeq resource(&resource##_start, &resource##_end);
 
 namespace P2PVR {
+namespace DVB {
 namespace Testing {
-void
-MockTuner::LZMA_ASSERT(int ret_xz)
-{
-	if (ret_xz != LZMA_OK) {
-		MockTuner::logger->messagebf(LOG::ERR, "%s: LZMA error (%d)", __PRETTY_FUNCTION__, ret_xz);
-		throw P2PVR::DeviceError("LZMA", "Decompressor error", ret_xz);
-	}
-}
-
-ResourceFile(network);
-ResourceFile(services);
-ResourceFile(events1);
-ResourceFile(events2);
-ResourceFile(pat);
-ResourceFile(pmt);
-ResourceFile(vid);
-
 int MockTuner::eventSet = 0;
 
 IceTray::Logging::LoggerPtr MockTuner::logger = LOGMANAGER()->getLogger<MockTuner>();
@@ -61,45 +45,11 @@ class MockFrontend : public DVB::Frontend {
 
 MockTuner::MockTuner(const boost::filesystem::path & deviceFrontend, Ice::CommunicatorPtr c) :
 	TunerI(deviceFrontend, DVB::FrontendPtr(new MockFrontend(this, dvb_frontend_info(), logger))),
+	MockData(c),
 	ic(c)
 {
 }
 
-Ice::ByteSeq MockTuner::Decompress(const Ice::ByteSeq & dataxz)
-{
-	lzma_stream strm = LZMA_STREAM_INIT;
-	const uint32_t flags = LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED;
-	const uint64_t memory_limit = UINT64_MAX;
-	LZMA_ASSERT(lzma_stream_decoder(&strm, memory_limit, flags));
-	strm.next_in = &dataxz.front();
-	strm.avail_in = dataxz.size();
-	uint8_t buf[BUFSIZ];
-
-	Ice::ByteSeq data;
-	data.reserve(dataxz.size() * 20);
-	do {
-		strm.next_out = buf;
-		strm.avail_out = BUFSIZ;
-		LZMA_ASSERT(lzma_code(&strm, LZMA_RUN));
-		for (auto idx = 0u; idx < BUFSIZ - strm.avail_out; idx += 1) {
-			data.push_back(buf[idx]);
-		}
-	} while (strm.avail_out == 0);
-	data.shrink_to_fit();
-	return data;
-}
-
-std::list<Ice::ByteSeq>
-MockTuner::DecompressAndRead(const Ice::ByteSeq & dataxz) const
-{
-	logger->messagef(LOG::DEBUG, "%s: decompress %zu bytes", __PRETTY_FUNCTION__, dataxz.size());
-	std::list<Ice::ByteSeq> packets;
-	auto istrm = Ice::createInputStream(ic, Decompress(dataxz));
-	istrm->read(packets);
-	logger->messagef(LOG::DEBUG, "%s: read %zu packets", __PRETTY_FUNCTION__, packets.size());
-	return packets;
-}
-
 AdHoc::FileUtils::FileHandle
 MockTuner::OpenDemux() const
 {
@@ -111,6 +61,7 @@ MockTuner::iopoll(struct pollfd *fds, nfds_t nfds, int) const
 {
 	auto & packets = selectedPackets[fds[0].fd];
 	if (packets.empty()) {
+		sleep(1);
 		return -2;
 	}
 	if (nfds > 0) {
@@ -169,19 +120,19 @@ MockTuner::RequestPID(int pid, int fh) const
 	logger->messagef(LOG::DEBUG, "%s: pid %x, fh %d", __PRETTY_FUNCTION__, pid, fh);
 	switch (pid) {
 		case 0: // pat
-			selectedPackets[fh] = DecompressAndRead(pat);
+			selectedPackets[fh] = getPAT();
 			break;
 		case 0x10:
-			selectedPackets[fh] = DecompressAndRead(network);
+			selectedPackets[fh] = getNetwork();
 			break;
 		case 0x11:
-			selectedPackets[fh] = DecompressAndRead(services);
+			selectedPackets[fh] = getServices();
 			break;
 		case 0x12:
-			selectedPackets[fh] = DecompressAndRead(eventSet == 0 ? events1 : events2);
+			selectedPackets[fh] = (eventSet == 0 ? getEvents1() : getEvents2());
 			break;
 		case 100: // sample pmt
-			selectedPackets[fh] = DecompressAndRead(pmt);
+			selectedPackets[fh] = getPMT();
 			break;
 	}
 }
@@ -190,9 +141,10 @@ void
 MockTuner::RequestTS(const PacketIds & pids, int fh) const
 {
 	logger->messagef(LOG::DEBUG, "%s: pids %zu, fh %d", __PRETTY_FUNCTION__, pids.size(), fh);
-	selectedPackets[fh] = DecompressAndRead(vid);
+	selectedPackets[fh] = getVideo();
 }
 
 }
 }
+}
 
diff --git a/p2pvr/devices/mockTuner.h b/p2pvr/devices/mockTuner.h
index 74578db..2277deb 100644
--- a/p2pvr/devices/mockTuner.h
+++ b/p2pvr/devices/mockTuner.h
@@ -7,10 +7,12 @@
 #include <boost/thread.hpp>
 #include <logger.h>
 #include "tuner.h"
+#include "mockData.h"
 
 namespace P2PVR {
+namespace DVB {
 namespace Testing {
-class DLL_PUBLIC MockTuner : public DVB::TunerI {
+class DLL_PUBLIC MockTuner : public DVB::TunerI, MockData {
 	public:
 		MockTuner(const boost::filesystem::path & deviceFrontend, Ice::CommunicatorPtr);
 
@@ -25,10 +27,6 @@ class DLL_PUBLIC MockTuner : public DVB::TunerI {
 		ssize_t ioread(int fd, void *buf, size_t count) const override;
 		int ioselect(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout) const override;
 
-		static Ice::ByteSeq Decompress(const Ice::ByteSeq &);
-		static void LZMA_ASSERT(int ret_xz);
-		std::list<Ice::ByteSeq> DecompressAndRead(const Ice::ByteSeq &) const;
-
 		Ice::CommunicatorPtr ic;
 		// The open fh -> list of DVB packets
 		mutable std::map<int, std::list<Ice::ByteSeq>> selectedPackets;
@@ -38,6 +36,7 @@ class DLL_PUBLIC MockTuner : public DVB::TunerI {
 };
 }
 }
+}
 
 #endif
 
diff --git a/p2pvr/devices/sampleSiData/events1.datxz b/p2pvr/devices/sampleSiData/events1.datxz
index 43358a9..dff36dc 100644
--- a/p2pvr/devices/sampleSiData/events1.datxz
+++ b/p2pvr/devices/sampleSiData/events1.datxz
diff --git a/p2pvr/devices/sampleSiData/events2.datxz b/p2pvr/devices/sampleSiData/events2.datxz
index b2fa9c2..035bfaf 100644
--- a/p2pvr/devices/sampleSiData/events2.datxz
+++ b/p2pvr/devices/sampleSiData/events2.datxz
diff --git a/p2pvr/devices/sampleSiData/network.datxz b/p2pvr/devices/sampleSiData/network.datxz
index f2e74cd..b779e5f 100644
--- a/p2pvr/devices/sampleSiData/network.datxz
+++ b/p2pvr/devices/sampleSiData/network.datxz
diff --git a/p2pvr/devices/sampleSiData/services.datxz b/p2pvr/devices/sampleSiData/services.datxz
index 0961e92..89509d7 100644
--- a/p2pvr/devices/sampleSiData/services.datxz
+++ b/p2pvr/devices/sampleSiData/services.datxz
diff --git a/p2pvr/devices/tuner.cpp b/p2pvr/devices/tuner.cpp
index d3d0488..7358428 100644
--- a/p2pvr/devices/tuner.cpp
+++ b/p2pvr/devices/tuner.cpp
@@ -181,7 +181,6 @@ TunerI::ReadDemuxAndSend(AdHoc::FileUtils::FileHandle demux, const RawDataClient
 		// Wait for data to appear
 		switch (iopoll(&ufd, 1, options->DemuxReadTimeout)) {
 			case -2:
-				//logger->messagef(LOG::DEBUG, "%s: All test data has been read... waiting...", __PRETTY_FUNCTION__);
 				break;
 			case -1:
 				logger->messagef(LOG::ERR, "%s: poll error reading demux (%d:%s)", __PRETTY_FUNCTION__, errno, strerror(errno));
@@ -203,12 +202,10 @@ TunerI::ReadDemuxAndSend(AdHoc::FileUtils::FileHandle demux, const RawDataClient
 					size_t n = nr;
 					buf.resize(n);
 
-						logger->messagef(LOG::DEBUG, "%s: sending", __PRETTY_FUNCTION__);
 					client->NewData(buf);
-						logger->messagef(LOG::DEBUG, "%s: sent", __PRETTY_FUNCTION__);
 				}
 		}
-	} while (!client->IsFinished() && client->HasPending());
+	} while (!client->IsFinished());
 	auto packetsSent = client->PacketsSent();
 	client.reset();
 	logger->messagebf(LOG::DEBUG, "%s: end (sent %d packets)", __PRETTY_FUNCTION__, packetsSent);
diff --git a/p2pvr/devices/tuner.h b/p2pvr/devices/tuner.h
index 983d3ff..5a70e94 100644
--- a/p2pvr/devices/tuner.h
+++ b/p2pvr/devices/tuner.h
@@ -31,7 +31,6 @@ class DLL_PUBLIC TunerI : public Tuner {
 
 				virtual void NewData(const Data &) = 0;
 				virtual bool IsFinished() = 0;
-				virtual bool HasPending() = 0;
 				uint64_t PacketsSent() const;
 				int fileHandle() const;
 
diff --git a/p2pvr/devices/tunerSendSi.cpp b/p2pvr/devices/tunerSendSi.cpp
index 822ec14..76eb2b7 100644
--- a/p2pvr/devices/tunerSendSi.cpp
+++ b/p2pvr/devices/tunerSendSi.cpp
@@ -66,12 +66,6 @@ SendSiStream::IsFinished()
 }
 
 bool
-SendSi::HasPending()
-{
-	return !asyncs.empty();
-}
-
-bool
 SendSi::IsValidSection(const Data & buf)
 {
 	auto n = buf.size();
diff --git a/p2pvr/devices/tunerSendSi.h b/p2pvr/devices/tunerSendSi.h
index e6517c6..a2b8b4f 100644
--- a/p2pvr/devices/tunerSendSi.h
+++ b/p2pvr/devices/tunerSendSi.h
@@ -13,7 +13,6 @@ class SendSi : public TunerI::IDataSender {
 
 		void NewData(const P2PVR::Data &) override;
 		bool IsFinished() override;
-		bool HasPending() override;
 
 	protected:
 		static bool crc32(const P2PVR::Data &);
diff --git a/p2pvr/devices/tunerSendTs.cpp b/p2pvr/devices/tunerSendTs.cpp
index 23bc3b4..0cddb93 100644
--- a/p2pvr/devices/tunerSendTs.cpp
+++ b/p2pvr/devices/tunerSendTs.cpp
@@ -77,11 +77,6 @@ SendTs::IsFinished()
 	}
 }
 
-bool
-SendTs::HasPending()
-{
-	return async;
-}
 }
 }
 
diff --git a/p2pvr/devices/tunerSendTs.h b/p2pvr/devices/tunerSendTs.h
index 5390882..a3f584c 100644
--- a/p2pvr/devices/tunerSendTs.h
+++ b/p2pvr/devices/tunerSendTs.h
@@ -13,7 +13,6 @@ class SendTs : public TunerI::IDataSender {
 
 		void NewData(const P2PVR::Data &);
 		bool IsFinished();
-		bool HasPending();
 
 	private:
 		void sendBufferChunk();