Fix up behaviour of table based SI parsers to stop only when they have received all on a required set of tables. Required updated sample SI packets which were actually complete and revisions to test cases. Allows for removal of hacky HasPending workaround.

8 files changed, 60 insertions, 23 deletions

diff --git a/p2pvr/daemon/maintenance/events.cpp b/p2pvr/daemon/maintenance/events.cpp
index 2dd91af..fbdeb51 100644
--- a/p2pvr/daemon/maintenance/events.cpp
+++ b/p2pvr/daemon/maintenance/events.cpp
@@ -8,13 +8,15 @@
 #include <sqlWriter.h>
 #include <slicer/modelPartsTypes.impl.h>
 #include "sql/maintenance/pruneEvents.sql.h"
+#include "sql/si/serviceIdsWithEit.sql.h"
 
 namespace P2PVR {
 class SiEventsStream : public Slicer::Stream<::DVBSI::EventPtr> {
 	private:
 		class SiEventsHandler : public DVBSI::SiEpgParser {
 			public:
-				SiEventsHandler(const Consumer & c, IceTray::Logging::LoggerPtr l) :
+				SiEventsHandler(const Consumer & c, DB::Connection * d, IceTray::Logging::LoggerPtr l) :
+					DVBSI::SiEpgParser(getIds(d)),
 					consumer(c),
 					logger(l)
 				{
@@ -28,14 +30,22 @@ class SiEventsStream : public Slicer::Stream<::DVBSI::EventPtr> {
 					return false;
 				}
 
+				static RequiredContentIds getIds(DB::Connection * db)
+				{
+					auto sel = sql::si::serviceIdsWithEit.select(db);
+					auto ids = Slicer::DeserializeAny<Slicer::SqlSelectDeserializer, std::vector<Ice::Int>>(*sel);
+					return {ids.begin(), ids.end()};
+				}
+
 			private:
 				const Consumer & consumer;
 				const IceTray::Logging::LoggerPtr logger;
 		};
 
 	public:
-		SiEventsStream(const Ice::Current & i, IceTray::Logging::LoggerPtr l) :
+		SiEventsStream(const Ice::Current & i, DB::Connection * d, IceTray::Logging::LoggerPtr l) :
 			ice(i),
+			db(d),
 			logger(l)
 		{
 		}
@@ -55,13 +65,14 @@ class SiEventsStream : public Slicer::Stream<::DVBSI::EventPtr> {
 			}
 			logger->messagebf(LOG::DEBUG, "%s: Getting a tuner", __PRETTY_FUNCTION__);
 			logger->messagebf(LOG::DEBUG, "%s: Fetching events", __PRETTY_FUNCTION__);
-			TemporaryIceAdapterObject<RawDataClient> parser(ice.adapter, new SiEventsHandler(ch, logger));
+			TemporaryIceAdapterObject<RawDataClient> parser(ice.adapter, new SiEventsHandler(ch, db, logger));
 
 			devs->SendEventInformation(delivery, parser);
 		}
 
 	private:
 		const Ice::Current & ice;
+		DB::Connection * db;
 		const IceTray::Logging::LoggerPtr logger;
 };
 
@@ -71,7 +82,7 @@ MaintenanceI::UpdateEvents(const Ice::Current & ice)
 	auto dbc = db->get();
 	auto ic = ice.adapter->getCommunicator();
 
-	SiEventsStream stream(ice, logger);
+	SiEventsStream stream(ice, dbc.get(), logger);
 
 	DB::TransactionScope tx(dbc.get());
 	DB::TablePatch tp;
diff --git a/p2pvr/daemon/maintenance/network.cpp b/p2pvr/daemon/maintenance/network.cpp
index c514175..2f74050 100644
--- a/p2pvr/daemon/maintenance/network.cpp
+++ b/p2pvr/daemon/maintenance/network.cpp
@@ -43,7 +43,9 @@ class SiNetworkInformationMerger : public DVBSI::SiNetworkInformationParser {
 			::DVBSI::NetworkServiceList services;
 			for (const auto & ts : n->TransportStreams) {
 				for (const auto & s : ts->Services) {
-					services.push_back(s);
+					if (std::find_if(services.begin(), services.end(), [&s](const auto & es) { return es.ServiceId == s.ServiceId; }) == services.end()) {
+						services.push_back(s);
+					}
 				}
 			}
 			DB::TablePatch mergeServices;
@@ -62,7 +64,9 @@ class SiNetworkInformationMerger : public DVBSI::SiNetworkInformationParser {
 			std::vector<Delivery> dels;
 			for (const auto & s : n->TransportStreams) {
 				if (auto d = s.get()->*member) {
-					dels.push_back(d);
+					if (std::find_if(dels.begin(), dels.end(), [&d](const auto & ed) { return ed->Frequency == d->Frequency; }) == dels.end()) {
+						dels.push_back(d);
+					}
 				}
 			}
 			Slicer::SerializeAny<Slicer::SqlTablePatchSerializer>(dels, dbc, tp);
diff --git a/p2pvr/daemon/maintenance/services.cpp b/p2pvr/daemon/maintenance/services.cpp
index 12b23ce..288dd99 100644
--- a/p2pvr/daemon/maintenance/services.cpp
+++ b/p2pvr/daemon/maintenance/services.cpp
@@ -6,11 +6,13 @@
 #include <slicer/slicer.h>
 #include <db/sqlTablePatchSerializer.h>
 #include <tablepatch.h>
+#include <sql/si/allDeliveriesTransportStreamId.sql.h>
 
 namespace P2PVR {
 class SiServicesMerger : public DVBSI::SiServicesParser {
 	public:
 		SiServicesMerger(IceTray::Logging::LoggerPtr l, DB::Connection * d) :
+			DVBSI::SiServicesParser(getIds(d)),
 			logger(l),
 			dbc(d)
 		{
@@ -34,6 +36,13 @@ class SiServicesMerger : public DVBSI::SiServicesParser {
 			return false;
 		}
 
+		static RequiredContentIds getIds(DB::Connection * db)
+		{
+			auto sel = sql::si::allDeliveriesTransportStreamId.select(db);
+			auto ids = Slicer::DeserializeAny<Slicer::SqlSelectDeserializer, std::vector<Ice::Int>>(*sel);
+			return {ids.begin(), ids.end()};
+		}
+
 	private:
 		IceTray::Logging::LoggerPtr logger;
 		DB::Connection * dbc;
diff --git a/p2pvr/daemon/sql/si/allDeliveriesTransportStreamId.sql b/p2pvr/daemon/sql/si/allDeliveriesTransportStreamId.sql
new file mode 100644
index 0000000..55ff3c8
--- /dev/null
+++ b/p2pvr/daemon/sql/si/allDeliveriesTransportStreamId.sql
@@ -0,0 +1,2 @@
+SELECT transportStreamId
+FROM allDeliveries
diff --git a/p2pvr/daemon/sql/si/serviceIdsWithEit.sql b/p2pvr/daemon/sql/si/serviceIdsWithEit.sql
new file mode 100644
index 0000000..ca4883d
--- /dev/null
+++ b/p2pvr/daemon/sql/si/serviceIdsWithEit.sql
@@ -0,0 +1,4 @@
+SELECT s.serviceId
+FROM allDeliveries d, services s
+WHERE d.transportStreamId = s.transportStreamId
+AND s.eitschedule
diff --git a/p2pvr/daemon/unittests/mockDevices.cpp b/p2pvr/daemon/unittests/mockDevices.cpp
index 5ff2cb9..e5ec6a4 100644
--- a/p2pvr/daemon/unittests/mockDevices.cpp
+++ b/p2pvr/daemon/unittests/mockDevices.cpp
@@ -13,7 +13,7 @@ namespace P2PVR {
 		TunerPtr
 		MockDevices::openTuner(const boost::filesystem::path & path) const
 		{
-			return new MockTuner(path, ic);
+			return new P2PVR::DVB::Testing::MockTuner(path, ic);
 		}
 	}
 }
diff --git a/p2pvr/daemon/unittests/testErrorHandling.cpp b/p2pvr/daemon/unittests/testErrorHandling.cpp
index 80af051..5b53999 100644
--- a/p2pvr/daemon/unittests/testErrorHandling.cpp
+++ b/p2pvr/daemon/unittests/testErrorHandling.cpp
@@ -32,10 +32,15 @@ class Core : public StandardMockDatabase {
 
 class TestDataClient : public RawDataClient {
 	public:
+		TestDataClient() :
+			packets(3)
+		{
+		}
 		virtual bool NewData(const Data &, const Ice::Current &) override
 		{
-			return false;
+			return !--packets;
 		}
+		unsigned int packets;
 };
 
 class FailingTestClient : public RawDataClient {
diff --git a/p2pvr/daemon/unittests/testMaint.cpp b/p2pvr/daemon/unittests/testMaint.cpp
index e918d74..a166d04 100644
--- a/p2pvr/daemon/unittests/testMaint.cpp
+++ b/p2pvr/daemon/unittests/testMaint.cpp
@@ -123,7 +123,7 @@ BOOST_AUTO_TEST_CASE( GetDeliveryForTransport )
 BOOST_AUTO_TEST_CASE( GetServices_stubs )
 {
 	auto services = si->GetServices();
-	BOOST_REQUIRE_EQUAL(services.size(), 145);
+	BOOST_REQUIRE_EQUAL(services.size(), 180);
 	BOOST_REQUIRE_EQUAL(services[0]->ServiceId, 4170);
 	BOOST_REQUIRE_EQUAL(services[0]->TransportStreamId, 4170);
 	// Definitely a stub
@@ -164,7 +164,7 @@ BOOST_AUTO_TEST_CASE( update_services )
 BOOST_AUTO_TEST_CASE( GetServices )
 {
 	auto services = si->GetServices();
-	BOOST_REQUIRE_EQUAL(services.size(), 145);
+	BOOST_REQUIRE_EQUAL(services.size(), 180);
 	BOOST_REQUIRE_EQUAL(services[0]->ServiceId, 4170);
 	BOOST_REQUIRE_EQUAL(services[0]->TransportStreamId, 4170);
 	BOOST_REQUIRE_EQUAL(services[0]->Name, "BBC ONE Yorks");
@@ -186,17 +186,17 @@ BOOST_AUTO_TEST_CASE( update_events )
 	auto db = DB::ConnectionPtr(DB::MockDatabase::openConnectionTo("postgres"));
 
 	BOOST_TEST_CHECKPOINT("Write first events");
-	MockTuner::SetEventsSet(0);
+	P2PVR::DVB::Testing::MockTuner::SetEventsSet(0);
 	maint->UpdateEvents();
 	auto dayOneEvents = si->EventSearch(IceUtil::Optional<std::string>(), IceUtil::Optional<int>(),
-			Common::DateTime {2014, 12, 18, 3, 0}, Common::DateTime {2014, 12, 19, 3, 0});
-	BOOST_REQUIRE_EQUAL(dayOneEvents.size(), 3345);
-	BOOST_REQUIRE(si->GetEvent(14448, 27052));
+			Common::DateTime {2017, 8, 22, 3, 0}, Common::DateTime {2017, 8, 23, 3, 0});
+	BOOST_REQUIRE_EQUAL(dayOneEvents.size(), 3023);
+	BOOST_REQUIRE(si->GetEvent(15048, 29747));
 	BOOST_REQUIRE_THROW(si->GetEvent(15856, 3591), P2PVR::NotFound);
 
 	BOOST_TEST_CHECKPOINT("Fake some recorded stuff");
-	auto keyEvent1 = si->GetEvent(25664, 55689);
-	auto keyEvent2 = si->GetEvent(24256, 1014);
+	auto keyEvent1 = dayOneEvents[1000];
+	auto keyEvent2 = dayOneEvents[2000];
 	db->execute("INSERT INTO schedules(repeats) VALUES(false)");
 	auto irecorded = boost::shared_ptr<DB::ModifyCommand>(
 			db->newModifyCommand("INSERT INTO recorded(scheduleId, eventUid) VALUES(?, ?)"));
@@ -211,21 +211,23 @@ BOOST_AUTO_TEST_CASE( update_events )
 	BOOST_REQUIRE(keyEvent1);
 	BOOST_REQUIRE(keyEvent2);
 	BOOST_REQUIRE(keyEvent1->Current);
-	BOOST_REQUIRE_EQUAL(keyEvent1->StartTime, Common::DateTime({2014, 12, 18, 21, 0}));
+	BOOST_REQUIRE_EQUAL(keyEvent1->StartTime, Common::DateTime({2017, 8, 22, 5, 0}));
 	BOOST_REQUIRE(keyEvent2->Current);
-	BOOST_REQUIRE_EQUAL(keyEvent2->StartTime, Common::DateTime({2014, 12, 17, 5, 30}));
+	BOOST_REQUIRE_EQUAL(keyEvent2->StartTime, Common::DateTime({2017, 8, 22, 6, 30}));
 
 	BOOST_TEST_CHECKPOINT("Write second events");
-	MockTuner::SetEventsSet(1);
+	P2PVR::DVB::Testing::MockTuner::SetEventsSet(1);
 	maint->UpdateEvents();
-	BOOST_REQUIRE_THROW(si->GetEvent(14448, 27052), P2PVR::NotFound);
-	BOOST_REQUIRE(si->GetEvent(15856, 3591));
+	auto dayTwoEvents = si->EventSearch(IceUtil::Optional<std::string>(), IceUtil::Optional<int>(),
+			Common::DateTime {2017, 8, 23, 3, 0}, Common::DateTime {2017, 8, 24, 3, 0});
+	BOOST_REQUIRE_THROW(si->GetEvent(15064, 22), P2PVR::NotFound);
+	BOOST_REQUIRE(si->GetEvent(15048, 29791));
 
 	BOOST_TEST_CHECKPOINT("Check our faked stuff is still there and right");
 	auto dayOneEventsOnDayTwo = si->EventSearch(IceUtil::Optional<std::string>(), IceUtil::Optional<int>(),
-			Common::DateTime {2014, 12, 18, 3, 0}, Common::DateTime {2014, 12, 19, 3, 0});
+			Common::DateTime {2017, 8, 22, 3, 0}, Common::DateTime {2017, 8, 23, 3, 0});
 	// Some datetime range overlap, but most are gone
-	BOOST_REQUIRE_EQUAL(dayOneEventsOnDayTwo.size(), 373);
+	BOOST_REQUIRE_EQUAL(dayOneEventsOnDayTwo.size(), 316);
 	BOOST_REQUIRE_EQUAL(std::count_if(dayOneEventsOnDayTwo.begin(), dayOneEventsOnDayTwo.end(),
 				[](const P2PVR::EventPtr & e) {
 					return (e->ServiceId == 25664 && e->EventId == 55689)