From 335a659fab3b0be39072c7564addd45f74dbd7ef Mon Sep 17 00:00:00 2001 From: KongQun Yang Date: Mon, 16 Jul 2018 10:26:16 -0700 Subject: [PATCH] [DASH] Fix possible incorrect segmentAlignment in static live profile DASH live profile with static MPD was incorrectly handled in the same way as live profile with dynamic MPD, i.e. by assuming the synchronized Representations, which is incorrect. It can be easily re-produced in multiple period outputs, i.e. with Ad cues. It may happen with regular contents as well, due to race condition, though with a much lower chance of occurring. Fixes #435. Bug: 111359775. Change-Id: I7de087f5dd8602b4c4e35cb697d589fa3699e8a5 --- packager/mpd/base/adaptation_set.cc | 57 ++++++++-------- packager/mpd/base/adaptation_set.h | 21 +++--- packager/mpd/base/adaptation_set_unittest.cc | 72 ++++++++++++++++++-- 3 files changed, 105 insertions(+), 45 deletions(-) diff --git a/packager/mpd/base/adaptation_set.cc b/packager/mpd/base/adaptation_set.cc index 23246c8d89..8959960350 100644 --- a/packager/mpd/base/adaptation_set.cc +++ b/packager/mpd/base/adaptation_set.cc @@ -274,8 +274,8 @@ xml::scoped_xml_ptr AdaptationSet::GetXml() { // Note: must be checked before checking segments_aligned_ (below). So that // segments_aligned_ is set before checking below. - if (mpd_options_.dash_profile == DashProfile::kOnDemand) { - CheckVodSegmentAlignment(); + if (mpd_options_.mpd_type == MpdType::kStatic) { + CheckStaticSegmentAlignment(); } if (segments_aligned_ == kSegmentAlignmentTrue) { @@ -348,18 +348,17 @@ void AdaptationSet::AddAdaptationSetSwitching( switchable_adaptation_sets_.push_back(adaptation_set); } -// Check segmentAlignment for Live here. Storing all start_time and duration -// will out-of-memory because there's no way of knowing when it will end. -// VOD subsegmentAlignment check is *not* done here because it is possible -// that some Representations might not have been added yet (e.g. a thread is -// assigned per muxer so one might run faster than others). -// To be clear, for Live, all Representations should be added before a -// segment is added. +// For dynamic MPD, storing all start_time and duration will out-of-memory +// because there's no way of knowing when it will end. Static MPD +// subsegmentAlignment check is *not* done here because it is possible that some +// Representations might not have been added yet (e.g. a thread is assigned per +// muxer so one might run faster than others). To be clear, for dynamic MPD, all +// Representations should be added before a segment is added. void AdaptationSet::OnNewSegmentForRepresentation(uint32_t representation_id, uint64_t start_time, uint64_t duration) { - if (mpd_options_.dash_profile == DashProfile::kLive) { - CheckLiveSegmentAlignment(representation_id, start_time, duration); + if (mpd_options_.mpd_type == MpdType::kDynamic) { + CheckDynamicSegmentAlignment(representation_id, start_time, duration); } else { representation_segment_start_times_[representation_id].push_back( start_time); @@ -442,34 +441,37 @@ void AdaptationSet::UpdateFromMediaInfo(const MediaInfo& media_info) { // They are not aligned but this will be marked as aligned. // But since this is unlikely to happen in the packager (and to save // computation), this isn't handled at the moment. -void AdaptationSet::CheckLiveSegmentAlignment(uint32_t representation_id, - uint64_t start_time, - uint64_t /* duration */) { +void AdaptationSet::CheckDynamicSegmentAlignment(uint32_t representation_id, + uint64_t start_time, + uint64_t /* duration */) { if (segments_aligned_ == kSegmentAlignmentFalse || force_set_segment_alignment_) { return; } - std::list& representation_start_times = + std::list& current_representation_start_times = representation_segment_start_times_[representation_id]; - representation_start_times.push_back(start_time); + current_representation_start_times.push_back(start_time); // There's no way to detemine whether the segments are aligned if some // representations do not have any segments. if (representation_segment_start_times_.size() != representation_map_.size()) return; - DCHECK(!representation_start_times.empty()); - const uint64_t expected_start_time = representation_start_times.front(); - for (RepresentationTimeline::const_iterator it = - representation_segment_start_times_.begin(); - it != representation_segment_start_times_.end(); ++it) { + DCHECK(!current_representation_start_times.empty()); + const uint64_t expected_start_time = + current_representation_start_times.front(); + for (const auto& key_value : representation_segment_start_times_) { + const std::list& representation_start_time = key_value.second; // If there are no entries in a list, then there is no way for the // segment alignment status to change. // Note that it can be empty because entries get deleted below. - if (it->second.empty()) + if (representation_start_time.empty()) return; - if (expected_start_time != it->second.front()) { + if (expected_start_time != representation_start_time.front()) { + VLOG(1) << "Seeing Misaligned segments with different start_times: " + << expected_start_time << " vs " + << representation_start_time.front(); // Flag as false and clear the start times data, no need to keep it // around. segments_aligned_ = kSegmentAlignmentFalse; @@ -479,16 +481,15 @@ void AdaptationSet::CheckLiveSegmentAlignment(uint32_t representation_id, } segments_aligned_ = kSegmentAlignmentTrue; - for (RepresentationTimeline::iterator it = - representation_segment_start_times_.begin(); - it != representation_segment_start_times_.end(); ++it) { - it->second.pop_front(); + for (auto& key_value : representation_segment_start_times_) { + std::list& representation_start_time = key_value.second; + representation_start_time.pop_front(); } } // Make sure all segements start times match for all Representations. // This assumes that the segments are contiguous. -void AdaptationSet::CheckVodSegmentAlignment() { +void AdaptationSet::CheckStaticSegmentAlignment() { if (segments_aligned_ == kSegmentAlignmentFalse || force_set_segment_alignment_) { return; diff --git a/packager/mpd/base/adaptation_set.h b/packager/mpd/base/adaptation_set.h index 340538fd1e..86a531e27b 100644 --- a/packager/mpd/base/adaptation_set.h +++ b/packager/mpd/base/adaptation_set.h @@ -213,18 +213,19 @@ class AdaptationSet { /// Called from OnNewSegmentForRepresentation(). Checks whether the segments /// are aligned. Sets segments_aligned_. - /// This is only for Live. For VOD, CheckVodSegmentAlignment() should be used. + /// This is only for dynamic MPD. For static MPD, + /// CheckStaticSegmentAlignment() should be used. /// @param representation_id is the id of the Representation with a new /// segment. /// @param start_time is the start time of the new segment. /// @param duration is the duration of the new segment. - void CheckLiveSegmentAlignment(uint32_t representation_id, - uint64_t start_time, - uint64_t duration); + void CheckDynamicSegmentAlignment(uint32_t representation_id, + uint64_t start_time, + uint64_t duration); // Checks representation_segment_start_times_ and sets segments_aligned_. - // Use this for VOD, do not use for Live. - void CheckVodSegmentAlignment(); + // Use this for static MPD, do not use for dynamic MPD. + void CheckStaticSegmentAlignment(); // Records the framerate of a Representation. void RecordFrameRate(uint32_t frame_duration, uint32_t timescale); @@ -279,12 +280,12 @@ class AdaptationSet { bool force_set_segment_alignment_; // Keeps track of segment start times of Representations. - // For VOD, this will not be cleared, all the segment start times are + // For static MPD, this will not be cleared, all the segment start times are // stored in this. This should not out-of-memory for a reasonable length // video and reasonable subsegment length. - // For Live, the entries are deleted (see CheckLiveSegmentAlignment() - // implementation comment) because storing the entire timeline is not - // reasonable and may cause an out-of-memory problem. + // For dynamic MPD, the entries are deleted (see + // CheckDynamicSegmentAlignment() implementation comment) because storing the + // entire timeline is not reasonable and may cause an out-of-memory problem. RepresentationTimeline representation_segment_start_times_; // Record the original AdaptationSets the trick play stream belongs to. There diff --git a/packager/mpd/base/adaptation_set_unittest.cc b/packager/mpd/base/adaptation_set_unittest.cc index d1bacec4f4..820dc4528f 100644 --- a/packager/mpd/base/adaptation_set_unittest.cc +++ b/packager/mpd/base/adaptation_set_unittest.cc @@ -743,8 +743,12 @@ TEST_F(OnDemandAdaptationSetTest, ForceSetsubsegmentAlignment) { } // Verify that segmentAlignment is set to true if all the Representations -// segments' are aligned and the DASH profile is Live. -TEST_F(LiveAdaptationSetTest, SegmentAlignment) { +// segments' are aligned and the DASH profile is Live and MPD type is dynamic. +TEST_F(LiveAdaptationSetTest, SegmentAlignmentDynamicMpd) { + const uint64_t kStartTime = 0u; + const uint64_t kDuration = 10u; + const uint64_t kAnySize = 19834u; + const char k480pMediaInfo[] = "video_info {\n" " codec: 'avc1'\n" @@ -767,17 +771,17 @@ TEST_F(LiveAdaptationSetTest, SegmentAlignment) { " pixel_height: 1\n" "}\n" "container_type: 1\n"; + + mpd_options_.mpd_type = MpdType::kDynamic; + + // For dynamic MPD, we expect the Reprensentations to be synchronized, so the + // Reprensentations are added to AdaptationSet before any segments are added. auto adaptation_set = CreateAdaptationSet(kNoLanguage); Representation* representation_480p = adaptation_set->AddRepresentation(ConvertToMediaInfo(k480pMediaInfo)); Representation* representation_360p = adaptation_set->AddRepresentation(ConvertToMediaInfo(k360pMediaInfo)); - // First use same start time and duration, and verify that that - // segmentAlignment is set. - const uint64_t kStartTime = 0u; - const uint64_t kDuration = 10u; - const uint64_t kAnySize = 19834u; representation_480p->AddNewSegment(kStartTime, kDuration, kAnySize); representation_360p->AddNewSegment(kStartTime, kDuration, kAnySize); xml::scoped_xml_ptr aligned(adaptation_set->GetXml()); @@ -792,6 +796,60 @@ TEST_F(LiveAdaptationSetTest, SegmentAlignment) { EXPECT_THAT(unaligned.get(), Not(AttributeSet("segmentAlignment"))); } +// Verify that segmentAlignment is set to true if all the Representations +// segments' are aligned and the DASH profile is Live and MPD type is static. +TEST_F(LiveAdaptationSetTest, SegmentAlignmentStaticMpd) { + const uint64_t kStartTime = 0u; + const uint64_t kDuration = 10u; + const uint64_t kAnySize = 19834u; + + const char k480pMediaInfo[] = + "video_info {\n" + " codec: 'avc1'\n" + " width: 720\n" + " height: 480\n" + " time_scale: 10\n" + " frame_duration: 10\n" + " pixel_width: 8\n" + " pixel_height: 9\n" + "}\n" + "container_type: 1\n"; + const char k360pMediaInfo[] = + "video_info {\n" + " codec: 'avc1'\n" + " width: 640\n" + " height: 360\n" + " time_scale: 10\n" + " frame_duration: 10\n" + " pixel_width: 1\n" + " pixel_height: 1\n" + "}\n" + "container_type: 1\n"; + + mpd_options_.mpd_type = MpdType::kStatic; + + auto adaptation_set = CreateAdaptationSet(kNoLanguage); + + // For static MPD, the Representations are not synchronized, so it is possible + // that the second Representation is added after adding segments to the first + // Representation. + Representation* representation_480p = + adaptation_set->AddRepresentation(ConvertToMediaInfo(k480pMediaInfo)); + representation_480p->AddNewSegment(kStartTime, kDuration, kAnySize); + + Representation* representation_360p = + adaptation_set->AddRepresentation(ConvertToMediaInfo(k360pMediaInfo)); + representation_360p->AddNewSegment(kStartTime, kDuration, kAnySize); + + representation_480p->AddNewSegment(kStartTime + kDuration, kDuration, + kAnySize); + representation_360p->AddNewSegment(kStartTime + kDuration, kDuration, + kAnySize); + + xml::scoped_xml_ptr aligned(adaptation_set->GetXml()); + EXPECT_THAT(aligned.get(), AttributeEqual("segmentAlignment", "true")); +} + // Verify that the width and height attribute are set if all the video // representations have the same width and height. TEST_F(OnDemandAdaptationSetTest, AdapatationSetWidthAndHeight) {