From 4bfa603b52f50586916341cc4ae705f0c92740ca Mon Sep 17 00:00:00 2001 From: sr90 Date: Tue, 24 Mar 2020 12:15:30 -0700 Subject: [PATCH] [DASH] Support TrickPlay using separate trick play specific streams (#736) Closes #732. --- .../media/trick_play/trick_play_handler.cc | 23 +++--- packager/mpd/base/period.cc | 76 ++++++++++++++----- packager/mpd/base/period.h | 34 +++++++-- packager/mpd/base/period_unittest.cc | 52 ++++++++++++- 4 files changed, 146 insertions(+), 39 deletions(-) diff --git a/packager/media/trick_play/trick_play_handler.cc b/packager/media/trick_play/trick_play_handler.cc index b6162162d7..d5382ca094 100644 --- a/packager/media/trick_play/trick_play_handler.cc +++ b/packager/media/trick_play/trick_play_handler.cc @@ -143,18 +143,6 @@ Status TrickPlayHandler::OnMediaSample(const MediaSample& sample) { return OnTrickFrame(sample); } } - - // Update this now as it may be sent out soon via the delay message queue. - if (total_trick_frames_ < 2) { - // At this point, video_info will be at the head of the delay message queue - // and can still be updated safely. - - // The play back rate is determined by the number of frames between the - // first two trick play frames. The first trick play frame will be the - // first frame in the video. - video_info_->set_playback_rate(total_frames_); - } - // If the frame is not a trick play frame, then take the duration of this // frame and add it to the previous trick play frame so that it will span the // gap created by not passing this frame through. @@ -182,6 +170,17 @@ Status TrickPlayHandler::OnTrickFrame(const MediaSample& sample) { return Status::OK; } + // Update this now as it may be sent out soon via the delay message queue. + if (total_trick_frames_ == 2) { + // At this point, video_info will be at the head of the delay message queue + // and can still be updated safely. + + // The play back rate is determined by the number of frames between the + // first two trick play frames. The first trick play frame will be the + // first frame in the video. + video_info_->set_playback_rate(total_frames_ - 1); + } + // Send out all delayed messages up until the new trick play frame we just // added. Status s; diff --git a/packager/mpd/base/period.cc b/packager/mpd/base/period.cc index 9de689dadb..f86902dd96 100644 --- a/packager/mpd/base/period.cc +++ b/packager/mpd/base/period.cc @@ -216,16 +216,30 @@ bool Period::SetNewAdaptationSetAttributes( } if (media_info.video_info().has_playback_rate()) { - const AdaptationSet* trick_play_reference_adaptation_set = - FindOriginalAdaptationSetForTrickPlay( - media_info, content_protection_in_adaptation_set); - if (!trick_play_reference_adaptation_set) { - LOG(ERROR) << "Failed to find original AdaptationSet for trick play."; - return false; + std::string trick_play_reference_adaptation_set_key; + AdaptationSet* trick_play_reference_adaptation_set = + FindMatchingAdaptationSetForTrickPlay( + media_info, content_protection_in_adaptation_set, + &trick_play_reference_adaptation_set_key); + if (trick_play_reference_adaptation_set) { + new_adaptation_set->AddTrickPlayReference( + trick_play_reference_adaptation_set); + } else { + trickplay_cache_[trick_play_reference_adaptation_set_key].push_back( + new_adaptation_set); + } + } else { + std::string trick_play_adaptation_set_key; + AdaptationSet* trickplay_adaptation_set = + FindMatchingAdaptationSetForTrickPlay( + media_info, content_protection_in_adaptation_set, + &trick_play_adaptation_set_key); + if (trickplay_adaptation_set) { + trickplay_adaptation_set->AddTrickPlayReference(new_adaptation_set); + trickplay_cache_.erase(trick_play_adaptation_set_key); } - new_adaptation_set->AddTrickPlayReference( - trick_play_reference_adaptation_set); } + } else if (media_info.has_text_info()) { // IOP requires all AdaptationSets to have (sub)segmentAlignment set to // true, so carelessly set it to true. @@ -235,26 +249,45 @@ bool Period::SetNewAdaptationSetAttributes( return true; } -const AdaptationSet* Period::FindOriginalAdaptationSetForTrickPlay( +AdaptationSet* Period::FindMatchingAdaptationSetForTrickPlay( const MediaInfo& media_info, - bool content_protection_in_adaptation_set) { - MediaInfo media_info_no_trickplay = media_info; - media_info_no_trickplay.mutable_video_info()->clear_playback_rate(); - - std::string key = GetAdaptationSetKey( - media_info_no_trickplay, mpd_options_.mpd_params.allow_codec_switching); - const std::list& adaptation_sets = - adaptation_set_list_map_[key]; - for (AdaptationSet* adaptation_set : adaptation_sets) { + bool content_protection_in_adaptation_set, + std::string* adaptation_set_key) { + std::list* adaptation_sets = nullptr; + const bool is_trickplay_adaptation_set = + media_info.video_info().has_playback_rate(); + if (is_trickplay_adaptation_set) { + *adaptation_set_key = GetAdaptationSetKeyForTrickPlay(media_info); + if (adaptation_set_list_map_.find(*adaptation_set_key) == + adaptation_set_list_map_.end()) + return nullptr; + adaptation_sets = &adaptation_set_list_map_[*adaptation_set_key]; + } else { + *adaptation_set_key = GetAdaptationSetKey( + media_info, mpd_options_.mpd_params.allow_codec_switching); + if (trickplay_cache_.find(*adaptation_set_key) == trickplay_cache_.end()) + return nullptr; + adaptation_sets = &trickplay_cache_[*adaptation_set_key]; + } + for (AdaptationSet* adaptation_set : *adaptation_sets) { if (protected_adaptation_set_map_.Match( *adaptation_set, media_info, content_protection_in_adaptation_set)) { return adaptation_set; } } + return nullptr; } +std::string Period::GetAdaptationSetKeyForTrickPlay( + const MediaInfo& media_info) { + MediaInfo media_info_no_trickplay = media_info; + media_info_no_trickplay.mutable_video_info()->clear_playback_rate(); + return GetAdaptationSetKey(media_info_no_trickplay, + mpd_options_.mpd_params.allow_codec_switching); +} + void Period::ProtectedAdaptationSetMap::Register( const AdaptationSet& adaptation_set, const MediaInfo& media_info) { @@ -303,4 +336,11 @@ bool Period::ProtectedAdaptationSetMap::Switchable( GetUUIDs(protected_content_it_b->second); } +Period::~Period() { + if (!trickplay_cache_.empty()) { + LOG(WARNING) << "Trickplay adaptation set did not get a valid adaptation " + "set match. Please check the command line options."; + } +} + } // namespace shaka diff --git a/packager/mpd/base/period.h b/packager/mpd/base/period.h index 3f939b813f..047e788745 100644 --- a/packager/mpd/base/period.h +++ b/packager/mpd/base/period.h @@ -29,7 +29,7 @@ class XmlNode; /// AdaptationSets. class Period { public: - virtual ~Period() = default; + virtual ~Period(); /// Check the existing AdaptationSets, if there is one matching the provided /// @a media_info, return it; otherwise a new AdaptationSet is created and @@ -66,6 +66,12 @@ class Period { duration_seconds_ = duration_seconds; } + /// @return trickplay_cache. + const std::map>& trickplay_cache() + const { + return trickplay_cache_; + } + protected: /// @param period_id is an ID number for this Period. /// @param start_time_in_seconds is the start time for this Period. @@ -98,13 +104,23 @@ class Period { bool content_protection_in_adaptation_set, AdaptationSet* new_adaptation_set); - // Gets the original AdaptationSet which the trick play video belongs to. - // It is assumed that the corresponding AdaptationSet has been created before - // the trick play AdaptationSet. - // Returns the original AdaptationSet if found, otherwise returns nullptr; - const AdaptationSet* FindOriginalAdaptationSetForTrickPlay( - const MediaInfo& media_info, bool content_protection_in_adaptation_set); + // If processing a trick play AdaptationSet, gets the original AdaptationSet + // which the trick play video belongs to.It is assumed that the corresponding + // AdaptationSet has been created before the trick play AdaptationSet. + // Returns the matching AdaptationSet if found, otherwise returns nullptr; + // If processing non-trick play AdaptationSet, gets the trick play + // AdaptationSet that belongs to current AdaptationSet from trick play cache. + // Returns nullptr if matching trick play AdaptationSet is not found. + AdaptationSet* FindMatchingAdaptationSetForTrickPlay( + const MediaInfo& media_info, + bool content_protection_in_adaptation_set, + std::string* adaptation_set_key); + // Returns AdaptationSet key without ':trickplay' in it for trickplay + // AdaptationSet. + std::string GetAdaptationSetKeyForTrickPlay(const MediaInfo& media_info); + + // FindMatchingAdaptationSetForTrickPlay const uint32_t id_; const double start_time_in_seconds_; double duration_seconds_ = 0; @@ -117,6 +133,10 @@ class Period { // if they contain identical ContentProtection elements. This map is only // useful when ContentProtection element is placed in AdaptationSet. std::map> adaptation_set_list_map_; + // Contains Trickplay AdaptationSets grouped by specific adaptation set + // grouping key. These AdaptationSets still have not found reference + // AdaptationSet. + std::map> trickplay_cache_; // Tracks ProtectedContent in AdaptationSet. class ProtectedAdaptationSetMap { diff --git a/packager/mpd/base/period_unittest.cc b/packager/mpd/base/period_unittest.cc index 59622a5662..a166e17a86 100644 --- a/packager/mpd/base/period_unittest.cc +++ b/packager/mpd/base/period_unittest.cc @@ -279,6 +279,52 @@ TEST_F(PeriodTest, TrickPlayWithMatchingAdaptationSet) { content_protection_in_adaptation_set_)); } +TEST_F(PeriodTest, TrickPlayCacheWithMatchingAdaptationSet) { + const char kVideoMediaInfo[] = + "video_info {\n" + " codec: 'avc1'\n" + " width: 1280\n" + " height: 720\n" + " time_scale: 10\n" + " frame_duration: 10\n" + " pixel_width: 1\n" + " pixel_height: 1\n" + "}\n" + "container_type: 1\n"; + const char kTrickPlayMediaInfo[] = + "video_info {\n" + " codec: 'avc1'\n" + " width: 1280\n" + " height: 720\n" + " time_scale: 10\n" + " frame_duration: 100\n" + " pixel_width: 1\n" + " pixel_height: 1\n" + " playback_rate: 10\n" + "}\n" + "container_type: 1\n"; + + std::unique_ptr> trick_play_adaptation_set( + new StrictMock()); + auto* trick_play_adaptation_set_ptr = trick_play_adaptation_set.get(); + + EXPECT_CALL(testable_period_, NewAdaptationSet(_, _, _)) + .WillOnce(Return(ByMove(std::move(trick_play_adaptation_set)))) + .WillOnce(Return(ByMove(std::move(default_adaptation_set_)))); + + EXPECT_CALL(*trick_play_adaptation_set_ptr, + AddTrickPlayReference(Eq(default_adaptation_set_ptr_))); + + ASSERT_EQ(trick_play_adaptation_set_ptr, + testable_period_.GetOrCreateAdaptationSet( + ConvertToMediaInfo(kTrickPlayMediaInfo), + content_protection_in_adaptation_set_)); + ASSERT_EQ(default_adaptation_set_ptr_, + testable_period_.GetOrCreateAdaptationSet( + ConvertToMediaInfo(kVideoMediaInfo), + content_protection_in_adaptation_set_)); +} + // Verify no AdaptationSet is returned on trickplay media info. TEST_F(PeriodTest, TrickPlayWithNoMatchingAdaptationSet) { const char kVideoMediaInfo[] = @@ -316,10 +362,12 @@ TEST_F(PeriodTest, TrickPlayWithNoMatchingAdaptationSet) { testable_period_.GetOrCreateAdaptationSet( ConvertToMediaInfo(kVideoMediaInfo), content_protection_in_adaptation_set_)); - // A nullptr is returned if it is not able to find matching AdaptationSet. - ASSERT_FALSE(testable_period_.GetOrCreateAdaptationSet( + + ASSERT_TRUE(testable_period_.GetOrCreateAdaptationSet( ConvertToMediaInfo(kVp9TrickPlayMediaInfo), content_protection_in_adaptation_set_)); + + ASSERT_TRUE(!testable_period_.trickplay_cache().empty()); } // Don't put different audio languages or codecs in the same AdaptationSet.