diff --git a/docs/source/options/dash_options.rst b/docs/source/options/dash_options.rst index 5d8b3b7d85..7a07ce9ae8 100644 --- a/docs/source/options/dash_options.rst +++ b/docs/source/options/dash_options.rst @@ -90,3 +90,8 @@ DASH options Optional. Defaults to 0 if not specified. If it is set to 1, indicates the stream is DASH only. + +--allow_codec_switching + + If enabled, allow adaptive switching between different codecs, if they have + the same language, media type (audio, video etc) and container type. diff --git a/packager/app/mpd_flags.cc b/packager/app/mpd_flags.cc index 62231840f4..9808ece178 100644 --- a/packager/app/mpd_flags.cc +++ b/packager/app/mpd_flags.cc @@ -59,3 +59,8 @@ DEFINE_bool( "completely." "Ignored if $Time$ is used in segment template, since $Time$ requires " "accurate Segment Timeline."); +DEFINE_bool(allow_codec_switching, + false, + "If enabled, allow adaptive switching between different codecs, " + "if they have the same language, media type (audio, video etc) and " + "container type."); diff --git a/packager/app/mpd_flags.h b/packager/app/mpd_flags.h index d6c9f2fe83..520739e083 100644 --- a/packager/app/mpd_flags.h +++ b/packager/app/mpd_flags.h @@ -21,5 +21,6 @@ DECLARE_double(suggested_presentation_delay); DECLARE_string(utc_timings); DECLARE_bool(generate_dash_if_iop_compliant_mpd); DECLARE_bool(allow_approximate_segment_timeline); +DECLARE_bool(allow_codec_switching); #endif // APP_MPD_FLAGS_H_ diff --git a/packager/app/packager_main.cc b/packager/app/packager_main.cc index 0d193ece15..35df360b8b 100644 --- a/packager/app/packager_main.cc +++ b/packager/app/packager_main.cc @@ -460,6 +460,7 @@ base::Optional GetPackagingParams() { FLAGS_generate_dash_if_iop_compliant_mpd; mpd_params.allow_approximate_segment_timeline = FLAGS_allow_approximate_segment_timeline; + mpd_params.allow_codec_switching = FLAGS_allow_codec_switching; HlsParams& hls_params = packaging_params.hls_params; if (!GetHlsPlaylistType(FLAGS_hls_playlist_type, &hls_params.playlist_type)) { diff --git a/packager/app/test/packager_test.py b/packager/app/test/packager_test.py index c30223e412..020b81dc77 100755 --- a/packager/app/test/packager_test.py +++ b/packager/app/test/packager_test.py @@ -447,7 +447,8 @@ class PackagerAppTest(unittest.TestCase): ad_cues=None, default_language=None, segment_duration=1.0, - use_fake_clock=True): + use_fake_clock=True, + allow_codec_switching=False): flags = [] if not strip_parameter_set_nalus: @@ -528,6 +529,9 @@ class PackagerAppTest(unittest.TestCase): if generate_static_live_mpd: flags += ['--generate_static_live_mpd'] + if allow_codec_switching: + flags += ['--allow_codec_switching'] + if ad_cues: flags += ['--ad_cues', ad_cues] @@ -1499,6 +1503,50 @@ class PackagerFunctionalTest(PackagerAppTest): self._GetFlags(output_dash=True, generate_static_live_mpd=True)) self._CheckTestResults('live-static-profile-with-time-in-segment-name') + def testAllowCodecSwitching(self): + streams = [ + self._GetStream('video', test_file='bear-640x360-hevc.mp4'), + self._GetStream('video', test_file='bear-640x360.mp4'), + self._GetStream('video', test_file='bear-1280x720.mp4'), + self._GetStream('audio', test_file='bear-640x360.mp4'), + ] + + self.assertPackageSuccess(streams, + self._GetFlags(output_dash=True, + allow_codec_switching=True)) + # Mpd cannot be validated right now since we don't generate determinstic + # mpd with multiple inputs due to thread racing. + # TODO(b/73349711): Generate determinstic mpd or at least validate mpd + # schema. + # See also https://github.com/google/shaka-packager/issues/177. + self._CheckTestResults( + 'audio-video-with-codec-switching', + diff_files_policy=DiffFilesPolicy( + allowed_diff_files=['output.mpd'], exact=False)) + + def testAllowCodecSwitchingWithEncryptionAndTrickplay(self): + streams = [ + self._GetStream('video', test_file='bear-640x360-hevc.mp4'), + self._GetStream('video', test_file='bear-640x360.mp4'), + self._GetStream('video', test_file='bear-1280x720.mp4'), + self._GetStream('video', test_file='bear-1280x720.mp4', + trick_play_factor=1), + self._GetStream('audio', test_file='bear-640x360.mp4'), + ] + + self.assertPackageSuccess(streams, self._GetFlags(output_dash=True, + allow_codec_switching=True, + encryption=True)) + # Mpd cannot be validated right now since we don't generate determinstic + # mpd with multiple inputs due to thread racing. + # TODO(b/73349711): Generate determinstic mpd or at least validate mpd + # schema. + # See also https://github.com/google/shaka-packager/issues/177. + self._CheckTestResults( + 'audio-video-with-codec-switching-encryption-trick-play', + diff_files_policy=DiffFilesPolicy( + allowed_diff_files=['output.mpd'], exact=False)) + def testLiveProfileAndEncryption(self): self.assertPackageSuccess( self._GetStreams(['audio', 'video'], segmented=True), diff --git a/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/bear-1280x720-video-trick_play_factor_1.mp4 b/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/bear-1280x720-video-trick_play_factor_1.mp4 new file mode 100644 index 0000000000..936f72e2a7 Binary files /dev/null and b/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/bear-1280x720-video-trick_play_factor_1.mp4 differ diff --git a/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/bear-1280x720-video.mp4 b/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/bear-1280x720-video.mp4 new file mode 100644 index 0000000000..7a6ea9ee19 Binary files /dev/null and b/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/bear-1280x720-video.mp4 differ diff --git a/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/bear-640x360-audio.mp4 b/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/bear-640x360-audio.mp4 new file mode 100644 index 0000000000..b82b25f618 Binary files /dev/null and b/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/bear-640x360-audio.mp4 differ diff --git a/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/bear-640x360-hevc-video.mp4 b/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/bear-640x360-hevc-video.mp4 new file mode 100644 index 0000000000..a8c72073be Binary files /dev/null and b/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/bear-640x360-hevc-video.mp4 differ diff --git a/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/bear-640x360-video.mp4 b/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/bear-640x360-video.mp4 new file mode 100644 index 0000000000..c82b090d9e Binary files /dev/null and b/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/bear-640x360-video.mp4 differ diff --git a/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/output.mpd b/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/output.mpd new file mode 100644 index 0000000000..54d9782cb8 --- /dev/null +++ b/packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/output.mpd @@ -0,0 +1,66 @@ + + + + + + + + AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA== + + + + + bear-640x360-hevc-video.mp4 + + + + + + + + + AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA== + + + + + bear-640x360-video.mp4 + + + + + + bear-1280x720-video.mp4 + + + + + + + + + AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA== + + + + bear-640x360-audio.mp4 + + + + + + + + + AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA== + + + + bear-1280x720-video-trick_play_factor_1.mp4 + + + + + + + diff --git a/packager/app/test/testdata/audio-video-with-codec-switching/bear-1280x720-video.mp4 b/packager/app/test/testdata/audio-video-with-codec-switching/bear-1280x720-video.mp4 new file mode 100644 index 0000000000..fe9098fadc Binary files /dev/null and b/packager/app/test/testdata/audio-video-with-codec-switching/bear-1280x720-video.mp4 differ diff --git a/packager/app/test/testdata/audio-video-with-codec-switching/bear-640x360-audio.mp4 b/packager/app/test/testdata/audio-video-with-codec-switching/bear-640x360-audio.mp4 new file mode 100644 index 0000000000..87f89a93c0 Binary files /dev/null and b/packager/app/test/testdata/audio-video-with-codec-switching/bear-640x360-audio.mp4 differ diff --git a/packager/app/test/testdata/audio-video-with-codec-switching/bear-640x360-hevc-video.mp4 b/packager/app/test/testdata/audio-video-with-codec-switching/bear-640x360-hevc-video.mp4 new file mode 100644 index 0000000000..298f9532bb Binary files /dev/null and b/packager/app/test/testdata/audio-video-with-codec-switching/bear-640x360-hevc-video.mp4 differ diff --git a/packager/app/test/testdata/audio-video-with-codec-switching/bear-640x360-video.mp4 b/packager/app/test/testdata/audio-video-with-codec-switching/bear-640x360-video.mp4 new file mode 100644 index 0000000000..9bc668f8f6 Binary files /dev/null and b/packager/app/test/testdata/audio-video-with-codec-switching/bear-640x360-video.mp4 differ diff --git a/packager/app/test/testdata/audio-video-with-codec-switching/output.mpd b/packager/app/test/testdata/audio-video-with-codec-switching/output.mpd new file mode 100644 index 0000000000..5a7630776d --- /dev/null +++ b/packager/app/test/testdata/audio-video-with-codec-switching/output.mpd @@ -0,0 +1,41 @@ + + + + + + + + + bear-640x360-hevc-video.mp4 + + + + + + + + + + bear-1280x720-video.mp4 + + + + + + bear-640x360-video.mp4 + + + + + + + + + bear-640x360-audio.mp4 + + + + + + + diff --git a/packager/mpd/base/adaptation_set.h b/packager/mpd/base/adaptation_set.h index c3d7a89894..a8fc70605e 100644 --- a/packager/mpd/base/adaptation_set.h +++ b/packager/mpd/base/adaptation_set.h @@ -176,6 +176,13 @@ class AdaptationSet { /// @return true if it is a video AdaptationSet. bool IsVideo() const; + /// @return codec. + const std::string& codec() const { return codec_; } + + /// Set AdaptationSet@codec. + /// @param codec is the new codec to be set. + void set_codec(const std::string& codec) { codec_ = codec; }; + protected: /// @param language is the language of this AdaptationSet. Mainly relevant for /// audio. @@ -269,6 +276,9 @@ class AdaptationSet { // Determined by examining the MediaInfo passed to AddRepresentation(). std::string content_type_; + // Codec of AdaptationSet. + std::string codec_; + // This does not have to be a set, it could be a list or vector because all we // really care is whether there is more than one entry. // Contains one entry if all the Representations have the same picture aspect diff --git a/packager/mpd/base/mock_mpd_builder.cc b/packager/mpd/base/mock_mpd_builder.cc index 5856c0199a..80e06e7d8a 100644 --- a/packager/mpd/base/mock_mpd_builder.cc +++ b/packager/mpd/base/mock_mpd_builder.cc @@ -21,6 +21,7 @@ MockPeriod::MockPeriod(uint32_t period_id, double start_time_in_seconds) MockAdaptationSet::MockAdaptationSet() : AdaptationSet(kEmptyLang, kDefaultMpdOptions, &sequence_counter_) {} + MockAdaptationSet::~MockAdaptationSet() {} MockRepresentation::MockRepresentation(uint32_t representation_id) diff --git a/packager/mpd/base/mpd_utils.cc b/packager/mpd/base/mpd_utils.cc index c8b99ee8a7..d0456f80fb 100644 --- a/packager/mpd/base/mpd_utils.cc +++ b/packager/mpd/base/mpd_utils.cc @@ -142,7 +142,8 @@ std::string GetBaseCodec(const MediaInfo& media_info) { return codec; } -std::string GetAdaptationSetKey(const MediaInfo& media_info) { +std::string GetAdaptationSetKey(const MediaInfo& media_info, + bool ignore_codec) { std::string key; if (media_info.has_video_info()) { @@ -157,8 +158,10 @@ std::string GetAdaptationSetKey(const MediaInfo& media_info) { } key.append(MediaInfo_ContainerType_Name(media_info.container_type())); - key.append(":"); - key.append(GetBaseCodec(media_info)); + if (!ignore_codec) { + key.append(":"); + key.append(GetBaseCodec(media_info)); + } key.append(":"); key.append(GetLanguage(media_info)); diff --git a/packager/mpd/base/mpd_utils.h b/packager/mpd/base/mpd_utils.h index 557d138ce6..817b8db331 100644 --- a/packager/mpd/base/mpd_utils.h +++ b/packager/mpd/base/mpd_utils.h @@ -47,7 +47,7 @@ std::string GetCodecs(const MediaInfo& media_info); std::string GetBaseCodec(const MediaInfo& media_info); // Returns a key made from the characteristics that separate AdaptationSets. -std::string GetAdaptationSetKey(const MediaInfo& media_info); +std::string GetAdaptationSetKey(const MediaInfo& media_info, bool ignore_codec); std::string SecondsToXmlDuration(double seconds); diff --git a/packager/mpd/base/period.cc b/packager/mpd/base/period.cc index aa9601a252..9de689dadb 100644 --- a/packager/mpd/base/period.cc +++ b/packager/mpd/base/period.cc @@ -79,28 +79,24 @@ AdaptationSet* Period::GetOrCreateAdaptationSet( if (duration_seconds_ == 0) duration_seconds_ = media_info.media_duration_seconds(); - // AdaptationSets with the same key should only differ in ContentProtection, - // which also means that if |content_protection_in_adaptation_set| is false, - // there should be at most one entry in |adaptation_sets|. - const std::string key = GetAdaptationSetKey(media_info); + const std::string key = GetAdaptationSetKey( + media_info, mpd_options_.mpd_params.allow_codec_switching); + std::list& adaptation_sets = adaptation_set_list_map_[key]; - if (content_protection_in_adaptation_set) { - for (AdaptationSet* adaptation_set : adaptation_sets) { - if (protected_adaptation_set_map_.Match(*adaptation_set, media_info)) - return adaptation_set; - } - } else { - if (!adaptation_sets.empty()) { - DCHECK_EQ(adaptation_sets.size(), 1u); - return adaptation_sets.front(); - } + + for (AdaptationSet* adaptation_set : adaptation_sets) { + if (protected_adaptation_set_map_.Match( + *adaptation_set, media_info, content_protection_in_adaptation_set)) + return adaptation_set; } + // None of the adaptation sets match with the new content protection. // Need a new one. const std::string language = GetLanguage(media_info); std::unique_ptr new_adaptation_set = NewAdaptationSet(language, mpd_options_, representation_counter_); if (!SetNewAdaptationSetAttributes(language, media_info, adaptation_sets, + content_protection_in_adaptation_set, new_adaptation_set.get())) { return nullptr; } @@ -109,15 +105,15 @@ AdaptationSet* Period::GetOrCreateAdaptationSet( media_info.has_protected_content()) { protected_adaptation_set_map_.Register(*new_adaptation_set, media_info); AddContentProtectionElements(media_info, new_adaptation_set.get()); - - for (AdaptationSet* adaptation_set : adaptation_sets) { - if (protected_adaptation_set_map_.Switchable(*adaptation_set, - *new_adaptation_set)) { - adaptation_set->AddAdaptationSetSwitching(new_adaptation_set.get()); - new_adaptation_set->AddAdaptationSetSwitching(adaptation_set); - } + } + for (AdaptationSet* adaptation_set : adaptation_sets) { + if (protected_adaptation_set_map_.Switchable(*adaptation_set, + *new_adaptation_set)) { + adaptation_set->AddAdaptationSetSwitching(new_adaptation_set.get()); + new_adaptation_set->AddAdaptationSetSwitching(adaptation_set); } } + AdaptationSet* adaptation_set_ptr = new_adaptation_set.get(); adaptation_sets.push_back(adaptation_set_ptr); adaptation_sets_.emplace_back(std::move(new_adaptation_set)); @@ -176,6 +172,7 @@ bool Period::SetNewAdaptationSetAttributes( const std::string& language, const MediaInfo& media_info, const std::list& adaptation_sets, + bool content_protection_in_adaptation_set, AdaptationSet* new_adaptation_set) { if (!media_info.dash_roles().empty()) { for (const std::string& role_str : media_info.dash_roles()) { @@ -206,6 +203,8 @@ bool Period::SetNewAdaptationSetAttributes( accessibility.substr(pos + 1)); } + new_adaptation_set->set_codec(GetBaseCodec(media_info)); + if (media_info.has_video_info()) { // Because 'language' is ignored for videos, |adaptation_sets| must have // all the video AdaptationSets. @@ -218,7 +217,8 @@ bool Period::SetNewAdaptationSetAttributes( if (media_info.video_info().has_playback_rate()) { const AdaptationSet* trick_play_reference_adaptation_set = - FindOriginalAdaptationSetForTrickPlay(media_info); + FindOriginalAdaptationSetForTrickPlay( + media_info, content_protection_in_adaptation_set); if (!trick_play_reference_adaptation_set) { LOG(ERROR) << "Failed to find original AdaptationSet for trick play."; return false; @@ -236,15 +236,19 @@ bool Period::SetNewAdaptationSetAttributes( } const AdaptationSet* Period::FindOriginalAdaptationSetForTrickPlay( - const MediaInfo& media_info) { + const MediaInfo& media_info, + bool content_protection_in_adaptation_set) { MediaInfo media_info_no_trickplay = media_info; media_info_no_trickplay.mutable_video_info()->clear_playback_rate(); - std::string key = GetAdaptationSetKey(media_info_no_trickplay); + std::string key = GetAdaptationSetKey( + media_info_no_trickplay, mpd_options_.mpd_params.allow_codec_switching); const std::list& adaptation_sets = adaptation_set_list_map_[key]; for (AdaptationSet* adaptation_set : adaptation_sets) { - if (protected_adaptation_set_map_.Match(*adaptation_set, media_info)) { + if (protected_adaptation_set_map_.Match( + *adaptation_set, media_info, + content_protection_in_adaptation_set)) { return adaptation_set; } } @@ -260,7 +264,14 @@ void Period::ProtectedAdaptationSetMap::Register( bool Period::ProtectedAdaptationSetMap::Match( const AdaptationSet& adaptation_set, - const MediaInfo& media_info) { + const MediaInfo& media_info, + bool content_protection_in_adaptation_set) { + if (adaptation_set.codec() != GetBaseCodec(media_info)) + return false; + + if (!content_protection_in_adaptation_set) + return true; + const auto protected_content_it = protected_content_map_.find(&adaptation_set); // If the AdaptationSet ID is not registered in the map, then it is clear @@ -269,6 +280,7 @@ bool Period::ProtectedAdaptationSetMap::Match( return !media_info.has_protected_content(); if (!media_info.has_protected_content()) return false; + return ProtectedContentEq(protected_content_it->second, media_info.protected_content()); } diff --git a/packager/mpd/base/period.h b/packager/mpd/base/period.h index 7a4b0ad37d..3f939b813f 100644 --- a/packager/mpd/base/period.h +++ b/packager/mpd/base/period.h @@ -95,6 +95,7 @@ class Period { const std::string& language, const MediaInfo& media_info, const std::list& adaptation_sets, + bool content_protection_in_adaptation_set, AdaptationSet* new_adaptation_set); // Gets the original AdaptationSet which the trick play video belongs to. @@ -102,7 +103,7 @@ class Period { // the trick play AdaptationSet. // Returns the original AdaptationSet if found, otherwise returns nullptr; const AdaptationSet* FindOriginalAdaptationSetForTrickPlay( - const MediaInfo& media_info); + const MediaInfo& media_info, bool content_protection_in_adaptation_set); const uint32_t id_; const double start_time_in_seconds_; @@ -127,7 +128,8 @@ class Period { // Check if the protected content associated with |adaptation_set| matches // with the one in |media_info|. bool Match(const AdaptationSet& adaptation_set, - const MediaInfo& media_info); + const MediaInfo& media_info, + bool content_protection_in_adaptation_set); // Check if the two adaptation sets are switchable. bool Switchable(const AdaptationSet& adaptation_set_a, const AdaptationSet& adaptation_set_b); diff --git a/packager/mpd/public/mpd_params.h b/packager/mpd/public/mpd_params.h index 7a194f8d0a..92e8719bbb 100644 --- a/packager/mpd/public/mpd_params.h +++ b/packager/mpd/public/mpd_params.h @@ -80,6 +80,9 @@ struct MpdParams { /// SegmentTimeline if it is enabled. It will be populated from segment /// duration specified in ChunkingParams if not specified. double target_segment_duration = 0; + /// If enabled, allow switching between different codecs, if they have the + /// same language, media type (audio, video etc) and container type. + bool allow_codec_switching = false; }; } // namespace shaka