Adding --allow_codec_switching (#726)

To allow adaptive switching between different codecs.

Closes #542.

docs/source/options/dash_options.rst

@@ -90,3 +90,8 @@ DASH options
 
     Optional. Defaults to 0 if not specified. If it is set to 1, indicates the
     stream is DASH only.
+
+--allow_codec_switching
+
+    If enabled, allow adaptive switching between different codecs, if they have 
+    the same language, media type (audio, video etc) and container type.

packager/app/mpd_flags.cc

@@ -59,3 +59,8 @@ DEFINE_bool(
     "completely."
     "Ignored if $Time$ is used in segment template, since $Time$ requires "
     "accurate Segment Timeline.");
+DEFINE_bool(allow_codec_switching,
+            false,
+            "If enabled, allow adaptive switching between different codecs, "
+            "if they have the same language, media type (audio, video etc) and "
+            "container type.");

packager/app/mpd_flags.h

@@ -21,5 +21,6 @@ DECLARE_double(suggested_presentation_delay);
 DECLARE_string(utc_timings);
 DECLARE_bool(generate_dash_if_iop_compliant_mpd);
 DECLARE_bool(allow_approximate_segment_timeline);
+DECLARE_bool(allow_codec_switching);
 
 #endif  // APP_MPD_FLAGS_H_

packager/app/packager_main.cc

@@ -460,6 +460,7 @@ base::Optional<PackagingParams> GetPackagingParams() {
       FLAGS_generate_dash_if_iop_compliant_mpd;
   mpd_params.allow_approximate_segment_timeline =
       FLAGS_allow_approximate_segment_timeline;
+  mpd_params.allow_codec_switching = FLAGS_allow_codec_switching;
 
   HlsParams& hls_params = packaging_params.hls_params;
   if (!GetHlsPlaylistType(FLAGS_hls_playlist_type, &hls_params.playlist_type)) {

packager/app/test/packager_test.py

@@ -447,7 +447,8 @@ class PackagerAppTest(unittest.TestCase):
                 ad_cues=None,
                 default_language=None,
                 segment_duration=1.0,
-                use_fake_clock=True):
+                use_fake_clock=True,
+                allow_codec_switching=False):
     flags = []
 
     if not strip_parameter_set_nalus:
@@ -528,6 +529,9 @@ class PackagerAppTest(unittest.TestCase):
     if generate_static_live_mpd:
       flags += ['--generate_static_live_mpd']
 
+    if allow_codec_switching:
+      flags += ['--allow_codec_switching']
+
     if ad_cues:
       flags += ['--ad_cues', ad_cues]
 
@@ -1499,6 +1503,50 @@ class PackagerFunctionalTest(PackagerAppTest):
         self._GetFlags(output_dash=True, generate_static_live_mpd=True))
     self._CheckTestResults('live-static-profile-with-time-in-segment-name')
 
+  def testAllowCodecSwitching(self):
+    streams = [
+        self._GetStream('video', test_file='bear-640x360-hevc.mp4'),
+        self._GetStream('video', test_file='bear-640x360.mp4'),
+        self._GetStream('video', test_file='bear-1280x720.mp4'),
+        self._GetStream('audio', test_file='bear-640x360.mp4'),
+    ]
+
+    self.assertPackageSuccess(streams, 
+                              self._GetFlags(output_dash=True,
+                                             allow_codec_switching=True))
+    # Mpd cannot be validated right now since we don't generate determinstic
+    # mpd with multiple inputs due to thread racing.
+    # TODO(b/73349711): Generate determinstic mpd or at least validate mpd
+    #                   schema.
+    # See also https://github.com/google/shaka-packager/issues/177.
+    self._CheckTestResults(
+        'audio-video-with-codec-switching',
+        diff_files_policy=DiffFilesPolicy(
+            allowed_diff_files=['output.mpd'], exact=False))
+
+  def testAllowCodecSwitchingWithEncryptionAndTrickplay(self):
+    streams = [
+        self._GetStream('video', test_file='bear-640x360-hevc.mp4'),
+        self._GetStream('video', test_file='bear-640x360.mp4'),
+        self._GetStream('video', test_file='bear-1280x720.mp4'),
+        self._GetStream('video', test_file='bear-1280x720.mp4',
+                        trick_play_factor=1),
+        self._GetStream('audio', test_file='bear-640x360.mp4'),
+    ]
+
+    self.assertPackageSuccess(streams, self._GetFlags(output_dash=True,
+                                            allow_codec_switching=True,
+                                            encryption=True))
+    # Mpd cannot be validated right now since we don't generate determinstic
+    # mpd with multiple inputs due to thread racing.
+    # TODO(b/73349711): Generate determinstic mpd or at least validate mpd
+    #                   schema.
+    # See also https://github.com/google/shaka-packager/issues/177.
+    self._CheckTestResults(
+        'audio-video-with-codec-switching-encryption-trick-play',
+        diff_files_policy=DiffFilesPolicy(
+            allowed_diff_files=['output.mpd'], exact=False))
+
   def testLiveProfileAndEncryption(self):
     self.assertPackageSuccess(
         self._GetStreams(['audio', 'video'], segmented=True),

packager/app/test/testdata/audio-video-with-codec-switching-encryption-trick-play/output.mpd

@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--Generated with https://github.com/google/shaka-packager version <tag>-<hash>-<test>-->
+<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" xmlns:cenc="urn:mpeg:cenc:2013" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.802799940109253S">
+  <Period id="0">
+    <AdaptationSet id="0" contentType="video" width="640" height="360" frameRate="30000/1001" subsegmentAlignment="true" par="16:9">
+      <ContentProtection value="cenc" schemeIdUri="urn:mpeg:dash:mp4protection:2011" cenc:default_KID="31323334-3536-3738-3930-313233343536"/>
+      <ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b">
+        <cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>
+      </ContentProtection>
+      <SupplementalProperty schemeIdUri="urn:mpeg:dash:adaptation-set-switching:2016" value="1"/>
+      <Role schemeIdUri="urn:mpeg:dash:role:2011" value="main"/>
+      <Representation id="0" bandwidth="281671" codecs="hev1.1.6.L63.90" mimeType="video/mp4" sar="1:1">
+        <BaseURL>bear-640x360-hevc-video.mp4</BaseURL>
+        <SegmentBase indexRange="3211-3278" timescale="30000">
+          <Initialization range="0-3210"/>
+        </SegmentBase>
+      </Representation>
+    </AdaptationSet>
+    <AdaptationSet id="1" contentType="video" maxWidth="1280" maxHeight="720" frameRate="30000/1001" subsegmentAlignment="true" par="16:9">
+      <ContentProtection value="cenc" schemeIdUri="urn:mpeg:dash:mp4protection:2011" cenc:default_KID="31323334-3536-3738-3930-313233343536"/>
+      <ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b">
+        <cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>
+      </ContentProtection>
+      <SupplementalProperty schemeIdUri="urn:mpeg:dash:adaptation-set-switching:2016" value="0"/>
+      <Role schemeIdUri="urn:mpeg:dash:role:2011" value="main"/>
+      <Representation id="1" bandwidth="977743" codecs="avc1.64001e" mimeType="video/mp4" sar="1:1" width="640" height="360">
+        <BaseURL>bear-640x360-video.mp4</BaseURL>
+        <SegmentBase indexRange="1127-1194" timescale="30000">
+          <Initialization range="0-1126"/>
+        </SegmentBase>
+      </Representation>
+      <Representation id="2" bandwidth="2631545" codecs="avc1.64001f" mimeType="video/mp4" sar="1:1" width="1280" height="720">
+        <BaseURL>bear-1280x720-video.mp4</BaseURL>
+        <SegmentBase indexRange="1125-1192" timescale="30000">
+          <Initialization range="0-1124"/>
+        </SegmentBase>
+      </Representation>
+    </AdaptationSet>
+    <AdaptationSet id="2" contentType="audio" subsegmentAlignment="true">
+      <ContentProtection value="cenc" schemeIdUri="urn:mpeg:dash:mp4protection:2011" cenc:default_KID="31323334-3536-3738-3930-313233343536"/>
+      <ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b">
+        <cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>
+      </ContentProtection>
+      <Representation id="3" bandwidth="133663" codecs="mp4a.40.2" mimeType="audio/mp4" audioSamplingRate="44100">
+        <AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="2"/>
+        <BaseURL>bear-640x360-audio.mp4</BaseURL>
+        <SegmentBase indexRange="1003-1070" timescale="44100">
+          <Initialization range="0-1002"/>
+        </SegmentBase>
+      </Representation>
+    </AdaptationSet>
+    <AdaptationSet id="3" contentType="video" width="1280" height="720" frameRate="30000/30030" subsegmentAlignment="true" par="16:9">
+      <ContentProtection value="cenc" schemeIdUri="urn:mpeg:dash:mp4protection:2011" cenc:default_KID="31323334-3536-3738-3930-313233343536"/>
+      <ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b">
+        <cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>
+      </ContentProtection>
+      <EssentialProperty schemeIdUri="http://dashif.org/guidelines/trickmode" value="1"/>
+      <Representation id="4" bandwidth="470530" codecs="avc1.64001f" mimeType="video/mp4" sar="1:1" maxPlayoutRate="30" codingDependency="false">
+        <BaseURL>bear-1280x720-video-trick_play_factor_1.mp4</BaseURL>
+        <SegmentBase indexRange="1125-1192" timescale="30000">
+          <Initialization range="0-1124"/>
+        </SegmentBase>
+      </Representation>
+    </AdaptationSet>
+  </Period>
+</MPD>

packager/app/test/testdata/audio-video-with-codec-switching/output.mpd

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--Generated with https://github.com/google/shaka-packager version <tag>-<hash>-<test>-->
+<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.802799940109253S">
+  <Period id="0">
+    <AdaptationSet id="0" contentType="video" width="640" height="360" frameRate="30000/1001" subsegmentAlignment="true" par="16:9">
+      <SupplementalProperty schemeIdUri="urn:mpeg:dash:adaptation-set-switching:2016" value="1"/>
+      <Role schemeIdUri="urn:mpeg:dash:role:2011" value="main"/>
+      <Representation id="0" bandwidth="277411" codecs="hev1.1.6.L63.90" mimeType="video/mp4" sar="1:1">
+        <BaseURL>bear-640x360-hevc-video.mp4</BaseURL>
+        <SegmentBase indexRange="1899-1966" timescale="30000">
+          <Initialization range="0-1898"/>
+        </SegmentBase>
+      </Representation>
+    </AdaptationSet>
+    <AdaptationSet id="1" contentType="video" maxWidth="1280" maxHeight="720" frameRate="30000/1001" subsegmentAlignment="true" par="16:9">
+      <SupplementalProperty schemeIdUri="urn:mpeg:dash:adaptation-set-switching:2016" value="0"/>
+      <Role schemeIdUri="urn:mpeg:dash:role:2011" value="main"/>
+      <Representation id="1" bandwidth="2627285" codecs="avc1.64001f" mimeType="video/mp4" sar="1:1" width="1280" height="720">
+        <BaseURL>bear-1280x720-video.mp4</BaseURL>
+        <SegmentBase indexRange="858-925" timescale="30000">
+          <Initialization range="0-857"/>
+        </SegmentBase>
+      </Representation>
+      <Representation id="2" bandwidth="973483" codecs="avc1.64001e" mimeType="video/mp4" sar="1:1" width="640" height="360">
+        <BaseURL>bear-640x360-video.mp4</BaseURL>
+        <SegmentBase indexRange="859-926" timescale="30000">
+          <Initialization range="0-858"/>
+        </SegmentBase>
+      </Representation>
+    </AdaptationSet>
+    <AdaptationSet id="2" contentType="audio" subsegmentAlignment="true">
+      <Representation id="3" bandwidth="133334" codecs="mp4a.40.2" mimeType="audio/mp4" audioSamplingRate="44100">
+        <AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="2"/>
+        <BaseURL>bear-640x360-audio.mp4</BaseURL>
+        <SegmentBase indexRange="793-860" timescale="44100">
+          <Initialization range="0-792"/>
+        </SegmentBase>
+      </Representation>
+    </AdaptationSet>
+  </Period>
+</MPD>

packager/mpd/base/adaptation_set.h

@@ -176,6 +176,13 @@ class AdaptationSet {
   /// @return true if it is a video AdaptationSet.
   bool IsVideo() const;
 
+  /// @return codec.
+  const std::string& codec() const { return codec_; }
+
+  /// Set AdaptationSet@codec.
+  /// @param codec is the new codec to be set.
+  void set_codec(const std::string& codec) { codec_ = codec; };
+
  protected:
   /// @param language is the language of this AdaptationSet. Mainly relevant for
   ///        audio.
@@ -269,6 +276,9 @@ class AdaptationSet {
   // Determined by examining the MediaInfo passed to AddRepresentation().
   std::string content_type_;
 
+  // Codec of AdaptationSet.
+  std::string codec_;
+
   // This does not have to be a set, it could be a list or vector because all we
   // really care is whether there is more than one entry.
   // Contains one entry if all the Representations have the same picture aspect

packager/mpd/base/mock_mpd_builder.cc

@@ -21,6 +21,7 @@ MockPeriod::MockPeriod(uint32_t period_id, double start_time_in_seconds)
 
 MockAdaptationSet::MockAdaptationSet()
     : AdaptationSet(kEmptyLang, kDefaultMpdOptions, &sequence_counter_) {}
+
 MockAdaptationSet::~MockAdaptationSet() {}
 
 MockRepresentation::MockRepresentation(uint32_t representation_id)

packager/mpd/base/mpd_utils.cc

@@ -142,7 +142,8 @@ std::string GetBaseCodec(const MediaInfo& media_info) {
   return codec;
 }
 
-std::string GetAdaptationSetKey(const MediaInfo& media_info) {
+std::string GetAdaptationSetKey(const MediaInfo& media_info,
+                                bool ignore_codec) {
   std::string key;
 
   if (media_info.has_video_info()) {
@@ -157,8 +158,10 @@ std::string GetAdaptationSetKey(const MediaInfo& media_info) {
   }
 
   key.append(MediaInfo_ContainerType_Name(media_info.container_type()));
+  if (!ignore_codec) {
     key.append(":");
     key.append(GetBaseCodec(media_info));
+  }
   key.append(":");
   key.append(GetLanguage(media_info));
 

packager/mpd/base/mpd_utils.h

@@ -47,7 +47,7 @@ std::string GetCodecs(const MediaInfo& media_info);
 std::string GetBaseCodec(const MediaInfo& media_info);
 
 // Returns a key made from the characteristics that separate AdaptationSets.
-std::string GetAdaptationSetKey(const MediaInfo& media_info);
+std::string GetAdaptationSetKey(const MediaInfo& media_info, bool ignore_codec);
 
 std::string SecondsToXmlDuration(double seconds);
 

packager/mpd/base/period.cc

@@ -79,28 +79,24 @@ AdaptationSet* Period::GetOrCreateAdaptationSet(
   if (duration_seconds_ == 0)
     duration_seconds_ = media_info.media_duration_seconds();
 
-  // AdaptationSets with the same key should only differ in ContentProtection,
+  const std::string key = GetAdaptationSetKey(
-  // which also means that if |content_protection_in_adaptation_set| is false,
+      media_info, mpd_options_.mpd_params.allow_codec_switching);
-  // there should be at most one entry in |adaptation_sets|.
+
-  const std::string key = GetAdaptationSetKey(media_info);
   std::list<AdaptationSet*>& adaptation_sets = adaptation_set_list_map_[key];
-  if (content_protection_in_adaptation_set) {
+
   for (AdaptationSet* adaptation_set : adaptation_sets) {
-      if (protected_adaptation_set_map_.Match(*adaptation_set, media_info))
+    if (protected_adaptation_set_map_.Match(
+            *adaptation_set, media_info, content_protection_in_adaptation_set))
       return adaptation_set;
   }
-  } else {
+
-    if (!adaptation_sets.empty()) {
-      DCHECK_EQ(adaptation_sets.size(), 1u);
-      return adaptation_sets.front();
-    }
-  }
   // None of the adaptation sets match with the new content protection.
   // Need a new one.
   const std::string language = GetLanguage(media_info);
   std::unique_ptr<AdaptationSet> new_adaptation_set =
       NewAdaptationSet(language, mpd_options_, representation_counter_);
   if (!SetNewAdaptationSetAttributes(language, media_info, adaptation_sets,
+                                     content_protection_in_adaptation_set,
                                      new_adaptation_set.get())) {
     return nullptr;
   }
@@ -109,7 +105,7 @@ AdaptationSet* Period::GetOrCreateAdaptationSet(
       media_info.has_protected_content()) {
     protected_adaptation_set_map_.Register(*new_adaptation_set, media_info);
     AddContentProtectionElements(media_info, new_adaptation_set.get());
-
+  }
   for (AdaptationSet* adaptation_set : adaptation_sets) {
     if (protected_adaptation_set_map_.Switchable(*adaptation_set,
                                                  *new_adaptation_set)) {
@@ -117,7 +113,7 @@ AdaptationSet* Period::GetOrCreateAdaptationSet(
       new_adaptation_set->AddAdaptationSetSwitching(adaptation_set);
     }
   }
-  }
+
   AdaptationSet* adaptation_set_ptr = new_adaptation_set.get();
   adaptation_sets.push_back(adaptation_set_ptr);
   adaptation_sets_.emplace_back(std::move(new_adaptation_set));
@@ -176,6 +172,7 @@ bool Period::SetNewAdaptationSetAttributes(
     const std::string& language,
     const MediaInfo& media_info,
     const std::list<AdaptationSet*>& adaptation_sets,
+    bool content_protection_in_adaptation_set,
     AdaptationSet* new_adaptation_set) {
   if (!media_info.dash_roles().empty()) {
     for (const std::string& role_str : media_info.dash_roles()) {
@@ -206,6 +203,8 @@ bool Period::SetNewAdaptationSetAttributes(
                                          accessibility.substr(pos + 1));
   }
 
+  new_adaptation_set->set_codec(GetBaseCodec(media_info));
+
   if (media_info.has_video_info()) {
     // Because 'language' is ignored for videos, |adaptation_sets| must have
     // all the video AdaptationSets.
@@ -218,7 +217,8 @@ bool Period::SetNewAdaptationSetAttributes(
 
     if (media_info.video_info().has_playback_rate()) {
       const AdaptationSet* trick_play_reference_adaptation_set =
-          FindOriginalAdaptationSetForTrickPlay(media_info);
+          FindOriginalAdaptationSetForTrickPlay(
+              media_info, content_protection_in_adaptation_set);
       if (!trick_play_reference_adaptation_set) {
         LOG(ERROR) << "Failed to find original AdaptationSet for trick play.";
         return false;
@@ -236,15 +236,19 @@ bool Period::SetNewAdaptationSetAttributes(
 }
 
 const AdaptationSet* Period::FindOriginalAdaptationSetForTrickPlay(
-    const MediaInfo& media_info) {
+    const MediaInfo& media_info,
+    bool content_protection_in_adaptation_set) {
   MediaInfo media_info_no_trickplay = media_info;
   media_info_no_trickplay.mutable_video_info()->clear_playback_rate();
 
-  std::string key = GetAdaptationSetKey(media_info_no_trickplay);
+  std::string key = GetAdaptationSetKey(
+      media_info_no_trickplay, mpd_options_.mpd_params.allow_codec_switching);
   const std::list<AdaptationSet*>& adaptation_sets =
       adaptation_set_list_map_[key];
   for (AdaptationSet* adaptation_set : adaptation_sets) {
-    if (protected_adaptation_set_map_.Match(*adaptation_set, media_info)) {
+    if (protected_adaptation_set_map_.Match(
+            *adaptation_set, media_info,
+            content_protection_in_adaptation_set)) {
       return adaptation_set;
     }
   }
@@ -260,7 +264,14 @@ void Period::ProtectedAdaptationSetMap::Register(
 
 bool Period::ProtectedAdaptationSetMap::Match(
     const AdaptationSet& adaptation_set,
-    const MediaInfo& media_info) {
+    const MediaInfo& media_info,
+    bool content_protection_in_adaptation_set) {
+  if (adaptation_set.codec() != GetBaseCodec(media_info))
+    return false;
+
+  if (!content_protection_in_adaptation_set)
+    return true;
+
   const auto protected_content_it =
       protected_content_map_.find(&adaptation_set);
   // If the AdaptationSet ID is not registered in the map, then it is clear
@@ -269,6 +280,7 @@ bool Period::ProtectedAdaptationSetMap::Match(
     return !media_info.has_protected_content();
   if (!media_info.has_protected_content())
     return false;
+
   return ProtectedContentEq(protected_content_it->second,
                             media_info.protected_content());
 }

packager/mpd/base/period.h

@@ -95,6 +95,7 @@ class Period {
       const std::string& language,
       const MediaInfo& media_info,
       const std::list<AdaptationSet*>& adaptation_sets,
+      bool content_protection_in_adaptation_set,
       AdaptationSet* new_adaptation_set);
 
   // Gets the original AdaptationSet which the trick play video belongs to.
@@ -102,7 +103,7 @@ class Period {
   // the trick play AdaptationSet.
   // Returns the original AdaptationSet if found, otherwise returns nullptr;
   const AdaptationSet* FindOriginalAdaptationSetForTrickPlay(
-      const MediaInfo& media_info);
+      const MediaInfo& media_info, bool content_protection_in_adaptation_set);
 
   const uint32_t id_;
   const double start_time_in_seconds_;
@@ -127,7 +128,8 @@ class Period {
     // Check if the protected content associated with |adaptation_set| matches
     // with the one in |media_info|.
     bool Match(const AdaptationSet& adaptation_set,
-               const MediaInfo& media_info);
+               const MediaInfo& media_info,
+               bool content_protection_in_adaptation_set);
     // Check if the two adaptation sets are switchable.
     bool Switchable(const AdaptationSet& adaptation_set_a,
                     const AdaptationSet& adaptation_set_b);

packager/mpd/public/mpd_params.h

@@ -80,6 +80,9 @@ struct MpdParams {
   /// SegmentTimeline if it is enabled. It will be populated from segment
   /// duration specified in ChunkingParams if not specified.
   double target_segment_duration = 0;
+  /// If enabled, allow switching between different codecs, if they have the
+  /// same language, media type (audio, video etc) and container type.
+  bool allow_codec_switching = false;
 };
 
 }  // namespace shaka