Adjust hls descriptor validation

- Try to determine container name from segment_template if output is
  not specified
- segment_template descriptor is required for TS
- output descriptor should not be specified for TS
- Generate hls_playlist_name from stream number if not specified
- Update packager_test.py to include playlist validation

b/29551186

Change-Id: Ide7169f1c26c85a6e38272dda0e1af77135b4d90

packager/app/packager_main.cc

@@ -238,13 +238,12 @@ bool CreateRemuxJobs(const StreamDescriptorList& stream_descriptors,
   DCHECK(!(mpd_notifier && hls_notifier));
   DCHECK(remux_jobs);
 
-  // This is the counter for audio that doesn't have a name set.
-  int hls_audio_name_counter = 0;
   std::string previous_input;
-  for (StreamDescriptorList::const_iterator stream_iter =
+  int stream_number = 0;
-           stream_descriptors.begin();
+  for (StreamDescriptorList::const_iterator
+           stream_iter = stream_descriptors.begin();
        stream_iter != stream_descriptors.end();
-       ++stream_iter) {
+       ++stream_iter, ++stream_number) {
     // Process stream descriptor.
     MuxerOptions stream_muxer_options(muxer_options);
     stream_muxer_options.output_file_name = stream_iter->output;
@@ -317,20 +316,8 @@ bool CreateRemuxJobs(const StreamDescriptorList& stream_descriptors,
     }
     DCHECK(!remux_jobs->empty());
 
-    MediaContainerName output_format = stream_iter->output_format;
-    if (output_format == CONTAINER_UNKNOWN) {
-      output_format =
-          DetermineContainerFromFileName(stream_muxer_options.output_file_name);
-
-      if (output_format == CONTAINER_UNKNOWN) {
-        LOG(ERROR) << "Unable to determine output format for file "
-                   << stream_muxer_options.output_file_name;
-        return false;
-      }
-    }
-
     scoped_ptr<Muxer> muxer(
-        CreateOutputMuxer(stream_muxer_options, output_format));
+        CreateOutputMuxer(stream_muxer_options, stream_iter->output_format));
     if (FLAGS_use_fake_clock_for_muxer) muxer->set_clock(fake_clock);
 
     if (key_source) {
@@ -359,17 +346,19 @@ bool CreateRemuxJobs(const StreamDescriptorList& stream_descriptors,
 
     if (hls_notifier) {
       // TODO(rkuroiwa): Do some smart stuff to group the audios, e.g. detect
-      // languages. Also detect whether it is audio so that the counter for
+      // languages.
-      // audio%d is continuous.
       std::string group_id = stream_iter->hls_group_id;
       std::string name = stream_iter->hls_name;
+      std::string hls_playlist_name = stream_iter->hls_playlist_name;
       if (group_id.empty())
         group_id = "audio";
       if (name.empty())
-        name = base::StringPrintf("audio%d", hls_audio_name_counter++);
+        name = base::StringPrintf("stream_%d", stream_number);
+      if (hls_playlist_name.empty())
+        hls_playlist_name = base::StringPrintf("stream_%d.m3u8", stream_number);
 
-      muxer_listener.reset(new HlsNotifyMuxerListener(
+      muxer_listener.reset(new HlsNotifyMuxerListener(hls_playlist_name, name,
-          stream_iter->hls_playlist_name, name, group_id, hls_notifier));
+                                                      group_id, hls_notifier));
     }
 
     if (muxer_listener)

packager/app/stream_descriptor.cc

@@ -10,6 +10,7 @@
 #include "packager/base/logging.h"
 #include "packager/base/strings/string_number_conversions.h"
 #include "packager/base/strings/string_split.h"
+#include "packager/media/base/container_names.h"
 #include "packager/mpd/base/language_utils.h"
 
 namespace shaka {
@@ -159,13 +160,39 @@ bool InsertStreamDescriptor(const std::string& descriptor_string,
     return false;
   }
 
-  // Note that MPEG2 TS doesn't need a separate initialization segment, so
+  if (descriptor.output_format == CONTAINER_UNKNOWN) {
-  // output field is ignored.
+    const std::string& output_name = descriptor.output.empty()
-  const bool is_mpeg2ts_with_segment_template =
+                                         ? descriptor.segment_template
-      descriptor.output_format == MediaContainerName::CONTAINER_MPEG2TS &&
+                                         : descriptor.output;
-      !descriptor.segment_template.empty();
+    if (!output_name.empty()) {
-  if (!FLAGS_dump_stream_info && descriptor.output.empty() &&
+      descriptor.output_format = DetermineContainerFromFileName(output_name);
-      !is_mpeg2ts_with_segment_template) {
+      if (descriptor.output_format == CONTAINER_UNKNOWN) {
+        LOG(ERROR) << "Unable to determine output format for file "
+                   << output_name;
+        return false;
+      }
+    }
+  }
+
+  if (descriptor.output_format == MediaContainerName::CONTAINER_MPEG2TS) {
+    if (descriptor.segment_template.empty()) {
+      LOG(ERROR) << "Please specify segment_template. Single file TS output is "
+                    "not supported.";
+      return false;
+    }
+    // Note that MPEG2 TS doesn't need a separate initialization segment, so
+    // output field is not needed.
+    if (!descriptor.output.empty()) {
+      LOG(WARNING) << "TS output '" << descriptor.output
+                   << "' ignored. TS muxer does not support initialization "
+                      "segment generation.";
+    }
+    // For convenience, set descriptor.output to descriptor.segment_template. It
+    // is only used for flag checking in variuos places.
+    descriptor.output = descriptor.segment_template;
+  }
+
+  if (!FLAGS_dump_stream_info && descriptor.output.empty()) {
     LOG(ERROR) << "Stream output not specified.";
     return false;
   }

packager/app/test/packager_app.py

@@ -5,10 +5,8 @@
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file or at
 # https://developers.google.com/open-source/licenses/bsd
-
 """Test wrapper for the sample packager binary."""
 
-
 import os
 import subprocess
 

packager/app/test/packager_test.py

@@ -30,6 +30,7 @@ class PackagerAppTest(unittest.TestCase):
                                         'test', 'testdata')
     self.output_prefix = os.path.join(self.tmp_dir, 'output')
     self.mpd_output = self.output_prefix + '.mpd'
+    self.hls_master_playlist_output = self.output_prefix + '.m3u8'
     self.output = None
 
   def tearDown(self):
@@ -107,15 +108,19 @@ class PackagerAppTest(unittest.TestCase):
                          output_format='ts',
                          live=True,
                          test_files=['bear-640x360.ts']),
-        self._GetFlags(live=True))
+        self._GetFlags(live=True, output_hls=True))
     self._DiffLiveGold(self.output[0],
                        'bear-640x360-a-golden',
-                       has_init_segment=False,
+                       output_format='ts')
-                       segment_extension='ts')
     self._DiffLiveGold(self.output[1],
                        'bear-640x360-v-golden',
-                       has_init_segment=False,
+                       output_format='ts')
-                       segment_extension='ts')
+    self._DiffGold(self.hls_master_playlist_output,
+                   'bear-640x360-av-master-golden.m3u8')
+    self._DiffGold(
+        os.path.join(self.tmp_dir, 'audio.m3u8'), 'bear-640x360-a-golden.m3u8')
+    self._DiffGold(
+        os.path.join(self.tmp_dir, 'video.m3u8'), 'bear-640x360-v-golden.m3u8')
 
   def testPackageVp8Webm(self):
     self.packager.Package(
@@ -188,6 +193,30 @@ class PackagerAppTest(unittest.TestCase):
     self._VerifyDecryption(self.output[0], 'bear-640x360-a-golden.mp4')
     self._VerifyDecryption(self.output[1], 'bear-640x360-v-golden.mp4')
 
+  def testPackageAvcTsWithEncryption(self):
+    # Currently we only support live packaging for ts.
+    self.packager.Package(
+        self._GetStreams(['audio', 'video'],
+                         output_format='ts',
+                         live=True,
+                         test_files=['bear-640x360.ts']),
+        self._GetFlags(encryption=True,
+                       live=True, output_hls=True))
+    self._DiffLiveGold(self.output[0],
+                       'bear-640x360-a-enc-golden',
+                       output_format='ts')
+    self._DiffLiveGold(self.output[1],
+                       'bear-640x360-v-enc-golden',
+                       output_format='ts')
+    self._DiffGold(self.hls_master_playlist_output,
+                   'bear-640x360-av-enc-master-golden.m3u8')
+    self._DiffGold(
+        os.path.join(self.tmp_dir, 'audio.m3u8'),
+        'bear-640x360-a-enc-golden.m3u8')
+    self._DiffGold(
+        os.path.join(self.tmp_dir, 'video.m3u8'),
+        'bear-640x360-v-enc-golden.m3u8')
+
   def testPackageWebmWithEncryption(self):
     self.packager.Package(
         self._GetStreams(['video'],
@@ -392,10 +421,18 @@ class PackagerAppTest(unittest.TestCase):
           output_prefix = '%s_%d_%s' % (self.output_prefix, test_file_index,
                                         stream_descriptor)
         if live:
-          stream = ('input=%s,stream=%s,format=%s,init_segment=%s-init.mp4,'
+          if output_format == 'ts':
-                    'segment_template=%s-$Number$.m4s')
+            stream = ('input=%s,stream=%s,format=%s,'
-          streams.append(stream % (test_file, stream_descriptor, output_format,
+                      'segment_template=%s-$Number$.ts,playlist_name=%s.m3u8')
-                                   output_prefix, output_prefix))
+            streams.append(stream % (test_file, stream_descriptor,
+                                     output_format, output_prefix,
+                                     stream_descriptor))
+          else:
+            stream = ('input=%s,stream=%s,format=%s,init_segment=%s-init.mp4,'
+                      'segment_template=%s-$Number$.m4s')
+            streams.append(stream % (test_file, stream_descriptor,
+                                     output_format, output_prefix,
+                                     output_prefix))
           self.output.append(output_prefix)
         else:
           output = '%s.%s' % (
@@ -423,6 +460,7 @@ class PackagerAppTest(unittest.TestCase):
                 live=False,
                 dash_if_iop=False,
                 output_media_info=False,
+                output_hls=False,
                 use_fake_clock=True):
     flags = []
     if widevine_encryption:
@@ -432,14 +470,16 @@ class PackagerAppTest(unittest.TestCase):
                 '--key_server_url=' + widevine_server_url,
                 '--content_id=3031323334353637', '--signer=widevine_test']
     elif encryption:
-      pssh_box = ('000000307073736800000000'  # PSSH header
-                  'edef8ba979d64acea3c827dcd51d21ed'  # Widevine system ID
-                  '00000010'  # Data size
-                  '31323334353637383930313233343536')  # Data
       flags += ['--enable_fixed_key_encryption',
                 '--key_id=31323334353637383930313233343536',
-                '--key=32333435363738393021323334353637', '--pssh=' + pssh_box,
+                '--key=32333435363738393021323334353637', '--clear_lead=1']
-                '--clear_lead=1']
+      if not output_hls:
+        pssh_box = ('000000307073736800000000'  # PSSH header
+                    'edef8ba979d64acea3c827dcd51d21ed'  # Widevine system ID
+                    '00000010'  # Data size
+                    '31323334353637383930313233343536')  # Data
+        flags.append('--pssh=' + pssh_box)
+
       if not random_iv:
         flags.append('--iv=3334353637383930')
     if protection_scheme:
@@ -459,6 +499,8 @@ class PackagerAppTest(unittest.TestCase):
       flags.append('--generate_dash_if_iop_compliant_mpd')
     if output_media_info:
       flags.append('--output_media_info')
+    elif output_hls:
+      flags += ['--hls_master_playlist_output', self.hls_master_playlist_output]
     else:
       flags += ['--mpd_output', self.mpd_output]
 
@@ -508,15 +550,18 @@ class PackagerAppTest(unittest.TestCase):
   def _DiffLiveGold(self,
                     test_output_prefix,
                     golden_file_name_prefix,
-                    has_init_segment=True,
+                    output_format='mp4'):
-                    segment_extension='m4s'):
     # Compare init and the first three segments.
-    if has_init_segment:
+    if output_format == 'ts':
+      for i in range(1, 4):
+        self._DiffGold('%s-%d.ts' % (test_output_prefix, i),
+                       '%s-%d.ts' % (golden_file_name_prefix, i))
+    else:
       self._DiffGold(test_output_prefix + '-init.mp4',
                      golden_file_name_prefix + '-init.mp4')
-    for i in range(1, 4):
+      for i in range(1, 4):
-      self._DiffGold('%s-%d.m4s' % (test_output_prefix, i), '%s-%d.%s' %
+        self._DiffGold('%s-%d.m4s' % (test_output_prefix, i),
-                     (golden_file_name_prefix, i, segment_extension))
+                       '%s-%d.m4s' % (golden_file_name_prefix, i))
 
   # Live mpd contains current availabilityStartTime and publishTime, which
   # needs to be replaced for comparison.

packager/app/test/testdata/bear-640x360-a-enc-golden.m3u8

@@ -0,0 +1,13 @@
+#EXTM3U
+#EXT-X-VERSION:5
+#EXT-X-TARGETDURATION:2
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXTINF:1.021,
+output_audio-1.ts
+#EXT-X-DISCONTINUITY
+#EXT-X-KEY:METHOD=SAMPLE-AES,URI="data:text/plain;base64,MTIzNDU2Nzg5MDEyMzQ1Ng==",IV=0x3334353637383930,KEYFORMAT="identity"
+#EXTINF:1.021,
+output_audio-2.ts
+#EXTINF:0.720,
+output_audio-3.ts
+#EXT-X-ENDLIST

packager/app/test/testdata/bear-640x360-a-golden.m3u8

@@ -0,0 +1,11 @@
+#EXTM3U
+#EXT-X-VERSION:5
+#EXT-X-TARGETDURATION:2
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXTINF:1.021,
+output_audio-1.ts
+#EXTINF:1.021,
+output_audio-2.ts
+#EXTINF:0.720,
+output_audio-3.ts
+#EXT-X-ENDLIST

packager/app/test/testdata/bear-640x360-av-enc-master-golden.m3u8

@@ -0,0 +1,4 @@
+#EXTM3U
+#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="stream_0",URI="audio.m3u8"
+#EXT-X-STREAM-INF:AUDIO="audio",CODECS="avc1.64001e,mp4a.40.2",BANDWIDTH=1217603
+video.m3u8

packager/app/test/testdata/bear-640x360-av-master-golden.m3u8

@@ -0,0 +1,4 @@
+#EXTM3U
+#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="stream_0",URI="audio.m3u8"
+#EXT-X-STREAM-INF:AUDIO="audio",CODECS="avc1.64001e,mp4a.40.2",BANDWIDTH=1217603
+video.m3u8

packager/app/test/testdata/bear-640x360-v-enc-golden.m3u8

@@ -0,0 +1,13 @@
+#EXTM3U
+#EXT-X-VERSION:5
+#EXT-X-TARGETDURATION:2
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXTINF:1.001,
+output_video-1.ts
+#EXT-X-DISCONTINUITY
+#EXT-X-KEY:METHOD=SAMPLE-AES,URI="data:text/plain;base64,MTIzNDU2Nzg5MDEyMzQ1Ng==",IV=0x3334353637383930,KEYFORMAT="identity"
+#EXTINF:1.001,
+output_video-2.ts
+#EXTINF:0.734,
+output_video-3.ts
+#EXT-X-ENDLIST

packager/app/test/testdata/bear-640x360-v-golden.m3u8

@@ -0,0 +1,11 @@
+#EXTM3U
+#EXT-X-VERSION:5
+#EXT-X-TARGETDURATION:2
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXTINF:1.001,
+output_video-1.ts
+#EXTINF:1.001,
+output_video-2.ts
+#EXTINF:0.734,
+output_video-3.ts
+#EXT-X-ENDLIST