Add driver program flags for HLS

- Add flags and stream descriptor fields for HLS.
- Remove fields from MuxerOptions. Instead pass them directly to
  HlsNotifyMuxerListener.
- Rebase segment names to the master playlist's path where it makes
  sense.

Change-Id: If4f54bc56ff46dc6140859a79ed66f7b99112ed7

packager/app/hls_flags.cc

@@ -0,0 +1,17 @@
+// Copyright 2016 Google Inc. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd
+
+#include "packager/app/hls_flags.h"
+
+DEFINE_string(hls_master_playlist_output,
+              "",
+              "Output path for the master playlist for HLS. This flag must be"
+              "used to output HLS.");
+
+DEFINE_string(hls_base_url,
+              "",
+              "The base URL for the Media Playlists and TS files listed in the "
+              "playlists. This is the prefix for the files.");

packager/app/hls_flags.h

@@ -0,0 +1,15 @@
+// Copyright 2016 Google Inc. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd
+
+#ifndef PACKAGER_APP_HLS_FLAGS_H_
+#define PACKAGER_APP_HLS_FLAGS_H_
+
+#include <gflags/gflags.h>
+
+DECLARE_string(hls_master_playlist_output);
+DECLARE_string(hls_base_url);
+
+#endif  // PACKAGER_APP_HLS_FLAGS_H_

packager/app/packager_main.cc

@@ -8,6 +8,7 @@
 #include <iostream>
 
 #include "packager/app/fixed_key_encryption_flags.h"
+#include "packager/app/hls_flags.h"
 #include "packager/app/libcrypto_threading.h"
 #include "packager/app/mpd_flags.h"
 #include "packager/app/muxer_flags.h"
@@ -17,18 +18,22 @@
 #include "packager/app/widevine_encryption_flags.h"
 #include "packager/base/at_exit.h"
 #include "packager/base/command_line.h"
+#include "packager/base/files/file_path.h"
 #include "packager/base/logging.h"
 #include "packager/base/stl_util.h"
 #include "packager/base/strings/string_split.h"
 #include "packager/base/strings/stringprintf.h"
 #include "packager/base/threading/simple_thread.h"
 #include "packager/base/time/clock.h"
+#include "packager/hls/base/hls_notifier.h"
+#include "packager/hls/base/simple_hls_notifier.h"
 #include "packager/media/base/container_names.h"
 #include "packager/media/base/demuxer.h"
 #include "packager/media/base/fourccs.h"
 #include "packager/media/base/key_source.h"
 #include "packager/media/base/muxer_options.h"
 #include "packager/media/base/muxer_util.h"
+#include "packager/media/event/hls_notify_muxer_listener.h"
 #include "packager/media/event/mpd_notify_muxer_listener.h"
 #include "packager/media/event/vod_media_info_dump_muxer_listener.h"
 #include "packager/media/file/file.h"
@@ -76,7 +81,15 @@ const char kUsage[] =
     "    metadata in the input track.\n"
     "  - output_format (format): Optional value which specifies the format\n"
     "    of the output files (MP4 or WebM).  If not specified, it will be\n"
-    "    derived from the file extension of the output file.\n";
+    "    derived from the file extension of the output file.\n"
+    "  - hls_name: Required for audio when outputting HLS.\n"
+    "    name of the output stream. This is not (necessarily) the same as\n"
+    "    output. This is used as the NAME attribute for EXT-X-MEDIA\n"
+    "  - hls_group_id: Required for audio when outputting HLS.\n"
+    "    The group ID for the output stream. For HLS this is used as the\n"
+    "    GROUP-ID attribute for EXT-X-MEDIA.\n"
+    "  - playlist_name: Required for HLS output.\n"
+    "    Name of the playlist for the stream. Usually ends with '.m3u8'.\n";
 
 const char kMediaInfoSuffix[] = ".media_info";
 
@@ -219,9 +232,14 @@ bool CreateRemuxJobs(const StreamDescriptorList& stream_descriptors,
                      FakeClock* fake_clock,
                      KeySource* key_source,
                      MpdNotifier* mpd_notifier,
+                     hls::HlsNotifier* hls_notifier,
                      std::vector<RemuxJob*>* remux_jobs) {
+  // No notifiers OR (mpd_notifier XOR hls_notifier); which is NAND.
+  DCHECK(!(mpd_notifier && hls_notifier));
   DCHECK(remux_jobs);
 
+  // This is the counter for audio that doesn't have a name set.
+  int hls_audio_name_counter = 0;
   std::string previous_input;
   for (StreamDescriptorList::const_iterator stream_iter =
            stream_descriptors.begin();
@@ -334,14 +352,30 @@ bool CreateRemuxJobs(const StreamDescriptorList& stream_descriptors,
       muxer_listener = mpd_notify_muxer_listener.Pass();
     }
 
+    if (hls_notifier) {
+      // TODO(rkuroiwa): Do some smart stuff to group the audios, e.g. detect
+      // languages. Also detect whether it is audio so that the counter for
+      // audio%d is continuous.
+      std::string group_id = stream_iter->hls_group_id;
+      std::string name = stream_iter->hls_name;
+      if (group_id.empty())
+        group_id = "audio";
+      if (name.empty())
+        name = base::StringPrintf("audio%d", hls_audio_name_counter++);
+
+      muxer_listener.reset(new HlsNotifyMuxerListener(
+          stream_iter->hls_playlist_name, name, group_id, hls_notifier));
+    }
+
     if (muxer_listener)
       muxer->SetMuxerListener(muxer_listener.Pass());
 
     if (!AddStreamToMuxer(remux_jobs->back()->demuxer()->streams(),
                           stream_iter->stream_selector,
                           stream_iter->language,
-                          muxer.get()))
+                          muxer.get())) {
       return false;
+    }
     remux_jobs->back()->AddMuxer(muxer.Pass());
   }
 
@@ -398,6 +432,13 @@ bool RunPackager(const StreamDescriptorList& stream_descriptors) {
     return false;
   }
 
+  // Since there isn't a muxer listener that can output both MPD and HLS,
+  // disallow specifying both MPD and HLS flags.
+  if (!FLAGS_mpd_output.empty() && !FLAGS_hls_master_playlist_output.empty()) {
+    LOG(ERROR) << "Cannot output both MPD and HLS.";
+    return false;
+  }
+
   // Get basic muxer options.
   MuxerOptions muxer_options;
   if (!GetMuxerOptions(&muxer_options))
@@ -434,12 +475,23 @@ bool RunPackager(const StreamDescriptorList& stream_descriptors) {
     }
   }
 
+  scoped_ptr<hls::HlsNotifier> hls_notifier;
+  if (!FLAGS_hls_master_playlist_output.empty()) {
+    base::FilePath master_playlist_path(FLAGS_hls_master_playlist_output);
+    base::FilePath master_playlist_name = master_playlist_path.BaseName();
+
+    hls_notifier.reset(new hls::SimpleHlsNotifier(
+        hls::HlsNotifier::HlsProfile::kOnDemandProfile, FLAGS_hls_base_url,
+        master_playlist_path.DirName().AsEndingWithSeparator().value(),
+        master_playlist_name.value()));
+  }
+
   std::vector<RemuxJob*> remux_jobs;
   STLElementDeleter<std::vector<RemuxJob*> > scoped_jobs_deleter(&remux_jobs);
   FakeClock fake_clock;
   if (!CreateRemuxJobs(stream_descriptors, muxer_options, &fake_clock,
                        encryption_key_source.get(), mpd_notifier.get(),
-                       &remux_jobs)) {
+                       hls_notifier.get(), &remux_jobs)) {
     return false;
   }
 
@@ -449,6 +501,15 @@ bool RunPackager(const StreamDescriptorList& stream_descriptors) {
     return false;
   }
 
+  if (hls_notifier) {
+    if (!hls_notifier->Flush())
+      return false;
+  }
+  if (mpd_notifier) {
+    if (!mpd_notifier->Flush())
+      return false;
+  }
+
   printf("Packaging completed successfully.\n");
   return true;
 }

packager/app/stream_descriptor.cc

@@ -26,6 +26,9 @@ enum FieldType {
   kBandwidthField,
   kLanguageField,
   kOutputFormatField,
+  kHlsNameField,
+  kHlsGroupIdField,
+  kHlsPlaylistNameField,
 };
 
 struct FieldNameToTypeMapping {
@@ -50,6 +53,9 @@ const FieldNameToTypeMapping kFieldNameTypeMappings[] = {
     {"lang", kLanguageField},
     {"output_format", kOutputFormatField},
     {"format", kOutputFormatField},
+    {"hls_name", kHlsNameField},
+    {"hls_group_id", kHlsGroupIdField},
+    {"playlist_name", kHlsPlaylistNameField},
 };
 
 FieldType GetFieldType(const std::string& field_name) {
@@ -124,6 +130,18 @@ bool InsertStreamDescriptor(const std::string& descriptor_string,
         descriptor.output_format = output_format;
         break;
       }
+      case kHlsNameField: {
+        descriptor.hls_name = iter->second;
+        break;
+      }
+      case kHlsGroupIdField: {
+        descriptor.hls_group_id = iter->second;
+        break;
+      }
+      case kHlsPlaylistNameField: {
+        descriptor.hls_playlist_name = iter->second;
+        break;
+      }
       default:
         LOG(ERROR) << "Unknown field in stream descriptor (\"" << iter->first
                    << "\").";
@@ -140,7 +158,14 @@ bool InsertStreamDescriptor(const std::string& descriptor_string,
     LOG(ERROR) << "Stream stream_selector not specified.";
     return false;
   }
-  if (!FLAGS_dump_stream_info && descriptor.output.empty()) {
+
+  // Note that MPEG2 TS doesn't need a separate initialization segment, so
+  // output field is ignored.
+  const bool is_mpeg2ts_with_segment_template =
+      descriptor.output_format == MediaContainerName::CONTAINER_MPEG2TS &&
+      !descriptor.segment_template.empty();
+  if (!FLAGS_dump_stream_info && descriptor.output.empty() &&
+      !is_mpeg2ts_with_segment_template) {
     LOG(ERROR) << "Stream output not specified.";
     return false;
   }

packager/app/stream_descriptor.h

@@ -30,6 +30,9 @@ struct StreamDescriptor {
   uint32_t bandwidth;
   std::string language;
   MediaContainerName output_format;
+  std::string hls_name;
+  std::string hls_group_id;
+  std::string hls_playlist_name;
 };
 
 class StreamDescriptorCompareFn {

packager/hls/base/simple_hls_notifier.cc

@@ -7,6 +7,7 @@
 #include "packager/hls/base/simple_hls_notifier.h"
 
 #include "packager/base/base64.h"
+#include "packager/base/files/file_path.h"
 #include "packager/base/logging.h"
 #include "packager/base/strings/string_number_conversions.h"
 #include "packager/base/strings/stringprintf.h"
@@ -24,6 +25,43 @@ bool IsWidevineSystemId(const std::vector<uint8_t>& system_id) {
   return system_id.size() == arraysize(kSystemIdWidevine) &&
          std::equal(system_id.begin(), system_id.end(), kSystemIdWidevine);
 }
+
+// TODO(rkuroiwa): Dedup these with the functions in MpdBuilder.
+std::string MakePathRelative(const std::string& original_path,
+                             const std::string& output_dir) {
+  return (original_path.find(output_dir) == 0)
+             ? original_path.substr(output_dir.size())
+             : original_path;
+}
+
+void MakePathsRelativeToOutputDirectory(const std::string& output_dir,
+                                        MediaInfo* media_info) {
+  DCHECK(media_info);
+  const std::string kFileProtocol("file://");
+  std::string prefix_stripped_output_dir =
+      (output_dir.find(kFileProtocol) == 0)
+          ? output_dir.substr(kFileProtocol.size())
+          : output_dir;
+
+  if (prefix_stripped_output_dir.empty())
+    return;
+
+  std::string directory_with_separator(
+      base::FilePath(prefix_stripped_output_dir)
+          .AsEndingWithSeparator()
+          .value());
+  if (directory_with_separator.empty())
+    return;
+
+  if (media_info->has_media_file_name()) {
+    media_info->set_media_file_name(MakePathRelative(
+        media_info->media_file_name(), directory_with_separator));
+  }
+  if (media_info->has_segment_template()) {
+    media_info->set_segment_template(MakePathRelative(
+        media_info->segment_template(), directory_with_separator));
+  }
+}
 }  // namespace
 
 MediaPlaylistFactory::~MediaPlaylistFactory() {}
@@ -74,9 +112,13 @@ bool SimpleHlsNotifier::NotifyNewStream(const MediaInfo& media_info,
       NOTREACHED();
       return false;
   }
+
+  MediaInfo adjusted_media_info(media_info);
+  MakePathsRelativeToOutputDirectory(output_dir_, &adjusted_media_info);
+
   scoped_ptr<MediaPlaylist> media_playlist =
       media_playlist_factory_->Create(type, playlist_name, name, group_id);
-  if (!media_playlist->SetMediaInfo(media_info)) {
+  if (!media_playlist->SetMediaInfo(adjusted_media_info)) {
     LOG(ERROR) << "Failed to set media info for playlist " << playlist_name;
     return false;
   }
@@ -99,8 +141,11 @@ bool SimpleHlsNotifier::NotifyNewSegment(uint32_t stream_id,
     LOG(ERROR) << "Cannot find stream with ID: " << stream_id;
     return false;
   }
+  const std::string relative_segment_name =
+      MakePathRelative(segment_name, output_dir_);
+
   auto& media_playlist = result->second;
-  media_playlist->AddSegment(prefix_ + segment_name, duration, size);
+  media_playlist->AddSegment(prefix_ + relative_segment_name, duration, size);
   return true;
 }
 

packager/hls/base/simple_hls_notifier_unittest.cc

@@ -55,6 +55,15 @@ class MockMediaPlaylistFactory : public MediaPlaylistFactory {
 const char kTestPrefix[] = "http://testprefix.com/";
 const char kAnyOutputDir[] = "anything/";
 
+const uint64_t kAnyStartTime = 10;
+const uint64_t kAnyDuration = 1000;
+const uint64_t kAnySize = 2000;
+
+MATCHER_P(SegmentTemplateEq, expected_template, "") {
+  *result_listener << " which is " << arg.segment_template();
+  return arg.segment_template() == expected_template;
+}
+
 }  // namespace
 
 class SimpleHlsNotifierTest : public ::testing::Test {
@@ -69,10 +78,20 @@ class SimpleHlsNotifierTest : public ::testing::Test {
     notifier_.media_playlist_factory_ = factory.Pass();
   }
 
+  void InjectMediaPlaylistFactory(scoped_ptr<MediaPlaylistFactory> factory,
+                                  SimpleHlsNotifier* notifier) {
+    notifier->media_playlist_factory_ = factory.Pass();
+  }
+
   void InjectMasterPlaylist(scoped_ptr<MasterPlaylist> playlist) {
     notifier_.master_playlist_ = playlist.Pass();
   }
 
+  void InjectMasterPlaylist(scoped_ptr<MasterPlaylist> playlist,
+                            SimpleHlsNotifier* notifier) {
+    notifier->master_playlist_ = playlist.Pass();
+  }
+
   const std::map<uint32_t, MediaPlaylist*>& GetMediaPlaylistMap() {
     return notifier_.media_playlist_map_;
   }
@@ -84,6 +103,128 @@ TEST_F(SimpleHlsNotifierTest, Init) {
   EXPECT_TRUE(notifier_.Init());
 }
 
+// Verify that relative paths can be handled.
+// For this test, since the prefix "anything/" matches, the prefix should be
+// stripped.
+TEST_F(SimpleHlsNotifierTest, RebaseSegmentTemplateRelative) {
+  scoped_ptr<MockMasterPlaylist> mock_master_playlist(new MockMasterPlaylist());
+  scoped_ptr<MockMediaPlaylistFactory> factory(new MockMediaPlaylistFactory());
+
+  // Pointer released by SimpleHlsNotifier.
+  MockMediaPlaylist* mock_media_playlist =
+      new MockMediaPlaylist(kVodPlaylist, "", "", "");
+  EXPECT_CALL(*mock_master_playlist, AddMediaPlaylist(mock_media_playlist));
+
+  EXPECT_CALL(
+      *mock_media_playlist,
+      SetMediaInfo(SegmentTemplateEq("path/to/media$Number$.ts")))
+      .WillOnce(Return(true));
+
+  // Verify that the common prefix is stripped for AddSegment().
+  EXPECT_CALL(*mock_media_playlist,
+              AddSegment("http://testprefix.com/path/to/media1.ts", _, _));
+  EXPECT_CALL(*factory, CreateMock(kVodPlaylist, StrEq("video_playlist.m3u8"),
+                                   StrEq("name"), StrEq("groupid")))
+      .WillOnce(Return(mock_media_playlist));
+
+  InjectMasterPlaylist(mock_master_playlist.Pass());
+  InjectMediaPlaylistFactory(factory.Pass());
+  EXPECT_TRUE(notifier_.Init());
+  MediaInfo media_info;
+  media_info.set_segment_template("anything/path/to/media$Number$.ts");
+  uint32_t stream_id;
+  EXPECT_TRUE(notifier_.NotifyNewStream(media_info, "video_playlist.m3u8",
+                                        "name", "groupid", &stream_id));
+
+  EXPECT_TRUE(
+      notifier_.NotifyNewSegment(stream_id, "anything/path/to/media1.ts",
+                                 kAnyStartTime, kAnyDuration, kAnySize));
+}
+
+// Verify that when segment template's prefix and output dir match, then the
+// prefix is stripped from segment template.
+TEST_F(SimpleHlsNotifierTest,
+       RebaseAbsoluteSegmentTemplatePrefixAndOutputDirMatch) {
+  const char kAbsoluteOutputDir[] = "/tmp/something/";
+  // Require a separate instance to set kAbsoluteOutputDir.
+  SimpleHlsNotifier test_notifier(HlsNotifier::HlsProfile::kOnDemandProfile,
+                                  kTestPrefix, kAbsoluteOutputDir,
+                                  kMasterPlaylistName);
+
+  scoped_ptr<MockMasterPlaylist> mock_master_playlist(new MockMasterPlaylist());
+  scoped_ptr<MockMediaPlaylistFactory> factory(new MockMediaPlaylistFactory());
+
+  // Pointer released by SimpleHlsNotifier.
+  MockMediaPlaylist* mock_media_playlist =
+      new MockMediaPlaylist(kVodPlaylist, "", "", "");
+  EXPECT_CALL(*mock_master_playlist, AddMediaPlaylist(mock_media_playlist));
+
+  EXPECT_CALL(*mock_media_playlist,
+              SetMediaInfo(SegmentTemplateEq("media$Number$.ts")))
+      .WillOnce(Return(true));
+
+  // Verify that the output_dir is stripped and then kTestPrefix is prepended.
+  EXPECT_CALL(*mock_media_playlist,
+              AddSegment("http://testprefix.com/media1.ts", _, _));
+  EXPECT_CALL(*factory, CreateMock(kVodPlaylist, StrEq("video_playlist.m3u8"),
+                                   StrEq("name"), StrEq("groupid")))
+      .WillOnce(Return(mock_media_playlist));
+
+  InjectMasterPlaylist(mock_master_playlist.Pass(), &test_notifier);
+  InjectMediaPlaylistFactory(factory.Pass(), &test_notifier);
+  EXPECT_TRUE(test_notifier.Init());
+  MediaInfo media_info;
+  media_info.set_segment_template("/tmp/something/media$Number$.ts");
+  uint32_t stream_id;
+  EXPECT_TRUE(test_notifier.NotifyNewStream(media_info, "video_playlist.m3u8",
+                                            "name", "groupid", &stream_id));
+
+  EXPECT_TRUE(
+      test_notifier.NotifyNewSegment(stream_id, "/tmp/something/media1.ts",
+                                     kAnyStartTime, kAnyDuration, kAnySize));
+}
+
+// If the paths don't match at all and they are both absolute and completely
+// different, then keep it as is.
+TEST_F(SimpleHlsNotifierTest,
+       RebaseAbsoluteSegmentTemplateCompletelyDifferentDirectory) {
+  const char kAbsoluteOutputDir[] = "/tmp/something/";
+  SimpleHlsNotifier test_notifier(HlsNotifier::HlsProfile::kOnDemandProfile,
+                                  kTestPrefix, kAbsoluteOutputDir,
+                                  kMasterPlaylistName);
+
+  scoped_ptr<MockMasterPlaylist> mock_master_playlist(new MockMasterPlaylist());
+  scoped_ptr<MockMediaPlaylistFactory> factory(new MockMediaPlaylistFactory());
+
+  // Pointer released by SimpleHlsNotifier.
+  MockMediaPlaylist* mock_media_playlist =
+      new MockMediaPlaylist(kVodPlaylist, "", "", "");
+  EXPECT_CALL(*mock_master_playlist, AddMediaPlaylist(mock_media_playlist));
+
+  EXPECT_CALL(
+      *mock_media_playlist,
+      SetMediaInfo(SegmentTemplateEq("/var/somewhereelse/media$Number$.ts")))
+      .WillOnce(Return(true));
+  EXPECT_CALL(
+      *mock_media_playlist,
+      AddSegment("http://testprefix.com//var/somewhereelse/media1.ts", _, _));
+  EXPECT_CALL(*factory, CreateMock(kVodPlaylist, StrEq("video_playlist.m3u8"),
+                                   StrEq("name"), StrEq("groupid")))
+      .WillOnce(Return(mock_media_playlist));
+
+  InjectMasterPlaylist(mock_master_playlist.Pass(), &test_notifier);
+  InjectMediaPlaylistFactory(factory.Pass(), &test_notifier);
+  EXPECT_TRUE(test_notifier.Init());
+  MediaInfo media_info;
+  media_info.set_segment_template("/var/somewhereelse/media$Number$.ts");
+  uint32_t stream_id;
+  EXPECT_TRUE(test_notifier.NotifyNewStream(media_info, "video_playlist.m3u8",
+                                            "name", "groupid", &stream_id));
+  EXPECT_TRUE(
+      test_notifier.NotifyNewSegment(stream_id, "/var/somewhereelse/media1.ts",
+                                     kAnyStartTime, kAnyDuration, kAnySize));
+}
+
 TEST_F(SimpleHlsNotifierTest, NotifyNewStream) {
   scoped_ptr<MockMasterPlaylist> mock_master_playlist(new MockMasterPlaylist());
   scoped_ptr<MockMediaPlaylistFactory> factory(new MockMediaPlaylistFactory());

packager/media/base/muxer_options.h

@@ -60,17 +60,6 @@ struct MuxerOptions {
   /// Optional.
   std::string segment_template;
 
-  /// name of the output stream. This is not (necessarily) the same as @a
-  /// output_file_name. For HLS this is used as the NAME attribute for
-  /// EXT-X-MEDIA.
-  /// Required for audio when outputting HLS.
-  std::string hls_name;
-
-  /// The group ID for the output stream.
-  /// For HLS this is used as the GROUP-ID attribute for EXT-X-MEDIA.
-  /// Required for audio when outputting HLS.
-  std::string hls_group_id;
-
   /// Specify temporary directory for intermediate files.
   std::string temp_dir;
 

packager/packager.gyp

@@ -15,6 +15,8 @@
       'sources': [
         'app/fixed_key_encryption_flags.cc',
         'app/fixed_key_encryption_flags.h',
+        'app/hls_flags.cc',
+        'app/hls_flags.h',
         'app/libcrypto_threading.cc',
         'app/libcrypto_threading.h',
         'app/mpd_flags.cc',