Add new URL fields in MediaInfo

The file_name fields will be used to solely indicate file paths on the
designated file system, and they are used to do normal file operations,
including file creation, file updating and file removal if needed;
added new xxx_url fields, for the URLs that should appear on DASH
manifest or HLS playlists.

xxx_url are the URIs of the media in the manifest. The fields are
converted from file_name fields but adjusted to be relative to DASH
manifest path or HLS playlist path, optionally with base_url prepended.

Previously the file_name fields are converted in place to indicate
URLs when passing to manifest / playlist builders. The original file
names were lost, which made it difficult to remove files outside of
live window.

Now that the input file names are preserved. File system APIs can
operate on the original file names while manifest / playlist generation
functions can operate on URLs.

Issue: #233

Change-Id: I36a64f16e3d1261ce91783a86588f24ad1371662

packager/app/mpd_generator.cc

@@ -78,9 +78,9 @@ ExitStatus RunMpdGenerator() {
   for (Iterator it = base_urls.begin(); it != base_urls.end(); ++it)
     mpd_writer.AddBaseUrl(*it);
 
-  for (Iterator it = input_files.begin(); it != input_files.end(); ++it) {
+  for (const std::string& file : input_files) {
-    if (!mpd_writer.AddFile(it->c_str(), FLAGS_output)) {
+    if (!mpd_writer.AddFile(file)) {
-      LOG(WARNING) << "MpdWriter failed to read " << *it << ", skipping.";
+      LOG(WARNING) << "MpdWriter failed to read " << file << ", skipping.";
     }
   }
 

packager/hls/base/media_playlist.cc

@@ -53,15 +53,15 @@ std::string GetLanguage(const MediaInfo& media_info) {
 }
 
 void AppendExtXMap(const MediaInfo& media_info, std::string* out) {
-  if (media_info.has_init_segment_name()) {
+  if (media_info.has_init_segment_url()) {
     Tag tag("#EXT-X-MAP", out);
-    tag.AddQuotedString("URI", media_info.init_segment_name().data());
+    tag.AddQuotedString("URI", media_info.init_segment_url().data());
     out->append("\n");
-  } else if (media_info.has_media_file_name() && media_info.has_init_range()) {
+  } else if (media_info.has_media_file_url() && media_info.has_init_range()) {
     // It only makes sense for single segment media to have EXT-X-MAP if
     // there is init_range.
     Tag tag("#EXT-X-MAP", out);
-    tag.AddQuotedString("URI", media_info.media_file_name().data());
+    tag.AddQuotedString("URI", media_info.media_file_url().data());
 
     if (media_info.has_init_range()) {
       const uint64_t begin = media_info.init_range().begin();
@@ -368,7 +368,7 @@ bool MediaPlaylist::SetMediaInfo(const MediaInfo& media_info) {
   time_scale_ = time_scale;
   media_info_ = media_info;
   language_ = GetLanguage(media_info);
-  use_byte_range_ = !media_info_.has_segment_template();
+  use_byte_range_ = !media_info_.has_segment_template_url();
   return true;
 }
 

packager/hls/base/media_playlist_unittest.cc

@@ -84,7 +84,7 @@ class MediaPlaylistMultiSegmentTest : public MediaPlaylistTest {
     // This is just set to be consistent with the multisegment format and used
     // as a switch in MediaPlaylist.
     // The template string doesn't really matter.
-    valid_video_media_info_.set_segment_template("file$Number$.ts");
+    valid_video_media_info_.set_segment_template_url("file$Number$.ts");
   }
 };
 
@@ -150,7 +150,7 @@ TEST_F(MediaPlaylistSingleSegmentTest, InitRange) {
       "#EXT-X-PLAYLIST-TYPE:VOD\n"
       "#EXT-X-MAP:URI=\"file.mp4\",BYTERANGE=\"501@0\"\n"
       "#EXT-X-ENDLIST\n";
-  valid_video_media_info_.set_media_file_name("file.mp4");
+  valid_video_media_info_.set_media_file_url("file.mp4");
   valid_video_media_info_.mutable_init_range()->set_begin(0);
   valid_video_media_info_.mutable_init_range()->set_end(500);
 
@@ -170,7 +170,7 @@ TEST_F(MediaPlaylistSingleSegmentTest, InitRangeWithOffset) {
       "#EXT-X-PLAYLIST-TYPE:VOD\n"
       "#EXT-X-MAP:URI=\"file.mp4\",BYTERANGE=\"485@16\"\n"
       "#EXT-X-ENDLIST\n";
-  valid_video_media_info_.set_media_file_name("file.mp4");
+  valid_video_media_info_.set_media_file_url("file.mp4");
   valid_video_media_info_.mutable_init_range()->set_begin(16);
   valid_video_media_info_.mutable_init_range()->set_end(500);
 
@@ -199,7 +199,7 @@ TEST_F(MediaPlaylistSingleSegmentTest, AddSegmentByteRange) {
       "#EXT-X-BYTERANGE:2000000\n"
       "file.mp4\n"
       "#EXT-X-ENDLIST\n";
-  valid_video_media_info_.set_media_file_name("file.mp4");
+  valid_video_media_info_.set_media_file_url("file.mp4");
   valid_video_media_info_.mutable_init_range()->set_begin(0);
   valid_video_media_info_.mutable_init_range()->set_end(500);
 
@@ -478,7 +478,7 @@ TEST_F(MediaPlaylistMultiSegmentTest, GetNumChannels) {
 
 TEST_F(MediaPlaylistMultiSegmentTest, InitSegment) {
   valid_video_media_info_.set_reference_time_scale(90000);
-  valid_video_media_info_.set_init_segment_name("init_segment.mp4");
+  valid_video_media_info_.set_init_segment_url("init_segment.mp4");
   ASSERT_TRUE(media_playlist_.SetMediaInfo(valid_video_media_info_));
 
   media_playlist_.AddSegment("file1.mp4", 0, 10 * kTimeScale, kZeroByteOffset,
@@ -785,7 +785,7 @@ TEST_F(IFrameMediaPlaylistTest, MediaPlaylistType) {
 }
 
 TEST_F(IFrameMediaPlaylistTest, SingleSegment) {
-  valid_video_media_info_.set_media_file_name("file.mp4");
+  valid_video_media_info_.set_media_file_url("file.mp4");
   valid_video_media_info_.mutable_init_range()->set_begin(0);
   valid_video_media_info_.mutable_init_range()->set_end(500);
 

packager/hls/base/simple_hls_notifier.cc

@@ -283,15 +283,20 @@ bool SimpleHlsNotifier::NotifyNewStream(const MediaInfo& media_info,
   // Update init_segment_name to be relative to playlist path if needed.
   MediaInfo media_info_copy = media_info;
   if (media_info_copy.has_init_segment_name()) {
-    media_info_copy.set_init_segment_name(
+    media_info_copy.set_init_segment_url(
         GenerateSegmentUrl(media_info_copy.init_segment_name(), prefix_,
                            output_dir_, media_playlist->file_name()));
   }
   if (media_info_copy.has_media_file_name()) {
-    media_info_copy.set_media_file_name(
+    media_info_copy.set_media_file_url(
         GenerateSegmentUrl(media_info_copy.media_file_name(), prefix_,
                            output_dir_, media_playlist->file_name()));
   }
+  if (media_info_copy.has_segment_template()) {
+    media_info_copy.set_segment_template_url(
+        GenerateSegmentUrl(media_info_copy.segment_template(), prefix_,
+                           output_dir_, media_playlist->file_name()));
+  }
   if (!media_playlist->SetMediaInfo(media_info_copy)) {
     LOG(ERROR) << "Failed to set media info for playlist " << playlist_name;
     return false;

packager/hls/base/simple_hls_notifier_unittest.cc

@@ -170,7 +170,7 @@ TEST_F(SimpleHlsNotifierTest, RebaseSegmentUrl) {
       new MockMediaPlaylist(kVodPlaylist, "playlist.m3u8", "", "");
 
   EXPECT_CALL(*mock_media_playlist,
-              SetMediaInfo(Property(&MediaInfo::init_segment_name, StrEq(""))))
+              SetMediaInfo(Property(&MediaInfo::init_segment_url, StrEq(""))))
       .WillOnce(Return(true));
 
   // Verify that the common prefix is stripped for AddSegment().
@@ -211,7 +211,7 @@ TEST_F(SimpleHlsNotifierTest, RebaseInitSegmentUrl) {
   // Verify that the common prefix is stripped in init segment.
   EXPECT_CALL(
       *mock_media_playlist,
-      SetMediaInfo(Property(&MediaInfo::init_segment_name,
+      SetMediaInfo(Property(&MediaInfo::init_segment_url,
                             StrEq("http://testprefix.com/path/to/init.mp4"))))
       .WillOnce(Return(true));
 
@@ -245,7 +245,7 @@ TEST_F(SimpleHlsNotifierTest, RebaseSegmentUrlRelativeToPlaylist) {
 
   // Verify that the init segment URL is relative to playlist path.
   EXPECT_CALL(*mock_media_playlist,
-              SetMediaInfo(Property(&MediaInfo::init_segment_name,
+              SetMediaInfo(Property(&MediaInfo::init_segment_url,
                                     StrEq("path/to/init.mp4"))))
       .WillOnce(Return(true));
 

packager/mpd/base/adaptation_set_unittest.cc

@@ -1023,7 +1023,7 @@ TEST_F(OnDemandAdaptationSetTest,
       "  begin: 864\n"
       "  end: 931\n"
       "}\n"
-      "media_file_name: 'encrypted_audio.mp4'\n"
+      "media_file_url: 'encrypted_audio.mp4'\n"
       "media_duration_seconds: 24.009434\n"
       "reference_time_scale: 44100\n"
       "container_type: CONTAINER_MP4\n";
@@ -1071,7 +1071,7 @@ TEST_F(OnDemandAdaptationSetTest, Text) {
       "}\n"
       "media_duration_seconds: 35\n"
       "bandwidth: 1000\n"
-      "media_file_name: 'subtitle.xml'\n"
+      "media_file_url: 'subtitle.xml'\n"
       "container_type: CONTAINER_TEXT\n";
 
   const char kExpectedOutput[] =

packager/mpd/base/media_info.proto

@@ -153,4 +153,11 @@ message MediaInfo {
   // MpdNotifier::NotifyNewSegment().
   optional float segment_duration_seconds = 12;
   // END LIVE only.
+
+  // URL fields for the corresponding file_name fields above.
+  // The file names are adjusted to be relative to DASH MPD or HLS media
+  // playlist, or with base url prepended.
+  optional string media_file_url = 17;
+  optional string init_segment_url = 18;
+  optional string segment_template_url = 19;
 }

packager/mpd/base/mpd_builder.cc

@@ -414,15 +414,15 @@ void MpdBuilder::MakePathsRelativeToMpd(const std::string& mpd_path,
                                .AsEndingWithSeparator());
     if (!mpd_dir.empty()) {
       if (media_info->has_media_file_name()) {
-        media_info->set_media_file_name(
+        media_info->set_media_file_url(
             MakePathRelative(media_info->media_file_name(), mpd_dir));
       }
       if (media_info->has_init_segment_name()) {
-        media_info->set_init_segment_name(
+        media_info->set_init_segment_url(
             MakePathRelative(media_info->init_segment_name(), mpd_dir));
       }
       if (media_info->has_segment_template()) {
-        media_info->set_segment_template(
+        media_info->set_segment_template_url(
             MakePathRelative(media_info->segment_template(), mpd_dir));
       }
     }

packager/mpd/base/mpd_builder_unittest.cc

@@ -355,9 +355,9 @@ TEST(RelativePaths, PathsModified) {
   media_info.set_init_segment_name(kInitSegment);
   media_info.set_segment_template(kSegmentTemplate);
   MpdBuilder::MakePathsRelativeToMpd(kPathModifiedMpd, &media_info);
-  EXPECT_EQ(kMediaFileBase, media_info.media_file_name());
+  EXPECT_EQ(kMediaFileBase, media_info.media_file_url());
-  EXPECT_EQ(kInitSegmentBase, media_info.init_segment_name());
+  EXPECT_EQ(kInitSegmentBase, media_info.init_segment_url());
-  EXPECT_EQ(kSegmentTemplateBase, media_info.segment_template());
+  EXPECT_EQ(kSegmentTemplateBase, media_info.segment_template_url());
 }
 
 TEST(RelativePaths, PathsNotModified) {
@@ -367,9 +367,9 @@ TEST(RelativePaths, PathsNotModified) {
   media_info.set_init_segment_name(kInitSegment);
   media_info.set_segment_template(kSegmentTemplate);
   MpdBuilder::MakePathsRelativeToMpd(kPathNotModifiedMpd, &media_info);
-  EXPECT_EQ(kMediaFile, media_info.media_file_name());
+  EXPECT_EQ(kMediaFile, media_info.media_file_url());
-  EXPECT_EQ(kInitSegment, media_info.init_segment_name());
+  EXPECT_EQ(kInitSegment, media_info.init_segment_url());
-  EXPECT_EQ(kSegmentTemplate, media_info.segment_template());
+  EXPECT_EQ(kSegmentTemplate, media_info.segment_template_url());
 }
 
 }  // namespace shaka

packager/mpd/base/mpd_utils.cc

@@ -53,12 +53,12 @@ std::string TextCodecString(const MediaInfo& media_info) {
 
 bool HasVODOnlyFields(const MediaInfo& media_info) {
   return media_info.has_init_range() || media_info.has_index_range() ||
-         media_info.has_media_file_name();
+         media_info.has_media_file_url();
 }
 
 bool HasLiveOnlyFields(const MediaInfo& media_info) {
-  return media_info.has_init_segment_name() ||
+  return media_info.has_init_segment_url() ||
-         media_info.has_segment_template() ||
+         media_info.has_segment_template_url() ||
          media_info.has_segment_duration_seconds();
 }
 

packager/mpd/base/representation_unittest.cc

@@ -409,8 +409,8 @@ std::string GetDefaultMediaInfo() {
       "}\n"
       "reference_time_scale: %u\n"
       "container_type: 1\n"
-      "init_segment_name: 'init.mp4'\n"
+      "init_segment_url: 'init.mp4'\n"
-      "segment_template: '$Time$.mp4'\n";
+      "segment_template_url: '$Time$.mp4'\n";
 
   return base::StringPrintf(kMediaInfo, kDefaultTimeScale);
 }
@@ -503,7 +503,7 @@ TEST_F(SegmentTemplateTest, OneSegmentNormal) {
 
 TEST_F(SegmentTemplateTest, RepresentationClone) {
   MediaInfo media_info = ConvertToMediaInfo(GetDefaultMediaInfo());
-  media_info.set_segment_template("$Number$.mp4");
+  media_info.set_segment_template_url("$Number$.mp4");
   representation_ =
       CreateRepresentation(media_info, kAnyRepresentationId, NoListener());
   ASSERT_TRUE(representation_->Init());
@@ -703,8 +703,8 @@ class TimeShiftBufferDepthTest : public SegmentTemplateTest {
         "}\n"
         "reference_time_scale: %u\n"
         "container_type: 1\n"
-        "init_segment_name: 'init.mp4'\n"
+        "init_segment_url: 'init.mp4'\n"
-        "segment_template: '$Number$.mp4'\n";
+        "segment_template_url: '$Number$.mp4'\n";
     const std::string& number_template_media_info =
         base::StringPrintf(kMediaInfo, kDefaultTimeScale);
     mpd_options_.mpd_type = MpdType::kDynamic;

packager/mpd/base/xml/xml_node.cc

@@ -276,9 +276,9 @@ bool RepresentationXmlNode::AddAudioInfo(const AudioInfo& audio_info) {
 }
 
 bool RepresentationXmlNode::AddVODOnlyInfo(const MediaInfo& media_info) {
-  if (media_info.has_media_file_name()) {
+  if (media_info.has_media_file_url()) {
     XmlNode base_url("BaseURL");
-    base_url.SetContent(media_info.media_file_name());
+    base_url.SetContent(media_info.media_file_url());
 
     if (!AddChild(base_url.PassScopedPtr()))
       return false;
@@ -336,24 +336,14 @@ bool RepresentationXmlNode::AddLiveOnlyInfo(
                                          media_info.presentation_time_offset());
   }
 
-  if (media_info.has_init_segment_name()) {
+  if (media_info.has_init_segment_url()) {
-    // The spec does not allow '$Number$' and '$Time$' in initialization
-    // attribute.
-    // TODO(rkuroiwa, kqyang): Swap this check out with a better check. These
-    // templates allow formatting as well.
-    const std::string& init_segment_name = media_info.init_segment_name();
-    if (init_segment_name.find("$Number$") != std::string::npos ||
-        init_segment_name.find("$Time$") != std::string::npos) {
-      LOG(ERROR) << "$Number$ and $Time$ cannot be used for "
-                    "SegmentTemplate@initialization";
-      return false;
-    }
     segment_template.SetStringAttribute("initialization",
-                                        media_info.init_segment_name());
+                                        media_info.init_segment_url());
   }
 
-  if (media_info.has_segment_template()) {
+  if (media_info.has_segment_template_url()) {
-    segment_template.SetStringAttribute("media", media_info.segment_template());
+    segment_template.SetStringAttribute("media",
+                                        media_info.segment_template_url());
     segment_template.SetIntegerAttribute("startNumber", start_number);
   }
 

packager/mpd/base/xml/xml_node_unittest.cc

@@ -191,28 +191,5 @@ TEST(XmlNodeTest, AddEC3AudioInfo) {
           "</Representation>\n"));
 }
 
-// Some template names cannot be used for init segment name.
-TEST(XmlNodeTest, InvalidLiveInitSegmentName) {
-  MediaInfo media_info;
-  const uint32_t kDefaultStartNumber = 1;
-  std::list<SegmentInfo> segment_infos;
-  RepresentationXmlNode representation;
-
-  // $Number$ cannot be used for segment name.
-  media_info.set_init_segment_name("$Number$.mp4");
-  ASSERT_FALSE(representation.AddLiveOnlyInfo(media_info, segment_infos,
-                                              kDefaultStartNumber));
-
-  // $Time$ as well.
-  media_info.set_init_segment_name("$Time$.mp4");
-  ASSERT_FALSE(representation.AddLiveOnlyInfo(media_info, segment_infos,
-                                              kDefaultStartNumber));
-
-  // This should be valid.
-  media_info.set_init_segment_name("some_non_template_name.mp4");
-  ASSERT_TRUE(representation.AddLiveOnlyInfo(media_info, segment_infos,
-                                             kDefaultStartNumber));
-}
-
 }  // namespace xml
 }  // namespace shaka

packager/mpd/test/data/audio_media_info1.txt

@@ -8,4 +8,5 @@ audio_info {
 reference_time_scale: 50
 container_type: 1
 media_file_name: "test_output_file_name_audio1.mp4"
+media_file_url: "test_output_file_name_audio1.mp4"
 media_duration_seconds: 10.5

packager/mpd/test/data/video_media_info1.txt

@@ -19,4 +19,5 @@ index_range {
 reference_time_scale: 1000
 container_type: 1
 media_file_name: "test_output_file_name1.mp4"
+media_file_url: "test_output_file_name1.mp4"
 media_duration_seconds: 10.5

packager/mpd/test/data/video_media_info2.txt

@@ -19,4 +19,5 @@ index_range {
 reference_time_scale: 50
 container_type: 1
 media_file_name: "test_output_file_name2.mp4"
+media_file_url: "test_output_file_name2.mp4"
 media_duration_seconds: 10.5

packager/mpd/util/mpd_writer.cc

@@ -44,8 +44,7 @@ class SimpleMpdNotifierFactory : public MpdNotifierFactory {
 MpdWriter::MpdWriter() : notifier_factory_(new SimpleMpdNotifierFactory()) {}
 MpdWriter::~MpdWriter() {}
 
-bool MpdWriter::AddFile(const std::string& media_info_path,
+bool MpdWriter::AddFile(const std::string& media_info_path) {
-                        const std::string& mpd_path) {
   std::string file_content;
   if (!File::ReadFileToString(media_info_path.c_str(), &file_content)) {
     LOG(ERROR) << "Failed to read " << media_info_path << " to string.";
@@ -59,7 +58,6 @@ bool MpdWriter::AddFile(const std::string& media_info_path,
     return false;
   }
 
-  MpdBuilder::MakePathsRelativeToMpd(mpd_path, &media_info);
   media_infos_.push_back(media_info);
   return true;
 }
@@ -82,13 +80,11 @@ bool MpdWriter::WriteMpdToFile(const char* file_name) {
     return false;
   }
 
-  for (std::list<MediaInfo>::const_iterator it = media_infos_.begin();
+  for (const MediaInfo& media_info : media_infos_) {
-       it != media_infos_.end();
-       ++it) {
     uint32_t unused_conatiner_id;
-    if (!notifier->NotifyNewContainer(*it, &unused_conatiner_id)) {
+    if (!notifier->NotifyNewContainer(media_info, &unused_conatiner_id)) {
       LOG(ERROR) << "Failed to add MediaInfo for media file: "
-                 << it->media_file_name();
+                 << media_info.media_file_name();
       return false;
     }
   }

packager/mpd/util/mpd_writer.h

@@ -55,8 +55,7 @@ class MpdWriter {
   // MediaInfo, i.e. the content should be a result of using
   // google::protobuf::TestFormat::Print*() methods.
   // If necessary, this method can be called after WriteMpd*() methods.
-  bool AddFile(const std::string& media_info_path,
+  bool AddFile(const std::string& media_info_path);
-               const std::string& mpd_path);
 
   // |base_url| will be used for <BaseURL> element for the MPD. The BaseURL
   // element will be a direct child element of the <MPD> element.

packager/mpd/util/mpd_writer_unittest.cc

@@ -88,8 +88,8 @@ TEST_F(MpdWriterTest, WriteMpdToFile) {
       GetTestDataFilePath(kFileNameVideoMediaInfo2);
 
   SetMpdNotifierFactoryForTest();
-  EXPECT_TRUE(mpd_writer_.AddFile(media_info_file1.AsUTF8Unsafe(), ""));
+  EXPECT_TRUE(mpd_writer_.AddFile(media_info_file1.AsUTF8Unsafe()));
-  EXPECT_TRUE(mpd_writer_.AddFile(media_info_file2.AsUTF8Unsafe(), ""));
+  EXPECT_TRUE(mpd_writer_.AddFile(media_info_file2.AsUTF8Unsafe()));
   mpd_writer_.AddBaseUrl(kBaseUrl1);
   mpd_writer_.AddBaseUrl(kBaseUrl2);