Support segment template identifier $Time$

Also add support for format tags.

Also change default fragment_duration to 10s, i.e. to have the same
value as --segment_duration. So by default, only one fragment per
(sub)segment is generated.

Change-Id: I21123723c3998b656037a397eb7b58b3d91721bb

app/muxer_flags.cc

@@ -9,7 +9,7 @@
 #include "app/muxer_flags.h"
 
 DEFINE_double(clear_lead,
-              10.0,
+              10.0f,
               "Clear lead in seconds if encryption is enabled.");
 
 DEFINE_bool(single_segment,
@@ -26,7 +26,7 @@ DEFINE_bool(segment_sap_aligned,
             true,
             "Force segments to begin with stream access points.");
 DEFINE_double(fragment_duration,
-              2.0f,
+              10.0f,
               "Fragment duration in seconds. Should not be larger than "
               "the segment duration. Actual fragment durations may not be "
               "exactly as requested.");

app/packager_main.cc

@@ -20,6 +20,7 @@
 #include "media/base/demuxer.h"
 #include "media/base/encryption_key_source.h"
 #include "media/base/muxer_options.h"
+#include "media/base/muxer_util.h"
 #include "media/formats/mp4/mp4_muxer.h"
 
 namespace {
@@ -105,8 +106,14 @@ bool CreateRemuxJobs(const StringVector& stream_descriptors,
     std::string file_path(descriptor[0].substr(0, hash_pos));
     std::string stream_selector(descriptor[0].substr(hash_pos + 1));
     stream_muxer_options.output_file_name = descriptor[1];
-    if (descriptor.size() == 3)
+    if (descriptor.size() == 3) {
       stream_muxer_options.segment_template = descriptor[2];
+      if (!ValidateSegmentTemplate(stream_muxer_options.segment_template)) {
+        LOG(ERROR) << "ERROR: segment template with '"
+                   << stream_muxer_options.segment_template << "' is invalid.";
+        return false;
+      }
+    }
 
     if (file_path != previous_file_path) {
       // New remux job needed. Create demux and job thread.

app/single_muxer_flags.cc

@@ -21,7 +21,5 @@ DEFINE_string(output,
               "initialization segment name.");
 DEFINE_string(segment_template,
               "",
-              "Output segment name pattern for generated segments. It "
-              "can furthermore be configured using a subset of "
-              "SegmentTemplate identifiers: $Number$, $Bandwidth$ and "
-              "$Time$.");
+              "Segment template pattern for generated segments. It should "
+              "comply with ISO/IEC 23009-1:2012 5.3.9.4.4.");

app/single_packager_main.cc

@@ -17,6 +17,7 @@
 #include "media/base/demuxer.h"
 #include "media/base/encryption_key_source.h"
 #include "media/base/muxer_options.h"
+#include "media/base/muxer_util.h"
 #include "media/event/vod_media_info_dump_muxer_listener.h"
 #include "media/file/file.h"
 #include "media/file/file_closer.h"
@@ -37,6 +38,12 @@ bool GetSingleMuxerOptions(MuxerOptions* muxer_options) {
 
   muxer_options->output_file_name = FLAGS_output;
   muxer_options->segment_template = FLAGS_segment_template;
+  if (!muxer_options->segment_template.empty() &&
+      !ValidateSegmentTemplate(muxer_options->segment_template)) {
+    LOG(ERROR) << "ERROR: segment template with '"
+               << muxer_options->segment_template << "' is invalid.";
+    return false;
+  }
 
   return true;
 }
@@ -44,6 +51,11 @@ bool GetSingleMuxerOptions(MuxerOptions* muxer_options) {
 bool RunPackager(const std::string& input) {
   Status status;
 
+  // Get muxer options from commandline flags.
+  MuxerOptions muxer_options;
+  if (!GetSingleMuxerOptions(&muxer_options))
+    return false;
+
   // Setup and initialize Demuxer.
   Demuxer demuxer(input, NULL);
   status = demuxer.Initialize();
@@ -62,10 +74,6 @@ bool RunPackager(const std::string& input) {
   }
 
   // Setup muxer.
-  MuxerOptions muxer_options;
-  if (!GetSingleMuxerOptions(&muxer_options))
-    return false;
-
   scoped_ptr<Muxer> muxer(new mp4::MP4Muxer(muxer_options));
   scoped_ptr<event::MuxerListener> muxer_listener;
   scoped_ptr<File, FileCloser> mpd_file;

media/base/media_base.gyp

@@ -86,6 +86,8 @@
         'muxer.h',
         'muxer_options.cc',
         'muxer_options.h',
+        'muxer_util.cc',
+        'muxer_util.h',
         'offset_byte_queue.cc',
         'offset_byte_queue.h',
         'producer_consumer_queue.h',
@@ -121,6 +123,7 @@
         'container_names_unittest.cc',
         'fake_prng.cc',  # For rsa_key_unittest
         'fake_prng.h',   # For rsa_key_unittest
+        'muxer_util_unittest.cc',
         'offset_byte_queue_unittest.cc',
         'producer_consumer_queue_unittest.cc',
         'rsa_key_unittest.cc',

media/base/muxer_util.cc

@@ -0,0 +1,148 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd
+
+#include "media/base/muxer_util.h"
+
+#include <string>
+#include <vector>
+
+#include "base/logging.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/strings/string_split.h"
+#include "base/strings/stringprintf.h"
+
+namespace {
+bool ValidateFormatTag(const std::string& format_tag) {
+  DCHECK(!format_tag.empty());
+  // Format tag should follow this prototype: %0[width]d.
+  if (format_tag.size() > 3 && format_tag[0] == '%' && format_tag[1] == '0' &&
+      format_tag[format_tag.size() - 1] == 'd') {
+    unsigned out;
+    if (base::StringToUint(format_tag.substr(2, format_tag.size() - 3), &out))
+      return true;
+  }
+  LOG(ERROR) << "SegmentTemplate: Format tag should follow this prototype: "
+             << "%0[width]d if exist.";
+  return false;
+}
+}  // namespace
+
+namespace media {
+
+bool ValidateSegmentTemplate(const std::string& segment_template) {
+  if (segment_template.empty())
+    return false;
+
+  std::vector<std::string> splits;
+  base::SplitString(segment_template, '$', &splits);
+
+  // ISO/IEC 23009-1:2012 5.3.9.4.4 Template-based Segment URL construction.
+  // Allowed identifiers: $$, $RepresentationID$, $Number$, $Bandwidth$, $Time$.
+  // "$" always appears in pairs, so there should be odd number of splits.
+  if (splits.size() % 2 == 0) {
+    LOG(ERROR) << "SegmentTemplate: '$' should appear in pairs.";
+    return false;
+  }
+
+  bool has_number = false;
+  bool has_time = false;
+  // Every second substring in split output should be an identifier.
+  for (size_t i = 1; i < splits.size(); i += 2) {
+    // Each identifier may be suffixed, within the enclosing ‘$’ characters,
+    // with an additional format tag aligned with the printf format tag as
+    // defined in IEEE 1003.1-2008 [10] following this prototype: %0[width]d.
+    size_t format_pos = splits[i].find('%');
+    std::string identifier = splits[i].substr(0, format_pos);
+    if (format_pos != std::string::npos) {
+      if (!ValidateFormatTag(splits[i].substr(format_pos)))
+        return false;
+    }
+
+    // TODO(kqyang): Support "RepresentationID" and "Bandwidth".
+    if (identifier == "RepresentationID" || identifier == "Bandwidth") {
+      NOTIMPLEMENTED() << "SegmentTemplate: $RepresentationID$ and $Bandwidth$ "
+                          "are not supported yet.";
+      return false;
+    } else if (identifier == "Number") {
+      has_number = true;
+    } else if (identifier == "Time") {
+      has_time = true;
+    } else if (identifier == "") {
+      if (format_pos != std::string::npos) {
+        LOG(ERROR) << "SegmentTemplate: $$ should not have any format tags.";
+        return false;
+      }
+    } else {
+      LOG(ERROR) << "SegmentTemplate: '$" << splits[i]
+                 << "$' is not a valid identifier.";
+      return false;
+    }
+  }
+  if (has_number && has_time) {
+    LOG(ERROR) << "SegmentTemplate: $Number$ and $Time$ should not co-exist.";
+    return false;
+  }
+  if (!has_number && !has_time) {
+    LOG(ERROR) << "SegmentTemplate: $Number$ or $Time$ should exist.";
+    return false;
+  }
+  // Note: The below check is skipped.
+  // Strings outside identifiers shall only contain characters that are
+  // permitted within URLs according to RFC 1738.
+  return true;
+}
+
+std::string GetSegmentName(const std::string& segment_template,
+                           uint64 segment_start_time,
+                           uint32 segment_index) {
+  DCHECK(ValidateSegmentTemplate(segment_template));
+
+  std::vector<std::string> splits;
+  base::SplitString(segment_template, '$', &splits);
+  // "$" always appears in pairs, so there should be odd number of splits.
+  DCHECK_EQ(1u, splits.size() % 2);
+
+  std::string segment_name;
+  for (size_t i = 0; i < splits.size(); ++i) {
+    // Every second substring in split output should be an identifier.
+    // Simply copy the non-identifier part.
+    if (i % 2 == 0) {
+      segment_name += splits[i];
+      continue;
+    }
+    if (splits[i].empty()) {
+      // "$$" is an escape sequence, replaced with a single "$".
+      segment_name += "$";
+      continue;
+    }
+    size_t format_pos = splits[i].find('%');
+    std::string identifier = splits[i].substr(0, format_pos);
+    DCHECK(identifier == "Number" || identifier == "Time");
+
+    std::string format_tag;
+    if (format_pos != std::string::npos) {
+      format_tag = splits[i].substr(format_pos);
+      DCHECK(ValidateFormatTag(format_tag));
+      // Replace %d formatting with %lu formatting to correctly format uint64.
+      format_tag = format_tag.substr(0, format_tag.size() - 1) + "lu";
+    } else {
+      // Default format tag "%01d", modified to format uint64 correctly.
+      format_tag = "%01lu";
+    }
+
+    if (identifier == "Number") {
+      // SegmentNumber starts from 1.
+      segment_name += base::StringPrintf(
+          format_tag.c_str(), static_cast<uint64>(segment_index + 1));
+    } else if (identifier == "Time") {
+      segment_name +=
+          base::StringPrintf(format_tag.c_str(), segment_start_time);
+    }
+  }
+  return segment_name;
+}
+
+} // namespace media

media/base/muxer_util.h

@@ -0,0 +1,37 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd
+//
+// Muxer utility functions.
+
+#ifndef MEDIA_BASE_MUXER_UTIL_H_
+#define MEDIA_BASE_MUXER_UTIL_H_
+
+#include <string>
+
+#include "base/basictypes.h"
+
+namespace media {
+
+/// Validates the segment template against segment URL construction rule
+/// specified in ISO/IEC 23009-1:2012 5.3.9.4.4.
+/// @param segment_template is the template to be validated.
+/// @return true if the segment template complies with
+//          ISO/IEC 23009-1:2012 5.3.9.4.4, false otherwise.
+bool ValidateSegmentTemplate(const std::string& segment_template);
+
+/// Build the segment name from provided input.
+/// @param segment_template is the segment template pattern, which should
+///        comply with ISO/IEC 23009-1:2012 5.3.9.4.4.
+/// @param segment_start_time specifies the segment start time.
+/// @param segment_index specifies the segment index.
+/// @return The segment name with identifier substituted.
+std::string GetSegmentName(const std::string& segment_template,
+                           uint64 segment_start_time,
+                           uint32 segment_index);
+
+}  // namespace media
+
+#endif  // MEDIA_BASE_MUXER_UTIL_H_

media/base/muxer_util_unittest.cc

@@ -0,0 +1,104 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd
+
+#include "media/base/muxer_util.h"
+
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace media {
+
+TEST(MuxerUtilTest, ValidateSegmentTemplate) {
+  EXPECT_FALSE(ValidateSegmentTemplate(""));
+
+  EXPECT_TRUE(ValidateSegmentTemplate("$Number$"));
+  EXPECT_TRUE(ValidateSegmentTemplate("$Time$"));
+  EXPECT_TRUE(ValidateSegmentTemplate("$Time$$Time$"));
+  EXPECT_TRUE(ValidateSegmentTemplate("foo$Time$goo"));
+  EXPECT_TRUE(ValidateSegmentTemplate("$Number$_$Number$"));
+
+  // Escape sequence "$$".
+  EXPECT_TRUE(ValidateSegmentTemplate("foo$Time$__$$loo"));
+  EXPECT_TRUE(ValidateSegmentTemplate("foo$Time$$$"));
+  EXPECT_TRUE(ValidateSegmentTemplate("$$$Time$$$"));
+
+  // Missing $Number$ / $Time$.
+  EXPECT_FALSE(ValidateSegmentTemplate("$$"));
+  EXPECT_FALSE(ValidateSegmentTemplate("foo$$goo"));
+
+  // $Number$, $Time$ should not co-exist.
+  EXPECT_FALSE(ValidateSegmentTemplate("$Number$$Time$"));
+  EXPECT_FALSE(ValidateSegmentTemplate("foo$Number$_$Time$loo"));
+
+  // $RepresentationID$ and $Bandwidth$ not implemented yet.
+  EXPECT_FALSE(ValidateSegmentTemplate("$RepresentationID$__$Time$"));
+  EXPECT_FALSE(ValidateSegmentTemplate("foo$Bandwidth$$Time$"));
+
+  // Unknown identifier.
+  EXPECT_FALSE(ValidateSegmentTemplate("$foo$$Time$"));
+}
+
+TEST(MuxerUtilTest, ValidateSegmentTemplateWithFormatTag) {
+  EXPECT_TRUE(ValidateSegmentTemplate("$Time%01d$"));
+  EXPECT_TRUE(ValidateSegmentTemplate("$Time%05d$"));
+  EXPECT_FALSE(ValidateSegmentTemplate("$Time%1d$"));
+  EXPECT_FALSE(ValidateSegmentTemplate("$Time%$"));
+  EXPECT_FALSE(ValidateSegmentTemplate("$Time%01$"));
+  EXPECT_FALSE(ValidateSegmentTemplate("$Time%0xd$"));
+  EXPECT_FALSE(ValidateSegmentTemplate("$Time%03xd$"));
+  // $$ should not have any format tag.
+  EXPECT_FALSE(ValidateSegmentTemplate("$%01d$$Time$"));
+  // Format specifier edge cases.
+  EXPECT_TRUE(ValidateSegmentTemplate("$Time%00d$"));
+  EXPECT_TRUE(ValidateSegmentTemplate("$Time%005d$"));
+}
+
+TEST(MuxerUtilTest, GetSegmentName) {
+  const uint64 kSegmentStartTime = 180180;
+  const uint32 kSegmentIndex = 11;
+
+  EXPECT_EQ("12", GetSegmentName("$Number$", kSegmentStartTime, kSegmentIndex));
+  EXPECT_EQ("012",
+            GetSegmentName("$Number%03d$", kSegmentStartTime, kSegmentIndex));
+  EXPECT_EQ(
+      "12$foo$00012",
+      GetSegmentName(
+          "$Number%01d$$$foo$$$Number%05d$", kSegmentStartTime, kSegmentIndex));
+
+  EXPECT_EQ("180180",
+            GetSegmentName("$Time$", kSegmentStartTime, kSegmentIndex));
+  EXPECT_EQ("foo$_$18018000180180.m4s",
+            GetSegmentName("foo$$_$$$Time%01d$$Time%08d$.m4s",
+                           kSegmentStartTime,
+                           kSegmentIndex));
+  // Format specifier edge cases.
+  EXPECT_EQ("12",
+            GetSegmentName("$Number%00d$", kSegmentStartTime, kSegmentIndex));
+  EXPECT_EQ("00012",
+            GetSegmentName("$Number%005d$", kSegmentStartTime, kSegmentIndex));
+}
+
+TEST(MuxerUtilTest, GetSegmentNameWithIndexZero) {
+  const uint64 kSegmentStartTime = 0;
+  const uint32 kSegmentIndex = 0;
+
+  EXPECT_EQ("1", GetSegmentName("$Number$", kSegmentStartTime, kSegmentIndex));
+  EXPECT_EQ("001",
+            GetSegmentName("$Number%03d$", kSegmentStartTime, kSegmentIndex));
+
+  EXPECT_EQ("0", GetSegmentName("$Time$", kSegmentStartTime, kSegmentIndex));
+  EXPECT_EQ("00000000.m4s",
+            GetSegmentName("$Time%08d$.m4s", kSegmentStartTime, kSegmentIndex));
+}
+
+TEST(MuxerUtilTest, GetSegmentNameLargeTime) {
+  const uint64 kSegmentStartTime = 1601599839840;
+  const uint32 kSegmentIndex = 8888888;
+
+  EXPECT_EQ("1601599839840",
+            GetSegmentName("$Time$", kSegmentStartTime, kSegmentIndex));
+}
+
+}  // namespace media

media/formats/mp4/multi_segment_segmenter.cc

@@ -11,6 +11,7 @@
 #include "media/base/buffer_writer.h"
 #include "media/base/media_stream.h"
 #include "media/base/muxer_options.h"
+#include "media/base/muxer_util.h"
 #include "media/event/muxer_listener.h"
 #include "media/file/file.h"
 #include "media/formats/mp4/box_definitions.h"
@@ -145,10 +146,10 @@ Status MultiSegmentSegmenter::WriteSegment() {
           "Cannot open file for append " + options().output_file_name);
     }
   } else {
-    file_name = options().segment_template;
-    ReplaceSubstringsAfterOffset(
-        &file_name, 0, "$Number$", base::UintToString(++num_segments_));
-    file = File::Open(file_name.c_str(), "w");
+    file = File::Open(GetSegmentName(options().segment_template,
+                                     sidx()->earliest_presentation_time,
+                                     num_segments_++).c_str(),
+                      "w");
     if (file == NULL) {
       return Status(error::FILE_FAILURE,
                     "Cannot open file for write " + file_name);