Enable segment auto removal for live packaging

The number of preserved segments outside live window can be
configured using flag --preserved_segments_outside_live_window,
which is default to 50, i.e. 5 minutes for 6s segment.

Note that the segment removal will be disabled if it is set to 0.

Only HLS live playlist and DASH dynamic MPD are affected by this flag.

- Also add end to end tests.

Fixes #223.

Change-Id: I8a566efebe2f1552c7d9509ab017bade5a4a1c98

docs/source/options/dash_options.rst

@@ -38,6 +38,16 @@ DASH options
     Guaranteed duration of the time shifting buffer for dynamic media
     presentations, in seconds.
 
+--preserved_segments_outside_live_window <num_segments>
+
+    Segments outside the live window (defined by `time_shift_buffer_depth`
+    above) are automatically removed except for the most recent X segments
+    defined by this parameter. This is needed to accommodate latencies in
+    various stages of content serving pipeline, so that the segments stay
+    accessible as they may still be accessed by the player.
+
+    The segments are not removed if the value is zero.
+
 --utc_timing {scheme_id_uri}={value}[,{scheme_id_uri}={value}]...
 
     Comma separated UTCTiming schemeIdUri and value pairs for the MPD.

docs/source/options/hls_options.rst

@@ -27,6 +27,16 @@ HLS options
     Guaranteed duration of the time shifting buffer for LIVE playlists, in
     seconds.
 
+--preserved_segments_outside_live_window <num_segments>
+
+    Segments outside the live window (defined by `time_shift_buffer_depth`
+    above) are automatically removed except for the most recent X segments
+    defined by this parameter. This is needed to accommodate latencies in
+    various stages of content serving pipeline, so that the segments stay
+    accessible as they may still be accessed by the player.
+
+    The segments are not removed if the value is zero.
+
 --default_language <language>
 
     The first audio/text rendition in a group tagged with this language will

docs/source/tutorials/live.rst

@@ -37,8 +37,9 @@ Here are some examples.
 
 .. note::
 
-    Packager does not support removing old segments internally. The user is
+    Packager supports removing old segments automatically. See
-    resposible for setting up a cron job to do so.
+    `preserved_segments_outside_live_window` option in
+    :doc:`/options/dash_options` or :doc:`/options/hls_options` for details.
 
 .. include:: /options/udp_file_options.rst
 

packager/app/manifest_flags.cc

@@ -10,6 +10,15 @@ DEFINE_double(time_shift_buffer_depth,
               1800.0,
               "Guaranteed duration of the time shifting buffer for HLS LIVE "
               "playlists and DASH dynamic media presentations, in seconds.");
+DEFINE_uint64(
+    preserved_segments_outside_live_window,
+    50,
+    "Segments outside the live window (defined by '--time_shift_buffer_depth') "
+    "are automatically removed except for the most recent X segments defined "
+    "by this parameter. This is needed to accommodate latencies in various "
+    "stages of content serving pipeline, so that the segments stay accessible "
+    "as they may still be accessed by the player."
+    "The segments are not removed if the value is zero.");
 DEFINE_string(default_language,
               "",
               "For DASH, any audio/text tracks tagged with this language will "

packager/app/manifest_flags.h

@@ -12,6 +12,7 @@
 #include <gflags/gflags.h>
 
 DECLARE_double(time_shift_buffer_depth);
+DECLARE_uint64(preserved_segments_outside_live_window);
 DECLARE_string(default_language);
 
 #endif  // PACKAGER_APP_MANIFEST_FLAGS_H_

packager/app/packager_main.cc

@@ -403,6 +403,8 @@ base::Optional<PackagingParams> GetPackagingParams() {
   mpd_params.minimum_update_period = FLAGS_minimum_update_period;
   mpd_params.suggested_presentation_delay = FLAGS_suggested_presentation_delay;
   mpd_params.time_shift_buffer_depth = FLAGS_time_shift_buffer_depth;
+  mpd_params.preserved_segments_outside_live_window =
+      FLAGS_preserved_segments_outside_live_window;
 
   if (!FLAGS_utc_timings.empty()) {
     base::StringPairs pairs;
@@ -431,6 +433,8 @@ base::Optional<PackagingParams> GetPackagingParams() {
   hls_params.base_url = FLAGS_hls_base_url;
   hls_params.key_uri = FLAGS_hls_key_uri;
   hls_params.time_shift_buffer_depth = FLAGS_time_shift_buffer_depth;
+  hls_params.preserved_segments_outside_live_window =
+      FLAGS_preserved_segments_outside_live_window;
   hls_params.default_language = FLAGS_default_language;
 
   TestParams& test_params = packaging_params.test_params;

packager/app/test/packager_test.py

@@ -279,10 +279,12 @@ class PackagerAppTest(unittest.TestCase):
                 output_hls=False,
                 hls_playlist_type=None,
                 time_shift_buffer_depth=0.0,
+                preserved_segments_outside_live_window=0,
                 utc_timings=None,
                 generate_static_mpd=False,
                 ad_cues=None,
                 default_language=None,
+                segment_duration=1.0,
                 use_fake_clock=True):
     flags = []
 
@@ -342,13 +344,17 @@ class PackagerAppTest(unittest.TestCase):
       flags += ['--hls_master_playlist_output', self.hls_master_playlist_output]
       if hls_playlist_type:
         flags += ['--hls_playlist_type', hls_playlist_type]
-      if time_shift_buffer_depth != 0.0:
-        flags += [
-            '--time_shift_buffer_depth={0}'.format(time_shift_buffer_depth)
-        ]
     else:
       flags += ['--mpd_output', self.mpd_output]
 
+    if time_shift_buffer_depth != 0.0:
+      flags += ['--time_shift_buffer_depth={0}'.format(time_shift_buffer_depth)]
+    if preserved_segments_outside_live_window != 0:
+      flags += [
+          '--preserved_segments_outside_live_window={0}'.format(
+              preserved_segments_outside_live_window)
+      ]
+
     if utc_timings:
       flags += ['--utc_timings', utc_timings]
 
@@ -361,7 +367,8 @@ class PackagerAppTest(unittest.TestCase):
     if default_language:
       flags += ['--default_language', default_language]
 
-    flags.append('--segment_duration=1')
+    flags.append('--segment_duration={0}'.format(segment_duration))
+
     # Use fake clock, so output can be compared.
     if use_fake_clock:
       flags.append('--use_fake_clock_for_muxer')
@@ -740,6 +747,36 @@ class PackagerFunctionalTest(PackagerAppTest):
             time_shift_buffer_depth=0.5))
     self._CheckTestResults('avc-ts-event-playlist')
 
+  def testPackageAvcTsLivePlaylistWithSegmentDeletion(self):
+    self.assertPackageSuccess(
+        self._GetStreams(
+            ['audio'],
+            output_format='mp4',
+            segmented=True,
+            hls=True,
+            test_files=['bear-640x360.ts']),
+        self._GetFlags(
+            output_hls=True,
+            hls_playlist_type='LIVE',
+            segment_duration=0.5,
+            time_shift_buffer_depth=0.5,
+            preserved_segments_outside_live_window=1))
+    self._CheckTestResults('avc-ts-live-playlist-with-segment-deletion')
+
+  def testPackageAvcTsDashDynamicWithSegmentDeletion(self):
+    self.assertPackageSuccess(
+        self._GetStreams(
+            ['audio'],
+            output_format='mp4',
+            segmented=True,
+            hls=True,
+            test_files=['bear-640x360.ts']),
+        self._GetFlags(
+            segment_duration=0.5,
+            time_shift_buffer_depth=0.5,
+            preserved_segments_outside_live_window=1))
+    self._CheckTestResults('avc-ts-dash-dynamic-with-segment-deletion')
+
   def testPackageVp8Webm(self):
     self.assertPackageSuccess(
         self._GetStreams(['video'],

packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/output.mpd

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--Generated with https://github.com/google/shaka-packager version <tag>-<hash>-<test>-->
+<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xlink="http://www.w3.org/1999/xlink" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" xmlns:cenc="urn:mpeg:cenc:2013" profiles="urn:mpeg:dash:profile:isoff-live:2011" minBufferTime="PT2S" type="dynamic" publishTime="some_time" availabilityStartTime="some_time" minimumUpdatePeriod="PT5S" timeShiftBufferDepth="PT.5S">
+  <Period id="0" start="PT0S">
+    <AdaptationSet id="0" contentType="audio" segmentAlignment="true">
+      <Representation id="0" bandwidth="133850" codecs="mp4a.40.2" mimeType="audio/mp4" audioSamplingRate="44100">
+        <AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="2"/>
+        <SegmentTemplate timescale="90000" initialization="bear-640x360-audio-init.mp4" media="bear-640x360-audio-$Number$.m4s" startNumber="4">
+          <SegmentTimeline>
+            <S t="135573" d="45958"/>
+            <S t="181549" d="43869"/>
+            <S t="225435" d="27157"/>
+          </SegmentTimeline>
+        </SegmentTemplate>
+      </Representation>
+    </AdaptationSet>
+  </Period>
+</MPD>

packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio.m3u8

@@ -0,0 +1,12 @@
+#EXTM3U
+#EXT-X-VERSION:6
+## Generated with https://github.com/google/shaka-packager version <tag>-<hash>-<test>
+#EXT-X-TARGETDURATION:1
+#EXT-X-MEDIA-SEQUENCE:3
+#EXT-X-MAP:URI="bear-640x360-audio-init.mp4"
+#EXTINF:0.511,
+bear-640x360-audio-4.m4s
+#EXTINF:0.487,
+bear-640x360-audio-5.m4s
+#EXTINF:0.302,
+bear-640x360-audio-6.m4s

packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/output.m3u8

@@ -0,0 +1,4 @@
+#EXTM3U
+## Generated with https://github.com/google/shaka-packager version <tag>-<hash>-<test>
+
+#EXT-X-MEDIA:TYPE=AUDIO,URI="bear-640x360-audio.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",AUTOSELECT=YES,CHANNELS="2"

packager/hls/public/hls_params.h

@@ -31,12 +31,12 @@ struct HlsParams {
   /// Defines the live window, or the guaranteed duration of the time shifting
   /// buffer for 'live' playlists.
   double time_shift_buffer_depth = 0;
-  /// Segments outside live window (defined by 'time_shift_buffer_depth' above)
+  /// Segments outside the live window (defined by 'time_shift_buffer_depth'
-  /// are automatically removed except the latest number of segments defined by
+  /// above) are automatically removed except for the most recent X segments
-  /// this parameter. This is needed to accommodate latencies in various stages
+  /// defined by this parameter. This is needed to accommodate latencies in
-  /// of content serving pipeline, so that the segments stay accessible as they
+  /// various stages of content serving pipeline, so that the segments stay
-  /// may still be accessed by the player.
+  /// accessible as they may still be accessed by the player. The segments are
-  /// The segments are not removed if the value is zero.
+  /// not removed if the value is zero.
   size_t preserved_segments_outside_live_window = 0;
   /// Defines the key uri for "identity" and "com.apple.streamingkeydelivery"
   /// key formats. Ignored if the playlist is not encrypted or not using the

packager/mpd/public/mpd_params.h

@@ -37,12 +37,12 @@ struct MpdParams {
   /// Set MPD@timeShiftBufferDepth attribute, which is the guaranteed duration
   /// of the time shifting buffer for 'dynamic' media presentations, in seconds.
   double time_shift_buffer_depth = 0;
-  /// Segments outside live window (defined by 'time_shift_buffer_depth' above)
+  /// Segments outside the live window (defined by 'time_shift_buffer_depth'
-  /// are automatically removed except the latest number of segments defined by
+  /// above) are automatically removed except for the most recent X segments
-  /// this parameter. This is needed to accommodate latencies in various stages
+  /// defined by this parameter. This is needed to accommodate latencies in
-  /// of content serving pipeline, so that the segments stay accessible as they
+  /// various stages of content serving pipeline, so that the segments stay
-  /// may still be accessed by the player.
+  /// accessible as they may still be accessed by the player. The segments are
-  /// The segments are not removed if the value is zero.
+  /// not removed if the value is zero.
   size_t preserved_segments_outside_live_window = 0;
   /// UTCTimings. For dynamic MPD only.
   struct UtcTiming {