diff --git a/docs/source/options/dash_options.rst b/docs/source/options/dash_options.rst index ee48eaad43..16cfa3f083 100644 --- a/docs/source/options/dash_options.rst +++ b/docs/source/options/dash_options.rst @@ -38,6 +38,16 @@ DASH options Guaranteed duration of the time shifting buffer for dynamic media presentations, in seconds. +--preserved_segments_outside_live_window + + Segments outside the live window (defined by `time_shift_buffer_depth` + above) are automatically removed except for the most recent X segments + defined by this parameter. This is needed to accommodate latencies in + various stages of content serving pipeline, so that the segments stay + accessible as they may still be accessed by the player. + + The segments are not removed if the value is zero. + --utc_timing {scheme_id_uri}={value}[,{scheme_id_uri}={value}]... Comma separated UTCTiming schemeIdUri and value pairs for the MPD. diff --git a/docs/source/options/hls_options.rst b/docs/source/options/hls_options.rst index 87a8ee6950..39293558b4 100644 --- a/docs/source/options/hls_options.rst +++ b/docs/source/options/hls_options.rst @@ -27,6 +27,16 @@ HLS options Guaranteed duration of the time shifting buffer for LIVE playlists, in seconds. +--preserved_segments_outside_live_window + + Segments outside the live window (defined by `time_shift_buffer_depth` + above) are automatically removed except for the most recent X segments + defined by this parameter. This is needed to accommodate latencies in + various stages of content serving pipeline, so that the segments stay + accessible as they may still be accessed by the player. + + The segments are not removed if the value is zero. + --default_language The first audio/text rendition in a group tagged with this language will diff --git a/docs/source/tutorials/live.rst b/docs/source/tutorials/live.rst index 98312197e4..f14f4ff100 100644 --- a/docs/source/tutorials/live.rst +++ b/docs/source/tutorials/live.rst @@ -37,8 +37,9 @@ Here are some examples. .. note:: - Packager does not support removing old segments internally. The user is - resposible for setting up a cron job to do so. + Packager supports removing old segments automatically. See + `preserved_segments_outside_live_window` option in + :doc:`/options/dash_options` or :doc:`/options/hls_options` for details. .. include:: /options/udp_file_options.rst diff --git a/packager/app/manifest_flags.cc b/packager/app/manifest_flags.cc index 393a9a45a6..4f7d8e582f 100644 --- a/packager/app/manifest_flags.cc +++ b/packager/app/manifest_flags.cc @@ -10,6 +10,15 @@ DEFINE_double(time_shift_buffer_depth, 1800.0, "Guaranteed duration of the time shifting buffer for HLS LIVE " "playlists and DASH dynamic media presentations, in seconds."); +DEFINE_uint64( + preserved_segments_outside_live_window, + 50, + "Segments outside the live window (defined by '--time_shift_buffer_depth') " + "are automatically removed except for the most recent X segments defined " + "by this parameter. This is needed to accommodate latencies in various " + "stages of content serving pipeline, so that the segments stay accessible " + "as they may still be accessed by the player." + "The segments are not removed if the value is zero."); DEFINE_string(default_language, "", "For DASH, any audio/text tracks tagged with this language will " diff --git a/packager/app/manifest_flags.h b/packager/app/manifest_flags.h index 1b9f50a570..7acecf3da9 100644 --- a/packager/app/manifest_flags.h +++ b/packager/app/manifest_flags.h @@ -12,6 +12,7 @@ #include DECLARE_double(time_shift_buffer_depth); +DECLARE_uint64(preserved_segments_outside_live_window); DECLARE_string(default_language); #endif // PACKAGER_APP_MANIFEST_FLAGS_H_ diff --git a/packager/app/packager_main.cc b/packager/app/packager_main.cc index ce762dfa51..8fb1d62600 100644 --- a/packager/app/packager_main.cc +++ b/packager/app/packager_main.cc @@ -403,6 +403,8 @@ base::Optional GetPackagingParams() { mpd_params.minimum_update_period = FLAGS_minimum_update_period; mpd_params.suggested_presentation_delay = FLAGS_suggested_presentation_delay; mpd_params.time_shift_buffer_depth = FLAGS_time_shift_buffer_depth; + mpd_params.preserved_segments_outside_live_window = + FLAGS_preserved_segments_outside_live_window; if (!FLAGS_utc_timings.empty()) { base::StringPairs pairs; @@ -431,6 +433,8 @@ base::Optional GetPackagingParams() { hls_params.base_url = FLAGS_hls_base_url; hls_params.key_uri = FLAGS_hls_key_uri; hls_params.time_shift_buffer_depth = FLAGS_time_shift_buffer_depth; + hls_params.preserved_segments_outside_live_window = + FLAGS_preserved_segments_outside_live_window; hls_params.default_language = FLAGS_default_language; TestParams& test_params = packaging_params.test_params; diff --git a/packager/app/test/packager_test.py b/packager/app/test/packager_test.py index 7c8aa8cc85..de02ee0539 100755 --- a/packager/app/test/packager_test.py +++ b/packager/app/test/packager_test.py @@ -279,10 +279,12 @@ class PackagerAppTest(unittest.TestCase): output_hls=False, hls_playlist_type=None, time_shift_buffer_depth=0.0, + preserved_segments_outside_live_window=0, utc_timings=None, generate_static_mpd=False, ad_cues=None, default_language=None, + segment_duration=1.0, use_fake_clock=True): flags = [] @@ -342,13 +344,17 @@ class PackagerAppTest(unittest.TestCase): flags += ['--hls_master_playlist_output', self.hls_master_playlist_output] if hls_playlist_type: flags += ['--hls_playlist_type', hls_playlist_type] - if time_shift_buffer_depth != 0.0: - flags += [ - '--time_shift_buffer_depth={0}'.format(time_shift_buffer_depth) - ] else: flags += ['--mpd_output', self.mpd_output] + if time_shift_buffer_depth != 0.0: + flags += ['--time_shift_buffer_depth={0}'.format(time_shift_buffer_depth)] + if preserved_segments_outside_live_window != 0: + flags += [ + '--preserved_segments_outside_live_window={0}'.format( + preserved_segments_outside_live_window) + ] + if utc_timings: flags += ['--utc_timings', utc_timings] @@ -361,7 +367,8 @@ class PackagerAppTest(unittest.TestCase): if default_language: flags += ['--default_language', default_language] - flags.append('--segment_duration=1') + flags.append('--segment_duration={0}'.format(segment_duration)) + # Use fake clock, so output can be compared. if use_fake_clock: flags.append('--use_fake_clock_for_muxer') @@ -740,6 +747,36 @@ class PackagerFunctionalTest(PackagerAppTest): time_shift_buffer_depth=0.5)) self._CheckTestResults('avc-ts-event-playlist') + def testPackageAvcTsLivePlaylistWithSegmentDeletion(self): + self.assertPackageSuccess( + self._GetStreams( + ['audio'], + output_format='mp4', + segmented=True, + hls=True, + test_files=['bear-640x360.ts']), + self._GetFlags( + output_hls=True, + hls_playlist_type='LIVE', + segment_duration=0.5, + time_shift_buffer_depth=0.5, + preserved_segments_outside_live_window=1)) + self._CheckTestResults('avc-ts-live-playlist-with-segment-deletion') + + def testPackageAvcTsDashDynamicWithSegmentDeletion(self): + self.assertPackageSuccess( + self._GetStreams( + ['audio'], + output_format='mp4', + segmented=True, + hls=True, + test_files=['bear-640x360.ts']), + self._GetFlags( + segment_duration=0.5, + time_shift_buffer_depth=0.5, + preserved_segments_outside_live_window=1)) + self._CheckTestResults('avc-ts-dash-dynamic-with-segment-deletion') + def testPackageVp8Webm(self): self.assertPackageSuccess( self._GetStreams(['video'], diff --git a/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/bear-640x360-audio-3.m4s b/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/bear-640x360-audio-3.m4s new file mode 100644 index 0000000000..8dcbed3059 Binary files /dev/null and b/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/bear-640x360-audio-3.m4s differ diff --git a/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/bear-640x360-audio-4.m4s b/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/bear-640x360-audio-4.m4s new file mode 100644 index 0000000000..e19520d815 Binary files /dev/null and b/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/bear-640x360-audio-4.m4s differ diff --git a/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/bear-640x360-audio-5.m4s b/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/bear-640x360-audio-5.m4s new file mode 100644 index 0000000000..0f3473d369 Binary files /dev/null and b/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/bear-640x360-audio-5.m4s differ diff --git a/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/bear-640x360-audio-6.m4s b/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/bear-640x360-audio-6.m4s new file mode 100644 index 0000000000..b187049c54 Binary files /dev/null and b/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/bear-640x360-audio-6.m4s differ diff --git a/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/bear-640x360-audio-init.mp4 b/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/bear-640x360-audio-init.mp4 new file mode 100644 index 0000000000..f6ff48bc01 Binary files /dev/null and b/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/bear-640x360-audio-init.mp4 differ diff --git a/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/output.mpd b/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/output.mpd new file mode 100644 index 0000000000..2d335c1f64 --- /dev/null +++ b/packager/app/test/testdata/avc-ts-dash-dynamic-with-segment-deletion/output.mpd @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio-3.m4s b/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio-3.m4s new file mode 100644 index 0000000000..8dcbed3059 Binary files /dev/null and b/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio-3.m4s differ diff --git a/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio-4.m4s b/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio-4.m4s new file mode 100644 index 0000000000..e19520d815 Binary files /dev/null and b/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio-4.m4s differ diff --git a/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio-5.m4s b/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio-5.m4s new file mode 100644 index 0000000000..0f3473d369 Binary files /dev/null and b/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio-5.m4s differ diff --git a/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio-6.m4s b/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio-6.m4s new file mode 100644 index 0000000000..b187049c54 Binary files /dev/null and b/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio-6.m4s differ diff --git a/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio-init.mp4 b/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio-init.mp4 new file mode 100644 index 0000000000..f6ff48bc01 Binary files /dev/null and b/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio-init.mp4 differ diff --git a/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio.m3u8 b/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio.m3u8 new file mode 100644 index 0000000000..eb4795a7d7 --- /dev/null +++ b/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/bear-640x360-audio.m3u8 @@ -0,0 +1,12 @@ +#EXTM3U +#EXT-X-VERSION:6 +## Generated with https://github.com/google/shaka-packager version -- +#EXT-X-TARGETDURATION:1 +#EXT-X-MEDIA-SEQUENCE:3 +#EXT-X-MAP:URI="bear-640x360-audio-init.mp4" +#EXTINF:0.511, +bear-640x360-audio-4.m4s +#EXTINF:0.487, +bear-640x360-audio-5.m4s +#EXTINF:0.302, +bear-640x360-audio-6.m4s diff --git a/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/output.m3u8 b/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/output.m3u8 new file mode 100644 index 0000000000..070810b3cc --- /dev/null +++ b/packager/app/test/testdata/avc-ts-live-playlist-with-segment-deletion/output.m3u8 @@ -0,0 +1,4 @@ +#EXTM3U +## Generated with https://github.com/google/shaka-packager version -- + +#EXT-X-MEDIA:TYPE=AUDIO,URI="bear-640x360-audio.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",AUTOSELECT=YES,CHANNELS="2" diff --git a/packager/hls/public/hls_params.h b/packager/hls/public/hls_params.h index 6d2cc489e6..5811fa7bfa 100644 --- a/packager/hls/public/hls_params.h +++ b/packager/hls/public/hls_params.h @@ -31,12 +31,12 @@ struct HlsParams { /// Defines the live window, or the guaranteed duration of the time shifting /// buffer for 'live' playlists. double time_shift_buffer_depth = 0; - /// Segments outside live window (defined by 'time_shift_buffer_depth' above) - /// are automatically removed except the latest number of segments defined by - /// this parameter. This is needed to accommodate latencies in various stages - /// of content serving pipeline, so that the segments stay accessible as they - /// may still be accessed by the player. - /// The segments are not removed if the value is zero. + /// Segments outside the live window (defined by 'time_shift_buffer_depth' + /// above) are automatically removed except for the most recent X segments + /// defined by this parameter. This is needed to accommodate latencies in + /// various stages of content serving pipeline, so that the segments stay + /// accessible as they may still be accessed by the player. The segments are + /// not removed if the value is zero. size_t preserved_segments_outside_live_window = 0; /// Defines the key uri for "identity" and "com.apple.streamingkeydelivery" /// key formats. Ignored if the playlist is not encrypted or not using the diff --git a/packager/mpd/public/mpd_params.h b/packager/mpd/public/mpd_params.h index b92d5d447d..0decd3a88c 100644 --- a/packager/mpd/public/mpd_params.h +++ b/packager/mpd/public/mpd_params.h @@ -37,12 +37,12 @@ struct MpdParams { /// Set MPD@timeShiftBufferDepth attribute, which is the guaranteed duration /// of the time shifting buffer for 'dynamic' media presentations, in seconds. double time_shift_buffer_depth = 0; - /// Segments outside live window (defined by 'time_shift_buffer_depth' above) - /// are automatically removed except the latest number of segments defined by - /// this parameter. This is needed to accommodate latencies in various stages - /// of content serving pipeline, so that the segments stay accessible as they - /// may still be accessed by the player. - /// The segments are not removed if the value is zero. + /// Segments outside the live window (defined by 'time_shift_buffer_depth' + /// above) are automatically removed except for the most recent X segments + /// defined by this parameter. This is needed to accommodate latencies in + /// various stages of content serving pipeline, so that the segments stay + /// accessible as they may still be accessed by the player. The segments are + /// not removed if the value is zero. size_t preserved_segments_outside_live_window = 0; /// UTCTimings. For dynamic MPD only. struct UtcTiming {