From cd018a71c30d2a79f0d97583b4ffbada76f71cbd Mon Sep 17 00:00:00 2001 From: Caitlin O'Callaghan <38890251+CaitlinOCallaghan@users.noreply.github.com> Date: Wed, 25 Aug 2021 08:38:05 -0700 Subject: [PATCH] Low latency DASH support (#979) # LL-DASH Support These changes add support for LL-DASH streaming. **NOTE:** LL-HLS support is still in progress, but it's coming. :) ## Testing `./chunking_unittest --gtest_filter="ChunkingHandlerTest.LowLatencyDash"` `./media_event_unittest --gtest_filter="MpdNotifyMuxerListenerTest.LowLatencyDash"` `./mpd_unittest --gtest_filter="PeriodTest.LowLatencyDashMpdGetXml"` `./mpd_unittest --gtest_filter="SimpleMpdNotifierTest.NotifyAvailabilityTimeOffset"` `./mpd_unittest --gtest_filter="SimpleMpdNotifierTest.NotifySegmentDuration"` `./mpd_unittest --gtest_filter="LowLatencySegmentTest.LowLatencySegmentTemplate"` Note, packager_test must be run from the main project directory `./out/Release/packager_test --gtest_filter="PackagerTest.LowLatencyDashEnabledAndUtcTimingNotSet"` `./out/Release/packager_test --gtest_filter="PackagerTest.LowLatencyDashEnabledAndUtcTimingNotSet"` --- docs/source/options/dash_options.rst | 5 + docs/source/tutorials/low_latency.rst | 103 +++++++++ docs/source/tutorials/tutorials.rst | 1 + packager/app/mpd_flags.cc | 8 + packager/app/mpd_flags.h | 1 + packager/app/packager_main.cc | 3 + packager/file/file.cc | 1 + packager/file/http_file.cc | 2 +- packager/media/base/media_handler.h | 2 + packager/media/chunking/chunking_handler.cc | 24 +- .../chunking/chunking_handler_unittest.cc | 43 ++++ .../media/event/combined_muxer_listener.cc | 12 + .../media/event/combined_muxer_listener.h | 2 + .../media/event/mpd_notify_muxer_listener.cc | 10 + .../media/event/mpd_notify_muxer_listener.h | 2 + .../mpd_notify_muxer_listener_unittest.cc | 71 +++++- packager/media/event/muxer_listener.h | 6 + .../mp4/low_latency_segment_segmenter.cc | 212 ++++++++++++++++++ .../mp4/low_latency_segment_segmenter.h | 72 ++++++ packager/media/formats/mp4/mp4.gyp | 2 + packager/media/formats/mp4/mp4_muxer.cc | 4 + packager/media/formats/mp4/segmenter.cc | 11 +- packager/media/formats/mp4/segmenter.h | 2 + packager/media/public/chunking_params.h | 6 + packager/media/public/mp4_output_params.h | 6 + packager/mpd/base/media_info.proto | 8 + packager/mpd/base/mock_mpd_builder.h | 2 + packager/mpd/base/mock_mpd_notifier.h | 2 + packager/mpd/base/mpd_notifier.h | 16 ++ packager/mpd/base/period.cc | 22 ++ packager/mpd/base/period_unittest.cc | 40 ++++ packager/mpd/base/representation.cc | 30 ++- packager/mpd/base/representation.h | 8 + packager/mpd/base/simple_mpd_notifier.cc | 22 ++ packager/mpd/base/simple_mpd_notifier.h | 2 + .../mpd/base/simple_mpd_notifier_unittest.cc | 50 +++++ packager/mpd/base/xml/xml_node.cc | 21 +- packager/mpd/base/xml/xml_node.h | 3 +- packager/mpd/base/xml/xml_node_unittest.cc | 71 ++++-- packager/mpd/public/mpd_params.h | 11 + packager/packager.cc | 21 ++ packager/packager_test.cc | 22 ++ 42 files changed, 937 insertions(+), 25 deletions(-) create mode 100644 docs/source/tutorials/low_latency.rst create mode 100644 packager/media/formats/mp4/low_latency_segment_segmenter.cc create mode 100644 packager/media/formats/mp4/low_latency_segment_segmenter.h diff --git a/docs/source/options/dash_options.rst b/docs/source/options/dash_options.rst index 7a07ce9ae8..e47ae3774f 100644 --- a/docs/source/options/dash_options.rst +++ b/docs/source/options/dash_options.rst @@ -95,3 +95,8 @@ DASH options If enabled, allow adaptive switching between different codecs, if they have the same language, media type (audio, video etc) and container type. + +--low_latency_dash_mode + + If enabled, LL-DASH streaming will be used, + reducing overall latency by decoupling latency from segment duration. \ No newline at end of file diff --git a/docs/source/tutorials/low_latency.rst b/docs/source/tutorials/low_latency.rst new file mode 100644 index 0000000000..2380ae12b4 --- /dev/null +++ b/docs/source/tutorials/low_latency.rst @@ -0,0 +1,103 @@ +#################################### +Low Latency DASH (LL-DASH) Streaming +#################################### + +************ +Introduction +************ + +If ``--low_latency_dash_mode`` is enabled, low latency DASH (LL-DASH) packaging will be used. + +This will reduce overall latency by ensuring that the media segments are chunk encoded and delivered via an aggregating response. +The combination of these features will ensure that overall latency can be decoupled from the segment duration. +For low latency to be achieved, the output of Shaka Packager must be combined with a delivery system which can chain together a set of aggregating responses, such as chunked transfer encoding under HTTP/1.1 or a HTTP/2 or HTTP/3 connection. +The output of Shaka Packager must be played with a DASH client that understands the availabilityTimeOffset MPD value. +Furthermore, the player should also understand the throughput estimation and ABR challenges that arise when operating in the low latency regime. + +This tutorial covers LL-DASH packaging and uses features from the DASH, HTTP upload, and FFmpeg piping tutorials. +For more information on DASH, see :doc:`dash`; for HTTP upload, see :doc:`http_upload`; +for FFmpeg piping, see :doc:`ffmpeg_piping`; +for full documentation, see :doc:`/documentation`. + +************* +Documentation +************* + +Getting started +=============== + +To enable LL-DASH mode, set the ``--low_latency_dash_mode`` flag to ``true``. + +All HTTP requests will use chunked transfer encoding: +``Transfer-Encoding: chunked``. + +.. note:: + + Only LL-DASH is supported. LL-HLS support is yet to come. + +Synopsis +======== + +Here is a basic example of the LL-DASH support. +The LL-DASH setup borrows features from "FFmpeg piping" and "HTTP upload", +see :doc:`ffmpeg_piping` and :doc:`http_upload`. + +Define UNIX pipe to connect ffmpeg with packager:: + + export PIPE=/tmp/bigbuckbunny.fifo + mkfifo ${PIPE} + +Acquire and transcode RTMP stream:: + + ffmpeg -fflags nobuffer -threads 0 -y \ + -i rtmp://184.72.239.149/vod/mp4:bigbuckbunny_450.mp4 \ + -pix_fmt yuv420p -vcodec libx264 -preset:v superfast -acodec aac \ + -f mpegts pipe: > ${PIPE} + +Configure and run packager:: + + # Define upload URL + export UPLOAD_URL=http://localhost:6767/ll-dash + + # Go + packager \ + "input=${PIPE},stream=audio,init_segment=${UPLOAD_URL}_init.m4s,segment_template=${UPLOAD_URL}/bigbuckbunny-audio-aac-\$Number%04d\$.m4s" \ + "input=${PIPE},stream=video,init_segment=${UPLOAD_URL}_init.m4s,segment_template=${UPLOAD_URL}/bigbuckbunny-video-h264-450-\$Number%04d\$.m4s" \ + --io_block_size 65536 \ + --segment_duration 2 \ + --low_latency_dash_mode=true \ + --utc_timings "urn:mpeg:dash:utc:http-xsdate:2014"="https://time.akamai.com/?iso" \ + --mpd_output "${UPLOAD_URL}/bigbuckbunny.mpd" \ + + +************************* +Low Latency Compatibility +************************* + +For low latency to be achieved, the processes handling Shaka Packager's output, such as the server and player, +must support LL-DASH streaming. + +Delivery Pipeline +================= +Shaka Packager will upload the LL-DASH content to the specified output via HTTP chunked transfer encoding. +The server must have the ability to handle this type of request. If using a proxy or shim for cloud authentication, +these services must also support HTTP chunked transfer encoding. + +Examples of supporting content delivery systems: + +* `AWS MediaStore `_ +* `s3-upload-proxy `_ +* `Streamline Low Latency DASH preview `_ +* `go-chunked-streaming-server `_ + +Player +====== +The player must support LL-DASH playout. +LL-DASH requires the player to be able to interpret ``availabilityTimeOffset`` values from the DASH MPD. +The player should also recognize the the throughput estimation and ABR challenges that arise with low latency streaming. + +Examples of supporting players: + +* `Shaka Player `_ +* `dash.js `_ +* `Streamline Low Latency DASH preview `_ \ No newline at end of file diff --git a/docs/source/tutorials/tutorials.rst b/docs/source/tutorials/tutorials.rst index a6a7234496..758c03f7a4 100644 --- a/docs/source/tutorials/tutorials.rst +++ b/docs/source/tutorials/tutorials.rst @@ -13,3 +13,4 @@ Tutorials ads.rst ffmpeg_piping.rst http_upload.rst + low_latency.rst diff --git a/packager/app/mpd_flags.cc b/packager/app/mpd_flags.cc index 5a881a0e6d..50c2b9eead 100644 --- a/packager/app/mpd_flags.cc +++ b/packager/app/mpd_flags.cc @@ -75,3 +75,11 @@ DEFINE_bool(dash_force_segment_list, "content is huge and the total number of (sub)segment references " "is greater than what the sidx atom allows (65535). Currently " "this flag is only supported in DASH ondemand profile."); +DEFINE_bool( + low_latency_dash_mode, + false, + "If enabled, LL-DASH streaming will be used, " + "reducing overall latency by decoupling latency from segment duration. " + "Please see " + "https://google.github.io/shaka-packager/html/tutorials/low_latency.html " + "for more information."); diff --git a/packager/app/mpd_flags.h b/packager/app/mpd_flags.h index ccb4192947..ec4519a2eb 100644 --- a/packager/app/mpd_flags.h +++ b/packager/app/mpd_flags.h @@ -24,5 +24,6 @@ DECLARE_bool(allow_approximate_segment_timeline); DECLARE_bool(allow_codec_switching); DECLARE_bool(include_mspr_pro_for_playready); DECLARE_bool(dash_force_segment_list); +DECLARE_bool(low_latency_dash_mode); #endif // APP_MPD_FLAGS_H_ diff --git a/packager/app/packager_main.cc b/packager/app/packager_main.cc index 179ecfd926..1035b1fe25 100644 --- a/packager/app/packager_main.cc +++ b/packager/app/packager_main.cc @@ -325,6 +325,7 @@ base::Optional GetPackagingParams() { ChunkingParams& chunking_params = packaging_params.chunking_params; chunking_params.segment_duration_in_seconds = FLAGS_segment_duration; chunking_params.subsegment_duration_in_seconds = FLAGS_fragment_duration; + chunking_params.low_latency_dash_mode = FLAGS_low_latency_dash_mode; chunking_params.segment_sap_aligned = FLAGS_segment_sap_aligned; chunking_params.subsegment_sap_aligned = FLAGS_fragment_sap_aligned; @@ -435,6 +436,7 @@ base::Optional GetPackagingParams() { mp4_params.generate_sidx_in_media_segments = FLAGS_generate_sidx_in_media_segments; mp4_params.include_pssh_in_stream = FLAGS_mp4_include_pssh_in_stream; + mp4_params.low_latency_dash_mode = FLAGS_low_latency_dash_mode; packaging_params.transport_stream_timestamp_offset_ms = FLAGS_transport_stream_timestamp_offset_ms; @@ -474,6 +476,7 @@ base::Optional GetPackagingParams() { FLAGS_allow_approximate_segment_timeline; mpd_params.allow_codec_switching = FLAGS_allow_codec_switching; mpd_params.include_mspr_pro = FLAGS_include_mspr_pro_for_playready; + mpd_params.low_latency_dash_mode = FLAGS_low_latency_dash_mode; HlsParams& hls_params = packaging_params.hls_params; if (!GetHlsPlaylistType(FLAGS_hls_playlist_type, &hls_params.playlist_type)) { diff --git a/packager/file/file.cc b/packager/file/file.cc index 653598b4eb..c10718128e 100644 --- a/packager/file/file.cc +++ b/packager/file/file.cc @@ -184,6 +184,7 @@ File* File::CreateInternalFile(const char* file_name, const char* mode) { base::StringPiece real_file_name; const FileTypeInfo* file_type = GetFileTypeInfo(file_name, &real_file_name); DCHECK(file_type); + // Calls constructor for the derived File class. return file_type->factory_function(real_file_name.data(), mode); } diff --git a/packager/file/http_file.cc b/packager/file/http_file.cc index b304746f99..c633b3345b 100644 --- a/packager/file/http_file.cc +++ b/packager/file/http_file.cc @@ -297,7 +297,7 @@ void HttpFile::SetupRequest() { curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &CurlWriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, - method_ == HttpMethod::kPut ? nullptr : &download_cache_); + method_ == HttpMethod::kGet ? &download_cache_ : nullptr); if (method_ != HttpMethod::kGet) { curl_easy_setopt(curl, CURLOPT_READFUNCTION, &CurlReadCallback); curl_easy_setopt(curl, CURLOPT_READDATA, &upload_cache_); diff --git a/packager/media/base/media_handler.h b/packager/media/base/media_handler.h index e9a8be5c3d..ad4cd710f2 100644 --- a/packager/media/base/media_handler.h +++ b/packager/media/base/media_handler.h @@ -54,6 +54,8 @@ struct CueEvent { struct SegmentInfo { bool is_subsegment = false; + bool is_chunk = false; + bool is_final_chunk_in_seg = false; bool is_encrypted = false; int64_t start_timestamp = -1; int64_t duration = 0; diff --git a/packager/media/chunking/chunking_handler.cc b/packager/media/chunking/chunking_handler.cc index c924a746e5..e345b6a378 100644 --- a/packager/media/chunking/chunking_handler.cc +++ b/packager/media/chunking/chunking_handler.cc @@ -112,7 +112,23 @@ Status ChunkingHandler::OnMediaSample( started_new_segment = true; } } - if (!started_new_segment && IsSubsegmentEnabled()) { + + // This handles the LL-DASH case. + // On each media sample, which is the basis for a chunk, + // we must increment the current_subsegment_index_ + // in order to hit FinalizeSegment() within Segmenter. + if (!started_new_segment && chunking_params_.low_latency_dash_mode) { + current_subsegment_index_++; + + RETURN_IF_ERROR(EndSubsegmentIfStarted()); + subsegment_start_time_ = timestamp; + } + + // Here, a subsegment refers to a fragment that is within a segment. + // This fragment size can be set with the 'fragment_duration' cmd arg. + // This is NOT for the LL-DASH case. + if (!started_new_segment && IsSubsegmentEnabled() && + !chunking_params_.low_latency_dash_mode) { const bool can_start_new_subsegment = sample->is_key_frame() || !chunking_params_.subsegment_sap_aligned; if (can_start_new_subsegment) { @@ -151,6 +167,10 @@ Status ChunkingHandler::EndSegmentIfStarted() const { auto segment_info = std::make_shared(); segment_info->start_timestamp = segment_start_time_.value(); segment_info->duration = max_segment_time_ - segment_start_time_.value(); + if (chunking_params_.low_latency_dash_mode) { + segment_info->is_chunk = true; + segment_info->is_final_chunk_in_seg = true; + } return DispatchSegmentInfo(kStreamIndex, std::move(segment_info)); } @@ -163,6 +183,8 @@ Status ChunkingHandler::EndSubsegmentIfStarted() const { subsegment_info->duration = max_segment_time_ - subsegment_start_time_.value(); subsegment_info->is_subsegment = true; + if (chunking_params_.low_latency_dash_mode) + subsegment_info->is_chunk = true; return DispatchSegmentInfo(kStreamIndex, std::move(subsegment_info)); } diff --git a/packager/media/chunking/chunking_handler_unittest.cc b/packager/media/chunking/chunking_handler_unittest.cc index 7c2dc71843..cffd8f246c 100644 --- a/packager/media/chunking/chunking_handler_unittest.cc +++ b/packager/media/chunking/chunking_handler_unittest.cc @@ -207,5 +207,48 @@ TEST_F(ChunkingHandlerTest, CueEvent) { kDuration, !kEncrypted, _))); } +TEST_F(ChunkingHandlerTest, LowLatencyDash) { + ChunkingParams chunking_params; + chunking_params.low_latency_dash_mode = true; + chunking_params.segment_duration_in_seconds = 1; + SetUpChunkingHandler(1, chunking_params); + + // Each completed segment will contain 2 chunks + const int64_t kChunkDurationInMs = 500; + const int64_t kSegmentDurationInMs = 1000; + + ASSERT_OK(Process(StreamData::FromStreamInfo( + kStreamIndex, GetVideoStreamInfo(kTimeScale1)))); + + for (int i = 0; i < 4; ++i) { + ASSERT_OK(Process(StreamData::FromMediaSample( + kStreamIndex, GetMediaSample(i * kChunkDurationInMs, kChunkDurationInMs, + kKeyFrame)))); + } + + // NOTE: Each MediaSample will create a chunk, dispatching SegmentInfo + EXPECT_THAT( + GetOutputStreamDataVector(), + ElementsAre( + IsStreamInfo(kStreamIndex, kTimeScale1, !kEncrypted, _), + // Chunk 1 for segment 1 + IsMediaSample(kStreamIndex, 0, kChunkDurationInMs, !kEncrypted, _), + IsSegmentInfo(kStreamIndex, 0, kChunkDurationInMs, kIsSubsegment, + !kEncrypted), + // Chunk 2 for segment 1 + IsMediaSample(kStreamIndex, kChunkDurationInMs, kChunkDurationInMs, + !kEncrypted, _), + IsSegmentInfo(kStreamIndex, 0, 2 * kChunkDurationInMs, !kIsSubsegment, + !kEncrypted), + // Chunk 1 for segment 2 + IsMediaSample(kStreamIndex, kSegmentDurationInMs, kChunkDurationInMs, + !kEncrypted, _), + IsSegmentInfo(kStreamIndex, kSegmentDurationInMs, kChunkDurationInMs, + kIsSubsegment, !kEncrypted), + // Chunk 2 for segment 2 + IsMediaSample(kStreamIndex, kSegmentDurationInMs + kChunkDurationInMs, + kChunkDurationInMs, !kEncrypted, _))); +} + } // namespace media } // namespace shaka diff --git a/packager/media/event/combined_muxer_listener.cc b/packager/media/event/combined_muxer_listener.cc index dd2bd4d2d7..7e1bc445a4 100644 --- a/packager/media/event/combined_muxer_listener.cc +++ b/packager/media/event/combined_muxer_listener.cc @@ -43,12 +43,24 @@ void CombinedMuxerListener::OnMediaStart(const MuxerOptions& muxer_options, } } +void CombinedMuxerListener::OnAvailabilityOffsetReady() { + for (auto& listener : muxer_listeners_) { + listener->OnAvailabilityOffsetReady(); + } +} + void CombinedMuxerListener::OnSampleDurationReady(int32_t sample_duration) { for (auto& listener : muxer_listeners_) { listener->OnSampleDurationReady(sample_duration); } } +void CombinedMuxerListener::OnSegmentDurationReady() { + for (auto& listener : muxer_listeners_) { + listener->OnSegmentDurationReady(); + } +} + void CombinedMuxerListener::OnMediaEnd(const MediaRanges& media_ranges, float duration_seconds) { for (auto& listener : muxer_listeners_) { diff --git a/packager/media/event/combined_muxer_listener.h b/packager/media/event/combined_muxer_listener.h index 776e4d37b6..ef66772bfd 100644 --- a/packager/media/event/combined_muxer_listener.h +++ b/packager/media/event/combined_muxer_listener.h @@ -36,7 +36,9 @@ class CombinedMuxerListener : public MuxerListener { const StreamInfo& stream_info, int32_t time_scale, ContainerType container_type) override; + void OnAvailabilityOffsetReady() override; void OnSampleDurationReady(int32_t sample_duration) override; + void OnSegmentDurationReady() override; void OnMediaEnd(const MediaRanges& media_ranges, float duration_seconds) override; void OnNewSegment(const std::string& file_name, diff --git a/packager/media/event/mpd_notify_muxer_listener.cc b/packager/media/event/mpd_notify_muxer_listener.cc index 5f7f5444ed..d16d7b1966 100644 --- a/packager/media/event/mpd_notify_muxer_listener.cc +++ b/packager/media/event/mpd_notify_muxer_listener.cc @@ -105,6 +105,11 @@ void MpdNotifyMuxerListener::OnMediaStart(const MuxerOptions& muxer_options, } } +// Record the availability time offset for LL-DASH manifests. +void MpdNotifyMuxerListener::OnAvailabilityOffsetReady() { + mpd_notifier_->NotifyAvailabilityTimeOffset(notification_id_.value()); +} + // Record the sample duration in the media info for VOD so that OnMediaEnd, all // the information is in the media info. void MpdNotifyMuxerListener::OnSampleDurationReady(int32_t sample_duration) { @@ -127,6 +132,11 @@ void MpdNotifyMuxerListener::OnSampleDurationReady(int32_t sample_duration) { media_info_->mutable_video_info()->set_frame_duration(sample_duration); } +// Record the segment duration for LL-DASH manifests. +void MpdNotifyMuxerListener::OnSegmentDurationReady() { + mpd_notifier_->NotifySegmentDuration(notification_id_.value()); +} + void MpdNotifyMuxerListener::OnMediaEnd(const MediaRanges& media_ranges, float duration_seconds) { if (mpd_notifier_->dash_profile() == DashProfile::kLive) { diff --git a/packager/media/event/mpd_notify_muxer_listener.h b/packager/media/event/mpd_notify_muxer_listener.h index a54999153b..271e7f3e25 100644 --- a/packager/media/event/mpd_notify_muxer_listener.h +++ b/packager/media/event/mpd_notify_muxer_listener.h @@ -44,7 +44,9 @@ class MpdNotifyMuxerListener : public MuxerListener { const StreamInfo& stream_info, int32_t time_scale, ContainerType container_type) override; + void OnAvailabilityOffsetReady() override; void OnSampleDurationReady(int32_t sample_duration) override; + void OnSegmentDurationReady() override; void OnMediaEnd(const MediaRanges& media_ranges, float duration_seconds) override; void OnNewSegment(const std::string& file_name, diff --git a/packager/media/event/mpd_notify_muxer_listener_unittest.cc b/packager/media/event/mpd_notify_muxer_listener_unittest.cc index 5b5e61dc00..01860be6e0 100644 --- a/packager/media/event/mpd_notify_muxer_listener_unittest.cc +++ b/packager/media/event/mpd_notify_muxer_listener_unittest.cc @@ -96,6 +96,17 @@ class MpdNotifyMuxerListenerTest : public ::testing::TestWithParam { listener_.reset(new MpdNotifyMuxerListener(notifier_.get())); } + void SetupForLowLatencyDash() { + MpdOptions mpd_options; + // Low Latency DASH streaming should be live. + mpd_options.dash_profile = DashProfile::kLive; + // Low Latency DASH live profile should be dynamic. + mpd_options.mpd_type = MpdType::kDynamic; + mpd_options.mpd_params.low_latency_dash_mode = true; + notifier_.reset(new MockMpdNotifier(mpd_options)); + listener_.reset(new MpdNotifyMuxerListener(notifier_.get())); + } + void FireOnMediaEndWithParams(const OnMediaEndParameters& params) { // On success, this writes the result to |temp_file_path_|. listener_->OnMediaEnd(params.media_ranges, params.duration_seconds); @@ -509,7 +520,6 @@ TEST_F(MpdNotifyMuxerListenerTest, VodMultipleFiles) { FireOnMediaEndWithParams(GetDefaultOnMediaEndParams()); } - TEST_F(MpdNotifyMuxerListenerTest, VodMultipleFilesSegmentList) { SetupForVodSegmentList(); MuxerOptions muxer_options1; @@ -571,6 +581,65 @@ TEST_F(MpdNotifyMuxerListenerTest, VodMultipleFilesSegmentList) { FireOnMediaEndWithParams(GetDefaultOnMediaEndParams()); } +TEST_F(MpdNotifyMuxerListenerTest, LowLatencyDash) { + SetupForLowLatencyDash(); + MuxerOptions muxer_options; + SetDefaultLiveMuxerOptions(&muxer_options); + VideoStreamInfoParameters video_params = GetDefaultVideoStreamInfoParams(); + std::shared_ptr video_stream_info = + CreateVideoStreamInfo(video_params); + + const std::string kExpectedMediaInfo = + "video_info {\n" + " codec: \"avc1.010101\"\n" + " width: 720\n" + " height: 480\n" + " time_scale: 10\n" + " pixel_width: 1\n" + " pixel_height: 1\n" + "}\n" + "media_duration_seconds: 20.0\n" + "init_segment_name: \"liveinit.mp4\"\n" + "segment_template: \"live-$NUMBER$.mp4\"\n" + "reference_time_scale: 1000\n" + "container_type: CONTAINER_MP4\n"; + + const uint64_t kStartTime1 = 0u; + const uint64_t kStartTime2 = 1001u; + const uint64_t kDuration = 1000u; + const uint64_t kSegmentSize1 = 29812u; + const uint64_t kSegmentSize2 = 30128u; + + EXPECT_CALL(*notifier_, + NotifyNewContainer(ExpectMediaInfoEq(kExpectedMediaInfo), _)) + .WillOnce(Return(true)); + EXPECT_CALL(*notifier_, NotifySampleDuration(_, kDuration)) + .WillOnce(Return(true)); + EXPECT_CALL(*notifier_, NotifyAvailabilityTimeOffset(_)) + .WillOnce(Return(true)); + EXPECT_CALL(*notifier_, NotifySegmentDuration(_)).WillOnce(Return(true)); + EXPECT_CALL(*notifier_, + NotifyNewSegment(_, kStartTime1, kDuration, kSegmentSize1)); + EXPECT_CALL(*notifier_, NotifyCueEvent(_, kStartTime2)); + EXPECT_CALL(*notifier_, + NotifyNewSegment(_, kStartTime2, kDuration, kSegmentSize2)); + EXPECT_CALL(*notifier_, Flush()).Times(2); + + listener_->OnMediaStart(muxer_options, *video_stream_info, + kDefaultReferenceTimeScale, + MuxerListener::kContainerMp4); + listener_->OnSampleDurationReady(kDuration); + listener_->OnAvailabilityOffsetReady(); + listener_->OnSegmentDurationReady(); + listener_->OnNewSegment("", kStartTime1, kDuration, kSegmentSize1); + listener_->OnCueEvent(kStartTime2, "dummy cue data"); + listener_->OnNewSegment("", kStartTime2, kDuration, kSegmentSize2); + ::testing::Mock::VerifyAndClearExpectations(notifier_.get()); + + EXPECT_CALL(*notifier_, Flush()).Times(0); + FireOnMediaEndWithParams(GetDefaultOnMediaEndParams()); +} + // Live without key rotation. Note that OnEncryptionInfoReady() is called before // OnMediaStart() but no more calls. TEST_P(MpdNotifyMuxerListenerTest, LiveNoKeyRotation) { diff --git a/packager/media/event/muxer_listener.h b/packager/media/event/muxer_listener.h index f444f81fef..1c1ee75a67 100644 --- a/packager/media/event/muxer_listener.h +++ b/packager/media/event/muxer_listener.h @@ -100,10 +100,16 @@ class MuxerListener { int32_t time_scale, ContainerType container_type) = 0; + /// Called when LL-DASH streaming starts. + virtual void OnAvailabilityOffsetReady() {} + /// Called when the average sample duration of the media is determined. /// @param sample_duration in timescale of the media. virtual void OnSampleDurationReady(int32_t sample_duration) = 0; + /// Called when LL-DASH streaming starts. + virtual void OnSegmentDurationReady() {} + /// Called when all files are written out and the muxer object does not output /// any more files. /// Note: This event might not be very interesting to MPEG DASH Live profile. diff --git a/packager/media/formats/mp4/low_latency_segment_segmenter.cc b/packager/media/formats/mp4/low_latency_segment_segmenter.cc new file mode 100644 index 0000000000..36574a633b --- /dev/null +++ b/packager/media/formats/mp4/low_latency_segment_segmenter.cc @@ -0,0 +1,212 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "packager/media/formats/mp4/low_latency_segment_segmenter.h" + +#include + +#include "packager/file/file.h" +#include "packager/file/file_closer.h" +#include "packager/media/base/buffer_writer.h" +#include "packager/media/base/media_handler.h" +#include "packager/media/base/muxer_options.h" +#include "packager/media/base/muxer_util.h" +#include "packager/media/event/muxer_listener.h" +#include "packager/media/formats/mp4/box_definitions.h" +#include "packager/media/formats/mp4/fragmenter.h" +#include "packager/media/formats/mp4/key_frame_info.h" +#include "packager/status_macros.h" + +namespace shaka { +namespace media { +namespace mp4 { + +LowLatencySegmentSegmenter::LowLatencySegmentSegmenter( + const MuxerOptions& options, + std::unique_ptr ftyp, + std::unique_ptr moov) + : Segmenter(options, std::move(ftyp), std::move(moov)), + styp_(new SegmentType), + num_segments_(0) { + // Use the same brands for styp as ftyp. + styp_->major_brand = Segmenter::ftyp()->major_brand; + styp_->compatible_brands = Segmenter::ftyp()->compatible_brands; + // Replace 'cmfc' with 'cmfs' for CMAF segments compatibility. + std::replace(styp_->compatible_brands.begin(), styp_->compatible_brands.end(), + FOURCC_cmfc, FOURCC_cmfs); +} + +LowLatencySegmentSegmenter::~LowLatencySegmentSegmenter() {} + +bool LowLatencySegmentSegmenter::GetInitRange(size_t* offset, size_t* size) { + VLOG(1) << "LowLatencySegmentSegmenter outputs init segment: " + << options().output_file_name; + return false; +} + +bool LowLatencySegmentSegmenter::GetIndexRange(size_t* offset, size_t* size) { + VLOG(1) << "LowLatencySegmentSegmenter does not have index range."; + return false; +} + +std::vector LowLatencySegmentSegmenter::GetSegmentRanges() { + VLOG(1) << "LowLatencySegmentSegmenter does not have media segment ranges."; + return std::vector(); +} + +Status LowLatencySegmentSegmenter::DoInitialize() { + return WriteInitSegment(); +} + +Status LowLatencySegmentSegmenter::DoFinalize() { + // Update init segment with media duration set. + RETURN_IF_ERROR(WriteInitSegment()); + SetComplete(); + return Status::OK; +} + +Status LowLatencySegmentSegmenter::DoFinalizeSegment() { + return FinalizeSegment(); +} + +Status LowLatencySegmentSegmenter::DoFinalizeChunk() { + if (is_initial_chunk_in_seg_) { + return WriteInitialChunk(); + } + return WriteChunk(); +} + +Status LowLatencySegmentSegmenter::WriteInitSegment() { + DCHECK(ftyp()); + DCHECK(moov()); + // Generate the output file with init segment. + std::unique_ptr file( + File::Open(options().output_file_name.c_str(), "w")); + if (!file) { + return Status(error::FILE_FAILURE, + "Cannot open file for write " + options().output_file_name); + } + std::unique_ptr buffer(new BufferWriter); + ftyp()->Write(buffer.get()); + moov()->Write(buffer.get()); + return buffer->WriteToFile(file.get()); +} + +Status LowLatencySegmentSegmenter::WriteInitialChunk() { + DCHECK(sidx()); + DCHECK(fragment_buffer()); + DCHECK(styp_); + + DCHECK(!sidx()->references.empty()); + // earliest_presentation_time is the earliest presentation time of any access + // unit in the reference stream in the first subsegment. + sidx()->earliest_presentation_time = + sidx()->references[0].earliest_presentation_time; + + if (options().segment_template.empty()) { + // Append the segment to output file if segment template is not specified. + file_name_ = options().output_file_name.c_str(); + } else { + file_name_ = GetSegmentName(options().segment_template, + sidx()->earliest_presentation_time, + num_segments_, options().bandwidth); + } + + // Create the segment file + segment_file_.reset(File::Open(file_name_.c_str(), "a")); + if (!segment_file_) { + return Status(error::FILE_FAILURE, + "Cannot open segment file: " + file_name_); + } + + std::unique_ptr buffer(new BufferWriter()); + + // Write the styp header to the beginning of the segment. + styp_->Write(buffer.get()); + + const size_t segment_header_size = buffer->Size(); + const size_t segment_size = segment_header_size + fragment_buffer()->Size(); + DCHECK_NE(segment_size, 0u); + + RETURN_IF_ERROR(buffer->WriteToFile(segment_file_.get())); + if (muxer_listener()) { + for (const KeyFrameInfo& key_frame_info : key_frame_infos()) { + muxer_listener()->OnKeyFrame( + key_frame_info.timestamp, + segment_header_size + key_frame_info.start_byte_offset, + key_frame_info.size); + } + } + + // Write the chunk data to the file + RETURN_IF_ERROR(fragment_buffer()->WriteToFile(segment_file_.get())); + + uint64_t segment_duration = GetSegmentDuration(); + UpdateProgress(segment_duration); + + if (muxer_listener()) { + if (!ll_dash_mpd_values_initialized_) { + // Set necessary values for LL-DASH mpd after the first chunk has been + // processed. + muxer_listener()->OnSampleDurationReady(sample_duration()); + muxer_listener()->OnAvailabilityOffsetReady(); + muxer_listener()->OnSegmentDurationReady(); + ll_dash_mpd_values_initialized_ = true; + } + // Add the current segment in the manifest. + // Following chunks will be appended to the open segment file. + muxer_listener()->OnNewSegment(file_name_, + sidx()->earliest_presentation_time, + segment_duration, segment_size); + is_initial_chunk_in_seg_ = false; + } + + return Status::OK; +} + +Status LowLatencySegmentSegmenter::WriteChunk() { + DCHECK(fragment_buffer()); + + // Write the chunk data to the file + RETURN_IF_ERROR(fragment_buffer()->WriteToFile(segment_file_.get())); + + UpdateProgress(GetSegmentDuration()); + + return Status::OK; +} + +Status LowLatencySegmentSegmenter::FinalizeSegment() { + // Close the file now that the final chunk has been written + if (!segment_file_.release()->Close()) { + return Status( + error::FILE_FAILURE, + "Cannot close file " + file_name_ + + ", possibly file permission issue or running out of disk space."); + } + + // Current segment is complete. Reset state in preparation for the next + // segment. + is_initial_chunk_in_seg_ = true; + num_segments_++; + + return Status::OK; +} + +uint64_t LowLatencySegmentSegmenter::GetSegmentDuration() { + DCHECK(sidx()); + + uint64_t segment_duration = 0; + // ISO/IEC 23009-1:2012: the value shall be identical to sum of the the + // values of all Subsegment_duration fields in the first ‘sidx’ box. + for (size_t i = 0; i < sidx()->references.size(); ++i) + segment_duration += sidx()->references[i].subsegment_duration; + + return segment_duration; +} + +} // namespace mp4 +} // namespace media +} // namespace shaka diff --git a/packager/media/formats/mp4/low_latency_segment_segmenter.h b/packager/media/formats/mp4/low_latency_segment_segmenter.h new file mode 100644 index 0000000000..a3dc3cd1b7 --- /dev/null +++ b/packager/media/formats/mp4/low_latency_segment_segmenter.h @@ -0,0 +1,72 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef PACKAGER_MEDIA_FORMATS_MP4_LOW_LATENCY_SEGMENT_SEGMENTER_H_ +#define PACKAGER_MEDIA_FORMATS_MP4_LOW_LATENCY_SEGMENT_SEGMENTER_H_ + +#include "packager/media/formats/mp4/segmenter.h" + +#include "packager/file/file.h" +#include "packager/file/file_closer.h" + +namespace shaka { +namespace media { +namespace mp4 { + +struct SegmentType; + +/// Segmenter for LL-DASH profiles. +/// Each segment constist of many fragments, and each fragment contains one +/// chunk. A chunk is the smallest unit and is constructed of a single moof and +/// mdat atom. A chunk is be generated for each recieved @b MediaSample. The +/// generated chunks are written as they are created to files defined by +/// @b MuxerOptions.segment_template if specified; otherwise, the chunks are +/// appended to the main output file specified by @b +/// MuxerOptions.output_file_name. +class LowLatencySegmentSegmenter : public Segmenter { + public: + LowLatencySegmentSegmenter(const MuxerOptions& options, + std::unique_ptr ftyp, + std::unique_ptr moov); + ~LowLatencySegmentSegmenter() override; + + /// @name Segmenter implementation overrides. + /// @{ + bool GetInitRange(size_t* offset, size_t* size) override; + bool GetIndexRange(size_t* offset, size_t* size) override; + std::vector GetSegmentRanges() override; + /// @} + + private: + // Segmenter implementation overrides. + Status DoInitialize() override; + Status DoFinalize() override; + Status DoFinalizeSegment() override; + Status DoFinalizeChunk() override; + + // Write segment to file. + Status WriteInitSegment(); + Status WriteChunk(); + Status WriteInitialChunk(); + Status FinalizeSegment(); + + uint64_t GetSegmentDuration(); + + std::unique_ptr styp_; + uint32_t num_segments_; + bool is_initial_chunk_in_seg_ = true; + bool ll_dash_mpd_values_initialized_ = false; + std::unique_ptr segment_file_; + std::string file_name_; + + DISALLOW_COPY_AND_ASSIGN(LowLatencySegmentSegmenter); +}; + +} // namespace mp4 +} // namespace media +} // namespace shaka + +#endif // PACKAGER_MEDIA_FORMATS_MP4_LOW_LATENCY_SEGMENT_SEGMENTER_H_ diff --git a/packager/media/formats/mp4/mp4.gyp b/packager/media/formats/mp4/mp4.gyp index 604e5979d1..41c66d0057 100644 --- a/packager/media/formats/mp4/mp4.gyp +++ b/packager/media/formats/mp4/mp4.gyp @@ -29,6 +29,8 @@ 'fragmenter.cc', 'fragmenter.h', 'key_frame_info.h', + 'low_latency_segment_segmenter.cc', + 'low_latency_segment_segmenter.h', 'mp4_media_parser.cc', 'mp4_media_parser.h', 'mp4_muxer.cc', diff --git a/packager/media/formats/mp4/mp4_muxer.cc b/packager/media/formats/mp4/mp4_muxer.cc index 7237b4472a..3c542bd93f 100644 --- a/packager/media/formats/mp4/mp4_muxer.cc +++ b/packager/media/formats/mp4/mp4_muxer.cc @@ -22,6 +22,7 @@ #include "packager/media/codecs/es_descriptor.h" #include "packager/media/event/muxer_listener.h" #include "packager/media/formats/mp4/box_definitions.h" +#include "packager/media/formats/mp4/low_latency_segment_segmenter.h" #include "packager/media/formats/mp4/multi_segment_segmenter.h" #include "packager/media/formats/mp4/single_segment_segmenter.h" #include "packager/media/formats/ttml/ttml_generator.h" @@ -298,6 +299,9 @@ Status MP4Muxer::DelayInitializeMuxer() { if (options().segment_template.empty()) { segmenter_.reset(new SingleSegmentSegmenter(options(), std::move(ftyp), std::move(moov))); + } else if (options().mp4_params.low_latency_dash_mode) { + segmenter_.reset(new LowLatencySegmentSegmenter(options(), std::move(ftyp), + std::move(moov))); } else { segmenter_.reset( new MultiSegmentSegmenter(options(), std::move(ftyp), std::move(moov))); diff --git a/packager/media/formats/mp4/segmenter.cc b/packager/media/formats/mp4/segmenter.cc index 65d81e8306..90769a57c6 100644 --- a/packager/media/formats/mp4/segmenter.cc +++ b/packager/media/formats/mp4/segmenter.cc @@ -225,7 +225,16 @@ Status Segmenter::FinalizeSegment(size_t stream_id, for (std::unique_ptr& fragmenter : fragmenters_) fragmenter->ClearFragmentFinalized(); - if (!segment_info.is_subsegment) { + + if (segment_info.is_chunk) { + // Finalize the completed chunk for the LL-DASH case. + Status status = DoFinalizeChunk(); + if (!status.ok()) + return status; + } + + if (!segment_info.is_subsegment || segment_info.is_final_chunk_in_seg) { + // Finalize the segment. Status status = DoFinalizeSegment(); // Reset segment information to initial state. sidx_->references.clear(); diff --git a/packager/media/formats/mp4/segmenter.h b/packager/media/formats/mp4/segmenter.h index 9b592299bc..cf3d7b2b13 100644 --- a/packager/media/formats/mp4/segmenter.h +++ b/packager/media/formats/mp4/segmenter.h @@ -126,6 +126,8 @@ class Segmenter { virtual Status DoFinalize() = 0; virtual Status DoFinalizeSegment() = 0; + virtual Status DoFinalizeChunk() { return Status::OK; } + uint32_t GetReferenceStreamId(); void FinalizeFragmentForKeyRotation( diff --git a/packager/media/public/chunking_params.h b/packager/media/public/chunking_params.h index c22e1b9230..ac756054c9 100644 --- a/packager/media/public/chunking_params.h +++ b/packager/media/public/chunking_params.h @@ -25,6 +25,12 @@ struct ChunkingParams { /// Setting to subsegment_sap_aligned to true but segment_sap_aligned to false /// is not allowed. bool subsegment_sap_aligned = true; + /// Enable LL-DASH streaming. + /// Each segment constists of many fragments, and each fragment contains one + /// chunk. A chunk is the smallest unit and is constructed of a single moof + /// and mdat atom. Each chunk is uploaded immediately upon creation, + /// decoupling latency from segment duration. + bool low_latency_dash_mode = false; }; } // namespace shaka diff --git a/packager/media/public/mp4_output_params.h b/packager/media/public/mp4_output_params.h index 0d286b0d6c..522d1bd805 100644 --- a/packager/media/public/mp4_output_params.h +++ b/packager/media/public/mp4_output_params.h @@ -20,6 +20,12 @@ struct Mp4OutputParams { /// Note that it is required by spec if segment_template contains $Times$ /// specifier. bool generate_sidx_in_media_segments = true; + /// Enable LL-DASH streaming. + /// Each segment constists of many fragments, and each fragment contains one + /// chunk. A chunk is the smallest unit and is constructed of a single moof + /// and mdat atom. Each chunk is uploaded immediately upon creation, + /// decoupling latency from segment duration. + bool low_latency_dash_mode = false; }; } // namespace shaka diff --git a/packager/mpd/base/media_info.proto b/packager/mpd/base/media_info.proto index d449461b2a..b626f6b32a 100644 --- a/packager/mpd/base/media_info.proto +++ b/packager/mpd/base/media_info.proto @@ -204,4 +204,12 @@ message MediaInfo { // Role value defined in "urn:mpeg:dash:role:2011" scheme or in the format: // scheme_id_uri=value (to be implemented). repeated string dash_roles = 22; + + // LOW LATENCY DASH only. Defines the availabilityTimeOffset in seconds. + // Equal to the segment time minus the chunk duration. + optional double availability_time_offset = 24; + // LOW LATENCY DASH only. Defines the segment duration + // with respect to the reference time scale. + // Equal to the target segment duration times the reference time scale. + optional uint64 segment_duration = 25; } diff --git a/packager/mpd/base/mock_mpd_builder.h b/packager/mpd/base/mock_mpd_builder.h index 5a870b04f6..e9c38340ed 100644 --- a/packager/mpd/base/mock_mpd_builder.h +++ b/packager/mpd/base/mock_mpd_builder.h @@ -77,6 +77,8 @@ class MockRepresentation : public Representation { void(const std::string& drm_uuid, const std::string& pssh)); MOCK_METHOD3(AddNewSegment, void(int64_t start_time, int64_t duration, uint64_t size)); + MOCK_METHOD0(SetSegmentDuration, void()); + MOCK_METHOD0(SetAvailabilityTimeOffset, void()); MOCK_METHOD1(SetSampleDuration, void(int32_t sample_duration)); MOCK_CONST_METHOD0(GetMediaInfo, const MediaInfo&()); }; diff --git a/packager/mpd/base/mock_mpd_notifier.h b/packager/mpd/base/mock_mpd_notifier.h index 1a97fc8858..60a4effdaa 100644 --- a/packager/mpd/base/mock_mpd_notifier.h +++ b/packager/mpd/base/mock_mpd_notifier.h @@ -31,6 +31,8 @@ class MockMpdNotifier : public MpdNotifier { int64_t start_time, int64_t duration, uint64_t size)); + MOCK_METHOD1(NotifyAvailabilityTimeOffset, bool(uint32_t container_id)); + MOCK_METHOD1(NotifySegmentDuration, bool(uint32_t container_id)); MOCK_METHOD2(NotifyCueEvent, bool(uint32_t container_id, int64_t timestamp)); MOCK_METHOD4(NotifyEncryptionUpdate, bool(uint32_t container_id, diff --git a/packager/mpd/base/mpd_notifier.h b/packager/mpd/base/mpd_notifier.h index 51efb4ae44..f6e788004e 100644 --- a/packager/mpd/base/mpd_notifier.h +++ b/packager/mpd/base/mpd_notifier.h @@ -46,6 +46,15 @@ class MpdNotifier { virtual bool NotifyNewContainer(const MediaInfo& media_info, uint32_t* container_id) = 0; + /// Record the availailityTimeOffset for Low Latency DASH streaming. + /// @param container_id Container ID obtained from calling + /// NotifyNewContainer(). + /// @return true on success, false otherwise. This may fail if the container + /// specified by @a container_id does not exist. + virtual bool NotifyAvailabilityTimeOffset(uint32_t container_id) { + return true; + } + /// Change the sample duration of container with @a container_id. /// @param container_id Container ID obtained from calling /// NotifyNewContainer(). @@ -56,6 +65,13 @@ class MpdNotifier { virtual bool NotifySampleDuration(uint32_t container_id, int32_t sample_duration) = 0; + /// Record the duration of a segment for Low Latency DASH streaming. + /// @param container_id Container ID obtained from calling + /// NotifyNewContainer(). + /// @return true on success, false otherwise. This may fail if the container + /// specified by @a container_id does not exist. + virtual bool NotifySegmentDuration(uint32_t container_id) { return true; } + /// Notifies MpdBuilder that there is a new segment ready. For live, this /// is usually a new segment, for VOD this is usually a subsegment. /// @param container_id Container ID obtained from calling diff --git a/packager/mpd/base/period.cc b/packager/mpd/base/period.cc index 73e656cd11..712cdd2e97 100644 --- a/packager/mpd/base/period.cc +++ b/packager/mpd/base/period.cc @@ -136,6 +136,28 @@ base::Optional Period::GetXml(bool output_period_duration) { // Required for 'dynamic' MPDs. if (!period.SetId(id_)) return base::nullopt; + + // Required for LL-DASH MPDs. + if (mpd_options_.mpd_params.low_latency_dash_mode) { + // Create ServiceDescription element. + xml::XmlNode service_description_node("ServiceDescription"); + if (!service_description_node.SetIntegerAttribute("id", id_)) + return base::nullopt; + + // Insert Latency into ServiceDescription element. + xml::XmlNode latency_node("Latency"); + uint64_t target_latency_ms = + mpd_options_.mpd_params.target_latency_seconds * 1000; + if (!latency_node.SetIntegerAttribute("target", target_latency_ms)) + return base::nullopt; + if (!service_description_node.AddChild(std::move(latency_node))) + return base::nullopt; + + // Insert ServiceDescription into Period element. + if (!period.AddChild(std::move(service_description_node))) + return base::nullopt; + } + // Iterate thru AdaptationSets and add them to one big Period element. for (const auto& adaptation_set : adaptation_sets_) { auto child = adaptation_set->GetXml(); diff --git a/packager/mpd/base/period_unittest.cc b/packager/mpd/base/period_unittest.cc index 151e041257..8bf1036e98 100644 --- a/packager/mpd/base/period_unittest.cc +++ b/packager/mpd/base/period_unittest.cc @@ -173,6 +173,46 @@ TEST_F(PeriodTest, DynamicMpdGetXml) { XmlNodeEqual(kExpectedXml)); } +TEST_F(PeriodTest, LowLatencyDashMpdGetXml) { + const char kVideoMediaInfo[] = + "video_info {\n" + " codec: 'avc1'\n" + " width: 1280\n" + " height: 720\n" + " time_scale: 10\n" + " frame_duration: 10\n" + " pixel_width: 1\n" + " pixel_height: 1\n" + "}\n" + "container_type: 1\n"; + mpd_options_.mpd_type = MpdType::kDynamic; + mpd_options_.mpd_params.low_latency_dash_mode = true; + mpd_options_.mpd_params.target_latency_seconds = 1; + + EXPECT_CALL(testable_period_, NewAdaptationSet(_, _, _)) + .WillOnce(Return(ByMove(std::move(default_adaptation_set_)))); + + ASSERT_EQ(default_adaptation_set_ptr_, + testable_period_.GetOrCreateAdaptationSet( + ConvertToMediaInfo(kVideoMediaInfo), + content_protection_in_adaptation_set_)); + + const char kExpectedXml[] = + "" + // LL-DASH standards require ServiceDescription and Latency elements + " " + // In LL-DASH MPD, the target latency is in ms, so the expected value is + // 1000. + " " + " " + // ContentType and Representation elements are populated after + // Representation::Init() is called. + " " + ""; + EXPECT_THAT(testable_period_.GetXml(!kOutputPeriodDuration), + XmlNodeEqual(kExpectedXml)); +} + TEST_F(PeriodTest, SetDurationAndGetXml) { const char kVideoMediaInfo[] = "video_info {\n" diff --git a/packager/mpd/base/representation.cc b/packager/mpd/base/representation.cc index a6db341271..7aa5fb39c8 100644 --- a/packager/mpd/base/representation.cc +++ b/packager/mpd/base/representation.cc @@ -208,6 +208,14 @@ void Representation::SetSampleDuration(int32_t frame_duration) { } } +void Representation::SetSegmentDuration() { + int64_t sd = mpd_options_.mpd_params.target_segment_duration * + media_info_.reference_time_scale(); + if (sd <= 0) + return; + media_info_.set_segment_duration(sd); +} + const MediaInfo& Representation::GetMediaInfo() const { return media_info_; } @@ -273,8 +281,9 @@ base::Optional Representation::GetXml() { } if (HasLiveOnlyFields(media_info_) && - !representation.AddLiveOnlyInfo(media_info_, segment_infos_, - start_number_)) { + !representation.AddLiveOnlyInfo( + media_info_, segment_infos_, start_number_, + mpd_options_.mpd_params.low_latency_dash_mode)) { LOG(ERROR) << "Failed to add Live info."; return base::nullopt; } @@ -297,6 +306,23 @@ void Representation::SetPresentationTimeOffset( media_info_.set_presentation_time_offset(pto); } +void Representation::SetAvailabilityTimeOffset() { + // Adjust the frame duration to units of seconds to match target segment + // duration. + const double frame_duration_sec = + (double)frame_duration_ / (double)media_info_.reference_time_scale(); + // availabilityTimeOffset = segment duration - chunk duration. + // Here, the frame duration is equivalent to the sample duration, + // see Representation::SetSampleDuration(uint32_t frame_duration). + // By definition, each chunk will contain only one sample; + // thus, chunk_duration = sample_duration = frame_duration. + const double ato = + mpd_options_.mpd_params.target_segment_duration - frame_duration_sec; + if (ato <= 0) + return; + media_info_.set_availability_time_offset(ato); +} + bool Representation::GetStartAndEndTimestamps( double* start_timestamp_seconds, double* end_timestamp_seconds) const { diff --git a/packager/mpd/base/representation.h b/packager/mpd/base/representation.h index 7c50e51c82..97c9e1d499 100644 --- a/packager/mpd/base/representation.h +++ b/packager/mpd/base/representation.h @@ -127,6 +127,14 @@ class Representation { /// Set @presentationTimeOffset in SegmentBase / SegmentTemplate. void SetPresentationTimeOffset(double presentation_time_offset); + /// Set @availabilityTimeOffset in SegmentTemplate. + /// This is necessary for Low Latency DASH streaming. + void SetAvailabilityTimeOffset(); + + /// Set @duration in SegmentTemplate. + /// This is necessary for Low Latency DASH streaming. + void SetSegmentDuration(); + /// Gets the start and end timestamps in seconds. /// @param start_timestamp_seconds contains the returned start timestamp in /// seconds on success. It can be nullptr, which means that start diff --git a/packager/mpd/base/simple_mpd_notifier.cc b/packager/mpd/base/simple_mpd_notifier.cc index 7fb74a3278..1dbc76f573 100644 --- a/packager/mpd/base/simple_mpd_notifier.cc +++ b/packager/mpd/base/simple_mpd_notifier.cc @@ -71,6 +71,17 @@ bool SimpleMpdNotifier::NotifyNewContainer(const MediaInfo& media_info, return true; } +bool SimpleMpdNotifier::NotifyAvailabilityTimeOffset(uint32_t container_id) { + base::AutoLock auto_lock(lock_); + auto it = representation_map_.find(container_id); + if (it == representation_map_.end()) { + LOG(ERROR) << "Unexpected container_id: " << container_id; + return false; + } + it->second->SetAvailabilityTimeOffset(); + return true; +} + bool SimpleMpdNotifier::NotifySampleDuration(uint32_t container_id, int32_t sample_duration) { base::AutoLock auto_lock(lock_); @@ -83,6 +94,17 @@ bool SimpleMpdNotifier::NotifySampleDuration(uint32_t container_id, return true; } +bool SimpleMpdNotifier::NotifySegmentDuration(uint32_t container_id) { + base::AutoLock auto_lock(lock_); + auto it = representation_map_.find(container_id); + if (it == representation_map_.end()) { + LOG(ERROR) << "Unexpected container_id: " << container_id; + return false; + } + it->second->SetSegmentDuration(); + return true; +} + bool SimpleMpdNotifier::NotifyNewSegment(uint32_t container_id, int64_t start_time, int64_t duration, diff --git a/packager/mpd/base/simple_mpd_notifier.h b/packager/mpd/base/simple_mpd_notifier.h index 79b22f6a08..783605c433 100644 --- a/packager/mpd/base/simple_mpd_notifier.h +++ b/packager/mpd/base/simple_mpd_notifier.h @@ -36,8 +36,10 @@ class SimpleMpdNotifier : public MpdNotifier { /// @{ bool Init() override; bool NotifyNewContainer(const MediaInfo& media_info, uint32_t* id) override; + bool NotifyAvailabilityTimeOffset(uint32_t container_id) override; bool NotifySampleDuration(uint32_t container_id, int32_t sample_duration) override; + bool NotifySegmentDuration(uint32_t container_id) override; bool NotifyNewSegment(uint32_t container_id, int64_t start_time, int64_t duration, diff --git a/packager/mpd/base/simple_mpd_notifier_unittest.cc b/packager/mpd/base/simple_mpd_notifier_unittest.cc index 44b12f5568..d9eca8c276 100644 --- a/packager/mpd/base/simple_mpd_notifier_unittest.cc +++ b/packager/mpd/base/simple_mpd_notifier_unittest.cc @@ -151,6 +151,56 @@ TEST_F(SimpleMpdNotifierTest, NotifySampleDuration) { notifier.NotifySampleDuration(kRepresentationId, kSampleDuration)); } +TEST_F(SimpleMpdNotifierTest, NotifySegmentDuration) { + SimpleMpdNotifier notifier(empty_mpd_option_); + + const uint32_t kRepresentationId = 9u; + std::unique_ptr mock_mpd_builder(new MockMpdBuilder()); + std::unique_ptr mock_representation( + new MockRepresentation(kRepresentationId)); + + EXPECT_CALL(*mock_mpd_builder, GetOrCreatePeriod(_)) + .WillOnce(Return(default_mock_period_.get())); + EXPECT_CALL(*default_mock_period_, GetOrCreateAdaptationSet(_, _)) + .WillOnce(Return(default_mock_adaptation_set_.get())); + EXPECT_CALL(*default_mock_adaptation_set_, AddRepresentation(_)) + .WillOnce(Return(mock_representation.get())); + + uint32_t container_id; + SetMpdBuilder(¬ifier, std::move(mock_mpd_builder)); + EXPECT_TRUE(notifier.NotifyNewContainer(valid_media_info1_, &container_id)); + EXPECT_EQ(kRepresentationId, container_id); + + mock_representation->SetSegmentDuration(); + + EXPECT_TRUE(notifier.NotifySegmentDuration(kRepresentationId)); +} + +TEST_F(SimpleMpdNotifierTest, NotifyAvailabilityTimeOffset) { + SimpleMpdNotifier notifier(empty_mpd_option_); + + const uint32_t kRepresentationId = 10u; + std::unique_ptr mock_mpd_builder(new MockMpdBuilder()); + std::unique_ptr mock_representation( + new MockRepresentation(kRepresentationId)); + + EXPECT_CALL(*mock_mpd_builder, GetOrCreatePeriod(_)) + .WillOnce(Return(default_mock_period_.get())); + EXPECT_CALL(*default_mock_period_, GetOrCreateAdaptationSet(_, _)) + .WillOnce(Return(default_mock_adaptation_set_.get())); + EXPECT_CALL(*default_mock_adaptation_set_, AddRepresentation(_)) + .WillOnce(Return(mock_representation.get())); + + uint32_t container_id; + SetMpdBuilder(¬ifier, std::move(mock_mpd_builder)); + EXPECT_TRUE(notifier.NotifyNewContainer(valid_media_info1_, &container_id)); + EXPECT_EQ(kRepresentationId, container_id); + + mock_representation->SetAvailabilityTimeOffset(); + + EXPECT_TRUE(notifier.NotifyAvailabilityTimeOffset(kRepresentationId)); +} + // This test is mainly for tsan. Using both the notifier and the MpdBuilder. // Although locks in MpdBuilder have been removed, // https://github.com/google/shaka-packager/issues/45 diff --git a/packager/mpd/base/xml/xml_node.cc b/packager/mpd/base/xml/xml_node.cc index 0fb5779902..a5630fca08 100644 --- a/packager/mpd/base/xml/xml_node.cc +++ b/packager/mpd/base/xml/xml_node.cc @@ -460,18 +460,29 @@ bool RepresentationXmlNode::AddVODOnlyInfo(const MediaInfo& media_info, bool RepresentationXmlNode::AddLiveOnlyInfo( const MediaInfo& media_info, const std::list& segment_infos, - uint32_t start_number) { + uint32_t start_number, + bool low_latency_dash_mode) { XmlNode segment_template("SegmentTemplate"); if (media_info.has_reference_time_scale()) { RCHECK(segment_template.SetIntegerAttribute( "timescale", media_info.reference_time_scale())); } + if (media_info.has_segment_duration()) { + RCHECK(segment_template.SetIntegerAttribute("duration", + media_info.segment_duration())); + } + if (media_info.has_presentation_time_offset()) { RCHECK(segment_template.SetIntegerAttribute( "presentationTimeOffset", media_info.presentation_time_offset())); } + if (media_info.has_availability_time_offset()) { + RCHECK(segment_template.SetFloatingPointAttribute( + "availabilityTimeOffset", media_info.availability_time_offset())); + } + if (media_info.has_init_segment_url()) { RCHECK(segment_template.SetStringAttribute("initialization", media_info.init_segment_url())); @@ -499,9 +510,11 @@ bool RepresentationXmlNode::AddLiveOnlyInfo( std::to_string(last_segment_number))); } } else { - XmlNode segment_timeline("SegmentTimeline"); - RCHECK(PopulateSegmentTimeline(segment_infos, &segment_timeline)); - RCHECK(segment_template.AddChild(std::move(segment_timeline))); + if (!low_latency_dash_mode) { + XmlNode segment_timeline("SegmentTimeline"); + RCHECK(PopulateSegmentTimeline(segment_infos, &segment_timeline)); + RCHECK(segment_template.AddChild(std::move(segment_timeline))); + } } } return AddChild(std::move(segment_template)); diff --git a/packager/mpd/base/xml/xml_node.h b/packager/mpd/base/xml/xml_node.h index b048d39627..6d8d4b2950 100644 --- a/packager/mpd/base/xml/xml_node.h +++ b/packager/mpd/base/xml/xml_node.h @@ -219,7 +219,8 @@ class RepresentationXmlNode : public RepresentationBaseXmlNode { /// SegmentInfos are sorted by its start time. bool AddLiveOnlyInfo(const MediaInfo& media_info, const std::list& segment_infos, - uint32_t start_number) WARN_UNUSED_RESULT; + uint32_t start_number, + bool low_latency_dash_mode) WARN_UNUSED_RESULT; private: // Add AudioChannelConfiguration element. Note that it is a required element diff --git a/packager/mpd/base/xml/xml_node_unittest.cc b/packager/mpd/base/xml/xml_node_unittest.cc index ad94bea0c3..da0216e249 100644 --- a/packager/mpd/base/xml/xml_node_unittest.cc +++ b/packager/mpd/base/xml/xml_node_unittest.cc @@ -368,13 +368,14 @@ TEST_F(LiveSegmentTimelineTest, OneSegmentInfo) { const int64_t kStartTime = 0; const int64_t kDuration = 100; const uint64_t kRepeat = 9; + const bool kIsLowLatency = false; std::list segment_infos = { {kStartTime, kDuration, kRepeat}, }; RepresentationXmlNode representation; - ASSERT_TRUE( - representation.AddLiveOnlyInfo(media_info_, segment_infos, kStartNumber)); + ASSERT_TRUE(representation.AddLiveOnlyInfo(media_info_, segment_infos, + kStartNumber, kIsLowLatency)); EXPECT_THAT( representation, @@ -389,13 +390,14 @@ TEST_F(LiveSegmentTimelineTest, OneSegmentInfoNonZeroStartTime) { const int64_t kNonZeroStartTime = 500; const int64_t kDuration = 100; const uint64_t kRepeat = 9; + const bool kIsLowLatency = false; std::list segment_infos = { {kNonZeroStartTime, kDuration, kRepeat}, }; RepresentationXmlNode representation; - ASSERT_TRUE( - representation.AddLiveOnlyInfo(media_info_, segment_infos, kStartNumber)); + ASSERT_TRUE(representation.AddLiveOnlyInfo(media_info_, segment_infos, + kStartNumber, kIsLowLatency)); EXPECT_THAT(representation, XmlNodeEqual( @@ -413,13 +415,14 @@ TEST_F(LiveSegmentTimelineTest, OneSegmentInfoMatchingStartTimeAndNumber) { const int64_t kNonZeroStartTime = 500; const int64_t kDuration = 100; const uint64_t kRepeat = 9; + const bool kIsLowLatency = false; std::list segment_infos = { {kNonZeroStartTime, kDuration, kRepeat}, }; RepresentationXmlNode representation; - ASSERT_TRUE( - representation.AddLiveOnlyInfo(media_info_, segment_infos, kStartNumber)); + ASSERT_TRUE(representation.AddLiveOnlyInfo(media_info_, segment_infos, + kStartNumber, kIsLowLatency)); EXPECT_THAT( representation, @@ -431,6 +434,7 @@ TEST_F(LiveSegmentTimelineTest, OneSegmentInfoMatchingStartTimeAndNumber) { TEST_F(LiveSegmentTimelineTest, AllSegmentsSameDurationExpectLastOne) { const uint32_t kStartNumber = 1; + const bool kIsLowLatency = false; const int64_t kStartTime1 = 0; const int64_t kDuration1 = 100; @@ -445,8 +449,8 @@ TEST_F(LiveSegmentTimelineTest, AllSegmentsSameDurationExpectLastOne) { {kStartTime2, kDuration2, kRepeat2}, }; RepresentationXmlNode representation; - ASSERT_TRUE( - representation.AddLiveOnlyInfo(media_info_, segment_infos, kStartNumber)); + ASSERT_TRUE(representation.AddLiveOnlyInfo(media_info_, segment_infos, + kStartNumber, kIsLowLatency)); EXPECT_THAT( representation, @@ -458,6 +462,7 @@ TEST_F(LiveSegmentTimelineTest, AllSegmentsSameDurationExpectLastOne) { TEST_F(LiveSegmentTimelineTest, SecondSegmentInfoNonZeroRepeat) { const uint32_t kStartNumber = 1; + const bool kIsLowLatency = false; const int64_t kStartTime1 = 0; const int64_t kDuration1 = 100; @@ -472,8 +477,8 @@ TEST_F(LiveSegmentTimelineTest, SecondSegmentInfoNonZeroRepeat) { {kStartTime2, kDuration2, kRepeat2}, }; RepresentationXmlNode representation; - ASSERT_TRUE( - representation.AddLiveOnlyInfo(media_info_, segment_infos, kStartNumber)); + ASSERT_TRUE(representation.AddLiveOnlyInfo(media_info_, segment_infos, + kStartNumber, kIsLowLatency)); EXPECT_THAT(representation, XmlNodeEqual( @@ -489,6 +494,7 @@ TEST_F(LiveSegmentTimelineTest, SecondSegmentInfoNonZeroRepeat) { TEST_F(LiveSegmentTimelineTest, TwoSegmentInfoWithGap) { const uint32_t kStartNumber = 1; + const bool kIsLowLatency = false; const int64_t kStartTime1 = 0; const int64_t kDuration1 = 100; @@ -504,8 +510,8 @@ TEST_F(LiveSegmentTimelineTest, TwoSegmentInfoWithGap) { {kStartTime2, kDuration2, kRepeat2}, }; RepresentationXmlNode representation; - ASSERT_TRUE( - representation.AddLiveOnlyInfo(media_info_, segment_infos, kStartNumber)); + ASSERT_TRUE(representation.AddLiveOnlyInfo(media_info_, segment_infos, + kStartNumber, kIsLowLatency)); EXPECT_THAT(representation, XmlNodeEqual( @@ -524,6 +530,7 @@ TEST_F(LiveSegmentTimelineTest, LastSegmentNumberSupplementalProperty) { const int64_t kStartTime = 0; const int64_t kDuration = 100; const uint64_t kRepeat = 9; + const bool kIsLowLatency = false; std::list segment_infos = { {kStartTime, kDuration, kRepeat}, @@ -531,8 +538,8 @@ TEST_F(LiveSegmentTimelineTest, LastSegmentNumberSupplementalProperty) { RepresentationXmlNode representation; FLAGS_dash_add_last_segment_number_when_needed = true; - ASSERT_TRUE( - representation.AddLiveOnlyInfo(media_info_, segment_infos, kStartNumber)); + ASSERT_TRUE(representation.AddLiveOnlyInfo(media_info_, segment_infos, + kStartNumber, kIsLowLatency)); EXPECT_THAT( representation, @@ -715,5 +722,41 @@ TEST_F(OnDemandVODSegmentTest, SegmentUrlWithMediaRanges) { "")); } +class LowLatencySegmentTest : public ::testing::Test { + protected: + void SetUp() override { + media_info_.set_init_segment_url("init.m4s"); + media_info_.set_segment_template_url("$Number$.m4s"); + media_info_.set_reference_time_scale(90000); + media_info_.set_availability_time_offset(4.9750987314); + media_info_.set_segment_duration(450000); + } + + MediaInfo media_info_; +}; + +TEST_F(LowLatencySegmentTest, LowLatencySegmentTemplate) { + const uint32_t kStartNumber = 1; + const uint64_t kDuration = 100; + const uint64_t kRepeat = 0; + const bool kIsLowLatency = true; + + std::list segment_infos = { + {kStartNumber, kDuration, kRepeat}, + }; + RepresentationXmlNode representation; + ASSERT_TRUE(representation.AddLiveOnlyInfo(media_info_, segment_infos, + kStartNumber, kIsLowLatency)); + EXPECT_THAT( + representation, + XmlNodeEqual("" + " " + "")); +} + } // namespace xml } // namespace shaka diff --git a/packager/mpd/public/mpd_params.h b/packager/mpd/public/mpd_params.h index 7f2faff290..676653a7de 100644 --- a/packager/mpd/public/mpd_params.h +++ b/packager/mpd/public/mpd_params.h @@ -91,6 +91,17 @@ struct MpdParams { /// content is huge and the total number of (sub)segment references /// is greater than what the sidx atom allows (65535). bool use_segment_list = false; + /// Enable LL-DASH streaming. + /// Each segment constists of many fragments, and each fragment contains one + /// chunk. A chunk is the smallest unit and is constructed of a single moof + /// and mdat atom. Each chunk is uploaded immediately upon creation, + /// decoupling latency from segment duration. + bool low_latency_dash_mode = false; + /// This is the target latency in seconds requested by the user. The actual + /// latency may be different to the target latency + /// and is greatly influnced by the player. + /// This parameter is required by DASH-IF Low Latency standards. + double target_latency_seconds = 1; }; } // namespace shaka diff --git a/packager/packager.cc b/packager/packager.cc index a81624c592..25a3d14f07 100644 --- a/packager/packager.cc +++ b/packager/packager.cc @@ -374,6 +374,27 @@ Status ValidateParams(const PackagingParams& packaging_params, "on-demand profile (not using segment_template or segment list)."); } + if (packaging_params.chunking_params.low_latency_dash_mode && + packaging_params.chunking_params.subsegment_duration_in_seconds) { + // Low latency streaming requires data to be shipped as chunks, + // the smallest unit of video. Right now, each chunk contains + // one frame. Therefore, in low latency mode, + // a user specified --fragment_duration is irrelevant. + // TODO(caitlinocallaghan): Add a feature for users to specify the number + // of desired frames per chunk. + return Status(error::INVALID_ARGUMENT, + "--fragment_duration cannot be set " + "if --low_latency_dash_mode is enabled."); + } + + if (packaging_params.mpd_params.low_latency_dash_mode && + packaging_params.mpd_params.utc_timings.empty()) { + // Low latency DASH MPD requires a UTC Timing value + return Status(error::INVALID_ARGUMENT, + "--utc_timings must be be set " + "if --low_latency_dash_mode is enabled."); + } + return Status::OK; } diff --git a/packager/packager_test.cc b/packager/packager_test.cc index a776f239ca..04b6668a96 100644 --- a/packager/packager_test.cc +++ b/packager/packager_test.cc @@ -39,6 +39,7 @@ const uint8_t kKey[]{ 0x3a, 0xed, 0xde, 0xc0, 0xbc, 0x42, 0x1f, 0x4d, }; const double kClearLeadInSeconds = 1.0; +const double kFragmentDurationInSeconds = 5.0; } // namespace @@ -266,6 +267,27 @@ TEST_F(PackagerTest, ReadFromBufferFailed) { ASSERT_EQ(error::FILE_FAILURE, packager.Run().error_code()); } +TEST_F(PackagerTest, LowLatencyDashEnabledAndFragmentDurationSet) { + auto packaging_params = SetupPackagingParams(); + packaging_params.chunking_params.low_latency_dash_mode = true; + packaging_params.chunking_params.subsegment_duration_in_seconds = + kFragmentDurationInSeconds; + Packager packager; + auto status = packager.Initialize(packaging_params, SetupStreamDescriptors()); + ASSERT_EQ(error::INVALID_ARGUMENT, status.error_code()); + EXPECT_THAT(status.error_message(), + HasSubstr("--fragment_duration cannot be set")); +} + +TEST_F(PackagerTest, LowLatencyDashEnabledAndUtcTimingNotSet) { + auto packaging_params = SetupPackagingParams(); + packaging_params.mpd_params.low_latency_dash_mode = true; + Packager packager; + auto status = packager.Initialize(packaging_params, SetupStreamDescriptors()); + ASSERT_EQ(error::INVALID_ARGUMENT, status.error_code()); + EXPECT_THAT(status.error_message(), + HasSubstr("--utc_timings must be be set")); +} // TODO(kqyang): Add more tests. } // namespace shaka