From e5fe2a76d2c675c577a82a5556e05e90c2ae2a66 Mon Sep 17 00:00:00 2001 From: Aaron Vaage Date: Thu, 29 Mar 2018 11:00:14 -0700 Subject: [PATCH] Created Text Padder Handler To Fill Gaps Created a media handler to come after parsers that will handle filling in gaps between text samples. The padder takes a min duration, and if the samples do not cover the min duration when flushed, one last empty sample will be injected so that the samples will go up to the min duration. Change-Id: I88605059664d09279676edac418ff3d4990d7556 --- .../bear-subtitle-english-text-1.vtt | 3 + packager/media/formats/webvtt/text_padder.cc | 61 +++++++++++++++++++ packager/media/formats/webvtt/text_padder.h | 42 +++++++++++++ packager/media/formats/webvtt/webvtt.gyp | 2 + packager/packager.cc | 30 ++++----- 5 files changed, 124 insertions(+), 14 deletions(-) create mode 100644 packager/media/formats/webvtt/text_padder.cc create mode 100644 packager/media/formats/webvtt/text_padder.h diff --git a/packager/app/test/testdata/hls-segmented-webvtt/bear-subtitle-english-text-1.vtt b/packager/app/test/testdata/hls-segmented-webvtt/bear-subtitle-english-text-1.vtt index 81150bae25..cf4aec6ee0 100644 --- a/packager/app/test/testdata/hls-segmented-webvtt/bear-subtitle-english-text-1.vtt +++ b/packager/app/test/testdata/hls-segmented-webvtt/bear-subtitle-english-text-1.vtt @@ -3,3 +3,6 @@ WEBVTT 00:00:00.000 --> 00:00:00.800 Yup, that's a bear, eh. +00:00:00.800 --> 00:00:01.000 + + diff --git a/packager/media/formats/webvtt/text_padder.cc b/packager/media/formats/webvtt/text_padder.cc new file mode 100644 index 0000000000..56219d6038 --- /dev/null +++ b/packager/media/formats/webvtt/text_padder.cc @@ -0,0 +1,61 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "packager/media/formats/webvtt/text_padder.h" + +#include + +#include "packager/status_macros.h" + +namespace shaka { +namespace media { +namespace { +const uint64_t kStreamIndex = 0; +} // namespace + +TextPadder::TextPadder(int64_t duration_ms) : duration_ms_(duration_ms) {} + +Status TextPadder::InitializeInternal() { + return Status::OK; +} + +Status TextPadder::Process(std::unique_ptr data) { + DCHECK_EQ(data->stream_index, kStreamIndex); + const bool is_text_sample = + data->stream_data_type == StreamDataType::kTextSample; + return is_text_sample ? OnTextSample(std::move(data)) + : Dispatch(std::move(data)); +} + +Status TextPadder::OnFlushRequest(size_t index) { + if (duration_ms_ > max_end_time_ms_) { + std::shared_ptr filler = std::make_shared(); + filler->SetTime(max_end_time_ms_, duration_ms_); + RETURN_IF_ERROR( + MediaHandler::DispatchTextSample(kStreamIndex, std::move(filler))); + } + + return FlushDownstream(index); +} + +Status TextPadder::OnTextSample(std::unique_ptr data) { + const TextSample& sample = *data->text_sample; + + // Check if there will be a gap between samples if we just dispatch this + // sample right away. If there will be one, create an empty sample that will + // fill in that gap. + if (sample.start_time() > max_end_time_ms_) { + std::shared_ptr filler = std::make_shared(); + filler->SetTime(max_end_time_ms_, sample.start_time()); + RETURN_IF_ERROR( + MediaHandler::DispatchTextSample(kStreamIndex, std::move(filler))); + } + + max_end_time_ms_ = std::max(max_end_time_ms_, sample.EndTime()); + return Dispatch(std::move(data)); +} +} // namespace media +} // namespace shaka diff --git a/packager/media/formats/webvtt/text_padder.h b/packager/media/formats/webvtt/text_padder.h new file mode 100644 index 0000000000..30fba431cd --- /dev/null +++ b/packager/media/formats/webvtt/text_padder.h @@ -0,0 +1,42 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef PACKAGER_MEDIA_FORMATS_WEBVTT_TEXT_PADDER_H_ +#define PACKAGER_MEDIA_FORMATS_WEBVTT_TEXT_PADDER_H_ + +#include "packager/media/base/media_handler.h" + +namespace shaka { +namespace media { + +/// A media handler that will inject empty text samples to fill any gaps +/// that may appear in the text stream. A min duration can be given to +/// ensure that the stream will have samples up to the given duration. +class TextPadder : public MediaHandler { + public: + /// Create a new text padder that will ensure the stream's duration is + // at least |duration_ms| long. + explicit TextPadder(int64_t duration_ms); + ~TextPadder() override = default; + + private: + TextPadder(const TextPadder&) = delete; + TextPadder& operator=(const TextPadder&) = delete; + + Status InitializeInternal() override; + + Status Process(std::unique_ptr data) override; + Status OnFlushRequest(size_t index) override; + Status OnTextSample(std::unique_ptr data); + + int64_t duration_ms_; + int64_t max_end_time_ms_ = 0; +}; + +} // namespace media +} // namespace shaka + +#endif // MEDIA_FORMATS_WEBVTT_TEXT_PADDER_H_ diff --git a/packager/media/formats/webvtt/webvtt.gyp b/packager/media/formats/webvtt/webvtt.gyp index 9b242c52bf..d983b7076c 100644 --- a/packager/media/formats/webvtt/webvtt.gyp +++ b/packager/media/formats/webvtt/webvtt.gyp @@ -15,6 +15,8 @@ 'sources': [ 'cue.cc', 'cue.h', + 'text_padder.cc', + 'text_padder.h', 'text_readers.cc', 'text_readers.h', 'webvtt_output_handler.cc', diff --git a/packager/packager.cc b/packager/packager.cc index ea1ff76a44..6cf36c3f6a 100644 --- a/packager/packager.cc +++ b/packager/packager.cc @@ -39,6 +39,7 @@ #include "packager/media/demuxer/demuxer.h" #include "packager/media/event/muxer_listener_factory.h" #include "packager/media/event/vod_media_info_dump_muxer_listener.h" +#include "packager/media/formats/webvtt/text_padder.h" #include "packager/media/formats/webvtt/text_readers.h" #include "packager/media/formats/webvtt/webvtt_output_handler.h" #include "packager/media/formats/webvtt/webvtt_parser.h" @@ -452,8 +453,10 @@ Status CreateHlsTextJob(const StreamDescriptor& stream, std::unique_ptr reader; RETURN_IF_ERROR(FileReader::Open(stream.input, &reader)); + const int64_t kNoDuration = 0; auto parser = std::make_shared(std::move(reader), stream.language); + auto padder = std::make_shared(kNoDuration); auto chunker = std::make_shared(segment_length_in_ms); // Build in reverse to allow us to move the pointers. @@ -461,10 +464,12 @@ Status CreateHlsTextJob(const StreamDescriptor& stream, auto cue_aligner = std::make_shared(sync_points); RETURN_IF_ERROR(chunker->AddHandler(std::move(output))); RETURN_IF_ERROR(cue_aligner->AddHandler(std::move(chunker))); - RETURN_IF_ERROR(parser->AddHandler(std::move(cue_aligner))); + RETURN_IF_ERROR(padder->AddHandler(std::move(cue_aligner))); + RETURN_IF_ERROR(parser->AddHandler(std::move(padder))); } else { RETURN_IF_ERROR(chunker->AddHandler(std::move(output))); - RETURN_IF_ERROR(parser->AddHandler(std::move(chunker))); + RETURN_IF_ERROR(padder->AddHandler(std::move(chunker))); + RETURN_IF_ERROR(parser->AddHandler(std::move(padder))); } job_manager->Add("Segmented Text Job", std::move(parser)); @@ -480,30 +485,27 @@ Status CreateWebVttToMp4TextJob(const StreamDescriptor& stream, std::shared_ptr* root) { // TODO(kqyang): Support Cue Alignment if |sync_points| is not null. - Status status; std::unique_ptr reader; - status = FileReader::Open(stream.input, &reader); - - if (!status.ok()) { - return status; - } + RETURN_IF_ERROR(FileReader::Open(stream.input, &reader)); + const int64_t kNoDuration = 0; auto parser = std::make_shared(std::move(reader), stream.language); + auto padder = std::make_shared(kNoDuration); auto text_to_mp4 = std::make_shared(); auto chunker = std::make_shared(packaging_params.chunking_params); - std::shared_ptr muxer = - muxer_factory->CreateMuxer(GetOutputFormat(stream), stream); + auto muxer = muxer_factory->CreateMuxer(GetOutputFormat(stream), stream); muxer->SetMuxerListener(std::move(muxer_listener)); - status.Update(chunker->AddHandler(std::move(muxer))); - status.Update(text_to_mp4->AddHandler(std::move(chunker))); - status.Update(parser->AddHandler(std::move(text_to_mp4))); + RETURN_IF_ERROR(chunker->AddHandler(std::move(muxer))); + RETURN_IF_ERROR(text_to_mp4->AddHandler(std::move(chunker))); + RETURN_IF_ERROR(padder->AddHandler(std::move(text_to_mp4))); + RETURN_IF_ERROR(parser->AddHandler(std::move(padder))); *root = std::move(parser); - return status; + return Status::OK; } Status CreateTextJobs(