From 9b036b764bd61fa8b547dd74bdd2a5f063e6aa01 Mon Sep 17 00:00:00 2001 From: Jacob Trimble Date: Fri, 16 Oct 2020 14:18:35 -0700 Subject: [PATCH] Add TextSample handling to MP2T parser. This also changes the callbacks a bit to (a) avoid passing references for already ref-counted types, and (b) don't pass PID since the parent knows this and gives it to the child parser. Issue #832 Change-Id: I7dd44436c8d1ad81d42a813d16f850175b85ad1a --- packager/media/formats/mp2t/es_parser.h | 8 +- .../media/formats/mp2t/es_parser_audio.cc | 2 +- .../formats/mp2t/es_parser_h264_unittest.cc | 4 +- packager/media/formats/mp2t/es_parser_h26x.cc | 5 +- .../formats/mp2t/es_parser_h26x_unittest.cc | 4 +- .../media/formats/mp2t/mp2t_media_parser.cc | 119 ++++++++++-------- .../media/formats/mp2t/mp2t_media_parser.h | 28 +++-- packager/media/formats/mp2t/ts_section_pmt.cc | 14 +-- packager/media/formats/mp2t/ts_section_pmt.h | 2 +- 9 files changed, 99 insertions(+), 87 deletions(-) diff --git a/packager/media/formats/mp2t/es_parser.h b/packager/media/formats/mp2t/es_parser.h index bcb4136f4e..4d377a2118 100644 --- a/packager/media/formats/mp2t/es_parser.h +++ b/packager/media/formats/mp2t/es_parser.h @@ -14,15 +14,15 @@ namespace media { class MediaSample; class StreamInfo; +class TextSample; namespace mp2t { class EsParser { public: - typedef base::Callback&)> - NewStreamInfoCB; - typedef base::Callback&)> - EmitSampleCB; + typedef base::Callback)> NewStreamInfoCB; + typedef base::Callback)> EmitSampleCB; + typedef base::Callback)> EmitTextSampleCB; EsParser(uint32_t pid) : pid_(pid) {} virtual ~EsParser() {} diff --git a/packager/media/formats/mp2t/es_parser_audio.cc b/packager/media/formats/mp2t/es_parser_audio.cc index 3a0d99683e..ceb65f4bc9 100644 --- a/packager/media/formats/mp2t/es_parser_audio.cc +++ b/packager/media/formats/mp2t/es_parser_audio.cc @@ -167,7 +167,7 @@ bool EsParserAudio::Parse(const uint8_t* buf, sample->set_pts(current_pts); sample->set_dts(current_pts); sample->set_duration(frame_duration); - emit_sample_cb_.Run(pid(), sample); + emit_sample_cb_.Run(sample); // Update the PTS of the next frame. audio_timestamp_helper_->AddFrames(audio_header_->GetSamplesPerFrame()); diff --git a/packager/media/formats/mp2t/es_parser_h264_unittest.cc b/packager/media/formats/mp2t/es_parser_h264_unittest.cc index 405ea49e9a..19273311b3 100644 --- a/packager/media/formats/mp2t/es_parser_h264_unittest.cc +++ b/packager/media/formats/mp2t/es_parser_h264_unittest.cc @@ -124,13 +124,13 @@ class EsParserH264Test : public testing::Test { void LoadStream(const char* filename); void ProcessPesPackets(const std::vector& pes_packets); - void EmitSample(uint32_t pid, const std::shared_ptr& sample) { + void EmitSample(std::shared_ptr sample) { sample_count_++; if (sample_count_ == 1) first_frame_is_key_frame_ = sample->is_key_frame(); } - void NewVideoConfig(const std::shared_ptr& config) { + void NewVideoConfig(std::shared_ptr config) { DVLOG(1) << config->ToString(); stream_map_[config->track_id()] = config; } diff --git a/packager/media/formats/mp2t/es_parser_h26x.cc b/packager/media/formats/mp2t/es_parser_h26x.cc index 9f3df7d61b..bf3838a264 100644 --- a/packager/media/formats/mp2t/es_parser_h26x.cc +++ b/packager/media/formats/mp2t/es_parser_h26x.cc @@ -101,8 +101,7 @@ void EsParserH26x::Flush() { // Flush pending sample. DCHECK(pending_sample_duration_); pending_sample_->set_duration(pending_sample_duration_); - emit_sample_cb_.Run(pid(), pending_sample_); - pending_sample_ = std::shared_ptr(); + emit_sample_cb_.Run(std::move(pending_sample_)); } } @@ -339,7 +338,7 @@ bool EsParserH26x::EmitFrame(int64_t access_unit_pos, pending_sample_duration_ = sample_duration; } - emit_sample_cb_.Run(pid(), std::move(pending_sample_)); + emit_sample_cb_.Run(std::move(pending_sample_)); } pending_sample_ = media_sample; diff --git a/packager/media/formats/mp2t/es_parser_h26x_unittest.cc b/packager/media/formats/mp2t/es_parser_h26x_unittest.cc index cd2aebe9ef..dd819aa395 100644 --- a/packager/media/formats/mp2t/es_parser_h26x_unittest.cc +++ b/packager/media/formats/mp2t/es_parser_h26x_unittest.cc @@ -168,7 +168,7 @@ class EsParserH26xTest : public testing::Test { const H26xNaluType* types, size_t types_count); - void EmitSample(uint32_t pid, const std::shared_ptr& sample) { + void EmitSample(std::shared_ptr sample) { size_t sample_id = sample_count_; sample_count_++; if (sample_count_ == 1) { @@ -181,7 +181,7 @@ class EsParserH26xTest : public testing::Test { EXPECT_EQ(samples_[sample_id], sample_data); } - void NewVideoConfig(const std::shared_ptr& config) { + void NewVideoConfig(std::shared_ptr config) { has_stream_info_ = true; } diff --git a/packager/media/formats/mp2t/mp2t_media_parser.cc b/packager/media/formats/mp2t/mp2t_media_parser.cc index e15e203ed1..665f25a517 100644 --- a/packager/media/formats/mp2t/mp2t_media_parser.cc +++ b/packager/media/formats/mp2t/mp2t_media_parser.cc @@ -5,9 +5,11 @@ #include "packager/media/formats/mp2t/mp2t_media_parser.h" #include + #include "packager/base/bind.h" #include "packager/media/base/media_sample.h" #include "packager/media/base/stream_info.h" +#include "packager/media/base/text_sample.h" #include "packager/media/formats/mp2t/es_parser.h" #include "packager/media/formats/mp2t/es_parser_audio.h" #include "packager/media/formats/mp2t/es_parser_h264.h" @@ -59,19 +61,20 @@ class PidState { config_ = config; } - SampleQueue& sample_queue() { return sample_queue_; } - private: + friend Mp2tMediaParser; void ResetState(); int pid_; PidType pid_type_; std::unique_ptr section_parser_; + std::deque> media_sample_queue_; + std::deque> text_sample_queue_; + bool enable_; int continuity_counter_; std::shared_ptr config_; - SampleQueue sample_queue_; }; PidState::PidState(int pid, @@ -157,9 +160,11 @@ void Mp2tMediaParser::Init(const InitCB& init_cb, DCHECK(init_cb_.is_null()); DCHECK(!init_cb.is_null()); DCHECK(!new_media_sample_cb.is_null()); + DCHECK(!new_text_sample_cb.is_null()); init_cb_ = init_cb; - new_sample_cb_ = new_media_sample_cb; + new_media_sample_cb_ = new_media_sample_cb; + new_text_sample_cb_ = new_text_sample_cb; } bool Mp2tMediaParser::Flush() { @@ -215,8 +220,7 @@ bool Mp2tMediaParser::Parse(const uint8_t* buf, int size) { << " start_unit=" << ts_packet->payload_unit_start_indicator(); // Parse the section. - std::map>::iterator it = - pids_.find(ts_packet->pid()); + auto it = pids_.find(ts_packet->pid()); if (it == pids_.end() && ts_packet->pid() == TsSection::kPidPat) { // Create the PAT state here if needed. @@ -225,10 +229,7 @@ bool Mp2tMediaParser::Parse(const uint8_t* buf, int size) { std::unique_ptr pat_pid_state(new PidState( ts_packet->pid(), PidState::kPidPat, std::move(pat_section_parser))); pat_pid_state->Enable(); - it = pids_ - .insert(std::pair>( - ts_packet->pid(), std::move(pat_pid_state))) - .first; + it = pids_.emplace(ts_packet->pid(), std::move(pat_pid_state)).first; } if (it != pids_.end()) { @@ -267,15 +268,13 @@ void Mp2tMediaParser::RegisterPmt(int program_number, int pmt_pid) { std::unique_ptr pmt_pid_state( new PidState(pmt_pid, PidState::kPidPmt, std::move(pmt_section_parser))); pmt_pid_state->Enable(); - pids_.insert(std::pair>( - pmt_pid, std::move(pmt_pid_state))); + pids_.emplace(pmt_pid, std::move(pmt_pid_state)); } void Mp2tMediaParser::RegisterPes(int pmt_pid, int pes_pid, - int stream_type) { - std::map>::iterator it = pids_.find(pes_pid); - if (it != pids_.end()) + uint8_t stream_type) { + if (pids_.count(pes_pid) != 0) return; DVLOG(1) << "RegisterPes:" << " pes_pid=" << pes_pid << " stream_type=" << std::hex @@ -286,21 +285,21 @@ void Mp2tMediaParser::RegisterPes(int pmt_pid, std::unique_ptr es_parser; auto on_new_stream = base::Bind(&Mp2tMediaParser::OnNewStreamInfo, base::Unretained(this), pes_pid); - auto on_emit = - base::Bind(&Mp2tMediaParser::OnEmitSample, base::Unretained(this)); + auto on_emit_media = base::Bind(&Mp2tMediaParser::OnEmitMediaSample, + base::Unretained(this), pes_pid); switch (static_cast(stream_type)) { case TsStreamType::kAvc: - es_parser.reset(new EsParserH264(pes_pid, on_new_stream, on_emit)); + es_parser.reset(new EsParserH264(pes_pid, on_new_stream, on_emit_media)); break; case TsStreamType::kHevc: - es_parser.reset(new EsParserH265(pes_pid, on_new_stream, on_emit)); + es_parser.reset(new EsParserH265(pes_pid, on_new_stream, on_emit_media)); break; case TsStreamType::kAdtsAac: case TsStreamType::kMpeg1Audio: case TsStreamType::kAc3: es_parser.reset( new EsParserAudio(pes_pid, static_cast(stream_type), - on_new_stream, on_emit, sbr_in_mimetype_)); + on_new_stream, on_emit_media, sbr_in_mimetype_)); is_audio = true; break; default: { @@ -321,18 +320,17 @@ void Mp2tMediaParser::RegisterPes(int pmt_pid, std::unique_ptr pes_pid_state( new PidState(pes_pid, pid_type, std::move(pes_section_parser))); pes_pid_state->Enable(); - pids_.insert(std::pair>( - pes_pid, std::move(pes_pid_state))); + pids_.emplace(pes_pid, std::move(pes_pid_state)); } void Mp2tMediaParser::OnNewStreamInfo( uint32_t pes_pid, - const std::shared_ptr& new_stream_info) { + std::shared_ptr new_stream_info) { DCHECK(!new_stream_info || new_stream_info->track_id() == pes_pid); DVLOG(1) << "OnVideoConfigChanged for pid=" << pes_pid << ", has_info=" << (new_stream_info ? "true" : "false"); - PidMap::iterator pid_state = pids_.find(pes_pid); + auto pid_state = pids_.find(pes_pid); if (pid_state == pids_.end()) { LOG(ERROR) << "PID State for new stream not found (pid = " << new_stream_info->track_id() << ")."; @@ -362,14 +360,13 @@ bool Mp2tMediaParser::FinishInitializationIfNeeded() { std::vector> all_stream_info; uint32_t num_es(0); - for (PidMap::const_iterator iter = pids_.begin(); iter != pids_.end(); - ++iter) { - if ((iter->second->pid_type() == PidState::kPidAudioPes || - iter->second->pid_type() == PidState::kPidVideoPes) && - iter->second->IsEnabled()) { + for (const auto& pair : pids_) { + if ((pair.second->pid_type() == PidState::kPidAudioPes || + pair.second->pid_type() == PidState::kPidVideoPes) && + pair.second->IsEnabled()) { ++num_es; - if (iter->second->config()) - all_stream_info.push_back(iter->second->config()); + if (pair.second->config()) + all_stream_info.push_back(pair.second->config()); } } if (num_es && (all_stream_info.size() == num_es)) { @@ -382,29 +379,41 @@ bool Mp2tMediaParser::FinishInitializationIfNeeded() { return true; } -void Mp2tMediaParser::OnEmitSample( +void Mp2tMediaParser::OnEmitMediaSample( uint32_t pes_pid, - const std::shared_ptr& new_sample) { + std::shared_ptr new_sample) { DCHECK(new_sample); - DVLOG(LOG_LEVEL_ES) - << "OnEmitSample: " - << " pid=" - << pes_pid - << " size=" - << new_sample->data_size() - << " dts=" - << new_sample->dts() - << " pts=" - << new_sample->pts(); + DVLOG(LOG_LEVEL_ES) << "OnEmitMediaSample: " + << " pid=" << pes_pid + << " size=" << new_sample->data_size() + << " dts=" << new_sample->dts() + << " pts=" << new_sample->pts(); // Add the sample to the appropriate PID sample queue. - PidMap::iterator pid_state = pids_.find(pes_pid); + auto pid_state = pids_.find(pes_pid); + if (pid_state == pids_.end()) { + LOG(ERROR) << "PID State for new sample not found (pid = " << pes_pid + << ")."; + return; + } + pid_state->second->media_sample_queue_.push_back(std::move(new_sample)); +} + +void Mp2tMediaParser::OnEmitTextSample(uint32_t pes_pid, + std::shared_ptr new_sample) { + DCHECK(new_sample); + DVLOG(LOG_LEVEL_ES) << "OnEmitTextSample: " + << " pid=" << pes_pid + << " start=" << new_sample->start_time(); + + // Add the sample to the appropriate PID sample queue. + auto pid_state = pids_.find(pes_pid); if (pid_state == pids_.end()) { LOG(ERROR) << "PID State for new sample not found (pid = " << pes_pid << ")."; return; } - pid_state->second->sample_queue().push_back(new_sample); + pid_state->second->text_sample_queue_.push_back(std::move(new_sample)); } bool Mp2tMediaParser::EmitRemainingSamples() { @@ -415,18 +424,22 @@ bool Mp2tMediaParser::EmitRemainingSamples() { return true; // Buffer emission. - for (PidMap::const_iterator pid_iter = pids_.begin(); pid_iter != pids_.end(); - ++pid_iter) { - SampleQueue& sample_queue = pid_iter->second->sample_queue(); - for (SampleQueue::iterator sample_iter = sample_queue.begin(); - sample_iter != sample_queue.end(); - ++sample_iter) { - if (!new_sample_cb_.Run(pid_iter->first, *sample_iter)) { + for (const auto& pid_pair : pids_) { + for (auto sample : pid_pair.second->media_sample_queue_) { + if (!new_media_sample_cb_.Run(pid_pair.first, sample)) { // Error processing sample. Propagate error condition. return false; } } - sample_queue.clear(); + pid_pair.second->media_sample_queue_.clear(); + + for (auto sample : pid_pair.second->text_sample_queue_) { + if (!new_text_sample_cb_.Run(pid_pair.first, sample)) { + // Error processing sample. Propagate error condition. + return false; + } + } + pid_pair.second->text_sample_queue_.clear(); } return true; diff --git a/packager/media/formats/mp2t/mp2t_media_parser.h b/packager/media/formats/mp2t/mp2t_media_parser.h index 4e1675cf3b..d3708ae5b4 100644 --- a/packager/media/formats/mp2t/mp2t_media_parser.h +++ b/packager/media/formats/mp2t/mp2t_media_parser.h @@ -5,6 +5,7 @@ #ifndef PACKAGER_MEDIA_FORMATS_MP2T_MP2T_MEDIA_PARSER_H_ #define PACKAGER_MEDIA_FORMATS_MP2T_MP2T_MEDIA_PARSER_H_ +#include #include #include #include @@ -24,8 +25,6 @@ class PidState; class TsPacket; class TsSection; -typedef std::deque> SampleQueue; - class Mp2tMediaParser : public MediaParser { public: Mp2tMediaParser(); @@ -42,8 +41,6 @@ class Mp2tMediaParser : public MediaParser { /// @} private: - typedef std::map> PidMap; - // Callback invoked to register a Program Map Table. // Note: Does nothing if the PID is already registered. void RegisterPmt(int program_number, int pmt_pid); @@ -52,17 +49,19 @@ class Mp2tMediaParser : public MediaParser { // Possible values for |media_type| are defined in: // ISO-13818.1 / ITU H.222 Table 2.34 "Media type assignments". // |pes_pid| is part of the Program Map Table refered by |pmt_pid|. - void RegisterPes(int pmt_pid, int pes_pid, int media_type); + void RegisterPes(int pmt_pid, int pes_pid, uint8_t media_type); // Callback invoked each time the audio/video decoder configuration is // changed. void OnNewStreamInfo(uint32_t pes_pid, - const std::shared_ptr& new_stream_info); + std::shared_ptr new_stream_info); // Callback invoked by the ES media parser // to emit a new audio/video access unit. - void OnEmitSample(uint32_t pes_pid, - const std::shared_ptr& new_sample); + void OnEmitMediaSample(uint32_t pes_pid, + std::shared_ptr new_sample); + void OnEmitTextSample(uint32_t pes_pid, + std::shared_ptr new_sample); // Invoke the initialization callback if needed. bool FinishInitializationIfNeeded(); @@ -72,26 +71,29 @@ class Mp2tMediaParser : public MediaParser { /// Set the value of the "SBR in mime-type" flag which leads to sample rate /// doubling. Default value is false. void set_sbr_in_mime_type(bool sbr_in_mimetype) { - sbr_in_mimetype_ = sbr_in_mimetype; } + sbr_in_mimetype_ = sbr_in_mimetype; + } // List of callbacks. InitCB init_cb_; - NewMediaSampleCB new_sample_cb_; + NewMediaSampleCB new_media_sample_cb_; + NewTextSampleCB new_text_sample_cb_; bool sbr_in_mimetype_; // Bytes of the TS media. ByteQueue ts_byte_queue_; - // List of PIDs and their states. - PidMap pids_; + // Map of PIDs and their states. Use an ordered map so manifest generation + // has a deterministic order. + std::map> pids_; // Whether |init_cb_| has been invoked. bool is_initialized_; // A map used to track unsupported stream types and make sure the error is // only logged once. - std::map stream_type_logged_once_; + std::bitset<256> stream_type_logged_once_; DISALLOW_COPY_AND_ASSIGN(Mp2tMediaParser); }; diff --git a/packager/media/formats/mp2t/ts_section_pmt.cc b/packager/media/formats/mp2t/ts_section_pmt.cc index 8ba854cb2e..7fbd22a528 100644 --- a/packager/media/formats/mp2t/ts_section_pmt.cc +++ b/packager/media/formats/mp2t/ts_section_pmt.cc @@ -75,15 +75,14 @@ bool TsSectionPmt::ParsePsiSection(BitReader* bit_reader) { // The end of the PID map if 4 bytes away from the end of the section // (4 bytes = size of the CRC). int pid_map_end_marker = section_start_marker - section_length + 4; - std::map pid_map; + std::map pid_map; while (static_cast(bit_reader->bits_available()) > 8 * pid_map_end_marker) { - int stream_type; - int reserved; + uint8_t stream_type; int pid_es; int es_info_length; RCHECK(bit_reader->ReadBits(8, &stream_type)); - RCHECK(bit_reader->ReadBits(3, &reserved)); + RCHECK(bit_reader->SkipBits(3)); // reserved RCHECK(bit_reader->ReadBits(13, &pid_es)); RCHECK(bit_reader->ReadBits(4, &reserved)); RCHECK(bit_reader->ReadBits(12, &es_info_length)); @@ -91,7 +90,7 @@ bool TsSectionPmt::ParsePsiSection(BitReader* bit_reader) { // Do not register the PID right away. // Wait for the end of the section to be fully parsed // to make sure there is no error. - pid_map.insert(std::pair(pid_es, stream_type)); + pid_map.emplace(pid_es, stream_type); // Read the ES info descriptors. // Defined in section 2.6 of ISO-13818. @@ -103,9 +102,8 @@ bool TsSectionPmt::ParsePsiSection(BitReader* bit_reader) { RCHECK(bit_reader->ReadBits(32, &crc32)); // Once the PMT has been proved to be correct, register the PIDs. - for (std::map::iterator it = pid_map.begin(); - it != pid_map.end(); ++it) - register_pes_cb_.Run(it->first, it->second); + for (auto& pair : pid_map) + register_pes_cb_.Run(pair.first, pair.second); return true; } diff --git a/packager/media/formats/mp2t/ts_section_pmt.h b/packager/media/formats/mp2t/ts_section_pmt.h index 7b14d82b70..8e600c1570 100644 --- a/packager/media/formats/mp2t/ts_section_pmt.h +++ b/packager/media/formats/mp2t/ts_section_pmt.h @@ -18,7 +18,7 @@ class TsSectionPmt : public TsSectionPsi { // RegisterPesCb::Run(int pes_pid, int stream_type); // Stream type is defined in // "Table 2-34 – Stream type assignments" in H.222 - typedef base::Callback RegisterPesCb; + typedef base::Callback RegisterPesCb; explicit TsSectionPmt(const RegisterPesCb& register_pes_cb); ~TsSectionPmt() override;