From 8e85862bdaa99db626fc1b8f82ac423e38fff01c Mon Sep 17 00:00:00 2001 From: Jacob Trimble Date: Tue, 6 Oct 2020 16:07:51 -0700 Subject: [PATCH] Ignore unsupported H26x streams. This adds a new path when parsing MPEG2-TS streams to ignore unsupported streams. This allows extracting supported streams when some of the streams are unsupported. For example, you can extract audio from a file that has unsupported video. Change-Id: I608fcb19d0a573bfd35e9272f60b0b69346ae11a --- packager/media/formats/mp2t/es_parser_h264.cc | 35 +++++++++---- packager/media/formats/mp2t/es_parser_h265.cc | 35 +++++++++---- .../media/formats/mp2t/mp2t_media_parser.cc | 52 ++++++++++--------- .../media/formats/mp2t/mp2t_media_parser.h | 3 +- 4 files changed, 78 insertions(+), 47 deletions(-) diff --git a/packager/media/formats/mp2t/es_parser_h264.cc b/packager/media/formats/mp2t/es_parser_h264.cc index 2bd3d26c11..10bde0c309 100644 --- a/packager/media/formats/mp2t/es_parser_h264.cc +++ b/packager/media/formats/mp2t/es_parser_h264.cc @@ -52,20 +52,29 @@ bool EsParserH264::ProcessNalu(const Nalu& nalu, case Nalu::H264_SPS: { DVLOG(LOG_LEVEL_ES) << "Nalu: SPS"; int sps_id; - if (h264_parser_->ParseSps(nalu, &sps_id) != H264Parser::kOk) + auto status = h264_parser_->ParseSps(nalu, &sps_id); + if (status == H264Parser::kOk) + decoder_config_check_pending_ = true; + else if (status == H264Parser::kUnsupportedStream) + // Indicate the stream can't be parsed. + new_stream_info_cb_.Run(nullptr); + else return false; - decoder_config_check_pending_ = true; break; } case Nalu::H264_PPS: { DVLOG(LOG_LEVEL_ES) << "Nalu: PPS"; int pps_id; - if (h264_parser_->ParsePps(nalu, &pps_id) != H264Parser::kOk) { + auto status = h264_parser_->ParsePps(nalu, &pps_id); + if (status == H264Parser::kOk) { + decoder_config_check_pending_ = true; + } else if (status == H264Parser::kUnsupportedStream) { + // Indicate the stream can't be parsed. + new_stream_info_cb_.Run(nullptr); + } else { // Allow PPS parsing to fail if waiting for SPS. if (last_video_decoder_config_) return false; - } else { - decoder_config_check_pending_ = true; } break; } @@ -74,16 +83,20 @@ bool EsParserH264::ProcessNalu(const Nalu& nalu, const bool is_key_frame = (nalu.type() == Nalu::H264_IDRSlice); DVLOG(LOG_LEVEL_ES) << "Nalu: slice IDR=" << is_key_frame; H264SliceHeader shdr; - if (h264_parser_->ParseSliceHeader(nalu, &shdr) != H264Parser::kOk) { - // Only accept an invalid SPS/PPS at the beginning when the stream - // does not necessarily start with an SPS/PPS/IDR. - if (last_video_decoder_config_) - return false; - } else { + auto status = h264_parser_->ParseSliceHeader(nalu, &shdr); + if (status == H264Parser::kOk) { video_slice_info->valid = true; video_slice_info->is_key_frame = is_key_frame; video_slice_info->frame_num = shdr.frame_num; video_slice_info->pps_id = shdr.pic_parameter_set_id; + } else if (status == H264Parser::kUnsupportedStream) { + // Indicate the stream can't be parsed. + new_stream_info_cb_.Run(nullptr); + } else { + // Only accept an invalid SPS/PPS at the beginning when the stream + // does not necessarily start with an SPS/PPS/IDR. + if (last_video_decoder_config_) + return false; } break; } diff --git a/packager/media/formats/mp2t/es_parser_h265.cc b/packager/media/formats/mp2t/es_parser_h265.cc index 4983392f22..3d270dffd8 100644 --- a/packager/media/formats/mp2t/es_parser_h265.cc +++ b/packager/media/formats/mp2t/es_parser_h265.cc @@ -55,20 +55,29 @@ bool EsParserH265::ProcessNalu(const Nalu& nalu, case Nalu::H265_SPS: { DVLOG(LOG_LEVEL_ES) << "Nalu: SPS"; int sps_id; - if (h265_parser_->ParseSps(nalu, &sps_id) != H265Parser::kOk) + auto status = h265_parser_->ParseSps(nalu, &sps_id); + if (status == H265Parser::kOk) + decoder_config_check_pending_ = true; + else if (status == H265Parser::kUnsupportedStream) + // Indicate the stream can't be parsed. + new_stream_info_cb_.Run(nullptr); + else return false; - decoder_config_check_pending_ = true; break; } case Nalu::H265_PPS: { DVLOG(LOG_LEVEL_ES) << "Nalu: PPS"; int pps_id; - if (h265_parser_->ParsePps(nalu, &pps_id) != H265Parser::kOk) { + auto status = h265_parser_->ParsePps(nalu, &pps_id); + if (status == H265Parser::kOk) { + decoder_config_check_pending_ = true; + } else if (status == H265Parser::kUnsupportedStream) { + // Indicate the stream can't be parsed. + new_stream_info_cb_.Run(nullptr); + } else { // Allow PPS parsing to fail if waiting for SPS. if (last_video_decoder_config_) return false; - } else { - decoder_config_check_pending_ = true; } break; } @@ -78,16 +87,20 @@ bool EsParserH265::ProcessNalu(const Nalu& nalu, nalu.type() == Nalu::H265_IDR_N_LP; DVLOG(LOG_LEVEL_ES) << "Nalu: slice KeyFrame=" << is_key_frame; H265SliceHeader shdr; - if (h265_parser_->ParseSliceHeader(nalu, &shdr) != H265Parser::kOk) { - // Only accept an invalid SPS/PPS at the beginning when the stream - // does not necessarily start with an SPS/PPS/IDR. - if (last_video_decoder_config_) - return false; - } else { + auto status = h265_parser_->ParseSliceHeader(nalu, &shdr); + if (status == H265Parser::kOk) { video_slice_info->valid = true; video_slice_info->is_key_frame = is_key_frame; video_slice_info->frame_num = 0; // frame_num is only for H264. video_slice_info->pps_id = shdr.pic_parameter_set_id; + } else if (status == H265Parser::kUnsupportedStream) { + // Indicate the stream can't be parsed. + new_stream_info_cb_.Run(nullptr); + } else { + // Only accept an invalid SPS/PPS at the beginning when the stream + // does not necessarily start with an SPS/PPS/IDR. + if (last_video_decoder_config_) + return false; } } else { DVLOG(LOG_LEVEL_ES) << "Nalu: " << nalu.type(); diff --git a/packager/media/formats/mp2t/mp2t_media_parser.cc b/packager/media/formats/mp2t/mp2t_media_parser.cc index d77622de41..e15e203ed1 100644 --- a/packager/media/formats/mp2t/mp2t_media_parser.cc +++ b/packager/media/formats/mp2t/mp2t_media_parser.cc @@ -181,7 +181,7 @@ bool Mp2tMediaParser::Flush() { } bool Mp2tMediaParser::Parse(const uint8_t* buf, int size) { - DVLOG(1) << "Mp2tMediaParser::Parse size=" << size; + DVLOG(2) << "Mp2tMediaParser::Parse size=" << size; // Add the data to the parser state. ts_byte_queue_.Push(buf, size); @@ -274,37 +274,33 @@ void Mp2tMediaParser::RegisterPmt(int program_number, int pmt_pid) { void Mp2tMediaParser::RegisterPes(int pmt_pid, int pes_pid, int stream_type) { - DVLOG(1) << "RegisterPes:" - << " pes_pid=" << pes_pid - << " stream_type=" << std::hex << stream_type << std::dec; std::map>::iterator it = pids_.find(pes_pid); if (it != pids_.end()) return; + DVLOG(1) << "RegisterPes:" + << " pes_pid=" << pes_pid << " stream_type=" << std::hex + << stream_type << std::dec; // Create a stream parser corresponding to the stream type. bool is_audio = false; std::unique_ptr es_parser; + auto on_new_stream = base::Bind(&Mp2tMediaParser::OnNewStreamInfo, + base::Unretained(this), pes_pid); + auto on_emit = + base::Bind(&Mp2tMediaParser::OnEmitSample, base::Unretained(this)); switch (static_cast(stream_type)) { case TsStreamType::kAvc: - es_parser.reset(new EsParserH264( - pes_pid, - base::Bind(&Mp2tMediaParser::OnNewStreamInfo, base::Unretained(this)), - base::Bind(&Mp2tMediaParser::OnEmitSample, base::Unretained(this)))); + es_parser.reset(new EsParserH264(pes_pid, on_new_stream, on_emit)); break; case TsStreamType::kHevc: - es_parser.reset(new EsParserH265( - pes_pid, - base::Bind(&Mp2tMediaParser::OnNewStreamInfo, base::Unretained(this)), - base::Bind(&Mp2tMediaParser::OnEmitSample, base::Unretained(this)))); + es_parser.reset(new EsParserH265(pes_pid, on_new_stream, on_emit)); break; case TsStreamType::kAdtsAac: case TsStreamType::kMpeg1Audio: case TsStreamType::kAc3: - es_parser.reset(new EsParserAudio( - pes_pid, static_cast(stream_type), - base::Bind(&Mp2tMediaParser::OnNewStreamInfo, base::Unretained(this)), - base::Bind(&Mp2tMediaParser::OnEmitSample, base::Unretained(this)), - sbr_in_mimetype_)); + es_parser.reset( + new EsParserAudio(pes_pid, static_cast(stream_type), + on_new_stream, on_emit, sbr_in_mimetype_)); is_audio = true; break; default: { @@ -330,19 +326,26 @@ void Mp2tMediaParser::RegisterPes(int pmt_pid, } void Mp2tMediaParser::OnNewStreamInfo( + uint32_t pes_pid, const std::shared_ptr& new_stream_info) { - DCHECK(new_stream_info); - DVLOG(1) << "OnVideoConfigChanged for pid=" << new_stream_info->track_id(); + DCHECK(!new_stream_info || new_stream_info->track_id() == pes_pid); + DVLOG(1) << "OnVideoConfigChanged for pid=" << pes_pid + << ", has_info=" << (new_stream_info ? "true" : "false"); - PidMap::iterator pid_state = pids_.find(new_stream_info->track_id()); + PidMap::iterator pid_state = pids_.find(pes_pid); if (pid_state == pids_.end()) { LOG(ERROR) << "PID State for new stream not found (pid = " << new_stream_info->track_id() << ")."; return; } - // Set the stream configuration information for the PID. - pid_state->second->set_config(new_stream_info); + if (new_stream_info) { + // Set the stream configuration information for the PID. + pid_state->second->set_config(new_stream_info); + } else { + LOG(WARNING) << "Ignoring unsupported stream with pid=" << pes_pid; + pid_state->second->Disable(); + } // Finish initialization if all streams have configs. FinishInitializationIfNeeded(); @@ -361,8 +364,9 @@ bool Mp2tMediaParser::FinishInitializationIfNeeded() { uint32_t num_es(0); for (PidMap::const_iterator iter = pids_.begin(); iter != pids_.end(); ++iter) { - if (((iter->second->pid_type() == PidState::kPidAudioPes) || - (iter->second->pid_type() == PidState::kPidVideoPes))) { + if ((iter->second->pid_type() == PidState::kPidAudioPes || + iter->second->pid_type() == PidState::kPidVideoPes) && + iter->second->IsEnabled()) { ++num_es; if (iter->second->config()) all_stream_info.push_back(iter->second->config()); diff --git a/packager/media/formats/mp2t/mp2t_media_parser.h b/packager/media/formats/mp2t/mp2t_media_parser.h index 06cfec1dc1..4e1675cf3b 100644 --- a/packager/media/formats/mp2t/mp2t_media_parser.h +++ b/packager/media/formats/mp2t/mp2t_media_parser.h @@ -56,7 +56,8 @@ class Mp2tMediaParser : public MediaParser { // Callback invoked each time the audio/video decoder configuration is // changed. - void OnNewStreamInfo(const std::shared_ptr& new_stream_info); + void OnNewStreamInfo(uint32_t pes_pid, + const std::shared_ptr& new_stream_info); // Callback invoked by the ES media parser // to emit a new audio/video access unit.