5 #include "packager/media/formats/mp4/mp4_media_parser.h"
9 #include "packager/base/callback.h"
10 #include "packager/base/callback_helpers.h"
11 #include "packager/base/logging.h"
12 #include "packager/base/memory/ref_counted.h"
13 #include "packager/base/strings/string_number_conversions.h"
14 #include "packager/media/base/audio_stream_info.h"
15 #include "packager/media/base/buffer_reader.h"
16 #include "packager/media/base/decrypt_config.h"
17 #include "packager/media/base/key_source.h"
18 #include "packager/media/base/macros.h"
19 #include "packager/media/base/media_sample.h"
20 #include "packager/media/base/rcheck.h"
21 #include "packager/media/base/video_stream_info.h"
22 #include "packager/media/codecs/avc_decoder_configuration_record.h"
23 #include "packager/media/codecs/es_descriptor.h"
24 #include "packager/media/codecs/hevc_decoder_configuration_record.h"
25 #include "packager/media/codecs/vp_codec_configuration_record.h"
26 #include "packager/media/file/file.h"
27 #include "packager/media/file/file_closer.h"
28 #include "packager/media/formats/mp4/box_definitions.h"
29 #include "packager/media/formats/mp4/box_reader.h"
30 #include "packager/media/formats/mp4/track_run_iterator.h"
37 uint64_t Rescale(uint64_t time_in_old_scale,
40 return (static_cast<double>(time_in_old_scale) / old_scale) * new_scale;
43 VideoCodec FourCCToVideoCodec(FourCC fourcc) {
58 return kUnknownVideoCodec;
62 AudioCodec FourCCToAudioCodec(FourCC fourcc) {
83 return kUnknownAudioCodec;
88 const uint8_t kDtsAudioNumChannels = 6;
89 const uint64_t kNanosecondsPerSecond = 1000000000ull;
93 MP4MediaParser::MP4MediaParser()
94 : state_(kWaitingForInit),
95 decryption_key_source_(NULL),
99 MP4MediaParser::~MP4MediaParser() {}
102 const NewSampleCB& new_sample_cb,
104 DCHECK_EQ(state_, kWaitingForInit);
105 DCHECK(init_cb_.is_null());
106 DCHECK(!init_cb.is_null());
107 DCHECK(!new_sample_cb.is_null());
109 ChangeState(kParsingBoxes);
111 new_sample_cb_ = new_sample_cb;
112 decryption_key_source_ = decryption_key_source;
113 if (decryption_key_source)
117 void MP4MediaParser::Reset() {
125 DCHECK_NE(state_, kWaitingForInit);
127 ChangeState(kParsingBoxes);
132 DCHECK_NE(state_, kWaitingForInit);
134 if (state_ == kError)
137 queue_.Push(buf, size);
139 bool result, err =
false;
142 if (state_ == kParsingBoxes) {
143 result = ParseBox(&err);
145 DCHECK_EQ(kEmittingSamples, state_);
146 result = EnqueueSample(&err);
148 int64_t max_clear = runs_->GetMaxClearOffset() + moof_head_;
149 err = !ReadAndDiscardMDATsUntil(max_clear);
152 }
while (result && !err);
155 DLOG(ERROR) <<
"Error while parsing MP4";
166 scoped_ptr<File, FileCloser> file(
169 LOG(ERROR) <<
"Unable to open media file '" << file_path <<
"'";
172 if (!file->Seek(0)) {
173 LOG(WARNING) <<
"Filesystem does not support seeking on file '" << file_path
178 uint64_t file_position(0);
179 bool mdat_seen(
false);
181 const uint32_t kBoxHeaderReadSize(16);
182 std::vector<uint8_t> buffer(kBoxHeaderReadSize);
183 int64_t bytes_read = file->Read(&buffer[0], kBoxHeaderReadSize);
184 if (bytes_read == 0) {
185 LOG(ERROR) <<
"Could not find 'moov' box in file '" << file_path <<
"'";
188 if (bytes_read < kBoxHeaderReadSize) {
189 LOG(ERROR) <<
"Error reading media file '" << file_path <<
"'";
197 LOG(ERROR) <<
"Could not start box from file '" << file_path <<
"'";
200 if (box_type == FOURCC_mdat) {
202 }
else if (box_type == FOURCC_moov) {
208 if (!
Parse(&buffer[0], bytes_read)) {
209 LOG(ERROR) <<
"Error parsing mp4 file '" << file_path <<
"'";
212 uint64_t bytes_to_read = box_size - bytes_read;
213 buffer.resize(bytes_to_read);
214 while (bytes_to_read > 0) {
215 bytes_read = file->Read(&buffer[0], bytes_to_read);
216 if (bytes_read <= 0) {
217 LOG(ERROR) <<
"Error reading 'moov' contents from file '" << file_path
221 if (!
Parse(&buffer[0], bytes_read)) {
222 LOG(ERROR) <<
"Error parsing mp4 file '" << file_path <<
"'";
225 bytes_to_read -= bytes_read;
231 file_position += box_size;
232 if (!file->Seek(file_position)) {
233 LOG(ERROR) <<
"Error skipping box in mp4 file '" << file_path <<
"'";
240 bool MP4MediaParser::ParseBox(
bool* err) {
243 queue_.Peek(&buf, &size);
248 if (reader.get() == NULL)
251 if (reader->type() == FOURCC_mdat) {
255 NOTIMPLEMENTED() <<
" Files with MDAT before MOOV is not supported yet.";
261 mdat_tail_ = queue_.
head() + reader->size();
263 if (reader->type() == FOURCC_moov) {
264 *err = !ParseMoov(reader.get());
265 }
else if (reader->type() == FOURCC_moof) {
266 moof_head_ = queue_.
head();
267 *err = !ParseMoof(reader.get());
275 VLOG(2) <<
"Skipping top-level box: " << FourCCToString(reader->type());
278 queue_.Pop(reader->size());
282 bool MP4MediaParser::ParseMoov(BoxReader* reader) {
286 moov_.reset(
new Movie);
287 RCHECK(moov_->Parse(reader));
290 std::vector<scoped_refptr<StreamInfo> > streams;
292 for (std::vector<Track>::const_iterator track = moov_->tracks.begin();
293 track != moov_->tracks.end(); ++track) {
294 const uint32_t timescale = track->media.header.timescale;
297 uint64_t duration = 0;
298 if (track->media.header.duration > 0) {
299 duration = track->media.header.duration;
300 }
else if (moov_->extends.header.fragment_duration > 0) {
301 DCHECK(moov_->header.timescale != 0);
302 duration = Rescale(moov_->extends.header.fragment_duration,
303 moov_->header.timescale,
305 }
else if (moov_->header.duration > 0 &&
306 moov_->header.duration != std::numeric_limits<uint64_t>::max()) {
307 DCHECK(moov_->header.timescale != 0);
309 Rescale(moov_->header.duration, moov_->header.timescale, timescale);
312 const SampleDescription& samp_descr =
313 track->media.information.sample_table.description;
319 if (moov_->extends.tracks.size() > 0) {
320 for (
size_t t = 0; t < moov_->extends.tracks.size(); t++) {
321 const TrackExtends& trex = moov_->extends.tracks[t];
322 if (trex.track_id == track->header.track_id) {
323 desc_idx = trex.default_sample_description_index;
328 const std::vector<ChunkInfo>& chunk_info =
329 track->media.information.sample_table.sample_to_chunk.chunk_info;
330 RCHECK(chunk_info.size() > 0);
331 desc_idx = chunk_info[0].sample_description_index;
333 RCHECK(desc_idx > 0);
336 if (samp_descr.type == kAudio) {
337 RCHECK(!samp_descr.audio_entries.empty());
341 if (desc_idx >= samp_descr.audio_entries.size())
344 const AudioSampleEntry& entry = samp_descr.audio_entries[desc_idx];
345 const FourCC actual_format = entry.GetActualFormat();
346 AudioCodec codec = FourCCToAudioCodec(actual_format);
347 uint8_t num_channels = 0;
348 uint32_t sampling_frequency = 0;
349 uint64_t codec_delay_ns = 0;
350 uint8_t audio_object_type = 0;
351 uint32_t max_bitrate = 0;
352 uint32_t avg_bitrate = 0;
353 std::vector<uint8_t> extra_data;
355 switch (actual_format) {
359 if (entry.esds.es_descriptor.IsAAC()) {
361 const AACAudioSpecificConfig& aac_audio_specific_config =
362 entry.esds.aac_audio_specific_config;
363 num_channels = aac_audio_specific_config.num_channels();
364 sampling_frequency = aac_audio_specific_config.frequency();
365 audio_object_type = aac_audio_specific_config.audio_object_type();
366 extra_data = entry.esds.es_descriptor.decoder_specific_info();
368 }
else if (entry.esds.es_descriptor.IsDTS()) {
369 ObjectType audio_type = entry.esds.es_descriptor.object_type();
370 switch (audio_type) {
384 LOG(ERROR) <<
"Unsupported audio type " << audio_type
388 num_channels = entry.esds.aac_audio_specific_config.num_channels();
391 if (num_channels != kDtsAudioNumChannels) {
392 LOG(ERROR) <<
"Unsupported channel count " << num_channels
393 <<
" for audio type " << audio_type <<
".";
396 sampling_frequency = entry.samplerate;
397 max_bitrate = entry.esds.es_descriptor.max_bitrate();
398 avg_bitrate = entry.esds.es_descriptor.avg_bitrate();
400 LOG(ERROR) <<
"Unsupported audio format 0x" << std::hex
401 << actual_format <<
" in stsd box.";
406 FALLTHROUGH_INTENDED;
408 FALLTHROUGH_INTENDED;
410 FALLTHROUGH_INTENDED;
412 FALLTHROUGH_INTENDED;
414 extra_data = entry.ddts.extra_data;
415 max_bitrate = entry.ddts.max_bitrate;
416 avg_bitrate = entry.ddts.avg_bitrate;
417 num_channels = entry.channelcount;
418 sampling_frequency = entry.samplerate;
421 extra_data = entry.dac3.data;
422 num_channels = entry.channelcount;
423 sampling_frequency = entry.samplerate;
426 extra_data = entry.dec3.data;
427 num_channels = entry.channelcount;
428 sampling_frequency = entry.samplerate;
431 extra_data = entry.dops.opus_identification_header;
432 num_channels = entry.channelcount;
433 sampling_frequency = entry.samplerate;
434 RCHECK(sampling_frequency != 0);
436 entry.dops.preskip * kNanosecondsPerSecond / sampling_frequency;
439 LOG(ERROR) <<
"Unsupported audio format 0x" << std::hex
440 << actual_format <<
" in stsd box.";
445 uint64_t seek_preroll_ns = 0;
446 for (
const auto& sample_group_description :
447 track->media.information.sample_table.sample_group_descriptions) {
448 if (sample_group_description.grouping_type != FOURCC_roll)
450 const auto& audio_roll_recovery_entries =
451 sample_group_description.audio_roll_recovery_entries;
452 if (audio_roll_recovery_entries.size() != 1) {
453 LOG(WARNING) <<
"Unexpected number of entries in "
454 "SampleGroupDescription table with grouping type "
458 const int16_t roll_distance_in_samples =
459 audio_roll_recovery_entries[0].roll_distance;
460 if (roll_distance_in_samples < 0) {
461 RCHECK(sampling_frequency != 0);
462 seek_preroll_ns = kNanosecondsPerSecond *
463 (-roll_distance_in_samples) / sampling_frequency;
466 <<
"Roll distance is supposed to be negative, but seeing "
467 << roll_distance_in_samples;
472 const bool is_encrypted =
473 entry.sinf.info.track_encryption.default_is_protected == 1;
474 DVLOG(1) <<
"is_audio_track_encrypted_: " << is_encrypted;
475 streams.push_back(
new AudioStreamInfo(
476 track->header.track_id,
481 track->media.header.language.code,
494 if (samp_descr.type == kVideo) {
495 RCHECK(!samp_descr.video_entries.empty());
496 if (desc_idx >= samp_descr.video_entries.size())
498 const VideoSampleEntry& entry = samp_descr.video_entries[desc_idx];
500 uint32_t coded_width = entry.width;
501 uint32_t coded_height = entry.height;
502 uint32_t pixel_width = entry.pixel_aspect.h_spacing;
503 uint32_t pixel_height = entry.pixel_aspect.v_spacing;
504 if (pixel_width == 0 && pixel_height == 0) {
508 std::string codec_string;
509 uint8_t nalu_length_size = 0;
511 const FourCC actual_format = entry.GetActualFormat();
512 const VideoCodec video_codec = FourCCToVideoCodec(actual_format);
513 switch (actual_format) {
515 AVCDecoderConfigurationRecord avc_config;
516 if (!avc_config.Parse(entry.codec_configuration.data)) {
517 LOG(ERROR) <<
"Failed to parse avcc.";
520 codec_string = avc_config.GetCodecString();
521 nalu_length_size = avc_config.nalu_length_size();
523 if (coded_width != avc_config.coded_width() ||
524 coded_height != avc_config.coded_height()) {
525 LOG(WARNING) <<
"Resolution in VisualSampleEntry (" << coded_width
526 <<
"," << coded_height
527 <<
") does not match with resolution in "
528 "AVCDecoderConfigurationRecord ("
529 << avc_config.coded_width() <<
","
530 << avc_config.coded_height()
531 <<
"). Use AVCDecoderConfigurationRecord.";
532 coded_width = avc_config.coded_width();
533 coded_height = avc_config.coded_height();
536 if (pixel_width != avc_config.pixel_width() ||
537 pixel_height != avc_config.pixel_height()) {
538 LOG_IF(WARNING, pixel_width != 1 || pixel_height != 1)
539 <<
"Pixel aspect ratio in PASP box (" << pixel_width <<
","
541 <<
") does not match with SAR in AVCDecoderConfigurationRecord "
543 << avc_config.pixel_width() <<
"," << avc_config.pixel_height()
544 <<
"). Use AVCDecoderConfigurationRecord.";
545 pixel_width = avc_config.pixel_width();
546 pixel_height = avc_config.pixel_height();
552 HEVCDecoderConfigurationRecord hevc_config;
553 if (!hevc_config.Parse(entry.codec_configuration.data)) {
554 LOG(ERROR) <<
"Failed to parse hevc.";
557 codec_string = hevc_config.GetCodecString(video_codec);
558 nalu_length_size = hevc_config.nalu_length_size();
564 VPCodecConfigurationRecord vp_config;
565 if (!vp_config.ParseMP4(entry.codec_configuration.data)) {
566 LOG(ERROR) <<
"Failed to parse vpcc.";
569 codec_string = vp_config.GetCodecString(video_codec);
573 LOG(ERROR) <<
"Unsupported video format "
574 << FourCCToString(actual_format) <<
" in stsd box.";
578 const bool is_encrypted =
579 entry.sinf.info.track_encryption.default_is_protected == 1;
580 DVLOG(1) <<
"is_video_track_encrypted_: " << is_encrypted;
581 streams.push_back(
new VideoStreamInfo(
582 track->header.track_id, timescale, duration, video_codec,
583 codec_string, track->media.header.language.code, coded_width,
584 coded_height, pixel_width, pixel_height,
586 nalu_length_size, entry.codec_configuration.data.data(),
587 entry.codec_configuration.data.size(), is_encrypted));
591 init_cb_.Run(streams);
592 if (!FetchKeysIfNecessary(moov_->pssh))
594 runs_.reset(
new TrackRunIterator(moov_.get()));
595 RCHECK(runs_->Init());
596 ChangeState(kEmittingSamples);
600 bool MP4MediaParser::ParseMoof(BoxReader* reader) {
604 RCHECK(moof.Parse(reader));
606 runs_.reset(
new TrackRunIterator(moov_.get()));
607 RCHECK(runs_->Init(moof));
608 if (!FetchKeysIfNecessary(moof.pssh))
610 ChangeState(kEmittingSamples);
614 bool MP4MediaParser::FetchKeysIfNecessary(
615 const std::vector<ProtectionSystemSpecificHeader>& headers) {
620 if (!decryption_key_source_)
624 for (std::vector<ProtectionSystemSpecificHeader>::const_iterator iter =
625 headers.begin(); iter != headers.end(); ++iter) {
626 status = decryption_key_source_->
FetchKeys(iter->raw_box);
630 VLOG(1) <<
"Unable to fetch decryption keys: " << status
631 <<
", trying the next PSSH box";
638 LOG(ERROR) <<
"Error fetching decryption keys: " << status;
642 LOG(ERROR) <<
"No viable 'pssh' box found for content decryption.";
646 bool MP4MediaParser::EnqueueSample(
bool* err) {
647 if (!runs_->IsRunValid()) {
650 if (!queue_.
Trim(mdat_tail_))
653 ChangeState(kParsingBoxes);
657 if (!runs_->IsSampleValid()) {
666 queue_.Peek(&buf, &buf_size);
671 if (!runs_->is_audio() && !runs_->is_video())
681 if (runs_->AuxInfoNeedsToBeCached()) {
682 queue_.
PeekAt(runs_->aux_info_offset() + moof_head_, &buf, &buf_size);
683 if (buf_size < runs_->aux_info_size())
685 *err = !runs_->CacheAuxInfo(buf, buf_size);
689 int64_t sample_offset = runs_->sample_offset() + moof_head_;
690 queue_.
PeekAt(sample_offset, &buf, &buf_size);
691 if (buf_size < runs_->sample_size()) {
692 if (sample_offset < queue_.
head()) {
693 LOG(ERROR) <<
"Incorrect sample offset " << sample_offset
694 <<
" < " << queue_.
head();
701 buf, runs_->sample_size(), runs_->is_keyframe()));
702 if (runs_->is_encrypted()) {
703 if (!decryptor_source_) {
705 LOG(ERROR) <<
"Encrypted media sample encountered, but decryption is not "
710 scoped_ptr<DecryptConfig> decrypt_config = runs_->GetDecryptConfig();
711 if (!decrypt_config ||
712 !decryptor_source_->DecryptSampleBuffer(decrypt_config.get(),
713 stream_sample->writable_data(),
714 stream_sample->data_size())) {
716 LOG(ERROR) <<
"Cannot decrypt samples.";
721 stream_sample->set_dts(runs_->dts());
722 stream_sample->set_pts(runs_->cts());
723 stream_sample->set_duration(runs_->duration());
725 DVLOG(3) <<
"Pushing frame: "
726 <<
", key=" << runs_->is_keyframe()
727 <<
", dur=" << runs_->duration()
728 <<
", dts=" << runs_->dts()
729 <<
", cts=" << runs_->cts()
730 <<
", size=" << runs_->sample_size();
732 if (!new_sample_cb_.Run(runs_->track_id(), stream_sample)) {
734 LOG(ERROR) <<
"Failed to process the sample.";
738 runs_->AdvanceSample();
742 bool MP4MediaParser::ReadAndDiscardMDATsUntil(
const int64_t offset) {
744 while (mdat_tail_ < offset) {
747 queue_.
PeekAt(mdat_tail_, &buf, &size);
754 mdat_tail_ += box_sz;
756 queue_.
Trim(std::min(mdat_tail_, offset));
760 void MP4MediaParser::ChangeState(State new_state) {
761 DVLOG(2) <<
"Changing state: " << new_state;