From a040fb711ceb8598a5fd762015f0c8128fbf7479 Mon Sep 17 00:00:00 2001 From: KongQun Yang Date: Thu, 7 Jan 2016 14:50:40 -0800 Subject: [PATCH] Handle side data properly in webm Change-Id: Ic8c81154ea66374df62a4a46d66c45b3035a7829 --- packager/media/formats/mp4/fragmenter.cc | 3 ++ .../webm/multi_segment_segmenter_unittest.cc | 17 +++++--- packager/media/formats/webm/segmenter.cc | 21 +++++++++- .../media/formats/webm/segmenter_test_base.cc | 30 ++++++++++---- .../media/formats/webm/segmenter_test_base.h | 14 ++++++- .../webm/single_segment_segmenter_unittest.cc | 41 +++++++++++++------ 6 files changed, 98 insertions(+), 28 deletions(-) diff --git a/packager/media/formats/mp4/fragmenter.cc b/packager/media/formats/mp4/fragmenter.cc index 7fa32fc8e4..ac6c28a466 100644 --- a/packager/media/formats/mp4/fragmenter.cc +++ b/packager/media/formats/mp4/fragmenter.cc @@ -43,6 +43,9 @@ Status Fragmenter::AddSample(scoped_refptr sample) { return status; } + if (sample->side_data_size() > 0) + LOG(WARNING) << "MP4 samples do not support side data. Side data ignored."; + // Fill in sample parameters. It will be optimized later. traf_->runs[0].sample_sizes.push_back(sample->data_size()); traf_->runs[0].sample_durations.push_back(sample->duration()); diff --git a/packager/media/formats/webm/multi_segment_segmenter_unittest.cc b/packager/media/formats/webm/multi_segment_segmenter_unittest.cc index b6328ef6ae..29e3b2f1ac 100644 --- a/packager/media/formats/webm/multi_segment_segmenter_unittest.cc +++ b/packager/media/formats/webm/multi_segment_segmenter_unittest.cc @@ -124,7 +124,8 @@ TEST_F(MultiSegmentSegmenterTest, BasicSupport) { // Write the samples to the Segmenter. for (int i = 0; i < 5; i++) { - scoped_refptr sample = CreateSample(true, kDuration); + scoped_refptr sample = + CreateSample(kKeyFrame, kDuration, kNoSideData); ASSERT_OK(segmenter_->AddSample(sample)); } ASSERT_OK(segmenter_->Finalize()); @@ -144,7 +145,8 @@ TEST_F(MultiSegmentSegmenterTest, SplitsFilesOnSegmentDuration) { // Write the samples to the Segmenter. for (int i = 0; i < 8; i++) { - scoped_refptr sample = CreateSample(true, kDuration); + scoped_refptr sample = + CreateSample(kKeyFrame, kDuration, kNoSideData); ASSERT_OK(segmenter_->AddSample(sample)); } ASSERT_OK(segmenter_->Finalize()); @@ -170,7 +172,9 @@ TEST_F(MultiSegmentSegmenterTest, RespectsSegmentSAPAlign) { // Write the samples to the Segmenter. for (int i = 0; i < 10; i++) { - scoped_refptr sample = CreateSample(i == 6, kDuration); + const KeyFrameFlag key_frame_flag = i == 6 ? kKeyFrame : kNotKeyFrame; + scoped_refptr sample = + CreateSample(key_frame_flag, kDuration, kNoSideData); ASSERT_OK(segmenter_->AddSample(sample)); } ASSERT_OK(segmenter_->Finalize()); @@ -195,7 +199,8 @@ TEST_F(MultiSegmentSegmenterTest, SplitsClustersOnFragmentDuration) { // Write the samples to the Segmenter. for (int i = 0; i < 8; i++) { - scoped_refptr sample = CreateSample(true, kDuration); + scoped_refptr sample = + CreateSample(kKeyFrame, kDuration, kNoSideData); ASSERT_OK(segmenter_->AddSample(sample)); } ASSERT_OK(segmenter_->Finalize()); @@ -218,7 +223,9 @@ TEST_F(MultiSegmentSegmenterTest, RespectsFragmentSAPAlign) { // Write the samples to the Segmenter. for (int i = 0; i < 10; i++) { - scoped_refptr sample = CreateSample(i == 6, kDuration); + const KeyFrameFlag key_frame_flag = i == 6 ? kKeyFrame : kNotKeyFrame; + scoped_refptr sample = + CreateSample(key_frame_flag, kDuration, kNoSideData); ASSERT_OK(segmenter_->AddSample(sample)); } ASSERT_OK(segmenter_->Finalize()); diff --git a/packager/media/formats/webm/segmenter.cc b/packager/media/formats/webm/segmenter.cc index a0f5dbde77..bed96e0f45 100644 --- a/packager/media/formats/webm/segmenter.cc +++ b/packager/media/formats/webm/segmenter.cc @@ -122,11 +122,28 @@ Status Segmenter::AddSample(scoped_refptr sample) { const int64_t time_ns = sample->pts() * kSecondsToNs / info_->time_scale(); - if (!cluster_->AddFrame(sample->data(), sample->data_size(), track_id_, - time_ns, sample->is_key_frame())) { + bool addframe_result; + if (sample->side_data_size() > 0) { + uint64_t block_add_id; + // First 8 bytes of side_data is the BlockAddID element's value, which is + // done to mimic ffmpeg behavior. See webm_cluster_parser.cc for details. + CHECK_GT(sample->side_data_size(), sizeof(block_add_id)); + memcpy(&block_add_id, sample->side_data(), sizeof(block_add_id)); + addframe_result = cluster_->AddFrameWithAdditional( + sample->data(), sample->data_size(), + sample->side_data() + sizeof(block_add_id), + sample->side_data_size() - sizeof(block_add_id), block_add_id, + track_id_, time_ns, sample->is_key_frame()); + } else { + addframe_result = + cluster_->AddFrame(sample->data(), sample->data_size(), track_id_, + time_ns, sample->is_key_frame()); + } + if (!addframe_result) { LOG(ERROR) << "Error adding sample to segment."; return Status(error::FILE_FAILURE, "Error adding sample to segment."); } + const double duration_sec = static_cast(sample->duration()) / info_->time_scale(); cluster_length_sec_ += duration_sec; diff --git a/packager/media/formats/webm/segmenter_test_base.cc b/packager/media/formats/webm/segmenter_test_base.cc index 6fb33f76d4..54100ac22b 100644 --- a/packager/media/formats/webm/segmenter_test_base.cc +++ b/packager/media/formats/webm/segmenter_test_base.cc @@ -15,8 +15,11 @@ namespace media { namespace { // The contents of a frame does not mater. -const uint8_t kTestMediaSampleData[] = {0xDE, 0xAD, 0xBE, 0xEF, 0x00}; -const size_t kTestMediaSampleDataSize = sizeof(kTestMediaSampleData); +const uint8_t kTestMediaSampleData[] = {0xde, 0xad, 0xbe, 0xef, 0x00}; +const uint8_t kTestMediaSampleSideData[] = { + // First 8 bytes of side_data is the BlockAddID element in big endian. + 0x12, 0x34, 0x56, 0x78, 0x9a, 0x00, 0x00, 0x00, + 0x73, 0x69, 0x64, 0x65, 0x00}; const int kTrackId = 1; const uint32_t kTimeScale = 1000; @@ -46,10 +49,21 @@ void SegmentTestBase::TearDown() { MemoryFile::DeleteAll(); } -scoped_refptr SegmentTestBase::CreateSample(bool is_key_frame, - uint64_t duration) { - scoped_refptr sample = MediaSample::CopyFrom( - kTestMediaSampleData, kTestMediaSampleDataSize, is_key_frame); +scoped_refptr SegmentTestBase::CreateSample( + KeyFrameFlag key_frame_flag, + uint64_t duration, + SideDataFlag side_data_flag) { + scoped_refptr sample; + const bool is_key_frame = key_frame_flag == kKeyFrame; + if (side_data_flag == kGenerateSideData) { + sample = MediaSample::CopyFrom( + kTestMediaSampleData, sizeof(kTestMediaSampleData), + kTestMediaSampleSideData, sizeof(kTestMediaSampleSideData), + is_key_frame); + } else { + sample = MediaSample::CopyFrom(kTestMediaSampleData, + sizeof(kTestMediaSampleData), is_key_frame); + } sample->set_dts(cur_time_timescale_); sample->set_pts(cur_time_timescale_); sample->set_duration(duration); @@ -170,8 +184,8 @@ bool SegmentTestBase::ClusterParser::OnFloat(int id, double val) { bool SegmentTestBase::ClusterParser::OnBinary(int id, const uint8_t* data, int size) { - if (in_cluster_ && id == kWebMIdSimpleBlock) { - cluster_sizes_[cluster_sizes_.size() - 1]++; + if (in_cluster_ && (id == kWebMIdSimpleBlock || id == kWebMIdBlock)) { + cluster_sizes_.back()++; } return true; diff --git a/packager/media/formats/webm/segmenter_test_base.h b/packager/media/formats/webm/segmenter_test_base.h index af1bb775b1..08903fd689 100644 --- a/packager/media/formats/webm/segmenter_test_base.h +++ b/packager/media/formats/webm/segmenter_test_base.h @@ -26,6 +26,16 @@ namespace edash_packager { namespace media { class SegmentTestBase : public ::testing::Test { + public: + enum KeyFrameFlag { + kKeyFrame, + kNotKeyFrame, + }; + enum SideDataFlag { + kGenerateSideData, + kNoSideData, + }; + protected: SegmentTestBase(); @@ -46,7 +56,9 @@ class SegmentTestBase : public ::testing::Test { } /// Creates a new media sample. - scoped_refptr CreateSample(bool is_key_frame, uint64_t duration); + scoped_refptr CreateSample(KeyFrameFlag key_frame_flag, + uint64_t duration, + SideDataFlag side_data_flag); /// Creates a Muxer options object for testing. MuxerOptions CreateMuxerOptions() const; /// Creates a video stream info object for testing. diff --git a/packager/media/formats/webm/single_segment_segmenter_unittest.cc b/packager/media/formats/webm/single_segment_segmenter_unittest.cc index 0033082c67..7681ab2c86 100644 --- a/packager/media/formats/webm/single_segment_segmenter_unittest.cc +++ b/packager/media/formats/webm/single_segment_segmenter_unittest.cc @@ -33,8 +33,8 @@ const uint8_t kBasicSupportData[] = { 0x42, 0x87, 0x81, 0x02, // DocTypeReadVersion: 2 0x42, 0x85, 0x81, 0x02, - // ID: Segment, Payload Size: 316 - 0x18, 0x53, 0x80, 0x67, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x3c, + // ID: Segment, Payload Size: 337 + 0x18, 0x53, 0x80, 0x67, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x51, // ID: SeekHead, Payload Size: 30 0x11, 0x4d, 0x9b, 0x74, 0x9e, // ID: Seek, Payload Size: 12 @@ -47,8 +47,8 @@ const uint8_t kBasicSupportData[] = { 0x4d, 0xbb, 0x8c, // SeekID: binary(4) (Cues) 0x53, 0xab, 0x84, 0x1c, 0x53, 0xbb, 0x6b, - // SeekPosition: 346 - 0x53, 0xac, 0x82, 0x01, 0x5a, + // SeekPosition: 367 + 0x53, 0xac, 0x82, 0x01, 0x6f, // ID: Void, Payload Size: 52 0xec, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -95,8 +95,8 @@ const uint8_t kBasicSupportData[] = { 0x54, 0xb0, 0x81, 0x64, // DisplayHeight: 100 0x54, 0xba, 0x81, 0x64, - // ID: Cluster, Payload Size: 58 - 0x1f, 0x43, 0xb6, 0x75, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3a, + // ID: Cluster, Payload Size: 79 + 0x1f, 0x43, 0xb6, 0x75, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4f, // Timecode: 0 0xe7, 0x81, 0x00, // ID: SimpleBlock, Payload Size: 9 @@ -105,8 +105,18 @@ const uint8_t kBasicSupportData[] = { 0xa3, 0x89, 0x81, 0x03, 0xe8, 0x80, 0xde, 0xad, 0xbe, 0xef, 0x00, // ID: SimpleBlock, Payload Size: 9 0xa3, 0x89, 0x81, 0x07, 0xd0, 0x80, 0xde, 0xad, 0xbe, 0xef, 0x00, - // ID: SimpleBlock, Payload Size: 9 - 0xa3, 0x89, 0x81, 0x0b, 0xb8, 0x80, 0xde, 0xad, 0xbe, 0xef, 0x00, + // ID: BlockGroup, Payload Size: 30 + 0xa0, 0x9e, + // ID: Block, Payload Size: 9 + 0xa1, 0x89, 0x81, 0x0b, 0xb8, 0x00, 0xde, 0xad, 0xbe, 0xef, 0x00, + // ID: BlockAdditions, Payload Size: 16 + 0x75, 0xa1, 0x90, + // ID: BlockMore, Payload Size: 14 + 0xa6, 0x8e, + // ID: BlockAddID, Payload Size: 1 + 0xee, 0x85, 0x9a, 0x78, 0x56, 0x34, 0x12, + // ID: BlockAdditional, Payload Size: 5 + 0xa5, 0x85, 0x73, 0x69, 0x64, 0x65, 0x00, // ID: SimpleBlock, Payload Size: 9 0xa3, 0x89, 0x81, 0x0f, 0xa0, 0x80, 0xde, 0xad, 0xbe, 0xef, 0x00, // ID: Cues, Payload Size: 13 @@ -156,7 +166,10 @@ TEST_P(SingleSegmentSegmenterTest, BasicSupport) { // Write the samples to the Segmenter. for (int i = 0; i < 5; i++) { - scoped_refptr sample = CreateSample(true, kDuration); + const SideDataFlag side_data_flag = + i == 3 ? kGenerateSideData : kNoSideData; + scoped_refptr sample = + CreateSample(kKeyFrame, kDuration, side_data_flag); ASSERT_OK(segmenter_->AddSample(sample)); } ASSERT_OK(segmenter_->Finalize()); @@ -171,7 +184,8 @@ TEST_P(SingleSegmentSegmenterTest, SplitsClustersOnSegmentDuration) { // Write the samples to the Segmenter. for (int i = 0; i < 8; i++) { - scoped_refptr sample = CreateSample(true, kDuration); + scoped_refptr sample = + CreateSample(kKeyFrame, kDuration, kNoSideData); ASSERT_OK(segmenter_->AddSample(sample)); } ASSERT_OK(segmenter_->Finalize()); @@ -191,7 +205,8 @@ TEST_P(SingleSegmentSegmenterTest, IgnoresFragmentDuration) { // Write the samples to the Segmenter. for (int i = 0; i < 8; i++) { - scoped_refptr sample = CreateSample(true, kDuration); + scoped_refptr sample = + CreateSample(kKeyFrame, kDuration, kNoSideData); ASSERT_OK(segmenter_->AddSample(sample)); } ASSERT_OK(segmenter_->Finalize()); @@ -211,7 +226,9 @@ TEST_P(SingleSegmentSegmenterTest, RespectsSAPAlign) { // Write the samples to the Segmenter. for (int i = 0; i < 10; i++) { - scoped_refptr sample = CreateSample(i == 6, kDuration); + const KeyFrameFlag key_frame_flag = i == 6 ? kKeyFrame : kNotKeyFrame; + scoped_refptr sample = + CreateSample(key_frame_flag, kDuration, kNoSideData); ASSERT_OK(segmenter_->AddSample(sample)); } ASSERT_OK(segmenter_->Finalize());