Handle side data properly in webm

Change-Id: Ic8c81154ea66374df62a4a46d66c45b3035a7829

packager/media/formats/mp4/fragmenter.cc

@@ -43,6 +43,9 @@ Status Fragmenter::AddSample(scoped_refptr<MediaSample> sample) {
       return status;
   }
 
+  if (sample->side_data_size() > 0)
+    LOG(WARNING) << "MP4 samples do not support side data. Side data ignored.";
+
   // Fill in sample parameters. It will be optimized later.
   traf_->runs[0].sample_sizes.push_back(sample->data_size());
   traf_->runs[0].sample_durations.push_back(sample->duration());

packager/media/formats/webm/multi_segment_segmenter_unittest.cc

@@ -124,7 +124,8 @@ TEST_F(MultiSegmentSegmenterTest, BasicSupport) {
 
   // Write the samples to the Segmenter.
   for (int i = 0; i < 5; i++) {
-    scoped_refptr<MediaSample> sample = CreateSample(true, kDuration);
+    scoped_refptr<MediaSample> sample =
+        CreateSample(kKeyFrame, kDuration, kNoSideData);
     ASSERT_OK(segmenter_->AddSample(sample));
   }
   ASSERT_OK(segmenter_->Finalize());
@@ -144,7 +145,8 @@ TEST_F(MultiSegmentSegmenterTest, SplitsFilesOnSegmentDuration) {
 
   // Write the samples to the Segmenter.
   for (int i = 0; i < 8; i++) {
-    scoped_refptr<MediaSample> sample = CreateSample(true, kDuration);
+    scoped_refptr<MediaSample> sample =
+        CreateSample(kKeyFrame, kDuration, kNoSideData);
     ASSERT_OK(segmenter_->AddSample(sample));
   }
   ASSERT_OK(segmenter_->Finalize());
@@ -170,7 +172,9 @@ TEST_F(MultiSegmentSegmenterTest, RespectsSegmentSAPAlign) {
 
   // Write the samples to the Segmenter.
   for (int i = 0; i < 10; i++) {
-    scoped_refptr<MediaSample> sample = CreateSample(i == 6, kDuration);
+    const KeyFrameFlag key_frame_flag = i == 6 ? kKeyFrame : kNotKeyFrame;
+    scoped_refptr<MediaSample> sample =
+        CreateSample(key_frame_flag, kDuration, kNoSideData);
     ASSERT_OK(segmenter_->AddSample(sample));
   }
   ASSERT_OK(segmenter_->Finalize());
@@ -195,7 +199,8 @@ TEST_F(MultiSegmentSegmenterTest, SplitsClustersOnFragmentDuration) {
 
   // Write the samples to the Segmenter.
   for (int i = 0; i < 8; i++) {
-    scoped_refptr<MediaSample> sample = CreateSample(true, kDuration);
+    scoped_refptr<MediaSample> sample =
+        CreateSample(kKeyFrame, kDuration, kNoSideData);
     ASSERT_OK(segmenter_->AddSample(sample));
   }
   ASSERT_OK(segmenter_->Finalize());
@@ -218,7 +223,9 @@ TEST_F(MultiSegmentSegmenterTest, RespectsFragmentSAPAlign) {
 
   // Write the samples to the Segmenter.
   for (int i = 0; i < 10; i++) {
-    scoped_refptr<MediaSample> sample = CreateSample(i == 6, kDuration);
+    const KeyFrameFlag key_frame_flag = i == 6 ? kKeyFrame : kNotKeyFrame;
+    scoped_refptr<MediaSample> sample =
+        CreateSample(key_frame_flag, kDuration, kNoSideData);
     ASSERT_OK(segmenter_->AddSample(sample));
   }
   ASSERT_OK(segmenter_->Finalize());

packager/media/formats/webm/segmenter.cc

@@ -122,11 +122,28 @@ Status Segmenter::AddSample(scoped_refptr<MediaSample> sample) {
 
   const int64_t time_ns =
       sample->pts() * kSecondsToNs / info_->time_scale();
-  if (!cluster_->AddFrame(sample->data(), sample->data_size(), track_id_,
+  bool addframe_result;
-                          time_ns, sample->is_key_frame())) {
+  if (sample->side_data_size() > 0) {
+    uint64_t block_add_id;
+    // First 8 bytes of side_data is the BlockAddID element's value, which is
+    // done to mimic ffmpeg behavior. See webm_cluster_parser.cc for details.
+    CHECK_GT(sample->side_data_size(), sizeof(block_add_id));
+    memcpy(&block_add_id, sample->side_data(), sizeof(block_add_id));
+    addframe_result = cluster_->AddFrameWithAdditional(
+        sample->data(), sample->data_size(),
+        sample->side_data() + sizeof(block_add_id),
+        sample->side_data_size() - sizeof(block_add_id), block_add_id,
+        track_id_, time_ns, sample->is_key_frame());
+  } else {
+    addframe_result =
+        cluster_->AddFrame(sample->data(), sample->data_size(), track_id_,
+                           time_ns, sample->is_key_frame());
+  }
+  if (!addframe_result) {
     LOG(ERROR) << "Error adding sample to segment.";
     return Status(error::FILE_FAILURE, "Error adding sample to segment.");
   }
+
   const double duration_sec =
       static_cast<double>(sample->duration()) / info_->time_scale();
   cluster_length_sec_ += duration_sec;

packager/media/formats/webm/segmenter_test_base.cc

@@ -15,8 +15,11 @@ namespace media {
 namespace {
 
 // The contents of a frame does not mater.
-const uint8_t kTestMediaSampleData[] = {0xDE, 0xAD, 0xBE, 0xEF, 0x00};
+const uint8_t kTestMediaSampleData[] = {0xde, 0xad, 0xbe, 0xef, 0x00};
-const size_t kTestMediaSampleDataSize = sizeof(kTestMediaSampleData);
+const uint8_t kTestMediaSampleSideData[] = {
+    // First 8 bytes of side_data is the BlockAddID element in big endian.
+    0x12, 0x34, 0x56, 0x78, 0x9a, 0x00, 0x00, 0x00,
+    0x73, 0x69, 0x64, 0x65, 0x00};
 
 const int kTrackId = 1;
 const uint32_t kTimeScale = 1000;
@@ -46,10 +49,21 @@ void SegmentTestBase::TearDown() {
   MemoryFile::DeleteAll();
 }
 
-scoped_refptr<MediaSample> SegmentTestBase::CreateSample(bool is_key_frame,
+scoped_refptr<MediaSample> SegmentTestBase::CreateSample(
-                                                         uint64_t duration) {
+    KeyFrameFlag key_frame_flag,
-  scoped_refptr<MediaSample> sample = MediaSample::CopyFrom(
+    uint64_t duration,
-      kTestMediaSampleData, kTestMediaSampleDataSize, is_key_frame);
+    SideDataFlag side_data_flag) {
+  scoped_refptr<MediaSample> sample;
+  const bool is_key_frame = key_frame_flag == kKeyFrame;
+  if (side_data_flag == kGenerateSideData) {
+    sample = MediaSample::CopyFrom(
+        kTestMediaSampleData, sizeof(kTestMediaSampleData),
+        kTestMediaSampleSideData, sizeof(kTestMediaSampleSideData),
+        is_key_frame);
+  } else {
+    sample = MediaSample::CopyFrom(kTestMediaSampleData,
+                                   sizeof(kTestMediaSampleData), is_key_frame);
+  }
   sample->set_dts(cur_time_timescale_);
   sample->set_pts(cur_time_timescale_);
   sample->set_duration(duration);
@@ -170,8 +184,8 @@ bool SegmentTestBase::ClusterParser::OnFloat(int id, double val) {
 bool SegmentTestBase::ClusterParser::OnBinary(int id,
                                              const uint8_t* data,
                                              int size) {
-  if (in_cluster_ && id == kWebMIdSimpleBlock) {
+  if (in_cluster_ && (id == kWebMIdSimpleBlock || id == kWebMIdBlock)) {
-    cluster_sizes_[cluster_sizes_.size() - 1]++;
+    cluster_sizes_.back()++;
   }
 
   return true;

packager/media/formats/webm/segmenter_test_base.h

@@ -26,6 +26,16 @@ namespace edash_packager {
 namespace media {
 
 class SegmentTestBase : public ::testing::Test {
+ public:
+  enum KeyFrameFlag {
+    kKeyFrame,
+    kNotKeyFrame,
+  };
+  enum SideDataFlag {
+    kGenerateSideData,
+    kNoSideData,
+  };
+
  protected:
   SegmentTestBase();
 
@@ -46,7 +56,9 @@ class SegmentTestBase : public ::testing::Test {
   }
 
   /// Creates a new media sample.
-  scoped_refptr<MediaSample> CreateSample(bool is_key_frame, uint64_t duration);
+  scoped_refptr<MediaSample> CreateSample(KeyFrameFlag key_frame_flag,
+                                          uint64_t duration,
+                                          SideDataFlag side_data_flag);
   /// Creates a Muxer options object for testing.
   MuxerOptions CreateMuxerOptions() const;
   /// Creates a video stream info object for testing.

packager/media/formats/webm/single_segment_segmenter_unittest.cc

@@ -33,8 +33,8 @@ const uint8_t kBasicSupportData[] = {
     0x42, 0x87, 0x81, 0x02,
     // DocTypeReadVersion: 2
     0x42, 0x85, 0x81, 0x02,
-  // ID: Segment, Payload Size: 316
+  // ID: Segment, Payload Size: 337
-  0x18, 0x53, 0x80, 0x67, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x3c,
+  0x18, 0x53, 0x80, 0x67, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x51,
     // ID: SeekHead, Payload Size: 30
     0x11, 0x4d, 0x9b, 0x74, 0x9e,
       // ID: Seek, Payload Size: 12
@@ -47,8 +47,8 @@ const uint8_t kBasicSupportData[] = {
       0x4d, 0xbb, 0x8c,
         // SeekID: binary(4) (Cues)
         0x53, 0xab, 0x84, 0x1c, 0x53, 0xbb, 0x6b,
-        // SeekPosition: 346
+        // SeekPosition: 367
-        0x53, 0xac, 0x82, 0x01, 0x5a,
+        0x53, 0xac, 0x82, 0x01, 0x6f,
     // ID: Void, Payload Size: 52
     0xec, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -95,8 +95,8 @@ const uint8_t kBasicSupportData[] = {
           0x54, 0xb0, 0x81, 0x64,
           // DisplayHeight: 100
           0x54, 0xba, 0x81, 0x64,
-    // ID: Cluster, Payload Size: 58
+    // ID: Cluster, Payload Size: 79
-    0x1f, 0x43, 0xb6, 0x75, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3a,
+    0x1f, 0x43, 0xb6, 0x75, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4f,
       // Timecode: 0
       0xe7, 0x81, 0x00,
       // ID: SimpleBlock, Payload Size: 9
@@ -105,8 +105,18 @@ const uint8_t kBasicSupportData[] = {
       0xa3, 0x89, 0x81, 0x03, 0xe8, 0x80, 0xde, 0xad, 0xbe, 0xef, 0x00,
       // ID: SimpleBlock, Payload Size: 9
       0xa3, 0x89, 0x81, 0x07, 0xd0, 0x80, 0xde, 0xad, 0xbe, 0xef, 0x00,
-      // ID: SimpleBlock, Payload Size: 9
+      // ID: BlockGroup, Payload Size: 30
-      0xa3, 0x89, 0x81, 0x0b, 0xb8, 0x80, 0xde, 0xad, 0xbe, 0xef, 0x00,
+      0xa0, 0x9e,
+        // ID: Block, Payload Size: 9
+        0xa1, 0x89, 0x81, 0x0b, 0xb8, 0x00, 0xde, 0xad, 0xbe, 0xef, 0x00,
+        // ID: BlockAdditions, Payload Size: 16
+        0x75, 0xa1, 0x90,
+          // ID: BlockMore, Payload Size: 14
+          0xa6, 0x8e,
+            // ID: BlockAddID, Payload Size: 1
+            0xee, 0x85, 0x9a, 0x78, 0x56, 0x34, 0x12,
+            // ID: BlockAdditional, Payload Size: 5
+            0xa5, 0x85, 0x73, 0x69, 0x64, 0x65, 0x00,
       // ID: SimpleBlock, Payload Size: 9
       0xa3, 0x89, 0x81, 0x0f, 0xa0, 0x80, 0xde, 0xad, 0xbe, 0xef, 0x00,
     // ID: Cues, Payload Size: 13
@@ -156,7 +166,10 @@ TEST_P(SingleSegmentSegmenterTest, BasicSupport) {
 
   // Write the samples to the Segmenter.
   for (int i = 0; i < 5; i++) {
-    scoped_refptr<MediaSample> sample = CreateSample(true, kDuration);
+    const SideDataFlag side_data_flag =
+        i == 3 ? kGenerateSideData : kNoSideData;
+    scoped_refptr<MediaSample> sample =
+        CreateSample(kKeyFrame, kDuration, side_data_flag);
     ASSERT_OK(segmenter_->AddSample(sample));
   }
   ASSERT_OK(segmenter_->Finalize());
@@ -171,7 +184,8 @@ TEST_P(SingleSegmentSegmenterTest, SplitsClustersOnSegmentDuration) {
 
   // Write the samples to the Segmenter.
   for (int i = 0; i < 8; i++) {
-    scoped_refptr<MediaSample> sample = CreateSample(true, kDuration);
+    scoped_refptr<MediaSample> sample =
+        CreateSample(kKeyFrame, kDuration, kNoSideData);
     ASSERT_OK(segmenter_->AddSample(sample));
   }
   ASSERT_OK(segmenter_->Finalize());
@@ -191,7 +205,8 @@ TEST_P(SingleSegmentSegmenterTest, IgnoresFragmentDuration) {
 
   // Write the samples to the Segmenter.
   for (int i = 0; i < 8; i++) {
-    scoped_refptr<MediaSample> sample = CreateSample(true, kDuration);
+    scoped_refptr<MediaSample> sample =
+        CreateSample(kKeyFrame, kDuration, kNoSideData);
     ASSERT_OK(segmenter_->AddSample(sample));
   }
   ASSERT_OK(segmenter_->Finalize());
@@ -211,7 +226,9 @@ TEST_P(SingleSegmentSegmenterTest, RespectsSAPAlign) {
 
   // Write the samples to the Segmenter.
   for (int i = 0; i < 10; i++) {
-    scoped_refptr<MediaSample> sample = CreateSample(i == 6, kDuration);
+    const KeyFrameFlag key_frame_flag = i == 6 ? kKeyFrame : kNotKeyFrame;
+    scoped_refptr<MediaSample> sample =
+        CreateSample(key_frame_flag, kDuration, kNoSideData);
     ASSERT_OK(segmenter_->AddSample(sample));
   }
   ASSERT_OK(segmenter_->Finalize());