Update NalUnitToByteStreamConvert to support escaping encrypted NAL only

- Also fixed a bug if nalu_length_size != 4.

Change-Id: Ia8c67c4b1ff08a6b64fa1e309b5ecd8f5c8ea421

packager/media/codecs/nal_unit_to_byte_stream_converter.cc

@@ -21,6 +21,7 @@ namespace media {
 
 namespace {
 
+const bool kEscapeData = true;
 const uint8_t kNaluStartCode[] = {0x00, 0x00, 0x00, 0x01};
 
 const uint8_t kEmulationPreventionByte = 0x03;
@@ -29,9 +30,15 @@ const uint8_t kAccessUnitDelimiterRbspAnyPrimaryPicType = 0xF0;
 
 void AppendNalu(const Nalu& nalu,
                 int nalu_length_size,
+                bool escape_data,
                 BufferWriter* buffer_writer) {
+  if (escape_data) {
+    EscapeNalByteSequence(nalu.data(), nalu.header_size() + nalu.payload_size(),
+                          buffer_writer);
+  } else {
     buffer_writer->AppendArray(nalu.data(),
                                nalu.header_size() + nalu.payload_size());
+  }
 }
 
 void AddAccessUnitDelimiter(BufferWriter* buffer_writer) {
@@ -40,7 +47,7 @@ void AddAccessUnitDelimiter(BufferWriter* buffer_writer) {
   buffer_writer->AppendInt(kAccessUnitDelimiterRbspAnyPrimaryPicType);
 }
 
-bool CheckSubsampleValid(const std::vector<SubsampleEntry>* subsamples,
+bool CheckIsClearNalu(const std::vector<SubsampleEntry>* subsamples,
                          size_t subsample_id,
                          size_t nalu_size,
                          bool* is_nalu_all_clear) {
@@ -133,11 +140,11 @@ bool NalUnitToByteStreamConverter::Initialize(
     const Nalu& nalu = decoder_config.nalu(i);
     if (nalu.type() == Nalu::H264NaluType::H264_SPS) {
       buffer_writer.AppendArray(kNaluStartCode, arraysize(kNaluStartCode));
-      AppendNalu(nalu, nalu_length_size_, &buffer_writer);
+      AppendNalu(nalu, nalu_length_size_, !kEscapeData, &buffer_writer);
       found_sps = true;
     } else if (nalu.type() == Nalu::H264NaluType::H264_PPS) {
       buffer_writer.AppendArray(kNaluStartCode, arraysize(kNaluStartCode));
-      AppendNalu(nalu, nalu_length_size_, &buffer_writer);
+      AppendNalu(nalu, nalu_length_size_, !kEscapeData, &buffer_writer);
       found_pps = true;
     }
   }
@@ -157,8 +164,9 @@ bool NalUnitToByteStreamConverter::ConvertUnitToByteStream(
     size_t sample_size,
     bool is_key_frame,
     std::vector<uint8_t>* output) {
+  LOG(INFO) << "ConvertUnitToByte";
   return ConvertUnitToByteStreamWithSubsamples(
-      sample, sample_size, is_key_frame, output,
+      sample, sample_size, is_key_frame, false, output,
       nullptr);  // Skip subsample update.
 }
 
@@ -168,6 +176,7 @@ bool NalUnitToByteStreamConverter::ConvertUnitToByteStreamWithSubsamples(
     const uint8_t* sample,
     size_t sample_size,
     bool is_key_frame,
+    bool escape_encrypted_nalu,
     std::vector<uint8_t>* output,
     std::vector<SubsampleEntry>* subsamples) {
   if (!sample || sample_size == 0) {
@@ -195,11 +204,11 @@ bool NalUnitToByteStreamConverter::ConvertUnitToByteStreamWithSubsamples(
       case Nalu::H264_SPS:
         FALLTHROUGH_INTENDED;
       case Nalu::H264_PPS:
-        if (subsamples) {
+        if (subsamples && !subsamples->empty()) {
           const size_t old_nalu_size =
               nalu_length_size_ + nalu.header_size() + nalu.payload_size();
           bool is_nalu_all_clear;
-          if (!CheckSubsampleValid(subsamples, subsample_id, old_nalu_size,
+          if (!CheckIsClearNalu(subsamples, subsample_id, old_nalu_size,
                                 &is_nalu_all_clear)) {
             return false;
           }
@@ -215,14 +224,12 @@ bool NalUnitToByteStreamConverter::ConvertUnitToByteStreamWithSubsamples(
         }
         break;
       default:
-        buffer_writer.AppendArray(kNaluStartCode, arraysize(kNaluStartCode));
+        bool escape_data = false;
-        AppendNalu(nalu, nalu_length_size_, &buffer_writer);
+        if (subsamples && !subsamples->empty()) {
-
-        if (subsamples) {
           const size_t old_nalu_size =
               nalu_length_size_ + nalu.header_size() + nalu.payload_size();
           bool is_nalu_all_clear;
-          if (!CheckSubsampleValid(subsamples, subsample_id, old_nalu_size,
+          if (!CheckIsClearNalu(subsamples, subsample_id, old_nalu_size,
                                 &is_nalu_all_clear)) {
             return false;
           }
@@ -231,15 +238,21 @@ bool NalUnitToByteStreamConverter::ConvertUnitToByteStreamWithSubsamples(
             DCHECK_LT(old_nalu_size, subsamples->at(subsample_id).clear_bytes);
             subsamples->at(subsample_id).clear_bytes -=
                 static_cast<uint16_t>(old_nalu_size);
-            adjustment += static_cast<int>(old_nalu_size);
+            adjustment += static_cast<int>(old_nalu_size) +
+                          arraysize(kNaluStartCode) - nalu_length_size_;
           } else {
+            if (escape_encrypted_nalu)
+              escape_data = subsamples->at(subsample_id).cipher_bytes != 0;
             // Apply the adjustment on the current subsample, reset the
             // adjustment and move to the next subsample.
-            subsamples->at(subsample_id).clear_bytes += adjustment;
+            subsamples->at(subsample_id).clear_bytes +=
+                adjustment + arraysize(kNaluStartCode) - nalu_length_size_;
             subsample_id++;
             adjustment = 0;
           }
         }
+        buffer_writer.AppendArray(kNaluStartCode, arraysize(kNaluStartCode));
+        AppendNalu(nalu, nalu_length_size_, escape_data, &buffer_writer);
         break;
     }
 

packager/media/codecs/nal_unit_to_byte_stream_converter.h

@@ -49,6 +49,7 @@ class NalUnitToByteStreamConverter {
   /// SAMPLE-AES encryption.
   /// @param sample is the sample to be converted.
   /// @param sample_size is the size of @a sample.
+  /// @param is_key_frame indicates if the sample is a key frame.
   /// @param[out] output is set to the the converted sample, on success.
   /// @return true on success, false otherwise.
   virtual bool ConvertUnitToByteStream(const uint8_t* sample,
@@ -62,14 +63,19 @@ class NalUnitToByteStreamConverter {
   /// SAMPLE-AES encryption.
   /// @param sample is the sample to be converted.
   /// @param sample_size is the size of @a sample.
+  /// @param is_key_frame indicates if the sample is a key frame.
+  /// @param escape_encrypted_nalu indicates whether an encrypted nalu should be
+  ///        escaped. This is needed for Apple Sample AES. Note that
+  ///        |subsamples| on return contains the sizes before escaping.
   /// @param[out] output is set to the the converted sample, on success.
   /// @param[in,out] subsamples has the input subsamples and output updated
-  ///                subsamples, on sucess.
+  ///                subsamples, on success.
   /// @return true on success, false otherwise.
   virtual bool ConvertUnitToByteStreamWithSubsamples(
       const uint8_t* sample,
       size_t sample_size,
       bool is_key_frame,
+      bool escape_encrypted_nalu,
       std::vector<uint8_t>* output,
       std::vector<SubsampleEntry>* subsamples);
 

packager/media/codecs/nal_unit_to_byte_stream_converter_unittest.cc

@@ -4,6 +4,7 @@
 // license that can be found in the LICENSE file or at
 // https://developers.google.com/open-source/licenses/bsd
 
+#include <gmock/gmock.h>
 #include <gtest/gtest.h>
 
 #include "packager/media/base/media_sample.h"
@@ -35,8 +36,27 @@ const uint8_t kTestAVCDecoderConfigurationRecord[] = {
     // The content of PPS is not checked except the type.
     0x68, 0xFE, 0xFD, 0xFC, 0xFB, 0x11, 0x12, 0x13, 0x14, 0x15,
 };
+const uint8_t kTestAVCDecoderConfigurationRecordNaluLengthSize2[] = {
+    0x01,        // configuration version (must be 1)
+    0x00,        // AVCProfileIndication (bogus)
+    0x00,        // profile_compatibility (bogus)
+    0x00,        // AVCLevelIndication (bogus)
+    0xFD,        // Length size minus 1 == 1
+    0xE1,        // 1 sps.
+    0x00, 0x1D,  // SPS length == 29
+    // Some valid SPS data.
+    0x67, 0x64, 0x00, 0x1E, 0xAC, 0xD9, 0x40, 0xB4,
+    0x2F, 0xF9, 0x7F, 0xF0, 0x00, 0x80, 0x00, 0x91,
+    0x00, 0x00, 0x03, 0x03, 0xE9, 0x00, 0x00, 0xEA,
+    0x60, 0x0F, 0x16, 0x2D, 0x96,
+    0x01,        // 1 pps.
+    0x00, 0x0A,  // PPS length == 10
+    // The content of PPS is not checked except the type.
+    0x68, 0xFE, 0xFD, 0xFC, 0xFB, 0x11, 0x12, 0x13, 0x14, 0x15,
+};
 
 const bool kIsKeyFrame = true;
+const bool kEscapeEncryptedNalu = true;
 
 }  // namespace
 
@@ -308,7 +328,7 @@ TEST(NalUnitToByteStreamConverterTest, NoClearNAL) {
   std::vector<uint8_t> output;
   EXPECT_TRUE(converter.ConvertUnitToByteStreamWithSubsamples(
       kUnitStreamLikeMediaSample, arraysize(kUnitStreamLikeMediaSample),
-      kIsKeyFrame, &output, &subsamples));
+      kIsKeyFrame, !kEscapeEncryptedNalu, &output, &subsamples));
 
   const uint8_t kExpectedOutput[] = {
       0x00, 0x00, 0x00, 0x01,              // Start code.
@@ -364,7 +384,7 @@ TEST(NalUnitToByteStreamConverterTest, WithSomeClearNAL) {
   std::vector<uint8_t> output;
   EXPECT_TRUE(converter.ConvertUnitToByteStreamWithSubsamples(
       kUnitStreamLikeMediaSample, arraysize(kUnitStreamLikeMediaSample),
-      kIsKeyFrame, &output, &subsamples));
+      kIsKeyFrame, !kEscapeEncryptedNalu, &output, &subsamples));
 
   const uint8_t kExpectedOutput[] = {
       0x00, 0x00, 0x00, 0x01,              // Start code.
@@ -397,7 +417,156 @@ TEST(NalUnitToByteStreamConverterTest, WithSomeClearNAL) {
   EXPECT_EQ(kExpectedOutputSubsamples, subsamples);
 }
 
-// A encrypted PPS NALU follows a clear NALU, the PPS will be removed. So the
+TEST(NalUnitToByteStreamConverterTest, WithSomeClearNALAndNaluLengthSize2) {
+  // Only the type of the NAL units are checked.
+  // This does not contain AUD, SPS, nor PPS.
+  const uint8_t kUnitStreamLikeMediaSample[] = {
+      0x00, 0x0A,  // Size 10 NALU.
+      0x06,        // NAL unit type.
+      0xFD, 0x78, 0xA4, 0xC3, 0x82, 0x62, 0x11,
+      0x29, 0x77,
+      0x00, 0x08,                                // Size 8 NALU.
+      0x02,                                      // NAL unit type.
+      0xFD, 0x78, 0xA4, 0x82, 0x62, 0x29, 0x77,  // Slice data
+  };
+
+  std::vector<SubsampleEntry> subsamples{SubsampleEntry(15, 7)};
+
+  NalUnitToByteStreamConverter converter;
+  EXPECT_TRUE(converter.Initialize(
+      kTestAVCDecoderConfigurationRecordNaluLengthSize2,
+      arraysize(kTestAVCDecoderConfigurationRecordNaluLengthSize2)));
+
+  std::vector<uint8_t> output;
+  EXPECT_TRUE(converter.ConvertUnitToByteStreamWithSubsamples(
+      kUnitStreamLikeMediaSample, arraysize(kUnitStreamLikeMediaSample),
+      kIsKeyFrame, !kEscapeEncryptedNalu, &output, &subsamples));
+
+  const uint8_t kExpectedOutput[] = {
+      0x00, 0x00, 0x00, 0x01,  // Start code.
+      0x09,                    // AUD type.
+      0xF0,                    // primary pic type is anything.
+      0x00, 0x00, 0x00, 0x01,  // Start code.
+      // Some valid SPS data.
+      0x67, 0x64, 0x00, 0x1E, 0xAC, 0xD9, 0x40, 0xB4, 0x2F, 0xF9, 0x7F, 0xF0,
+      0x00, 0x80, 0x00, 0x91, 0x00, 0x00, 0x03, 0x03, 0xE9, 0x00, 0x00, 0xEA,
+      0x60, 0x0F, 0x16, 0x2D, 0x96, 0x00, 0x00, 0x00, 0x01,  // Start code.
+      0x68, 0xFE, 0xFD, 0xFC, 0xFB, 0x11, 0x12, 0x13, 0x14, 0x15,  // PPS.
+      0x00, 0x00, 0x00, 0x01,  // Start code.
+      // The input NALU 1.
+      0x06,  // NALU type.
+      0xFD, 0x78, 0xA4, 0xC3, 0x82, 0x62, 0x11, 0x29, 0x77, 0x00, 0x00, 0x00,
+      0x01,  // Start code.
+      // The input NALU 2.
+      0x02,  // NALU type.
+      0xFD, 0x78, 0xA4, 0x82, 0x62, 0x29, 0x77,
+  };
+
+  const std::vector<SubsampleEntry> kExpectedOutputSubsamples{
+      SubsampleEntry(72, 7)};
+
+  EXPECT_EQ(std::vector<uint8_t>(kExpectedOutput,
+                                 kExpectedOutput + arraysize(kExpectedOutput)),
+            output);
+  EXPECT_EQ(kExpectedOutputSubsamples, subsamples);
+}
+
+TEST(NalUnitToByteStreamConverterTest, EscapeEncryptedNalu) {
+  // Only the type of the NAL units are checked.
+  // This does not contain AUD, SPS, nor PPS.
+  const uint8_t kUnitStreamLikeMediaSample[] = {
+      0x00, 0x00, 0x00, 0x0A,  // Size 10 NALU.
+      0x06,                    // NAL unit type.
+      // Unencrypted NALU with 0x000000 pattern (no need to escaped).
+      0xFD, 0x00, 0x00, 0x00, 0x82, 0x62, 0x11, 0x29, 0x77,
+      0x00, 0x00, 0x00, 0x08,  // Size 8 NALU.
+      0x02,  // NAL unit type.
+      // Encrypted NALU with 0x000000 pattern (need to escape).
+      0xFD, 0x00, 0x00, 0x00, 0x62, 0x29, 0x77,
+      0x00, 0x00, 0x00, 0x09,  // Size 9 NALU.
+      0x01,  // NAL unit types.
+      // Partially encrypted NALU with 0x000000 pattern at the boundary (need to
+      // escape).
+      0xFD, 0x01, 0x02, 0x00, 0x00, 0x01, 0x02, 0x03,
+  };
+
+  std::vector<SubsampleEntry> subsamples{
+      SubsampleEntry(19, 7), SubsampleEntry(9, 4), SubsampleEntry(7, 3)};
+
+  NalUnitToByteStreamConverter converter;
+  EXPECT_TRUE(
+      converter.Initialize(kTestAVCDecoderConfigurationRecord,
+                           arraysize(kTestAVCDecoderConfigurationRecord)));
+
+  std::vector<uint8_t> output;
+  ASSERT_TRUE(converter.ConvertUnitToByteStreamWithSubsamples(
+      kUnitStreamLikeMediaSample, arraysize(kUnitStreamLikeMediaSample),
+      !kIsKeyFrame, kEscapeEncryptedNalu, &output, &subsamples));
+
+  const uint8_t kExpectedOutput[] = {
+      0x00, 0x00, 0x00, 0x01,  // Start code.
+      0x09,                    // AUD type.
+      0xF0,                    // primary pic type is anything.
+      // The input NALU 1.
+      0x00, 0x00, 0x00, 0x01,  // Start code.
+      0x06,                    // NALU type.
+      0xFD, 0x00, 0x00, 0x00, 0x82, 0x62, 0x11, 0x29, 0x77,
+      0x00, 0x00, 0x00, 0x01,  // Start code.
+      // The input NALU 2.
+      0x02,  // NALU type.
+      0xFD, 0x00, 0x00, 0x03, 0x00, 0x62, 0x29, 0x77,
+      // The input NALU 3.
+      0x00, 0x00, 0x00, 0x01,  // Start code.
+      0x01,                    // NAL unit types.
+      0xFD, 0x01, 0x02, 0x00, 0x00, 0x03, 0x01, 0x02, 0x03,
+  };
+  EXPECT_EQ(std::vector<uint8_t>(std::begin(kExpectedOutput),
+                                 std::end(kExpectedOutput)),
+            output);
+  // The result subsample does not include emulation prevention bytes.
+  EXPECT_THAT(subsamples, ::testing::ElementsAre(SubsampleEntry(25, 7),
+                                                 SubsampleEntry(9, 4),
+                                                 SubsampleEntry(7, 3)));
+}
+
+TEST(NalUnitToByteStreamConverterTest, EncryptedNaluEndingWithZero) {
+  // Only the type of the NAL units are checked.
+  // This does not contain AUD, SPS, nor PPS.
+  const uint8_t kUnitStreamLikeMediaSample[] = {
+      0x00, 0x00, 0x00, 0x06,  // Size 6 NALU.
+      0x01,                    // NALU unit types.
+      // Encrypted NALU with 0x0003 pattern in the end (need to escape).
+      0xFD, 0x00, 0x01, 0x02, 0x00,
+  };
+
+  std::vector<SubsampleEntry> subsamples{SubsampleEntry(7, 3)};
+
+  NalUnitToByteStreamConverter converter;
+  EXPECT_TRUE(
+      converter.Initialize(kTestAVCDecoderConfigurationRecord,
+                           arraysize(kTestAVCDecoderConfigurationRecord)));
+
+  std::vector<uint8_t> output;
+  ASSERT_TRUE(converter.ConvertUnitToByteStreamWithSubsamples(
+      kUnitStreamLikeMediaSample, arraysize(kUnitStreamLikeMediaSample),
+      !kIsKeyFrame, kEscapeEncryptedNalu, &output, &subsamples));
+
+  const uint8_t kExpectedOutput[] = {
+      0x00, 0x00, 0x00, 0x01,  // Start code.
+      0x09,                    // AUD type.
+      0xF0,                    // primary pic type is anything.
+      0x00, 0x00, 0x00, 0x01,  // Start code.
+      0x01,                    // NALU unit types.
+      // Encrypted NALU with 0x0003 pattern in the end (need to escape).
+      0xFD, 0x00, 0x01, 0x02, 0x00, 0x03,
+  };
+  EXPECT_EQ(std::vector<uint8_t>(std::begin(kExpectedOutput),
+                                 std::end(kExpectedOutput)),
+            output);
+  // The result subsample does not include emulation prevention bytes.
+  EXPECT_THAT(subsamples, ::testing::ElementsAre(SubsampleEntry(13, 3)));
+}
+
 // corresponding subsample needs to be removed.
 TEST(NalUnitToByteStreamConverterTest, EncryptedPps) {
   // Only the type of the NAL units are checked.
@@ -426,7 +595,7 @@ TEST(NalUnitToByteStreamConverterTest, EncryptedPps) {
   std::vector<uint8_t> output;
   EXPECT_TRUE(converter.ConvertUnitToByteStreamWithSubsamples(
       kUnitStreamLikeMediaSample, arraysize(kUnitStreamLikeMediaSample),
-      kIsKeyFrame, &output, &subsamples));
+      kIsKeyFrame, !kEscapeEncryptedNalu, &output, &subsamples));
 
   const uint8_t kExpectedOutput[] = {
       0x00, 0x00, 0x00, 0x01,              // Start code.
@@ -456,7 +625,7 @@ TEST(NalUnitToByteStreamConverterTest, EncryptedPps) {
   EXPECT_EQ(std::vector<uint8_t>(kExpectedOutput,
                                  kExpectedOutput + arraysize(kExpectedOutput)),
             output);
-  EXPECT_EQ(kExpectedOutputSubsamples, subsamples);
+  EXPECT_THAT(kExpectedOutputSubsamples, subsamples);
 }
 
 // A clear PPS NALU follows a clear NALU, the PPS will be removed. So the
@@ -487,7 +656,7 @@ TEST(NalUnitToByteStreamConverterTest, ClearPps) {
   std::vector<uint8_t> output;
   EXPECT_TRUE(converter.ConvertUnitToByteStreamWithSubsamples(
       kUnitStreamLikeMediaSample, arraysize(kUnitStreamLikeMediaSample),
-      kIsKeyFrame, &output, &subsamples));
+      kIsKeyFrame, !kEscapeEncryptedNalu, &output, &subsamples));
 
   const uint8_t kExpectedOutput[] = {
       0x00, 0x00, 0x00, 0x01,              // Start code.