From 16615095f0313012071c007199b6bb1b1294ec38 Mon Sep 17 00:00:00 2001 From: KongQun Yang Date: Fri, 10 Mar 2017 18:49:45 -0800 Subject: [PATCH] Update NalUnitToByteStreamConvert to support escaping encrypted NAL only - Also fixed a bug if nalu_length_size != 4. Change-Id: Ia8c67c4b1ff08a6b64fa1e309b5ecd8f5c8ea421 --- .../nal_unit_to_byte_stream_converter.cc | 49 +++-- .../nal_unit_to_byte_stream_converter.h | 8 +- ..._unit_to_byte_stream_converter_unittest.cc | 181 +++++++++++++++++- 3 files changed, 213 insertions(+), 25 deletions(-) diff --git a/packager/media/codecs/nal_unit_to_byte_stream_converter.cc b/packager/media/codecs/nal_unit_to_byte_stream_converter.cc index db46e6d094..d33a7ce463 100644 --- a/packager/media/codecs/nal_unit_to_byte_stream_converter.cc +++ b/packager/media/codecs/nal_unit_to_byte_stream_converter.cc @@ -21,6 +21,7 @@ namespace media { namespace { +const bool kEscapeData = true; const uint8_t kNaluStartCode[] = {0x00, 0x00, 0x00, 0x01}; const uint8_t kEmulationPreventionByte = 0x03; @@ -29,9 +30,15 @@ const uint8_t kAccessUnitDelimiterRbspAnyPrimaryPicType = 0xF0; void AppendNalu(const Nalu& nalu, int nalu_length_size, + bool escape_data, BufferWriter* buffer_writer) { - buffer_writer->AppendArray(nalu.data(), - nalu.header_size() + nalu.payload_size()); + if (escape_data) { + EscapeNalByteSequence(nalu.data(), nalu.header_size() + nalu.payload_size(), + buffer_writer); + } else { + buffer_writer->AppendArray(nalu.data(), + nalu.header_size() + nalu.payload_size()); + } } void AddAccessUnitDelimiter(BufferWriter* buffer_writer) { @@ -40,7 +47,7 @@ void AddAccessUnitDelimiter(BufferWriter* buffer_writer) { buffer_writer->AppendInt(kAccessUnitDelimiterRbspAnyPrimaryPicType); } -bool CheckSubsampleValid(const std::vector* subsamples, +bool CheckIsClearNalu(const std::vector* subsamples, size_t subsample_id, size_t nalu_size, bool* is_nalu_all_clear) { @@ -133,11 +140,11 @@ bool NalUnitToByteStreamConverter::Initialize( const Nalu& nalu = decoder_config.nalu(i); if (nalu.type() == Nalu::H264NaluType::H264_SPS) { buffer_writer.AppendArray(kNaluStartCode, arraysize(kNaluStartCode)); - AppendNalu(nalu, nalu_length_size_, &buffer_writer); + AppendNalu(nalu, nalu_length_size_, !kEscapeData, &buffer_writer); found_sps = true; } else if (nalu.type() == Nalu::H264NaluType::H264_PPS) { buffer_writer.AppendArray(kNaluStartCode, arraysize(kNaluStartCode)); - AppendNalu(nalu, nalu_length_size_, &buffer_writer); + AppendNalu(nalu, nalu_length_size_, !kEscapeData, &buffer_writer); found_pps = true; } } @@ -157,9 +164,10 @@ bool NalUnitToByteStreamConverter::ConvertUnitToByteStream( size_t sample_size, bool is_key_frame, std::vector* output) { + LOG(INFO) << "ConvertUnitToByte"; return ConvertUnitToByteStreamWithSubsamples( - sample, sample_size, is_key_frame, output, - nullptr); // Skip subsample update. + sample, sample_size, is_key_frame, false, output, + nullptr); // Skip subsample update. } // This ignores all AUD, SPS, and PPS in the sample. Instead uses the data @@ -168,6 +176,7 @@ bool NalUnitToByteStreamConverter::ConvertUnitToByteStreamWithSubsamples( const uint8_t* sample, size_t sample_size, bool is_key_frame, + bool escape_encrypted_nalu, std::vector* output, std::vector* subsamples) { if (!sample || sample_size == 0) { @@ -195,12 +204,12 @@ bool NalUnitToByteStreamConverter::ConvertUnitToByteStreamWithSubsamples( case Nalu::H264_SPS: FALLTHROUGH_INTENDED; case Nalu::H264_PPS: - if (subsamples) { + if (subsamples && !subsamples->empty()) { const size_t old_nalu_size = nalu_length_size_ + nalu.header_size() + nalu.payload_size(); bool is_nalu_all_clear; - if (!CheckSubsampleValid(subsamples, subsample_id, old_nalu_size, - &is_nalu_all_clear)) { + if (!CheckIsClearNalu(subsamples, subsample_id, old_nalu_size, + &is_nalu_all_clear)) { return false; } if (is_nalu_all_clear) { @@ -215,15 +224,13 @@ bool NalUnitToByteStreamConverter::ConvertUnitToByteStreamWithSubsamples( } break; default: - buffer_writer.AppendArray(kNaluStartCode, arraysize(kNaluStartCode)); - AppendNalu(nalu, nalu_length_size_, &buffer_writer); - - if (subsamples) { + bool escape_data = false; + if (subsamples && !subsamples->empty()) { const size_t old_nalu_size = nalu_length_size_ + nalu.header_size() + nalu.payload_size(); bool is_nalu_all_clear; - if (!CheckSubsampleValid(subsamples, subsample_id, old_nalu_size, - &is_nalu_all_clear)) { + if (!CheckIsClearNalu(subsamples, subsample_id, old_nalu_size, + &is_nalu_all_clear)) { return false; } if (is_nalu_all_clear) { @@ -231,15 +238,21 @@ bool NalUnitToByteStreamConverter::ConvertUnitToByteStreamWithSubsamples( DCHECK_LT(old_nalu_size, subsamples->at(subsample_id).clear_bytes); subsamples->at(subsample_id).clear_bytes -= static_cast(old_nalu_size); - adjustment += static_cast(old_nalu_size); + adjustment += static_cast(old_nalu_size) + + arraysize(kNaluStartCode) - nalu_length_size_; } else { + if (escape_encrypted_nalu) + escape_data = subsamples->at(subsample_id).cipher_bytes != 0; // Apply the adjustment on the current subsample, reset the // adjustment and move to the next subsample. - subsamples->at(subsample_id).clear_bytes += adjustment; + subsamples->at(subsample_id).clear_bytes += + adjustment + arraysize(kNaluStartCode) - nalu_length_size_; subsample_id++; adjustment = 0; } } + buffer_writer.AppendArray(kNaluStartCode, arraysize(kNaluStartCode)); + AppendNalu(nalu, nalu_length_size_, escape_data, &buffer_writer); break; } diff --git a/packager/media/codecs/nal_unit_to_byte_stream_converter.h b/packager/media/codecs/nal_unit_to_byte_stream_converter.h index 8bb0cfd872..9dc5a116c9 100644 --- a/packager/media/codecs/nal_unit_to_byte_stream_converter.h +++ b/packager/media/codecs/nal_unit_to_byte_stream_converter.h @@ -49,6 +49,7 @@ class NalUnitToByteStreamConverter { /// SAMPLE-AES encryption. /// @param sample is the sample to be converted. /// @param sample_size is the size of @a sample. + /// @param is_key_frame indicates if the sample is a key frame. /// @param[out] output is set to the the converted sample, on success. /// @return true on success, false otherwise. virtual bool ConvertUnitToByteStream(const uint8_t* sample, @@ -62,14 +63,19 @@ class NalUnitToByteStreamConverter { /// SAMPLE-AES encryption. /// @param sample is the sample to be converted. /// @param sample_size is the size of @a sample. + /// @param is_key_frame indicates if the sample is a key frame. + /// @param escape_encrypted_nalu indicates whether an encrypted nalu should be + /// escaped. This is needed for Apple Sample AES. Note that + /// |subsamples| on return contains the sizes before escaping. /// @param[out] output is set to the the converted sample, on success. /// @param[in,out] subsamples has the input subsamples and output updated - /// subsamples, on sucess. + /// subsamples, on success. /// @return true on success, false otherwise. virtual bool ConvertUnitToByteStreamWithSubsamples( const uint8_t* sample, size_t sample_size, bool is_key_frame, + bool escape_encrypted_nalu, std::vector* output, std::vector* subsamples); diff --git a/packager/media/codecs/nal_unit_to_byte_stream_converter_unittest.cc b/packager/media/codecs/nal_unit_to_byte_stream_converter_unittest.cc index 94db57247a..078ea0520e 100644 --- a/packager/media/codecs/nal_unit_to_byte_stream_converter_unittest.cc +++ b/packager/media/codecs/nal_unit_to_byte_stream_converter_unittest.cc @@ -4,6 +4,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd +#include #include #include "packager/media/base/media_sample.h" @@ -35,8 +36,27 @@ const uint8_t kTestAVCDecoderConfigurationRecord[] = { // The content of PPS is not checked except the type. 0x68, 0xFE, 0xFD, 0xFC, 0xFB, 0x11, 0x12, 0x13, 0x14, 0x15, }; +const uint8_t kTestAVCDecoderConfigurationRecordNaluLengthSize2[] = { + 0x01, // configuration version (must be 1) + 0x00, // AVCProfileIndication (bogus) + 0x00, // profile_compatibility (bogus) + 0x00, // AVCLevelIndication (bogus) + 0xFD, // Length size minus 1 == 1 + 0xE1, // 1 sps. + 0x00, 0x1D, // SPS length == 29 + // Some valid SPS data. + 0x67, 0x64, 0x00, 0x1E, 0xAC, 0xD9, 0x40, 0xB4, + 0x2F, 0xF9, 0x7F, 0xF0, 0x00, 0x80, 0x00, 0x91, + 0x00, 0x00, 0x03, 0x03, 0xE9, 0x00, 0x00, 0xEA, + 0x60, 0x0F, 0x16, 0x2D, 0x96, + 0x01, // 1 pps. + 0x00, 0x0A, // PPS length == 10 + // The content of PPS is not checked except the type. + 0x68, 0xFE, 0xFD, 0xFC, 0xFB, 0x11, 0x12, 0x13, 0x14, 0x15, +}; const bool kIsKeyFrame = true; +const bool kEscapeEncryptedNalu = true; } // namespace @@ -308,7 +328,7 @@ TEST(NalUnitToByteStreamConverterTest, NoClearNAL) { std::vector output; EXPECT_TRUE(converter.ConvertUnitToByteStreamWithSubsamples( kUnitStreamLikeMediaSample, arraysize(kUnitStreamLikeMediaSample), - kIsKeyFrame, &output, &subsamples)); + kIsKeyFrame, !kEscapeEncryptedNalu, &output, &subsamples)); const uint8_t kExpectedOutput[] = { 0x00, 0x00, 0x00, 0x01, // Start code. @@ -364,7 +384,7 @@ TEST(NalUnitToByteStreamConverterTest, WithSomeClearNAL) { std::vector output; EXPECT_TRUE(converter.ConvertUnitToByteStreamWithSubsamples( kUnitStreamLikeMediaSample, arraysize(kUnitStreamLikeMediaSample), - kIsKeyFrame, &output, &subsamples)); + kIsKeyFrame, !kEscapeEncryptedNalu, &output, &subsamples)); const uint8_t kExpectedOutput[] = { 0x00, 0x00, 0x00, 0x01, // Start code. @@ -397,7 +417,156 @@ TEST(NalUnitToByteStreamConverterTest, WithSomeClearNAL) { EXPECT_EQ(kExpectedOutputSubsamples, subsamples); } -// A encrypted PPS NALU follows a clear NALU, the PPS will be removed. So the +TEST(NalUnitToByteStreamConverterTest, WithSomeClearNALAndNaluLengthSize2) { + // Only the type of the NAL units are checked. + // This does not contain AUD, SPS, nor PPS. + const uint8_t kUnitStreamLikeMediaSample[] = { + 0x00, 0x0A, // Size 10 NALU. + 0x06, // NAL unit type. + 0xFD, 0x78, 0xA4, 0xC3, 0x82, 0x62, 0x11, + 0x29, 0x77, + 0x00, 0x08, // Size 8 NALU. + 0x02, // NAL unit type. + 0xFD, 0x78, 0xA4, 0x82, 0x62, 0x29, 0x77, // Slice data + }; + + std::vector subsamples{SubsampleEntry(15, 7)}; + + NalUnitToByteStreamConverter converter; + EXPECT_TRUE(converter.Initialize( + kTestAVCDecoderConfigurationRecordNaluLengthSize2, + arraysize(kTestAVCDecoderConfigurationRecordNaluLengthSize2))); + + std::vector output; + EXPECT_TRUE(converter.ConvertUnitToByteStreamWithSubsamples( + kUnitStreamLikeMediaSample, arraysize(kUnitStreamLikeMediaSample), + kIsKeyFrame, !kEscapeEncryptedNalu, &output, &subsamples)); + + const uint8_t kExpectedOutput[] = { + 0x00, 0x00, 0x00, 0x01, // Start code. + 0x09, // AUD type. + 0xF0, // primary pic type is anything. + 0x00, 0x00, 0x00, 0x01, // Start code. + // Some valid SPS data. + 0x67, 0x64, 0x00, 0x1E, 0xAC, 0xD9, 0x40, 0xB4, 0x2F, 0xF9, 0x7F, 0xF0, + 0x00, 0x80, 0x00, 0x91, 0x00, 0x00, 0x03, 0x03, 0xE9, 0x00, 0x00, 0xEA, + 0x60, 0x0F, 0x16, 0x2D, 0x96, 0x00, 0x00, 0x00, 0x01, // Start code. + 0x68, 0xFE, 0xFD, 0xFC, 0xFB, 0x11, 0x12, 0x13, 0x14, 0x15, // PPS. + 0x00, 0x00, 0x00, 0x01, // Start code. + // The input NALU 1. + 0x06, // NALU type. + 0xFD, 0x78, 0xA4, 0xC3, 0x82, 0x62, 0x11, 0x29, 0x77, 0x00, 0x00, 0x00, + 0x01, // Start code. + // The input NALU 2. + 0x02, // NALU type. + 0xFD, 0x78, 0xA4, 0x82, 0x62, 0x29, 0x77, + }; + + const std::vector kExpectedOutputSubsamples{ + SubsampleEntry(72, 7)}; + + EXPECT_EQ(std::vector(kExpectedOutput, + kExpectedOutput + arraysize(kExpectedOutput)), + output); + EXPECT_EQ(kExpectedOutputSubsamples, subsamples); +} + +TEST(NalUnitToByteStreamConverterTest, EscapeEncryptedNalu) { + // Only the type of the NAL units are checked. + // This does not contain AUD, SPS, nor PPS. + const uint8_t kUnitStreamLikeMediaSample[] = { + 0x00, 0x00, 0x00, 0x0A, // Size 10 NALU. + 0x06, // NAL unit type. + // Unencrypted NALU with 0x000000 pattern (no need to escaped). + 0xFD, 0x00, 0x00, 0x00, 0x82, 0x62, 0x11, 0x29, 0x77, + 0x00, 0x00, 0x00, 0x08, // Size 8 NALU. + 0x02, // NAL unit type. + // Encrypted NALU with 0x000000 pattern (need to escape). + 0xFD, 0x00, 0x00, 0x00, 0x62, 0x29, 0x77, + 0x00, 0x00, 0x00, 0x09, // Size 9 NALU. + 0x01, // NAL unit types. + // Partially encrypted NALU with 0x000000 pattern at the boundary (need to + // escape). + 0xFD, 0x01, 0x02, 0x00, 0x00, 0x01, 0x02, 0x03, + }; + + std::vector subsamples{ + SubsampleEntry(19, 7), SubsampleEntry(9, 4), SubsampleEntry(7, 3)}; + + NalUnitToByteStreamConverter converter; + EXPECT_TRUE( + converter.Initialize(kTestAVCDecoderConfigurationRecord, + arraysize(kTestAVCDecoderConfigurationRecord))); + + std::vector output; + ASSERT_TRUE(converter.ConvertUnitToByteStreamWithSubsamples( + kUnitStreamLikeMediaSample, arraysize(kUnitStreamLikeMediaSample), + !kIsKeyFrame, kEscapeEncryptedNalu, &output, &subsamples)); + + const uint8_t kExpectedOutput[] = { + 0x00, 0x00, 0x00, 0x01, // Start code. + 0x09, // AUD type. + 0xF0, // primary pic type is anything. + // The input NALU 1. + 0x00, 0x00, 0x00, 0x01, // Start code. + 0x06, // NALU type. + 0xFD, 0x00, 0x00, 0x00, 0x82, 0x62, 0x11, 0x29, 0x77, + 0x00, 0x00, 0x00, 0x01, // Start code. + // The input NALU 2. + 0x02, // NALU type. + 0xFD, 0x00, 0x00, 0x03, 0x00, 0x62, 0x29, 0x77, + // The input NALU 3. + 0x00, 0x00, 0x00, 0x01, // Start code. + 0x01, // NAL unit types. + 0xFD, 0x01, 0x02, 0x00, 0x00, 0x03, 0x01, 0x02, 0x03, + }; + EXPECT_EQ(std::vector(std::begin(kExpectedOutput), + std::end(kExpectedOutput)), + output); + // The result subsample does not include emulation prevention bytes. + EXPECT_THAT(subsamples, ::testing::ElementsAre(SubsampleEntry(25, 7), + SubsampleEntry(9, 4), + SubsampleEntry(7, 3))); +} + +TEST(NalUnitToByteStreamConverterTest, EncryptedNaluEndingWithZero) { + // Only the type of the NAL units are checked. + // This does not contain AUD, SPS, nor PPS. + const uint8_t kUnitStreamLikeMediaSample[] = { + 0x00, 0x00, 0x00, 0x06, // Size 6 NALU. + 0x01, // NALU unit types. + // Encrypted NALU with 0x0003 pattern in the end (need to escape). + 0xFD, 0x00, 0x01, 0x02, 0x00, + }; + + std::vector subsamples{SubsampleEntry(7, 3)}; + + NalUnitToByteStreamConverter converter; + EXPECT_TRUE( + converter.Initialize(kTestAVCDecoderConfigurationRecord, + arraysize(kTestAVCDecoderConfigurationRecord))); + + std::vector output; + ASSERT_TRUE(converter.ConvertUnitToByteStreamWithSubsamples( + kUnitStreamLikeMediaSample, arraysize(kUnitStreamLikeMediaSample), + !kIsKeyFrame, kEscapeEncryptedNalu, &output, &subsamples)); + + const uint8_t kExpectedOutput[] = { + 0x00, 0x00, 0x00, 0x01, // Start code. + 0x09, // AUD type. + 0xF0, // primary pic type is anything. + 0x00, 0x00, 0x00, 0x01, // Start code. + 0x01, // NALU unit types. + // Encrypted NALU with 0x0003 pattern in the end (need to escape). + 0xFD, 0x00, 0x01, 0x02, 0x00, 0x03, + }; + EXPECT_EQ(std::vector(std::begin(kExpectedOutput), + std::end(kExpectedOutput)), + output); + // The result subsample does not include emulation prevention bytes. + EXPECT_THAT(subsamples, ::testing::ElementsAre(SubsampleEntry(13, 3))); +} + // corresponding subsample needs to be removed. TEST(NalUnitToByteStreamConverterTest, EncryptedPps) { // Only the type of the NAL units are checked. @@ -426,7 +595,7 @@ TEST(NalUnitToByteStreamConverterTest, EncryptedPps) { std::vector output; EXPECT_TRUE(converter.ConvertUnitToByteStreamWithSubsamples( kUnitStreamLikeMediaSample, arraysize(kUnitStreamLikeMediaSample), - kIsKeyFrame, &output, &subsamples)); + kIsKeyFrame, !kEscapeEncryptedNalu, &output, &subsamples)); const uint8_t kExpectedOutput[] = { 0x00, 0x00, 0x00, 0x01, // Start code. @@ -456,7 +625,7 @@ TEST(NalUnitToByteStreamConverterTest, EncryptedPps) { EXPECT_EQ(std::vector(kExpectedOutput, kExpectedOutput + arraysize(kExpectedOutput)), output); - EXPECT_EQ(kExpectedOutputSubsamples, subsamples); + EXPECT_THAT(kExpectedOutputSubsamples, subsamples); } // A clear PPS NALU follows a clear NALU, the PPS will be removed. So the @@ -487,7 +656,7 @@ TEST(NalUnitToByteStreamConverterTest, ClearPps) { std::vector output; EXPECT_TRUE(converter.ConvertUnitToByteStreamWithSubsamples( kUnitStreamLikeMediaSample, arraysize(kUnitStreamLikeMediaSample), - kIsKeyFrame, &output, &subsamples)); + kIsKeyFrame, !kEscapeEncryptedNalu, &output, &subsamples)); const uint8_t kExpectedOutput[] = { 0x00, 0x00, 0x00, 0x01, // Start code.