Mirror-Tools
/
shaka-packager
mirror of https://github.com/shaka-project/shaka-packager.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve handling of unescaped NAL units in byte stream 

The new algorithm will parse NAL unit header and only starts a new
NAL unit if it is valid, otherwise it will be considered part of
the previous NAL unit.

Closes #96

Change-Id: I45f2a0f37d51841ee8345d6d0d38fcda57e0a009
This commit is contained in:
KongQun Yang 2016-03-21 17:39:22 -07:00
parent d5cdd00ba1
commit 0c46943177
7 changed files with 205 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
packager/app/test/testdata/bear-640x360-hevc-v-cenc-golden.mp4 vendored
View File

Binary file not shown.

24
packager/media/filters/nal_unit_to_byte_stream_converter_unittest.cc
View File

@ -152,7 +152,7 @@ TEST(NalUnitToByteStreamConverterTest, ConvertUnitToByteStream) {
// This does not contain AUD, SPS, nor PPS.
const uint8_t kUnitStreamLikeMediaSample[] = {
0x00, 0x00, 0x00, 0x0A, // Size 10 NALU.
0x00, // Unspecified NAL unit type.
0x06, // NAL unit type.
0xFD, 0x78, 0xA4, 0xC3, 0x82, 0x62, 0x11, 0x29, 0x77,
};
NalUnitToByteStreamConverter converter;
@ -180,7 +180,7 @@ TEST(NalUnitToByteStreamConverterTest, ConvertUnitToByteStream) {
0x68, 0xFE, 0xFD, 0xFC, 0xFB, 0x11, 0x12, 0x13, 0x14, 0x15, // PPS.
0x00, 0x00, 0x00, 0x01, // Start code.
// The input NALU.
0x00, // Unspecified NALU type.
0x06, // NALU type.
0xFD, 0x78, 0xA4, 0xC3, 0x82, 0x62, 0x11, 0x29, 0x77,
};
@ -195,7 +195,7 @@ TEST(NalUnitToByteStreamConverterTest, ConvertUnitToByteStreamWithEscape) {
// This does not contain AUD, SPS, nor PPS.
const uint8_t kUnitStreamLikeMediaSample[] = {
0x00, 0x00, 0x00, 0x0A, // Size 10 NALU.
0x00, // Unspecified NAL unit type.
0x06, // NAL unit type.
0x06, 0x00, 0x00, 0x00, 0xDF, 0x62, 0x11, 0x29, 0x77,
};
NalUnitToByteStreamConverter converter;
@ -224,7 +224,7 @@ TEST(NalUnitToByteStreamConverterTest, ConvertUnitToByteStreamWithEscape) {
0x68, 0xFE, 0xFD, 0xFC, 0xFB, 0x11, 0x12, 0x13, 0x14, 0x15, // PPS.
0x00, 0x00, 0x00, 0x01, // Start code.
// The input NALU.
0x00, // Unspecified NALU type.
0x06, // NALU type.
0x06, 0x00, 0x00, 0x03, 0x00, 0xDF, 0x62, 0x11, 0x29, 0x77,
};
@ -237,7 +237,7 @@ TEST(NalUnitToByteStreamConverterTest, ConvertUnitToByteStreamWithEscape) {
TEST(NalUnitToByteStreamConverterTest, NaluEndingWithZero) {
const uint8_t kNaluEndingWithZero[] = {
0x00, 0x00, 0x00, 0x03, // Size 10 NALU.
0x00, // Unspecified NAL unit type.
0x06, // NAL unit type.
0xAA, 0x00, // Ends with 0.
};
NalUnitToByteStreamConverter converter;
@ -257,7 +257,7 @@ TEST(NalUnitToByteStreamConverterTest, NaluEndingWithZero) {
0xF0, // primary pic type is anything.
0x00, 0x00, 0x00, 0x01, // Start code.
// The input NALU.
0x00, // Unspecified NALU type.
0x06, // NALU type.
0xAA, 0x00, 0x03, // 0x03 at the end because the original ends with 0.
};
@ -271,7 +271,7 @@ TEST(NalUnitToByteStreamConverterTest, NaluEndingWithZero) {
TEST(NalUnitToByteStreamConverterTest, NonKeyFrameSample) {
const uint8_t kNonKeyFrameStream[] = {
0x00, 0x00, 0x00, 0x03, // Size 10 NALU.
0x00, // Unspecified NAL unit type.
0x06, // NAL unit type.
0x33, 0x88,
};
NalUnitToByteStreamConverter converter;
@ -291,7 +291,7 @@ TEST(NalUnitToByteStreamConverterTest, NonKeyFrameSample) {
0xF0, // Anything.
0x00, 0x00, 0x00, 0x01, // Start code.
// The input NALU.
0x00, // Unspecified NALU type.
0x06, // NALU type.
0x33, 0x88,
};
@ -305,7 +305,7 @@ TEST(NalUnitToByteStreamConverterTest, NonKeyFrameSample) {
TEST(NalUnitToByteStreamConverterTest, DispersedZeros) {
const uint8_t kDispersedZeros[] = {
0x00, 0x00, 0x00, 0x08, // Size 10 NALU.
0x00, // Unspecified NAL unit type.
0x06, // NAL unit type.
// After 2 zeros (including the first byte of the NALU followed by 0, 1,
// 2, or 3 caused it to insert the escape byte.
0x11, 0x00,
@ -327,7 +327,7 @@ TEST(NalUnitToByteStreamConverterTest, DispersedZeros) {
0xF0, // Anything.
0x00, 0x00, 0x00, 0x01, // Start code.
// The input NALU.
0x00, // Unspecified NAL unit type.
0x06, // NAL unit type.
0x11, 0x00, 0x01, 0x00, 0x02, 0x00, 0x44,
};
@ -341,7 +341,7 @@ TEST(NalUnitToByteStreamConverterTest, DoNotEscape) {
// This has sequences that should be escaped if escape_data = true.
const uint8_t kNotEscaped[] = {
0x00, 0x00, 0x00, 0x0C, // Size 12 NALU.
0x00, // Unspecified NAL unit type.
0x06, // NAL unit type.
0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x03,
};
@ -361,7 +361,7 @@ TEST(NalUnitToByteStreamConverterTest, DoNotEscape) {
0xF0, // Anything.
0x00, 0x00, 0x00, 0x01, // Start code.
// Should be the same as the input.
0x00,
0x06,
0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x03,
};

116
packager/media/filters/nalu_reader.cc
View File

@ -6,6 +6,8 @@
#include "packager/media/filters/nalu_reader.h"
#include <iostream>
#include "packager/base/logging.h"
#include "packager/media/base/buffer_reader.h"
#include "packager/media/filters/h264_parser.h"
@ -29,31 +31,66 @@ Nalu::Nalu()
type_(0),
is_video_slice_(false) {}
// ITU-T H.264 (02/2014) 7.4.1 NAL unit semantics
bool Nalu::InitializeFromH264(const uint8_t* data, uint64_t size) {
DCHECK(data);
if (size == 0)
return false;
uint8_t header = data[0];
if ((header & 0x80) != 0)
const uint8_t header = data[0];
if ((header & 0x80) != 0) {
LOG(WARNING) << "forbidden_zero_bit shall be equal to 0 (header 0x"
<< std::hex << static_cast<int>(header) << ").";
return false;
}
data_ = data;
header_size_ = 1;
payload_size_ = size - header_size_;
ref_idc_ = (header >> 5) & 0x3;
type_ = header & 0x1F;
// Reserved NAL units are not treated as valid NAL units here.
if (type_ == Nalu::H264_Unspecified || type_ == Nalu::H264_Reserved17 ||
type_ == Nalu::H264_Reserved18 || type_ >= Nalu::H264_Reserved22) {
LOG(WARNING) << "Unspecified or reserved nal_unit_type " << type_
<< " (header 0x" << std::hex << static_cast<int>(header)
<< ").";
return false;
} else if (type_ == Nalu::H264_IDRSlice || type_ == Nalu::H264_SPS ||
type_ == Nalu::H264_SPSExtension || type_ == Nalu::H264_SubsetSPS ||
type_ == Nalu::H264_PPS) {
if (ref_idc_ == 0) {
LOG(WARNING) << "nal_ref_idc shall not be equal to 0 for nalu type "
<< type_ << " (header 0x" << std::hex
<< static_cast<int>(header) << ").";
return false;
}
} else if (type_ == Nalu::H264_SEIMessage ||
(type_ >= Nalu::H264_AUD && type_ <= Nalu::H264_FillerData)) {
if (ref_idc_ != 0) {
LOG(WARNING) << "nal_ref_idc shall be equal to 0 for nalu type " << type_
<< " (header 0x" << std::hex << static_cast<int>(header)
<< ").";
return false;
}
}
is_video_slice_ = (type_ >= Nalu::H264_NonIDRSlice &&
type_ <= Nalu::H264_IDRSlice);
return true;
}
// ITU-T H.265 (04/2015) 7.4.2.2 NAL unit header semantics
bool Nalu::InitializeFromH265(const uint8_t* data, uint64_t size) {
DCHECK(data);
if (size < 2)
return false;
uint16_t header = (data[0] << 8) | data[1];
if ((header & 0x8000) != 0)
const uint16_t header = (data[0] << 8) | data[1];
if ((header & 0x8000) != 0) {
LOG(WARNING) << "forbidden_zero_bit shall be equal to 0 (header 0x"
<< std::hex << header << ").";
return false;
}
data_ = data;
header_size_ = 2;
@ -61,12 +98,47 @@ bool Nalu::InitializeFromH265(const uint8_t* data, uint64_t size) {
type_ = (header >> 9) & 0x3F;
nuh_layer_id_ = (header >> 3) & 0x3F;
nuh_temporal_id_ = (header & 0x7) - 1;
const int nuh_temporal_id_plus1 = header & 0x7;
if (nuh_temporal_id_plus1 == 0) {
LOG(WARNING) << "nul_temporal_id_plus1 shall not be equal to 0 (header 0x"
<< std::hex << header << ").";
return false;
}
nuh_temporal_id_ = nuh_temporal_id_plus1 - 1;
// Don't treat reserved VCL types as video slices since we cannot parse them.
is_video_slice_ =
(type_ >= Nalu::H265_TRAIL_N && type_ <= Nalu::H265_RASL_R) ||
(type_ >= Nalu::H265_BLA_W_LP && type_ <= Nalu::H265_CRA_NUT);
if (type_ == Nalu::H265_EOB && nuh_layer_id_ != 0) {
LOG(WARNING) << "nuh_layer_id shall be equal to 0 for nalu type " << type_
<< " (header 0x" << std::hex << header << ").";
return false;
}
// Reserved NAL units are not treated as valid NAL units here.
if ((type_ >= Nalu::H265_RSV_VCL_N10 && type_ <= Nalu::H265_RSV_VCL_R15) ||
(type_ >= Nalu::H265_RSV_IRAP_VCL22 && type_ < Nalu::H265_RSV_VCL31) ||
(type_ >= Nalu::H265_RSV_NVCL41)) {
LOG(WARNING) << "Unspecified or reserved nal_unit_type " << type_
<< " (header 0x" << std::hex << header << ").";
return false;
} else if ((type_ >= Nalu::H265_BLA_W_LP &&
type_ <= Nalu::H265_RSV_IRAP_VCL23) ||
type_ == Nalu::H265_VPS || type_ == Nalu::H265_SPS ||
type_ == Nalu::H265_EOS || type_ == Nalu::H265_EOB) {
if (nuh_temporal_id_ != 0) {
LOG(WARNING) << "TemporalId shall be equal to 0 for nalu type " << type_
<< " (header 0x" << std::hex << header << ").";
return false;
}
} else if (type_ == Nalu::H265_TSA_N || type_ == Nalu::H265_TSA_R ||
(nuh_layer_id_ == 0 &&
(type_ == Nalu::H265_STSA_N || type_ == Nalu::H265_STSA_R))) {
if (nuh_temporal_id_ == 0) {
LOG(WARNING) << "TemporalId shall not be equal to 0 for nalu type "
<< type_ << " (header 0x" << std::hex << header << ").";
return false;
}
}
is_video_slice_ = type_ >= Nalu::H265_TRAIL_N && type_ <= Nalu::H265_CRA_NUT;
return true;
}
@ -218,11 +290,29 @@ bool NaluReader::LocateNaluByStartCode(uint64_t* nalu_size,
// belong to the current NALU.
uint64_t nalu_size_without_start_code = 0;
uint8_t next_start_code_size = 0;
if (!FindStartCode(nalu_data, max_nalu_data_size,
&nalu_size_without_start_code, &next_start_code_size)) {
nalu_size_without_start_code = max_nalu_data_size;
while (true) {
if (!FindStartCode(nalu_data, max_nalu_data_size,
&nalu_size_without_start_code, &next_start_code_size)) {
nalu_data += max_nalu_data_size;
break;
}
nalu_data += nalu_size_without_start_code + next_start_code_size;
max_nalu_data_size -= nalu_size_without_start_code + next_start_code_size;
// If it is not a valid NAL unit, we will continue searching. This is to
// handle the case where emulation prevention are not applied.
Nalu nalu;
if (nalu_type_ == kH264
? nalu.InitializeFromH264(nalu_data, max_nalu_data_size)
: nalu.InitializeFromH265(nalu_data, max_nalu_data_size)) {
nalu_data -= next_start_code_size;
break;
}
LOG(WARNING) << "Seeing invalid NAL unit. Emulation prevention may not "
"have been applied properly. Assuming it is part of the "
"previous NAL unit.";
}
*nalu_size = nalu_size_without_start_code + annexb_start_code_size;
*nalu_size = nalu_data - stream_;
*start_code_size = annexb_start_code_size;
return true;
}

21
packager/media/filters/nalu_reader.h
View File

@ -34,22 +34,43 @@ class Nalu {
H264_PPS = 8,
H264_AUD = 9,
H264_EOSeq = 10,
H264_FillerData = 12,
H264_SPSExtension = 13,
H264_SubsetSPS = 15,
H264_Reserved17 = 17,
H264_Reserved18 = 18,
H264_CodedSliceExtension = 20,
H264_Reserved22 = 22,
};
enum H265NaluType {
H265_TRAIL_N = 0,
H265_TSA_N = 2,
H265_TSA_R = 3,
H265_STSA_N = 4,
H265_STSA_R = 5,
H265_RASL_R = 9,
H265_RSV_VCL_N10 = 10,
H265_RSV_VCL_R15 = 15,
H265_BLA_W_LP = 16,
H265_IDR_W_RADL = 19,
H265_IDR_N_LP = 20,
H265_CRA_NUT = 21,
H265_RSV_IRAP_VCL22 = 22,
H265_RSV_IRAP_VCL23 = 23,
H265_RSV_VCL31 = 31,
H265_VPS = 32,
H265_SPS = 33,
H265_PPS = 34,
H265_AUD = 35,
H265_EOS = 36,
H265_EOB = 37,
H265_RSV_NVCL41 = 41,
};
Nalu();

86
packager/media/filters/nalu_reader_unittest.cc
View File

@ -13,11 +13,11 @@ namespace media {
TEST(NaluReaderTest, StartCodeSearch) {
const uint8_t kNaluData[] = {
0x01, 0x00, 0x00, 0x04, 0x23, 0x56,
// First NALU
0x00, 0x00, 0x01, 0x12, 0x34, 0x56, 0x78,
// Second NALU
0x00, 0x00, 0x00, 0x01, 0x67, 0xbb, 0xcc, 0xdd
0x01, 0x00, 0x00, 0x04, 0x23, 0x56,
// First NALU
0x00, 0x00, 0x01, 0x14, 0x34, 0x56, 0x78,
// Second NALU
0x00, 0x00, 0x00, 0x01, 0x67, 0xbb, 0xcc, 0xdd,
};
NaluReader reader(NaluReader::kH264, kIsAnnexbByteStream, kNaluData,
@ -29,7 +29,7 @@ TEST(NaluReaderTest, StartCodeSearch) {
EXPECT_EQ(3u, nalu.payload_size());
EXPECT_EQ(1u, nalu.header_size());
EXPECT_EQ(0, nalu.ref_idc());
EXPECT_EQ(0x12, nalu.type());
EXPECT_EQ(0x14, nalu.type());
ASSERT_EQ(NaluReader::kOk, reader.Advance(&nalu));
EXPECT_EQ(kNaluData + 17, nalu.data());
@ -41,12 +41,46 @@ TEST(NaluReaderTest, StartCodeSearch) {
EXPECT_EQ(NaluReader::kEOStream, reader.Advance(&nalu));
}
TEST(NaluReaderTest, StartCodeSearchWithStartCodeInsideNalUnit) {
const uint8_t kNaluData[] = {
0x01, 0x00, 0x00, 0x04, 0x23, 0x56,
// First NALU
0x00, 0x00, 0x01, 0x14, 0x34, 0x56, 0x78,
// This is part of the first NALU as it is not a valid NALU.
0x00, 0x00, 0x00, 0x01, 0x07, 0xbb, 0xcc, 0xdd,
// Second NALU
0x00, 0x00, 0x01, 0x67, 0x03, 0x04,
// This is part of the second NALU.
0x00, 0x00, 0x01,
};
NaluReader reader(NaluReader::kH264, kIsAnnexbByteStream, kNaluData,
arraysize(kNaluData));
Nalu nalu;
ASSERT_EQ(NaluReader::kOk, reader.Advance(&nalu));
EXPECT_EQ(kNaluData + 9, nalu.data());
EXPECT_EQ(11u, nalu.payload_size());
EXPECT_EQ(1u, nalu.header_size());
EXPECT_EQ(0, nalu.ref_idc());
EXPECT_EQ(0x14, nalu.type());
ASSERT_EQ(NaluReader::kOk, reader.Advance(&nalu));
EXPECT_EQ(kNaluData + 24, nalu.data());
EXPECT_EQ(5u, nalu.payload_size());
EXPECT_EQ(1u, nalu.header_size());
EXPECT_EQ(3, nalu.ref_idc());
EXPECT_EQ(7, nalu.type());
EXPECT_EQ(NaluReader::kEOStream, reader.Advance(&nalu));
}
TEST(NaluReaderTest, OneByteNaluLength) {
const uint8_t kNaluData[] = {
// First NALU
0x05, 0x08, 0x01, 0x02, 0x03, 0x04,
// Second NALU
0x06, 0x67, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e
// First NALU
0x05, 0x06, 0x01, 0x02, 0x03, 0x04,
// Second NALU
0x06, 0x67, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
};
NaluReader reader(NaluReader::kH264, 1, kNaluData, arraysize(kNaluData));
@ -57,7 +91,7 @@ TEST(NaluReaderTest, OneByteNaluLength) {
EXPECT_EQ(4u, nalu.payload_size());
EXPECT_EQ(1u, nalu.header_size());
EXPECT_EQ(0, nalu.ref_idc());
EXPECT_EQ(8, nalu.type());
EXPECT_EQ(6, nalu.type());
ASSERT_EQ(NaluReader::kOk, reader.Advance(&nalu));
EXPECT_EQ(kNaluData + 7, nalu.data());
@ -71,10 +105,10 @@ TEST(NaluReaderTest, OneByteNaluLength) {
TEST(NaluReaderTest, FourByteNaluLength) {
const uint8_t kNaluData[] = {
// First NALU
0x00, 0x00, 0x00, 0x07, 0x08, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
// Second NALU
0x00, 0x00, 0x00, 0x03, 0x67, 0x0a, 0x0b
// First NALU
0x00, 0x00, 0x00, 0x07, 0x06, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
// Second NALU
0x00, 0x00, 0x00, 0x03, 0x67, 0x0a, 0x0b,
};
NaluReader reader(NaluReader::kH264, 4, kNaluData, arraysize(kNaluData));
@ -85,7 +119,7 @@ TEST(NaluReaderTest, FourByteNaluLength) {
EXPECT_EQ(6u, nalu.payload_size());
EXPECT_EQ(1u, nalu.header_size());
EXPECT_EQ(0, nalu.ref_idc());
EXPECT_EQ(8, nalu.type());
EXPECT_EQ(6, nalu.type());
ASSERT_EQ(NaluReader::kOk, reader.Advance(&nalu));
EXPECT_EQ(kNaluData + 15, nalu.data());
@ -99,8 +133,8 @@ TEST(NaluReaderTest, FourByteNaluLength) {
TEST(NaluReaderTest, ErrorForNotEnoughForNaluLength) {
const uint8_t kNaluData[] = {
// First NALU
0x00
// First NALU
0x00,
};
NaluReader reader(NaluReader::kH264, 3, kNaluData, arraysize(kNaluData));
@ -111,8 +145,8 @@ TEST(NaluReaderTest, ErrorForNotEnoughForNaluLength) {
TEST(NaluReaderTest, ErrorForNaluLengthExceedsRemainingData) {
const uint8_t kNaluData[] = {
// First NALU
0xFF, 0x08, 0x00
// First NALU
0xFF, 0x08, 0x00,
};
NaluReader reader(NaluReader::kH264, 1, kNaluData, arraysize(kNaluData));
@ -122,8 +156,8 @@ TEST(NaluReaderTest, ErrorForNaluLengthExceedsRemainingData) {
// Another test for off by one.
const uint8_t kNaluData2[] = {
// First NALU
0x04, 0x08, 0x00, 0x00
// First NALU
0x04, 0x08, 0x00, 0x00,
};
NaluReader reader2(NaluReader::kH264, 1, kNaluData2, arraysize(kNaluData2));
@ -132,8 +166,8 @@ TEST(NaluReaderTest, ErrorForNaluLengthExceedsRemainingData) {
TEST(NaluReaderTest, ErrorForForbiddenBitSet) {
const uint8_t kNaluData[] = {
// First NALU
0x03, 0x80, 0x00, 0x00
// First NALU
0x03, 0x80, 0x00, 0x00,
};
NaluReader reader(NaluReader::kH264, 1, kNaluData, arraysize(kNaluData));
@ -144,8 +178,8 @@ TEST(NaluReaderTest, ErrorForForbiddenBitSet) {
TEST(NaluReaderTest, ErrorForZeroSize) {
const uint8_t kNaluData[] = {
// First NALU
0x03, 0x80, 0x00, 0x00
// First NALU
0x03, 0x80, 0x00, 0x00,
};
Nalu nalu;

14
packager/media/formats/mp4/encrypting_fragmenter.cc
View File

@ -70,15 +70,15 @@ EncryptingFragmenter::EncryptingFragmenter(
info_(info),
encryption_key_(encryption_key.Pass()),
nalu_length_size_(GetNaluLengthSize(*info)),
video_codec_(GetVideoCodec(*info)),
clear_time_(clear_time),
encryption_mode_(encryption_mode) {
DCHECK(encryption_key_);
VideoCodec video_codec = GetVideoCodec(*info);
if (video_codec == kCodecVP8) {
if (video_codec_ == kCodecVP8) {
vpx_parser_.reset(new VP8Parser);
} else if (video_codec == kCodecVP9) {
} else if (video_codec_ == kCodecVP9) {
vpx_parser_.reset(new VP9Parser);
} else if (video_codec == kCodecH264) {
} else if (video_codec_ == kCodecH264) {
header_parser_.reset(new H264VideoSliceHeaderParser);
}
// TODO(modmaker): Support H.265.
@ -231,8 +231,10 @@ Status EncryptingFragmenter::EncryptSample(scoped_refptr<MediaSample> sample) {
data += frame.frame_size;
}
} else {
// TODO(modmaker): Support H.265.
const NaluReader::NaluType nalu_type = NaluReader::kH264;
const NaluReader::NaluType nalu_type =
(video_codec_ == kCodecHVC1 || video_codec_ == kCodecHEV1)
? NaluReader::kH265
: NaluReader::kH264;
NaluReader reader(nalu_type, nalu_length_size_, data,
sample->data_size());

1
packager/media/formats/mp4/encrypting_fragmenter.h
View File

@ -78,6 +78,7 @@ class EncryptingFragmenter : public Fragmenter {
// and type of NAL units remain unencrypted. This function returns the size of
// the size field in bytes. Can be 1, 2 or 4 bytes.
const uint8_t nalu_length_size_;
const VideoCodec video_codec_;
int64_t clear_time_;
EncryptionMode encryption_mode_;