From bbf9c6849b1f77f993ff0f70be425bb274078f16 Mon Sep 17 00:00:00 2001 From: Jacob Trimble Date: Wed, 13 Apr 2016 14:46:13 -0700 Subject: [PATCH] Parse vui_parameters in H.265. Parsing vui_parameters in the SPS is required to be able to extract the resolution of the video. This also adds a method to extract the resolution to the H.265 parser. Issue #46 Change-Id: Idf498d2afdb36a689490151a4ae6baef0b6b73f6 --- .../h265_byte_to_unit_stream_converter.cc | 10 +- packager/media/filters/h265_parser.cc | 269 ++++++++++++++++++ packager/media/filters/h265_parser.h | 37 +++ .../media/filters/h265_parser_unittest.cc | 46 +++ 4 files changed, 359 insertions(+), 3 deletions(-) diff --git a/packager/media/filters/h265_byte_to_unit_stream_converter.cc b/packager/media/filters/h265_byte_to_unit_stream_converter.cc index db9a2c7785..95b3d195a9 100644 --- a/packager/media/filters/h265_byte_to_unit_stream_converter.cc +++ b/packager/media/filters/h265_byte_to_unit_stream_converter.cc @@ -49,9 +49,13 @@ bool H265ByteToUnitStreamConverter::GetDecoderConfigurationRecord( // Skip Nalu header (2) and the first byte of the SPS to get the // profile_tier_level. buffer.AppendArray(&last_sps_[2+1], 12); - // min_spacial_segmentation_idc = 0 (Unknown) - // TODO(modmaker): Parse vui_parameters and update this. - buffer.AppendInt(static_cast(0xf000)); + + // The default value for this field is 0, which is Unknown. + int min_spatial_segmentation_idc = + sps->vui_parameters.min_spatial_segmentation_idc; + + buffer.AppendInt( + static_cast(0xf000 | min_spatial_segmentation_idc)); buffer.AppendInt(static_cast(0xfc) /* parallelismType = 0 */); buffer.AppendInt(static_cast(0xfc | sps->chroma_format_idc)); buffer.AppendInt(static_cast(0xf8 | sps->bit_depth_luma_minus8)); diff --git a/packager/media/filters/h265_parser.cc b/packager/media/filters/h265_parser.cc index 19b8f10389..2a101cbe95 100644 --- a/packager/media/filters/h265_parser.cc +++ b/packager/media/filters/h265_parser.cc @@ -11,6 +11,7 @@ #include "packager/base/logging.h" #include "packager/base/stl_util.h" +#include "packager/media/base/macros.h" #include "packager/media/filters/nalu_reader.h" #define TRUE_OR_RETURN(a) \ @@ -51,8 +52,99 @@ int GetNumPicTotalCurr(const H265SliceHeader& slice_header, return num_pic_total_curr + slice_header.used_by_curr_pic_lt; } + +void GetAspectRatioInfo(const H265Sps& sps, + uint32_t* pixel_width, + uint32_t* pixel_height) { + // The default value is 0; so if this is not in the SPS, it will correctly + // assume unspecified. + int aspect_ratio_idc = sps.vui_parameters.aspect_ratio_idc; + + // Table E.1 + switch (aspect_ratio_idc) { + case 1: *pixel_width = 1; *pixel_height = 1; break; + case 2: *pixel_width = 12; *pixel_height = 11; break; + case 3: *pixel_width = 10; *pixel_height = 11; break; + case 4: *pixel_width = 16; *pixel_height = 11; break; + case 5: *pixel_width = 40; *pixel_height = 33; break; + case 6: *pixel_width = 24; *pixel_height = 11; break; + case 7: *pixel_width = 20; *pixel_height = 11; break; + case 8: *pixel_width = 32; *pixel_height = 11; break; + case 9: *pixel_width = 80; *pixel_height = 33; break; + case 10: *pixel_width = 18; *pixel_height = 11; break; + case 11: *pixel_width = 15; *pixel_height = 11; break; + case 12: *pixel_width = 64; *pixel_height = 33; break; + case 13: *pixel_width = 160; *pixel_height = 99; break; + case 14: *pixel_width = 4; *pixel_height = 3; break; + case 15: *pixel_width = 3; *pixel_height = 2; break; + case 16: *pixel_width = 2; *pixel_height = 1; break; + + case 255: + *pixel_width = sps.vui_parameters.sar_width; + *pixel_height = sps.vui_parameters.sar_height; + break; + + default: + // Section E.3.1 specifies that other values should be interpreted as 0. + LOG(WARNING) << "Unknown aspect_ratio_idc " << aspect_ratio_idc; + FALLTHROUGH_INTENDED; + case 0: + // Unlike the spec, assume 1:1 if not specified. + *pixel_width = 1; + *pixel_height = 1; + break; + } +} } // namespace +bool ExtractResolutionFromSps(const H265Sps& sps, + uint32_t* coded_width, + uint32_t* coded_height, + uint32_t* pixel_width, + uint32_t* pixel_height) { + int crop_x = 0; + int crop_y = 0; + if (sps.conformance_window_flag) { + int sub_width_c = 0; + int sub_height_c = 0; + + // Table 6-1 + switch (sps.chroma_format_idc) { + case 0: // Monochrome + sub_width_c = 1; + sub_height_c = 1; + break; + case 1: // 4:2:0 + sub_width_c = 2; + sub_height_c = 2; + break; + case 2: // 4:2:2 + sub_width_c = 2; + sub_height_c = 1; + break; + case 3: // 4:4:4 + sub_width_c = 1; + sub_height_c = 1; + break; + default: + LOG(ERROR) << "Unexpected chroma_format_idc " << sps.chroma_format_idc; + return false; + } + + // Formula D-28, D-29 + crop_x = + sub_width_c * (sps.conf_win_right_offset + sps.conf_win_left_offset); + crop_y = + sub_height_c * (sps.conf_win_bottom_offset + sps.conf_win_top_offset); + } + + // Formula D-28, D-29 + *coded_width = sps.pic_width_in_luma_samples - crop_x; + *coded_height = sps.pic_height_in_luma_samples - crop_y; + GetAspectRatioInfo(sps, pixel_width, pixel_height); + return true; +} + H265Pps::H265Pps() {} H265Pps::~H265Pps() {} @@ -519,6 +611,12 @@ H265Parser::Result H265Parser::ParseSps(const Nalu& nalu, int* sps_id) { TRUE_OR_RETURN(br->ReadBool(&sps->temporal_mvp_enabled_flag)); TRUE_OR_RETURN(br->ReadBool(&sps->strong_intra_smoothing_enabled_flag)); + TRUE_OR_RETURN(br->ReadBool(&sps->vui_parameters_present)); + if (sps->vui_parameters_present) { + OK_OR_RETURN(ParseVuiParameters(sps->max_sub_layers_minus1, br, + &sps->vui_parameters)); + } + // Ignore remaining extension data. // This will replace any existing SPS instance. @@ -537,6 +635,98 @@ const H265Sps* H265Parser::GetSps(int sps_id) { return active_spses_[sps_id]; } +H265Parser::Result H265Parser::ParseVuiParameters(int max_num_sub_layers_minus1, + H26xBitReader* br, + H265VuiParameters* vui) { + // Reads whole element but ignores most of it. + int ignored; + + TRUE_OR_RETURN(br->ReadBool(&vui->aspect_ratio_info_present_flag)); + if (vui->aspect_ratio_info_present_flag) { + TRUE_OR_RETURN(br->ReadBits(8, &vui->aspect_ratio_idc)); + if (vui->aspect_ratio_idc == H265VuiParameters::kExtendedSar) { + TRUE_OR_RETURN(br->ReadBits(16, &vui->sar_width)); + TRUE_OR_RETURN(br->ReadBits(16, &vui->sar_height)); + } + } + + bool overscan_info_present_flag; + TRUE_OR_RETURN(br->ReadBool(&overscan_info_present_flag)); + if (overscan_info_present_flag) { + TRUE_OR_RETURN(br->SkipBits(1)); // overscan_appropriate_flag + } + + bool video_signal_type_present_flag; + TRUE_OR_RETURN(br->ReadBool(&video_signal_type_present_flag)); + if (video_signal_type_present_flag) { + TRUE_OR_RETURN(br->SkipBits(3)); // video_format + TRUE_OR_RETURN(br->SkipBits(1)); // video_full_range_flag + + bool colour_description_present_flag; + TRUE_OR_RETURN(br->ReadBool(&colour_description_present_flag)); + if (colour_description_present_flag) { + // colour_primaries, transfer_characteristics, matrix_coeffs + TRUE_OR_RETURN(br->SkipBits(8 + 8 + 8)); + } + } + + bool chroma_loc_info_present_flag; + TRUE_OR_RETURN(br->ReadBool(&chroma_loc_info_present_flag)); + if (chroma_loc_info_present_flag) { + // chroma_sample_log_type_top_field, chroma_sample_log_type_bottom_field + TRUE_OR_RETURN(br->ReadUE(&ignored)); + TRUE_OR_RETURN(br->ReadUE(&ignored)); + } + + // neutral_chroma_indication_flag, field_seq_flag, + // frame_field_info_present_flag. + TRUE_OR_RETURN(br->SkipBits(3)); + + bool default_display_window_flag; + TRUE_OR_RETURN(br->ReadBool(&default_display_window_flag)); + if (default_display_window_flag) { + TRUE_OR_RETURN(br->ReadUE(&ignored)); // def_disp_win_left_offset + TRUE_OR_RETURN(br->ReadUE(&ignored)); // def_disp_win_right_offset + TRUE_OR_RETURN(br->ReadUE(&ignored)); // def_disp_win_top_offset + TRUE_OR_RETURN(br->ReadUE(&ignored)); // def_disp_win_bottom_offset + } + + bool vui_timing_info_present_flag; + TRUE_OR_RETURN(br->ReadBool(&vui_timing_info_present_flag)); + if (vui_timing_info_present_flag) { + // vui_num_units_in_tick, vui_time_scale + TRUE_OR_RETURN(br->SkipBits(32 + 32)); + + bool vui_poc_proportional_to_timing_flag; + TRUE_OR_RETURN(br->ReadBool(&vui_poc_proportional_to_timing_flag)); + if (vui_poc_proportional_to_timing_flag) { + // vui_num_ticks_poc_diff_one_minus1 + TRUE_OR_RETURN(br->ReadUE(&ignored)); + } + + bool vui_hdr_parameters_present_flag; + TRUE_OR_RETURN(br->ReadBool(&vui_hdr_parameters_present_flag)); + if (vui_hdr_parameters_present_flag) { + OK_OR_RETURN(SkipHrdParameters(max_num_sub_layers_minus1, br)); + } + } + + TRUE_OR_RETURN(br->ReadBool(&vui->bitstream_restriction_flag)); + if (vui->bitstream_restriction_flag) { + // tiles_fixed_structure_flag, motion_vectors_over_pic_boundaries_flag, + // restricted_ref_pic_lists_flag. + TRUE_OR_RETURN(br->SkipBits(3)); + + TRUE_OR_RETURN(br->ReadUE(&vui->min_spatial_segmentation_idc)); + TRUE_OR_RETURN(br->ReadUE(&ignored)); // max_bytes_per_pic_denom + TRUE_OR_RETURN(br->ReadUE(&ignored)); // max_bits_per_min_cu_denum + TRUE_OR_RETURN(br->ReadUE(&ignored)); // log2_max_mv_length_horizontal + TRUE_OR_RETURN(br->ReadUE(&ignored)); // log2_max_mv_length_vertical + } + + return kOk; +} + H265Parser::Result H265Parser::ParseReferencePictureSet( int num_short_term_ref_pic_sets, int st_rps_idx, @@ -843,5 +1033,84 @@ H265Parser::Result H265Parser::SkipScalingListData(H26xBitReader* br) { return kOk; } +H265Parser::Result H265Parser::SkipHrdParameters(int max_num_sub_layers_minus1, + H26xBitReader* br) { + // common_inf_present_flag is always 1 when parsing vui_parameters. + const bool common_inf_present_flag = true; + + int ignored; + bool nal_hdr_parameters_present_flag; + bool vcl_hdr_parameters_present_flag; + bool sub_pic_hdr_params_present_flag = false; + if (common_inf_present_flag) { + TRUE_OR_RETURN(br->ReadBool(&nal_hdr_parameters_present_flag)); + TRUE_OR_RETURN(br->ReadBool(&vcl_hdr_parameters_present_flag)); + if (nal_hdr_parameters_present_flag || vcl_hdr_parameters_present_flag) { + TRUE_OR_RETURN(br->ReadBool(&sub_pic_hdr_params_present_flag)); + if (sub_pic_hdr_params_present_flag) { + // tick_divisor_minus2, du_cpb_removal_delay_increment_length_minus1, + // sub_pic_cpb_params_in_pic_timing_sei_flag + // dpb_output_delay_du_length_minus1 + TRUE_OR_RETURN(br->SkipBits(8 + 5 + 1 + 5)); + } + + // bit_rate_scale, cpb_size_scale + TRUE_OR_RETURN(br->SkipBits(4 + 4)); + if (sub_pic_hdr_params_present_flag) + TRUE_OR_RETURN(br->SkipBits(4)); // cpb_size_du_scale + + // initial_cpb_removal_delay_length_minus1, + // au_cpb_removal_delay_length_minus1, dpb_output_delay_length_minus1 + TRUE_OR_RETURN(br->SkipBits(5 + 5 + 5)); + } + } + + for (int i = 0; i <= max_num_sub_layers_minus1; i++) { + bool fixed_pic_rate_general_flag; + bool fixed_pic_rate_within_cvs_flag = true; + bool low_delay_hdr_flag = false; + int cpb_cnt_minus1 = 0; + TRUE_OR_RETURN(br->ReadBool(&fixed_pic_rate_general_flag)); + if (!fixed_pic_rate_general_flag) + TRUE_OR_RETURN(br->ReadBool(&fixed_pic_rate_within_cvs_flag)); + if (fixed_pic_rate_within_cvs_flag) + TRUE_OR_RETURN(br->ReadUE(&ignored)); // elemental_duration_ic_tc_minus1 + else + TRUE_OR_RETURN(br->ReadBool(&low_delay_hdr_flag)); + if (!low_delay_hdr_flag) + TRUE_OR_RETURN(br->ReadUE(&cpb_cnt_minus1)); + + if (nal_hdr_parameters_present_flag) { + OK_OR_RETURN(SkipSubLayerHrdParameters( + cpb_cnt_minus1, sub_pic_hdr_params_present_flag, br)); + } + if (vcl_hdr_parameters_present_flag) { + OK_OR_RETURN(SkipSubLayerHrdParameters( + cpb_cnt_minus1, sub_pic_hdr_params_present_flag, br)); + } + } + + return kOk; +} + +H265Parser::Result H265Parser::SkipSubLayerHrdParameters( + int cpb_cnt_minus1, + bool sub_pic_hdr_params_present_flag, + H26xBitReader* br) { + int ignored; + for (int i = 0; i <= cpb_cnt_minus1; i++) { + TRUE_OR_RETURN(br->ReadUE(&ignored)); // bit_rate_value_minus1 + TRUE_OR_RETURN(br->ReadUE(&ignored)); // cpb_size_value_minus1 + if (sub_pic_hdr_params_present_flag) { + TRUE_OR_RETURN(br->ReadUE(&ignored)); // cpb_size_du_value_minus1 + TRUE_OR_RETURN(br->ReadUE(&ignored)); // bit_rate_du_value_minus1 + } + + TRUE_OR_RETURN(br->SkipBits(1)); // cbr_flag + } + + return kOk; +} + } // namespace media } // namespace edash_packager diff --git a/packager/media/filters/h265_parser.h b/packager/media/filters/h265_parser.h index 615fc12fae..0fe7a596ca 100644 --- a/packager/media/filters/h265_parser.h +++ b/packager/media/filters/h265_parser.h @@ -22,6 +22,16 @@ enum H265SliceType { kBSlice = 0, kPSlice = 1, kISlice = 2 }; const int kMaxRefPicSetCount = 16; +// On success, |coded_width| and |coded_height| contains coded resolution after +// cropping; |pixel_width:pixel_height| contains pixel aspect ratio, 1:1 is +// assigned if it is not present in SPS. +struct H265Sps; +bool ExtractResolutionFromSps(const H265Sps& sps, + uint32_t* coded_width, + uint32_t* coded_height, + uint32_t* pixel_width, + uint32_t* pixel_height); + struct H265ReferencePictureSet { int delta_poc_s0[kMaxRefPicSetCount]; int delta_poc_s1[kMaxRefPicSetCount]; @@ -33,6 +43,20 @@ struct H265ReferencePictureSet { int num_delta_pocs; }; +struct H265VuiParameters { + enum { kExtendedSar = 255 }; + + bool aspect_ratio_info_present_flag = false; + int aspect_ratio_idc = 0; + int sar_width = 0; + int sar_height = 0; + + bool bitstream_restriction_flag = false; + int min_spatial_segmentation_idc = 0; + + // Incomplete... +}; + struct H265Pps { H265Pps(); ~H265Pps(); @@ -162,6 +186,9 @@ struct H265Sps { bool temporal_mvp_enabled_flag = false; bool strong_intra_smoothing_enabled_flag = false; + bool vui_parameters_present = false; + H265VuiParameters vui_parameters; + // Ignored: extensions... }; @@ -277,6 +304,10 @@ class H265Parser { const H265Sps* GetSps(int sps_id); private: + Result ParseVuiParameters(int max_num_sub_layers_minus1, + H26xBitReader* br, + H265VuiParameters* vui); + Result ParseReferencePictureSet( int num_short_term_ref_pic_sets, int st_rpx_idx, @@ -305,6 +336,12 @@ class H265Parser { Result SkipScalingListData(H26xBitReader* br); + Result SkipHrdParameters(int max_num_sub_layers_minus1, H26xBitReader* br); + + Result SkipSubLayerHrdParameters(int cpb_cnt_minus1, + bool sub_pic_hdr_params_present_flag, + H26xBitReader* br); + typedef std::map SpsById; typedef std::map PpsById; diff --git a/packager/media/filters/h265_parser_unittest.cc b/packager/media/filters/h265_parser_unittest.cc index b7c7302bca..0f1d9b4c0c 100644 --- a/packager/media/filters/h265_parser_unittest.cc +++ b/packager/media/filters/h265_parser_unittest.cc @@ -125,6 +125,52 @@ TEST(H265ParserTest, ParsePps) { EXPECT_EQ(0, pps->log2_parallel_merge_level_minus2); } +TEST(H265ParserTest, ExtractResolutionFromSpsData) { + H265Parser parser; + int sps_id = 0; + Nalu nalu; + ASSERT_TRUE(nalu.InitializeFromH265(kSpsData, arraysize(kSpsData))); + ASSERT_EQ(H265Parser::kOk, parser.ParseSps(nalu, &sps_id)); + + uint32_t coded_width = 0; + uint32_t coded_height = 0; + uint32_t pixel_width = 0; + uint32_t pixel_height = 0; + ASSERT_TRUE(ExtractResolutionFromSps(*parser.GetSps(sps_id), &coded_width, + &coded_height, &pixel_width, + &pixel_height)); + EXPECT_EQ(640u, coded_width); + EXPECT_EQ(360u, coded_height); + EXPECT_EQ(1u, pixel_width); + EXPECT_EQ(1u, pixel_height); +} + +TEST(H265ParserTest, ExtractResolutionFromSpsDataWithCrop) { + const uint8_t kSpsCropData[] = { + 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, + 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x3c, 0xa0, 0x0f, 0x08, 0x0f, + 0x16, 0x59, 0x99, 0xa4, 0x93, 0x2b, 0xff, 0xc0, 0xd5, 0xc0, 0xd6, + 0x40, 0x40, 0x00, 0x00, 0x03, 0x00, 0x40, 0x00, 0x00, 0x06, 0x02, + }; + H265Parser parser; + int sps_id = 0; + Nalu nalu; + ASSERT_TRUE(nalu.InitializeFromH265(kSpsCropData, arraysize(kSpsCropData))); + ASSERT_EQ(H265Parser::kOk, parser.ParseSps(nalu, &sps_id)); + + uint32_t coded_width = 0; + uint32_t coded_height = 0; + uint32_t pixel_width = 0; + uint32_t pixel_height = 0; + ASSERT_TRUE(ExtractResolutionFromSps(*parser.GetSps(sps_id), &coded_width, + &coded_height, &pixel_width, + &pixel_height)); + EXPECT_EQ(480u, coded_width); + EXPECT_EQ(240u, coded_height); + EXPECT_EQ(855u, pixel_width); + EXPECT_EQ(857u, pixel_height); +} + } // namespace H265 } // namespace media } // namespace edash_packager