From cc9a691aef946dfb4d68077d3a741ef1b88d2f21 Mon Sep 17 00:00:00 2001 From: Caitlin O'Callaghan <38890251+CaitlinOCallaghan@users.noreply.github.com> Date: Fri, 4 Aug 2023 09:00:59 -0700 Subject: [PATCH] feat: Generate the entire AV1 codec string when the colr atom is present (#1205) As per the AV1 spec, the codec string may contain optional color values. This extracts the missing color information from the mp4 `colr` atom, if present, and generates the full AV1 codec string. Closes #1007 --- packager/media/base/fourccs.h | 3 ++ .../codecs/av1_codec_configuration_record.cc | 19 +++++++++-- .../codecs/av1_codec_configuration_record.h | 5 +++ ...av1_codec_configuration_record_unittest.cc | 4 +++ packager/media/formats/mp4/box_definitions.cc | 34 +++++++++++++++++++ packager/media/formats/mp4/box_definitions.h | 11 ++++++ .../formats/mp4/box_definitions_unittest.cc | 9 +++++ .../media/formats/mp4/mp4_media_parser.cc | 10 +++++- 8 files changed, 92 insertions(+), 3 deletions(-) diff --git a/packager/media/base/fourccs.h b/packager/media/base/fourccs.h index 12884a6db7..c721f2c81f 100644 --- a/packager/media/base/fourccs.h +++ b/packager/media/base/fourccs.h @@ -35,6 +35,7 @@ enum FourCC : uint32_t { FOURCC_cbcs = 0x63626373, FOURCC_cenc = 0x63656e63, FOURCC_cens = 0x63656e73, + FOURCC_colr = 0x636f6c72, FOURCC_co64 = 0x636f3634, FOURCC_cmfc = 0x636d6663, FOURCC_cmfs = 0x636d6673, @@ -102,6 +103,8 @@ enum FourCC : uint32_t { FOURCC_mp4v = 0x6d703476, FOURCC_mvex = 0x6d766578, FOURCC_mvhd = 0x6d766864, + FOURCC_nclc = 0x6e636c63, + FOURCC_nclx = 0x6e636c78, FOURCC_nmhd = 0x6e6d6864, FOURCC_pasp = 0x70617370, FOURCC_payl = 0x7061796c, diff --git a/packager/media/codecs/av1_codec_configuration_record.cc b/packager/media/codecs/av1_codec_configuration_record.cc index c88b313360..13eab3c1a7 100644 --- a/packager/media/codecs/av1_codec_configuration_record.cc +++ b/packager/media/codecs/av1_codec_configuration_record.cc @@ -81,12 +81,27 @@ bool AV1CodecConfigurationRecord::Parse(const uint8_t* data, size_t data_size) { // mandatory fields. // All the other fields (including their leading '.') are optional, mutually // inclusive (all or none) fields. -// Since some of the optional fields (e.g. colorPrimaries) are not present in -// AV1CodecConfigurationRecord, we omit all the optional fields. + +// When color info is NOT available, generate the basic codec string without the +// optional fields std::string AV1CodecConfigurationRecord::GetCodecString() const { return base::StringPrintf("av01.%d.%02d%c.%02d", profile_, level_, tier_ ? 'H' : 'M', bit_depth_); } +// When color info IS available, generate the full codec string with optional +// fields +std::string AV1CodecConfigurationRecord::GetCodecString( + uint16_t color_primaries, + uint16_t transfer_characteristics, + uint16_t matrix_coefficients, + uint8_t video_full_range_flag) const { + return base::StringPrintf( + "av01.%d.%02d%c.%02d.%d.%d%d%d.%02d.%02d.%02d.%d", profile_, level_, + tier_ ? 'H' : 'M', bit_depth_, mono_chrome_, chroma_subsampling_x_, + chroma_subsampling_y_, chroma_sample_position_, color_primaries, + transfer_characteristics, matrix_coefficients, video_full_range_flag); +} + } // namespace media } // namespace shaka diff --git a/packager/media/codecs/av1_codec_configuration_record.h b/packager/media/codecs/av1_codec_configuration_record.h index 463e3ba708..0517ab9e81 100644 --- a/packager/media/codecs/av1_codec_configuration_record.h +++ b/packager/media/codecs/av1_codec_configuration_record.h @@ -33,6 +33,11 @@ class AV1CodecConfigurationRecord { /// @return The codec string. std::string GetCodecString() const; + std::string GetCodecString(uint16_t color_primaries, + uint16_t transfer_characteristics, + uint16_t matrix_coefficients, + uint8_t video_full_range_flag) const; + private: int profile_ = 0; int level_ = 0; diff --git a/packager/media/codecs/av1_codec_configuration_record_unittest.cc b/packager/media/codecs/av1_codec_configuration_record_unittest.cc index 8ec180a086..34360f0d79 100644 --- a/packager/media/codecs/av1_codec_configuration_record_unittest.cc +++ b/packager/media/codecs/av1_codec_configuration_record_unittest.cc @@ -28,6 +28,8 @@ TEST(AV1CodecConfigurationRecordTest, Success) { std::end(kAV1CodecConfigurationData)))); EXPECT_EQ(av1_config.GetCodecString(), "av01.0.04M.10"); + EXPECT_EQ(av1_config.GetCodecString(10, 8, 4, 1), + "av01.0.04M.10.0.112.10.08.04.1"); } TEST(AV1CodecConfigurationRecordTest, Success2) { @@ -47,6 +49,8 @@ TEST(AV1CodecConfigurationRecordTest, Success2) { std::end(kAV1CodecConfigurationData)))); EXPECT_EQ(av1_config.GetCodecString(), "av01.1.21H.12"); + EXPECT_EQ(av1_config.GetCodecString(1, 1, 1, 0), + "av01.1.21H.12.1.010.01.01.01.0"); } TEST(AV1CodecConfigurationRecordTest, InsufficientData) { diff --git a/packager/media/formats/mp4/box_definitions.cc b/packager/media/formats/mp4/box_definitions.cc index b4c35714d3..06a55a5c04 100644 --- a/packager/media/formats/mp4/box_definitions.cc +++ b/packager/media/formats/mp4/box_definitions.cc @@ -1462,6 +1462,39 @@ size_t CodecConfiguration::ComputeSizeInternal() { return HeaderSize() + (box_type == FOURCC_vpcC ? 4 : 0) + data.size(); } +ColorParameters::ColorParameters() = default; +ColorParameters::~ColorParameters() = default; + +FourCC ColorParameters::BoxType() const { + return FOURCC_colr; +} + +bool ColorParameters::ReadWriteInternal(BoxBuffer* buffer) { + if (buffer->reader()) { + RCHECK((buffer->reader())->ReadFourCC(&color_parameter_type) && + (buffer->reader())->Read2(&color_primaries) && + (buffer->reader())->Read2(&transfer_characteristics) && + (buffer->reader())->Read2(&matrix_coefficients)); + // Type nclc does not contain video_full_range_flag data, and thus, it has 1 + // less byte than nclx. Only extract video_full_range_flag if of type nclx. + if (color_parameter_type == FOURCC_nclx) { + RCHECK((buffer->reader())->Read1(&video_full_range_flag)); + } + } + // TODO(caitlinocallaghan) Add the ability to write the colr atom and include + // it in the muxed mp4. + return true; +} + +size_t ColorParameters::ComputeSizeInternal() { + // This box is optional. Skip it if it is not initialized. + if (color_parameter_type == FOURCC_NULL) + return 0; + return HeaderSize() + kFourCCSize + sizeof(color_primaries) + + sizeof(transfer_characteristics) + sizeof(matrix_coefficients) + + sizeof(video_full_range_flag); +} + PixelAspectRatio::PixelAspectRatio() = default; PixelAspectRatio::~PixelAspectRatio() = default; @@ -1597,6 +1630,7 @@ bool VideoSampleEntry::ReadWriteInternal(BoxBuffer* buffer) { RCHECK(buffer->ReadWriteChild(&extra_codec_config)); } + RCHECK(buffer->TryReadWriteChild(&colr)); RCHECK(buffer->TryReadWriteChild(&pixel_aspect)); // Somehow Edge does not support having sinf box before codec_configuration, diff --git a/packager/media/formats/mp4/box_definitions.h b/packager/media/formats/mp4/box_definitions.h index 989d49c8af..ba9c49504a 100644 --- a/packager/media/formats/mp4/box_definitions.h +++ b/packager/media/formats/mp4/box_definitions.h @@ -268,6 +268,16 @@ struct CodecConfiguration : Box { std::vector data; }; +struct ColorParameters : Box { + DECLARE_BOX_METHODS(ColorParameters); + + FourCC color_parameter_type = FOURCC_NULL; + uint16_t color_primaries = 1; + uint16_t transfer_characteristics = 1; + uint16_t matrix_coefficients = 1; + uint8_t video_full_range_flag = 0; +}; + struct PixelAspectRatio : Box { DECLARE_BOX_METHODS(PixelAspectRatio); @@ -297,6 +307,7 @@ struct VideoSampleEntry : Box { uint16_t width = 0u; uint16_t height = 0u; + ColorParameters colr; PixelAspectRatio pixel_aspect; ProtectionSchemeInfo sinf; CodecConfiguration codec_configuration; diff --git a/packager/media/formats/mp4/box_definitions_unittest.cc b/packager/media/formats/mp4/box_definitions_unittest.cc index 075e490779..1d96251fe5 100644 --- a/packager/media/formats/mp4/box_definitions_unittest.cc +++ b/packager/media/formats/mp4/box_definitions_unittest.cc @@ -326,6 +326,14 @@ class BoxDefinitionsTestGeneral : public testing::Test { Modify(&metadata->id3v2); } + void Fill(ColorParameters* colr) { + colr->color_parameter_type = FOURCC_nclc; + colr->color_primaries = 9; + colr->transfer_characteristics = 16; + colr->matrix_coefficients = 9; + colr->video_full_range_flag = 0; + } + void Fill(PixelAspectRatio* pasp) { pasp->h_spacing = 5; pasp->v_spacing = 8; @@ -360,6 +368,7 @@ class BoxDefinitionsTestGeneral : public testing::Test { entry->data_reference_index = 1; entry->width = 800; entry->height = 600; + Fill(&entry->colr); Fill(&entry->pixel_aspect); Fill(&entry->sinf); Fill(&entry->codec_configuration); diff --git a/packager/media/formats/mp4/mp4_media_parser.cc b/packager/media/formats/mp4/mp4_media_parser.cc index a7fc1f8a20..8f2b504851 100644 --- a/packager/media/formats/mp4/mp4_media_parser.cc +++ b/packager/media/formats/mp4/mp4_media_parser.cc @@ -604,7 +604,15 @@ bool MP4MediaParser::ParseMoov(BoxReader* reader) { LOG(ERROR) << "Failed to parse av1c."; return false; } - codec_string = av1_config.GetCodecString(); + // Generate the full codec string if the colr atom is present. + if (entry.colr.color_parameter_type != FOURCC_NULL) { + codec_string = av1_config.GetCodecString( + entry.colr.color_primaries, entry.colr.transfer_characteristics, + entry.colr.matrix_coefficients, + entry.colr.video_full_range_flag); + } else { + codec_string = av1_config.GetCodecString(); + } break; } case FOURCC_avc1: