Mirror-Tools
/
shaka-packager
mirror of https://github.com/shaka-project/shaka-packager.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: Add IAMF support (#1415) (#1416)

This commit is contained in:
Felicia Lim 2024-10-25 09:56:28 -07:00 committed by GitHub
parent d88ed2798c
commit dc6196d3d1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
37 changed files with 858 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CONTRIBUTORS
View File

@ -32,6 +32,7 @@ Daniel Cantarín <canta@canta.com.ar>
David Cavar <pal3thorn@gmail.com>
Dennis E. Mungai (Brainiarc7) <dmngaie@gmail.com>
Evgeny Zajcev <zevlg@yandex.ru>
Felicia Lim <flim@google.com>
Gabe Kopley <gabe@philo.com>
Geoff Jukes <geoff@jukes.org>
Haoming Chen <hmchen@google.com>

1
README.md
View File

@ -46,6 +46,7 @@ Shaka Packager supports:
| FLAC | I / O | - | - | - | - |
| Opus | I / O³ | I / O | - | - | - |
| Vorbis | - | I / O | - | - | - |
| IAMF | I / O | - | - | - | - |
NOTES:
- I for input and O for output.

32
packager/app/test/packager_test.py
View File

@ -1104,6 +1104,38 @@ class PackagerFunctionalTest(PackagerAppTest):
self._GetFlags(output_dash=True, output_hls=True))
self._CheckTestResults('av1-webm')
def testIamfWithBaseProfileAndPcm(self):
self.assertPackageSuccess(
self._GetStreams(['audio'],
output_format='mp4',
test_files=['bear-iamf-base-pcm.mp4']),
self._GetFlags(output_dash=True, output_hls=True))
self._CheckTestResults('iamf-base-pcm-mp4')
def testIamfWithBaseProfileAndOpus(self):
self.assertPackageSuccess(
self._GetStreams(['audio'],
output_format='mp4',
test_files=['bear-iamf-base-opus.mp4']),
self._GetFlags(output_dash=True, output_hls=True))
self._CheckTestResults('iamf-base-opus-mp4')
def testIamfWithSimpleProfileAndAacLc(self):
self.assertPackageSuccess(
self._GetStreams(['audio'],
output_format='mp4',
test_files=['bear-iamf-simple-aac-lc.mp4']),
self._GetFlags(output_dash=True, output_hls=True))
self._CheckTestResults('iamf-simple-aac-lc-mp4')
def testIamfWithSimpleProfileAndFlac(self):
self.assertPackageSuccess(
self._GetStreams(['audio'],
output_format='mp4',
test_files=['bear-iamf-simple-flac.mp4']),
self._GetFlags(output_dash=True, output_hls=True))
self._CheckTestResults('iamf-simple-flac-mp4')
def testEncryption(self):
self.assertPackageSuccess(
self._GetStreams(['audio', 'video']),

BIN
packager/app/test/testdata/iamf-base-opus-mp4/bear-iamf-base-opus-audio.mp4 vendored Normal file
View File

Binary file not shown.

9
packager/app/test/testdata/iamf-base-opus-mp4/output.m3u8 vendored Normal file
View File

@ -0,0 +1,9 @@
#EXTM3U
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
#EXT-X-INDEPENDENT-SEGMENTS
#EXT-X-MEDIA:TYPE=AUDIO,URI="stream_0.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="0"
#EXT-X-STREAM-INF:BANDWIDTH=136796,AVERAGE-BANDWIDTH=134443,CODECS="iamf.001.001.Opus",AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
stream_0.m3u8

15
packager/app/test/testdata/iamf-base-opus-mp4/output.mpd vendored Normal file
View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.739958S">
<Period id="0">
<AdaptationSet id="0" contentType="audio" subsegmentAlignment="true">
<Representation id="0" bandwidth="136796" codecs="iamf.001.001.Opus" mimeType="audio/mp4" audioSamplingRate="0">
<AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="0"/>
<BaseURL>bear-iamf-base-opus-audio.mp4</BaseURL>
<SegmentBase indexRange="892-959" timescale="48000">
<Initialization range="0-891"/>
</SegmentBase>
</Representation>
</AdaptationSet>
</Period>
</MPD>

16
packager/app/test/testdata/iamf-base-opus-mp4/stream_0.m3u8 vendored Normal file
View File

@ -0,0 +1,16 @@
#EXTM3U
#EXT-X-VERSION:6
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
#EXT-X-TARGETDURATION:2
#EXT-X-PLAYLIST-TYPE:VOD
#EXT-X-MAP:URI="bear-iamf-base-opus-audio.mp4",BYTERANGE="892@0"
#EXTINF:1.014,
#EXT-X-BYTERANGE:16835@960
bear-iamf-base-opus-audio.mp4
#EXTINF:1.000,
#EXT-X-BYTERANGE:16789
bear-iamf-base-opus-audio.mp4
#EXTINF:0.726,
#EXT-X-BYTERANGE:12422
bear-iamf-base-opus-audio.mp4
#EXT-X-ENDLIST

BIN
packager/app/test/testdata/iamf-base-pcm-mp4/bear-iamf-base-pcm-audio.mp4 vendored Normal file
View File

Binary file not shown.

9
packager/app/test/testdata/iamf-base-pcm-mp4/output.m3u8 vendored Normal file
View File

@ -0,0 +1,9 @@
#EXTM3U
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
#EXT-X-INDEPENDENT-SEGMENTS
#EXT-X-MEDIA:TYPE=AUDIO,URI="stream_0.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="0"
#EXT-X-STREAM-INF:BANDWIDTH=2177226,AVERAGE-BANDWIDTH=1803234,CODECS="iamf.001.001.ipcm",AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
stream_0.m3u8

15
packager/app/test/testdata/iamf-base-pcm-mp4/output.mpd vendored Normal file
View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.739958S">
<Period id="0">
<AdaptationSet id="0" contentType="audio" subsegmentAlignment="true">
<Representation id="0" bandwidth="2177226" codecs="iamf.001.001.ipcm" mimeType="audio/mp4" audioSamplingRate="0">
<AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="0"/>
<BaseURL>bear-iamf-base-pcm-audio.mp4</BaseURL>
<SegmentBase indexRange="850-917" timescale="48000">
<Initialization range="0-849"/>
</SegmentBase>
</Representation>
</AdaptationSet>
</Period>
</MPD>

16
packager/app/test/testdata/iamf-base-pcm-mp4/stream_0.m3u8 vendored Normal file
View File

@ -0,0 +1,16 @@
#EXTM3U
#EXT-X-VERSION:6
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
#EXT-X-TARGETDURATION:1
#EXT-X-PLAYLIST-TYPE:VOD
#EXT-X-MAP:URI="bear-iamf-base-pcm-audio.mp4",BYTERANGE="850@0"
#EXTINF:1.000,
#EXT-X-BYTERANGE:208108@918
bear-iamf-base-pcm-audio.mp4
#EXTINF:1.000,
#EXT-X-BYTERANGE:208108
bear-iamf-base-pcm-audio.mp4
#EXTINF:0.740,
#EXT-X-BYTERANGE:201382
bear-iamf-base-pcm-audio.mp4
#EXT-X-ENDLIST

BIN
packager/app/test/testdata/iamf-simple-aac-lc-mp4/bear-iamf-simple-aac-lc-audio.mp4 vendored Normal file
View File

Binary file not shown.

9
packager/app/test/testdata/iamf-simple-aac-lc-mp4/output.m3u8 vendored Normal file
View File

@ -0,0 +1,9 @@
#EXTM3U
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
#EXT-X-INDEPENDENT-SEGMENTS
#EXT-X-MEDIA:TYPE=AUDIO,URI="stream_0.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="0"
#EXT-X-STREAM-INF:BANDWIDTH=604125,AVERAGE-BANDWIDTH=590976,CODECS="iamf.000.000.mp4a.40.2",AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
stream_0.m3u8

15
packager/app/test/testdata/iamf-simple-aac-lc-mp4/output.mpd vendored Normal file
View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.739958S">
<Period id="0">
<AdaptationSet id="0" contentType="audio" subsegmentAlignment="true">
<Representation id="0" bandwidth="604125" codecs="iamf.000.000.mp4a.40.2" mimeType="audio/mp4" audioSamplingRate="0">
<AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="0"/>
<BaseURL>bear-iamf-simple-aac-lc-audio.mp4</BaseURL>
<SegmentBase indexRange="900-967" timescale="48000">
<Initialization range="0-899"/>
</SegmentBase>
</Representation>
</AdaptationSet>
</Period>
</MPD>

16
packager/app/test/testdata/iamf-simple-aac-lc-mp4/stream_0.m3u8 vendored Normal file
View File

@ -0,0 +1,16 @@
#EXTM3U
#EXT-X-VERSION:6
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
#EXT-X-TARGETDURATION:2
#EXT-X-PLAYLIST-TYPE:VOD
#EXT-X-MAP:URI="bear-iamf-simple-aac-lc-audio.mp4",BYTERANGE="900@0"
#EXTINF:1.003,
#EXT-X-BYTERANGE:75717@968
bear-iamf-simple-aac-lc-audio.mp4
#EXTINF:1.003,
#EXT-X-BYTERANGE:72441
bear-iamf-simple-aac-lc-audio.mp4
#EXTINF:0.735,
#EXT-X-BYTERANGE:54248
bear-iamf-simple-aac-lc-audio.mp4
#EXT-X-ENDLIST

BIN
packager/app/test/testdata/iamf-simple-flac-mp4/bear-iamf-simple-flac-audio.mp4 vendored Normal file
View File

Binary file not shown.

9
packager/app/test/testdata/iamf-simple-flac-mp4/output.m3u8 vendored Normal file
View File

@ -0,0 +1,9 @@
#EXTM3U
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
#EXT-X-INDEPENDENT-SEGMENTS
#EXT-X-MEDIA:TYPE=AUDIO,URI="stream_0.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="0"
#EXT-X-STREAM-INF:BANDWIDTH=977240,AVERAGE-BANDWIDTH=926327,CODECS="iamf.000.000.fLaC",AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
stream_0.m3u8

15
packager/app/test/testdata/iamf-simple-flac-mp4/output.mpd vendored Normal file
View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.739958S">
<Period id="0">
<AdaptationSet id="0" contentType="audio" subsegmentAlignment="true">
<Representation id="0" bandwidth="977240" codecs="iamf.000.000.fLaC" mimeType="audio/mp4" audioSamplingRate="0">
<AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="0"/>
<BaseURL>bear-iamf-simple-flac-audio.mp4</BaseURL>
<SegmentBase indexRange="883-950" timescale="48000">
<Initialization range="0-882"/>
</SegmentBase>
</Representation>
</AdaptationSet>
</Period>
</MPD>

16
packager/app/test/testdata/iamf-simple-flac-mp4/stream_0.m3u8 vendored Normal file
View File

@ -0,0 +1,16 @@
#EXTM3U
#EXT-X-VERSION:6
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
#EXT-X-TARGETDURATION:1
#EXT-X-PLAYLIST-TYPE:VOD
#EXT-X-MAP:URI="bear-iamf-simple-flac-audio.mp4",BYTERANGE="883@0"
#EXTINF:1.000,
#EXT-X-BYTERANGE:116763@951
bear-iamf-simple-flac-audio.mp4
#EXTINF:1.000,
#EXT-X-BYTERANGE:122155
bear-iamf-simple-flac-audio.mp4
#EXTINF:0.740,
#EXT-X-BYTERANGE:78344
bear-iamf-simple-flac-audio.mp4
#EXT-X-ENDLIST

1
packager/media/base/CMakeLists.txt
View File

@ -80,6 +80,7 @@ target_link_libraries(media_handler_test_base
add_executable(media_base_unittest
aes_cryptor_unittest.cc
aes_pattern_cryptor_unittest.cc
audio_stream_info_unittest.cc
audio_timestamp_helper_unittest.cc
bit_reader_unittest.cc
bit_writer_unittest.cc

35
packager/media/base/audio_stream_info.cc
View File

@ -45,6 +45,8 @@ std::string AudioCodecToString(Codec codec) {
return "AC4";
case kCodecFlac:
return "FLAC";
case kCodecIAMF:
return "IAMF";
case kCodecOpus:
return "Opus";
case kCodecVorbis:
@ -168,6 +170,39 @@ std::string AudioStreamInfo::GetCodecString(Codec codec,
(audio_object_type & 0x18) >> 3, audio_object_type & 0x7);
case kCodecFlac:
return "flac";
case kCodecIAMF: {
// https://aomediacodec.github.io/iamf/#codecsparameter
// The codecs parameter string is composed as
//
// iamf.xxx.yyy.<standalone_codec_string>
//
// - xxx is the IAMF primary profile
// - yyy is the IAMF additional profile
// - <standalone_codec_string> are the elements of the codecs parameter
// string if that stream was carried in its own track
//
// audio_object_type is composed of primary_profile (2 bits),
// additional_profile (2 bits) and (IAMF codec - kCodecAudio) (4 bits).
const int iamf_codec = (audio_object_type & 0xF) + kCodecAudio;
const std::string iamf_codec_string =
absl::StrFormat("iamf.%03d.%03d", (audio_object_type & 0xC0) >> 6,
(audio_object_type & 0x30) >> 4);
switch (iamf_codec) {
case kCodecOpus:
return absl::StrFormat("%s.%s", iamf_codec_string, "Opus");
case kCodecAAC:
return absl::StrFormat("%s.%s", iamf_codec_string, "mp4a.40.2");
case kCodecFlac:
return absl::StrFormat("%s.%s", iamf_codec_string, "fLaC");
case kCodecPcm:
return absl::StrFormat("%s.%s", iamf_codec_string, "ipcm");
default:
LOG(WARNING) << "Unknown IAMF codec: " << iamf_codec;
return "unknown";
}
}
case kCodecOpus:
return "opus";
case kCodecMP3:

73
packager/media/base/audio_stream_info_unittest.cc Normal file
View File

@ -0,0 +1,73 @@
// Copyright 2024 Google LLC. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#include <packager/media/base/audio_stream_info.h>
#include <gtest/gtest.h>
namespace shaka {
namespace media {
const int kSimpleProfile = 0;
const int kBaseProfile = 1;
TEST(AudioStreamInfo, IamfGetCodecStringForSimpleProfilesAndPcm) {
const uint8_t audio_object_type =
((kSimpleProfile << 6) | // primary_profile
((kSimpleProfile << 4) & 0x3F) | // additional_profile
((kCodecPcm - kCodecAudio) & 0xF)); // IAMF codec
std::string codec_string =
AudioStreamInfo::GetCodecString(kCodecIAMF, audio_object_type);
EXPECT_EQ("iamf.000.000.ipcm", codec_string);
}
TEST(AudioStreamInfo, IamfGetCodecStringForSimpleProfilesAndOpus) {
const uint8_t audio_object_type =
((kSimpleProfile << 6) | // primary_profile
((kSimpleProfile << 4) & 0x3F) | // additional_profile
((kCodecOpus - kCodecAudio) & 0xF)); // IAMF codec
std::string codec_string =
AudioStreamInfo::GetCodecString(kCodecIAMF, audio_object_type);
EXPECT_EQ("iamf.000.000.Opus", codec_string);
}
TEST(AudioStreamInfo, IamfGetCodecStringForSimpleProfilesAndMp4a) {
const uint8_t audio_object_type =
((kSimpleProfile << 6) | // primary_profile
((kSimpleProfile << 4) & 0x3F) | // additional_profile
((kCodecAAC - kCodecAudio) & 0xF)); // IAMF codec
std::string codec_string =
AudioStreamInfo::GetCodecString(kCodecIAMF, audio_object_type);
EXPECT_EQ("iamf.000.000.mp4a.40.2", codec_string);
}
TEST(AudioStreamInfo, IamfGetCodecStringForSimpleProfilesAndFlac) {
const uint8_t audio_object_type =
((kSimpleProfile << 6) | // primary_profile
((kSimpleProfile << 4) & 0x3F) | // additional_profile
((kCodecFlac - kCodecAudio) & 0xF)); // IAMF codec
std::string codec_string =
AudioStreamInfo::GetCodecString(kCodecIAMF, audio_object_type);
EXPECT_EQ("iamf.000.000.fLaC", codec_string);
}
TEST(AudioStreamInfo, IamfGetCodecStringForBaseProfilesAndPcm) {
const uint8_t audio_object_type =
((kBaseProfile << 6) | // primary_profile = 1
((kBaseProfile << 4) & 0x3F) | // additional_profile = 1
((kCodecPcm - kCodecAudio) & 0xF)); // IAMF codec
std::string codec_string =
AudioStreamInfo::GetCodecString(kCodecIAMF, audio_object_type);
EXPECT_EQ("iamf.001.001.ipcm", codec_string);
}
} // namespace media
} // namespace shaka

3
packager/media/base/fourccs.h
View File

@ -86,7 +86,10 @@ enum FourCC : uint32_t {
FOURCC_hvc1 = 0x68766331,
FOURCC_hvcC = 0x68766343,
FOURCC_hvcE = 0x68766345,
FOURCC_iamf = 0x69616d66,
FOURCC_iacb = 0x69616362,
FOURCC_iden = 0x6964656e,
FOURCC_ipcm = 0x6970636d,
FOURCC_iso6 = 0x69736f36,
FOURCC_iso8 = 0x69736f38,
FOURCC_isom = 0x69736f6d,

2
packager/media/base/stream_info.h
View File

@ -53,7 +53,9 @@ enum Codec {
kCodecDTSX,
kCodecEAC3,
kCodecFlac,
kCodecIAMF,
kCodecOpus,
kCodecPcm,
kCodecVorbis,
kCodecMP3,
kCodecMha1,

2
packager/media/codecs/CMakeLists.txt
View File

@ -24,6 +24,7 @@ add_library(media_codecs STATIC
h26x_byte_to_unit_stream_converter.cc
hevc_decoder_configuration_record.cc
hls_audio_util.cc
iamf_audio_util.cc
nal_unit_to_byte_stream_converter.cc
nalu_reader.cc
video_slice_header_parser.cc
@ -52,6 +53,7 @@ add_executable(media_codecs_unittest
h26x_bit_reader_unittest.cc
hevc_decoder_configuration_record_unittest.cc
hls_audio_util_unittest.cc
iamf_audio_util_unittest.cc
nal_unit_to_byte_stream_converter_unittest.cc
nalu_reader_unittest.cc
video_slice_header_parser_unittest.cc

209
packager/media/codecs/iamf_audio_util.cc Normal file
View File

@ -0,0 +1,209 @@
// Copyright 2024 Google LLC. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#include <packager/media/codecs/iamf_audio_util.h>
#include <iomanip>
#include <packager/media/base/bit_reader.h>
#include <packager/media/base/fourccs.h>
#include <packager/media/base/rcheck.h>
#include <packager/media/base/stream_info.h>
namespace shaka {
namespace media {
namespace {
const uint8_t kMaxIamfProfile = 2;
// 3.2. OBU type
// Only the IA Sequence Header and Codec Configs are used in this
// implementation.
enum ObuType {
OBU_IA_Codec_Config = 0,
OBU_IA_Sequence_Header = 31,
};
// 8.1.1. leb128(). Unsigned integer represented by a variable number of
// little-endian bytes.
bool ReadLeb128(BitReader& reader, size_t* size, size_t* leb128_bytes) {
size_t value = 0;
size_t bytes_read = 0;
for (int i = 0; i < 8; i++) {
size_t leb128_byte = 0;
RCHECK(reader.ReadBits(8, &leb128_byte));
value |= (leb128_byte & 0x7f) << (i * 7);
bytes_read += 1;
if (!(leb128_byte & 0x80))
break;
}
// It is a requirement of bitstream conformance that the value returned from
// the leb128 parsing process is less than or equal to (1<<32) - 1.
RCHECK(value <= ((1ull << 32) - 1));
if (size != nullptr) {
*size = value;
}
if (leb128_bytes != nullptr) {
*leb128_bytes = bytes_read;
}
return true;
}
bool ParseObuHeader(BitReader& reader, int& obu_type, size_t& obu_size) {
size_t leb128_bytes;
bool obu_trimming_status_flag;
bool obu_extension_flag;
RCHECK(reader.ReadBits(5, &obu_type));
RCHECK(reader.SkipBits(1)); // Skip obu_redundant_copy
RCHECK(reader.ReadBits(1, &obu_trimming_status_flag));
RCHECK(reader.ReadBits(1, &obu_extension_flag));
RCHECK(ReadLeb128(reader, &obu_size, &leb128_bytes));
if (obu_trimming_status_flag) {
// Skip num_samples_to_trim_at_end
RCHECK(ReadLeb128(reader, nullptr, &leb128_bytes));
obu_size -= leb128_bytes;
// Skip num_samples_to_trim_at_start
RCHECK(ReadLeb128(reader, nullptr, &leb128_bytes));
obu_size -= leb128_bytes;
}
if (obu_extension_flag) {
size_t extension_header_size;
RCHECK(ReadLeb128(reader, &extension_header_size, &leb128_bytes));
obu_size -= leb128_bytes;
RCHECK(reader.SkipBits(extension_header_size * 8));
obu_size -= extension_header_size * 8;
}
return true;
}
bool ParseSequenceHeaderObu(BitReader& reader,
uint8_t& primary_profile,
uint8_t& additional_profile) {
uint32_t ia_code;
RCHECK(reader.ReadBits(32, &ia_code));
if (ia_code != FOURCC_iamf) {
LOG(WARNING) << "Unknown ia_code= " << std::setfill('0') << std::setw(8)
<< std::hex << ia_code;
return false;
}
RCHECK(reader.ReadBits(8, &primary_profile));
if (primary_profile > kMaxIamfProfile) {
LOG(WARNING) << "Unknown primary_profile= " << primary_profile;
return false;
}
RCHECK(reader.ReadBits(8, &additional_profile));
if (additional_profile > kMaxIamfProfile) {
LOG(WARNING) << "Unknown additional_profile= " << additional_profile;
return false;
}
return true;
}
bool ParseCodecConfigObu(BitReader& reader, size_t obu_size, Codec& codec) {
uint32_t codec_id;
size_t leb128_bytes;
// Skip codec_config_id
RCHECK(ReadLeb128(reader, nullptr, &leb128_bytes));
obu_size -= leb128_bytes;
RCHECK(reader.ReadBits(32, &codec_id));
obu_size -= 4;
// Skip the remainder of the OBU.
RCHECK(reader.SkipBits(obu_size * 8));
switch (codec_id) {
case FOURCC_Opus:
codec = kCodecOpus;
break;
case FOURCC_mp4a:
codec = kCodecAAC;
break;
case FOURCC_fLaC:
codec = kCodecFlac;
break;
case FOURCC_ipcm:
codec = kCodecPcm;
break;
default:
LOG(WARNING) << "Unknown codec_id= " << std::setfill('0') << std::setw(8)
<< std::hex << codec_id;
return false;
}
return true;
}
} // namespace
bool GetIamfCodecStringInfo(const std::vector<uint8_t>& iacb,
uint8_t& codec_string_info) {
uint8_t primary_profile;
uint8_t additional_profile;
Codec iamf_codec; // codec used to encode IAMF audio substreams
int obu_type;
size_t obu_size;
BitReader reader(iacb.data(), iacb.size());
// configurationVersion
RCHECK(reader.SkipBits(8));
// configOBUs_size
RCHECK(ReadLeb128(reader, &obu_size, nullptr));
while (reader.bits_available() > 0) {
RCHECK(ParseObuHeader(reader, obu_type, obu_size));
switch (obu_type) {
case OBU_IA_Sequence_Header:
RCHECK(ParseSequenceHeaderObu(reader, primary_profile,
additional_profile));
break;
case OBU_IA_Codec_Config:
RCHECK(ParseCodecConfigObu(reader, obu_size, iamf_codec));
break;
default:
// Skip other irrelevant OBUs.
RCHECK(reader.SkipBits(obu_size * 8));
break;
}
}
// In IAMF v1.1 (https://aomediacodec.github.io/iamf),
// the valid values of primary_profile and additional_profile are {0, 1, 2}.
// The valid codec_ids are {Opus, mp4a, fLaC, ipcm}.
//
// This can be represented in uint8_t as:
// primary_profile (2bits) + additional_profile (2bits) + iamf_codec (4bits),
// where iamf_codec is represented using the Codec enum.
//
// Since iamf_codec is limited to 16 values, subtract the value of kCodecAudio
// to ensure it fits. If future audio codecs are added to the Codec enum,
// it may break the assumption that IAMF supported codecs are present within
// the first 16 audio codec entries.
// Further, if these values change in future version of IAMF, this format may
// need to be changed, and AudioStreamInfo::GetCodecString needs to be updated
// accordingly.
codec_string_info =
((primary_profile << 6) | ((additional_profile << 4) & 0x3F) |
((iamf_codec - kCodecAudio) & 0xF));
return true;
}
} // namespace media
} // namespace shaka

27
packager/media/codecs/iamf_audio_util.h Normal file
View File

@ -0,0 +1,27 @@
// Copyright 2024 Google LLC. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
//
// IAMF audio utility functions.
#ifndef PACKAGER_MEDIA_CODECS_IAMF_AUDIO_UTIL_H_
#define PACKAGER_MEDIA_CODECS_IAMF_AUDIO_UTIL_H_
#include <cstdint>
#include <vector>
namespace shaka {
namespace media {
/// Parse data from IAMFSpecific box and obtain the profile and codec
/// information needed to construct its Codec String (Section 6.4).
/// @return false if there are parsing errors.
bool GetIamfCodecStringInfo(const std::vector<uint8_t>& iamf_data,
uint8_t& codec_string_info);
} // namespace media
} // namespace shaka
#endif // PACKAGER_MEDIA_CODECS_IAMF_AUDIO_UTIL_H_

227
packager/media/codecs/iamf_audio_util_unittest.cc Normal file
View File

@ -0,0 +1,227 @@
// Copyright 2024 Google LLC. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#include <packager/media/codecs/iamf_audio_util.h>
#include <gtest/gtest.h>
#include <packager/media/base/stream_info.h>
#include <packager/media/test/test_data_util.h>
namespace shaka {
namespace media {
namespace {
const int kSimpleProfile = 0;
const int kBaseProfile = 1;
const std::vector<uint8_t> kSimpleIaSequenceObu = {
// OBU header
0xf8, // obu_type (5 bits), obu_redundant_copy (1 bit),
// obu_trimming_status_flag (1 bit), obu_extension_flag (1 bit)
0x06, // obu_size
// IASequenceHeaderOBU
0x69, 0x61, 0x6d, 0x66, // ia_code
0x00, // primary_profile = simple
0x00 // additional_profile = simple
};
const std::vector<uint8_t> kOpusCodecConfigObu = {
// OBU header
0x00, // obu_type (5 bits), obu_redundant_copy (1 bit),
// obu_trimming_status_flag (1 bit), obu_extension_flag (1 bit)
0x0A, // obu_size
// CodecConfigOBU
0xc8, 0x01, // codec_config_id
0x4f, 0x70, 0x75, 0x73, // codec_id = 'Opus'
0x46, 0x41, 0x4B, 0x45 // 'F''A''K''E' remainder codec config OBU
};
const std::vector<uint8_t> kFakeAudioElementObu = {
// OBU header
0x08, // obu_type (5 bits), obu_redundant_copy (1 bit),
// obu_trimming_status_flag (1 bit), obu_extension_flag (1 bit)
0x04, // obu_size
// 'F''A''K''E' AudioElementOBU
0x46, 0x41, 0x4B, 0x45};
const std::vector<uint8_t> kFakeMixPresentationObu = {
// OBU header
0x10, // obu_type (5 bits), obu_redundant_copy (1 bit),
// obu_trimming_status_flag (1 bit), obu_extension_flag (1 bit)
0x04, // obu_size
// 'F''A''K''E' MixPresentationOBU
0x46, 0x41, 0x4B, 0x45};
} // namespace
TEST(IamfAudioUtilTest, GetCodecStringInfoWithSimpleProfiles) {
std::vector<uint8_t> iacb = {
0x01, // configurationVersion
0x20 // configOBUs_size
};
iacb.insert(iacb.end(), kSimpleIaSequenceObu.begin(),
kSimpleIaSequenceObu.end());
iacb.insert(iacb.end(), kOpusCodecConfigObu.begin(),
kOpusCodecConfigObu.end());
iacb.insert(iacb.end(), kFakeAudioElementObu.begin(),
kFakeAudioElementObu.end());
iacb.insert(iacb.end(), kFakeMixPresentationObu.begin(),
kFakeMixPresentationObu.end());
uint8_t codec_string_info;
uint8_t expected_codec_string_info =
((kSimpleProfile << 6) | // primary_profile
((kSimpleProfile << 4) & 0x3F) | // additional_profile
((kCodecOpus - kCodecAudio) & 0xF)); // IAMF codec
ASSERT_TRUE(GetIamfCodecStringInfo(iacb, codec_string_info));
EXPECT_EQ(expected_codec_string_info, codec_string_info);
}
TEST(IamfAudioUtilTest, CodecStringInfoTestWithBaseProfiles) {
const std::vector<uint8_t> base_ia_sequence_obu = {
// OBU header
0xf8, // obu_type (5 bits), obu_redundant_copy (1 bit),
// obu_trimming_status_flag (1 bit), obu_extension_flag (1 bit)
0x06, // obu_size
// IASequenceHeaderOBU
0x69, 0x61, 0x6d, 0x66, // ia_code
0x01, // primary_profile = base
0x01 // additional_profile = base
};
std::vector<uint8_t> iacb = {
0x01, // configurationVersion
0x20 // configOBUs_size
};
iacb.insert(iacb.end(), base_ia_sequence_obu.begin(),
base_ia_sequence_obu.end());
iacb.insert(iacb.end(), kOpusCodecConfigObu.begin(),
kOpusCodecConfigObu.end());
iacb.insert(iacb.end(), kFakeAudioElementObu.begin(),
kFakeAudioElementObu.end());
iacb.insert(iacb.end(), kFakeMixPresentationObu.begin(),
kFakeMixPresentationObu.end());
uint8_t codec_string_info;
uint8_t expected_codec_string_info =
((kBaseProfile << 6) | // primary_profile = 1
((kBaseProfile << 4) & 0x3F) | // additional_profile = 1
((kCodecOpus - kCodecAudio) & 0xF)); // IAMF codec
ASSERT_TRUE(GetIamfCodecStringInfo(iacb, codec_string_info));
EXPECT_EQ(expected_codec_string_info, codec_string_info);
}
TEST(IamfAudioUtilTest, CodecStringInfoWithPcm) {
const std::vector<uint8_t> pcm_codec_config_obu = {
// OBU header
0x00, // obu_type (5 bits), obu_redundant_copy (1 bit),
// obu_trimming_status_flag (1 bit), obu_extension_flag (1 bit)
0x0A, // obu_size
// CodecConfigOBU
0xc8, 0x01, // codec_config_id
0x69, 0x70, 0x63, 0x6d, // codec_id = 'ipcm'
0x46, 0x41, 0x4B, 0x45 // 'F''A''K''E' remainder codec config OBU
};
std::vector<uint8_t> iacb = {
0x01, // configurationVersion
0x20 // configOBUs_size
};
iacb.insert(iacb.end(), kSimpleIaSequenceObu.begin(),
kSimpleIaSequenceObu.end());
iacb.insert(iacb.end(), pcm_codec_config_obu.begin(),
pcm_codec_config_obu.end());
iacb.insert(iacb.end(), kFakeAudioElementObu.begin(),
kFakeAudioElementObu.end());
iacb.insert(iacb.end(), kFakeMixPresentationObu.begin(),
kFakeMixPresentationObu.end());
uint8_t codec_string_info;
uint8_t expected_codec_string_info =
((kSimpleProfile << 6) | // primary_profile
((kSimpleProfile << 4) & 0x3F) | // additional_profile
((kCodecPcm - kCodecAudio) & 0xF)); // IAMF codec
ASSERT_TRUE(GetIamfCodecStringInfo(iacb, codec_string_info));
EXPECT_EQ(expected_codec_string_info, codec_string_info);
}
TEST(IamfAudioUtilTest, CodecStringInfoWithMp4a) {
const std::vector<uint8_t> mp4a_codec_config_obu = {
// OBU header
0x00, // obu_type (5 bits), obu_redundant_copy (1 bit),
// obu_trimming_status_flag (1 bit), obu_extension_flag (1 bit)
0x0A, // obu_size
// CodecConfigOBU
0xc8, 0x01, // codec_config_id
0x6d, 0x70, 0x34, 0x61, // codec_id = 'mp4a'
0x46, 0x41, 0x4B, 0x45 // 'F''A''K''E' remainder codec config OBU
};
std::vector<uint8_t> iacb = {
0x01, // configurationVersion
0x20 // configOBUs_size
};
iacb.insert(iacb.end(), kSimpleIaSequenceObu.begin(),
kSimpleIaSequenceObu.end());
iacb.insert(iacb.end(), mp4a_codec_config_obu.begin(),
mp4a_codec_config_obu.end());
iacb.insert(iacb.end(), kFakeAudioElementObu.begin(),
kFakeAudioElementObu.end());
iacb.insert(iacb.end(), kFakeMixPresentationObu.begin(),
kFakeMixPresentationObu.end());
uint8_t codec_string_info;
uint8_t expected_codec_string_info =
((kSimpleProfile << 6) | // primary_profile
((kSimpleProfile << 4) & 0x3F) | // additional_profile
((kCodecAAC - kCodecAudio) & 0xF)); // IAMF codec
ASSERT_TRUE(GetIamfCodecStringInfo(iacb, codec_string_info));
EXPECT_EQ(expected_codec_string_info, codec_string_info);
}
TEST(IamfAudioUtilTest, CodecStringInfoWithFlac) {
const std::vector<uint8_t> flac_codec_config_obu = {
// OBU header
0x00, // obu_type (5 bits), obu_redundant_copy (1 bit),
// obu_trimming_status_flag (1 bit), obu_extension_flag (1 bit)
0x0A, // obu_size
// CodecConfigOBU
0xc8, 0x01, // codec_config_id
0x66, 0x4C, 0x61, 0x43, // codec_id = 'fLaC'
0x46, 0x41, 0x4B, 0x45 // 'F''A''K''E' remainder codec config OBU
};
std::vector<uint8_t> iacb = {
0x01, // configurationVersion
0x20 // configOBUs_size
};
iacb.insert(iacb.end(), kSimpleIaSequenceObu.begin(),
kSimpleIaSequenceObu.end());
iacb.insert(iacb.end(), flac_codec_config_obu.begin(),
flac_codec_config_obu.end());
iacb.insert(iacb.end(), kFakeAudioElementObu.begin(),
kFakeAudioElementObu.end());
iacb.insert(iacb.end(), kFakeMixPresentationObu.begin(),
kFakeMixPresentationObu.end());
uint8_t codec_string_info;
uint8_t expected_codec_string_info =
((kSimpleProfile << 6) | // primary_profile
((kSimpleProfile << 4) & 0x3F) | // additional_profile
((kCodecFlac - kCodecAudio) & 0xF)); // IAMF codec
ASSERT_TRUE(GetIamfCodecStringInfo(iacb, codec_string_info));
EXPECT_EQ(expected_codec_string_info, codec_string_info);
}
} // namespace media
} // namespace shaka

24
packager/media/formats/mp4/box_definitions.cc
View File

@ -1958,6 +1958,27 @@ size_t OpusSpecific::ComputeSizeInternal() {
kOpusMagicSignatureSize;
}
IAMFSpecific::IAMFSpecific() = default;
IAMFSpecific::~IAMFSpecific() = default;
FourCC IAMFSpecific::BoxType() const {
return FOURCC_iacb;
}
bool IAMFSpecific::ReadWriteInternal(BoxBuffer* buffer) {
RCHECK(ReadWriteHeaderInternal(buffer));
size_t size = buffer->Reading() ? buffer->BytesLeft() : data.size();
RCHECK(buffer->ReadWriteVector(&data, size));
return true;
}
size_t IAMFSpecific::ComputeSizeInternal() {
// This box is optional. Skip it if not initialized.
if (data.empty())
return 0;
return HeaderSize() + data.size();
}
FlacSpecific::FlacSpecific() = default;
FlacSpecific::~FlacSpecific() = default;
@ -2040,6 +2061,7 @@ bool AudioSampleEntry::ReadWriteInternal(BoxBuffer* buffer) {
RCHECK(buffer->TryReadWriteChild(&dec3));
RCHECK(buffer->TryReadWriteChild(&dac4));
RCHECK(buffer->TryReadWriteChild(&dops));
RCHECK(buffer->TryReadWriteChild(&iacb));
RCHECK(buffer->TryReadWriteChild(&dfla));
RCHECK(buffer->TryReadWriteChild(&mhac));
RCHECK(buffer->TryReadWriteChild(&alac));
@ -2069,7 +2091,7 @@ size_t AudioSampleEntry::ComputeSizeInternal() {
esds.ComputeSize() + ddts.ComputeSize() + dac3.ComputeSize() +
dec3.ComputeSize() + dops.ComputeSize() + dfla.ComputeSize() +
dac4.ComputeSize() + mhac.ComputeSize() + udts.ComputeSize() +
alac.ComputeSize() +
alac.ComputeSize() + iacb.ComputeSize() +
// Reserved and predefined bytes.
6 + 8 + // 6 + 8 bytes reserved.
4; // 4 bytes predefined.

7
packager/media/formats/mp4/box_definitions.h
View File

@ -373,6 +373,12 @@ struct OpusSpecific : Box {
uint16_t preskip = 0u;
};
struct IAMFSpecific : Box {
DECLARE_BOX_METHODS(IAMFSpecific);
std::vector<uint8_t> data;
};
// FLAC specific decoder configuration box:
// https://github.com/xiph/flac/blob/master/doc/isoflac.txt
// We do not care about the actual data inside, which is simply copied over.
@ -416,6 +422,7 @@ struct AudioSampleEntry : Box {
EC3Specific dec3;
AC4Specific dac4;
OpusSpecific dops;
IAMFSpecific iacb;
FlacSpecific dfla;
MHAConfiguration mhac;
ALACSpecific alac;

15
packager/media/formats/mp4/box_definitions_unittest.cc
View File

@ -1316,6 +1316,21 @@ TEST_F(BoxDefinitionsTest, OpusSampleEntry) {
ASSERT_EQ(entry, entry_readback);
}
TEST_F(BoxDefinitionsTest, IAMFSampleEntry) {
AudioSampleEntry entry;
entry.format = FOURCC_iamf;
entry.data_reference_index = 2;
entry.channelcount = 0;
entry.samplesize = 16;
entry.samplerate = 0;
Fill(&entry.iacb);
entry.Write(this->buffer_.get());
AudioSampleEntry entry_readback;
ASSERT_TRUE(ReadBack(&entry_readback));
ASSERT_EQ(entry, entry_readback);
}
TEST_F(BoxDefinitionsTest, FlacSampleEntry) {
AudioSampleEntry entry;
entry.format = FOURCC_fLaC;

19
packager/media/formats/mp4/mp4_media_parser.cc
View File

@ -32,6 +32,7 @@
#include <packager/media/codecs/ec3_audio_util.h>
#include <packager/media/codecs/es_descriptor.h>
#include <packager/media/codecs/hevc_decoder_configuration_record.h>
#include <packager/media/codecs/iamf_audio_util.h>
#include <packager/media/codecs/vp_codec_configuration_record.h>
#include <packager/media/formats/mp4/box_definitions.h>
#include <packager/media/formats/mp4/box_reader.h>
@ -113,6 +114,10 @@ Codec FourCCToCodec(FourCC fourcc) {
return kCodecALAC;
case FOURCC_fLaC:
return kCodecFlac;
case FOURCC_iamf:
return kCodecIAMF;
case FOURCC_ipcm:
return kCodecPcm;
case FOURCC_mha1:
return kCodecMha1;
case FOURCC_mhm1:
@ -583,6 +588,13 @@ bool MP4MediaParser::ParseMoov(BoxReader* reader) {
codec_delay_ns =
entry.dops.preskip * kNanosecondsPerSecond / sampling_frequency;
break;
case FOURCC_iamf:
codec_config = entry.iacb.data;
if (!GetIamfCodecStringInfo(codec_config, audio_object_type)) {
LOG(ERROR) << "Failed to parse iamf.";
return false;
}
break;
case FOURCC_mha1:
case FOURCC_mhm1:
codec_config = entry.mhac.data;
@ -616,7 +628,12 @@ bool MP4MediaParser::ParseMoov(BoxReader* reader) {
const int16_t roll_distance_in_samples =
audio_roll_recovery_entries[0].roll_distance;
if (roll_distance_in_samples < 0) {
RCHECK(sampling_frequency != 0);
// IAMF requires the `samplerate` field to be set to 0.
// (https://aomediacodec.github.io/iamf/#iasampleentry-section)
if (actual_format == FOURCC_iamf)
continue;
RCHECK((sampling_frequency != 0));
seek_preroll_ns = kNanosecondsPerSecond *
(-roll_distance_in_samples) / sampling_frequency;
} else {

23
packager/media/formats/mp4/mp4_muxer.cc
View File

@ -98,6 +98,8 @@ FourCC CodecToFourCC(Codec codec, H26xStreamFormat h26x_stream_format) {
return FOURCC_fLaC;
case kCodecOpus:
return FOURCC_Opus;
case kCodecIAMF:
return FOURCC_iamf;
case kCodecMha1:
return FOURCC_mha1;
case kCodecMhm1:
@ -260,6 +262,13 @@ Status MP4Muxer::DelayInitializeMuxer() {
// supported yet.
if (codec_fourcc != FOURCC_avc3 && codec_fourcc != FOURCC_hev1)
ftyp->compatible_brands.push_back(FOURCC_cmfc);
if (streams()[0]->stream_type() == kStreamAudio) {
codec_fourcc =
CodecToFourCC(streams()[0]->codec(), H26xStreamFormat::kUnSpecified);
if (codec_fourcc == FOURCC_iamf)
ftyp->compatible_brands.push_back(FOURCC_iamf);
}
}
moov->header.creation_time = IsoTimeNow();
@ -556,6 +565,9 @@ bool MP4Muxer::GenerateAudioTrak(const AudioStreamInfo* audio_info,
case kCodecOpus:
audio.dops.opus_identification_header = audio_info->codec_config();
break;
case kCodecIAMF:
audio.iacb.data = audio_info->codec_config();
break;
case kCodecMha1:
case kCodecMhm1:
audio.mhac.data = audio_info->codec_config();
@ -576,11 +588,20 @@ bool MP4Muxer::GenerateAudioTrak(const AudioStreamInfo* audio_info,
audio.channelcount = audio_info->num_channels();
//ETSI TS 103 190-2, E.4.6 samplesize shall be set to 16.
audio.samplesize = 16;
} else if (audio_info->codec() == kCodecIAMF) {
// IAMF sets channelcount to 0
// https://aomediacodec.github.io/iamf/#iasampleentry-section
audio.channelcount = 0;
} else {
audio.channelcount = audio_info->num_channels();
audio.samplesize = audio_info->sample_bits();
}
audio.samplerate = audio_info->sampling_frequency();
// IAMF sets samplerate to 0
// https://aomediacodec.github.io/iamf/#iasampleentry-section
audio.samplerate =
audio_info->codec() == kCodecIAMF ? 0 : audio_info->sampling_frequency();
SampleTable& sample_table = trak->media.information.sample_table;
SampleDescription& sample_description = sample_table.description;
sample_description.type = kAudio;

BIN
packager/media/test/data/bear-iamf-base-opus.mp4 Normal file
View File

Binary file not shown.

BIN
packager/media/test/data/bear-iamf-base-pcm.mp4 Normal file
View File

Binary file not shown.

BIN
packager/media/test/data/bear-iamf-simple-aac-lc.mp4 Normal file
View File

Binary file not shown.

BIN
packager/media/test/data/bear-iamf-simple-flac.mp4 Normal file
View File

Binary file not shown.