From ac47e529ad7b69cc232f7f96e2a042990505776f Mon Sep 17 00:00:00 2001 From: sr90 Date: Wed, 12 Jul 2023 09:01:55 -0700 Subject: [PATCH 1/4] fix: Update golden files for ttml tests and failing hls unit tests. (#1226) Updating golden files for failing ttml tests in packager_test.py related to https://github.com/shaka-project/shaka-packager/pull/1179 and failing HLS unit tests related to https://github.com/shaka-project/shaka-packager/pull/1170. --- .../bear-english-text-1.m4s | Bin 500 -> 501 bytes .../bear-english-text-2.m4s | Bin 514 -> 515 bytes .../bear-english-text-3.m4s | Bin 514 -> 515 bytes .../bear-english-text-4.m4s | Bin 514 -> 515 bytes .../bear-english-text-5.m4s | Bin 514 -> 515 bytes .../testdata/segmented-ttml-mp4/output.mpd | 2 +- .../bear-english-text-1.ttml | 2 +- .../bear-english-text-2.ttml | 2 +- .../bear-english-text-3.ttml | 2 +- .../bear-english-text-4.ttml | 2 +- .../bear-english-text-5.ttml | 2 +- .../testdata/segmented-ttml-text/output.mpd | 2 +- packager/hls/base/master_playlist_unittest.cc | 23 +++++++++--------- 13 files changed, 19 insertions(+), 18 deletions(-) diff --git a/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-1.m4s b/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-1.m4s index 9cba9773d22091518761c02a7089f6074d08dd94..1de1cd08d379e025f12b28fb626004637aeb1013 100644 GIT binary patch delta 25 hcmeyu{FQlvJLA%c9?p!G6MM879VhO%Ie7wO3IKum3HJa1 delta 25 hcmey${DpafJL8gx9?pyw6MM879VYI$Ie8LeG5~?u3H1N~ diff --git a/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-2.m4s b/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-2.m4s index 473e7de4bada9c88fbf6d6d50539590200dbc75f..29351536a0b23a737f7580f67602a936eab3e22f 100644 GIT binary patch delta 25 hcmZo-X=a(=&bWD^hcl!5#2#%%|A{+pPM*M61ORUL2~7Y1 delta 25 hcmZo>X=0h+&bVo!hclzw#2#%%zll3;PM*YA2mo%^2}=L~ diff --git a/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-3.m4s b/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-3.m4s index f015e810cf64a30cefc9a611533087aec58b4852..8b81e6d983dbd6b6e508150c5298ab300b05cb77 100644 GIT binary patch delta 25 hcmZo-X=a(=&bWD^hcl!5#2#%%|A{+pPM*M61ORUL2~7Y1 delta 25 hcmZo>X=0h+&bVo!hclzw#2#%%zll3;PM*YA2mo%^2}=L~ diff --git a/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-4.m4s b/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-4.m4s index 4253cd74b70017e5407f541b68de4b1b81a3e556..9c86b0e5e2e6a43d1c06dc5679918d672a0762d4 100644 GIT binary patch delta 25 hcmZo-X=a(=&bWD^hcl!5#2#%%|A{+pPM*M61ORUL2~7Y1 delta 25 hcmZo>X=0h+&bVo!hclzw#2#%%zll3;PM*YA2mo%^2}=L~ diff --git a/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-5.m4s b/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-5.m4s index 8232dc03b592b701225fb394fec1b8b6ae108998..e0dc6e3b8d4604c09d49a4e1c2f1615c4d116d59 100644 GIT binary patch delta 25 hcmZo-X=a(=&bWD^hcl!5#2#%%|A{+pPM*M61ORUL2~7Y1 delta 25 hcmZo>X=0h+&bVo!hclzw#2#%%zll3;PM*YA2mo%^2}=L~ diff --git a/packager/app/test/testdata/segmented-ttml-mp4/output.mpd b/packager/app/test/testdata/segmented-ttml-mp4/output.mpd index 10019a55be..35ddebf4a1 100644 --- a/packager/app/test/testdata/segmented-ttml-mp4/output.mpd +++ b/packager/app/test/testdata/segmented-ttml-mp4/output.mpd @@ -4,7 +4,7 @@ - + diff --git a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-1.ttml b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-1.ttml index 70c1bdb8a8..94a5092c9d 100644 --- a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-1.ttml +++ b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-1.ttml @@ -3,7 +3,7 @@
-

Yup, that's a bear, eh.

+

Yup, that's a bear, eh.

diff --git a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-2.ttml b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-2.ttml index 07dfd93d87..8048787ec7 100644 --- a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-2.ttml +++ b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-2.ttml @@ -3,7 +3,7 @@
-

He 's... um... doing bear-like stuff.

+

He 's... um... doing bear-like stuff.

diff --git a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-3.ttml b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-3.ttml index 07dfd93d87..8048787ec7 100644 --- a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-3.ttml +++ b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-3.ttml @@ -3,7 +3,7 @@
-

He 's... um... doing bear-like stuff.

+

He 's... um... doing bear-like stuff.

diff --git a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-4.ttml b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-4.ttml index 07dfd93d87..8048787ec7 100644 --- a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-4.ttml +++ b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-4.ttml @@ -3,7 +3,7 @@
-

He 's... um... doing bear-like stuff.

+

He 's... um... doing bear-like stuff.

diff --git a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-5.ttml b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-5.ttml index 07dfd93d87..8048787ec7 100644 --- a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-5.ttml +++ b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-5.ttml @@ -3,7 +3,7 @@
-

He 's... um... doing bear-like stuff.

+

He 's... um... doing bear-like stuff.

diff --git a/packager/app/test/testdata/segmented-ttml-text/output.mpd b/packager/app/test/testdata/segmented-ttml-text/output.mpd index 8b506062bf..9b9e87e2ed 100644 --- a/packager/app/test/testdata/segmented-ttml-text/output.mpd +++ b/packager/app/test/testdata/segmented-ttml-text/output.mpd @@ -4,7 +4,7 @@ - + diff --git a/packager/hls/base/master_playlist_unittest.cc b/packager/hls/base/master_playlist_unittest.cc index 450583341c..f72cdede53 100644 --- a/packager/hls/base/master_playlist_unittest.cc +++ b/packager/hls/base/master_playlist_unittest.cc @@ -324,7 +324,7 @@ TEST_F(MasterPlaylistTest, WriteMasterPlaylistVideoAndAudio) { "DEFAULT=YES,AUTOSELECT=YES,CHANNELS=\"2\"\n" "#EXT-X-MEDIA:TYPE=AUDIO,URI=\"http://playlists.org/spa.m3u8\"," "GROUP-ID=\"audiogroup\",LANGUAGE=\"es\",NAME=\"espanol\"," - "AUTOSELECT=YES,CHANNELS=\"5\"\n" + "DEFAULT=NO,AUTOSELECT=YES,CHANNELS=\"5\"\n" "\n" "#EXT-X-STREAM-INF:BANDWIDTH=360000,AVERAGE-BANDWIDTH=240000," "CODECS=\"sdvideocodec,audiocodec\"," @@ -430,7 +430,8 @@ TEST_F(MasterPlaylistTest, WriteMasterPlaylistSameAudioGroupSameLanguage) { "GROUP-ID=\"audio\",LANGUAGE=\"en\",NAME=\"english\"," "DEFAULT=YES,AUTOSELECT=YES,CHANNELS=\"1\"\n" "#EXT-X-MEDIA:TYPE=AUDIO,URI=\"http://anydomain.com/eng_hi.m3u8\"," - "GROUP-ID=\"audio\",LANGUAGE=\"en\",NAME=\"english\",CHANNELS=\"8\"\n" + "GROUP-ID=\"audio\",LANGUAGE=\"en\",NAME=\"english\",DEFAULT=NO," + "CHANNELS=\"8\"\n" "\n" "#EXT-X-STREAM-INF:BANDWIDTH=400000,AVERAGE-BANDWIDTH=280000," "CODECS=\"videocodec,audiocodec\",RESOLUTION=800x600,AUDIO=\"audio\"," @@ -472,7 +473,7 @@ TEST_F(MasterPlaylistTest, WriteMasterPlaylistVideosAndTexts) { "\n" "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/eng.m3u8\"," "GROUP-ID=\"textgroup\",LANGUAGE=\"en\",NAME=\"english\"," - "AUTOSELECT=YES\n" + "DEFAULT=NO,AUTOSELECT=YES\n" "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/fr.m3u8\"," "GROUP-ID=\"textgroup\",LANGUAGE=\"fr\",NAME=\"french\",DEFAULT=YES," "AUTOSELECT=YES\n" @@ -513,8 +514,8 @@ TEST_F(MasterPlaylistTest, WriteMasterPlaylistVideoAndTextWithCharacteritics) { "version test\n" "\n" "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/eng.m3u8\"," - "GROUP-ID=\"textgroup\",LANGUAGE=\"en\",NAME=\"english\",AUTOSELECT=YES," - "CHARACTERISTICS=\"" + "GROUP-ID=\"textgroup\",LANGUAGE=\"en\",NAME=\"english\",DEFAULT=NO," + "AUTOSELECT=YES,CHARACTERISTICS=\"" "public.accessibility.transcribes-spoken-dialog,public.easy-to-read\"\n" "\n" "#EXT-X-STREAM-INF:BANDWIDTH=300000,AVERAGE-BANDWIDTH=200000," @@ -555,7 +556,7 @@ TEST_F(MasterPlaylistTest, WriteMasterPlaylistVideoAndDvsAudio) { "version test\n" "\n" "#EXT-X-MEDIA:TYPE=AUDIO,URI=\"http://playlists.org/dvs_eng.m3u8\"," - "GROUP-ID=\"audiogroup\",LANGUAGE=\"en\",NAME=\"DVS english\"," + "GROUP-ID=\"audiogroup\",LANGUAGE=\"en\",NAME=\"DVS english\",DEFAULT=NO," "AUTOSELECT=YES,CHARACTERISTICS=\"public.accessibility.describes-video\"," "CHANNELS=\"2\"\n" "#EXT-X-MEDIA:TYPE=AUDIO,URI=\"http://playlists.org/eng.m3u8\"," @@ -598,7 +599,7 @@ TEST_F(MasterPlaylistTest, WriteMasterPlaylistVideoAndTextGroups) { "\n" "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/eng.m3u8\"," "GROUP-ID=\"en-text-group\",LANGUAGE=\"en\",NAME=\"english\"," - "AUTOSELECT=YES\n" + "DEFAULT=NO,AUTOSELECT=YES\n" "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/fr.m3u8\"," "GROUP-ID=\"fr-text-group\",LANGUAGE=\"fr\",NAME=\"french\"," "DEFAULT=YES,AUTOSELECT=YES\n" @@ -648,7 +649,7 @@ TEST_F(MasterPlaylistTest, WriteMasterPlaylistVideoAndAudioAndText) { "\n" "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/eng.m3u8\"," "GROUP-ID=\"textgroup\",LANGUAGE=\"en\",NAME=\"english\"," - "AUTOSELECT=YES\n" + "DEFAULT=NO,AUTOSELECT=YES\n" "\n" "#EXT-X-STREAM-INF:BANDWIDTH=350000,AVERAGE-BANDWIDTH=230000," "CODECS=\"sdvideocodec,audiocodec,textcodec\",RESOLUTION=800x600," @@ -720,11 +721,11 @@ TEST_F(MasterPlaylistTest, WriteMasterPlaylistMixedPlaylistsDifferentGroups) { "DEFAULT=YES,AUTOSELECT=YES,CHANNELS=\"2\"\n" "#EXT-X-MEDIA:TYPE=AUDIO,URI=\"http://playlists.org/audio-2.m3u8\"," "GROUP-ID=\"audio-group-2\",LANGUAGE=\"fr\",NAME=\"audio 2\"," - "AUTOSELECT=YES,CHANNELS=\"2\"\n" + "DEFAULT=NO,AUTOSELECT=YES,CHANNELS=\"2\"\n" "\n" "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/text-1.m3u8\"," "GROUP-ID=\"text-group-1\",LANGUAGE=\"en\",NAME=\"text 1\"," - "AUTOSELECT=YES\n" + "DEFAULT=NO,AUTOSELECT=YES\n" "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/text-2.m3u8\"," "GROUP-ID=\"text-group-2\",LANGUAGE=\"fr\",NAME=\"text 2\"," "DEFAULT=YES,AUTOSELECT=YES\n" @@ -823,7 +824,7 @@ TEST_F(MasterPlaylistTest, WriteMasterPlaylistAudioOnly) { "DEFAULT=YES,AUTOSELECT=YES,CHANNELS=\"2\"\n" "#EXT-X-MEDIA:TYPE=AUDIO,URI=\"http://playlists.org/audio-2.m3u8\"," "GROUP-ID=\"audio-group-2\",LANGUAGE=\"fr\",NAME=\"audio 2\"," - "AUTOSELECT=YES,CHANNELS=\"2\"\n" + "DEFAULT=NO,AUTOSELECT=YES,CHANNELS=\"2\"\n" "\n" "#EXT-X-STREAM-INF:BANDWIDTH=50000,AVERAGE-BANDWIDTH=30000," "CODECS=\"audiocodec\",AUDIO=\"audio-group-1\",CLOSED-CAPTIONS=NONE\n" From d9d3c7f8be13e493a99b2ff4b72a402c441c1666 Mon Sep 17 00:00:00 2001 From: Allan Lei Date: Thu, 13 Jul 2023 00:03:43 +0800 Subject: [PATCH 2/4] fix: Indexing `bytes` produces `int` on python3 for `pssh-box.py` (#1228) Fixes #1227 --- packager/tools/pssh/pssh-box.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/packager/tools/pssh/pssh-box.py b/packager/tools/pssh/pssh-box.py index 7c2b20ffa0..45306247d1 100755 --- a/packager/tools/pssh/pssh-box.py +++ b/packager/tools/pssh/pssh-box.py @@ -20,6 +20,17 @@ import os import struct import sys + +def to_code_point(value): + """ + Return the unicode code point with `int` passthrough + """ + if isinstance(value, int): + return value + + return ord(value) + + _script_dir = os.path.dirname(os.path.realpath(__file__)) _proto_path = os.path.join(_script_dir, 'pyproto') _widevine_proto_path = os.path.join(_proto_path, 'packager/media/base') @@ -64,9 +75,9 @@ class BinaryReader(object): ret = 0 for i in range(0, size): if self.little_endian: - ret |= (ord(data[i]) << (8 * i)) + ret |= (to_code_point(data[i]) << (8 * i)) else: - ret |= (ord(data[i]) << (8 * (size - i - 1))) + ret |= (to_code_point(data[i]) << (8 * (size - i - 1))) return ret From dab165d3e5d979e2e5ff783d91d948357b932078 Mon Sep 17 00:00:00 2001 From: xyb Date: Wed, 12 Jul 2023 09:07:07 -0700 Subject: [PATCH 3/4] fix: Fix failure on very short WebVTT files (#1216) Fix a bug that if the webvtt file is very short, e.g. only contains one block WEBVTT 00:00:00.500 --> 00:00:02.000 The Web is always changing shaka packager will report error: "Packaging Error: 6 (END_OF_STREAM)". Fixes #1217 --- packager/media/demuxer/demuxer.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packager/media/demuxer/demuxer.cc b/packager/media/demuxer/demuxer.cc index 10e69a0155..20fae4ac7e 100644 --- a/packager/media/demuxer/demuxer.cc +++ b/packager/media/demuxer/demuxer.cc @@ -162,13 +162,16 @@ Status Demuxer::InitializeParser() { // Read enough bytes before detecting the container. int64_t bytes_read = 0; + bool eof = false; while (static_cast(bytes_read) < kInitBufSize) { int64_t read_result = media_file_->Read(buffer_.get() + bytes_read, kInitBufSize); if (read_result < 0) return Status(error::FILE_FAILURE, "Cannot read file " + file_name_); - if (read_result == 0) + if (read_result == 0) { + eof = true; break; + } bytes_read += read_result; } container_name_ = DetermineContainer(buffer_.get(), bytes_read); @@ -224,7 +227,7 @@ Status Demuxer::InitializeParser() { // descriptor |media_file_| instead of opening the same file again. static_cast(parser_.get())->LoadMoov(file_name_); } - if (!parser_->Parse(buffer_.get(), bytes_read)) { + if (!parser_->Parse(buffer_.get(), bytes_read) || (eof && !parser_->Flush())) { return Status(error::PARSER_FAILURE, "Cannot parse media file " + file_name_); } From d6f28d456c6ec5ecf39c868447d85294a698166d Mon Sep 17 00:00:00 2001 From: Sergio Garcia Murillo Date: Wed, 12 Jul 2023 23:51:00 +0200 Subject: [PATCH 4/4] fix: Prevent crash in GetEarliestTimestamp() if periods are empty (#1173) While I have not yet found why the periods are empty, this will prevent shaka from seg faulting Fixes #1172 --- packager/mpd/base/mpd_builder.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packager/mpd/base/mpd_builder.cc b/packager/mpd/base/mpd_builder.cc index f09381601e..e2241dc2e3 100644 --- a/packager/mpd/base/mpd_builder.cc +++ b/packager/mpd/base/mpd_builder.cc @@ -342,6 +342,8 @@ float MpdBuilder::GetStaticMpdDuration() { bool MpdBuilder::GetEarliestTimestamp(double* timestamp_seconds) { DCHECK(timestamp_seconds); DCHECK(!periods_.empty()); + if (periods_.empty()) + return false; double timestamp = 0; double earliest_timestamp = -1; // TODO(kqyang): This is used to set availabilityStartTime. We may consider