From e2dae2d960675a3af822760619a17bdc35942e93 Mon Sep 17 00:00:00 2001 From: Aaron Vaage Date: Wed, 31 Jan 2018 11:14:15 -0800 Subject: [PATCH] Add Subtitle Support to HLS Playlists The master playlist and media playlist did not have implementations for handling text streams. This change adds support for both. Bug: #205 Change-Id: I1329b8cc2585f15b89959071db9dd16d35847cba --- packager/hls/base/master_playlist.cc | 120 +++++++++- packager/hls/base/master_playlist.h | 7 +- packager/hls/base/master_playlist_unittest.cc | 223 ++++++++++++++++++ packager/hls/base/media_playlist.cc | 5 +- packager/hls/base/media_playlist_unittest.cc | 5 +- 5 files changed, 346 insertions(+), 14 deletions(-) diff --git a/packager/hls/base/master_playlist.cc b/packager/hls/base/master_playlist.cc index 761cc6df75..df3c4f154c 100644 --- a/packager/hls/base/master_playlist.cc +++ b/packager/hls/base/master_playlist.cc @@ -22,10 +22,13 @@ namespace shaka { namespace hls { namespace { const char* kDefaultAudioGroupId = "default-audio-group"; +const char* kDefaultSubtitleGroupId = "default-text-group"; +const char* kUnexpectedGroupId = "unexpected-group"; struct Variant { std::string audio_codec; const std::string* audio_group_id = nullptr; + const std::string* text_group_id = nullptr; uint64_t audio_bitrate = 0; }; @@ -66,10 +69,72 @@ std::list AudioGroupsToVariants( } const char* GetGroupId(const MediaPlaylist& playlist) { - // TODO(vaage): Add support to get a subtitle group id when text support - // is added. const std::string& group_id = playlist.group_id(); - return group_id.empty() ? kDefaultAudioGroupId : group_id.c_str(); + + if (!group_id.empty()) { + return group_id.c_str(); + } + + switch (playlist.stream_type()) { + case MediaPlaylist::MediaPlaylistStreamType::kAudio: + return kDefaultAudioGroupId; + + case MediaPlaylist::MediaPlaylistStreamType::kSubtitle: + return kDefaultSubtitleGroupId; + + default: + return kUnexpectedGroupId; + } +} + +std::list SubtitleGroupsToVariants( + const std::map>& groups) { + std::list variants; + + for (const auto& group : groups) { + Variant variant; + variant.text_group_id = &group.first; + + variants.push_back(variant); + } + + // Make sure we return at least one variant so create a null variant if there + // are no variants. + if (variants.empty()) { + variants.emplace_back(); + } + + return variants; +} + +std::list BuildVariants( + const std::map>& audio_groups, + const std::map>& + subtitle_groups) { + std::list audio_variants = AudioGroupsToVariants(audio_groups); + std::list subtitle_variants = + SubtitleGroupsToVariants(subtitle_groups); + + DCHECK_GE(audio_variants.size(), 1u); + DCHECK_GE(subtitle_variants.size(), 1u); + + std::list merged; + + for (const auto& audio_variant : audio_variants) { + for (const auto& subtitle_variant : subtitle_variants) { + Variant variant; + variant.audio_codec = audio_variant.audio_codec; + variant.audio_group_id = audio_variant.audio_group_id; + variant.text_group_id = subtitle_variant.text_group_id; + variant.audio_bitrate = audio_variant.audio_bitrate; + + merged.push_back(variant); + } + } + + DCHECK_GE(merged.size(), 1u); + + return merged; } void BuildAudioTag(const std::string& base_url, @@ -105,6 +170,7 @@ void BuildVideoTag(const MediaPlaylist& playlist, uint64_t max_audio_bitrate, const std::string& audio_codec, const std::string* audio_group_id, + const std::string* text_group_id, const std::string& base_url, std::string* out) { DCHECK(out); @@ -130,9 +196,31 @@ void BuildVideoTag(const MediaPlaylist& playlist, tag.AddQuotedString("AUDIO", *audio_group_id); } + if (text_group_id) { + tag.AddQuotedString("SUBTITLES", *text_group_id); + } + base::StringAppendF(out, "\n%s%s\n", base_url.c_str(), playlist.file_name().c_str()); } + +void BuildSubtitleTag(const MediaPlaylist& playlist, + const std::string& base_url, + const std::string& group_id, + std::string* out) { + Tag tag("#EXT-X-MEDIA", out); + + tag.AddString("TYPE", "SUBTITLES"); + tag.AddQuotedString("URI", base_url + playlist.file_name()); + tag.AddQuotedString("GROUP-ID", group_id); + const std::string& language = playlist.GetLanguage(); + if (!language.empty()) { + tag.AddQuotedString("LANGUAGE", language); + } + tag.AddQuotedString("NAME", playlist.name()); + + out->append("\n"); +} } // namespace MasterPlaylist::MasterPlaylist(const std::string& file_name, @@ -152,6 +240,11 @@ void MasterPlaylist::AddMediaPlaylist(MediaPlaylist* media_playlist) { video_playlists_.push_back(media_playlist); break; } + case MediaPlaylist::MediaPlaylistStreamType::kSubtitle: { + std::string group_id = GetGroupId(*media_playlist); + subtitle_playlist_groups_[group_id].push_back(media_playlist); + break; + } default: { NOTIMPLEMENTED() << static_cast(media_playlist->stream_type()) << " not handled."; @@ -168,6 +261,7 @@ bool MasterPlaylist::WriteMasterPlaylist(const std::string& base_url, // TODO(rkuroiwa): Handle audio only. std::string audio_output; std::string video_output; + std::string subtitle_output; // Write out all the audio tags. for (const auto& group : audio_playlist_groups_) { @@ -203,13 +297,24 @@ bool MasterPlaylist::WriteMasterPlaylist(const std::string& base_url, } } - std::list variants = AudioGroupsToVariants(audio_playlist_groups_); + // Write out all the text tags. + for (const auto& group : subtitle_playlist_groups_) { + const auto& group_id = group.first; + const auto& playlists = group.second; + for (const auto& playlist : playlists) { + BuildSubtitleTag(*playlist, base_url, group_id, &subtitle_output); + } + } + + std::list variants = + BuildVariants(audio_playlist_groups_, subtitle_playlist_groups_); // Write all the video tags out. for (const auto& playlist : video_playlists_) { for (const auto& variant : variants) { BuildVideoTag(*playlist, variant.audio_bitrate, variant.audio_codec, - variant.audio_group_id, base_url, &video_output); + variant.audio_group_id, variant.text_group_id, base_url, + &video_output); } } @@ -222,8 +327,9 @@ bool MasterPlaylist::WriteMasterPlaylist(const std::string& base_url, } std::string content = ""; - base::StringAppendF(&content, "#EXTM3U\n%s%s%s", version_line.c_str(), - audio_output.c_str(), video_output.c_str()); + base::StringAppendF(&content, "#EXTM3U\n%s%s%s%s", version_line.c_str(), + audio_output.c_str(), subtitle_output.c_str(), + video_output.c_str()); // Skip if the playlist is already written. if (content == written_playlist_) diff --git a/packager/hls/base/master_playlist.h b/packager/hls/base/master_playlist.h index 4f521314dc..9a49cd2a50 100644 --- a/packager/hls/base/master_playlist.h +++ b/packager/hls/base/master_playlist.h @@ -52,8 +52,13 @@ class MasterPlaylist { const std::string default_language_; std::list all_playlists_; std::list video_playlists_; - // The key is the audio group name. + + // The ID is the group name, and the value is the list of all media playlists + // in that group. Keep audio and subtitle separate as they are processed + // separately. std::map> audio_playlist_groups_; + std::map> + subtitle_playlist_groups_; DISALLOW_COPY_AND_ASSIGN(MasterPlaylist); }; diff --git a/packager/hls/base/master_playlist_unittest.cc b/packager/hls/base/master_playlist_unittest.cc index 71fce7e628..499be5d53c 100644 --- a/packager/hls/base/master_playlist_unittest.cc +++ b/packager/hls/base/master_playlist_unittest.cc @@ -84,6 +84,20 @@ std::unique_ptr CreateAudioPlaylist( return playlist; } + +std::unique_ptr MakeText(const std::string& filename, + const std::string& name, + const std::string& group, + const std::string& language) { + std::unique_ptr playlist( + new MockMediaPlaylist(kVodPlaylist, filename, name, group)); + + EXPECT_CALL(*playlist, GetLanguage()).WillRepeatedly(Return(language)); + playlist->SetStreamTypeForTesting( + MediaPlaylist::MediaPlaylistStreamType::kSubtitle); + + return playlist; +} } // namespace class MasterPlaylistTest : public ::testing::Test { @@ -279,5 +293,214 @@ TEST_F(MasterPlaylistTest, WriteMasterPlaylistSameAudioGroupSameLanguage) { ASSERT_EQ(expected, actual); } + +TEST_F(MasterPlaylistTest, WriteMasterPlaylistVideosAndTexts) { + // Video, sd.m3u8. + std::unique_ptr video1 = + CreateVideoPlaylist("sd.m3u8", "sdvideocodec", 300000); + master_playlist_.AddMediaPlaylist(video1.get()); + + // Video, hd.m3u8. + std::unique_ptr video2 = + CreateVideoPlaylist("hd.m3u8", "sdvideocodec", 600000); + master_playlist_.AddMediaPlaylist(video2.get()); + + // Text, eng.m3u8. + std::unique_ptr text_eng = + MakeText("eng.m3u8", "english", "textgroup", "en"); + master_playlist_.AddMediaPlaylist(text_eng.get()); + + // Text, fr.m3u8. + std::unique_ptr text_fr = + MakeText("fr.m3u8", "french", "textgroup", "fr"); + master_playlist_.AddMediaPlaylist(text_fr.get()); + + const char kBaseUrl[] = "http://playlists.org/"; + EXPECT_TRUE(master_playlist_.WriteMasterPlaylist(kBaseUrl, test_output_dir_)); + + std::string actual; + ASSERT_TRUE(File::ReadFileToString(master_playlist_path_.c_str(), &actual)); + + const std::string expected = + "#EXTM3U\n" + "## Generated with https://github.com/google/shaka-packager version " + "test\n" + "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/eng.m3u8\"," + "GROUP-ID=\"textgroup\",LANGUAGE=\"en\",NAME=\"english\"\n" + "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/fr.m3u8\"," + "GROUP-ID=\"textgroup\",LANGUAGE=\"fr\",NAME=\"french\"\n" + "#EXT-X-STREAM-INF:BANDWIDTH=300000,CODECS=\"sdvideocodec\"," + "RESOLUTION=800x600,SUBTITLES=\"textgroup\"\n" + "http://playlists.org/sd.m3u8\n" + "#EXT-X-STREAM-INF:BANDWIDTH=600000,CODECS=\"sdvideocodec\"," + "RESOLUTION=800x600,SUBTITLES=\"textgroup\"\n" + "http://playlists.org/hd.m3u8\n"; + + ASSERT_EQ(expected, actual); +} + +TEST_F(MasterPlaylistTest, WriteMasterPlaylistVideoAndTextGroups) { + // Video, sd.m3u8. + std::unique_ptr video = + CreateVideoPlaylist("sd.m3u8", "sdvideocodec", 300000); + master_playlist_.AddMediaPlaylist(video.get()); + + // Text, eng.m3u8. + std::unique_ptr text_eng = + MakeText("eng.m3u8", "english", "en-text-group", "en"); + master_playlist_.AddMediaPlaylist(text_eng.get()); + + // Text, fr.m3u8. + std::unique_ptr text_fr = + MakeText("fr.m3u8", "french", "fr-text-group", "fr"); + master_playlist_.AddMediaPlaylist(text_fr.get()); + + const char kBaseUrl[] = "http://playlists.org/"; + EXPECT_TRUE(master_playlist_.WriteMasterPlaylist(kBaseUrl, test_output_dir_)); + + std::string actual; + ASSERT_TRUE(File::ReadFileToString(master_playlist_path_.c_str(), &actual)); + + const std::string expected = + "#EXTM3U\n" + "## Generated with https://github.com/google/shaka-packager version " + "test\n" + "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/eng.m3u8\"," + "GROUP-ID=\"en-text-group\",LANGUAGE=\"en\",NAME=\"english\"\n" + "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/fr.m3u8\"," + "GROUP-ID=\"fr-text-group\",LANGUAGE=\"fr\",NAME=\"french\"\n" + "#EXT-X-STREAM-INF:BANDWIDTH=300000,CODECS=\"sdvideocodec\"," + "RESOLUTION=800x600,SUBTITLES=\"en-text-group\"\n" + "http://playlists.org/sd.m3u8\n" + "#EXT-X-STREAM-INF:BANDWIDTH=300000,CODECS=\"sdvideocodec\"," + "RESOLUTION=800x600,SUBTITLES=\"fr-text-group\"\n" + "http://playlists.org/sd.m3u8\n"; + + ASSERT_EQ(expected, actual); +} + +TEST_F(MasterPlaylistTest, WriteMasterPlaylistVideoAndAudioAndText) { + // Video, sd.m3u8. + std::unique_ptr video = + CreateVideoPlaylist("sd.m3u8", "sdvideocodec", 300000); + master_playlist_.AddMediaPlaylist(video.get()); + + // Audio, english.m3u8. + std::unique_ptr audio = CreateAudioPlaylist( + "eng.m3u8", "english", "audiogroup", "audiocodec", "en", 2, 50000); + master_playlist_.AddMediaPlaylist(audio.get()); + + // Text, english.m3u8. + std::unique_ptr text = + MakeText("eng.m3u8", "english", "textgroup", "en"); + master_playlist_.AddMediaPlaylist(text.get()); + + const char kBaseUrl[] = "http://playlists.org/"; + EXPECT_TRUE(master_playlist_.WriteMasterPlaylist(kBaseUrl, test_output_dir_)); + + std::string actual; + ASSERT_TRUE(File::ReadFileToString(master_playlist_path_.c_str(), &actual)); + + const std::string expected = + "#EXTM3U\n" + "## Generated with https://github.com/google/shaka-packager version " + "test\n" + "#EXT-X-MEDIA:TYPE=AUDIO,URI=\"http://playlists.org/eng.m3u8\"," + "GROUP-ID=\"audiogroup\",LANGUAGE=\"en\",NAME=\"english\"," + "DEFAULT=YES,AUTOSELECT=YES,CHANNELS=\"2\"\n" + "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/eng.m3u8\"," + "GROUP-ID=\"textgroup\",LANGUAGE=\"en\",NAME=\"english\"\n" + "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\"," + "RESOLUTION=800x600,AUDIO=\"audiogroup\",SUBTITLES=\"textgroup\"\n" + "http://playlists.org/sd.m3u8\n"; + + ASSERT_EQ(expected, actual); +} + +TEST_F(MasterPlaylistTest, WriteMasterPlaylistVidesAudiosTextsDifferentGroups) { + const uint64_t kAudioChannels = 2; + const uint64_t kAudioBitRate = 50000; + const uint64_t kVideoBitRate = 300000; + + std::unique_ptr media_playlists[] = { + // AUDIO + CreateAudioPlaylist("audio-1.m3u8", "audio 1", "audio-group-1", + "audiocodec", "en", kAudioChannels, kAudioBitRate), + CreateAudioPlaylist("audio-2.m3u8", "audio 2", "audio-group-2", + "audiocodec", "en", kAudioChannels, kAudioBitRate), + + // SUBTITLES + MakeText("text-1.m3u8", "text 1", "text-group-1", "en"), + MakeText("text-2.m3u8", "text 2", "text-group-2", "en"), + + // VIDEO + CreateVideoPlaylist("video-1.m3u8", "sdvideocodec", kVideoBitRate), + CreateVideoPlaylist("video-2.m3u8", "sdvideocodec", kVideoBitRate), + }; + + // Add all the media playlists to the master playlist. + for (const auto& media_playlist : media_playlists) { + master_playlist_.AddMediaPlaylist(media_playlist.get()); + } + + const char kBaseUrl[] = "http://playlists.org/"; + EXPECT_TRUE(master_playlist_.WriteMasterPlaylist(kBaseUrl, test_output_dir_)); + + std::string actual; + ASSERT_TRUE(File::ReadFileToString(master_playlist_path_.c_str(), &actual)); + + const std::string expected = + "#EXTM3U\n" + "## Generated with https://github.com/google/shaka-packager version " + "test\n" + + "#EXT-X-MEDIA:TYPE=AUDIO,URI=\"http://playlists.org/audio-1.m3u8\"," + "GROUP-ID=\"audio-group-1\",LANGUAGE=\"en\",NAME=\"audio 1\"," + "DEFAULT=YES,AUTOSELECT=YES,CHANNELS=\"2\"\n" + + "#EXT-X-MEDIA:TYPE=AUDIO,URI=\"http://playlists.org/audio-2.m3u8\"," + "GROUP-ID=\"audio-group-2\",LANGUAGE=\"en\",NAME=\"audio 2\"," + "DEFAULT=YES,AUTOSELECT=YES,CHANNELS=\"2\"\n" + + "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/text-1.m3u8\"," + "GROUP-ID=\"text-group-1\",LANGUAGE=\"en\",NAME=\"text 1\"\n" + + "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/text-2.m3u8\"," + "GROUP-ID=\"text-group-2\",LANGUAGE=\"en\",NAME=\"text 2\"\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\"," + "RESOLUTION=800x600,AUDIO=\"audio-group-1\",SUBTITLES=\"text-group-1\"\n" + "http://playlists.org/video-1.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\"," + "RESOLUTION=800x600,AUDIO=\"audio-group-1\",SUBTITLES=\"text-group-2\"\n" + "http://playlists.org/video-1.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\"," + "RESOLUTION=800x600,AUDIO=\"audio-group-2\",SUBTITLES=\"text-group-1\"\n" + "http://playlists.org/video-1.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\"," + "RESOLUTION=800x600,AUDIO=\"audio-group-2\",SUBTITLES=\"text-group-2\"\n" + "http://playlists.org/video-1.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\"," + "RESOLUTION=800x600,AUDIO=\"audio-group-1\",SUBTITLES=\"text-group-1\"\n" + "http://playlists.org/video-2.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\"," + "RESOLUTION=800x600,AUDIO=\"audio-group-1\",SUBTITLES=\"text-group-2\"\n" + "http://playlists.org/video-2.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\"," + "RESOLUTION=800x600,AUDIO=\"audio-group-2\",SUBTITLES=\"text-group-1\"\n" + "http://playlists.org/video-2.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\"," + "RESOLUTION=800x600,AUDIO=\"audio-group-2\",SUBTITLES=\"text-group-2\"\n" + "http://playlists.org/video-2.m3u8\n"; + + ASSERT_EQ(expected, actual); +} } // namespace hls } // namespace shaka diff --git a/packager/hls/base/media_playlist.cc b/packager/hls/base/media_playlist.cc index 417c985c70..ffddeaeca3 100644 --- a/packager/hls/base/media_playlist.cc +++ b/packager/hls/base/media_playlist.cc @@ -221,7 +221,6 @@ std::string EncryptionInfoEntry::ToString() { std::string tag_string; Tag tag("#EXT-X-KEY", &tag_string); - std::string method_attribute; if (method_ == MediaPlaylist::EncryptionMethod::kSampleAes) { tag.AddString("METHOD", "SAMPLE-AES"); } else if (method_ == MediaPlaylist::EncryptionMethod::kAes128) { @@ -342,8 +341,8 @@ bool MediaPlaylist::SetMediaInfo(const MediaInfo& media_info) { stream_type_ = MediaPlaylistStreamType::kAudio; codec_ = media_info.audio_info().codec(); } else { - NOTIMPLEMENTED(); - return false; + stream_type_ = MediaPlaylistStreamType::kSubtitle; + codec_ = media_info.text_info().format(); } time_scale_ = time_scale; diff --git a/packager/hls/base/media_playlist_unittest.cc b/packager/hls/base/media_playlist_unittest.cc index 42dd8835e5..a481a57c76 100644 --- a/packager/hls/base/media_playlist_unittest.cc +++ b/packager/hls/base/media_playlist_unittest.cc @@ -99,13 +99,12 @@ TEST_F(MediaPlaylistMultiSegmentTest, NoTimeScale) { EXPECT_FALSE(media_playlist_.SetMediaInfo(media_info)); } -// The current implementation only handles video and audio. -TEST_F(MediaPlaylistMultiSegmentTest, NoAudioOrVideo) { +TEST_F(MediaPlaylistMultiSegmentTest, SetMediaInfoText) { MediaInfo media_info; media_info.set_reference_time_scale(kTimeScale); MediaInfo::TextInfo* text_info = media_info.mutable_text_info(); text_info->set_format("vtt"); - EXPECT_FALSE(media_playlist_.SetMediaInfo(media_info)); + EXPECT_TRUE(media_playlist_.SetMediaInfo(media_info)); } TEST_F(MediaPlaylistMultiSegmentTest, SetMediaInfo) {