Add Subtitle Support to HLS Playlists

The master playlist and media playlist did not have implementations
for handling text streams. This change adds support for both.

Bug: #205

Change-Id: I1329b8cc2585f15b89959071db9dd16d35847cba

packager/hls/base/master_playlist.cc

@@ -22,10 +22,13 @@ namespace shaka {
 namespace hls {
 namespace {
 const char* kDefaultAudioGroupId = "default-audio-group";
+const char* kDefaultSubtitleGroupId = "default-text-group";
+const char* kUnexpectedGroupId = "unexpected-group";
 
 struct Variant {
   std::string audio_codec;
   const std::string* audio_group_id = nullptr;
+  const std::string* text_group_id = nullptr;
   uint64_t audio_bitrate = 0;
 };
 
@@ -66,10 +69,72 @@ std::list<Variant> AudioGroupsToVariants(
 }
 
 const char* GetGroupId(const MediaPlaylist& playlist) {
-  // TODO(vaage): Add support to get a subtitle group id when text support
-  //              is added.
   const std::string& group_id = playlist.group_id();
-  return group_id.empty() ? kDefaultAudioGroupId : group_id.c_str();
+
+  if (!group_id.empty()) {
+    return group_id.c_str();
+  }
+
+  switch (playlist.stream_type()) {
+    case MediaPlaylist::MediaPlaylistStreamType::kAudio:
+      return kDefaultAudioGroupId;
+
+    case MediaPlaylist::MediaPlaylistStreamType::kSubtitle:
+      return kDefaultSubtitleGroupId;
+
+    default:
+      return kUnexpectedGroupId;
+  }
+}
+
+std::list<Variant> SubtitleGroupsToVariants(
+    const std::map<std::string, std::list<const MediaPlaylist*>>& groups) {
+  std::list<Variant> variants;
+
+  for (const auto& group : groups) {
+    Variant variant;
+    variant.text_group_id = &group.first;
+
+    variants.push_back(variant);
+  }
+
+  // Make sure we return at least one variant so create a null variant if there
+  // are no variants.
+  if (variants.empty()) {
+    variants.emplace_back();
+  }
+
+  return variants;
+}
+
+std::list<Variant> BuildVariants(
+    const std::map<std::string, std::list<const MediaPlaylist*>>& audio_groups,
+    const std::map<std::string, std::list<const MediaPlaylist*>>&
+        subtitle_groups) {
+  std::list<Variant> audio_variants = AudioGroupsToVariants(audio_groups);
+  std::list<Variant> subtitle_variants =
+      SubtitleGroupsToVariants(subtitle_groups);
+
+  DCHECK_GE(audio_variants.size(), 1u);
+  DCHECK_GE(subtitle_variants.size(), 1u);
+
+  std::list<Variant> merged;
+
+  for (const auto& audio_variant : audio_variants) {
+    for (const auto& subtitle_variant : subtitle_variants) {
+      Variant variant;
+      variant.audio_codec = audio_variant.audio_codec;
+      variant.audio_group_id = audio_variant.audio_group_id;
+      variant.text_group_id = subtitle_variant.text_group_id;
+      variant.audio_bitrate = audio_variant.audio_bitrate;
+
+      merged.push_back(variant);
+    }
+  }
+
+  DCHECK_GE(merged.size(), 1u);
+
+  return merged;
 }
 
 void BuildAudioTag(const std::string& base_url,
@@ -105,6 +170,7 @@ void BuildVideoTag(const MediaPlaylist& playlist,
                    uint64_t max_audio_bitrate,
                    const std::string& audio_codec,
                    const std::string* audio_group_id,
+                   const std::string* text_group_id,
                    const std::string& base_url,
                    std::string* out) {
   DCHECK(out);
@@ -130,9 +196,31 @@ void BuildVideoTag(const MediaPlaylist& playlist,
     tag.AddQuotedString("AUDIO", *audio_group_id);
   }
 
+  if (text_group_id) {
+    tag.AddQuotedString("SUBTITLES", *text_group_id);
+  }
+
   base::StringAppendF(out, "\n%s%s\n", base_url.c_str(),
                       playlist.file_name().c_str());
 }
+
+void BuildSubtitleTag(const MediaPlaylist& playlist,
+                      const std::string& base_url,
+                      const std::string& group_id,
+                      std::string* out) {
+  Tag tag("#EXT-X-MEDIA", out);
+
+  tag.AddString("TYPE", "SUBTITLES");
+  tag.AddQuotedString("URI", base_url + playlist.file_name());
+  tag.AddQuotedString("GROUP-ID", group_id);
+  const std::string& language = playlist.GetLanguage();
+  if (!language.empty()) {
+    tag.AddQuotedString("LANGUAGE", language);
+  }
+  tag.AddQuotedString("NAME", playlist.name());
+
+  out->append("\n");
+}
 }  // namespace
 
 MasterPlaylist::MasterPlaylist(const std::string& file_name,
@@ -152,6 +240,11 @@ void MasterPlaylist::AddMediaPlaylist(MediaPlaylist* media_playlist) {
       video_playlists_.push_back(media_playlist);
       break;
     }
+    case MediaPlaylist::MediaPlaylistStreamType::kSubtitle: {
+      std::string group_id = GetGroupId(*media_playlist);
+      subtitle_playlist_groups_[group_id].push_back(media_playlist);
+      break;
+    }
     default: {
       NOTIMPLEMENTED() << static_cast<int>(media_playlist->stream_type())
                        << " not handled.";
@@ -168,6 +261,7 @@ bool MasterPlaylist::WriteMasterPlaylist(const std::string& base_url,
   // TODO(rkuroiwa): Handle audio only.
   std::string audio_output;
   std::string video_output;
+  std::string subtitle_output;
 
   // Write out all the audio tags.
   for (const auto& group : audio_playlist_groups_) {
@@ -203,13 +297,24 @@ bool MasterPlaylist::WriteMasterPlaylist(const std::string& base_url,
     }
   }
 
-  std::list<Variant> variants = AudioGroupsToVariants(audio_playlist_groups_);
+  // Write out all the text tags.
+  for (const auto& group : subtitle_playlist_groups_) {
+    const auto& group_id = group.first;
+    const auto& playlists = group.second;
+    for (const auto& playlist : playlists) {
+      BuildSubtitleTag(*playlist, base_url, group_id, &subtitle_output);
+    }
+  }
+
+  std::list<Variant> variants =
+      BuildVariants(audio_playlist_groups_, subtitle_playlist_groups_);
 
   // Write all the video tags out.
   for (const auto& playlist : video_playlists_) {
     for (const auto& variant : variants) {
       BuildVideoTag(*playlist, variant.audio_bitrate, variant.audio_codec,
-                    variant.audio_group_id, base_url, &video_output);
+                    variant.audio_group_id, variant.text_group_id, base_url,
+                    &video_output);
     }
   }
 
@@ -222,8 +327,9 @@ bool MasterPlaylist::WriteMasterPlaylist(const std::string& base_url,
   }
 
   std::string content = "";
-  base::StringAppendF(&content, "#EXTM3U\n%s%s%s", version_line.c_str(),
+  base::StringAppendF(&content, "#EXTM3U\n%s%s%s%s", version_line.c_str(),
-                      audio_output.c_str(), video_output.c_str());
+                      audio_output.c_str(), subtitle_output.c_str(),
+                      video_output.c_str());
 
   // Skip if the playlist is already written.
   if (content == written_playlist_)

packager/hls/base/master_playlist.h

@@ -52,8 +52,13 @@ class MasterPlaylist {
   const std::string default_language_;
   std::list<MediaPlaylist*> all_playlists_;
   std::list<const MediaPlaylist*> video_playlists_;
-  // The key is the audio group name.
+
+  // The ID is the group name, and the value is the list of all media playlists
+  // in that group. Keep audio and subtitle separate as they are processed
+  // separately.
   std::map<std::string, std::list<const MediaPlaylist*>> audio_playlist_groups_;
+  std::map<std::string, std::list<const MediaPlaylist*>>
+      subtitle_playlist_groups_;
 
   DISALLOW_COPY_AND_ASSIGN(MasterPlaylist);
 };

packager/hls/base/master_playlist_unittest.cc

@@ -84,6 +84,20 @@ std::unique_ptr<MockMediaPlaylist> CreateAudioPlaylist(
 
   return playlist;
 }
+
+std::unique_ptr<MockMediaPlaylist> MakeText(const std::string& filename,
+                                            const std::string& name,
+                                            const std::string& group,
+                                            const std::string& language) {
+  std::unique_ptr<MockMediaPlaylist> playlist(
+      new MockMediaPlaylist(kVodPlaylist, filename, name, group));
+
+  EXPECT_CALL(*playlist, GetLanguage()).WillRepeatedly(Return(language));
+  playlist->SetStreamTypeForTesting(
+      MediaPlaylist::MediaPlaylistStreamType::kSubtitle);
+
+  return playlist;
+}
 }  // namespace
 
 class MasterPlaylistTest : public ::testing::Test {
@@ -279,5 +293,214 @@ TEST_F(MasterPlaylistTest, WriteMasterPlaylistSameAudioGroupSameLanguage) {
 
   ASSERT_EQ(expected, actual);
 }
+
+TEST_F(MasterPlaylistTest, WriteMasterPlaylistVideosAndTexts) {
+  // Video, sd.m3u8.
+  std::unique_ptr<MockMediaPlaylist> video1 =
+      CreateVideoPlaylist("sd.m3u8", "sdvideocodec", 300000);
+  master_playlist_.AddMediaPlaylist(video1.get());
+
+  // Video, hd.m3u8.
+  std::unique_ptr<MockMediaPlaylist> video2 =
+      CreateVideoPlaylist("hd.m3u8", "sdvideocodec", 600000);
+  master_playlist_.AddMediaPlaylist(video2.get());
+
+  // Text, eng.m3u8.
+  std::unique_ptr<MockMediaPlaylist> text_eng =
+      MakeText("eng.m3u8", "english", "textgroup", "en");
+  master_playlist_.AddMediaPlaylist(text_eng.get());
+
+  // Text, fr.m3u8.
+  std::unique_ptr<MockMediaPlaylist> text_fr =
+      MakeText("fr.m3u8", "french", "textgroup", "fr");
+  master_playlist_.AddMediaPlaylist(text_fr.get());
+
+  const char kBaseUrl[] = "http://playlists.org/";
+  EXPECT_TRUE(master_playlist_.WriteMasterPlaylist(kBaseUrl, test_output_dir_));
+
+  std::string actual;
+  ASSERT_TRUE(File::ReadFileToString(master_playlist_path_.c_str(), &actual));
+
+  const std::string expected =
+      "#EXTM3U\n"
+      "## Generated with https://github.com/google/shaka-packager version "
+      "test\n"
+      "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/eng.m3u8\","
+      "GROUP-ID=\"textgroup\",LANGUAGE=\"en\",NAME=\"english\"\n"
+      "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/fr.m3u8\","
+      "GROUP-ID=\"textgroup\",LANGUAGE=\"fr\",NAME=\"french\"\n"
+      "#EXT-X-STREAM-INF:BANDWIDTH=300000,CODECS=\"sdvideocodec\","
+      "RESOLUTION=800x600,SUBTITLES=\"textgroup\"\n"
+      "http://playlists.org/sd.m3u8\n"
+      "#EXT-X-STREAM-INF:BANDWIDTH=600000,CODECS=\"sdvideocodec\","
+      "RESOLUTION=800x600,SUBTITLES=\"textgroup\"\n"
+      "http://playlists.org/hd.m3u8\n";
+
+  ASSERT_EQ(expected, actual);
+}
+
+TEST_F(MasterPlaylistTest, WriteMasterPlaylistVideoAndTextGroups) {
+  // Video, sd.m3u8.
+  std::unique_ptr<MockMediaPlaylist> video =
+      CreateVideoPlaylist("sd.m3u8", "sdvideocodec", 300000);
+  master_playlist_.AddMediaPlaylist(video.get());
+
+  // Text, eng.m3u8.
+  std::unique_ptr<MockMediaPlaylist> text_eng =
+      MakeText("eng.m3u8", "english", "en-text-group", "en");
+  master_playlist_.AddMediaPlaylist(text_eng.get());
+
+  // Text, fr.m3u8.
+  std::unique_ptr<MockMediaPlaylist> text_fr =
+      MakeText("fr.m3u8", "french", "fr-text-group", "fr");
+  master_playlist_.AddMediaPlaylist(text_fr.get());
+
+  const char kBaseUrl[] = "http://playlists.org/";
+  EXPECT_TRUE(master_playlist_.WriteMasterPlaylist(kBaseUrl, test_output_dir_));
+
+  std::string actual;
+  ASSERT_TRUE(File::ReadFileToString(master_playlist_path_.c_str(), &actual));
+
+  const std::string expected =
+      "#EXTM3U\n"
+      "## Generated with https://github.com/google/shaka-packager version "
+      "test\n"
+      "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/eng.m3u8\","
+      "GROUP-ID=\"en-text-group\",LANGUAGE=\"en\",NAME=\"english\"\n"
+      "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/fr.m3u8\","
+      "GROUP-ID=\"fr-text-group\",LANGUAGE=\"fr\",NAME=\"french\"\n"
+      "#EXT-X-STREAM-INF:BANDWIDTH=300000,CODECS=\"sdvideocodec\","
+      "RESOLUTION=800x600,SUBTITLES=\"en-text-group\"\n"
+      "http://playlists.org/sd.m3u8\n"
+      "#EXT-X-STREAM-INF:BANDWIDTH=300000,CODECS=\"sdvideocodec\","
+      "RESOLUTION=800x600,SUBTITLES=\"fr-text-group\"\n"
+      "http://playlists.org/sd.m3u8\n";
+
+  ASSERT_EQ(expected, actual);
+}
+
+TEST_F(MasterPlaylistTest, WriteMasterPlaylistVideoAndAudioAndText) {
+  // Video, sd.m3u8.
+  std::unique_ptr<MockMediaPlaylist> video =
+      CreateVideoPlaylist("sd.m3u8", "sdvideocodec", 300000);
+  master_playlist_.AddMediaPlaylist(video.get());
+
+  // Audio, english.m3u8.
+  std::unique_ptr<MockMediaPlaylist> audio = CreateAudioPlaylist(
+      "eng.m3u8", "english", "audiogroup", "audiocodec", "en", 2, 50000);
+  master_playlist_.AddMediaPlaylist(audio.get());
+
+  // Text, english.m3u8.
+  std::unique_ptr<MockMediaPlaylist> text =
+      MakeText("eng.m3u8", "english", "textgroup", "en");
+  master_playlist_.AddMediaPlaylist(text.get());
+
+  const char kBaseUrl[] = "http://playlists.org/";
+  EXPECT_TRUE(master_playlist_.WriteMasterPlaylist(kBaseUrl, test_output_dir_));
+
+  std::string actual;
+  ASSERT_TRUE(File::ReadFileToString(master_playlist_path_.c_str(), &actual));
+
+  const std::string expected =
+      "#EXTM3U\n"
+      "## Generated with https://github.com/google/shaka-packager version "
+      "test\n"
+      "#EXT-X-MEDIA:TYPE=AUDIO,URI=\"http://playlists.org/eng.m3u8\","
+      "GROUP-ID=\"audiogroup\",LANGUAGE=\"en\",NAME=\"english\","
+      "DEFAULT=YES,AUTOSELECT=YES,CHANNELS=\"2\"\n"
+      "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/eng.m3u8\","
+      "GROUP-ID=\"textgroup\",LANGUAGE=\"en\",NAME=\"english\"\n"
+      "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\","
+      "RESOLUTION=800x600,AUDIO=\"audiogroup\",SUBTITLES=\"textgroup\"\n"
+      "http://playlists.org/sd.m3u8\n";
+
+  ASSERT_EQ(expected, actual);
+}
+
+TEST_F(MasterPlaylistTest, WriteMasterPlaylistVidesAudiosTextsDifferentGroups) {
+  const uint64_t kAudioChannels = 2;
+  const uint64_t kAudioBitRate = 50000;
+  const uint64_t kVideoBitRate = 300000;
+
+  std::unique_ptr<MockMediaPlaylist> media_playlists[] = {
+      // AUDIO
+      CreateAudioPlaylist("audio-1.m3u8", "audio 1", "audio-group-1",
+                          "audiocodec", "en", kAudioChannels, kAudioBitRate),
+      CreateAudioPlaylist("audio-2.m3u8", "audio 2", "audio-group-2",
+                          "audiocodec", "en", kAudioChannels, kAudioBitRate),
+
+      // SUBTITLES
+      MakeText("text-1.m3u8", "text 1", "text-group-1", "en"),
+      MakeText("text-2.m3u8", "text 2", "text-group-2", "en"),
+
+      // VIDEO
+      CreateVideoPlaylist("video-1.m3u8", "sdvideocodec", kVideoBitRate),
+      CreateVideoPlaylist("video-2.m3u8", "sdvideocodec", kVideoBitRate),
+  };
+
+  // Add all the media playlists to the master playlist.
+  for (const auto& media_playlist : media_playlists) {
+    master_playlist_.AddMediaPlaylist(media_playlist.get());
+  }
+
+  const char kBaseUrl[] = "http://playlists.org/";
+  EXPECT_TRUE(master_playlist_.WriteMasterPlaylist(kBaseUrl, test_output_dir_));
+
+  std::string actual;
+  ASSERT_TRUE(File::ReadFileToString(master_playlist_path_.c_str(), &actual));
+
+  const std::string expected =
+      "#EXTM3U\n"
+      "## Generated with https://github.com/google/shaka-packager version "
+      "test\n"
+
+      "#EXT-X-MEDIA:TYPE=AUDIO,URI=\"http://playlists.org/audio-1.m3u8\","
+      "GROUP-ID=\"audio-group-1\",LANGUAGE=\"en\",NAME=\"audio 1\","
+      "DEFAULT=YES,AUTOSELECT=YES,CHANNELS=\"2\"\n"
+
+      "#EXT-X-MEDIA:TYPE=AUDIO,URI=\"http://playlists.org/audio-2.m3u8\","
+      "GROUP-ID=\"audio-group-2\",LANGUAGE=\"en\",NAME=\"audio 2\","
+      "DEFAULT=YES,AUTOSELECT=YES,CHANNELS=\"2\"\n"
+
+      "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/text-1.m3u8\","
+      "GROUP-ID=\"text-group-1\",LANGUAGE=\"en\",NAME=\"text 1\"\n"
+
+      "#EXT-X-MEDIA:TYPE=SUBTITLES,URI=\"http://playlists.org/text-2.m3u8\","
+      "GROUP-ID=\"text-group-2\",LANGUAGE=\"en\",NAME=\"text 2\"\n"
+
+      "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\","
+      "RESOLUTION=800x600,AUDIO=\"audio-group-1\",SUBTITLES=\"text-group-1\"\n"
+      "http://playlists.org/video-1.m3u8\n"
+
+      "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\","
+      "RESOLUTION=800x600,AUDIO=\"audio-group-1\",SUBTITLES=\"text-group-2\"\n"
+      "http://playlists.org/video-1.m3u8\n"
+
+      "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\","
+      "RESOLUTION=800x600,AUDIO=\"audio-group-2\",SUBTITLES=\"text-group-1\"\n"
+      "http://playlists.org/video-1.m3u8\n"
+
+      "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\","
+      "RESOLUTION=800x600,AUDIO=\"audio-group-2\",SUBTITLES=\"text-group-2\"\n"
+      "http://playlists.org/video-1.m3u8\n"
+
+      "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\","
+      "RESOLUTION=800x600,AUDIO=\"audio-group-1\",SUBTITLES=\"text-group-1\"\n"
+      "http://playlists.org/video-2.m3u8\n"
+
+      "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\","
+      "RESOLUTION=800x600,AUDIO=\"audio-group-1\",SUBTITLES=\"text-group-2\"\n"
+      "http://playlists.org/video-2.m3u8\n"
+
+      "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\","
+      "RESOLUTION=800x600,AUDIO=\"audio-group-2\",SUBTITLES=\"text-group-1\"\n"
+      "http://playlists.org/video-2.m3u8\n"
+
+      "#EXT-X-STREAM-INF:BANDWIDTH=350000,CODECS=\"sdvideocodec,audiocodec\","
+      "RESOLUTION=800x600,AUDIO=\"audio-group-2\",SUBTITLES=\"text-group-2\"\n"
+      "http://playlists.org/video-2.m3u8\n";
+
+  ASSERT_EQ(expected, actual);
+}
 }  // namespace hls
 }  // namespace shaka

packager/hls/base/media_playlist.cc

@@ -221,7 +221,6 @@ std::string EncryptionInfoEntry::ToString() {
   std::string tag_string;
   Tag tag("#EXT-X-KEY", &tag_string);
 
-  std::string method_attribute;
   if (method_ == MediaPlaylist::EncryptionMethod::kSampleAes) {
     tag.AddString("METHOD", "SAMPLE-AES");
   } else if (method_ == MediaPlaylist::EncryptionMethod::kAes128) {
@@ -342,8 +341,8 @@ bool MediaPlaylist::SetMediaInfo(const MediaInfo& media_info) {
     stream_type_ = MediaPlaylistStreamType::kAudio;
     codec_ = media_info.audio_info().codec();
   } else {
-    NOTIMPLEMENTED();
+    stream_type_ = MediaPlaylistStreamType::kSubtitle;
-    return false;
+    codec_ = media_info.text_info().format();
   }
 
   time_scale_ = time_scale;

packager/hls/base/media_playlist_unittest.cc

@@ -99,13 +99,12 @@ TEST_F(MediaPlaylistMultiSegmentTest, NoTimeScale) {
   EXPECT_FALSE(media_playlist_.SetMediaInfo(media_info));
 }
 
-// The current implementation only handles video and audio.
+TEST_F(MediaPlaylistMultiSegmentTest, SetMediaInfoText) {
-TEST_F(MediaPlaylistMultiSegmentTest, NoAudioOrVideo) {
   MediaInfo media_info;
   media_info.set_reference_time_scale(kTimeScale);
   MediaInfo::TextInfo* text_info = media_info.mutable_text_info();
   text_info->set_format("vtt");
-  EXPECT_FALSE(media_playlist_.SetMediaInfo(media_info));
+  EXPECT_TRUE(media_playlist_.SetMediaInfo(media_info));
 }
 
 TEST_F(MediaPlaylistMultiSegmentTest, SetMediaInfo) {