From fe44b5e13c198ad60927e2cd0af104e295c2c703 Mon Sep 17 00:00:00 2001
From: Jacob Trimble <modmaker@google.com>
Date: Fri, 20 Nov 2020 13:03:16 -0800
Subject: [PATCH] Add background image to TextSample and TTML output

Issue #832

Change-Id: I50f23223fa4362559087ada9b40488c089594450
---
 packager/media/base/text_sample.cc            |  2 +-
 packager/media/base/text_sample.h             |  7 +++-
 packager/media/formats/ttml/ttml_generator.cc | 36 ++++++++++++++++---
 packager/media/formats/ttml/ttml_generator.h  |  9 +++--
 .../formats/ttml/ttml_generator_unittest.cc   | 26 ++++++++++++++
 5 files changed, 71 insertions(+), 9 deletions(-)

diff --git a/packager/media/base/text_sample.cc b/packager/media/base/text_sample.cc
index fec427123e..9562b0d6fc 100644
--- a/packager/media/base/text_sample.cc
+++ b/packager/media/base/text_sample.cc
@@ -17,7 +17,7 @@ namespace media {
 bool TextFragment::is_empty() const {
   return std::all_of(sub_fragments.begin(), sub_fragments.end(),
                      std::mem_fn(&TextFragment::is_empty)) &&
-         body.empty();
+         body.empty() && image.empty();
 }
 
 TextSample::TextSample(const std::string& id,
diff --git a/packager/media/base/text_sample.h b/packager/media/base/text_sample.h
index ca0a4f9daa..98be5720c6 100644
--- a/packager/media/base/text_sample.h
+++ b/packager/media/base/text_sample.h
@@ -80,7 +80,7 @@ struct TextFragmentStyle {
 };
 
 /// Represents a recursive structure of styled blocks of text.  Only one of
-/// sub_fragments, body, or newline will be set.
+/// sub_fragments, body, image, or newline will be set.
 struct TextFragment {
   TextFragment() {}
   TextFragment(const TextFragmentStyle& style,
@@ -90,6 +90,9 @@ struct TextFragment {
       : style(style), body(body) {}
   TextFragment(const TextFragmentStyle& style, const std::string& body)
       : style(style), body(body) {}
+  TextFragment(const TextFragmentStyle& style,
+               const std::vector<uint8_t>& image)
+      : style(style), image(image) {}
   TextFragment(const TextFragmentStyle& style, bool newline)
       : style(style), newline(newline) {}
 
@@ -97,6 +100,8 @@ struct TextFragment {
 
   std::vector<TextFragment> sub_fragments;
   std::string body;
+  /// PNG image data.
+  std::vector<uint8_t> image;
   bool newline = false;
 
   bool is_empty() const;
diff --git a/packager/media/formats/ttml/ttml_generator.cc b/packager/media/formats/ttml/ttml_generator.cc
index cc4d228cc1..4124aaefa7 100644
--- a/packager/media/formats/ttml/ttml_generator.cc
+++ b/packager/media/formats/ttml/ttml_generator.cc
@@ -6,6 +6,7 @@
 
 #include "packager/media/formats/ttml/ttml_generator.h"
 
+#include "packager/base/base64.h"
 #include "packager/base/strings/stringprintf.h"
 #include "packager/media/base/rcheck.h"
 
@@ -87,12 +88,19 @@ bool TtmlGenerator::Dump(std::string* result) const {
   }
   RCHECK(root.AddChild(std::move(head)));
 
+  size_t image_count = 0;
+  xml::XmlNode metadata("metadata");
   xml::XmlNode body("body");
   xml::XmlNode div("div");
   for (const auto& sample : samples_) {
-    RCHECK(AddSampleToXml(sample, &div));
+    RCHECK(AddSampleToXml(sample, &div, &metadata, &image_count));
   }
   RCHECK(body.AddChild(std::move(div)));
+  if (image_count > 0) {
+    RCHECK(root.SetStringAttribute(
+        "xmlns:smpte", "http://www.smpte-ra.org/schemas/2052-1/2010/smpte-tt"));
+    RCHECK(root.AddChild(std::move(metadata)));
+  }
   RCHECK(root.AddChild(std::move(body)));
 
   *result = root.ToString(/* comment= */ "");
@@ -100,14 +108,16 @@ bool TtmlGenerator::Dump(std::string* result) const {
 }
 
 bool TtmlGenerator::AddSampleToXml(const TextSample& sample,
-                                   xml::XmlNode* body) const {
+                                   xml::XmlNode* body,
+                                   xml::XmlNode* metadata,
+                                   size_t* image_count) const {
   xml::XmlNode p("p");
   RCHECK(p.SetStringAttribute("xml:space", "preserve"));
   RCHECK(p.SetStringAttribute("begin",
                               ToTtmlTime(sample.start_time(), time_scale_)));
   RCHECK(
       p.SetStringAttribute("end", ToTtmlTime(sample.EndTime(), time_scale_)));
-  RCHECK(ConvertFragmentToXml(sample.body(), &p));
+  RCHECK(ConvertFragmentToXml(sample.body(), &p, metadata, image_count));
   if (!sample.id().empty())
     RCHECK(p.SetStringAttribute("xml:id", sample.id()));
 
@@ -151,7 +161,9 @@ bool TtmlGenerator::AddSampleToXml(const TextSample& sample,
 }
 
 bool TtmlGenerator::ConvertFragmentToXml(const TextFragment& body,
-                                         xml::XmlNode* parent) const {
+                                         xml::XmlNode* parent,
+                                         xml::XmlNode* metadata,
+                                         size_t* image_count) const {
   if (body.newline) {
     xml::XmlNode br("br");
     return parent->AddChild(std::move(br));
@@ -179,9 +191,23 @@ bool TtmlGenerator::ConvertFragmentToXml(const TextFragment& body,
 
   if (!body.body.empty()) {
     node->AddContent(body.body);
+  } else if (!body.image.empty()) {
+    std::string image_data(body.image.begin(), body.image.end());
+    std::string base64_data;
+    base::Base64Encode(image_data, &base64_data);
+    std::string id = "img_" + std::to_string(++*image_count);
+
+    xml::XmlNode image_xml("smpte:image");
+    RCHECK(image_xml.SetStringAttribute("imagetype", "PNG"));
+    RCHECK(image_xml.SetStringAttribute("encoding", "Base64"));
+    RCHECK(image_xml.SetStringAttribute("xml:id", id));
+    image_xml.SetContent(base64_data);
+    RCHECK(metadata->AddChild(std::move(image_xml)));
+
+    RCHECK(node->SetStringAttribute("smpte:backgroundImage", "#" + id));
   } else {
     for (const auto& frag : body.sub_fragments) {
-      if (!ConvertFragmentToXml(frag, node))
+      if (!ConvertFragmentToXml(frag, node, metadata, image_count))
         return false;
     }
   }
diff --git a/packager/media/formats/ttml/ttml_generator.h b/packager/media/formats/ttml/ttml_generator.h
index 817339f01d..d28e55a656 100644
--- a/packager/media/formats/ttml/ttml_generator.h
+++ b/packager/media/formats/ttml/ttml_generator.h
@@ -35,9 +35,14 @@ class TtmlGenerator {
   bool Dump(std::string* result) const;
 
  private:
-  bool AddSampleToXml(const TextSample& sample, xml::XmlNode* body) const;
+  bool AddSampleToXml(const TextSample& sample,
+                      xml::XmlNode* body,
+                      xml::XmlNode* metadata,
+                      size_t* image_count) const;
   bool ConvertFragmentToXml(const TextFragment& fragment,
-                            xml::XmlNode* parent) const;
+                            xml::XmlNode* parent,
+                            xml::XmlNode* metadata,
+                            size_t* image_count) const;
 
   std::list<TextSample> samples_;
   std::map<std::string, TextRegion> regions_;
diff --git a/packager/media/formats/ttml/ttml_generator_unittest.cc b/packager/media/formats/ttml/ttml_generator_unittest.cc
index 3737cafa48..13d815ad1b 100644
--- a/packager/media/formats/ttml/ttml_generator_unittest.cc
+++ b/packager/media/formats/ttml/ttml_generator_unittest.cc
@@ -322,6 +322,32 @@ TEST_F(TtmlMuxerTest, HandlesReset) {
   ASSERT_EQ(results, kExpectedOutput2);
 }
 
+TEST_F(TtmlMuxerTest, HandlesImage) {
+  const char* kExpectedOutput =
+      "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+      "<tt xmlns=\"http://www.w3.org/ns/ttml\" "
+      "xmlns:tts=\"http://www.w3.org/ns/ttml#styling\" xml:lang=\"\" "
+      "xmlns:smpte=\"http://www.smpte-ra.org/schemas/2052-1/2010/smpte-tt\">\n"
+      "  <head/>\n"
+      "  <metadata>\n"
+      "    <smpte:image imagetype=\"PNG\" encoding=\"Base64\" xml:id=\"img_1\">"
+      "AQID</smpte:image>\n"
+      "  </metadata>\n"
+      "  <body>\n"
+      "    <div>\n"
+      "      <p xml:space=\"preserve\" begin=\"00:00:05.00\" "
+      "end=\"00:00:06.00\" smpte:backgroundImage=\"#img_1\" xml:id=\"foo\"/>\n"
+      "    </div>\n"
+      "  </body>\n"
+      "</tt>\n";
+
+  TestProperties properties;
+  properties.id = "foo";
+  properties.body.image = {1, 2, 3};
+
+  ParseSingleCue(kExpectedOutput, properties);
+}
+
 }  // namespace ttml
 }  // namespace media
 }  // namespace shaka