diff --git a/CONTRIBUTORS b/CONTRIBUTORS index d7757949bb..9ba03ca1b7 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -54,6 +54,7 @@ Sanil Raut Sergio Ammirata Thomas Inskip Tim Lansen +Torbjörn Einarsson Vincent Nguyen Weiguo Shao diff --git a/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-1.m4s b/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-1.m4s index 1de1cd08d3..319e9a7cdd 100644 Binary files a/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-1.m4s and b/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-1.m4s differ diff --git a/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-2.m4s b/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-2.m4s index 29351536a0..ac847dfc25 100644 Binary files a/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-2.m4s and b/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-2.m4s differ diff --git a/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-3.m4s b/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-3.m4s index 8b81e6d983..db5c1dd6f6 100644 Binary files a/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-3.m4s and b/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-3.m4s differ diff --git a/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-4.m4s b/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-4.m4s index 9c86b0e5e2..32792d0cfc 100644 Binary files a/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-4.m4s and b/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-4.m4s differ diff --git a/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-5.m4s b/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-5.m4s index e0dc6e3b8d..224ccd2c68 100644 Binary files a/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-5.m4s and b/packager/app/test/testdata/segmented-ttml-mp4/bear-english-text-5.m4s differ diff --git a/packager/app/test/testdata/segmented-ttml-mp4/output.mpd b/packager/app/test/testdata/segmented-ttml-mp4/output.mpd index 35ddebf4a1..bc8c0c7591 100644 --- a/packager/app/test/testdata/segmented-ttml-mp4/output.mpd +++ b/packager/app/test/testdata/segmented-ttml-mp4/output.mpd @@ -4,7 +4,7 @@ - + diff --git a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-1.ttml b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-1.ttml index 94a5092c9d..2bd77c0bed 100644 --- a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-1.ttml +++ b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-1.ttml @@ -1,6 +1,10 @@ - + + + + +

Yup, that's a bear, eh.

diff --git a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-2.ttml b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-2.ttml index 8048787ec7..10a7664c86 100644 --- a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-2.ttml +++ b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-2.ttml @@ -1,6 +1,10 @@ - + + + + +

He 's... um... doing bear-like stuff.

diff --git a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-3.ttml b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-3.ttml index 8048787ec7..10a7664c86 100644 --- a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-3.ttml +++ b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-3.ttml @@ -1,6 +1,10 @@ - + + + + +

He 's... um... doing bear-like stuff.

diff --git a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-4.ttml b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-4.ttml index 8048787ec7..10a7664c86 100644 --- a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-4.ttml +++ b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-4.ttml @@ -1,6 +1,10 @@ - + + + + +

He 's... um... doing bear-like stuff.

diff --git a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-5.ttml b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-5.ttml index 8048787ec7..10a7664c86 100644 --- a/packager/app/test/testdata/segmented-ttml-text/bear-english-text-5.ttml +++ b/packager/app/test/testdata/segmented-ttml-text/bear-english-text-5.ttml @@ -1,6 +1,10 @@ - + + + + +

He 's... um... doing bear-like stuff.

diff --git a/packager/app/test/testdata/segmented-ttml-text/output.mpd b/packager/app/test/testdata/segmented-ttml-text/output.mpd index 9b9e87e2ed..398aeeca60 100644 --- a/packager/app/test/testdata/segmented-ttml-text/output.mpd +++ b/packager/app/test/testdata/segmented-ttml-text/output.mpd @@ -4,7 +4,7 @@ - + diff --git a/packager/media/base/text_sample.h b/packager/media/base/text_sample.h index 81f78eac67..22b4647984 100644 --- a/packager/media/base/text_sample.h +++ b/packager/media/base/text_sample.h @@ -80,6 +80,11 @@ struct TextFragmentStyle { std::optional underline; std::optional bold; std::optional italic; + // The colors could be any string that can be interpreted as + // a color in TTML (or WebVTT). As a start, the 8 teletext colors are used, + // i.e. black, red, green, yellow, blue, magenta, cyan, and white + std::string color; + std::string backgroundColor; }; /// Represents a recursive structure of styled blocks of text. Only one of diff --git a/packager/media/formats/mp2t/es_parser_teletext.cc b/packager/media/formats/mp2t/es_parser_teletext.cc index 8e859d1b1d..ac42ce2f23 100644 --- a/packager/media/formats/mp2t/es_parser_teletext.cc +++ b/packager/media/formats/mp2t/es_parser_teletext.cc @@ -7,10 +7,10 @@ #include #include -#include #include #include #include +#include namespace shaka { namespace media { @@ -18,6 +18,8 @@ namespace mp2t { namespace { +constexpr const char* kRegionTeletextPrefix = "ttx_"; + const uint8_t EBU_TELETEXT_WITH_SUBTITLING = 0x03; const int kPayloadSize = 40; const int kNumTriplets = 13; @@ -94,14 +96,6 @@ bool ParseSubtitlingDescriptor( return true; } -std::string RemoveTrailingSpaces(const std::string& input) { - const auto index = input.find_last_not_of(' '); - if (index == std::string::npos) { - return ""; - } - return input.substr(0, index + 1); -} - } // namespace EsParserTeletext::EsParserTeletext(const uint32_t pid, @@ -169,7 +163,7 @@ bool EsParserTeletext::ParseInternal(const uint8_t* data, const int64_t pts) { BitReader reader(data, size); RCHECK(reader.SkipBits(8)); - std::vector lines; + std::vector rows; while (reader.bits_available()) { uint8_t data_unit_id; @@ -178,15 +172,17 @@ bool EsParserTeletext::ParseInternal(const uint8_t* data, uint8_t data_unit_length; RCHECK(reader.ReadBits(8, &data_unit_length)); + if (data_unit_id != EBU_TELETEXT_WITH_SUBTITLING) { + RCHECK(reader.SkipBytes(data_unit_length)); + continue; + } + if (data_unit_length != 44) { + // Teletext data unit length is always 44 bytes LOG(ERROR) << "Bad Teletext data length"; break; } - if (data_unit_id != EBU_TELETEXT_WITH_SUBTITLING) { - RCHECK(reader.SkipBytes(44)); - continue; - } RCHECK(reader.SkipBits(16)); @@ -207,27 +203,26 @@ bool EsParserTeletext::ParseInternal(const uint8_t* data, const uint8_t* data_block = reader.current_byte_ptr(); RCHECK(reader.SkipBytes(40)); - std::string display_text; - if (ParseDataBlock(pts, data_block, packet_nr, magazine, display_text)) { - lines.emplace_back(std::move(display_text)); + TextRow row; + if (ParseDataBlock(pts, data_block, packet_nr, magazine, row)) { + rows.emplace_back(std::move(row)); } } - if (lines.empty()) { + if (rows.empty()) { return true; } - const uint16_t index = magazine_ * 100 + page_number_; auto page_state_itr = page_state_.find(index); if (page_state_itr == page_state_.end()) { - page_state_.emplace(index, TextBlock{std::move(lines), {}, last_pts_}); + page_state_.emplace(index, TextBlock{std::move(rows), {}, last_pts_}); } else { - for (auto& line : lines) { - auto& page_state_lines = page_state_itr->second.lines; - page_state_lines.emplace_back(std::move(line)); + for (auto& row : rows) { + auto& page_state_lines = page_state_itr->second.rows; + page_state_lines.emplace_back(std::move(row)); } - lines.clear(); + rows.clear(); } return true; @@ -237,13 +232,17 @@ bool EsParserTeletext::ParseDataBlock(const int64_t pts, const uint8_t* data_block, const uint8_t packet_nr, const uint8_t magazine, - std::string& display_text) { + TextRow& row) { if (packet_nr == 0) { last_pts_ = pts; BitReader reader(data_block, 32); const uint8_t page_number_units = ReadHamming(reader); const uint8_t page_number_tens = ReadHamming(reader); + if (page_number_units == 0xf || page_number_tens == 0xf) { + RCHECK(reader.SkipBits(40)); + return false; + } const uint8_t page_number = 10 * page_number_tens + page_number_units; const uint16_t index = magazine * 100 + page_number; @@ -251,9 +250,6 @@ bool EsParserTeletext::ParseDataBlock(const int64_t pts, page_number_ = page_number; magazine_ = magazine; - if (page_number == 0xFF) { - return false; - } RCHECK(reader.SkipBits(40)); const uint8_t subcode_c11_c14 = ReadHamming(reader); @@ -273,7 +269,7 @@ bool EsParserTeletext::ParseDataBlock(const int64_t pts, return false; } - display_text = BuildText(data_block, packet_nr); + row = BuildRow(data_block, packet_nr); return true; } @@ -317,45 +313,81 @@ void EsParserTeletext::SendPending(const uint16_t index, const int64_t pts) { auto page_state_itr = page_state_.find(index); if (page_state_itr == page_state_.end() || - page_state_itr->second.lines.empty()) { + page_state_itr->second.rows.empty()) { return; } - const auto& pending_lines = page_state_itr->second.lines; + const auto& pending_rows = page_state_itr->second.rows; const auto pending_pts = page_state_itr->second.pts; - TextFragmentStyle text_fragment_style; TextSettings text_settings; std::shared_ptr text_sample; + std::vector sub_fragments; - if (pending_lines.size() == 1) { - TextFragment text_fragment(text_fragment_style, pending_lines[0].c_str()); - text_sample = std::make_shared("", pending_pts, pts, - text_settings, text_fragment); - + if (pending_rows.size() == 1) { + // This is a single line of formatted text. + // Propagate row number/2 and alignment + const float line_nr = float(pending_rows[0].row_number) / 2.0; + text_settings.line = TextNumber(line_nr, TextUnitType::kLines); + text_settings.region = kRegionTeletextPrefix + std::to_string(int(line_nr)); + text_settings.text_alignment = pending_rows[0].alignment; + text_sample = std::make_shared( + "", pending_pts, pts, text_settings, pending_rows[0].fragment); + text_sample->set_sub_stream_index(index); + emit_sample_cb_(text_sample); + page_state_.erase(index); + return; } else { - std::vector sub_fragments; - for (const auto& line : pending_lines) { - sub_fragments.emplace_back(text_fragment_style, line.c_str()); - sub_fragments.emplace_back(text_fragment_style, true); + int32_t latest_row_nr = -1; + bool last_double_height = false; + bool new_sample = true; + for (const auto& row : pending_rows) { + int row_nr = row.row_number; + bool double_height = row.double_height; + int row_step = last_double_height ? 2 : 1; + if (latest_row_nr != -1) { // Not the first row + if (row_nr != latest_row_nr + row_step) { + // Send what has been collected since not adjacent + text_sample = + std::make_shared("", pending_pts, pts, text_settings, + TextFragment({}, sub_fragments)); + text_sample->set_sub_stream_index(index); + emit_sample_cb_(text_sample); + new_sample = true; + } else { + // Add a newline and the next row to the current sample + sub_fragments.push_back(TextFragment({}, true)); + sub_fragments.push_back(row.fragment); + new_sample = false; + } + } + if (new_sample) { + const float line_nr = float(row.row_number) / 2.0; + text_settings.line = TextNumber(line_nr, TextUnitType::kLines); + text_settings.region = + kRegionTeletextPrefix + std::to_string(int(line_nr)); + text_settings.text_alignment = row.alignment; + sub_fragments.clear(); + sub_fragments.push_back(row.fragment); + } + last_double_height = double_height; + latest_row_nr = row_nr; } - sub_fragments.pop_back(); - TextFragment text_fragment(text_fragment_style, sub_fragments); - text_sample = std::make_shared("", pending_pts, pts, - text_settings, text_fragment); } + text_sample = std::make_shared( + "", pending_pts, pts, text_settings, TextFragment({}, sub_fragments)); text_sample->set_sub_stream_index(index); emit_sample_cb_(text_sample); page_state_.erase(index); } -std::string EsParserTeletext::BuildText(const uint8_t* data_block, - const uint8_t row) const { +// BuildRow builds a row with alignment information. +EsParserTeletext::TextRow EsParserTeletext::BuildRow(const uint8_t* data_block, + const uint8_t row) const { std::string next_string; next_string.reserve(kPayloadSize * 2); - bool leading_spaces = true; const uint16_t index = magazine_ * 100 + page_number_; const auto page_state_itr = page_state_.find(index); @@ -370,12 +402,19 @@ std::string EsParserTeletext::BuildText(const uint8_t* data_block, } } + int32_t start_pos = 0; + int32_t end_pos = 0; + bool double_height = false; + TextFragmentStyle text_style = TextFragmentStyle(); + text_style.color = "white"; + text_style.backgroundColor = "black"; + // A typical 40 character line looks like: + // doubleHeight, [color] spaces, Start, Start, text, End End, spaces for (size_t i = 0; i < kPayloadSize; ++i) { if (column_replacement_map) { const auto column_itr = column_replacement_map->find(i); if (column_itr != column_replacement_map->cend()) { next_string.append(column_itr->second); - leading_spaces = false; continue; } } @@ -383,17 +422,68 @@ std::string EsParserTeletext::BuildText(const uint8_t* data_block, char next_char = static_cast(TELETEXT_BITREVERSE_8[data_block[i]] & 0x7f); - if (next_char < 32) { - next_char = 0x20; - } - - if (leading_spaces) { - if (next_char == 0x20) { - continue; + if (next_char < 0x20) { + // Here are control characters, which are not printable. + // These include colors, double-height, flashing, etc. + // We only handle one-foreground color and double-height. + switch (next_char) { + case 0x0: // Alpha Black (not included in Level 1.5) + // color = ColorBlack + break; + case 0x1: + text_style.color = "red"; + break; + case 0x2: + text_style.color = "green"; + break; + case 0x3: + text_style.color = "yellow"; + break; + case 0x4: + text_style.color = "blue"; + break; + case 0x5: + text_style.color = "magenta"; + break; + case 0x6: + text_style.color = "cyan"; + break; + case 0x7: + text_style.color = "white"; + break; + case 0x08: // Flash (not handled) + break; + case 0x09: // Steady (not handled) + break; + case 0xa: // End Box + end_pos = i - 1; + break; + case 0xb: // Start Box, typically twice due to double height + start_pos = i + 1; + continue; // Do not propagate as a space + break; + case 0xc: // Normal size + break; + case 0xd: // Double height, typically always used + double_height = true; + break; + case 0x1c: // Black background (not handled) + break; + case 0x1d: // Set background color from text color. + text_style.backgroundColor = text_style.color; + text_style.color = "black"; // Avoid having same as background + break; + default: + // Rest of codes below 0x20 are not part of Level 1.5 or related to + // mosaic graphics (non-text) + break; } - leading_spaces = false; + next_char = + 0x20; // These characters result in a space if between start and end + } + if (start_pos == 0 || end_pos != 0) { // Not between start and end + continue; } - switch (next_char) { case '&': next_string.append("&"); @@ -407,8 +497,25 @@ std::string EsParserTeletext::BuildText(const uint8_t* data_block, } break; } } + if (end_pos == 0) { + end_pos = kPayloadSize - 1; + } - return RemoveTrailingSpaces(next_string); + // Using start_pos and end_pos we approximated alignment of text + // depending on the number of spaces to the left and right of the text. + auto left_right_diff = start_pos - (kPayloadSize - 1 - end_pos); + TextAlignment alignment; + if (left_right_diff > 4) { + alignment = TextAlignment::kRight; + } else if (left_right_diff < -4) { + alignment = TextAlignment::kLeft; + } else { + alignment = TextAlignment::kCenter; + } + const auto text_row = TextRow( + {alignment, row, double_height, {TextFragment(text_style, next_string)}}); + + return text_row; } void EsParserTeletext::ParsePacket26(const uint8_t* data_block) { diff --git a/packager/media/formats/mp2t/es_parser_teletext.h b/packager/media/formats/mp2t/es_parser_teletext.h index dda578bf21..0efde38f3e 100644 --- a/packager/media/formats/mp2t/es_parser_teletext.h +++ b/packager/media/formats/mp2t/es_parser_teletext.h @@ -12,6 +12,7 @@ #include #include +#include #include namespace shaka { @@ -37,8 +38,15 @@ class EsParserTeletext : public EsParser { using RowColReplacementMap = std::unordered_map>; + struct TextRow { + TextAlignment alignment; + int row_number; + bool double_height; + TextFragment fragment; + }; + struct TextBlock { - std::vector lines; + std::vector rows; RowColReplacementMap packet_26_replacements; int64_t pts; }; @@ -48,10 +56,10 @@ class EsParserTeletext : public EsParser { const uint8_t* data_block, const uint8_t packet_nr, const uint8_t magazine, - std::string& display_text); + TextRow& display_text); void UpdateCharset(); void SendPending(const uint16_t index, const int64_t pts); - std::string BuildText(const uint8_t* data_block, const uint8_t row) const; + TextRow BuildRow(const uint8_t* data_block, const uint8_t row) const; void ParsePacket26(const uint8_t* data_block); void UpdateNationalSubset(const uint8_t national_subset[13][3]); diff --git a/packager/media/formats/mp2t/es_parser_teletext_unittest.cc b/packager/media/formats/mp2t/es_parser_teletext_unittest.cc index 3cb9d59f48..64f52b97a8 100644 --- a/packager/media/formats/mp2t/es_parser_teletext_unittest.cc +++ b/packager/media/formats/mp2t/es_parser_teletext_unittest.cc @@ -166,6 +166,65 @@ const uint8_t PES_8937764[] = { 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x23, 0xc7, 0x75, 0x8c, 0x1c, 0x04, 0x04, 0x04, 0x86, 0x4f, 0xce, 0x75, 0x75, 0x75, 0x8c, 0x8c}; +// Start (packet0, page88) with packet26 and row 18 left (packet18) +const uint8_t PES_867681[] = { + 0x10, 0x03, 0x2c, 0xf6, 0xe4, 0xa8, 0xa8, 0x0b, 0x0b, 0xa8, 0x0b, 0xa8, + 0x0b, 0xf4, 0xa8, 0xb3, 0x83, 0x32, 0xa2, 0x73, 0x2a, 0xa2, 0x73, 0x23, + 0x83, 0x73, 0x2a, 0xcb, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, + 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x03, + 0x2c, 0xd4, 0xe4, 0xa8, 0x6d, 0xa8, 0x9e, 0xc9, 0x00, 0x4e, 0x93, 0xa7, + 0x90, 0x53, 0xa7, 0x2e, 0xfe, 0xff, 0x2e, 0xfe, 0xff, 0x2e, 0xfe, 0xff, + 0x2e, 0xfe, 0xff, 0x2e, 0xfe, 0xff, 0x2e, 0xfe, 0xff, 0x2e, 0xfe, 0xff, + 0x2e, 0xfe, 0xff, 0x2e, 0xfe, 0xff, 0x2e, 0xfe, 0xff, 0x03, 0x2c, 0xd6, + 0xe4, 0xa8, 0xe3, 0xb0, 0x04, 0x04, 0x04, 0x04, 0xd0, 0xd0, 0xb5, 0x32, + 0xae, 0xd6, 0xa7, 0x04, 0x85, 0x51, 0x51, 0x04, 0x04, 0x04, 0x04, 0x04, + 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, + 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04}; + +// row 22 right (packet22) +const uint8_t PES_871281[] = { + 0x10, 0x03, 0x2c, 0xf4, 0xe4, 0xa8, 0xd9, 0xb0, 0x04, 0x04, 0x04, 0x04, + 0x04, 0x04, 0xd0, 0xd0, 0xb5, 0x52, 0xa7, 0x04, 0x6e, 0x86, 0x97, 0xce, + 0x04, 0x86, 0xae, 0x1f, 0x04, 0xc7, 0xf7, 0xae, 0x4f, 0xce, 0x04, 0x26, + 0xe5, 0xa7, 0x2f, 0xa7, 0x75, 0x51, 0x51, 0x04, 0x04, 0x04, 0x04}; + +// End (packet 0, page 88) +const uint8_t PES_1011695[] = { + 0x10, 0x03, 0x2c, 0xf6, 0xe4, 0xa8, 0xa8, 0x0b, 0x0b, 0xa8, 0x0b, 0xa8, + 0x0b, 0xf4, 0xa8, 0xb3, 0x83, 0x32, 0xa2, 0x73, 0x2a, 0xa2, 0x73, 0x23, + 0x83, 0x73, 0x2a, 0xcb, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, + 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04}; + +// Start (packet0, page 88) with packet26 and row 20 (centered yellow) +// (packet20) +const uint8_t PES_1033297[] = { + 0x10, 0x03, 0x2c, 0xf6, 0xe4, 0xa8, 0xa8, 0x0b, 0x0b, 0xa8, 0x0b, 0xa8, + 0x0b, 0xf4, 0xa8, 0xb3, 0x83, 0x32, 0xa2, 0x73, 0x2a, 0xa2, 0x73, 0x23, + 0x83, 0x73, 0x2a, 0xcb, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, + 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x03, + 0x2c, 0xd4, 0xe4, 0xa8, 0x6d, 0xa8, 0x06, 0xc9, 0x01, 0x62, 0x93, 0xa6, + 0x9e, 0xc9, 0x00, 0xf4, 0xa2, 0x86, 0x06, 0xa3, 0xa7, 0x2e, 0xfe, 0xff, + 0x2e, 0xfe, 0xff, 0x2e, 0xfe, 0xff, 0x2e, 0xfe, 0xff, 0x2e, 0xfe, 0xff, + 0x2e, 0xfe, 0xff, 0x2e, 0xfe, 0xff, 0x2e, 0xfe, 0xff, 0x03, 0x2c, 0xd6, + 0xe4, 0xa8, 0x31, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, + 0x04, 0x04, 0x04, 0xb0, 0xc1, 0xd0, 0xd0, 0x52, 0xe5, 0x86, 0x97, 0x04, + 0x37, 0xf7, 0xae, 0x0e, 0xa7, 0x51, 0x51, 0x04, 0x04, 0x04, 0x04, 0x04, + 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04}; + +// row 22 centered blue on yellow background (packet22) +const uint8_t PES_1036900[] = { + 0x10, 0x03, 0x2c, 0xf4, 0xe4, 0xa8, 0xd9, 0x04, 0x04, 0xb0, 0xc1, 0xb9, + 0x20, 0xd0, 0xd0, 0x37, 0xe5, 0x97, 0x76, 0x97, 0x2f, 0x97, 0x86, 0x2f, + 0x97, 0xf7, 0x76, 0x04, 0x86, 0x04, 0x37, 0xe5, 0x86, 0x37, 0xe6, 0xa7, + 0x46, 0x4f, 0xa7, 0x75, 0x51, 0x51, 0x04, 0x04, 0x04, 0x04, 0x04}; + +// End (packet 0, page 88) +const uint8_t PES_1173713[] = { + 0x10, 0x03, 0x2c, 0xf6, 0xe4, 0xa8, 0xa8, 0x0b, 0x0b, 0xa8, 0x0b, 0xa8, + 0x0b, 0xf4, 0xa8, 0xb3, 0x83, 0x32, 0xa2, 0x73, 0x2a, 0xa2, 0x73, 0x23, + 0x83, 0x73, 0x2a, 0xcb, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, + 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04}; + const uint32_t kPesPid = 123; } // namespace @@ -179,10 +238,12 @@ class EsParserTeletextTest : public ::testing::Test { void OnEmitTextSample(uint32_t pes_pid, std::shared_ptr text_sample) { text_sample_ = text_sample; + text_samples_.push_back(text_sample); } protected: std::shared_ptr stream_info_; + std::vector> text_samples_; std::shared_ptr text_sample_; }; @@ -230,6 +291,13 @@ TEST_F(EsParserTeletextTest, pes_283413_line_emitted_on_next_pes) { EXPECT_EQ(283413, text_sample_->start_time()); EXPECT_EQ(407876, text_sample_->EndTime()); EXPECT_EQ("Bon dia!", text_sample_->body().body); + EXPECT_EQ("black", text_sample_->body().style.backgroundColor); + EXPECT_EQ("white", text_sample_->body().style.color); + TextSettings settings = text_sample_->settings(); + EXPECT_EQ(TextAlignment::kCenter, settings.text_alignment); + EXPECT_TRUE(settings.line.has_value()); + EXPECT_EQ(11, settings.line.value().value); + EXPECT_EQ(TextUnitType::kLines, settings.line.value().type); } TEST_F(EsParserTeletextTest, multiple_lines_with_same_pts) { @@ -260,6 +328,98 @@ TEST_F(EsParserTeletextTest, multiple_lines_with_same_pts) { EXPECT_EQ("-Sí?", text_sample_->body().sub_fragments[0].body); EXPECT_TRUE(text_sample_->body().sub_fragments[1].newline); EXPECT_EQ("-Sí.", text_sample_->body().sub_fragments[2].body); + TextSettings settings = text_sample_->settings(); + EXPECT_EQ(10, settings.line.value().value); + EXPECT_EQ("ttx_10", settings.region); + EXPECT_EQ(1, text_samples_.size()); +} + +// separate_lines_with_slightly_different_pts has the original lines +// 18 and 22, with different alignment, which means that they should +// result in two parallel text samples. +TEST_F(EsParserTeletextTest, separate_lines_with_slightly_different_pts) { + auto on_new_stream = std::bind(&EsParserTeletextTest::OnNewStreamInfo, this, + kPesPid, std::placeholders::_1); + auto on_emit_text = std::bind(&EsParserTeletextTest::OnEmitTextSample, this, + kPesPid, std::placeholders::_1); + + std::unique_ptr es_parser_teletext(new EsParserTeletext( + kPesPid, on_new_stream, on_emit_text, DESCRIPTOR, 12)); + + auto parse_result = + es_parser_teletext->Parse(PES_867681, sizeof(PES_867681), 867681, 0); + EXPECT_TRUE(parse_result); + + parse_result = + es_parser_teletext->Parse(PES_871281, sizeof(PES_871281), 871281, 0); + EXPECT_TRUE(parse_result); + + parse_result = + es_parser_teletext->Parse(PES_1011695, sizeof(PES_1011695), 1011695, 0); + EXPECT_TRUE(parse_result); + + EXPECT_NE(nullptr, text_sample_.get()); + EXPECT_EQ(2, text_samples_.size()); + // The subtitles should get the same start and end time + EXPECT_EQ(867681, text_samples_[0]->start_time()); + EXPECT_EQ(867681, text_samples_[1]->start_time()); + EXPECT_EQ(1011695, text_samples_[0]->EndTime()); + EXPECT_EQ(1011695, text_samples_[0]->EndTime()); + EXPECT_EQ(1, text_samples_[0]->body().sub_fragments.size()); + EXPECT_EQ(1, text_samples_[1]->body().sub_fragments.size()); + EXPECT_EQ("-Luke !", text_samples_[0]->body().sub_fragments[0].body); + EXPECT_EQ("ttx_9", text_samples_[0]->settings().region); + EXPECT_EQ(TextAlignment::kLeft, text_samples_[0]->settings().text_alignment); + EXPECT_EQ("-Je vais aux cours d'été.", + text_samples_[1]->body().sub_fragments[0].body); + EXPECT_EQ(11, text_samples_[1]->settings().line.value().value); + EXPECT_EQ("ttx_11", text_samples_[1]->settings().region); + EXPECT_EQ(TextAlignment::kCenter, + text_samples_[1]->settings().text_alignment); +} + +// consecutive_lines_with_slightly_different_pts has the original lines +// 20 and 22 with same alignment, which means that they should +// result in one text sample with two lines. +TEST_F(EsParserTeletextTest, consecutive_lines_with_slightly_different_pts) { + auto on_new_stream = std::bind(&EsParserTeletextTest::OnNewStreamInfo, this, + kPesPid, std::placeholders::_1); + auto on_emit_text = std::bind(&EsParserTeletextTest::OnEmitTextSample, this, + kPesPid, std::placeholders::_1); + + std::unique_ptr es_parser_teletext(new EsParserTeletext( + kPesPid, on_new_stream, on_emit_text, DESCRIPTOR, 12)); + + auto parse_result = + es_parser_teletext->Parse(PES_1033297, sizeof(PES_1033297), 1033297, 0); + EXPECT_TRUE(parse_result); + + parse_result = + es_parser_teletext->Parse(PES_1036900, sizeof(PES_1036900), 1036900, 0); + EXPECT_TRUE(parse_result); + + parse_result = + es_parser_teletext->Parse(PES_1173713, sizeof(PES_1173713), 1173713, 0); + EXPECT_TRUE(parse_result); + + EXPECT_NE(nullptr, text_sample_.get()); + EXPECT_EQ(1, text_samples_.size()); + // The subtitles should get the same start and end time + EXPECT_EQ(1033297, text_sample_->start_time()); + EXPECT_EQ(1173713, text_sample_->EndTime()); + EXPECT_EQ(3, text_sample_->body().sub_fragments.size()); + TextSettings settings = text_sample_->settings(); + EXPECT_EQ(10, settings.line.value().value); + EXPECT_EQ("ttx_10", settings.region); + EXPECT_EQ(TextAlignment::kCenter, settings.text_alignment); + EXPECT_EQ("J'ai loupé", text_sample_->body().sub_fragments[0].body); + EXPECT_EQ("yellow", text_sample_->body().sub_fragments[0].style.color); + EXPECT_TRUE(text_sample_->body().sub_fragments[1].newline); + EXPECT_EQ("l'initiation à l'algèbre.", + text_sample_->body().sub_fragments[2].body); + EXPECT_EQ("yellow", + text_sample_->body().sub_fragments[2].style.backgroundColor); + EXPECT_EQ("blue", text_sample_->body().sub_fragments[2].style.color); } } // namespace mp2t diff --git a/packager/media/formats/ttml/ttml_generator.cc b/packager/media/formats/ttml/ttml_generator.cc index e7fc596fc3..230582cb4f 100644 --- a/packager/media/formats/ttml/ttml_generator.cc +++ b/packager/media/formats/ttml/ttml_generator.cc @@ -18,6 +18,7 @@ namespace ttml { namespace { constexpr const char* kRegionIdPrefix = "_shaka_region_"; +constexpr const char* kRegionTeletextPrefix = "ttx_"; std::string ToTtmlTime(int64_t time, int32_t timescale) { int64_t remaining = time * 1000 / timescale; @@ -54,6 +55,18 @@ void TtmlGenerator::Initialize(const std::map& regions, regions_ = regions; language_ = language; time_scale_ = time_scale; + // Add ebu_tt_d_regions + float step = 74.1f / 11; + for (int i = 0; i < 12; i++) { + TextRegion region; + float verPos = 10.0 + int(float(step) * float(i)); + region.width = TextNumber(80, TextUnitType::kPercent); + region.height = TextNumber(15, TextUnitType::kPercent); + region.window_anchor_x = TextNumber(10, TextUnitType::kPercent); + region.window_anchor_y = TextNumber(verPos, TextUnitType::kPercent); + const std::string id = kRegionTeletextPrefix + std::to_string(i); + regions_.emplace(id, region); + } } void TtmlGenerator::AddSample(const TextSample& sample) { @@ -66,72 +79,92 @@ void TtmlGenerator::Reset() { bool TtmlGenerator::Dump(std::string* result) const { xml::XmlNode root("tt"); + bool ebuTTDFormat = isEbuTTTD(); RCHECK(root.SetStringAttribute("xmlns", kTtNamespace)); RCHECK(root.SetStringAttribute("xmlns:tts", "http://www.w3.org/ns/ttml#styling")); - - bool did_log = false; - xml::XmlNode head("head"); + RCHECK(root.SetStringAttribute("xmlns:tts", + "http://www.w3.org/ns/ttml#styling")); RCHECK(root.SetStringAttribute("xml:lang", language_)); - for (const auto& pair : regions_) { - if (!did_log && (pair.second.region_anchor_x.value != 0 && - pair.second.region_anchor_y.value != 0)) { - LOG(WARNING) << "TTML doesn't support non-0 region anchor"; - did_log = true; - } - xml::XmlNode region("region"); - const auto origin = - ToTtmlSize(pair.second.window_anchor_x, pair.second.window_anchor_y); - const auto extent = ToTtmlSize(pair.second.width, pair.second.height); - RCHECK(region.SetStringAttribute("xml:id", pair.first)); - RCHECK(region.SetStringAttribute("tts:origin", origin)); - RCHECK(region.SetStringAttribute("tts:extent", extent)); - RCHECK(head.AddChild(std::move(region))); + if (ebuTTDFormat) { + RCHECK(root.SetStringAttribute("xmlns:ttp", + "http://www.w3.org/ns/ttml#parameter")); + RCHECK(root.SetStringAttribute("xmlns:ttm", + "http://www.w3.org/ns/ttml#metadata")); + RCHECK(root.SetStringAttribute("xmlns:ebuttm", "urn:ebu:tt:metadata")); + RCHECK(root.SetStringAttribute("xmlns:ebutts", "urn:ebu:tt:style")); + RCHECK(root.SetStringAttribute("xml:space", "default")); + RCHECK(root.SetStringAttribute("ttp:timeBase", "media")); + RCHECK(root.SetStringAttribute("ttp:cellResolution", "32 15")); } - RCHECK(root.AddChild(std::move(head))); - size_t image_count = 0; + xml::XmlNode head("head"); + xml::XmlNode styling("styling"); xml::XmlNode metadata("metadata"); + xml::XmlNode layout("layout"); + RCHECK(addRegions(layout)); + xml::XmlNode body("body"); + if (ebuTTDFormat) { + RCHECK(body.SetStringAttribute("style", "default")); + } + size_t image_count = 0; + std::unordered_set fragmentStyles; xml::XmlNode div("div"); for (const auto& sample : samples_) { - RCHECK(AddSampleToXml(sample, &div, &metadata, &image_count)); + RCHECK( + AddSampleToXml(sample, &div, &metadata, fragmentStyles, &image_count)); } - RCHECK(body.AddChild(std::move(div))); if (image_count > 0) { RCHECK(root.SetStringAttribute( "xmlns:smpte", "http://www.smpte-ra.org/schemas/2052-1/2010/smpte-tt")); - RCHECK(root.AddChild(std::move(metadata))); } + RCHECK(body.AddChild(std::move(div))); + RCHECK(head.AddChild(std::move(metadata))); + RCHECK(addStyling(styling, fragmentStyles)); + RCHECK(head.AddChild(std::move(styling))); + RCHECK(head.AddChild(std::move(layout))); + RCHECK(root.AddChild(std::move(head))); + RCHECK(root.AddChild(std::move(body))); *result = root.ToString(/* comment= */ ""); return true; } -bool TtmlGenerator::AddSampleToXml(const TextSample& sample, - xml::XmlNode* body, - xml::XmlNode* metadata, - size_t* image_count) const { +bool TtmlGenerator::AddSampleToXml( + const TextSample& sample, + xml::XmlNode* body, + xml::XmlNode* metadata, + std::unordered_set& fragmentStyles, + size_t* image_count) const { xml::XmlNode p("p"); - RCHECK(p.SetStringAttribute("xml:space", "preserve")); + if (!isEbuTTTD()) { + RCHECK(p.SetStringAttribute("xml:space", "preserve")); + } RCHECK(p.SetStringAttribute("begin", ToTtmlTime(sample.start_time(), time_scale_))); RCHECK( p.SetStringAttribute("end", ToTtmlTime(sample.EndTime(), time_scale_))); - RCHECK(ConvertFragmentToXml(sample.body(), &p, metadata, image_count)); + RCHECK(ConvertFragmentToXml(sample.body(), &p, metadata, fragmentStyles, + image_count)); if (!sample.id().empty()) RCHECK(p.SetStringAttribute("xml:id", sample.id())); const auto& settings = sample.settings(); - if (settings.line || settings.position || settings.width || settings.height) { - // TTML positioning needs to be from a region. - if (!settings.region.empty()) { - LOG(WARNING) - << "Using both text regions and positioning isn't supported in TTML"; + bool regionFound = false; + if (!settings.region.empty()) { + auto reg = regions_.find(settings.region); + if (reg != regions_.end()) { + regionFound = true; + RCHECK(p.SetStringAttribute("region", settings.region)); } + } + if (!regionFound && (settings.line || settings.position || settings.width || + settings.height)) { + // TTML positioning needs to be from a region. const auto origin = ToTtmlSize( settings.position.value_or(TextNumber(0, TextUnitType::kPixels)), settings.line.value_or(TextNumber(0, TextUnitType::kPixels))); @@ -146,8 +179,6 @@ bool TtmlGenerator::AddSampleToXml(const TextSample& sample, RCHECK(region.SetStringAttribute("tts:extent", extent)); RCHECK(p.SetStringAttribute("region", id)); RCHECK(body->AddChild(std::move(region))); - } else if (!settings.region.empty()) { - RCHECK(p.SetStringAttribute("region", settings.region)); } if (settings.writing_direction != WritingDirection::kHorizontal) { @@ -179,19 +210,22 @@ bool TtmlGenerator::AddSampleToXml(const TextSample& sample, return true; } -bool TtmlGenerator::ConvertFragmentToXml(const TextFragment& body, - xml::XmlNode* parent, - xml::XmlNode* metadata, - size_t* image_count) const { +bool TtmlGenerator::ConvertFragmentToXml( + const TextFragment& body, + xml::XmlNode* parent, + xml::XmlNode* metadata, + std::unordered_set& fragmentStyles, + size_t* image_count) const { if (body.newline) { xml::XmlNode br("br"); return parent->AddChild(std::move(br)); } - - // If we have new styles, add a new . xml::XmlNode span("span"); xml::XmlNode* node = parent; - if (body.style.bold || body.style.italic || body.style.underline) { + bool useSpan = + (body.style.bold || body.style.italic || body.style.underline || + !body.style.color.empty() || !body.style.backgroundColor.empty()); + if (useSpan) { node = &span; if (body.style.bold) { RCHECK(span.SetStringAttribute("tts:fontWeight", @@ -206,6 +240,20 @@ bool TtmlGenerator::ConvertFragmentToXml(const TextFragment& body, "tts:textDecoration", *body.style.underline ? "underline" : "noUnderline")); } + std::string color = "white"; + std::string backgroundColor = "black"; + + if (!body.style.color.empty()) { + color = body.style.color; + } + + if (!body.style.backgroundColor.empty()) { + backgroundColor = body.style.backgroundColor; + } + + const std::string fragStyle = color + "_" + backgroundColor; + fragmentStyles.insert(fragStyle); + RCHECK(span.SetStringAttribute("style", fragStyle)); } if (!body.body.empty()) { @@ -226,16 +274,91 @@ bool TtmlGenerator::ConvertFragmentToXml(const TextFragment& body, RCHECK(node->SetStringAttribute("smpte:backgroundImage", "#" + id)); } else { for (const auto& frag : body.sub_fragments) { - if (!ConvertFragmentToXml(frag, node, metadata, image_count)) + if (!ConvertFragmentToXml(frag, node, metadata, fragmentStyles, + image_count)) return false; } } - if (body.style.bold || body.style.italic || body.style.underline) + if (useSpan) RCHECK(parent->AddChild(std::move(span))); return true; } +std::vector TtmlGenerator::usedRegions() const { + std::vector uRegions; + for (const auto& sample : samples_) { + if (!sample.settings().region.empty()) { + uRegions.push_back(sample.settings().region); + } + } + return uRegions; +} + +bool TtmlGenerator::addRegions(xml::XmlNode& layout) const { + auto regNames = usedRegions(); + for (const auto& r : regions_) { + bool used = false; + for (const auto& name : regNames) { + if (r.first == name) { + used = true; + } + } + if (used) { + xml::XmlNode region("region"); + const auto origin = + ToTtmlSize(r.second.window_anchor_x, r.second.window_anchor_y); + const auto extent = ToTtmlSize(r.second.width, r.second.height); + RCHECK(region.SetStringAttribute("xml:id", r.first)); + RCHECK(region.SetStringAttribute("tts:origin", origin)); + RCHECK(region.SetStringAttribute("tts:extent", extent)); + RCHECK(region.SetStringAttribute("tts:overflow", "visible")); + RCHECK(layout.AddChild(std::move(region))); + } + } + return true; +} + +bool TtmlGenerator::addStyling( + xml::XmlNode& styling, + const std::unordered_set& fragmentStyles) const { + if (fragmentStyles.empty()) { + return true; + } + // Add default style + xml::XmlNode defaultStyle("style"); + RCHECK(defaultStyle.SetStringAttribute("xml:id", "default")); + RCHECK(defaultStyle.SetStringAttribute("tts:fontStyle", "normal")); + RCHECK(defaultStyle.SetStringAttribute("tts:fontFamily", "sansSerif")); + RCHECK(defaultStyle.SetStringAttribute("tts:fontSize", "100%")); + RCHECK(defaultStyle.SetStringAttribute("tts:lineHeight", "normal")); + RCHECK(defaultStyle.SetStringAttribute("tts:textAlign", "center")); + RCHECK(defaultStyle.SetStringAttribute("ebutts:linePadding", "0.5c")); + RCHECK(styling.AddChild(std::move(defaultStyle))); + + for (const auto& name : fragmentStyles) { + auto pos = name.find('_'); + auto color = name.substr(0, pos); + auto backgroundColor = name.substr(pos + 1, name.size()); + xml::XmlNode fragStyle("style"); + RCHECK(fragStyle.SetStringAttribute("xml:id", name)); + RCHECK( + fragStyle.SetStringAttribute("tts:backgroundColor", backgroundColor)); + RCHECK(fragStyle.SetStringAttribute("tts:color", color)); + RCHECK(styling.AddChild(std::move(fragStyle))); + } + return true; +} + +bool TtmlGenerator::isEbuTTTD() const { + for (const auto& sample : samples_) { + if (sample.settings().region.rfind(kRegionTeletextPrefix, 0) == 0) { + return true; + } + } + return false; +} + } // namespace ttml } // namespace media } // namespace shaka diff --git a/packager/media/formats/ttml/ttml_generator.h b/packager/media/formats/ttml/ttml_generator.h index 303a3e4b33..9c1afbdfaf 100644 --- a/packager/media/formats/ttml/ttml_generator.h +++ b/packager/media/formats/ttml/ttml_generator.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -38,12 +39,20 @@ class TtmlGenerator { bool AddSampleToXml(const TextSample& sample, xml::XmlNode* body, xml::XmlNode* metadata, + std::unordered_set& fragmentStyles, size_t* image_count) const; bool ConvertFragmentToXml(const TextFragment& fragment, xml::XmlNode* parent, xml::XmlNode* metadata, + std::unordered_set& fragmentStyles, size_t* image_count) const; + bool addStyling(xml::XmlNode& styling, + const std::unordered_set& fragmentStyles) const; + bool addRegions(xml::XmlNode& layout) const; + std::vector usedRegions() const; + bool isEbuTTTD() const; + std::list samples_; std::map regions_; std::string language_; diff --git a/packager/media/formats/ttml/ttml_generator_unittest.cc b/packager/media/formats/ttml/ttml_generator_unittest.cc index 23728d223f..0f0d04dcbd 100644 --- a/packager/media/formats/ttml/ttml_generator_unittest.cc +++ b/packager/media/formats/ttml/ttml_generator_unittest.cc @@ -64,7 +64,11 @@ TEST_F(TtmlMuxerTest, WithOneSegmentAndWithOneSample) { "\n" "\n" - " \n" + " \n" + " \n" + " \n" + " \n" + " \n" " \n" "
\n" "

\n" "\n" - " \n" + " \n" + " \n" + " \n" + " \n" + " \n" " \n" "

\n" "

\n" "\n" - " \n" + " \n" + " \n" + " \n" + " \n" + " \n" " \n" "

\n" "

\n" " \n" - " \n" + " \n" + " \n" + " \n" + " \n" + " \n" " \n" " \n" "

\n" @@ -166,7 +182,11 @@ TEST_F(TtmlMuxerTest, HandlesLanguage) { "\n" "\n" - " \n" + " \n" + " \n" + " \n" + " \n" + " \n" " \n" "
\n" "

\n" "\n" - " \n" + " \n" + " \n" + " \n" + " \n" + " \n" " \n" "

\n" " \n" "\n" - " \n" + " \n" + " \n" + " \n" + " \n" + " \n" " \n" "
\n" "

\n" "\n" - " \n" + " \n" + " \n" + " \n" + " \n" + " \n" " \n" "

\n" "

\n" " \n" - " \n" + " \n" + " \n" + " \n" + " \n" + " \n" " \n" " \n" "

\n" @@ -287,7 +323,11 @@ TEST_F(TtmlMuxerTest, HandlesReset) { "\n" "\n" - " \n" + " \n" + " \n" + " \n" + " \n" + " \n" " \n" "
\n" "

\n" "\n" - " \n" + " \n" + " \n" + " \n" + " \n" + " \n" " \n" "

\n" "

\n" - " \n" - " \n" - " " + " \n" + " \n" + " " "AQID\n" - " \n" + " \n" + " \n" + " \n" + " \n" " \n" "

\n" "

\n" "\n" - " \n" + " \n" + " \n" + " \n" + " \n" + " \n" " \n" "

\n" "

\n" + "\n" + " \n" + " \n" + " \n" + "