shaka-packager/packager/media/formats/mp2t/es_parser_teletext.cc

513 lines
15 KiB
C++
Raw Normal View History

// Copyright 2020 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#include <packager/media/formats/mp2t/es_parser_teletext.h>
#include <packager/media/base/bit_reader.h>
#include <packager/media/base/text_stream_info.h>
#include <packager/media/base/timestamp.h>
#include <packager/media/formats/mp2t/es_parser_teletext_tables.h>
#include <packager/media/formats/mp2t/mp2t_common.h>
namespace shaka {
namespace media {
namespace mp2t {
namespace {
const uint8_t EBU_TELETEXT_WITH_SUBTITLING = 0x03;
const int kPayloadSize = 40;
const int kNumTriplets = 13;
template <typename T>
constexpr T bit(T value, const size_t bit_pos) {
return (value >> bit_pos) & 0x1;
}
uint8_t ReadHamming(BitReader& reader) {
uint8_t bits;
RCHECK(reader.ReadBits(8, &bits));
return TELETEXT_HAMMING_8_4[bits];
}
bool Hamming_24_18(const uint32_t value, uint32_t& out_result) {
uint32_t result = value;
uint8_t test = 0;
for (uint8_t i = 0; i < 23; i++) {
test ^= ((result >> i) & 0x01) * (i + 0x21);
}
test ^= ((result >> 0x17) & 0x01) * 0x20;
if ((test & 0x1f) != 0x1f) {
if ((test & 0x20) == 0x20) {
return false;
}
result ^= 1 << (0x1e - test);
}
out_result = (result & 0x000004) >> 2 | (result & 0x000070) >> 3 |
(result & 0x007f00) >> 4 | (result & 0x7f0000) >> 5;
return true;
}
bool ParseSubtitlingDescriptor(
const uint8_t* descriptor,
const size_t size,
std::unordered_map<uint16_t, std::string>& result) {
BitReader reader(descriptor, size);
RCHECK(reader.SkipBits(8));
size_t data_size;
RCHECK(reader.ReadBits(8, &data_size));
RCHECK(data_size + 2 <= size);
for (size_t i = 0; i < data_size; i += 8) {
uint32_t lang_code;
RCHECK(reader.ReadBits(24, &lang_code));
uint8_t ignored_teletext_type;
RCHECK(reader.ReadBits(5, &ignored_teletext_type));
uint8_t magazine_number;
RCHECK(reader.ReadBits(3, &magazine_number));
if (magazine_number == 0) {
magazine_number = 8;
}
uint8_t page_number_tens;
RCHECK(reader.ReadBits(4, &page_number_tens));
uint8_t page_number_units;
RCHECK(reader.ReadBits(4, &page_number_units));
const uint8_t page_number = page_number_tens * 10 + page_number_units;
std::string lang(3, '\0');
lang[0] = static_cast<char>((lang_code >> 16) & 0xff);
lang[1] = static_cast<char>((lang_code >> 8) & 0xff);
lang[2] = static_cast<char>((lang_code >> 0) & 0xff);
const uint16_t index = magazine_number * 100 + page_number;
result.emplace(index, std::move(lang));
}
return true;
}
std::string RemoveTrailingSpaces(const std::string& input) {
const auto index = input.find_last_not_of(' ');
if (index == std::string::npos) {
return "";
}
return input.substr(0, index + 1);
}
} // namespace
EsParserTeletext::EsParserTeletext(const uint32_t pid,
const NewStreamInfoCB& new_stream_info_cb,
const EmitTextSampleCB& emit_sample_cb,
const uint8_t* descriptor,
const size_t descriptor_length)
: EsParser(pid),
new_stream_info_cb_(new_stream_info_cb),
emit_sample_cb_(emit_sample_cb),
magazine_(0),
page_number_(0),
charset_code_(0),
current_charset_{},
last_pts_(0) {
if (!ParseSubtitlingDescriptor(descriptor, descriptor_length, languages_)) {
LOG(ERROR) << "Unable to parse teletext_descriptor";
}
UpdateCharset();
}
bool EsParserTeletext::Parse(const uint8_t* buf,
int size,
int64_t pts,
int64_t dts) {
if (!sent_info_) {
sent_info_ = true;
auto info = std::make_shared<TextStreamInfo>(pid(), kMpeg2Timescale,
kInfiniteDuration, kCodecText,
"", "", 0, 0, "");
for (const auto& pair : languages_) {
info->AddSubStream(pair.first, {pair.second});
}
new_stream_info_cb_(info);
}
return ParseInternal(buf, size, pts);
}
bool EsParserTeletext::Flush() {
std::vector<uint16_t> keys;
for (const auto& entry : page_state_) {
keys.push_back(entry.first);
}
for (const auto key : keys) {
SendPending(key, last_pts_);
}
return true;
}
void EsParserTeletext::Reset() {
page_state_.clear();
magazine_ = 0;
page_number_ = 0;
sent_info_ = false;
charset_code_ = 0;
UpdateCharset();
}
bool EsParserTeletext::ParseInternal(const uint8_t* data,
const size_t size,
const int64_t pts) {
BitReader reader(data, size);
RCHECK(reader.SkipBits(8));
std::vector<std::string> lines;
while (reader.bits_available()) {
uint8_t data_unit_id;
RCHECK(reader.ReadBits(8, &data_unit_id));
uint8_t data_unit_length;
RCHECK(reader.ReadBits(8, &data_unit_length));
if (data_unit_length != 44) {
LOG(ERROR) << "Bad Teletext data length";
break;
}
if (data_unit_id != EBU_TELETEXT_WITH_SUBTITLING) {
RCHECK(reader.SkipBytes(44));
continue;
}
RCHECK(reader.SkipBits(16));
uint16_t address_bits;
RCHECK(reader.ReadBits(16, &address_bits));
uint8_t magazine = bit(address_bits, 14) + 2 * bit(address_bits, 12) +
4 * bit(address_bits, 10);
if (magazine == 0) {
magazine = 8;
}
const uint8_t packet_nr =
(bit(address_bits, 8) + 2 * bit(address_bits, 6) +
4 * bit(address_bits, 4) + 8 * bit(address_bits, 2) +
16 * bit(address_bits, 0));
const uint8_t* data_block = reader.current_byte_ptr();
RCHECK(reader.SkipBytes(40));
std::string display_text;
if (ParseDataBlock(pts, data_block, packet_nr, magazine, display_text)) {
lines.emplace_back(std::move(display_text));
}
}
if (lines.empty()) {
return true;
}
const uint16_t index = magazine_ * 100 + page_number_;
auto page_state_itr = page_state_.find(index);
if (page_state_itr == page_state_.end()) {
page_state_.emplace(index, TextBlock{std::move(lines), {}, last_pts_});
} else {
for (auto& line : lines) {
auto& page_state_lines = page_state_itr->second.lines;
page_state_lines.emplace_back(std::move(line));
}
lines.clear();
}
return true;
}
bool EsParserTeletext::ParseDataBlock(const int64_t pts,
const uint8_t* data_block,
const uint8_t packet_nr,
const uint8_t magazine,
std::string& display_text) {
if (packet_nr == 0) {
last_pts_ = pts;
BitReader reader(data_block, 32);
const uint8_t page_number_units = ReadHamming(reader);
const uint8_t page_number_tens = ReadHamming(reader);
const uint8_t page_number = 10 * page_number_tens + page_number_units;
const uint16_t index = magazine * 100 + page_number;
SendPending(index, pts);
page_number_ = page_number;
magazine_ = magazine;
if (page_number == 0xFF) {
return false;
}
RCHECK(reader.SkipBits(40));
const uint8_t subcode_c11_c14 = ReadHamming(reader);
const uint8_t charset_code = subcode_c11_c14 >> 1;
if (charset_code != charset_code_) {
charset_code_ = charset_code;
UpdateCharset();
}
return false;
} else if (packet_nr == 26) {
ParsePacket26(data_block);
return false;
} else if (packet_nr > 26) {
return false;
}
display_text = BuildText(data_block, packet_nr);
return true;
}
void EsParserTeletext::UpdateCharset() {
memcpy(current_charset_, TELETEXT_CHARSET_G0_LATIN, sizeof(TELETEXT_CHARSET_G0_LATIN));
if (charset_code_ > 7) {
return;
}
const auto teletext_national_subset =
static_cast<TELETEXT_NATIONAL_SUBSET>(charset_code_);
switch (teletext_national_subset) {
case TELETEXT_NATIONAL_SUBSET::ENGLISH:
UpdateNationalSubset(TELETEXT_NATIONAL_SUBSET_ENGLISH);
break;
case TELETEXT_NATIONAL_SUBSET::FRENCH:
UpdateNationalSubset(TELETEXT_NATIONAL_SUBSET_FRENCH);
break;
case TELETEXT_NATIONAL_SUBSET::SWEDISH_FINNISH_HUNGARIAN:
UpdateNationalSubset(TELETEXT_NATIONAL_SUBSET_SWEDISH_FINNISH_HUNGARIAN);
break;
case TELETEXT_NATIONAL_SUBSET::CZECH_SLOVAK:
UpdateNationalSubset(TELETEXT_NATIONAL_SUBSET_CZECH_SLOVAK);
break;
case TELETEXT_NATIONAL_SUBSET::GERMAN:
UpdateNationalSubset(TELETEXT_NATIONAL_SUBSET_GERMAN);
break;
case TELETEXT_NATIONAL_SUBSET::PORTUGUESE_SPANISH:
UpdateNationalSubset(TELETEXT_NATIONAL_SUBSET_PORTUGUESE_SPANISH);
break;
case TELETEXT_NATIONAL_SUBSET::ITALIAN:
UpdateNationalSubset(TELETEXT_NATIONAL_SUBSET_ITALIAN);
break;
case TELETEXT_NATIONAL_SUBSET::NONE:
default:
break;
}
}
void EsParserTeletext::SendPending(const uint16_t index, const int64_t pts) {
auto page_state_itr = page_state_.find(index);
if (page_state_itr == page_state_.end() ||
page_state_itr->second.lines.empty()) {
return;
}
const auto& pending_lines = page_state_itr->second.lines;
const auto pending_pts = page_state_itr->second.pts;
TextFragmentStyle text_fragment_style;
TextSettings text_settings;
std::shared_ptr<TextSample> text_sample;
if (pending_lines.size() == 1) {
TextFragment text_fragment(text_fragment_style, pending_lines[0].c_str());
text_sample = std::make_shared<TextSample>("", pending_pts, pts,
text_settings, text_fragment);
} else {
std::vector<TextFragment> sub_fragments;
for (const auto& line : pending_lines) {
sub_fragments.emplace_back(text_fragment_style, line.c_str());
sub_fragments.emplace_back(text_fragment_style, true);
}
sub_fragments.pop_back();
TextFragment text_fragment(text_fragment_style, sub_fragments);
text_sample = std::make_shared<TextSample>("", pending_pts, pts,
text_settings, text_fragment);
}
text_sample->set_sub_stream_index(index);
emit_sample_cb_(text_sample);
page_state_.erase(index);
}
std::string EsParserTeletext::BuildText(const uint8_t* data_block,
const uint8_t row) const {
std::string next_string;
next_string.reserve(kPayloadSize * 2);
bool leading_spaces = true;
const uint16_t index = magazine_ * 100 + page_number_;
const auto page_state_itr = page_state_.find(index);
const std::unordered_map<uint8_t, std::string>* column_replacement_map =
nullptr;
if (page_state_itr != page_state_.cend()) {
const auto row_itr =
page_state_itr->second.packet_26_replacements.find(row);
if (row_itr != page_state_itr->second.packet_26_replacements.cend()) {
column_replacement_map = &(row_itr->second);
}
}
for (size_t i = 0; i < kPayloadSize; ++i) {
if (column_replacement_map) {
const auto column_itr = column_replacement_map->find(i);
if (column_itr != column_replacement_map->cend()) {
next_string.append(column_itr->second);
leading_spaces = false;
continue;
}
}
char next_char =
static_cast<char>(TELETEXT_BITREVERSE_8[data_block[i]] & 0x7f);
if (next_char < 32) {
next_char = 0x20;
}
if (leading_spaces) {
if (next_char == 0x20) {
continue;
}
leading_spaces = false;
}
switch (next_char) {
case '&':
next_string.append("&amp;");
break;
case '<':
next_string.append("&lt;");
break;
default: {
const std::string replacement(current_charset_[next_char - 0x20]);
next_string.append(replacement);
} break;
}
}
return RemoveTrailingSpaces(next_string);
}
void EsParserTeletext::ParsePacket26(const uint8_t* data_block) {
const uint16_t index = magazine_ * 100 + page_number_;
auto page_state_itr = page_state_.find(index);
if (page_state_itr == page_state_.end()) {
page_state_.emplace(index, TextBlock{{}, {}, last_pts_});
}
auto& replacement_map = page_state_[index].packet_26_replacements;
uint8_t row = 0;
std::vector<uint32_t> x26_triplets;
x26_triplets.reserve(kNumTriplets);
for (uint8_t i = 1; i < kPayloadSize; i += 3) {
const uint32_t bytes = (TELETEXT_BITREVERSE_8[data_block[i + 2]] << 16) |
(TELETEXT_BITREVERSE_8[data_block[i + 1]] << 8) |
TELETEXT_BITREVERSE_8[data_block[i]];
uint32_t triplet;
if (Hamming_24_18(bytes, triplet)) {
x26_triplets.emplace_back(triplet);
}
}
for (const auto triplet : x26_triplets) {
const uint8_t mode = (triplet & 0x7c0) >> 6;
const uint8_t address = triplet & 0x3f;
const uint8_t row_address_group = (address >= 0x28) && (address <= 0x3f);
if ((mode == 0x4) && (row_address_group == 0x1)) {
row = address - 0x28;
if (row == 0x0) {
row = 0x18;
}
}
if (mode >= 0x11 && mode <= 0x1f && row_address_group == 0x1) {
break;
}
const uint8_t data = (triplet & 0x3f800) >> 11;
if (mode == 0x0f && row_address_group == 0x0 && data > 0x1f) {
SetPacket26ReplacementString(replacement_map, row, address,
reinterpret_cast<const char*>(
TELETEXT_CHARSET_G2_LATIN[data - 0x20]));
}
if (mode == 0x10 && row_address_group == 0x0 && data == 0x40) {
SetPacket26ReplacementString(replacement_map, row, address, "@");
}
if (mode < 0x11 || mode > 0x1f || row_address_group != 0x0) {
continue;
}
if (data >= 0x41 && data <= 0x5a) {
SetPacket26ReplacementString(
replacement_map, row, address,
reinterpret_cast<const char*>(
TELETEXT_G2_LATIN_ACCENTS[mode - 0x11][data - 0x41]));
} else if (data >= 0x61 && data <= 0x7a) {
SetPacket26ReplacementString(
replacement_map, row, address,
reinterpret_cast<const char*>(
TELETEXT_G2_LATIN_ACCENTS[mode - 0x11][data - 0x47]));
} else if ((data & 0x7f) >= 0x20) {
SetPacket26ReplacementString(
replacement_map, row, address,
reinterpret_cast<const char*>(
TELETEXT_CHARSET_G0_LATIN[(data & 0x7f) - 0x20]));
}
}
}
void EsParserTeletext::UpdateNationalSubset(
const uint8_t national_subset[13][3]) {
for (size_t i = 0; i < 13; ++i) {
const size_t position = TELETEXT_NATIONAL_CHAR_INDEX_G0[i];
memcpy(current_charset_[position], national_subset[i], 3);
}
}
void EsParserTeletext::SetPacket26ReplacementString(
RowColReplacementMap& replacement_map,
const uint8_t row,
const uint8_t column,
std::string&& replacement_string) {
auto replacement_map_itr = replacement_map.find(row);
if (replacement_map_itr == replacement_map.cend()) {
replacement_map.emplace(row, std::unordered_map<uint8_t, std::string>{});
}
auto& column_map = replacement_map[row];
column_map.emplace(column, std::move(replacement_string));
}
} // namespace mp2t
} // namespace media
} // namespace shaka