Mirror-Tools
/
shaka-packager
mirror of https://github.com/shaka-project/shaka-packager.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add subtitle composition to DVB-sub parser. 

Issue #832

Change-Id: Iababe884619e1e48f1abe0806e8b863c95a3c1ef
This commit is contained in:
Jacob Trimble 2020-10-05 15:31:31 -07:00
parent 32c5393fba
commit 89d407f9ae
5 changed files with 514 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
packager/media/formats/dvb/dvb.gyp
View File

@ -15,6 +15,8 @@
'sources': [
'dvb_image.cc',
'dvb_image.h',
'subtitle_composer.cc',
'subtitle_composer.h',
],
'dependencies': [
'../../base/media_base.gyp:media_base',
@ -26,6 +28,7 @@
'type': '<(gtest_target_type)',
'sources': [
'dvb_image_unittest.cc',
'subtitle_composer_unittest.cc',
],
'dependencies': [
'../../../testing/gtest.gyp:gtest',

254
packager/media/formats/dvb/subtitle_composer.cc Normal file
View File

@ -0,0 +1,254 @@
// Copyright 2020 Google LLC. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#include "packager/media/formats/dvb/subtitle_composer.h"
#include <png.h>
#include <string.h>
#include "packager/base/logging.h"
namespace shaka {
namespace media {
namespace {
const uint16_t kDefaultWidth = 720;
const uint16_t kDefaultHeight = 576;
const RgbaColor kTransparent{0, 0, 0, 0};
struct PngFreeHelper {
PngFreeHelper(png_structp* png, png_infop* info) : png(png), info(info) {}
~PngFreeHelper() { png_destroy_write_struct(png, info); }
png_structp* png;
png_infop* info;
};
void PngWriteData(png_structp png, png_bytep data, png_size_t length) {
auto* output = reinterpret_cast<std::vector<uint8_t>*>(png_get_io_ptr(png));
output->insert(output->end(), data, data + length);
}
void PngFlushData(png_structp png) {}
bool IsTransparent(const RgbaColor* colors, uint16_t width, uint16_t height) {
for (size_t i = 0; i < static_cast<size_t>(width) * height; i++) {
if (colors[i].a != 0)
return false;
}
return true;
}
bool GetImageData(const DvbImageBuilder* image,
std::vector<uint8_t>* data,
uint16_t* width,
uint16_t* height) {
// CAREFUL in this method since this uses long-jumps. A long-jump causes the
// execution to jump to another point *without executing returns*. This
// causes C++ objects to not get destroyed. This also causes the same code to
// be executed twice, so C++ objects can be initialized twice.
//
// So long as we don't create C++ objects after the long-jump point,
// everything should work fine. If we early-return after the long-jump, the
// destructors will still be called; if we long-jump, we won't call the
// constructors since we're past that point.
auto png =
png_create_write_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr);
auto info = png_create_info_struct(png);
PngFreeHelper helper(&png, &info);
if (!png || !info) {
LOG(ERROR) << "Error creating libpng struct";
return false;
}
if (setjmp(png_jmpbuf(png))) {
// If any png_* functions fail, the code will jump back to here.
LOG(ERROR) << "Error writing PNG image";
return false;
}
png_set_write_fn(png, data, &PngWriteData, &PngFlushData);
const RgbaColor* pixels;
if (!image->GetPixels(&pixels, width, height))
return false;
if (IsTransparent(pixels, *width, *height))
return true; // Skip empty/transparent images.
png_set_IHDR(png, info, *width, *height, 8, PNG_COLOR_TYPE_RGBA,
PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_BASE,
PNG_FILTER_TYPE_BASE);
png_write_info(png, info);
const uint8_t* in_data = reinterpret_cast<const uint8_t*>(pixels);
for (size_t y = 0; y < *height; y++) {
size_t offset = image->max_width() * y * sizeof(RgbaColor);
png_write_row(png, in_data + offset);
}
png_write_end(png, nullptr);
return true;
}
} // namespace
SubtitleComposer::SubtitleComposer()
: display_width_(kDefaultWidth), display_height_(kDefaultHeight) {}
SubtitleComposer::~SubtitleComposer() {}
void SubtitleComposer::SetDisplaySize(uint16_t width, uint16_t height) {
display_width_ = width;
display_height_ = height;
}
bool SubtitleComposer::SetRegionInfo(uint8_t region_id,
uint8_t color_space_id,
uint16_t width,
uint16_t height) {
auto* region = &regions_[region_id];
if (region->x + width > display_width_ ||
region->y + height > display_height_) {
LOG(ERROR) << "DVB-sub region won't fit within display";
return false;
}
if (width == 0 || height == 0) {
LOG(ERROR) << "DVB-sub width/height cannot be 0";
return false;
}
region->width = width;
region->height = height;
region->color_space = &color_spaces_[color_space_id];
return true;
}
bool SubtitleComposer::SetRegionPosition(uint8_t region_id,
uint16_t x,
uint16_t y) {
auto* region = &regions_[region_id];
if (x + region->width > display_width_ ||
y + region->height > display_height_) {
LOG(ERROR) << "DVB-sub region won't fit within display";
return false;
}
region->x = x;
region->y = y;
return true;
}
bool SubtitleComposer::SetObjectInfo(uint16_t object_id,
uint8_t region_id,
uint16_t x,
uint16_t y,
int default_color_code) {
auto region = regions_.find(region_id);
if (region == regions_.end()) {
LOG(ERROR) << "Unknown DVB-sub region: " << (int)region_id
<< ", in object: " << object_id;
return false;
}
if (x >= region->second.width || y >= region->second.height) {
LOG(ERROR) << "DVB-sub object is outside region: " << object_id;
return false;
}
auto* object = &objects_[object_id];
object->region = &region->second;
object->default_color_code = default_color_code;
object->x = x;
object->y = y;
return true;
}
DvbImageColorSpace* SubtitleComposer::GetColorSpace(uint8_t color_space_id) {
return &color_spaces_[color_space_id];
}
DvbImageColorSpace* SubtitleComposer::GetColorSpaceForObject(
uint16_t object_id) {
auto info = objects_.find(object_id);
if (info == objects_.end()) {
LOG(ERROR) << "Unknown DVB-sub object: " << object_id;
return nullptr;
}
return info->second.region->color_space;
}
DvbImageBuilder* SubtitleComposer::GetObjectImage(uint16_t object_id) {
auto it = images_.find(object_id);
if (it == images_.end()) {
auto info = objects_.find(object_id);
if (info == objects_.end()) {
LOG(ERROR) << "Unknown DVB-sub object: " << object_id;
return nullptr;
}
auto color = info->second.default_color_code < 0
? kTransparent
: info->second.region->color_space->GetColor(
BitDepth::k8Bit, info->second.default_color_code);
it = images_
.emplace(std::piecewise_construct, std::make_tuple(object_id),
std::make_tuple(
info->second.region->color_space, color,
info->second.region->width - info->second.region->x,
info->second.region->height - info->second.region->y))
.first;
}
return &it->second;
}
bool SubtitleComposer::GetSamples(
int64_t start,
int64_t end,
std::vector<std::shared_ptr<TextSample>>* samples) const {
for (const auto& pair : objects_) {
auto it = images_.find(pair.first);
if (it == images_.end()) {
LOG(WARNING) << "DVB-sub object " << pair.first
<< " doesn't include object data";
continue;
}
uint16_t width, height;
std::vector<uint8_t> image_data;
if (!GetImageData(&it->second, &image_data, &width, &height))
return false;
if (image_data.empty()) {
VLOG(1) << "Skipping transparent object";
continue;
}
TextFragment body({}, image_data);
DCHECK_LE(width, display_width_);
DCHECK_LE(height, display_height_);
TextSettings settings;
settings.position.emplace(
(pair.second.x + pair.second.region->x) * 100.0f / display_width_,
TextUnitType::kPercent);
settings.line.emplace(
(pair.second.y + pair.second.region->y) * 100.0f / display_height_,
TextUnitType::kPercent);
settings.width.emplace(width * 100.0f / display_width_,
TextUnitType::kPercent);
settings.height.emplace(height * 100.0f / display_height_,
TextUnitType::kPercent);
samples->emplace_back(
std::make_shared<TextSample>("", start, end, settings, body));
}
return true;
}
void SubtitleComposer::ClearObjects() {
regions_.clear();
objects_.clear();
images_.clear();
}
} // namespace media
} // namespace shaka

79
packager/media/formats/dvb/subtitle_composer.h Normal file
View File

@ -0,0 +1,79 @@
// Copyright 2020 Google LLC. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#ifndef PACKAGER_MEDIA_DVB_SUBTITLE_COMPOSER_H_
#define PACKAGER_MEDIA_DVB_SUBTITLE_COMPOSER_H_
#include <memory>
#include <unordered_map>
#include <vector>
#include "packager/base/macros.h"
#include "packager/media/base/text_sample.h"
#include "packager/media/formats/dvb/dvb_image.h"
namespace shaka {
namespace media {
/// Holds pixel/caption data for a single DVB-sub page. This composes
/// multiple objects and creates TextSample objects from it.
class SubtitleComposer {
public:
SubtitleComposer();
~SubtitleComposer();
DISALLOW_COPY_AND_ASSIGN(SubtitleComposer);
void SetDisplaySize(uint16_t width, uint16_t height);
bool SetRegionPosition(uint8_t region_id, uint16_t x, uint16_t y);
bool SetRegionInfo(uint8_t region_id,
uint8_t color_space_id,
uint16_t width,
uint16_t height);
bool SetObjectInfo(uint16_t object_id,
uint8_t region_id,
uint16_t x,
uint16_t y,
int default_color_code);
DvbImageColorSpace* GetColorSpace(uint8_t color_space_id);
DvbImageColorSpace* GetColorSpaceForObject(uint16_t object_id);
DvbImageBuilder* GetObjectImage(uint16_t object_id);
bool GetSamples(int64_t start,
int64_t end,
std::vector<std::shared_ptr<TextSample>>* samples) const;
void ClearObjects();
private:
struct RegionInfo {
DvbImageColorSpace* color_space = nullptr;
uint16_t x = 0;
uint16_t y = 0;
uint16_t width = 0;
uint16_t height = 0;
};
struct ObjectInfo {
RegionInfo* region = nullptr;
int default_color_code = -1;
uint16_t x = 0;
uint16_t y = 0;
};
// Maps of IDs to their respective object.
std::unordered_map<uint8_t, RegionInfo> regions_;
std::unordered_map<uint8_t, DvbImageColorSpace> color_spaces_;
std::unordered_map<uint16_t, ObjectInfo> objects_;
std::unordered_map<uint16_t, DvbImageBuilder> images_; // Uses object_id.
uint16_t display_width_;
uint16_t display_height_;
};
} // namespace media
} // namespace shaka
#endif // PACKAGER_MEDIA_DVB_SUBTITLE_COMPOSER_H_

177
packager/media/formats/dvb/subtitle_composer_unittest.cc Normal file
View File

@ -0,0 +1,177 @@
// Copyright 2020 Google LLC. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#include "packager/media/formats/dvb/subtitle_composer.h"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
namespace shaka {
namespace media {
namespace {
constexpr const int kNoBgColor = -1;
void CreateDefaultImage(SubtitleComposer* composer, uint16_t object_id) {
auto* image = composer->GetObjectImage(object_id);
EXPECT_TRUE(image->AddPixel(BitDepth::k8Bit, 1, true));
image->NewRow(true);
}
} // namespace
TEST(SubtitleComposerTest, PositionsSamples) {
const uint8_t kRegionId = 3, kColorSpaceId = 9;
const uint16_t kObjectId1 = 5, kObjectId2 = 11;
SubtitleComposer composer;
composer.SetDisplaySize(100, 100);
ASSERT_TRUE(composer.SetRegionPosition(kRegionId, 4, 5));
ASSERT_TRUE(composer.SetRegionInfo(kRegionId, kColorSpaceId, 96, 95));
ASSERT_TRUE(composer.SetObjectInfo(kObjectId1, kRegionId, 8, 9, kNoBgColor));
ASSERT_TRUE(
composer.SetObjectInfo(kObjectId2, kRegionId, 12, 14, kNoBgColor));
CreateDefaultImage(&composer, kObjectId1);
CreateDefaultImage(&composer, kObjectId2);
std::vector<std::shared_ptr<TextSample>> samples;
ASSERT_TRUE(composer.GetSamples(0, 1, &samples));
ASSERT_EQ(samples.size(), 2u);
for (size_t i = 0; i < samples.size(); i++) {
ASSERT_TRUE(samples[i]->settings().line);
ASSERT_EQ(samples[i]->settings().line->type, TextUnitType::kPercent);
ASSERT_TRUE(samples[i]->settings().position);
ASSERT_EQ(samples[i]->settings().position->type, TextUnitType::kPercent);
ASSERT_TRUE(samples[i]->settings().width);
ASSERT_EQ(samples[i]->settings().width->type, TextUnitType::kPercent);
ASSERT_EQ(samples[i]->settings().width->value, 1);
ASSERT_TRUE(samples[i]->settings().height);
ASSERT_EQ(samples[i]->settings().height->type, TextUnitType::kPercent);
ASSERT_EQ(samples[i]->settings().height->value, 1);
ASSERT_FALSE(samples[i]->body().image.empty());
}
// Allow in either order.
if (samples[0]->settings().position->value == 12) {
EXPECT_EQ(samples[0]->settings().line->value, 14);
EXPECT_EQ(samples[1]->settings().position->value, 16);
EXPECT_EQ(samples[1]->settings().line->value, 19);
} else {
EXPECT_EQ(samples[0]->settings().position->value, 16);
EXPECT_EQ(samples[0]->settings().line->value, 19);
EXPECT_EQ(samples[1]->settings().position->value, 12);
EXPECT_EQ(samples[1]->settings().line->value, 14);
}
}
TEST(SubtitleComposerTest, EnsuresRegionsFit) {
SubtitleComposer composer;
composer.SetDisplaySize(0xfff0, 0xfff0);
ASSERT_FALSE(composer.SetRegionPosition(1, 0xffff, 0xffff));
ASSERT_TRUE(composer.SetRegionPosition(1, 0xff00, 0xff00));
ASSERT_FALSE(composer.SetRegionInfo(1, 0, 0xf1, 0));
ASSERT_FALSE(composer.SetRegionInfo(1, 0, 0, 0xf1));
ASSERT_FALSE(composer.SetRegionInfo(1, 0, 0xff1, 0));
ASSERT_FALSE(composer.SetRegionInfo(1, 0, 0, 0xff1));
ASSERT_TRUE(composer.SetRegionInfo(1, 0, 25, 25));
ASSERT_FALSE(composer.SetObjectInfo(1, 1, 0xf1, 0, kNoBgColor));
ASSERT_FALSE(composer.SetObjectInfo(1, 1, 0, 0xf1, kNoBgColor));
ASSERT_FALSE(composer.SetObjectInfo(1, 1, 0xff1, 0, kNoBgColor));
ASSERT_FALSE(composer.SetObjectInfo(1, 1, 0, 0xff1, kNoBgColor));
ASSERT_TRUE(composer.SetObjectInfo(1, 1, 20, 20, kNoBgColor));
}
TEST(SubtitleComposerTest, MustInitRegionFirst) {
SubtitleComposer composer;
EXPECT_FALSE(composer.SetObjectInfo(0, 0, 0, 0, kNoBgColor));
}
TEST(SubtitleComposerTest, ReturnsConsistentColorSpace) {
const uint8_t kColorSpaceId = 2;
const uint16_t kObjectId = 5;
const uint16_t kRegionId = 1;
// Initially created in GetColorSpace.
{
SubtitleComposer composer;
auto* color_space = composer.GetColorSpace(kColorSpaceId);
ASSERT_TRUE(composer.SetRegionInfo(kRegionId, kColorSpaceId, 1, 1));
ASSERT_TRUE(composer.SetObjectInfo(kObjectId, kRegionId, 0, 0, kNoBgColor));
ASSERT_EQ(composer.GetColorSpace(kColorSpaceId), color_space);
ASSERT_EQ(composer.GetColorSpaceForObject(kObjectId), color_space);
}
// Initially created in SetRegionInfo.
{
SubtitleComposer composer;
ASSERT_TRUE(composer.SetRegionInfo(kRegionId, kColorSpaceId, 1, 1));
ASSERT_TRUE(composer.SetObjectInfo(kObjectId, kRegionId, 0, 0, kNoBgColor));
auto* color_space = composer.GetColorSpace(kColorSpaceId);
ASSERT_EQ(composer.GetColorSpace(kColorSpaceId), color_space);
ASSERT_EQ(composer.GetColorSpaceForObject(kObjectId), color_space);
}
}
TEST(SubtitleComposerTest, ClearObjects) {
const uint8_t kColorSpaceId = 2;
const uint16_t kObjectId1 = 5;
const uint16_t kObjectId2 = 6;
const uint16_t kRegionId = 1;
SubtitleComposer composer;
ASSERT_TRUE(composer.SetRegionInfo(kRegionId, kColorSpaceId, 10, 10));
ASSERT_TRUE(composer.SetObjectInfo(kObjectId1, kRegionId, 0, 0, kNoBgColor));
CreateDefaultImage(&composer, kObjectId1);
std::vector<std::shared_ptr<TextSample>> samples;
ASSERT_TRUE(composer.GetSamples(0, 1, &samples));
EXPECT_EQ(samples.size(), 1u);
composer.ClearObjects();
samples.clear();
// Should clear regions too.
ASSERT_FALSE(composer.SetObjectInfo(kObjectId2, kRegionId, 3, 3, kNoBgColor));
ASSERT_TRUE(composer.SetRegionInfo(kRegionId, kColorSpaceId, 10, 10));
ASSERT_TRUE(composer.SetObjectInfo(kObjectId2, kRegionId, 3, 3, kNoBgColor));
CreateDefaultImage(&composer, kObjectId2);
ASSERT_TRUE(composer.GetSamples(0, 1, &samples));
EXPECT_EQ(samples.size(), 1u);
}
TEST(SubtitleComposerTest, IgnoresEmptyImages) {
const uint8_t kColorSpaceId = 1;
const uint16_t kRegionId = 1;
const uint16_t kObjectId1 = 2;
const uint16_t kObjectId2 = 3;
const uint16_t kObjectId3 = 4;
const uint8_t kColorId = 10;
SubtitleComposer composer;
ASSERT_TRUE(composer.SetRegionInfo(kRegionId, kColorSpaceId, 10, 10));
ASSERT_TRUE(composer.SetObjectInfo(kObjectId1, kRegionId, 0, 0, kNoBgColor));
ASSERT_TRUE(composer.SetObjectInfo(kObjectId2, kRegionId, 5, 0, kNoBgColor));
ASSERT_TRUE(composer.SetObjectInfo(kObjectId3, kRegionId, 0, 5, kNoBgColor));
// Leave kObjectId1 with nothing.
CreateDefaultImage(&composer, kObjectId2);
{
// Add a transparent color.
auto* color_space = composer.GetColorSpace(kColorSpaceId);
color_space->SetColor(BitDepth::k8Bit, kColorId, RgbaColor{0, 0, 0, 0});
auto* image = composer.GetObjectImage(kObjectId3);
EXPECT_TRUE(image->AddPixel(BitDepth::k8Bit, kColorId, true));
image->NewRow(true);
}
std::vector<std::shared_ptr<TextSample>> samples;
ASSERT_TRUE(composer.GetSamples(0, 1, &samples));
EXPECT_EQ(samples.size(), 1u);
}
} // namespace media
} // namespace shaka

1
packager/third_party/libpng/libpng.gyp vendored
View File

@ -35,6 +35,7 @@
],
'direct_dependent_settings': {
'include_dirs': [
'.',
'src',
],
},