From 89d407f9ae44e9b0911becf433c67cd25cf180c5 Mon Sep 17 00:00:00 2001 From: Jacob Trimble Date: Mon, 5 Oct 2020 15:31:31 -0700 Subject: [PATCH] Add subtitle composition to DVB-sub parser. Issue #832 Change-Id: Iababe884619e1e48f1abe0806e8b863c95a3c1ef --- packager/media/formats/dvb/dvb.gyp | 3 + .../media/formats/dvb/subtitle_composer.cc | 254 ++++++++++++++++++ .../media/formats/dvb/subtitle_composer.h | 79 ++++++ .../formats/dvb/subtitle_composer_unittest.cc | 177 ++++++++++++ packager/third_party/libpng/libpng.gyp | 1 + 5 files changed, 514 insertions(+) create mode 100644 packager/media/formats/dvb/subtitle_composer.cc create mode 100644 packager/media/formats/dvb/subtitle_composer.h create mode 100644 packager/media/formats/dvb/subtitle_composer_unittest.cc diff --git a/packager/media/formats/dvb/dvb.gyp b/packager/media/formats/dvb/dvb.gyp index fd85d91c73..55519ef89b 100644 --- a/packager/media/formats/dvb/dvb.gyp +++ b/packager/media/formats/dvb/dvb.gyp @@ -15,6 +15,8 @@ 'sources': [ 'dvb_image.cc', 'dvb_image.h', + 'subtitle_composer.cc', + 'subtitle_composer.h', ], 'dependencies': [ '../../base/media_base.gyp:media_base', @@ -26,6 +28,7 @@ 'type': '<(gtest_target_type)', 'sources': [ 'dvb_image_unittest.cc', + 'subtitle_composer_unittest.cc', ], 'dependencies': [ '../../../testing/gtest.gyp:gtest', diff --git a/packager/media/formats/dvb/subtitle_composer.cc b/packager/media/formats/dvb/subtitle_composer.cc new file mode 100644 index 0000000000..11d7a17a55 --- /dev/null +++ b/packager/media/formats/dvb/subtitle_composer.cc @@ -0,0 +1,254 @@ +// Copyright 2020 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "packager/media/formats/dvb/subtitle_composer.h" + +#include +#include + +#include "packager/base/logging.h" + +namespace shaka { +namespace media { + +namespace { + +const uint16_t kDefaultWidth = 720; +const uint16_t kDefaultHeight = 576; +const RgbaColor kTransparent{0, 0, 0, 0}; + +struct PngFreeHelper { + PngFreeHelper(png_structp* png, png_infop* info) : png(png), info(info) {} + ~PngFreeHelper() { png_destroy_write_struct(png, info); } + + png_structp* png; + png_infop* info; +}; + +void PngWriteData(png_structp png, png_bytep data, png_size_t length) { + auto* output = reinterpret_cast*>(png_get_io_ptr(png)); + output->insert(output->end(), data, data + length); +} + +void PngFlushData(png_structp png) {} + +bool IsTransparent(const RgbaColor* colors, uint16_t width, uint16_t height) { + for (size_t i = 0; i < static_cast(width) * height; i++) { + if (colors[i].a != 0) + return false; + } + return true; +} + +bool GetImageData(const DvbImageBuilder* image, + std::vector* data, + uint16_t* width, + uint16_t* height) { + // CAREFUL in this method since this uses long-jumps. A long-jump causes the + // execution to jump to another point *without executing returns*. This + // causes C++ objects to not get destroyed. This also causes the same code to + // be executed twice, so C++ objects can be initialized twice. + // + // So long as we don't create C++ objects after the long-jump point, + // everything should work fine. If we early-return after the long-jump, the + // destructors will still be called; if we long-jump, we won't call the + // constructors since we're past that point. + auto png = + png_create_write_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr); + auto info = png_create_info_struct(png); + PngFreeHelper helper(&png, &info); + if (!png || !info) { + LOG(ERROR) << "Error creating libpng struct"; + return false; + } + if (setjmp(png_jmpbuf(png))) { + // If any png_* functions fail, the code will jump back to here. + LOG(ERROR) << "Error writing PNG image"; + return false; + } + png_set_write_fn(png, data, &PngWriteData, &PngFlushData); + + const RgbaColor* pixels; + if (!image->GetPixels(&pixels, width, height)) + return false; + if (IsTransparent(pixels, *width, *height)) + return true; // Skip empty/transparent images. + png_set_IHDR(png, info, *width, *height, 8, PNG_COLOR_TYPE_RGBA, + PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_BASE, + PNG_FILTER_TYPE_BASE); + png_write_info(png, info); + + const uint8_t* in_data = reinterpret_cast(pixels); + for (size_t y = 0; y < *height; y++) { + size_t offset = image->max_width() * y * sizeof(RgbaColor); + png_write_row(png, in_data + offset); + } + png_write_end(png, nullptr); + + return true; +} + +} // namespace + +SubtitleComposer::SubtitleComposer() + : display_width_(kDefaultWidth), display_height_(kDefaultHeight) {} + +SubtitleComposer::~SubtitleComposer() {} + +void SubtitleComposer::SetDisplaySize(uint16_t width, uint16_t height) { + display_width_ = width; + display_height_ = height; +} + +bool SubtitleComposer::SetRegionInfo(uint8_t region_id, + uint8_t color_space_id, + uint16_t width, + uint16_t height) { + auto* region = ®ions_[region_id]; + if (region->x + width > display_width_ || + region->y + height > display_height_) { + LOG(ERROR) << "DVB-sub region won't fit within display"; + return false; + } + if (width == 0 || height == 0) { + LOG(ERROR) << "DVB-sub width/height cannot be 0"; + return false; + } + + region->width = width; + region->height = height; + region->color_space = &color_spaces_[color_space_id]; + return true; +} + +bool SubtitleComposer::SetRegionPosition(uint8_t region_id, + uint16_t x, + uint16_t y) { + auto* region = ®ions_[region_id]; + if (x + region->width > display_width_ || + y + region->height > display_height_) { + LOG(ERROR) << "DVB-sub region won't fit within display"; + return false; + } + + region->x = x; + region->y = y; + return true; +} + +bool SubtitleComposer::SetObjectInfo(uint16_t object_id, + uint8_t region_id, + uint16_t x, + uint16_t y, + int default_color_code) { + auto region = regions_.find(region_id); + if (region == regions_.end()) { + LOG(ERROR) << "Unknown DVB-sub region: " << (int)region_id + << ", in object: " << object_id; + return false; + } + if (x >= region->second.width || y >= region->second.height) { + LOG(ERROR) << "DVB-sub object is outside region: " << object_id; + return false; + } + + auto* object = &objects_[object_id]; + object->region = ®ion->second; + object->default_color_code = default_color_code; + object->x = x; + object->y = y; + return true; +} + +DvbImageColorSpace* SubtitleComposer::GetColorSpace(uint8_t color_space_id) { + return &color_spaces_[color_space_id]; +} + +DvbImageColorSpace* SubtitleComposer::GetColorSpaceForObject( + uint16_t object_id) { + auto info = objects_.find(object_id); + if (info == objects_.end()) { + LOG(ERROR) << "Unknown DVB-sub object: " << object_id; + return nullptr; + } + return info->second.region->color_space; +} + +DvbImageBuilder* SubtitleComposer::GetObjectImage(uint16_t object_id) { + auto it = images_.find(object_id); + if (it == images_.end()) { + auto info = objects_.find(object_id); + if (info == objects_.end()) { + LOG(ERROR) << "Unknown DVB-sub object: " << object_id; + return nullptr; + } + + auto color = info->second.default_color_code < 0 + ? kTransparent + : info->second.region->color_space->GetColor( + BitDepth::k8Bit, info->second.default_color_code); + it = images_ + .emplace(std::piecewise_construct, std::make_tuple(object_id), + std::make_tuple( + info->second.region->color_space, color, + info->second.region->width - info->second.region->x, + info->second.region->height - info->second.region->y)) + .first; + } + return &it->second; +} + +bool SubtitleComposer::GetSamples( + int64_t start, + int64_t end, + std::vector>* samples) const { + for (const auto& pair : objects_) { + auto it = images_.find(pair.first); + if (it == images_.end()) { + LOG(WARNING) << "DVB-sub object " << pair.first + << " doesn't include object data"; + continue; + } + + uint16_t width, height; + std::vector image_data; + if (!GetImageData(&it->second, &image_data, &width, &height)) + return false; + if (image_data.empty()) { + VLOG(1) << "Skipping transparent object"; + continue; + } + TextFragment body({}, image_data); + DCHECK_LE(width, display_width_); + DCHECK_LE(height, display_height_); + + TextSettings settings; + settings.position.emplace( + (pair.second.x + pair.second.region->x) * 100.0f / display_width_, + TextUnitType::kPercent); + settings.line.emplace( + (pair.second.y + pair.second.region->y) * 100.0f / display_height_, + TextUnitType::kPercent); + settings.width.emplace(width * 100.0f / display_width_, + TextUnitType::kPercent); + settings.height.emplace(height * 100.0f / display_height_, + TextUnitType::kPercent); + + samples->emplace_back( + std::make_shared("", start, end, settings, body)); + } + + return true; +} + +void SubtitleComposer::ClearObjects() { + regions_.clear(); + objects_.clear(); + images_.clear(); +} + +} // namespace media +} // namespace shaka diff --git a/packager/media/formats/dvb/subtitle_composer.h b/packager/media/formats/dvb/subtitle_composer.h new file mode 100644 index 0000000000..743bc602d5 --- /dev/null +++ b/packager/media/formats/dvb/subtitle_composer.h @@ -0,0 +1,79 @@ +// Copyright 2020 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef PACKAGER_MEDIA_DVB_SUBTITLE_COMPOSER_H_ +#define PACKAGER_MEDIA_DVB_SUBTITLE_COMPOSER_H_ + +#include +#include +#include + +#include "packager/base/macros.h" +#include "packager/media/base/text_sample.h" +#include "packager/media/formats/dvb/dvb_image.h" + +namespace shaka { +namespace media { + +/// Holds pixel/caption data for a single DVB-sub page. This composes +/// multiple objects and creates TextSample objects from it. +class SubtitleComposer { + public: + SubtitleComposer(); + ~SubtitleComposer(); + + DISALLOW_COPY_AND_ASSIGN(SubtitleComposer); + + void SetDisplaySize(uint16_t width, uint16_t height); + bool SetRegionPosition(uint8_t region_id, uint16_t x, uint16_t y); + bool SetRegionInfo(uint8_t region_id, + uint8_t color_space_id, + uint16_t width, + uint16_t height); + bool SetObjectInfo(uint16_t object_id, + uint8_t region_id, + uint16_t x, + uint16_t y, + int default_color_code); + + DvbImageColorSpace* GetColorSpace(uint8_t color_space_id); + DvbImageColorSpace* GetColorSpaceForObject(uint16_t object_id); + DvbImageBuilder* GetObjectImage(uint16_t object_id); + + bool GetSamples(int64_t start, + int64_t end, + std::vector>* samples) const; + void ClearObjects(); + + private: + struct RegionInfo { + DvbImageColorSpace* color_space = nullptr; + uint16_t x = 0; + uint16_t y = 0; + uint16_t width = 0; + uint16_t height = 0; + }; + + struct ObjectInfo { + RegionInfo* region = nullptr; + int default_color_code = -1; + uint16_t x = 0; + uint16_t y = 0; + }; + + // Maps of IDs to their respective object. + std::unordered_map regions_; + std::unordered_map color_spaces_; + std::unordered_map objects_; + std::unordered_map images_; // Uses object_id. + uint16_t display_width_; + uint16_t display_height_; +}; + +} // namespace media +} // namespace shaka + +#endif // PACKAGER_MEDIA_DVB_SUBTITLE_COMPOSER_H_ diff --git a/packager/media/formats/dvb/subtitle_composer_unittest.cc b/packager/media/formats/dvb/subtitle_composer_unittest.cc new file mode 100644 index 0000000000..ace8ec8217 --- /dev/null +++ b/packager/media/formats/dvb/subtitle_composer_unittest.cc @@ -0,0 +1,177 @@ +// Copyright 2020 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "packager/media/formats/dvb/subtitle_composer.h" + +#include +#include + +namespace shaka { +namespace media { + +namespace { + +constexpr const int kNoBgColor = -1; + +void CreateDefaultImage(SubtitleComposer* composer, uint16_t object_id) { + auto* image = composer->GetObjectImage(object_id); + EXPECT_TRUE(image->AddPixel(BitDepth::k8Bit, 1, true)); + image->NewRow(true); +} + +} // namespace + +TEST(SubtitleComposerTest, PositionsSamples) { + const uint8_t kRegionId = 3, kColorSpaceId = 9; + const uint16_t kObjectId1 = 5, kObjectId2 = 11; + SubtitleComposer composer; + composer.SetDisplaySize(100, 100); + ASSERT_TRUE(composer.SetRegionPosition(kRegionId, 4, 5)); + ASSERT_TRUE(composer.SetRegionInfo(kRegionId, kColorSpaceId, 96, 95)); + ASSERT_TRUE(composer.SetObjectInfo(kObjectId1, kRegionId, 8, 9, kNoBgColor)); + ASSERT_TRUE( + composer.SetObjectInfo(kObjectId2, kRegionId, 12, 14, kNoBgColor)); + CreateDefaultImage(&composer, kObjectId1); + CreateDefaultImage(&composer, kObjectId2); + + std::vector> samples; + ASSERT_TRUE(composer.GetSamples(0, 1, &samples)); + ASSERT_EQ(samples.size(), 2u); + for (size_t i = 0; i < samples.size(); i++) { + ASSERT_TRUE(samples[i]->settings().line); + ASSERT_EQ(samples[i]->settings().line->type, TextUnitType::kPercent); + ASSERT_TRUE(samples[i]->settings().position); + ASSERT_EQ(samples[i]->settings().position->type, TextUnitType::kPercent); + ASSERT_TRUE(samples[i]->settings().width); + ASSERT_EQ(samples[i]->settings().width->type, TextUnitType::kPercent); + ASSERT_EQ(samples[i]->settings().width->value, 1); + ASSERT_TRUE(samples[i]->settings().height); + ASSERT_EQ(samples[i]->settings().height->type, TextUnitType::kPercent); + ASSERT_EQ(samples[i]->settings().height->value, 1); + ASSERT_FALSE(samples[i]->body().image.empty()); + } + // Allow in either order. + if (samples[0]->settings().position->value == 12) { + EXPECT_EQ(samples[0]->settings().line->value, 14); + EXPECT_EQ(samples[1]->settings().position->value, 16); + EXPECT_EQ(samples[1]->settings().line->value, 19); + } else { + EXPECT_EQ(samples[0]->settings().position->value, 16); + EXPECT_EQ(samples[0]->settings().line->value, 19); + EXPECT_EQ(samples[1]->settings().position->value, 12); + EXPECT_EQ(samples[1]->settings().line->value, 14); + } +} + +TEST(SubtitleComposerTest, EnsuresRegionsFit) { + SubtitleComposer composer; + composer.SetDisplaySize(0xfff0, 0xfff0); + ASSERT_FALSE(composer.SetRegionPosition(1, 0xffff, 0xffff)); + ASSERT_TRUE(composer.SetRegionPosition(1, 0xff00, 0xff00)); + ASSERT_FALSE(composer.SetRegionInfo(1, 0, 0xf1, 0)); + ASSERT_FALSE(composer.SetRegionInfo(1, 0, 0, 0xf1)); + ASSERT_FALSE(composer.SetRegionInfo(1, 0, 0xff1, 0)); + ASSERT_FALSE(composer.SetRegionInfo(1, 0, 0, 0xff1)); + ASSERT_TRUE(composer.SetRegionInfo(1, 0, 25, 25)); + + ASSERT_FALSE(composer.SetObjectInfo(1, 1, 0xf1, 0, kNoBgColor)); + ASSERT_FALSE(composer.SetObjectInfo(1, 1, 0, 0xf1, kNoBgColor)); + ASSERT_FALSE(composer.SetObjectInfo(1, 1, 0xff1, 0, kNoBgColor)); + ASSERT_FALSE(composer.SetObjectInfo(1, 1, 0, 0xff1, kNoBgColor)); + ASSERT_TRUE(composer.SetObjectInfo(1, 1, 20, 20, kNoBgColor)); +} + +TEST(SubtitleComposerTest, MustInitRegionFirst) { + SubtitleComposer composer; + EXPECT_FALSE(composer.SetObjectInfo(0, 0, 0, 0, kNoBgColor)); +} + +TEST(SubtitleComposerTest, ReturnsConsistentColorSpace) { + const uint8_t kColorSpaceId = 2; + const uint16_t kObjectId = 5; + const uint16_t kRegionId = 1; + + // Initially created in GetColorSpace. + { + SubtitleComposer composer; + auto* color_space = composer.GetColorSpace(kColorSpaceId); + ASSERT_TRUE(composer.SetRegionInfo(kRegionId, kColorSpaceId, 1, 1)); + ASSERT_TRUE(composer.SetObjectInfo(kObjectId, kRegionId, 0, 0, kNoBgColor)); + ASSERT_EQ(composer.GetColorSpace(kColorSpaceId), color_space); + ASSERT_EQ(composer.GetColorSpaceForObject(kObjectId), color_space); + } + + // Initially created in SetRegionInfo. + { + SubtitleComposer composer; + ASSERT_TRUE(composer.SetRegionInfo(kRegionId, kColorSpaceId, 1, 1)); + ASSERT_TRUE(composer.SetObjectInfo(kObjectId, kRegionId, 0, 0, kNoBgColor)); + auto* color_space = composer.GetColorSpace(kColorSpaceId); + ASSERT_EQ(composer.GetColorSpace(kColorSpaceId), color_space); + ASSERT_EQ(composer.GetColorSpaceForObject(kObjectId), color_space); + } +} + +TEST(SubtitleComposerTest, ClearObjects) { + const uint8_t kColorSpaceId = 2; + const uint16_t kObjectId1 = 5; + const uint16_t kObjectId2 = 6; + const uint16_t kRegionId = 1; + + SubtitleComposer composer; + ASSERT_TRUE(composer.SetRegionInfo(kRegionId, kColorSpaceId, 10, 10)); + ASSERT_TRUE(composer.SetObjectInfo(kObjectId1, kRegionId, 0, 0, kNoBgColor)); + CreateDefaultImage(&composer, kObjectId1); + + std::vector> samples; + ASSERT_TRUE(composer.GetSamples(0, 1, &samples)); + EXPECT_EQ(samples.size(), 1u); + + composer.ClearObjects(); + samples.clear(); + + // Should clear regions too. + ASSERT_FALSE(composer.SetObjectInfo(kObjectId2, kRegionId, 3, 3, kNoBgColor)); + ASSERT_TRUE(composer.SetRegionInfo(kRegionId, kColorSpaceId, 10, 10)); + ASSERT_TRUE(composer.SetObjectInfo(kObjectId2, kRegionId, 3, 3, kNoBgColor)); + CreateDefaultImage(&composer, kObjectId2); + + ASSERT_TRUE(composer.GetSamples(0, 1, &samples)); + EXPECT_EQ(samples.size(), 1u); +} + +TEST(SubtitleComposerTest, IgnoresEmptyImages) { + const uint8_t kColorSpaceId = 1; + const uint16_t kRegionId = 1; + const uint16_t kObjectId1 = 2; + const uint16_t kObjectId2 = 3; + const uint16_t kObjectId3 = 4; + const uint8_t kColorId = 10; + + SubtitleComposer composer; + ASSERT_TRUE(composer.SetRegionInfo(kRegionId, kColorSpaceId, 10, 10)); + ASSERT_TRUE(composer.SetObjectInfo(kObjectId1, kRegionId, 0, 0, kNoBgColor)); + ASSERT_TRUE(composer.SetObjectInfo(kObjectId2, kRegionId, 5, 0, kNoBgColor)); + ASSERT_TRUE(composer.SetObjectInfo(kObjectId3, kRegionId, 0, 5, kNoBgColor)); + // Leave kObjectId1 with nothing. + CreateDefaultImage(&composer, kObjectId2); + { + // Add a transparent color. + auto* color_space = composer.GetColorSpace(kColorSpaceId); + color_space->SetColor(BitDepth::k8Bit, kColorId, RgbaColor{0, 0, 0, 0}); + + auto* image = composer.GetObjectImage(kObjectId3); + EXPECT_TRUE(image->AddPixel(BitDepth::k8Bit, kColorId, true)); + image->NewRow(true); + } + + std::vector> samples; + ASSERT_TRUE(composer.GetSamples(0, 1, &samples)); + EXPECT_EQ(samples.size(), 1u); +} + +} // namespace media +} // namespace shaka diff --git a/packager/third_party/libpng/libpng.gyp b/packager/third_party/libpng/libpng.gyp index 708c941ca8..36571a515d 100644 --- a/packager/third_party/libpng/libpng.gyp +++ b/packager/third_party/libpng/libpng.gyp @@ -35,6 +35,7 @@ ], 'direct_dependent_settings': { 'include_dirs': [ + '.', 'src', ], },