2022-08-26 15:44:59 +00:00
|
|
|
// Copyright 2022 Google LLC. All rights reserved.
|
feat: First phase of CMake build system implementation (#1072)
There are a lot of changes in this first phase, because there was a
lot of infrastructure required to get some meaningful amount of
porting done. Future PRs should be simpler.
<b>Summary of changes:</b><details>
- Remove old deps:
- boringssl (replaced with mbedtls, lighter, easier to build)
- gflags (replaced with absl::flags)
- Chromium build tools
- New deps to replace parts of Chromium base:
- abseil-cpp
- glog
- nlohmann::json (for tests only)
- Submodules, updates, and CMake build rules for third-party
libraries:
- curl
- gmock/gtest
- Ported internal libraries and their tests by removing Chromium deps
and adding CMake build rules:
- file (now using C++17 filesystem APIs)
- license_notice
- status
- version
- Test improvements
- Removed file tests that can never be re-enabled
- Re-enabled all other disabled file tests
- Debug JSON values when HTTP tests fail
- Fixed chunked-encoding issues in HTTP tests
- Updated and refactored Dockerfiles testing
- All docker files working, with OS versions updated to meet the
new tool requirements
- Local docker builds no longer write files to your working
directory as root
- Local docker builds can now be run in parallel without clobbering
each others' build outputs
- DEBUG=1 can drop you into an interactive shell when a docker
build fails
- Updated and heavily refactored workflows and Dockerfiles
- All docker files now tested in parallel on GitHub, speeding up CI
- All common workflow components broken out and using workflow_call
instead of custom actions
- Self-hosted runners now optional, to make testing easier on forks
- CMake porting works-in-process can now be fully tested on GitHub
- Building ported libraries and passing ported tests on all three
platforms!
- CI hacks for macOS removed, now testing on macos-latest!
- Python2 no longer required! (Only Python3)
- Using strict build flags, treating all warnings as errors.
</details>
<b>Required to build:</b>
- CMake >= 3.16
- Python 3
- A compiler supporting C++ >= 17
- g++ >= 9 if using GCC (Clang also fine)
- MSVC for Windows
<b>Still needs work:</b><details>
- Moving other dependencies into submodules (if we keep them):
- apple_apsl
- icu
- libevent
- libpng
- libwebm
- libxml
- modp_b64
- protobuf
- zlib
- Port remaining internal libraries:
- app
- hls
- media/base
- media/chunking
- media/codecs
- media/crypto
- media/demuxer
- media/event
- media/formats/dvb
- media/formats/mp2t
- media/formats/mp4
- media/formats/packed_audio
- media/formats/ttml
- media/formats/webm
- media/formats/webvtt
- media/formats/wvm
- media/origin
- media/public
- media/replicator
- media/trick_play
- mpd
- Port main application
- Add logging flags in absl and connect them to glog (which expects
gflags)
- Port pssh-box.py
- Port main test targets (packager_test.py and packager_app.py)
- Updating all requirement and build documentation
- Remove any remaining refs to gclient, depot_tools, ninja
- Update and complete release workflows using release-please
</details>
Issue #346 (Switch to abseil)
Issue #1047 (New build system)
2022-08-16 18:34:51 +00:00
|
|
|
//
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file or at
|
|
|
|
// https://developers.google.com/open-source/licenses/bsd
|
|
|
|
|
2023-10-10 23:51:11 +00:00
|
|
|
#include <packager/file/thread_pool.h>
|
feat: First phase of CMake build system implementation (#1072)
There are a lot of changes in this first phase, because there was a
lot of infrastructure required to get some meaningful amount of
porting done. Future PRs should be simpler.
<b>Summary of changes:</b><details>
- Remove old deps:
- boringssl (replaced with mbedtls, lighter, easier to build)
- gflags (replaced with absl::flags)
- Chromium build tools
- New deps to replace parts of Chromium base:
- abseil-cpp
- glog
- nlohmann::json (for tests only)
- Submodules, updates, and CMake build rules for third-party
libraries:
- curl
- gmock/gtest
- Ported internal libraries and their tests by removing Chromium deps
and adding CMake build rules:
- file (now using C++17 filesystem APIs)
- license_notice
- status
- version
- Test improvements
- Removed file tests that can never be re-enabled
- Re-enabled all other disabled file tests
- Debug JSON values when HTTP tests fail
- Fixed chunked-encoding issues in HTTP tests
- Updated and refactored Dockerfiles testing
- All docker files working, with OS versions updated to meet the
new tool requirements
- Local docker builds no longer write files to your working
directory as root
- Local docker builds can now be run in parallel without clobbering
each others' build outputs
- DEBUG=1 can drop you into an interactive shell when a docker
build fails
- Updated and heavily refactored workflows and Dockerfiles
- All docker files now tested in parallel on GitHub, speeding up CI
- All common workflow components broken out and using workflow_call
instead of custom actions
- Self-hosted runners now optional, to make testing easier on forks
- CMake porting works-in-process can now be fully tested on GitHub
- Building ported libraries and passing ported tests on all three
platforms!
- CI hacks for macOS removed, now testing on macos-latest!
- Python2 no longer required! (Only Python3)
- Using strict build flags, treating all warnings as errors.
</details>
<b>Required to build:</b>
- CMake >= 3.16
- Python 3
- A compiler supporting C++ >= 17
- g++ >= 9 if using GCC (Clang also fine)
- MSVC for Windows
<b>Still needs work:</b><details>
- Moving other dependencies into submodules (if we keep them):
- apple_apsl
- icu
- libevent
- libpng
- libwebm
- libxml
- modp_b64
- protobuf
- zlib
- Port remaining internal libraries:
- app
- hls
- media/base
- media/chunking
- media/codecs
- media/crypto
- media/demuxer
- media/event
- media/formats/dvb
- media/formats/mp2t
- media/formats/mp4
- media/formats/packed_audio
- media/formats/ttml
- media/formats/webm
- media/formats/webvtt
- media/formats/wvm
- media/origin
- media/public
- media/replicator
- media/trick_play
- mpd
- Port main application
- Add logging flags in absl and connect them to glog (which expects
gflags)
- Port pssh-box.py
- Port main test targets (packager_test.py and packager_app.py)
- Updating all requirement and build documentation
- Remove any remaining refs to gclient, depot_tools, ninja
- Update and complete release workflows using release-please
</details>
Issue #346 (Switch to abseil)
Issue #1047 (New build system)
2022-08-16 18:34:51 +00:00
|
|
|
|
|
|
|
#include <thread>
|
|
|
|
|
2023-10-13 19:42:47 +00:00
|
|
|
#include <absl/log/check.h>
|
|
|
|
#include <absl/log/log.h>
|
2023-10-09 23:21:41 +00:00
|
|
|
#include <absl/time/time.h>
|
feat: First phase of CMake build system implementation (#1072)
There are a lot of changes in this first phase, because there was a
lot of infrastructure required to get some meaningful amount of
porting done. Future PRs should be simpler.
<b>Summary of changes:</b><details>
- Remove old deps:
- boringssl (replaced with mbedtls, lighter, easier to build)
- gflags (replaced with absl::flags)
- Chromium build tools
- New deps to replace parts of Chromium base:
- abseil-cpp
- glog
- nlohmann::json (for tests only)
- Submodules, updates, and CMake build rules for third-party
libraries:
- curl
- gmock/gtest
- Ported internal libraries and their tests by removing Chromium deps
and adding CMake build rules:
- file (now using C++17 filesystem APIs)
- license_notice
- status
- version
- Test improvements
- Removed file tests that can never be re-enabled
- Re-enabled all other disabled file tests
- Debug JSON values when HTTP tests fail
- Fixed chunked-encoding issues in HTTP tests
- Updated and refactored Dockerfiles testing
- All docker files working, with OS versions updated to meet the
new tool requirements
- Local docker builds no longer write files to your working
directory as root
- Local docker builds can now be run in parallel without clobbering
each others' build outputs
- DEBUG=1 can drop you into an interactive shell when a docker
build fails
- Updated and heavily refactored workflows and Dockerfiles
- All docker files now tested in parallel on GitHub, speeding up CI
- All common workflow components broken out and using workflow_call
instead of custom actions
- Self-hosted runners now optional, to make testing easier on forks
- CMake porting works-in-process can now be fully tested on GitHub
- Building ported libraries and passing ported tests on all three
platforms!
- CI hacks for macOS removed, now testing on macos-latest!
- Python2 no longer required! (Only Python3)
- Using strict build flags, treating all warnings as errors.
</details>
<b>Required to build:</b>
- CMake >= 3.16
- Python 3
- A compiler supporting C++ >= 17
- g++ >= 9 if using GCC (Clang also fine)
- MSVC for Windows
<b>Still needs work:</b><details>
- Moving other dependencies into submodules (if we keep them):
- apple_apsl
- icu
- libevent
- libpng
- libwebm
- libxml
- modp_b64
- protobuf
- zlib
- Port remaining internal libraries:
- app
- hls
- media/base
- media/chunking
- media/codecs
- media/crypto
- media/demuxer
- media/event
- media/formats/dvb
- media/formats/mp2t
- media/formats/mp4
- media/formats/packed_audio
- media/formats/ttml
- media/formats/webm
- media/formats/webvtt
- media/formats/wvm
- media/origin
- media/public
- media/replicator
- media/trick_play
- mpd
- Port main application
- Add logging flags in absl and connect them to glog (which expects
gflags)
- Port pssh-box.py
- Port main test targets (packager_test.py and packager_app.py)
- Updating all requirement and build documentation
- Remove any remaining refs to gclient, depot_tools, ninja
- Update and complete release workflows using release-please
</details>
Issue #346 (Switch to abseil)
Issue #1047 (New build system)
2022-08-16 18:34:51 +00:00
|
|
|
|
|
|
|
namespace shaka {
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
|
|
|
const absl::Duration kMaxThreadIdleTime = absl::Minutes(10);
|
|
|
|
|
|
|
|
} // namespace
|
|
|
|
|
|
|
|
// static
|
|
|
|
ThreadPool ThreadPool::instance;
|
|
|
|
|
|
|
|
ThreadPool::ThreadPool() : num_idle_threads_(0), terminated_(false) {}
|
|
|
|
|
|
|
|
ThreadPool::~ThreadPool() {
|
|
|
|
Terminate();
|
|
|
|
}
|
|
|
|
|
|
|
|
void ThreadPool::PostTask(const std::function<void()>& task) {
|
|
|
|
absl::MutexLock lock(&mutex_);
|
|
|
|
|
|
|
|
DCHECK(!terminated_) << "Should not call PostTask after Terminate!";
|
|
|
|
|
|
|
|
if (terminated_) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// An empty task is used internally to signal the thread to terminate. This
|
|
|
|
// should never be sent on input.
|
|
|
|
if (!task) {
|
|
|
|
DLOG(ERROR) << "Should not post an empty task!";
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
tasks_.push(std::move(task));
|
|
|
|
|
|
|
|
if (num_idle_threads_ >= tasks_.size()) {
|
|
|
|
// We have enough threads available.
|
|
|
|
tasks_available_.SignalAll();
|
|
|
|
} else {
|
|
|
|
// We need to start an additional thread.
|
|
|
|
std::thread thread(std::bind(&ThreadPool::ThreadMain, this));
|
|
|
|
thread.detach();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void ThreadPool::Terminate() {
|
|
|
|
{
|
|
|
|
absl::MutexLock lock(&mutex_);
|
|
|
|
terminated_ = true;
|
|
|
|
while (!tasks_.empty()) {
|
|
|
|
tasks_.pop();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
tasks_available_.SignalAll();
|
|
|
|
}
|
|
|
|
|
|
|
|
ThreadPool::Task ThreadPool::WaitForTask() {
|
|
|
|
absl::MutexLock lock(&mutex_);
|
|
|
|
if (terminated_) {
|
|
|
|
// The pool is terminated. Terminate this thread.
|
|
|
|
return Task();
|
|
|
|
}
|
|
|
|
|
|
|
|
if (tasks_.empty()) {
|
|
|
|
num_idle_threads_++;
|
|
|
|
// Wait for a task, up to the maximum idle time.
|
|
|
|
tasks_available_.WaitWithTimeout(&mutex_, kMaxThreadIdleTime);
|
|
|
|
num_idle_threads_--;
|
|
|
|
|
|
|
|
if (tasks_.empty()) {
|
|
|
|
// No work before the timeout. Terminate this thread.
|
|
|
|
return Task();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get the next task from the queue.
|
|
|
|
Task task = tasks_.front();
|
|
|
|
tasks_.pop();
|
|
|
|
return task;
|
|
|
|
}
|
|
|
|
|
|
|
|
void ThreadPool::ThreadMain() {
|
|
|
|
while (true) {
|
|
|
|
auto task = WaitForTask();
|
|
|
|
if (!task) {
|
|
|
|
// An empty task signals the thread to terminate.
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Run the task, then loop to wait for another.
|
|
|
|
task();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace shaka
|