2022-08-26 15:44:59 +00:00
|
|
|
// Copyright 2014 Google LLC. All rights reserved.
|
2014-02-14 23:21:05 +00:00
|
|
|
//
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file or at
|
|
|
|
// https://developers.google.com/open-source/licenses/bsd
|
2013-10-11 21:44:55 +00:00
|
|
|
|
|
|
|
#ifndef PACKAGER_FILE_FILE_H_
|
|
|
|
#define PACKAGER_FILE_FILE_H_
|
|
|
|
|
2014-09-30 23:52:58 +00:00
|
|
|
#include <stdint.h>
|
|
|
|
|
2013-11-12 20:37:58 +00:00
|
|
|
#include <string>
|
|
|
|
|
feat: First phase of CMake build system implementation (#1072)
There are a lot of changes in this first phase, because there was a
lot of infrastructure required to get some meaningful amount of
porting done. Future PRs should be simpler.
<b>Summary of changes:</b><details>
- Remove old deps:
- boringssl (replaced with mbedtls, lighter, easier to build)
- gflags (replaced with absl::flags)
- Chromium build tools
- New deps to replace parts of Chromium base:
- abseil-cpp
- glog
- nlohmann::json (for tests only)
- Submodules, updates, and CMake build rules for third-party
libraries:
- curl
- gmock/gtest
- Ported internal libraries and their tests by removing Chromium deps
and adding CMake build rules:
- file (now using C++17 filesystem APIs)
- license_notice
- status
- version
- Test improvements
- Removed file tests that can never be re-enabled
- Re-enabled all other disabled file tests
- Debug JSON values when HTTP tests fail
- Fixed chunked-encoding issues in HTTP tests
- Updated and refactored Dockerfiles testing
- All docker files working, with OS versions updated to meet the
new tool requirements
- Local docker builds no longer write files to your working
directory as root
- Local docker builds can now be run in parallel without clobbering
each others' build outputs
- DEBUG=1 can drop you into an interactive shell when a docker
build fails
- Updated and heavily refactored workflows and Dockerfiles
- All docker files now tested in parallel on GitHub, speeding up CI
- All common workflow components broken out and using workflow_call
instead of custom actions
- Self-hosted runners now optional, to make testing easier on forks
- CMake porting works-in-process can now be fully tested on GitHub
- Building ported libraries and passing ported tests on all three
platforms!
- CI hacks for macOS removed, now testing on macos-latest!
- Python2 no longer required! (Only Python3)
- Using strict build flags, treating all warnings as errors.
</details>
<b>Required to build:</b>
- CMake >= 3.16
- Python 3
- A compiler supporting C++ >= 17
- g++ >= 9 if using GCC (Clang also fine)
- MSVC for Windows
<b>Still needs work:</b><details>
- Moving other dependencies into submodules (if we keep them):
- apple_apsl
- icu
- libevent
- libpng
- libwebm
- libxml
- modp_b64
- protobuf
- zlib
- Port remaining internal libraries:
- app
- hls
- media/base
- media/chunking
- media/codecs
- media/crypto
- media/demuxer
- media/event
- media/formats/dvb
- media/formats/mp2t
- media/formats/mp4
- media/formats/packed_audio
- media/formats/ttml
- media/formats/webm
- media/formats/webvtt
- media/formats/wvm
- media/origin
- media/public
- media/replicator
- media/trick_play
- mpd
- Port main application
- Add logging flags in absl and connect them to glog (which expects
gflags)
- Port pssh-box.py
- Port main test targets (packager_test.py and packager_app.py)
- Updating all requirement and build documentation
- Remove any remaining refs to gclient, depot_tools, ninja
- Update and complete release workflows using release-please
</details>
Issue #346 (Switch to abseil)
Issue #1047 (New build system)
2022-08-16 18:34:51 +00:00
|
|
|
#include "packager/common.h"
|
2017-07-17 18:15:07 +00:00
|
|
|
#include "packager/file/public/buffer_callback_params.h"
|
feat: First phase of CMake build system implementation (#1072)
There are a lot of changes in this first phase, because there was a
lot of infrastructure required to get some meaningful amount of
porting done. Future PRs should be simpler.
<b>Summary of changes:</b><details>
- Remove old deps:
- boringssl (replaced with mbedtls, lighter, easier to build)
- gflags (replaced with absl::flags)
- Chromium build tools
- New deps to replace parts of Chromium base:
- abseil-cpp
- glog
- nlohmann::json (for tests only)
- Submodules, updates, and CMake build rules for third-party
libraries:
- curl
- gmock/gtest
- Ported internal libraries and their tests by removing Chromium deps
and adding CMake build rules:
- file (now using C++17 filesystem APIs)
- license_notice
- status
- version
- Test improvements
- Removed file tests that can never be re-enabled
- Re-enabled all other disabled file tests
- Debug JSON values when HTTP tests fail
- Fixed chunked-encoding issues in HTTP tests
- Updated and refactored Dockerfiles testing
- All docker files working, with OS versions updated to meet the
new tool requirements
- Local docker builds no longer write files to your working
directory as root
- Local docker builds can now be run in parallel without clobbering
each others' build outputs
- DEBUG=1 can drop you into an interactive shell when a docker
build fails
- Updated and heavily refactored workflows and Dockerfiles
- All docker files now tested in parallel on GitHub, speeding up CI
- All common workflow components broken out and using workflow_call
instead of custom actions
- Self-hosted runners now optional, to make testing easier on forks
- CMake porting works-in-process can now be fully tested on GitHub
- Building ported libraries and passing ported tests on all three
platforms!
- CI hacks for macOS removed, now testing on macos-latest!
- Python2 no longer required! (Only Python3)
- Using strict build flags, treating all warnings as errors.
</details>
<b>Required to build:</b>
- CMake >= 3.16
- Python 3
- A compiler supporting C++ >= 17
- g++ >= 9 if using GCC (Clang also fine)
- MSVC for Windows
<b>Still needs work:</b><details>
- Moving other dependencies into submodules (if we keep them):
- apple_apsl
- icu
- libevent
- libpng
- libwebm
- libxml
- modp_b64
- protobuf
- zlib
- Port remaining internal libraries:
- app
- hls
- media/base
- media/chunking
- media/codecs
- media/crypto
- media/demuxer
- media/event
- media/formats/dvb
- media/formats/mp2t
- media/formats/mp4
- media/formats/packed_audio
- media/formats/ttml
- media/formats/webm
- media/formats/webvtt
- media/formats/wvm
- media/origin
- media/public
- media/replicator
- media/trick_play
- mpd
- Port main application
- Add logging flags in absl and connect them to glog (which expects
gflags)
- Port pssh-box.py
- Port main test targets (packager_test.py and packager_app.py)
- Updating all requirement and build documentation
- Remove any remaining refs to gclient, depot_tools, ninja
- Update and complete release workflows using release-please
</details>
Issue #346 (Switch to abseil)
Issue #1047 (New build system)
2022-08-16 18:34:51 +00:00
|
|
|
#include "packager/status/status.h"
|
2013-10-11 21:44:55 +00:00
|
|
|
|
2016-05-20 21:19:33 +00:00
|
|
|
namespace shaka {
|
2013-10-11 21:44:55 +00:00
|
|
|
|
2017-07-17 18:15:07 +00:00
|
|
|
extern const char* kCallbackFilePrefix;
|
2013-10-11 21:44:55 +00:00
|
|
|
extern const char* kLocalFilePrefix;
|
2016-04-06 06:32:44 +00:00
|
|
|
extern const char* kMemoryFilePrefix;
|
2017-07-17 18:15:07 +00:00
|
|
|
extern const char* kUdpFilePrefix;
|
2021-02-02 18:51:50 +00:00
|
|
|
extern const char* kHttpFilePrefix;
|
2015-10-28 17:23:08 +00:00
|
|
|
const int64_t kWholeFile = -1;
|
2013-10-11 21:44:55 +00:00
|
|
|
|
2014-01-24 00:26:00 +00:00
|
|
|
/// Define an abstract file interface.
|
2021-06-14 22:31:44 +00:00
|
|
|
class SHAKA_EXPORT File {
|
2013-10-11 21:44:55 +00:00
|
|
|
public:
|
2014-01-24 00:26:00 +00:00
|
|
|
/// Open the specified file.
|
|
|
|
/// This is a file factory method, it opens a proper file automatically
|
|
|
|
/// based on prefix, e.g. "file://" for LocalFile.
|
|
|
|
/// @param file_name contains the name of the file to be accessed.
|
|
|
|
/// @param mode contains file access mode. Implementation dependent.
|
|
|
|
/// @return A File pointer on success, false otherwise.
|
2014-01-15 22:29:56 +00:00
|
|
|
static File* Open(const char* file_name, const char* mode);
|
2013-10-11 21:44:55 +00:00
|
|
|
|
2015-05-20 17:14:36 +00:00
|
|
|
/// Open the specified file in direct-access mode (no buffering).
|
|
|
|
/// This is a file factory method, it opens a proper file automatically
|
|
|
|
/// based on prefix, e.g. "file://" for LocalFile.
|
|
|
|
/// @param file_name contains the name of the file to be accessed.
|
|
|
|
/// @param mode contains file access mode. Implementation dependent.
|
|
|
|
/// @return A File pointer on success, false otherwise.
|
|
|
|
static File* OpenWithNoBuffering(const char* file_name, const char* mode);
|
|
|
|
|
2015-03-11 19:18:17 +00:00
|
|
|
/// Delete the specified file.
|
|
|
|
/// @param file_name contains the path of the file to be deleted.
|
|
|
|
/// @return true if successful, false otherwise.
|
|
|
|
static bool Delete(const char* file_name);
|
|
|
|
|
2014-01-24 00:26:00 +00:00
|
|
|
/// Flush() and de-allocate resources associated with this file, and
|
|
|
|
/// delete this File object. THIS IS THE ONE TRUE WAY TO DEALLOCATE
|
|
|
|
/// THIS OBJECT.
|
|
|
|
/// @return true on success. For writable files, returning false MAY
|
|
|
|
/// INDICATE DATA LOSS.
|
2013-10-11 21:44:55 +00:00
|
|
|
virtual bool Close() = 0;
|
|
|
|
|
2014-01-24 00:26:00 +00:00
|
|
|
/// Read data and return it in buffer.
|
|
|
|
/// @param[out] buffer points to a block of memory with a size of at least
|
|
|
|
/// @a length bytes.
|
|
|
|
/// @param length indicates number of bytes to be read.
|
|
|
|
/// @return Number of bytes read, or a value < 0 on error.
|
|
|
|
/// Zero on end-of-file, or if 'length' is zero.
|
2014-09-30 21:52:21 +00:00
|
|
|
virtual int64_t Read(void* buffer, uint64_t length) = 0;
|
2013-10-11 21:44:55 +00:00
|
|
|
|
2014-01-24 00:26:00 +00:00
|
|
|
/// Write block of data.
|
|
|
|
/// @param buffer points to a block of memory with at least @a length bytes.
|
|
|
|
/// @param length indicates number of bytes to write.
|
|
|
|
/// @return Number of bytes written, or a value < 0 on error.
|
2014-09-30 21:52:21 +00:00
|
|
|
virtual int64_t Write(const void* buffer, uint64_t length) = 0;
|
2013-10-11 21:44:55 +00:00
|
|
|
|
2014-01-24 00:26:00 +00:00
|
|
|
/// @return Size of the file in bytes. A return value less than zero
|
|
|
|
/// indicates a problem getting the size.
|
2014-09-30 21:52:21 +00:00
|
|
|
virtual int64_t Size() = 0;
|
2013-10-11 21:44:55 +00:00
|
|
|
|
2014-01-24 00:26:00 +00:00
|
|
|
/// Flush the file so that recently written data will survive an
|
|
|
|
/// application crash (but not necessarily an OS crash). For
|
|
|
|
/// instance, in LocalFile the data is flushed into the OS but not
|
|
|
|
/// necessarily to disk.
|
|
|
|
/// @return true on success, false otherwise.
|
2013-10-11 21:44:55 +00:00
|
|
|
virtual bool Flush() = 0;
|
|
|
|
|
2015-05-20 17:14:36 +00:00
|
|
|
/// Seek to the specifield position in the file.
|
|
|
|
/// @param position is the position to seek to.
|
|
|
|
/// @return true on success, false otherwise.
|
|
|
|
virtual bool Seek(uint64_t position) = 0;
|
|
|
|
|
|
|
|
/// Get the current file position.
|
|
|
|
/// @param position is a pointer to contain the current file position upon
|
|
|
|
/// successful return.
|
|
|
|
/// @return true on succcess, false otherwise.
|
|
|
|
virtual bool Tell(uint64_t* position) = 0;
|
|
|
|
|
2017-07-17 18:15:07 +00:00
|
|
|
/// @return The file name. Note that the file type prefix has been stripped
|
|
|
|
/// off.
|
2013-11-12 20:37:58 +00:00
|
|
|
const std::string& file_name() const { return file_name_; }
|
|
|
|
|
2013-10-11 21:44:55 +00:00
|
|
|
// ************************************************************
|
|
|
|
// * Static Methods: File-on-the-filesystem status
|
|
|
|
// ************************************************************
|
|
|
|
|
2014-01-24 00:26:00 +00:00
|
|
|
/// @return The size of a file in bytes on success, a value < 0 otherwise.
|
|
|
|
/// The file will be opened and closed in the process.
|
2014-09-30 21:52:21 +00:00
|
|
|
static int64_t GetFileSize(const char* file_name);
|
2014-01-15 22:29:56 +00:00
|
|
|
|
2017-08-01 22:34:20 +00:00
|
|
|
/// Read the contents of a file into string.
|
2014-01-24 00:26:00 +00:00
|
|
|
/// @param file_name is the file to be read. It should be a valid file.
|
|
|
|
/// @param contents[out] points to the output string. Should not be NULL.
|
|
|
|
/// @return true on success, false otherwise.
|
2014-01-15 22:29:56 +00:00
|
|
|
static bool ReadFileToString(const char* file_name, std::string* contents);
|
2013-10-11 21:44:55 +00:00
|
|
|
|
2017-08-01 22:34:20 +00:00
|
|
|
/// Writes the data to file.
|
|
|
|
/// @param file_name is the file to write to.
|
|
|
|
/// @param contents is the data to write.
|
|
|
|
/// @return true on success, false otherwise.
|
|
|
|
static bool WriteStringToFile(const char* file_name,
|
|
|
|
const std::string& contents);
|
|
|
|
|
2017-06-15 20:00:28 +00:00
|
|
|
/// Save `contents` to `file_name` in an atomic manner.
|
|
|
|
/// @param file_name is the destination file name.
|
|
|
|
/// @param contents is the data to be saved.
|
|
|
|
/// @return true on success, false otherwise.
|
|
|
|
static bool WriteFileAtomically(const char* file_name,
|
|
|
|
const std::string& contents);
|
|
|
|
|
2015-11-05 01:49:35 +00:00
|
|
|
/// Copies files. This is not good for copying huge files. Although not
|
|
|
|
/// recommended, it is safe to have source file and destination file name be
|
|
|
|
/// the same.
|
|
|
|
/// @param from_file_name is the source file name.
|
|
|
|
/// @param to_file_name is the destination file name.
|
|
|
|
/// @return true on success, false otherwise.
|
|
|
|
static bool Copy(const char* from_file_name, const char* to_file_name);
|
|
|
|
|
2015-10-28 17:23:08 +00:00
|
|
|
/// Copies the contents from source to destination.
|
|
|
|
/// @param source The file to copy from.
|
|
|
|
/// @param destination The file to copy to.
|
|
|
|
/// @return Number of bytes written, or a value < 0 on error.
|
|
|
|
static int64_t CopyFile(File* source, File* destination);
|
|
|
|
|
|
|
|
/// Copies the contents from source to destination.
|
|
|
|
/// @param source The file to copy from.
|
|
|
|
/// @param destination The file to copy to.
|
|
|
|
/// @param max_copy The maximum number of bytes to copy; < 0 to copy to EOF.
|
|
|
|
/// @return Number of bytes written, or a value < 0 on error.
|
|
|
|
static int64_t CopyFile(File* source, File* destination, int64_t max_copy);
|
|
|
|
|
2019-10-18 22:15:31 +00:00
|
|
|
/// @param file_name is the name of the file to be checked.
|
|
|
|
/// @return true if `file_name` is a local and regular file.
|
|
|
|
static bool IsLocalRegularFile(const char* file_name);
|
|
|
|
|
2017-07-17 18:15:07 +00:00
|
|
|
/// Generate callback file name.
|
|
|
|
/// NOTE: THE GENERATED NAME IS ONLY VAID WHILE @a callback_params IS VALID.
|
|
|
|
/// @param callback_params references BufferCallbackParams, which will be
|
|
|
|
/// embedded in the generated callback file name.
|
|
|
|
/// @param name is the name of the buffer, which will be embedded in the
|
|
|
|
/// generated callback file name.
|
|
|
|
static std::string MakeCallbackFileName(
|
|
|
|
const BufferCallbackParams& callback_params,
|
|
|
|
const std::string& name);
|
|
|
|
|
|
|
|
/// Parse and extract callback params.
|
|
|
|
/// @param callback_file_name is the name of the callback file which contains
|
|
|
|
/// @a callback_params and @a name.
|
|
|
|
/// @param callback_params points to the parsed BufferCallbackParams pointer.
|
|
|
|
/// @param name points to the parsed name.
|
|
|
|
/// @return true on success, false otherwise.
|
|
|
|
static bool ParseCallbackFileName(
|
|
|
|
const std::string& callback_file_name,
|
|
|
|
const BufferCallbackParams** callback_params,
|
|
|
|
std::string* name);
|
|
|
|
|
2013-10-11 21:44:55 +00:00
|
|
|
protected:
|
2013-11-12 20:37:58 +00:00
|
|
|
explicit File(const std::string& file_name) : file_name_(file_name) {}
|
2014-01-24 00:26:00 +00:00
|
|
|
/// Do *not* call the destructor directly (with the "delete" keyword)
|
2016-08-17 17:41:40 +00:00
|
|
|
/// nor use std::unique_ptr; instead use Close().
|
2013-10-11 21:44:55 +00:00
|
|
|
virtual ~File() {}
|
|
|
|
|
2014-01-24 00:26:00 +00:00
|
|
|
/// Internal open. Should not be used directly.
|
2013-10-11 21:44:55 +00:00
|
|
|
virtual bool Open() = 0;
|
|
|
|
|
|
|
|
private:
|
2015-03-19 18:28:04 +00:00
|
|
|
friend class ThreadedIoFile;
|
|
|
|
|
2013-10-11 21:44:55 +00:00
|
|
|
// This is a file factory method, it creates a proper file, e.g.
|
|
|
|
// LocalFile, MemFile based on prefix.
|
2014-01-15 22:29:56 +00:00
|
|
|
static File* Create(const char* file_name, const char* mode);
|
2013-10-11 21:44:55 +00:00
|
|
|
|
2015-05-20 17:14:36 +00:00
|
|
|
static File* CreateInternalFile(const char* file_name, const char* mode);
|
|
|
|
|
2017-07-17 18:15:07 +00:00
|
|
|
// Note that the file type prefix has been stripped off.
|
2013-11-12 20:37:58 +00:00
|
|
|
std::string file_name_;
|
2017-07-17 18:15:07 +00:00
|
|
|
|
2013-10-11 21:44:55 +00:00
|
|
|
DISALLOW_COPY_AND_ASSIGN(File);
|
|
|
|
};
|
|
|
|
|
2016-05-20 21:19:33 +00:00
|
|
|
} // namespace shaka
|
2013-10-11 21:44:55 +00:00
|
|
|
|
|
|
|
#endif // PACKAGER_FILE_FILE_H_
|