From 00af1926269b1b2fef851d39a91e487e0bf2a3a6 Mon Sep 17 00:00:00 2001 From: Jacob Trimble Date: Tue, 2 Feb 2021 13:09:18 -0800 Subject: [PATCH] Cleanup HttpFile and related PR. This implements many of the comments made on the PR and cleans up those files. Closes #149 Change-Id: Ice73fe3c04a6f595da6986a4c070e50cb20f9435 --- docs/source/tutorials/http_upload.rst | 59 +- packager/app/packager_main.cc | 6 - packager/app/packager_util.cc | 22 +- .../app/playready_key_encryption_flags.cc | 9 - packager/app/playready_key_encryption_flags.h | 4 - packager/file/file.cc | 10 +- packager/file/http_file.cc | 516 +++++++++--------- packager/file/http_file.h | 97 ++-- packager/file/http_file_unittest.cc | 195 ++++++- packager/file/local_file.cc | 4 - packager/media/base/buffer_writer.cc | 2 +- packager/media/base/http_key_fetcher.cc | 220 +------- packager/media/base/http_key_fetcher.h | 36 +- packager/media/base/playready_key_source.cc | 21 - packager/media/base/playready_key_source.h | 21 - .../packed_audio/packed_audio_writer.cc | 1 - 16 files changed, 532 insertions(+), 691 deletions(-) diff --git a/docs/source/tutorials/http_upload.rst b/docs/source/tutorials/http_upload.rst index 2c5aeed8da..0061b35db7 100644 --- a/docs/source/tutorials/http_upload.rst +++ b/docs/source/tutorials/http_upload.rst @@ -9,10 +9,7 @@ contributions are always welcome. The discussion about this feature currently happens at - `Add HTTP PUT output #149 `_, - its development on the - `http-upload `_ branch, - feel free to join us. + `Add HTTP PUT output #149 `_. ########### HTTP upload @@ -46,15 +43,10 @@ Documentation Getting started =============== -For enabling the HTTP upload transfer mode, please populate -the ``segment_template`` attribute in the ``stream_descriptor`` -parameter as well as the ``--hls_master_playlist_output`` parameter -with appropriate URLs where the HTTP PUT requests will be issued to. +To enable the HTTP upload transfer mode, use ``https:`` file paths for any +output files (e.g. ``segment_template``). -You can also supply the ``--user_agent`` flag to specify a custom -User-Agent string for all HTTP PUT requests. - -For pragmatic reasons, all HTTP requests will be declared as +All HTTP requests will be declared as ``Content-Type: application/octet-stream``. Synopsis @@ -90,17 +82,18 @@ Configure and run packager:: --hls_playlist_type LIVE \ --vmodule=http_file=1 -******* -HTTPS -******* -If your ingest uses HTTPS and requires specific certificates, these -can be specified on the command line similar to how it's done for -:doc:`playready`, with the following arguments: +********************* +Client Authentication +********************* +If your server requires client authentication, you can add the following +arguments to enable it: -- ``--https_ca_file``: Absolute path to the Certificate Authority file for the server cert. PEM format. -- ``--https_cert_file``: Absolute path to client certificate file. -- ``--https_cert_private_key_file``: Absolute path to the private Key file. -- ``--https_cert_private_key_password``: Password to the private key file. +- ``--ca_file``: (optional) Absolute path to the Certificate Authority file for + the server cert. PEM format. +- ``--client_cert_file``: Absolute path to client certificate file. +- ``--client_cert_private_key_file``: Absolute path to the private Key file. +- ``--client_cert_private_key_password``: (optional) Password to the private + key file. ******* Backlog @@ -108,20 +101,6 @@ Backlog Please note the HTTP upload feature still lacks some features probably important for production. Contributions are welcome! -DASH -==== -While the current implementation works for HLS_, -we should also check DASH_. - -Basic Auth -========== -There's no support for authentication yet. - -HTTPS -===== -While there's already some code in place, -HTTPS is currently not supported yet. - HTTP DELETE =========== Nothing has be done to support this yet: @@ -146,9 +125,9 @@ Miscellaneous - Make ``io_cache_size`` configurable? -******* -Backend -******* +*************** +Example Backend +*************** HTTP PUT file uploads to Nginx ============================== @@ -230,8 +209,6 @@ Grab and run `httpd-reflector.py`_ to use it as a dummy HTTP sink:: ---- -Have fun! - .. _HLS: https://en.wikipedia.org/wiki/HTTP_Live_Streaming .. _DASH: https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP .. _M3U: https://en.wikipedia.org/wiki/M3U diff --git a/packager/app/packager_main.cc b/packager/app/packager_main.cc index b6f1add2e0..c8653ce7bc 100644 --- a/packager/app/packager_main.cc +++ b/packager/app/packager_main.cc @@ -387,12 +387,6 @@ base::Optional GetPackagingParams() { PlayReadyEncryptionParams& playready = encryption_params.playready; playready.key_server_url = FLAGS_playready_server_url; playready.program_identifier = FLAGS_program_identifier; - playready.ca_file = FLAGS_ca_file; - playready.client_cert_file = FLAGS_client_cert_file; - playready.client_cert_private_key_file = - FLAGS_client_cert_private_key_file; - playready.client_cert_private_key_password = - FLAGS_client_cert_private_key_password; break; } case KeyProvider::kRawKey: { diff --git a/packager/app/packager_util.cc b/packager/app/packager_util.cc index fead5c1809..e4cbe02eb7 100644 --- a/packager/app/packager_util.cc +++ b/packager/app/packager_util.cc @@ -100,26 +100,8 @@ std::unique_ptr CreateEncryptionKeySource( } std::unique_ptr playready_key_source; // private_key_password is allowed to be empty for unencrypted key. - if (!playready.client_cert_file.empty() || - !playready.client_cert_private_key_file.empty()) { - if (playready.client_cert_file.empty() || - playready.client_cert_private_key_file.empty()) { - LOG(ERROR) << "Either PlayReady client_cert_file or " - "client_cert_private_key_file is not set."; - return nullptr; - } - playready_key_source.reset(new PlayReadyKeySource( - playready.key_server_url, playready.client_cert_file, - playready.client_cert_private_key_file, - playready.client_cert_private_key_password, - encryption_params.protection_systems)); - } else { - playready_key_source.reset(new PlayReadyKeySource( - playready.key_server_url, encryption_params.protection_systems)); - } - if (!playready.ca_file.empty()) { - playready_key_source->SetCaFile(playready.ca_file); - } + playready_key_source.reset(new PlayReadyKeySource( + playready.key_server_url, encryption_params.protection_systems)); Status status = playready_key_source->FetchKeysWithProgramIdentifier( playready.program_identifier); if (!status.ok()) { diff --git a/packager/app/playready_key_encryption_flags.cc b/packager/app/playready_key_encryption_flags.cc index 4a28110a06..374475f5ed 100644 --- a/packager/app/playready_key_encryption_flags.cc +++ b/packager/app/playready_key_encryption_flags.cc @@ -16,15 +16,6 @@ DEFINE_bool(enable_playready_encryption, DEFINE_string(playready_server_url, "", "PlayReady packaging server url."); DEFINE_string(program_identifier, "", "Program identifier for packaging request."); -DEFINE_string(ca_file, "", - "Absolute path to the Certificate Authority file for the " - "server cert. PEM format"); -DEFINE_string(client_cert_file, "", - "Absolute path to client certificate file."); -DEFINE_string(client_cert_private_key_file, "", - "Absolute path to the Private Key file."); -DEFINE_string(client_cert_private_key_password, "", - "Password to the private key file."); namespace shaka { namespace { diff --git a/packager/app/playready_key_encryption_flags.h b/packager/app/playready_key_encryption_flags.h index cdbe76ca16..a82c2be75e 100644 --- a/packager/app/playready_key_encryption_flags.h +++ b/packager/app/playready_key_encryption_flags.h @@ -16,10 +16,6 @@ DECLARE_bool(enable_playready_encryption); DECLARE_string(playready_server_url); DECLARE_string(program_identifier); -DECLARE_string(ca_file); -DECLARE_string(client_cert_file); -DECLARE_string(client_cert_private_key_file); -DECLARE_string(client_cert_private_key_password); namespace shaka { diff --git a/packager/file/file.cc b/packager/file/file.cc index 3e61c68d9f..92825b97b5 100644 --- a/packager/file/file.cc +++ b/packager/file/file.cc @@ -100,11 +100,11 @@ File* CreateUdpFile(const char* file_name, const char* mode) { } File* CreateHttpsFile(const char* file_name, const char* mode) { - return new HttpFile(file_name, mode, true); + return new HttpFile(HttpMethod::kPut, std::string("https://") + file_name); } File* CreateHttpFile(const char* file_name, const char* mode) { - return new HttpFile(file_name, mode, false); + return new HttpFile(HttpMethod::kPut, std::string("http://") + file_name); } File* CreateMemoryFile(const char* file_name, const char* mode) { @@ -137,7 +137,6 @@ base::StringPiece GetFileTypePrefix(base::StringPiece file_name) { const FileTypeInfo* GetFileTypeInfo(base::StringPiece file_name, base::StringPiece* real_file_name) { - base::StringPiece file_type_prefix = GetFileTypePrefix(file_name); for (const FileTypeInfo& file_type : kFileTypeInfo) { if (file_type_prefix == file_type.type) { @@ -297,7 +296,7 @@ bool File::WriteFileAtomically(const char* file_name, bool File::Copy(const char* from_file_name, const char* to_file_name) { std::string content; - VLOG(1) << "File::Copy from " << from_file_name << " to " << to_file_name; + VLOG(2) << "File::Copy from " << from_file_name << " to " << to_file_name; if (!ReadFileToString(from_file_name, &content)) { LOG(ERROR) << "Failed to open file " << from_file_name; return false; @@ -341,7 +340,8 @@ int64_t File::CopyFile(File* source, File* destination, int64_t max_copy) { if (max_copy < 0) max_copy = std::numeric_limits::max(); - VLOG(1) << "File::CopyFile from " << source->file_name() << " to " << destination->file_name(); + VLOG(2) << "File::CopyFile from " << source->file_name() << " to " + << destination->file_name(); const int64_t kBufferSize = 0x40000; // 256KB. std::unique_ptr buffer(new uint8_t[kBufferSize]); diff --git a/packager/file/http_file.cc b/packager/file/http_file.cc index 9fe513a839..8ba7b4719a 100644 --- a/packager/file/http_file.cc +++ b/packager/file/http_file.cc @@ -1,4 +1,4 @@ -// Copyright 2018 Google Inc. All rights reserved. +// Copyright 2020 Google LLC. All rights reserved. // // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file or at @@ -6,49 +6,113 @@ #include "packager/file/http_file.h" +#include #include + #include "packager/base/bind.h" #include "packager/base/files/file_util.h" #include "packager/base/logging.h" #include "packager/base/strings/string_number_conversions.h" #include "packager/base/strings/stringprintf.h" -#include "packager/base/synchronization/lock.h" #include "packager/base/threading/worker_pool.h" -DEFINE_int32(libcurl_verbosity, 0, - "Set verbosity level for libcurl."); -DEFINE_string(user_agent, "", - "Set a custom User-Agent string for HTTP ingest."); -DEFINE_string(https_ca_file, "", +DEFINE_string(ca_file, + "", "Absolute path to the Certificate Authority file for the " "server cert. PEM format"); -DEFINE_string(https_cert_file, "", +DEFINE_string(client_cert_file, + "", "Absolute path to client certificate file."); -DEFINE_string(https_cert_private_key_file, "", - "Absolute path to the private Key file."); -DEFINE_string(https_cert_private_key_password, "", +DEFINE_string(client_cert_private_key_file, + "", + "Absolute path to the Private Key file."); +DEFINE_string(client_cert_private_key_password, + "", "Password to the private key file."); +DEFINE_bool(disable_peer_verification, + false, + "Disable peer verification. This is needed to talk to servers " + "without valid certificates."); DECLARE_uint64(io_cache_size); namespace shaka { -// curl_ primitives stolen from `http_key_fetcher.cc`. namespace { -const char kUserAgentString[] = "shaka-packager-uploader/0.1"; +constexpr const char* kBinaryContentType = "application/octet-stream"; +constexpr const char* kUserAgent = "shaka-packager-http-fetch/1.0"; +constexpr const int kMinLogLevelForCurlDebugFunction = 2; -size_t AppendToString(char* ptr, - size_t size, - size_t nmemb, - std::string* response) { - DCHECK(ptr); - DCHECK(response); - const size_t total_size = size * nmemb; - response->append(ptr, total_size); - return total_size; +size_t CurlWriteCallback(char* buffer, size_t size, size_t nmemb, void* user) { + IoCache* cache = reinterpret_cast(user); + size_t length = cache->Write(buffer, size * nmemb); + VLOG(3) << "CurlWriteCallback length=" << length; + return length; } -} // namespace +size_t CurlReadCallback(char* buffer, size_t size, size_t nitems, void* user) { + IoCache* cache = reinterpret_cast(user); + size_t length = cache->Read(buffer, size * nitems); + VLOG(3) << "CurlRead length=" << length; + return length; +} + +int CurlDebugCallback(CURL* /* handle */, + curl_infotype type, + const char* data, + size_t size, + void* /* userptr */) { + const char* type_text; + int log_level; + bool in_hex; + switch (type) { + case CURLINFO_TEXT: + type_text = "== Info"; + log_level = kMinLogLevelForCurlDebugFunction + 1; + in_hex = false; + break; + case CURLINFO_HEADER_IN: + type_text = "<= Recv header"; + log_level = kMinLogLevelForCurlDebugFunction; + in_hex = false; + break; + case CURLINFO_HEADER_OUT: + type_text = "=> Send header"; + log_level = kMinLogLevelForCurlDebugFunction; + in_hex = false; + break; + case CURLINFO_DATA_IN: + type_text = "<= Recv data"; + log_level = kMinLogLevelForCurlDebugFunction + 1; + in_hex = true; + break; + case CURLINFO_DATA_OUT: + type_text = "=> Send data"; + log_level = kMinLogLevelForCurlDebugFunction + 1; + in_hex = true; + break; + case CURLINFO_SSL_DATA_IN: + type_text = "<= Recv SSL data"; + log_level = kMinLogLevelForCurlDebugFunction + 2; + in_hex = true; + break; + case CURLINFO_SSL_DATA_OUT: + type_text = "=> Send SSL data"; + log_level = kMinLogLevelForCurlDebugFunction + 2; + in_hex = true; + break; + default: + // Ignore other debug data. + return 0; + } + + VLOG(log_level) << "\n\n" + << type_text << " (0x" << std::hex << size << std::dec + << " bytes)\n" + << (in_hex ? base::HexEncode(data, size) + : std::string(data, size)); + return 0; +} class LibCurlInitializer { public: @@ -60,309 +124,221 @@ class LibCurlInitializer { curl_global_cleanup(); } - private: - DISALLOW_COPY_AND_ASSIGN(LibCurlInitializer); + LibCurlInitializer(const LibCurlInitializer&) = delete; + LibCurlInitializer& operator=(const LibCurlInitializer&) = delete; }; -/// Create a HTTP/HTTPS client -HttpFile::HttpFile(const char* file_name, const char* mode, bool https) - : File(file_name), - file_mode_(mode), - user_agent_(FLAGS_user_agent), - ca_file_(FLAGS_https_ca_file), - cert_file_(FLAGS_https_cert_file), - cert_private_key_file_(FLAGS_https_cert_private_key_file), - cert_private_key_pass_(FLAGS_https_cert_private_key_password), - timeout_in_seconds_(0), - cache_(FLAGS_io_cache_size), - scoped_curl(curl_easy_init(), &curl_easy_cleanup), - task_exit_event_(base::WaitableEvent::ResetPolicy::AUTOMATIC, - base::WaitableEvent::InitialState::NOT_SIGNALED) { - if (https) { - resource_url_ = "https://" + std::string(file_name); +template +bool AppendHeader(const std::string& header, List* list) { + auto* temp = curl_slist_append(list->get(), header.c_str()); + if (temp) { + list->release(); // Don't free old list since it's part of the new one. + list->reset(temp); + return true; } else { - resource_url_ = "http://" + std::string(file_name); - } - - static LibCurlInitializer lib_curl_initializer; - - // Setup libcurl scope - if (!scoped_curl.get()) { - LOG(ERROR) << "curl_easy_init() failed."; - // return Status(error::HTTP_FAILURE, "curl_easy_init() failed."); - delete this; + return false; } } -HttpFile::HttpFile(const char* file_name, const char* mode) - : HttpFile(file_name, mode, false) -{} +} // namespace + +HttpFile::HttpFile(HttpMethod method, const std::string& url) + : HttpFile(method, url, kBinaryContentType, {}, 0) {} + +HttpFile::HttpFile(HttpMethod method, + const std::string& url, + const std::string& upload_content_type, + const std::vector& headers, + uint32_t timeout_in_seconds) + : File(url.c_str()), + url_(url), + upload_content_type_(upload_content_type), + timeout_in_seconds_(timeout_in_seconds), + method_(method), + download_cache_(FLAGS_io_cache_size), + upload_cache_(FLAGS_io_cache_size), + curl_(curl_easy_init()), + status_(Status::OK), + task_exit_event_(base::WaitableEvent::ResetPolicy::MANUAL, + base::WaitableEvent::InitialState::NOT_SIGNALED) { + static LibCurlInitializer lib_curl_initializer; + + // We will have at least one header, so use a null header to signal error + // to Open. + + // Don't wait for 100-Continue. + std::unique_ptr temp_headers; + if (!AppendHeader("Expect:", &temp_headers)) + return; + if (!upload_content_type.empty() && + !AppendHeader("Content-Type: " + upload_content_type_, &temp_headers)) { + return; + } + if (method != HttpMethod::kGet && + !AppendHeader("Transfer-Encoding: chunked", &temp_headers)) { + return; + } + for (const auto& item : headers) { + if (!AppendHeader(item, &temp_headers)) { + return; + } + } + request_headers_ = std::move(temp_headers); +} -// Destructor HttpFile::~HttpFile() {} bool HttpFile::Open() { + VLOG(2) << "Opening " << url_; - VLOG(1) << "Opening " << resource_url() << " with file mode \"" << file_mode_ << "\"."; - - // Ignore read requests as they would truncate the target - // file by propagating as zero-length PUT requests. - // See also https://github.com/google/shaka-packager/issues/149#issuecomment-437203701 - if (file_mode_ == "r") { - VLOG(1) << "HttpFile only supports write mode, skipping further operations"; - task_exit_event_.Signal(); + if (!curl_ || !request_headers_) { + LOG(ERROR) << "curl_easy_init() failed."; return false; } - - // Run progressive upload in separate thread. - base::WorkerPool::PostTask( - FROM_HERE, base::Bind(&HttpFile::CurlPut, base::Unretained(this)), - true // task_is_slow - ); - - return true; -} - -void HttpFile::CurlPut() { - // Setup libcurl handle with HTTP PUT upload transfer mode. - std::string request_body; - Request(PUT, resource_url(), request_body, &response_body_); -} - -bool HttpFile::Close() { - VLOG(1) << "Closing " << resource_url() << "."; - cache_.Close(); - task_exit_event_.Wait(); - delete this; - return true; -} - -int64_t HttpFile::Read(void* buffer, uint64_t length) { - LOG(WARNING) << "HttpFile does not support Read()."; - return -1; -} - -int64_t HttpFile::Write(const void* buffer, uint64_t length) { - std::string url = resource_url(); - - VLOG(2) << "Writing to " << url << ", length=" << length; + // TODO: Try to connect initially so we can return connection error here. // TODO: Implement retrying with exponential backoff, see // "widevine_key_source.cc" - Status status; - uint64_t bytes_written = cache_.Write(buffer, length); - VLOG(3) << "PUT CHUNK bytes_written: " << bytes_written; - return bytes_written; + base::WorkerPool::PostTask( + FROM_HERE, base::Bind(&HttpFile::ThreadMain, base::Unretained(this)), + /* task_is_slow= */ true); - // Debugging based on response status - /* - if (status.ok()) { - VLOG(1) << "Writing chunk succeeded"; + return true; +} - } else { - VLOG(1) << "Writing chunk failed"; - if (!response_body.empty()) { - VLOG(2) << "Response:\n" << response_body; - } - } - */ +Status HttpFile::CloseWithStatus() { + VLOG(2) << "Closing " << url_; + // Close the cache first so the thread will finish uploading. Otherwise it + // will wait for more data forever. + download_cache_.Close(); + upload_cache_.Close(); + task_exit_event_.Wait(); - // Always signal success to the downstream pipeline - return length; + const Status result = status_; + LOG_IF(ERROR, !result.ok()) << "HttpFile request failed: " << result; + delete this; + return result; +} + +bool HttpFile::Close() { + return CloseWithStatus().ok(); +} + +int64_t HttpFile::Read(void* buffer, uint64_t length) { + VLOG(2) << "Reading from " << url_ << ", length=" << length; + return download_cache_.Read(buffer, length); +} + +int64_t HttpFile::Write(const void* buffer, uint64_t length) { + VLOG(2) << "Writing to " << url_ << ", length=" << length; + return upload_cache_.Write(buffer, length); } int64_t HttpFile::Size() { - VLOG(1) << "HttpFile does not support Size()."; + LOG(ERROR) << "HttpFile does not support Size()."; return -1; } bool HttpFile::Flush() { - // Do nothing on Flush. + upload_cache_.Close(); return true; } bool HttpFile::Seek(uint64_t position) { - VLOG(1) << "HttpFile does not support Seek()."; + LOG(ERROR) << "HttpFile does not support Seek()."; return false; } bool HttpFile::Tell(uint64_t* position) { - VLOG(1) << "HttpFile does not support Tell()."; + LOG(ERROR) << "HttpFile does not support Tell()."; return false; } -// Perform HTTP request -Status HttpFile::Request(HttpMethod http_method, - const std::string& url, - const std::string& data, - std::string* response) { +void HttpFile::CurlDelete::operator()(CURL* curl) { + curl_easy_cleanup(curl); +} - // TODO: Sanity checks. - // DCHECK(http_method == GET || http_method == POST); +void HttpFile::CurlDelete::operator()(curl_slist* headers) { + curl_slist_free_all(headers); +} - VLOG(1) << "Sending request to URL " << url; +void HttpFile::SetupRequest() { + auto* curl = curl_.get(); - // Setup HTTP method and libcurl options - SetupRequestBase(http_method, url, response); + switch (method_) { + case HttpMethod::kGet: + curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L); + break; + case HttpMethod::kPost: + curl_easy_setopt(curl, CURLOPT_POST, 1L); + break; + case HttpMethod::kPut: + curl_easy_setopt(curl, CURLOPT_PUT, 1L); + break; + } - // Setup HTTP request headers and body - SetupRequestData(data); + curl_easy_setopt(curl, CURLOPT_URL, url_.c_str()); + curl_easy_setopt(curl, CURLOPT_USERAGENT, kUserAgent); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout_in_seconds_); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &CurlWriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &download_cache_); + if (method_ != HttpMethod::kGet) { + curl_easy_setopt(curl, CURLOPT_READFUNCTION, &CurlReadCallback); + curl_easy_setopt(curl, CURLOPT_READDATA, &upload_cache_); + } - // Perform HTTP request - CURLcode res = curl_easy_perform(scoped_curl.get()); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers_.get()); - // Assume successful request - Status status = Status::OK; + if (FLAGS_disable_peer_verification) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); - // Handle request failure + // Client authentication + if (!FLAGS_client_cert_private_key_file.empty() && + !FLAGS_client_cert_file.empty()) { + curl_easy_setopt(curl, CURLOPT_SSLKEY, + FLAGS_client_cert_private_key_file.data()); + curl_easy_setopt(curl, CURLOPT_SSLCERT, FLAGS_client_cert_file.data()); + curl_easy_setopt(curl, CURLOPT_SSLKEYTYPE, "PEM"); + curl_easy_setopt(curl, CURLOPT_SSLCERTTYPE, "PEM"); + + if (!FLAGS_client_cert_private_key_password.empty()) { + curl_easy_setopt(curl, CURLOPT_KEYPASSWD, + FLAGS_client_cert_private_key_password.data()); + } + } + if (!FLAGS_ca_file.empty()) { + curl_easy_setopt(curl, CURLOPT_CAINFO, FLAGS_ca_file.data()); + } + + if (VLOG_IS_ON(kMinLogLevelForCurlDebugFunction)) { + curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, CurlDebugCallback); + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + } +} + +void HttpFile::ThreadMain() { + SetupRequest(); + + CURLcode res = curl_easy_perform(curl_.get()); if (res != CURLE_OK) { - std::string method_text = method_as_text(http_method); - std::string error_message = base::StringPrintf( - "%s request for %s failed. Reason: %s.", method_text.c_str(), - url.c_str(), curl_easy_strerror(res)); + std::string error_message = curl_easy_strerror(res); if (res == CURLE_HTTP_RETURNED_ERROR) { long response_code = 0; - curl_easy_getinfo(scoped_curl.get(), CURLINFO_RESPONSE_CODE, &response_code); + curl_easy_getinfo(curl_.get(), CURLINFO_RESPONSE_CODE, &response_code); error_message += - base::StringPrintf(" Response code: %ld.", response_code); + base::StringPrintf(", response code: %ld.", response_code); } - // Signal error to logfile - LOG(ERROR) << error_message; - - // Signal error to caller - status = Status( + status_ = Status( res == CURLE_OPERATION_TIMEDOUT ? error::TIME_OUT : error::HTTP_FAILURE, error_message); } - // Signal task completion + download_cache_.Close(); task_exit_event_.Signal(); - - // Return request status to caller - return status; -} - -// Configure curl_ handle with reasonable defaults -void HttpFile::SetupRequestBase(HttpMethod http_method, - const std::string& url, - std::string* response) { - response->clear(); - - // Configure HTTP request method/verb - switch (http_method) { - case GET: - curl_easy_setopt(scoped_curl.get(), CURLOPT_HTTPGET, 1L); - break; - case POST: - curl_easy_setopt(scoped_curl.get(), CURLOPT_POST, 1L); - break; - case PUT: - curl_easy_setopt(scoped_curl.get(), CURLOPT_PUT, 1L); - break; - case PATCH: - curl_easy_setopt(scoped_curl.get(), CURLOPT_CUSTOMREQUEST, "PATCH"); - break; - } - - // Configure HTTP request - curl_easy_setopt(scoped_curl.get(), CURLOPT_URL, url.c_str()); - - if (user_agent_.empty()) { - curl_easy_setopt(scoped_curl.get(), CURLOPT_USERAGENT, kUserAgentString); - } else { - curl_easy_setopt(scoped_curl.get(), CURLOPT_USERAGENT, user_agent_.data()); - } - - curl_easy_setopt(scoped_curl.get(), CURLOPT_TIMEOUT, timeout_in_seconds_); - curl_easy_setopt(scoped_curl.get(), CURLOPT_FAILONERROR, 1L); - curl_easy_setopt(scoped_curl.get(), CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(scoped_curl.get(), CURLOPT_WRITEFUNCTION, AppendToString); - curl_easy_setopt(scoped_curl.get(), CURLOPT_WRITEDATA, response); - - // HTTPS - if (!cert_private_key_file_.empty() && !cert_file_.empty()) { - curl_easy_setopt(scoped_curl.get(), CURLOPT_SSLKEY, - cert_private_key_file_.data()); - - if (!cert_private_key_pass_.empty()) { - curl_easy_setopt(scoped_curl.get(), CURLOPT_KEYPASSWD, - cert_private_key_pass_.data()); - } - - curl_easy_setopt(scoped_curl.get(), CURLOPT_SSLKEYTYPE, "PEM"); - curl_easy_setopt(scoped_curl.get(), CURLOPT_SSLCERTTYPE, "PEM"); - curl_easy_setopt(scoped_curl.get(), CURLOPT_SSLCERT, cert_file_.data()); - } - if (!ca_file_.empty()) { - // Host validation needs to be off when using self-signed certificates. - curl_easy_setopt(scoped_curl.get(), CURLOPT_SSL_VERIFYHOST, 0L); - curl_easy_setopt(scoped_curl.get(), CURLOPT_CAINFO, ca_file_.data()); - } - - // Propagate log level indicated by "--libcurl_verbosity" to libcurl. - curl_easy_setopt(scoped_curl.get(), CURLOPT_VERBOSE, FLAGS_libcurl_verbosity); - -} - -// https://ec.haxx.se/callback-read.html -size_t read_callback(char* buffer, size_t size, size_t nitems, void* stream) { - VLOG(3) << "read_callback"; - - // Cast stream back to what is actually is - // IoCache* cache = reinterpret_cast(stream); - IoCache* cache = (IoCache*)stream; - VLOG(3) << "read_callback, cache: " << cache; - - // Copy cache content into buffer - size_t length = cache->Read(buffer, size * nitems); - VLOG(3) << "read_callback, length: " << length << "; buffer: " << buffer; - return length; -} - -// Configure curl_ handle for HTTP PUT upload -void HttpFile::SetupRequestData(const std::string& data) { - - // TODO: Sanity checks. - // if (method == POST || method == PUT || method == PATCH) - - // Build list of HTTP request headers. - struct curl_slist* headers = nullptr; - - headers = curl_slist_append(headers, "Content-Type: application/octet-stream"); - headers = curl_slist_append(headers, "Transfer-Encoding: chunked"); - - // Don't stop on 200 OK responses. - headers = curl_slist_append(headers, "Expect:"); - - // Enable progressive upload with chunked transfer encoding. - curl_easy_setopt(scoped_curl.get(), CURLOPT_READFUNCTION, read_callback); - curl_easy_setopt(scoped_curl.get(), CURLOPT_READDATA, &cache_); - curl_easy_setopt(scoped_curl.get(), CURLOPT_UPLOAD, 1L); - - // Add HTTP request headers. - curl_easy_setopt(scoped_curl.get(), CURLOPT_HTTPHEADER, headers); -} - -// Return HTTP request method (verb) as string -std::string HttpFile::method_as_text(HttpMethod method) { - std::string method_text = "UNKNOWN"; - switch (method) { - case GET: - method_text = "GET"; - break; - case POST: - method_text = "POST"; - break; - case PUT: - method_text = "PUT"; - break; - case PATCH: - method_text = "PATCH"; - break; - } - return method_text; } } // namespace shaka diff --git a/packager/file/http_file.h b/packager/file/http_file.h index fc23347883..c0bd681093 100644 --- a/packager/file/http_file.h +++ b/packager/file/http_file.h @@ -1,4 +1,4 @@ -// Copyright 2018 Google Inc. All rights reserved. +// Copyright 2020 Google LLC. All rights reserved. // // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file or at @@ -7,35 +7,47 @@ #ifndef PACKAGER_FILE_HTTP_H_ #define PACKAGER_FILE_HTTP_H_ -#include #include +#include -#include "packager/base/compiler_specific.h" #include "packager/base/synchronization/waitable_event.h" #include "packager/file/file.h" #include "packager/file/io_cache.h" #include "packager/status.h" -namespace shaka { -using ScopedCurl = std::unique_ptr; +typedef void CURL; +struct curl_slist; -/// HttpFile delegates write calls to HTTP PUT requests. +namespace shaka { + +enum class HttpMethod { + kGet, + kPost, + kPut, +}; + +/// HttpFile reads or writes network requests. /// -/// About how to use this, please visit the corresponding documentation [1,2]. +/// Note that calling Flush will indicate EOF for the upload and no more can be +/// uploaded. +/// +/// About how to use this, please visit the corresponding documentation [1]. /// /// [1] https://google.github.io/shaka-packager/html/tutorials/http_upload.html -/// [2] -/// https://github.com/3QSDN/shaka-packager/blob/http-upload/docs/source/tutorials/http_upload.rst /// class HttpFile : public File { public: + HttpFile(HttpMethod method, const std::string& url); + HttpFile(HttpMethod method, + const std::string& url, + const std::string& upload_content_type, + const std::vector& headers, + uint32_t timeout_in_seconds); - /// Create a HTTP client - /// @param file_name contains the url of the resource to be accessed. - /// Note that the file type prefix should be stripped off already. - /// @param mode contains file access mode. Implementation dependent. - HttpFile(const char* file_name, const char* mode, bool https); - HttpFile(const char* file_name, const char* mode); + HttpFile(const HttpFile&) = delete; + HttpFile& operator=(const HttpFile&) = delete; + + Status CloseWithStatus(); /// @name File implementation overrides. /// @{ @@ -46,56 +58,31 @@ class HttpFile : public File { bool Flush() override; bool Seek(uint64_t position) override; bool Tell(uint64_t* position) override; + bool Open() override; /// @} - /// @return The full resource url - const std::string& resource_url() const { return resource_url_; } - protected: - // Destructor ~HttpFile() override; - bool Open() override; - private: - enum HttpMethod { - GET, - POST, - PUT, - PATCH, + struct CurlDelete { + void operator()(CURL* curl); + void operator()(curl_slist* headers); }; - HttpFile(const HttpFile&) = delete; - HttpFile& operator=(const HttpFile&) = delete; - - // Internal implementation of HTTP functions, e.g. Get and Post. - Status Request(HttpMethod http_method, - const std::string& url, - const std::string& data, - std::string* response); - - void SetupRequestBase(HttpMethod http_method, - const std::string& url, - std::string* response); - - void SetupRequestData(const std::string& data); - - void CurlPut(); - - std::string method_as_text(HttpMethod method); - - std::string file_mode_; - std::string resource_url_; - std::string user_agent_; - std::string ca_file_; - std::string cert_file_; - std::string cert_private_key_file_; - std::string cert_private_key_pass_; + void SetupRequest(); + void ThreadMain(); + const std::string url_; + const std::string upload_content_type_; const uint32_t timeout_in_seconds_; - IoCache cache_; - ScopedCurl scoped_curl; - std::string response_body_; + const HttpMethod method_; + IoCache download_cache_; + IoCache upload_cache_; + std::unique_ptr curl_; + // The headers need to remain alive for the duration of the request. + std::unique_ptr request_headers_; + Status status_; // Signaled when the "curl easy perform" task completes. base::WaitableEvent task_exit_event_; diff --git a/packager/file/http_file_unittest.cc b/packager/file/http_file_unittest.cc index c11fed8897..a3aec0c937 100644 --- a/packager/file/http_file_unittest.cc +++ b/packager/file/http_file_unittest.cc @@ -1,25 +1,202 @@ +// Copyright 2020 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + #include "packager/file/http_file.h" + #include + #include +#include + +#include "packager/base/json/json_reader.h" +#include "packager/base/values.h" #include "packager/file/file.h" #include "packager/file/file_closer.h" +#define ASSERT_JSON_STRING(json, key, value) \ + do { \ + std::string actual; \ + ASSERT_TRUE((json)->GetString((key), &actual)); \ + ASSERT_EQ(actual, (value)); \ + } while (false) + namespace shaka { + namespace { -const uint8_t kWriteBuffer[] = {1, 2, 3, 4, 5, 6, 7, 8}; -const int64_t kWriteBufferSize = sizeof(kWriteBuffer); +using FilePtr = std::unique_ptr; + +std::unique_ptr HandleResponse(const FilePtr& file) { + std::string result; + while (true) { + char buffer[64 * 1024]; + auto ret = file->Read(buffer, sizeof(buffer)); + if (ret < 0) + return nullptr; + if (ret == 0) + break; + result.append(buffer, buffer + ret); + } + VLOG(1) << "Response:\n" << result; + + auto value = base::JSONReader::Read(result); + if (!value || !value->IsType(base::Value::TYPE_DICTIONARY)) + return nullptr; + return std::unique_ptr{ + static_cast(value.release())}; +} } // namespace -class HttpFileTest : public testing::Test {}; +TEST(HttpFileTest, DISABLED_BasicGet) { + FilePtr file(new HttpFile(HttpMethod::kGet, "https://httpbin.org/anything")); + ASSERT_TRUE(file); + ASSERT_TRUE(file->Open()); -TEST_F(HttpFileTest, PutChunkedTranser) { - std::unique_ptr writer( - File::Open("http://127.0.0.1:8080/test_out", "w")); - ASSERT_TRUE(writer); - ASSERT_EQ(kWriteBufferSize, writer->Write(kWriteBuffer, kWriteBufferSize)); - writer.release()->Close(); + auto json = HandleResponse(file); + ASSERT_TRUE(json); + ASSERT_TRUE(file.release()->Close()); + ASSERT_JSON_STRING(json, "method", "GET"); +} + +TEST(HttpFileTest, DISABLED_CustomHeaders) { + std::vector headers{"Host: foo", "X-My-Header: Something"}; + FilePtr file(new HttpFile(HttpMethod::kGet, "https://httpbin.org/anything", + "", headers, 0)); + ASSERT_TRUE(file); + ASSERT_TRUE(file->Open()); + + auto json = HandleResponse(file); + ASSERT_TRUE(json); + ASSERT_TRUE(file.release()->Close()); + + ASSERT_JSON_STRING(json, "method", "GET"); + ASSERT_JSON_STRING(json, "headers.Host", "foo"); + ASSERT_JSON_STRING(json, "headers.X-My-Header", "Something"); +} + +TEST(HttpFileTest, DISABLED_BasicPost) { + FilePtr file(new HttpFile(HttpMethod::kPost, "https://httpbin.org/anything")); + ASSERT_TRUE(file); + ASSERT_TRUE(file->Open()); + + const std::string data = "abcd"; + + ASSERT_EQ(file->Write(data.data(), data.size()), + static_cast(data.size())); + ASSERT_TRUE(file->Flush()); + + auto json = HandleResponse(file); + ASSERT_TRUE(json); + ASSERT_TRUE(file.release()->Close()); + + ASSERT_JSON_STRING(json, "method", "POST"); + ASSERT_JSON_STRING(json, "data", data); + ASSERT_JSON_STRING(json, "headers.Content-Type", "application/octet-stream"); + ASSERT_JSON_STRING(json, "headers.Content-Length", + std::to_string(data.size())); +} + +TEST(HttpFileTest, DISABLED_BasicPut) { + FilePtr file(new HttpFile(HttpMethod::kPut, "https://httpbin.org/anything")); + ASSERT_TRUE(file); + ASSERT_TRUE(file->Open()); + + const std::string data = "abcd"; + + ASSERT_EQ(file->Write(data.data(), data.size()), + static_cast(data.size())); + ASSERT_TRUE(file->Flush()); + + auto json = HandleResponse(file); + ASSERT_TRUE(json); + ASSERT_TRUE(file.release()->Close()); + + ASSERT_JSON_STRING(json, "method", "PUT"); + ASSERT_JSON_STRING(json, "data", data); + ASSERT_JSON_STRING(json, "headers.Content-Type", "application/octet-stream"); + ASSERT_JSON_STRING(json, "headers.Content-Length", + std::to_string(data.size())); +} + +TEST(HttpFileTest, DISABLED_MultipleWrites) { + FilePtr file(new HttpFile(HttpMethod::kPut, "https://httpbin.org/anything")); + ASSERT_TRUE(file); + ASSERT_TRUE(file->Open()); + + const std::string data1 = "abcd"; + const std::string data2 = "efgh"; + const std::string data3 = "ijkl"; + const std::string data4 = "mnop"; + + ASSERT_EQ(file->Write(data1.data(), data1.size()), + static_cast(data1.size())); + ASSERT_EQ(file->Write(data2.data(), data2.size()), + static_cast(data2.size())); + ASSERT_EQ(file->Write(data3.data(), data3.size()), + static_cast(data3.size())); + ASSERT_EQ(file->Write(data4.data(), data4.size()), + static_cast(data4.size())); + ASSERT_TRUE(file->Flush()); + + auto json = HandleResponse(file); + ASSERT_TRUE(json); + ASSERT_TRUE(file.release()->Close()); + + ASSERT_JSON_STRING(json, "method", "PUT"); + ASSERT_JSON_STRING(json, "data", data1 + data2 + data3 + data4); + ASSERT_JSON_STRING(json, "headers.Content-Type", "application/octet-stream"); + ASSERT_JSON_STRING(json, "headers.Content-Length", + std::to_string(data1.size() + data2.size() + data3.size() + + data4.size())); +} + +// TODO: Test chunked uploads. Since we can only read the response, we have no +// way to detect if we are streaming the upload like we want. httpbin seems to +// populate the Content-Length even if we don't give it in the request. + +TEST(HttpFileTest, DISABLED_Error404) { + FilePtr file( + new HttpFile(HttpMethod::kGet, "https://httpbin.org/status/404")); + ASSERT_TRUE(file); + ASSERT_TRUE(file->Open()); + + // The site returns an empty response. + uint8_t buffer[1]; + ASSERT_EQ(file->Read(buffer, sizeof(buffer)), 0); + + auto status = file.release()->CloseWithStatus(); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.error_code(), error::HTTP_FAILURE); +} + +TEST(HttpFileTest, DISABLED_TimeoutTriggered) { + FilePtr file( + new HttpFile(HttpMethod::kGet, "https://httpbin.org/delay/8", "", {}, 1)); + ASSERT_TRUE(file); + ASSERT_TRUE(file->Open()); + + // Request should timeout; error is reported in Close/CloseWithStatus. + uint8_t buffer[1]; + ASSERT_EQ(file->Read(buffer, sizeof(buffer)), 0); + + auto status = file.release()->CloseWithStatus(); + ASSERT_FALSE(status.ok()); + ASSERT_EQ(status.error_code(), error::TIME_OUT); +} + +TEST(HttpFileTest, DISABLED_TimeoutNotTriggered) { + FilePtr file( + new HttpFile(HttpMethod::kGet, "https://httpbin.org/delay/1", "", {}, 5)); + ASSERT_TRUE(file); + ASSERT_TRUE(file->Open()); + + auto json = HandleResponse(file); + ASSERT_TRUE(json); + ASSERT_TRUE(file.release()->Close()); } } // namespace shaka diff --git a/packager/file/local_file.cc b/packager/file/local_file.cc index 2b2b9a7913..4120df1812 100644 --- a/packager/file/local_file.cc +++ b/packager/file/local_file.cc @@ -124,10 +124,6 @@ int64_t LocalFile::Read(void* buffer, uint64_t length) { } int64_t LocalFile::Write(const void* buffer, uint64_t length) { - - base::FilePath file_path(base::FilePath::FromUTF8Unsafe(file_name())); - VLOG(2) << "Writing to " << file_path.AsUTF8Unsafe() << ", length=" << length; - DCHECK(buffer != NULL); DCHECK(internal_file_ != NULL); size_t bytes_written = fwrite(buffer, sizeof(char), length, internal_file_); diff --git a/packager/media/base/buffer_writer.cc b/packager/media/base/buffer_writer.cc index d974a5a3be..d3c32fece3 100644 --- a/packager/media/base/buffer_writer.cc +++ b/packager/media/base/buffer_writer.cc @@ -70,8 +70,8 @@ void BufferWriter::AppendBuffer(const BufferWriter& buffer) { Status BufferWriter::WriteToFile(File* file) { DCHECK(file); DCHECK(!buf_.empty()); + size_t remaining_size = buf_.size(); - VLOG(1) << "BufferWriter::WriteToFile " << file->file_name() << " with " << remaining_size << " octets"; const uint8_t* buf = &buf_[0]; while (remaining_size > 0) { int64_t size_written = file->Write(buf, remaining_size); diff --git a/packager/media/base/http_key_fetcher.cc b/packager/media/base/http_key_fetcher.cc index 4acb11d69e..5cea5f3b5f 100644 --- a/packager/media/base/http_key_fetcher.cc +++ b/packager/media/base/http_key_fetcher.cc @@ -6,134 +6,22 @@ #include "packager/media/base/http_key_fetcher.h" -#include -#include - -#include "packager/base/logging.h" -#include "packager/base/strings/string_number_conversions.h" -#include "packager/base/strings/stringprintf.h" -#include "packager/base/synchronization/lock.h" - -DEFINE_bool(disable_peer_verification, - false, - "Disable peer verification. This is needed to talk to servers " - "without valid certificates."); +#include "packager/file/file_closer.h" namespace shaka { +namespace media { namespace { -const char kUserAgentString[] = "shaka-packager-http_fetcher/1.0"; + const char kSoapActionHeader[] = "SOAPAction: \"http://schemas.microsoft.com/DRM/2007/03/protocols/" "AcquirePackagingData\""; -const char kXmlContentTypeHeader[] = "Content-Type: text/xml; charset=UTF-8"; -const char kJsonContentTypeHeader[] = "Content-Type: application/json"; - -const int kMinLogLevelForCurlDebugFunction = 2; - -int CurlDebugFunction(CURL* /* handle */, - curl_infotype type, - const char* data, - size_t size, - void* /* userptr */) { - const char* type_text; - int log_level = kMinLogLevelForCurlDebugFunction; - switch (type) { - case CURLINFO_TEXT: - type_text = "== Info"; - log_level = kMinLogLevelForCurlDebugFunction + 1; - break; - case CURLINFO_HEADER_IN: - type_text = "<= Recv header"; - log_level = kMinLogLevelForCurlDebugFunction; - break; - case CURLINFO_HEADER_OUT: - type_text = "=> Send header"; - log_level = kMinLogLevelForCurlDebugFunction; - break; - case CURLINFO_DATA_IN: - type_text = "<= Recv data"; - log_level = kMinLogLevelForCurlDebugFunction + 1; - break; - case CURLINFO_DATA_OUT: - type_text = "=> Send data"; - log_level = kMinLogLevelForCurlDebugFunction + 1; - break; - case CURLINFO_SSL_DATA_IN: - type_text = "<= Recv SSL data"; - log_level = kMinLogLevelForCurlDebugFunction + 2; - break; - case CURLINFO_SSL_DATA_OUT: - type_text = "=> Send SSL data"; - log_level = kMinLogLevelForCurlDebugFunction + 2; - break; - default: - // Ignore other debug data. - return 0; - } - - VLOG(log_level) << "\n\n" - << type_text << " (0x" << std::hex << size << std::dec - << " bytes)" - << "\n" - << std::string(data, size) << "\nHex Format: \n" - << base::HexEncode(data, size); - return 0; -} - -// Scoped CURL implementation which cleans up itself when goes out of scope. -class ScopedCurl { - public: - ScopedCurl() { ptr_ = curl_easy_init(); } - ~ScopedCurl() { - if (ptr_) - curl_easy_cleanup(ptr_); - } - - CURL* get() { return ptr_; } - - private: - CURL* ptr_; - DISALLOW_COPY_AND_ASSIGN(ScopedCurl); -}; - -size_t AppendToString(char* ptr, size_t size, size_t nmemb, std::string* response) { - DCHECK(ptr); - DCHECK(response); - const size_t total_size = size * nmemb; - response->append(ptr, total_size); - return total_size; -} - -class LibCurlInitializer { - public: - LibCurlInitializer() : initialized_(false) { - base::AutoLock lock(lock_); - if (!initialized_) { - curl_global_init(CURL_GLOBAL_DEFAULT); - initialized_ = true; - } - } - - ~LibCurlInitializer() { - base::AutoLock lock(lock_); - if (initialized_) { - curl_global_cleanup(); - initialized_ = false; - } - } - - private: - base::Lock lock_; - bool initialized_; - - DISALLOW_COPY_AND_ASSIGN(LibCurlInitializer); -}; +const char kXmlContentType[] = "text/xml; charset=UTF-8"; +const char kJsonContentType[] = "application/json"; +constexpr size_t kBufferSize = 64 * 1024; } // namespace -namespace media { - HttpKeyFetcher::HttpKeyFetcher() : timeout_in_seconds_(0) {} HttpKeyFetcher::HttpKeyFetcher(uint32_t timeout_in_seconds) @@ -148,95 +36,45 @@ Status HttpKeyFetcher::FetchKeys(const std::string& url, } Status HttpKeyFetcher::Get(const std::string& path, std::string* response) { - return FetchInternal(GET, path, "", response); + return FetchInternal(HttpMethod::kGet, path, "", response); } Status HttpKeyFetcher::Post(const std::string& path, const std::string& data, std::string* response) { - return FetchInternal(POST, path, data, response); + return FetchInternal(HttpMethod::kPost, path, data, response); } Status HttpKeyFetcher::FetchInternal(HttpMethod method, const std::string& path, const std::string& data, std::string* response) { - DCHECK(method == GET || method == POST); - static LibCurlInitializer lib_curl_initializer; - - ScopedCurl scoped_curl; - CURL* curl = scoped_curl.get(); - if (!curl) { - LOG(ERROR) << "curl_easy_init() failed."; - return Status(error::HTTP_FAILURE, "curl_easy_init() failed."); - } - response->clear(); - - curl_easy_setopt(curl, CURLOPT_URL, path.c_str()); - curl_easy_setopt(curl, CURLOPT_USERAGENT, kUserAgentString); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout_in_seconds_); - curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, AppendToString); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, response); - - if (FLAGS_disable_peer_verification) - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); - - if (!client_cert_private_key_file_.empty() && !client_cert_file_.empty()) { - // Some PlayReady packaging servers only allow connects via HTTPS with - // client certificates. - curl_easy_setopt(curl, CURLOPT_SSLKEY, - client_cert_private_key_file_.data()); - if (!client_cert_private_key_password_.empty()) { - curl_easy_setopt(curl, CURLOPT_KEYPASSWD, - client_cert_private_key_password_.data()); - } - curl_easy_setopt(curl, CURLOPT_SSLKEYTYPE, "PEM"); - curl_easy_setopt(curl, CURLOPT_SSLCERTTYPE, "PEM"); - curl_easy_setopt(curl, CURLOPT_SSLCERT, client_cert_file_.data()); - } - if (!ca_file_.empty()) { - // Host validation needs to be off when using self-signed certificates. - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); - curl_easy_setopt(curl, CURLOPT_CAINFO, ca_file_.data()); - } - if (method == POST) { - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.c_str()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data.size()); - - curl_slist* chunk = nullptr; - if (data.find("soap:Envelope") != std::string::npos) { - // Adds Http headers for SOAP requests. - chunk = curl_slist_append(chunk, kXmlContentTypeHeader); - chunk = curl_slist_append(chunk, kSoapActionHeader); - } else { - chunk = curl_slist_append(chunk, kJsonContentTypeHeader); - } - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); + std::string content_type; + std::vector headers; + if (data.find("soap:Envelope") != std::string::npos) { + // Adds Http headers for SOAP requests. + content_type = kXmlContentType; + headers.push_back(kSoapActionHeader); + } else { + content_type = kJsonContentType; } - if (VLOG_IS_ON(kMinLogLevelForCurlDebugFunction)) { - curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, CurlDebugFunction); - curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + std::unique_ptr file( + new HttpFile(method, path, content_type, headers, timeout_in_seconds_)); + if (!file->Open()) { + return Status(error::INTERNAL_ERROR, "Cannot open URL"); } + file->Write(data.data(), data.size()); + file->Flush(); - CURLcode res = curl_easy_perform(curl); - if (res != CURLE_OK) { - std::string error_message = base::StringPrintf( - "curl_easy_perform() failed: %s.", curl_easy_strerror(res)); - if (res == CURLE_HTTP_RETURNED_ERROR) { - long response_code = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); - error_message += base::StringPrintf(" Response code: %ld.", response_code); - } - - LOG(ERROR) << error_message; - return Status( - res == CURLE_OPERATION_TIMEDOUT ? error::TIME_OUT : error::HTTP_FAILURE, - error_message); + while (true) { + char temp[kBufferSize]; + int64_t ret = file->Read(temp, kBufferSize); + if (ret <= 0) + break; + response->append(temp, ret); } - return Status::OK; + return file.release()->CloseWithStatus(); } } // namespace media diff --git a/packager/media/base/http_key_fetcher.h b/packager/media/base/http_key_fetcher.h index f62e1a4864..7f6ccbb2fc 100644 --- a/packager/media/base/http_key_fetcher.h +++ b/packager/media/base/http_key_fetcher.h @@ -3,15 +3,14 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -// -/// NOTE: Inclusion of this module will cause curl_global_init and -/// curl_global_cleanup to be called at static initialization / -/// deinitialization time. #ifndef PACKAGER_MEDIA_BASE_HTTP_KEY_FETCHER_H_ #define PACKAGER_MEDIA_BASE_HTTP_KEY_FETCHER_H_ +#include + #include "packager/base/compiler_specific.h" +#include "packager/file/http_file.h" #include "packager/media/base/key_fetcher.h" #include "packager/status.h" @@ -53,40 +52,11 @@ class HttpKeyFetcher : public KeyFetcher { const std::string& data, std::string* response); - /// Sets client certificate information for http requests. - /// @param cert_file absolute path to the client certificate. - /// @param private_key_file absolute path to the client certificate - /// private key file. - /// @param private_key_password private key password. - void SetClientCertInfo(const std::string& cert_file, - const std::string& private_key_file, - const std::string& private_key_password) { - client_cert_file_ = cert_file; - client_cert_private_key_file_ = private_key_file; - client_cert_private_key_password_ = private_key_password; - } - /// Sets the Certifiate Authority file information for http requests. - /// @param ca_file absolute path to the client certificate - void SetCaFile(const std::string& ca_file) { - ca_file_ = ca_file; - } - private: - enum HttpMethod { - GET, - POST, - PUT - }; - - // Internal implementation of HTTP functions, e.g. Get and Post. Status FetchInternal(HttpMethod method, const std::string& url, const std::string& data, std::string* response); const uint32_t timeout_in_seconds_; - std::string ca_file_; - std::string client_cert_file_; - std::string client_cert_private_key_file_; - std::string client_cert_private_key_password_; DISALLOW_COPY_AND_ASSIGN(HttpKeyFetcher); }; diff --git a/packager/media/base/playready_key_source.cc b/packager/media/base/playready_key_source.cc index 4734f77df3..125a1971dc 100644 --- a/packager/media/base/playready_key_source.cc +++ b/packager/media/base/playready_key_source.cc @@ -73,19 +73,6 @@ PlayReadyKeySource::PlayReadyKeySource(const std::string& server_url, encryption_key_(new EncryptionKey), server_url_(server_url) {} -PlayReadyKeySource::PlayReadyKeySource( - const std::string& server_url, - const std::string& client_cert_file, - const std::string& client_cert_private_key_file, - const std::string& client_cert_private_key_password, - ProtectionSystem protection_systems) - // PlayReady PSSH is retrived from PlayReady server response. - : encryption_key_(new EncryptionKey), - server_url_(server_url), - client_cert_file_(client_cert_file), - client_cert_private_key_file_(client_cert_private_key_file), - client_cert_private_key_password_(client_cert_private_key_password) {} - PlayReadyKeySource::~PlayReadyKeySource() = default; Status RetrieveTextInXMLElement(const std::string& element, @@ -161,14 +148,6 @@ Status PlayReadyKeySource::FetchKeysWithProgramIdentifier( const std::string& program_identifier) { std::unique_ptr encryption_key(new EncryptionKey); HttpKeyFetcher key_fetcher(kHttpFetchTimeout); - if (!client_cert_file_.empty() && !client_cert_private_key_file_.empty()) { - key_fetcher.SetClientCertInfo(client_cert_file_, - client_cert_private_key_file_, - client_cert_private_key_password_); - } - if (!ca_file_.empty()) { - key_fetcher.SetCaFile(ca_file_); - } std::string acquire_license_request = kAcquireLicenseRequest; base::ReplaceFirstSubstringAfterOffset( diff --git a/packager/media/base/playready_key_source.h b/packager/media/base/playready_key_source.h index 80fe4b17f8..036c5c25d6 100644 --- a/packager/media/base/playready_key_source.h +++ b/packager/media/base/playready_key_source.h @@ -25,19 +25,6 @@ class PlayReadyKeySource : public KeySource { /// be included. PlayReadyKeySource(const std::string& server_url, ProtectionSystem protection_systems); - /// Creates a new PlayReadyKeySource from the given packaging information. - /// @param server_url PlayReady packaging server url. - /// @param client_cert_file absolute path to a client certificate. - /// @param client_cert_private_key_file absolute path to the private file - /// for the client certificate. - /// @param client_cert_private_key_password password for the private key. - /// @param protection_systems is the enum indicating which PSSH should - /// be included. - PlayReadyKeySource(const std::string& server_url, - const std::string& client_cert_file, - const std::string& client_cert_private_key_file, - const std::string& client_cert_private_key_password, - ProtectionSystem protection_systems); ~PlayReadyKeySource() override; /// @name KeySource implementation overrides. @@ -61,10 +48,6 @@ class PlayReadyKeySource : public KeySource { static std::unique_ptr CreateFromKeyAndKeyId( const std::vector& key_id, const std::vector& key); - /// Sets the Certificate Authority file for validating self-signed certificates. - void SetCaFile(const std::string& ca_file) { - ca_file_ = ca_file; - } private: Status GetKeyInternal(); @@ -75,10 +58,6 @@ class PlayReadyKeySource : public KeySource { std::unique_ptr encryption_key_; std::string server_url_; - std::string ca_file_; - std::string client_cert_file_; - std::string client_cert_private_key_file_; - std::string client_cert_private_key_password_; DISALLOW_COPY_AND_ASSIGN(PlayReadyKeySource); }; diff --git a/packager/media/formats/packed_audio/packed_audio_writer.cc b/packager/media/formats/packed_audio/packed_audio_writer.cc index bfe89f16e3..0b17cf3872 100644 --- a/packager/media/formats/packed_audio/packed_audio_writer.cc +++ b/packager/media/formats/packed_audio/packed_audio_writer.cc @@ -107,7 +107,6 @@ Status PackedAudioWriter::WriteSegment(const std::string& segment_path, range.end = range.start + segment_buffer->Size() - 1; media_ranges_.subsegment_ranges.push_back(range); } else { - VLOG(2) << "PackedAudioWriter::WriteSegment: File::Open(" << segment_path << ")"; file.reset(File::Open(segment_path.c_str(), "w")); if (!file) { return Status(error::FILE_FAILURE,