Cleanup HttpFile and related PR.

This implements many of the comments made on the PR and cleans up those
files.

Closes #149

Change-Id: Ice73fe3c04a6f595da6986a4c070e50cb20f9435

docs/source/tutorials/http_upload.rst

@@ -9,10 +9,7 @@
     contributions are always welcome.
 
     The discussion about this feature currently happens at
-    `Add HTTP PUT output #149 <https://github.com/google/shaka-packager/issues/149>`_,
+    `Add HTTP PUT output #149 <https://github.com/google/shaka-packager/issues/149>`_.
-    its development on the
-    `http-upload <https://github.com/3QSDN/shaka-packager/tree/http-upload>`_ branch,
-    feel free to join us.
 
 ###########
 HTTP upload
@@ -46,15 +43,10 @@ Documentation
 
 Getting started
 ===============
-For enabling the HTTP upload transfer mode, please populate
+To enable the HTTP upload transfer mode, use ``https:`` file paths for any
-the ``segment_template`` attribute in the ``stream_descriptor``
+output files (e.g. ``segment_template``).
-parameter as well as the ``--hls_master_playlist_output`` parameter
-with appropriate URLs where the HTTP PUT requests will be issued to.
 
-You can also supply the ``--user_agent`` flag to specify a custom
+All HTTP requests will be declared as
-User-Agent string for all HTTP PUT requests.
-
-For pragmatic reasons, all HTTP requests will be declared as
 ``Content-Type: application/octet-stream``.
 
 Synopsis
@@ -90,17 +82,18 @@ Configure and run packager::
         --hls_playlist_type LIVE \
         --vmodule=http_file=1
 
-*******
+*********************
-HTTPS
+Client Authentication
-*******
+*********************
-If your ingest uses HTTPS and requires specific certificates, these
+If your server requires client authentication, you can add the following
-can be specified on the command line similar to how it's done for
+arguments to enable it:
-:doc:`playready`, with the following arguments:
 
-- ``--https_ca_file``: Absolute path to the Certificate Authority file for the server cert. PEM format.
+- ``--ca_file``: (optional) Absolute path to the Certificate Authority file for
-- ``--https_cert_file``: Absolute path to client certificate file.
+  the server cert. PEM format.
-- ``--https_cert_private_key_file``: Absolute path to the private Key file.
+- ``--client_cert_file``: Absolute path to client certificate file.
-- ``--https_cert_private_key_password``: Password to the private key file.
+- ``--client_cert_private_key_file``: Absolute path to the private Key file.
+- ``--client_cert_private_key_password``: (optional) Password to the private
+  key file.
 
 *******
 Backlog
@@ -108,20 +101,6 @@ Backlog
 Please note the HTTP upload feature still lacks some features
 probably important for production. Contributions are welcome!
 
-DASH
-====
-While the current implementation works for HLS_,
-we should also check DASH_.
-
-Basic Auth
-==========
-There's no support for authentication yet.
-
-HTTPS
-=====
-While there's already some code in place,
-HTTPS is currently not supported yet.
-
 HTTP DELETE
 ===========
 Nothing has be done to support this yet:
@@ -146,9 +125,9 @@ Miscellaneous
 - Make ``io_cache_size`` configurable?
 
 
-*******
+***************
-Backend
+Example Backend
-*******
+***************
 
 HTTP PUT file uploads to Nginx
 ==============================
@@ -230,8 +209,6 @@ Grab and run `httpd-reflector.py`_ to use it as a dummy HTTP sink::
 
 ----
 
-Have fun!
-
 .. _HLS: https://en.wikipedia.org/wiki/HTTP_Live_Streaming
 .. _DASH: https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP
 .. _M3U: https://en.wikipedia.org/wiki/M3U

packager/app/packager_main.cc

@@ -387,12 +387,6 @@ base::Optional<PackagingParams> GetPackagingParams() {
       PlayReadyEncryptionParams& playready = encryption_params.playready;
       playready.key_server_url = FLAGS_playready_server_url;
       playready.program_identifier = FLAGS_program_identifier;
-      playready.ca_file = FLAGS_ca_file;
-      playready.client_cert_file = FLAGS_client_cert_file;
-      playready.client_cert_private_key_file =
-          FLAGS_client_cert_private_key_file;
-      playready.client_cert_private_key_password =
-          FLAGS_client_cert_private_key_password;
       break;
     }
     case KeyProvider::kRawKey: {

packager/app/packager_util.cc

@@ -100,26 +100,8 @@ std::unique_ptr<KeySource> CreateEncryptionKeySource(
         }
         std::unique_ptr<PlayReadyKeySource> playready_key_source;
         // private_key_password is allowed to be empty for unencrypted key.
-        if (!playready.client_cert_file.empty() ||
-            !playready.client_cert_private_key_file.empty()) {
-          if (playready.client_cert_file.empty() ||
-              playready.client_cert_private_key_file.empty()) {
-            LOG(ERROR) << "Either PlayReady client_cert_file or "
-                          "client_cert_private_key_file is not set.";
-            return nullptr;
-          }
-          playready_key_source.reset(new PlayReadyKeySource(
-              playready.key_server_url, playready.client_cert_file,
-              playready.client_cert_private_key_file,
-              playready.client_cert_private_key_password,
-              encryption_params.protection_systems));
-        } else {
         playready_key_source.reset(new PlayReadyKeySource(
             playready.key_server_url, encryption_params.protection_systems));
-        }
-        if (!playready.ca_file.empty()) {
-          playready_key_source->SetCaFile(playready.ca_file);
-        }
         Status status = playready_key_source->FetchKeysWithProgramIdentifier(
             playready.program_identifier);
         if (!status.ok()) {

packager/app/playready_key_encryption_flags.cc

@@ -16,15 +16,6 @@ DEFINE_bool(enable_playready_encryption,
 DEFINE_string(playready_server_url, "", "PlayReady packaging server url.");
 DEFINE_string(program_identifier, "",
               "Program identifier for packaging request.");
-DEFINE_string(ca_file, "",
-              "Absolute path to the Certificate Authority file for the "
-              "server cert. PEM format");
-DEFINE_string(client_cert_file, "",
-              "Absolute path to client certificate file.");
-DEFINE_string(client_cert_private_key_file, "",
-              "Absolute path to the Private Key file.");
-DEFINE_string(client_cert_private_key_password, "",
-              "Password to the private key file.");
 
 namespace shaka {
 namespace {

packager/app/playready_key_encryption_flags.h

@@ -16,10 +16,6 @@
 DECLARE_bool(enable_playready_encryption);
 DECLARE_string(playready_server_url);
 DECLARE_string(program_identifier);
-DECLARE_string(ca_file);
-DECLARE_string(client_cert_file);
-DECLARE_string(client_cert_private_key_file);
-DECLARE_string(client_cert_private_key_password);
 
 namespace shaka {
 

packager/file/file.cc

@@ -100,11 +100,11 @@ File* CreateUdpFile(const char* file_name, const char* mode) {
 }
 
 File* CreateHttpsFile(const char* file_name, const char* mode) {
-  return new HttpFile(file_name, mode, true);
+  return new HttpFile(HttpMethod::kPut, std::string("https://") + file_name);
 }
 
 File* CreateHttpFile(const char* file_name, const char* mode) {
-  return new HttpFile(file_name, mode, false);
+  return new HttpFile(HttpMethod::kPut, std::string("http://") + file_name);
 }
 
 File* CreateMemoryFile(const char* file_name, const char* mode) {
@@ -137,7 +137,6 @@ base::StringPiece GetFileTypePrefix(base::StringPiece file_name) {
 
 const FileTypeInfo* GetFileTypeInfo(base::StringPiece file_name,
                                     base::StringPiece* real_file_name) {
-
   base::StringPiece file_type_prefix = GetFileTypePrefix(file_name);
   for (const FileTypeInfo& file_type : kFileTypeInfo) {
     if (file_type_prefix == file_type.type) {
@@ -297,7 +296,7 @@ bool File::WriteFileAtomically(const char* file_name,
 
 bool File::Copy(const char* from_file_name, const char* to_file_name) {
   std::string content;
-  VLOG(1) << "File::Copy from " << from_file_name << " to " << to_file_name;
+  VLOG(2) << "File::Copy from " << from_file_name << " to " << to_file_name;
   if (!ReadFileToString(from_file_name, &content)) {
     LOG(ERROR) << "Failed to open file " << from_file_name;
     return false;
@@ -341,7 +340,8 @@ int64_t File::CopyFile(File* source, File* destination, int64_t max_copy) {
   if (max_copy < 0)
     max_copy = std::numeric_limits<int64_t>::max();
 
-  VLOG(1) << "File::CopyFile from " << source->file_name() << " to " << destination->file_name();
+  VLOG(2) << "File::CopyFile from " << source->file_name() << " to "
+          << destination->file_name();
 
   const int64_t kBufferSize = 0x40000;  // 256KB.
   std::unique_ptr<uint8_t[]> buffer(new uint8_t[kBufferSize]);

packager/file/http_file.cc

@@ -1,4 +1,4 @@
-// Copyright 2018 Google Inc. All rights reserved.
+// Copyright 2020 Google LLC. All rights reserved.
 //
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file or at
@@ -6,49 +6,113 @@
 
 #include "packager/file/http_file.h"
 
+#include <curl/curl.h>
 #include <gflags/gflags.h>
+
 #include "packager/base/bind.h"
 #include "packager/base/files/file_util.h"
 #include "packager/base/logging.h"
 #include "packager/base/strings/string_number_conversions.h"
 #include "packager/base/strings/stringprintf.h"
-#include "packager/base/synchronization/lock.h"
 #include "packager/base/threading/worker_pool.h"
 
-DEFINE_int32(libcurl_verbosity, 0,
+DEFINE_string(ca_file,
-             "Set verbosity level for libcurl.");
+              "",
-DEFINE_string(user_agent, "",
-              "Set a custom User-Agent string for HTTP ingest.");
-DEFINE_string(https_ca_file, "",
               "Absolute path to the Certificate Authority file for the "
               "server cert. PEM format");
-DEFINE_string(https_cert_file, "",
+DEFINE_string(client_cert_file,
+              "",
               "Absolute path to client certificate file.");
-DEFINE_string(https_cert_private_key_file, "",
+DEFINE_string(client_cert_private_key_file,
-              "Absolute path to the private Key file.");
+              "",
-DEFINE_string(https_cert_private_key_password, "",
+              "Absolute path to the Private Key file.");
+DEFINE_string(client_cert_private_key_password,
+              "",
               "Password to the private key file.");
+DEFINE_bool(disable_peer_verification,
+            false,
+            "Disable peer verification. This is needed to talk to servers "
+            "without valid certificates.");
 DECLARE_uint64(io_cache_size);
 
 namespace shaka {
 
-// curl_ primitives stolen from `http_key_fetcher.cc`.
 namespace {
 
-const char kUserAgentString[] = "shaka-packager-uploader/0.1";
+constexpr const char* kBinaryContentType = "application/octet-stream";
+constexpr const char* kUserAgent = "shaka-packager-http-fetch/1.0";
+constexpr const int kMinLogLevelForCurlDebugFunction = 2;
 
-size_t AppendToString(char* ptr,
+size_t CurlWriteCallback(char* buffer, size_t size, size_t nmemb, void* user) {
-                      size_t size,
+  IoCache* cache = reinterpret_cast<IoCache*>(user);
-                      size_t nmemb,
+  size_t length = cache->Write(buffer, size * nmemb);
-                      std::string* response) {
+  VLOG(3) << "CurlWriteCallback length=" << length;
-  DCHECK(ptr);
+  return length;
-  DCHECK(response);
-  const size_t total_size = size * nmemb;
-  response->append(ptr, total_size);
-  return total_size;
 }
 
-}  // namespace
+size_t CurlReadCallback(char* buffer, size_t size, size_t nitems, void* user) {
+  IoCache* cache = reinterpret_cast<IoCache*>(user);
+  size_t length = cache->Read(buffer, size * nitems);
+  VLOG(3) << "CurlRead length=" << length;
+  return length;
+}
+
+int CurlDebugCallback(CURL* /* handle */,
+                      curl_infotype type,
+                      const char* data,
+                      size_t size,
+                      void* /* userptr */) {
+  const char* type_text;
+  int log_level;
+  bool in_hex;
+  switch (type) {
+    case CURLINFO_TEXT:
+      type_text = "== Info";
+      log_level = kMinLogLevelForCurlDebugFunction + 1;
+      in_hex = false;
+      break;
+    case CURLINFO_HEADER_IN:
+      type_text = "<= Recv header";
+      log_level = kMinLogLevelForCurlDebugFunction;
+      in_hex = false;
+      break;
+    case CURLINFO_HEADER_OUT:
+      type_text = "=> Send header";
+      log_level = kMinLogLevelForCurlDebugFunction;
+      in_hex = false;
+      break;
+    case CURLINFO_DATA_IN:
+      type_text = "<= Recv data";
+      log_level = kMinLogLevelForCurlDebugFunction + 1;
+      in_hex = true;
+      break;
+    case CURLINFO_DATA_OUT:
+      type_text = "=> Send data";
+      log_level = kMinLogLevelForCurlDebugFunction + 1;
+      in_hex = true;
+      break;
+    case CURLINFO_SSL_DATA_IN:
+      type_text = "<= Recv SSL data";
+      log_level = kMinLogLevelForCurlDebugFunction + 2;
+      in_hex = true;
+      break;
+    case CURLINFO_SSL_DATA_OUT:
+      type_text = "=> Send SSL data";
+      log_level = kMinLogLevelForCurlDebugFunction + 2;
+      in_hex = true;
+      break;
+    default:
+      // Ignore other debug data.
+      return 0;
+  }
+
+  VLOG(log_level) << "\n\n"
+                  << type_text << " (0x" << std::hex << size << std::dec
+                  << " bytes)\n"
+                  << (in_hex ? base::HexEncode(data, size)
+                             : std::string(data, size));
+  return 0;
+}
 
 class LibCurlInitializer {
  public:
@@ -60,309 +124,221 @@ class LibCurlInitializer {
     curl_global_cleanup();
   }
 
- private:
+  LibCurlInitializer(const LibCurlInitializer&) = delete;
-  DISALLOW_COPY_AND_ASSIGN(LibCurlInitializer);
+  LibCurlInitializer& operator=(const LibCurlInitializer&) = delete;
 };
 
-/// Create a HTTP/HTTPS client
+template <typename List>
-HttpFile::HttpFile(const char* file_name, const char* mode, bool https)
+bool AppendHeader(const std::string& header, List* list) {
-    : File(file_name),
+  auto* temp = curl_slist_append(list->get(), header.c_str());
-      file_mode_(mode),
+  if (temp) {
-      user_agent_(FLAGS_user_agent),
+    list->release();  // Don't free old list since it's part of the new one.
-      ca_file_(FLAGS_https_ca_file),
+    list->reset(temp);
-      cert_file_(FLAGS_https_cert_file),
+    return true;
-      cert_private_key_file_(FLAGS_https_cert_private_key_file),
-      cert_private_key_pass_(FLAGS_https_cert_private_key_password),
-      timeout_in_seconds_(0),
-      cache_(FLAGS_io_cache_size),
-      scoped_curl(curl_easy_init(), &curl_easy_cleanup),
-      task_exit_event_(base::WaitableEvent::ResetPolicy::AUTOMATIC,
-                       base::WaitableEvent::InitialState::NOT_SIGNALED) {
-  if (https) {
-    resource_url_ = "https://" + std::string(file_name);
   } else {
-    resource_url_ = "http://" + std::string(file_name);
+    return false;
-  }
-
-  static LibCurlInitializer lib_curl_initializer;
-
-  // Setup libcurl scope
-  if (!scoped_curl.get()) {
-    LOG(ERROR) << "curl_easy_init() failed.";
-    // return Status(error::HTTP_FAILURE, "curl_easy_init() failed.");
-    delete this;
   }
 }
 
-HttpFile::HttpFile(const char* file_name, const char* mode)
+}  // namespace
-    : HttpFile(file_name, mode, false)
+
-{}
+HttpFile::HttpFile(HttpMethod method, const std::string& url)
+    : HttpFile(method, url, kBinaryContentType, {}, 0) {}
+
+HttpFile::HttpFile(HttpMethod method,
+                   const std::string& url,
+                   const std::string& upload_content_type,
+                   const std::vector<std::string>& headers,
+                   uint32_t timeout_in_seconds)
+    : File(url.c_str()),
+      url_(url),
+      upload_content_type_(upload_content_type),
+      timeout_in_seconds_(timeout_in_seconds),
+      method_(method),
+      download_cache_(FLAGS_io_cache_size),
+      upload_cache_(FLAGS_io_cache_size),
+      curl_(curl_easy_init()),
+      status_(Status::OK),
+      task_exit_event_(base::WaitableEvent::ResetPolicy::MANUAL,
+                       base::WaitableEvent::InitialState::NOT_SIGNALED) {
+  static LibCurlInitializer lib_curl_initializer;
+
+  // We will have at least one header, so use a null header to signal error
+  // to Open.
+
+  // Don't wait for 100-Continue.
+  std::unique_ptr<curl_slist, CurlDelete> temp_headers;
+  if (!AppendHeader("Expect:", &temp_headers))
+    return;
+  if (!upload_content_type.empty() &&
+      !AppendHeader("Content-Type: " + upload_content_type_, &temp_headers)) {
+    return;
+  }
+  if (method != HttpMethod::kGet &&
+      !AppendHeader("Transfer-Encoding: chunked", &temp_headers)) {
+    return;
+  }
+  for (const auto& item : headers) {
+    if (!AppendHeader(item, &temp_headers)) {
+      return;
+    }
+  }
+  request_headers_ = std::move(temp_headers);
+}
 
-// Destructor
 HttpFile::~HttpFile() {}
 
 bool HttpFile::Open() {
+  VLOG(2) << "Opening " << url_;
 
-  VLOG(1) << "Opening " << resource_url() << " with file mode \"" << file_mode_ << "\".";
+  if (!curl_ || !request_headers_) {
-
+    LOG(ERROR) << "curl_easy_init() failed.";
-  // Ignore read requests as they would truncate the target
-  // file by propagating as zero-length PUT requests.
-  // See also https://github.com/google/shaka-packager/issues/149#issuecomment-437203701
-  if (file_mode_ == "r") {
-    VLOG(1) << "HttpFile only supports write mode, skipping further operations";
-    task_exit_event_.Signal();
     return false;
   }
-
+  // TODO: Try to connect initially so we can return connection error here.
-  // Run progressive upload in separate thread.
-  base::WorkerPool::PostTask(
-      FROM_HERE, base::Bind(&HttpFile::CurlPut, base::Unretained(this)),
-      true  // task_is_slow
-  );
-
-  return true;
-}
-
-void HttpFile::CurlPut() {
-  // Setup libcurl handle with HTTP PUT upload transfer mode.
-  std::string request_body;
-  Request(PUT, resource_url(), request_body, &response_body_);
-}
-
-bool HttpFile::Close() {
-  VLOG(1) << "Closing " << resource_url() << ".";
-  cache_.Close();
-  task_exit_event_.Wait();
-  delete this;
-  return true;
-}
-
-int64_t HttpFile::Read(void* buffer, uint64_t length) {
-  LOG(WARNING) << "HttpFile does not support Read().";
-  return -1;
-}
-
-int64_t HttpFile::Write(const void* buffer, uint64_t length) {
-  std::string url = resource_url();
-
-  VLOG(2) << "Writing to " << url << ", length=" << length;
 
   // TODO: Implement retrying with exponential backoff, see
   // "widevine_key_source.cc"
-  Status status;
 
-  uint64_t bytes_written = cache_.Write(buffer, length);
+  base::WorkerPool::PostTask(
-  VLOG(3) << "PUT CHUNK bytes_written: " << bytes_written;
+      FROM_HERE, base::Bind(&HttpFile::ThreadMain, base::Unretained(this)),
-  return bytes_written;
+      /* task_is_slow= */ true);
 
-  // Debugging based on response status
+  return true;
-  /*
+}
-  if (status.ok()) {
-    VLOG(1) << "Writing chunk succeeded";
 
-  } else {
+Status HttpFile::CloseWithStatus() {
-    VLOG(1) << "Writing chunk failed";
+  VLOG(2) << "Closing " << url_;
-    if (!response_body.empty()) {
+  // Close the cache first so the thread will finish uploading. Otherwise it
-      VLOG(2) << "Response:\n" << response_body;
+  // will wait for more data forever.
-    }
+  download_cache_.Close();
-  }
+  upload_cache_.Close();
-  */
+  task_exit_event_.Wait();
 
-  // Always signal success to the downstream pipeline
+  const Status result = status_;
-  return length;
+  LOG_IF(ERROR, !result.ok()) << "HttpFile request failed: " << result;
+  delete this;
+  return result;
+}
+
+bool HttpFile::Close() {
+  return CloseWithStatus().ok();
+}
+
+int64_t HttpFile::Read(void* buffer, uint64_t length) {
+  VLOG(2) << "Reading from " << url_ << ", length=" << length;
+  return download_cache_.Read(buffer, length);
+}
+
+int64_t HttpFile::Write(const void* buffer, uint64_t length) {
+  VLOG(2) << "Writing to " << url_ << ", length=" << length;
+  return upload_cache_.Write(buffer, length);
 }
 
 int64_t HttpFile::Size() {
-  VLOG(1) << "HttpFile does not support Size().";
+  LOG(ERROR) << "HttpFile does not support Size().";
   return -1;
 }
 
 bool HttpFile::Flush() {
-  // Do nothing on Flush.
+  upload_cache_.Close();
   return true;
 }
 
 bool HttpFile::Seek(uint64_t position) {
-  VLOG(1) << "HttpFile does not support Seek().";
+  LOG(ERROR) << "HttpFile does not support Seek().";
   return false;
 }
 
 bool HttpFile::Tell(uint64_t* position) {
-  VLOG(1) << "HttpFile does not support Tell().";
+  LOG(ERROR) << "HttpFile does not support Tell().";
   return false;
 }
 
-// Perform HTTP request
+void HttpFile::CurlDelete::operator()(CURL* curl) {
-Status HttpFile::Request(HttpMethod http_method,
+  curl_easy_cleanup(curl);
-                         const std::string& url,
+}
-                         const std::string& data,
-                         std::string* response) {
 
-  // TODO: Sanity checks.
+void HttpFile::CurlDelete::operator()(curl_slist* headers) {
-  // DCHECK(http_method == GET || http_method == POST);
+  curl_slist_free_all(headers);
+}
 
-  VLOG(1) << "Sending request to URL " << url;
+void HttpFile::SetupRequest() {
+  auto* curl = curl_.get();
 
-  // Setup HTTP method and libcurl options
+  switch (method_) {
-  SetupRequestBase(http_method, url, response);
+    case HttpMethod::kGet:
-
+      curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
-  // Setup HTTP request headers and body
+      break;
-  SetupRequestData(data);
+    case HttpMethod::kPost:
-
+      curl_easy_setopt(curl, CURLOPT_POST, 1L);
-  // Perform HTTP request
+      break;
-  CURLcode res = curl_easy_perform(scoped_curl.get());
+    case HttpMethod::kPut:
-
+      curl_easy_setopt(curl, CURLOPT_PUT, 1L);
-  // Assume successful request
+      break;
-  Status status = Status::OK;
-
-  // Handle request failure
-  if (res != CURLE_OK) {
-    std::string method_text = method_as_text(http_method);
-    std::string error_message = base::StringPrintf(
-        "%s request for %s failed. Reason: %s.", method_text.c_str(),
-        url.c_str(), curl_easy_strerror(res));
-    if (res == CURLE_HTTP_RETURNED_ERROR) {
-      long response_code = 0;
-      curl_easy_getinfo(scoped_curl.get(), CURLINFO_RESPONSE_CODE, &response_code);
-      error_message +=
-          base::StringPrintf(" Response code: %ld.", response_code);
   }
 
-    // Signal error to logfile
+  curl_easy_setopt(curl, CURLOPT_URL, url_.c_str());
-    LOG(ERROR) << error_message;
+  curl_easy_setopt(curl, CURLOPT_USERAGENT, kUserAgent);
+  curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout_in_seconds_);
+  curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
+  curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &CurlWriteCallback);
+  curl_easy_setopt(curl, CURLOPT_WRITEDATA, &download_cache_);
+  if (method_ != HttpMethod::kGet) {
+    curl_easy_setopt(curl, CURLOPT_READFUNCTION, &CurlReadCallback);
+    curl_easy_setopt(curl, CURLOPT_READDATA, &upload_cache_);
+  }
 
-    // Signal error to caller
+  curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers_.get());
-    status = Status(
+
+  if (FLAGS_disable_peer_verification)
+    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
+
+  // Client authentication
+  if (!FLAGS_client_cert_private_key_file.empty() &&
+      !FLAGS_client_cert_file.empty()) {
+    curl_easy_setopt(curl, CURLOPT_SSLKEY,
+                     FLAGS_client_cert_private_key_file.data());
+    curl_easy_setopt(curl, CURLOPT_SSLCERT, FLAGS_client_cert_file.data());
+    curl_easy_setopt(curl, CURLOPT_SSLKEYTYPE, "PEM");
+    curl_easy_setopt(curl, CURLOPT_SSLCERTTYPE, "PEM");
+
+    if (!FLAGS_client_cert_private_key_password.empty()) {
+      curl_easy_setopt(curl, CURLOPT_KEYPASSWD,
+                       FLAGS_client_cert_private_key_password.data());
+    }
+  }
+  if (!FLAGS_ca_file.empty()) {
+    curl_easy_setopt(curl, CURLOPT_CAINFO, FLAGS_ca_file.data());
+  }
+
+  if (VLOG_IS_ON(kMinLogLevelForCurlDebugFunction)) {
+    curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, CurlDebugCallback);
+    curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
+  }
+}
+
+void HttpFile::ThreadMain() {
+  SetupRequest();
+
+  CURLcode res = curl_easy_perform(curl_.get());
+  if (res != CURLE_OK) {
+    std::string error_message = curl_easy_strerror(res);
+    if (res == CURLE_HTTP_RETURNED_ERROR) {
+      long response_code = 0;
+      curl_easy_getinfo(curl_.get(), CURLINFO_RESPONSE_CODE, &response_code);
+      error_message +=
+          base::StringPrintf(", response code: %ld.", response_code);
+    }
+
+    status_ = Status(
         res == CURLE_OPERATION_TIMEDOUT ? error::TIME_OUT : error::HTTP_FAILURE,
         error_message);
   }
 
-  // Signal task completion
+  download_cache_.Close();
   task_exit_event_.Signal();
-
-  // Return request status to caller
-  return status;
-}
-
-// Configure curl_ handle with reasonable defaults
-void HttpFile::SetupRequestBase(HttpMethod http_method,
-                                const std::string& url,
-                                std::string* response) {
-  response->clear();
-
-  // Configure HTTP request method/verb
-  switch (http_method) {
-    case GET:
-      curl_easy_setopt(scoped_curl.get(), CURLOPT_HTTPGET, 1L);
-      break;
-    case POST:
-      curl_easy_setopt(scoped_curl.get(), CURLOPT_POST, 1L);
-      break;
-    case PUT:
-      curl_easy_setopt(scoped_curl.get(), CURLOPT_PUT, 1L);
-      break;
-    case PATCH:
-      curl_easy_setopt(scoped_curl.get(), CURLOPT_CUSTOMREQUEST, "PATCH");
-      break;
-  }
-
-  // Configure HTTP request
-  curl_easy_setopt(scoped_curl.get(), CURLOPT_URL, url.c_str());
-
-  if (user_agent_.empty()) {
-    curl_easy_setopt(scoped_curl.get(), CURLOPT_USERAGENT, kUserAgentString);
-  } else {
-    curl_easy_setopt(scoped_curl.get(), CURLOPT_USERAGENT, user_agent_.data());
-  }
-
-  curl_easy_setopt(scoped_curl.get(), CURLOPT_TIMEOUT, timeout_in_seconds_);
-  curl_easy_setopt(scoped_curl.get(), CURLOPT_FAILONERROR, 1L);
-  curl_easy_setopt(scoped_curl.get(), CURLOPT_FOLLOWLOCATION, 1L);
-  curl_easy_setopt(scoped_curl.get(), CURLOPT_WRITEFUNCTION, AppendToString);
-  curl_easy_setopt(scoped_curl.get(), CURLOPT_WRITEDATA, response);
-
-  // HTTPS
-  if (!cert_private_key_file_.empty() && !cert_file_.empty()) {
-    curl_easy_setopt(scoped_curl.get(), CURLOPT_SSLKEY,
-                     cert_private_key_file_.data());
-
-    if (!cert_private_key_pass_.empty()) {
-      curl_easy_setopt(scoped_curl.get(), CURLOPT_KEYPASSWD,
-                       cert_private_key_pass_.data());
-    }
-
-    curl_easy_setopt(scoped_curl.get(), CURLOPT_SSLKEYTYPE, "PEM");
-    curl_easy_setopt(scoped_curl.get(), CURLOPT_SSLCERTTYPE, "PEM");
-    curl_easy_setopt(scoped_curl.get(), CURLOPT_SSLCERT, cert_file_.data());
-  }
-  if (!ca_file_.empty()) {
-    // Host validation needs to be off when using self-signed certificates.
-    curl_easy_setopt(scoped_curl.get(), CURLOPT_SSL_VERIFYHOST, 0L);
-    curl_easy_setopt(scoped_curl.get(), CURLOPT_CAINFO, ca_file_.data());
-  }
-
-  // Propagate log level indicated by "--libcurl_verbosity" to libcurl.
-  curl_easy_setopt(scoped_curl.get(), CURLOPT_VERBOSE, FLAGS_libcurl_verbosity);
-
-}
-
-// https://ec.haxx.se/callback-read.html
-size_t read_callback(char* buffer, size_t size, size_t nitems, void* stream) {
-  VLOG(3) << "read_callback";
-
-  // Cast stream back to what is actually is
-  // IoCache* cache = reinterpret_cast<IoCache*>(stream);
-  IoCache* cache = (IoCache*)stream;
-  VLOG(3) << "read_callback, cache: " << cache;
-
-  // Copy cache content into buffer
-  size_t length = cache->Read(buffer, size * nitems);
-  VLOG(3) << "read_callback, length: " << length << "; buffer: " << buffer;
-  return length;
-}
-
-// Configure curl_ handle for HTTP PUT upload
-void HttpFile::SetupRequestData(const std::string& data) {
-
-  // TODO: Sanity checks.
-  // if (method == POST || method == PUT || method == PATCH)
-
-  // Build list of HTTP request headers.
-  struct curl_slist* headers = nullptr;
-
-  headers = curl_slist_append(headers, "Content-Type: application/octet-stream");
-  headers = curl_slist_append(headers, "Transfer-Encoding: chunked");
-
-  // Don't stop on 200 OK responses.
-  headers = curl_slist_append(headers, "Expect:");
-
-  // Enable progressive upload with chunked transfer encoding.
-  curl_easy_setopt(scoped_curl.get(), CURLOPT_READFUNCTION, read_callback);
-  curl_easy_setopt(scoped_curl.get(), CURLOPT_READDATA, &cache_);
-  curl_easy_setopt(scoped_curl.get(), CURLOPT_UPLOAD, 1L);
-
-  // Add HTTP request headers.
-  curl_easy_setopt(scoped_curl.get(), CURLOPT_HTTPHEADER, headers);
-}
-
-// Return HTTP request method (verb) as string
-std::string HttpFile::method_as_text(HttpMethod method) {
-  std::string method_text = "UNKNOWN";
-  switch (method) {
-    case GET:
-      method_text = "GET";
-      break;
-    case POST:
-      method_text = "POST";
-      break;
-    case PUT:
-      method_text = "PUT";
-      break;
-    case PATCH:
-      method_text = "PATCH";
-      break;
-  }
-  return method_text;
 }
 
 }  // namespace shaka

packager/file/http_file.h

@@ -1,4 +1,4 @@
-// Copyright 2018 Google Inc. All rights reserved.
+// Copyright 2020 Google LLC. All rights reserved.
 //
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file or at
@@ -7,35 +7,47 @@
 #ifndef PACKAGER_FILE_HTTP_H_
 #define PACKAGER_FILE_HTTP_H_
 
-#include <curl/curl.h>
 #include <memory>
+#include <string>
 
-#include "packager/base/compiler_specific.h"
 #include "packager/base/synchronization/waitable_event.h"
 #include "packager/file/file.h"
 #include "packager/file/io_cache.h"
 #include "packager/status.h"
 
-namespace shaka {
+typedef void CURL;
-using ScopedCurl = std::unique_ptr<CURL, decltype(&curl_easy_cleanup)>;
+struct curl_slist;
 
-/// HttpFile delegates write calls to HTTP PUT requests.
+namespace shaka {
+
+enum class HttpMethod {
+  kGet,
+  kPost,
+  kPut,
+};
+
+/// HttpFile reads or writes network requests.
 ///
-/// About how to use this, please visit the corresponding documentation [1,2].
+/// Note that calling Flush will indicate EOF for the upload and no more can be
+/// uploaded.
+///
+/// About how to use this, please visit the corresponding documentation [1].
 ///
 /// [1] https://google.github.io/shaka-packager/html/tutorials/http_upload.html
-/// [2]
-/// https://github.com/3QSDN/shaka-packager/blob/http-upload/docs/source/tutorials/http_upload.rst
 ///
 class HttpFile : public File {
  public:
+  HttpFile(HttpMethod method, const std::string& url);
+  HttpFile(HttpMethod method,
+           const std::string& url,
+           const std::string& upload_content_type,
+           const std::vector<std::string>& headers,
+           uint32_t timeout_in_seconds);
 
-  /// Create a HTTP client
+  HttpFile(const HttpFile&) = delete;
-  /// @param file_name contains the url of the resource to be accessed.
+  HttpFile& operator=(const HttpFile&) = delete;
-  ///        Note that the file type prefix should be stripped off already.
+
-  /// @param mode contains file access mode. Implementation dependent.
+  Status CloseWithStatus();
-  HttpFile(const char* file_name, const char* mode, bool https);
-  HttpFile(const char* file_name, const char* mode);
 
   /// @name File implementation overrides.
   /// @{
@@ -46,56 +58,31 @@ class HttpFile : public File {
   bool Flush() override;
   bool Seek(uint64_t position) override;
   bool Tell(uint64_t* position) override;
+  bool Open() override;
   /// @}
 
-  /// @return The full resource url
-  const std::string& resource_url() const { return resource_url_; }
-
  protected:
-  // Destructor
   ~HttpFile() override;
 
-  bool Open() override;
-
  private:
-  enum HttpMethod {
+  struct CurlDelete {
-    GET,
+    void operator()(CURL* curl);
-    POST,
+    void operator()(curl_slist* headers);
-    PUT,
-    PATCH,
   };
 
-  HttpFile(const HttpFile&) = delete;
+  void SetupRequest();
-  HttpFile& operator=(const HttpFile&) = delete;
+  void ThreadMain();
-
-  // Internal implementation of HTTP functions, e.g. Get and Post.
-  Status Request(HttpMethod http_method,
-                 const std::string& url,
-                 const std::string& data,
-                 std::string* response);
-
-  void SetupRequestBase(HttpMethod http_method,
-                        const std::string& url,
-                        std::string* response);
-
-  void SetupRequestData(const std::string& data);
-
-  void CurlPut();
-
-  std::string method_as_text(HttpMethod method);
-
-  std::string file_mode_;
-  std::string resource_url_;
-  std::string user_agent_;
-  std::string ca_file_;
-  std::string cert_file_;
-  std::string cert_private_key_file_;
-  std::string cert_private_key_pass_;
 
+  const std::string url_;
+  const std::string upload_content_type_;
   const uint32_t timeout_in_seconds_;
-  IoCache cache_;
+  const HttpMethod method_;
-  ScopedCurl scoped_curl;
+  IoCache download_cache_;
-  std::string response_body_;
+  IoCache upload_cache_;
+  std::unique_ptr<CURL, CurlDelete> curl_;
+  // The headers need to remain alive for the duration of the request.
+  std::unique_ptr<curl_slist, CurlDelete> request_headers_;
+  Status status_;
 
   // Signaled when the "curl easy perform" task completes.
   base::WaitableEvent task_exit_event_;

packager/file/http_file_unittest.cc

@@ -1,25 +1,202 @@
+// Copyright 2020 Google LLC. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd
+
 #include "packager/file/http_file.h"
+
 #include <gtest/gtest.h>
+
 #include <memory>
+#include <vector>
+
+#include "packager/base/json/json_reader.h"
+#include "packager/base/values.h"
 #include "packager/file/file.h"
 #include "packager/file/file_closer.h"
 
+#define ASSERT_JSON_STRING(json, key, value)        \
+  do {                                              \
+    std::string actual;                             \
+    ASSERT_TRUE((json)->GetString((key), &actual)); \
+    ASSERT_EQ(actual, (value));                     \
+  } while (false)
+
 namespace shaka {
+
 namespace {
 
-const uint8_t kWriteBuffer[] = {1, 2, 3, 4, 5, 6, 7, 8};
+using FilePtr = std::unique_ptr<HttpFile, FileCloser>;
-const int64_t kWriteBufferSize = sizeof(kWriteBuffer);
+
+std::unique_ptr<base::DictionaryValue> HandleResponse(const FilePtr& file) {
+  std::string result;
+  while (true) {
+    char buffer[64 * 1024];
+    auto ret = file->Read(buffer, sizeof(buffer));
+    if (ret < 0)
+      return nullptr;
+    if (ret == 0)
+      break;
+    result.append(buffer, buffer + ret);
+  }
+  VLOG(1) << "Response:\n" << result;
+
+  auto value = base::JSONReader::Read(result);
+  if (!value || !value->IsType(base::Value::TYPE_DICTIONARY))
+    return nullptr;
+  return std::unique_ptr<base::DictionaryValue>{
+      static_cast<base::DictionaryValue*>(value.release())};
+}
 
 }  // namespace
 
-class HttpFileTest : public testing::Test {};
+TEST(HttpFileTest, DISABLED_BasicGet) {
+  FilePtr file(new HttpFile(HttpMethod::kGet, "https://httpbin.org/anything"));
+  ASSERT_TRUE(file);
+  ASSERT_TRUE(file->Open());
 
-TEST_F(HttpFileTest, PutChunkedTranser) {
+  auto json = HandleResponse(file);
-  std::unique_ptr<File, FileCloser> writer(
+  ASSERT_TRUE(json);
-      File::Open("http://127.0.0.1:8080/test_out", "w"));
+  ASSERT_TRUE(file.release()->Close());
-  ASSERT_TRUE(writer);
+  ASSERT_JSON_STRING(json, "method", "GET");
-  ASSERT_EQ(kWriteBufferSize, writer->Write(kWriteBuffer, kWriteBufferSize));
+}
-  writer.release()->Close();
+
+TEST(HttpFileTest, DISABLED_CustomHeaders) {
+  std::vector<std::string> headers{"Host: foo", "X-My-Header: Something"};
+  FilePtr file(new HttpFile(HttpMethod::kGet, "https://httpbin.org/anything",
+                            "", headers, 0));
+  ASSERT_TRUE(file);
+  ASSERT_TRUE(file->Open());
+
+  auto json = HandleResponse(file);
+  ASSERT_TRUE(json);
+  ASSERT_TRUE(file.release()->Close());
+
+  ASSERT_JSON_STRING(json, "method", "GET");
+  ASSERT_JSON_STRING(json, "headers.Host", "foo");
+  ASSERT_JSON_STRING(json, "headers.X-My-Header", "Something");
+}
+
+TEST(HttpFileTest, DISABLED_BasicPost) {
+  FilePtr file(new HttpFile(HttpMethod::kPost, "https://httpbin.org/anything"));
+  ASSERT_TRUE(file);
+  ASSERT_TRUE(file->Open());
+
+  const std::string data = "abcd";
+
+  ASSERT_EQ(file->Write(data.data(), data.size()),
+            static_cast<int64_t>(data.size()));
+  ASSERT_TRUE(file->Flush());
+
+  auto json = HandleResponse(file);
+  ASSERT_TRUE(json);
+  ASSERT_TRUE(file.release()->Close());
+
+  ASSERT_JSON_STRING(json, "method", "POST");
+  ASSERT_JSON_STRING(json, "data", data);
+  ASSERT_JSON_STRING(json, "headers.Content-Type", "application/octet-stream");
+  ASSERT_JSON_STRING(json, "headers.Content-Length",
+                     std::to_string(data.size()));
+}
+
+TEST(HttpFileTest, DISABLED_BasicPut) {
+  FilePtr file(new HttpFile(HttpMethod::kPut, "https://httpbin.org/anything"));
+  ASSERT_TRUE(file);
+  ASSERT_TRUE(file->Open());
+
+  const std::string data = "abcd";
+
+  ASSERT_EQ(file->Write(data.data(), data.size()),
+            static_cast<int64_t>(data.size()));
+  ASSERT_TRUE(file->Flush());
+
+  auto json = HandleResponse(file);
+  ASSERT_TRUE(json);
+  ASSERT_TRUE(file.release()->Close());
+
+  ASSERT_JSON_STRING(json, "method", "PUT");
+  ASSERT_JSON_STRING(json, "data", data);
+  ASSERT_JSON_STRING(json, "headers.Content-Type", "application/octet-stream");
+  ASSERT_JSON_STRING(json, "headers.Content-Length",
+                     std::to_string(data.size()));
+}
+
+TEST(HttpFileTest, DISABLED_MultipleWrites) {
+  FilePtr file(new HttpFile(HttpMethod::kPut, "https://httpbin.org/anything"));
+  ASSERT_TRUE(file);
+  ASSERT_TRUE(file->Open());
+
+  const std::string data1 = "abcd";
+  const std::string data2 = "efgh";
+  const std::string data3 = "ijkl";
+  const std::string data4 = "mnop";
+
+  ASSERT_EQ(file->Write(data1.data(), data1.size()),
+            static_cast<int64_t>(data1.size()));
+  ASSERT_EQ(file->Write(data2.data(), data2.size()),
+            static_cast<int64_t>(data2.size()));
+  ASSERT_EQ(file->Write(data3.data(), data3.size()),
+            static_cast<int64_t>(data3.size()));
+  ASSERT_EQ(file->Write(data4.data(), data4.size()),
+            static_cast<int64_t>(data4.size()));
+  ASSERT_TRUE(file->Flush());
+
+  auto json = HandleResponse(file);
+  ASSERT_TRUE(json);
+  ASSERT_TRUE(file.release()->Close());
+
+  ASSERT_JSON_STRING(json, "method", "PUT");
+  ASSERT_JSON_STRING(json, "data", data1 + data2 + data3 + data4);
+  ASSERT_JSON_STRING(json, "headers.Content-Type", "application/octet-stream");
+  ASSERT_JSON_STRING(json, "headers.Content-Length",
+                     std::to_string(data1.size() + data2.size() + data3.size() +
+                                    data4.size()));
+}
+
+// TODO: Test chunked uploads.  Since we can only read the response, we have no
+// way to detect if we are streaming the upload like we want.  httpbin seems to
+// populate the Content-Length even if we don't give it in the request.
+
+TEST(HttpFileTest, DISABLED_Error404) {
+  FilePtr file(
+      new HttpFile(HttpMethod::kGet, "https://httpbin.org/status/404"));
+  ASSERT_TRUE(file);
+  ASSERT_TRUE(file->Open());
+
+  // The site returns an empty response.
+  uint8_t buffer[1];
+  ASSERT_EQ(file->Read(buffer, sizeof(buffer)), 0);
+
+  auto status = file.release()->CloseWithStatus();
+  ASSERT_FALSE(status.ok());
+  ASSERT_EQ(status.error_code(), error::HTTP_FAILURE);
+}
+
+TEST(HttpFileTest, DISABLED_TimeoutTriggered) {
+  FilePtr file(
+      new HttpFile(HttpMethod::kGet, "https://httpbin.org/delay/8", "", {}, 1));
+  ASSERT_TRUE(file);
+  ASSERT_TRUE(file->Open());
+
+  // Request should timeout; error is reported in Close/CloseWithStatus.
+  uint8_t buffer[1];
+  ASSERT_EQ(file->Read(buffer, sizeof(buffer)), 0);
+
+  auto status = file.release()->CloseWithStatus();
+  ASSERT_FALSE(status.ok());
+  ASSERT_EQ(status.error_code(), error::TIME_OUT);
+}
+
+TEST(HttpFileTest, DISABLED_TimeoutNotTriggered) {
+  FilePtr file(
+      new HttpFile(HttpMethod::kGet, "https://httpbin.org/delay/1", "", {}, 5));
+  ASSERT_TRUE(file);
+  ASSERT_TRUE(file->Open());
+
+  auto json = HandleResponse(file);
+  ASSERT_TRUE(json);
+  ASSERT_TRUE(file.release()->Close());
 }
 
 }  // namespace shaka

packager/file/local_file.cc

@@ -124,10 +124,6 @@ int64_t LocalFile::Read(void* buffer, uint64_t length) {
 }
 
 int64_t LocalFile::Write(const void* buffer, uint64_t length) {
-
-  base::FilePath file_path(base::FilePath::FromUTF8Unsafe(file_name()));
-  VLOG(2) << "Writing to " << file_path.AsUTF8Unsafe() << ", length=" << length;
-
   DCHECK(buffer != NULL);
   DCHECK(internal_file_ != NULL);
   size_t bytes_written = fwrite(buffer, sizeof(char), length, internal_file_);

packager/media/base/buffer_writer.cc

@@ -70,8 +70,8 @@ void BufferWriter::AppendBuffer(const BufferWriter& buffer) {
 Status BufferWriter::WriteToFile(File* file) {
   DCHECK(file);
   DCHECK(!buf_.empty());
+
   size_t remaining_size = buf_.size();
-  VLOG(1) << "BufferWriter::WriteToFile " << file->file_name() << " with " << remaining_size << " octets";
   const uint8_t* buf = &buf_[0];
   while (remaining_size > 0) {
     int64_t size_written = file->Write(buf, remaining_size);

packager/media/base/http_key_fetcher.cc

@@ -6,134 +6,22 @@
 
 #include "packager/media/base/http_key_fetcher.h"
 
-#include <curl/curl.h>
+#include "packager/file/file_closer.h"
-#include <gflags/gflags.h>
-
-#include "packager/base/logging.h"
-#include "packager/base/strings/string_number_conversions.h"
-#include "packager/base/strings/stringprintf.h"
-#include "packager/base/synchronization/lock.h"
-
-DEFINE_bool(disable_peer_verification,
-            false,
-            "Disable peer verification. This is needed to talk to servers "
-            "without valid certificates.");
 
 namespace shaka {
+namespace media {
 
 namespace {
-const char kUserAgentString[] = "shaka-packager-http_fetcher/1.0";
+
 const char kSoapActionHeader[] =
     "SOAPAction: \"http://schemas.microsoft.com/DRM/2007/03/protocols/"
     "AcquirePackagingData\"";
-const char kXmlContentTypeHeader[] = "Content-Type: text/xml; charset=UTF-8";
+const char kXmlContentType[] = "text/xml; charset=UTF-8";
-const char kJsonContentTypeHeader[] = "Content-Type: application/json";
+const char kJsonContentType[] = "application/json";
-
+constexpr size_t kBufferSize = 64 * 1024;
-const int kMinLogLevelForCurlDebugFunction = 2;
-
-int CurlDebugFunction(CURL* /* handle */,
-                      curl_infotype type,
-                      const char* data,
-                      size_t size,
-                      void* /* userptr */) {
-  const char* type_text;
-  int log_level = kMinLogLevelForCurlDebugFunction;
-  switch (type) {
-    case CURLINFO_TEXT:
-      type_text = "== Info";
-      log_level = kMinLogLevelForCurlDebugFunction + 1;
-      break;
-    case CURLINFO_HEADER_IN:
-      type_text = "<= Recv header";
-      log_level = kMinLogLevelForCurlDebugFunction;
-      break;
-    case CURLINFO_HEADER_OUT:
-      type_text = "=> Send header";
-      log_level = kMinLogLevelForCurlDebugFunction;
-      break;
-    case CURLINFO_DATA_IN:
-      type_text = "<= Recv data";
-      log_level = kMinLogLevelForCurlDebugFunction + 1;
-      break;
-    case CURLINFO_DATA_OUT:
-      type_text = "=> Send data";
-      log_level = kMinLogLevelForCurlDebugFunction + 1;
-      break;
-    case CURLINFO_SSL_DATA_IN:
-      type_text = "<= Recv SSL data";
-      log_level = kMinLogLevelForCurlDebugFunction + 2;
-      break;
-    case CURLINFO_SSL_DATA_OUT:
-      type_text = "=> Send SSL data";
-      log_level = kMinLogLevelForCurlDebugFunction + 2;
-      break;
-    default:
-      // Ignore other debug data.
-      return 0;
-  }
-
-  VLOG(log_level) << "\n\n"
-                  << type_text << " (0x" << std::hex << size << std::dec
-                  << " bytes)"
-                  << "\n"
-                  << std::string(data, size) << "\nHex Format: \n"
-                  << base::HexEncode(data, size);
-  return 0;
-}
-
-// Scoped CURL implementation which cleans up itself when goes out of scope.
-class ScopedCurl {
- public:
-  ScopedCurl() { ptr_ = curl_easy_init(); }
-  ~ScopedCurl() {
-    if (ptr_)
-      curl_easy_cleanup(ptr_);
-  }
-
-  CURL* get() { return ptr_; }
-
- private:
-  CURL* ptr_;
-  DISALLOW_COPY_AND_ASSIGN(ScopedCurl);
-};
-
-size_t AppendToString(char* ptr, size_t size, size_t nmemb, std::string* response) {
-  DCHECK(ptr);
-  DCHECK(response);
-  const size_t total_size = size * nmemb;
-  response->append(ptr, total_size);
-  return total_size;
-}
-
-class LibCurlInitializer {
- public:
-  LibCurlInitializer() : initialized_(false) {
-    base::AutoLock lock(lock_);
-    if (!initialized_) {
-      curl_global_init(CURL_GLOBAL_DEFAULT);
-      initialized_ = true;
-    }
-  }
-
-  ~LibCurlInitializer() {
-    base::AutoLock lock(lock_);
-    if (initialized_) {
-      curl_global_cleanup();
-      initialized_ = false;
-    }
-  }
-
- private:
-  base::Lock lock_;
-  bool initialized_;
-
-  DISALLOW_COPY_AND_ASSIGN(LibCurlInitializer);
-};
 
 }  // namespace
 
-namespace media {
-
 HttpKeyFetcher::HttpKeyFetcher() : timeout_in_seconds_(0) {}
 
 HttpKeyFetcher::HttpKeyFetcher(uint32_t timeout_in_seconds)
@@ -148,95 +36,45 @@ Status HttpKeyFetcher::FetchKeys(const std::string& url,
 }
 
 Status HttpKeyFetcher::Get(const std::string& path, std::string* response) {
-  return FetchInternal(GET, path, "", response);
+  return FetchInternal(HttpMethod::kGet, path, "", response);
 }
 
 Status HttpKeyFetcher::Post(const std::string& path,
                             const std::string& data,
                             std::string* response) {
-  return FetchInternal(POST, path, data, response);
+  return FetchInternal(HttpMethod::kPost, path, data, response);
 }
 
 Status HttpKeyFetcher::FetchInternal(HttpMethod method,
                                      const std::string& path,
                                      const std::string& data,
                                      std::string* response) {
-  DCHECK(method == GET || method == POST);
+  std::string content_type;
-  static LibCurlInitializer lib_curl_initializer;
+  std::vector<std::string> headers;
-
-  ScopedCurl scoped_curl;
-  CURL* curl = scoped_curl.get();
-  if (!curl) {
-    LOG(ERROR) << "curl_easy_init() failed.";
-    return Status(error::HTTP_FAILURE, "curl_easy_init() failed.");
-  }
-  response->clear();
-
-  curl_easy_setopt(curl, CURLOPT_URL, path.c_str());
-  curl_easy_setopt(curl, CURLOPT_USERAGENT, kUserAgentString);
-  curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout_in_seconds_);
-  curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
-  curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
-  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, AppendToString);
-  curl_easy_setopt(curl, CURLOPT_WRITEDATA, response);
-
-  if (FLAGS_disable_peer_verification)
-    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
-
-  if (!client_cert_private_key_file_.empty() && !client_cert_file_.empty()) {
-    // Some PlayReady packaging servers only allow connects via HTTPS with
-    // client certificates.
-    curl_easy_setopt(curl, CURLOPT_SSLKEY,
-                     client_cert_private_key_file_.data());
-    if (!client_cert_private_key_password_.empty()) {
-      curl_easy_setopt(curl, CURLOPT_KEYPASSWD,
-                       client_cert_private_key_password_.data());
-    }
-    curl_easy_setopt(curl, CURLOPT_SSLKEYTYPE, "PEM");
-    curl_easy_setopt(curl, CURLOPT_SSLCERTTYPE, "PEM");
-    curl_easy_setopt(curl, CURLOPT_SSLCERT, client_cert_file_.data());
-  }
-  if (!ca_file_.empty()) {
-    // Host validation needs to be off when using self-signed certificates.
-    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
-    curl_easy_setopt(curl, CURLOPT_CAINFO, ca_file_.data());
-  }
-  if (method == POST) {
-    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.c_str());
-    curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data.size());
-
-    curl_slist* chunk = nullptr;
   if (data.find("soap:Envelope") != std::string::npos) {
     // Adds Http headers for SOAP requests.
-      chunk = curl_slist_append(chunk, kXmlContentTypeHeader);
+    content_type = kXmlContentType;
-      chunk = curl_slist_append(chunk, kSoapActionHeader);
+    headers.push_back(kSoapActionHeader);
   } else {
-      chunk = curl_slist_append(chunk, kJsonContentTypeHeader);
+    content_type = kJsonContentType;
-    }
-    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk);
   }
 
-  if (VLOG_IS_ON(kMinLogLevelForCurlDebugFunction)) {
+  std::unique_ptr<HttpFile, FileCloser> file(
-    curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, CurlDebugFunction);
+      new HttpFile(method, path, content_type, headers, timeout_in_seconds_));
-    curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
+  if (!file->Open()) {
+    return Status(error::INTERNAL_ERROR, "Cannot open URL");
   }
+  file->Write(data.data(), data.size());
+  file->Flush();
 
-  CURLcode res = curl_easy_perform(curl);
+  while (true) {
-  if (res != CURLE_OK) {
+    char temp[kBufferSize];
-    std::string error_message = base::StringPrintf(
+    int64_t ret = file->Read(temp, kBufferSize);
-        "curl_easy_perform() failed: %s.", curl_easy_strerror(res));
+    if (ret <= 0)
-    if (res == CURLE_HTTP_RETURNED_ERROR) {
+      break;
-      long response_code = 0;
+    response->append(temp, ret);
-      curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
-      error_message += base::StringPrintf(" Response code: %ld.", response_code);
   }
-
+  return file.release()->CloseWithStatus();
-    LOG(ERROR) << error_message;
-    return Status(
-        res == CURLE_OPERATION_TIMEDOUT ? error::TIME_OUT : error::HTTP_FAILURE,
-        error_message);
-  }
-  return Status::OK;
 }
 
 }  // namespace media

packager/media/base/http_key_fetcher.h

@@ -3,15 +3,14 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file or at
 // https://developers.google.com/open-source/licenses/bsd
-//
-/// NOTE: Inclusion of this module will cause curl_global_init and
-///       curl_global_cleanup to be called at static initialization /
-///       deinitialization time.
 
 #ifndef PACKAGER_MEDIA_BASE_HTTP_KEY_FETCHER_H_
 #define PACKAGER_MEDIA_BASE_HTTP_KEY_FETCHER_H_
 
+#include <string>
+
 #include "packager/base/compiler_specific.h"
+#include "packager/file/http_file.h"
 #include "packager/media/base/key_fetcher.h"
 #include "packager/status.h"
 
@@ -53,40 +52,11 @@ class HttpKeyFetcher : public KeyFetcher {
                       const std::string& data,
                       std::string* response);
 
-  /// Sets client certificate information for http requests.
-  /// @param cert_file absolute path to the client certificate.
-  /// @param private_key_file absolute path to the client certificate
-  ///        private key file.
-  /// @param private_key_password private key password.
-  void SetClientCertInfo(const std::string& cert_file,
-                         const std::string& private_key_file,
-                         const std::string& private_key_password) {
-    client_cert_file_ = cert_file;
-    client_cert_private_key_file_ = private_key_file;
-    client_cert_private_key_password_ = private_key_password;
-  }
-  /// Sets the Certifiate Authority file information for http requests.
-  /// @param ca_file absolute path to the client certificate
-  void SetCaFile(const std::string& ca_file) {
-    ca_file_ = ca_file;
-  }
-
  private:
-  enum HttpMethod {
-    GET,
-    POST,
-    PUT
-  };
-
-  // Internal implementation of HTTP functions, e.g. Get and Post.
   Status FetchInternal(HttpMethod method, const std::string& url,
                        const std::string& data, std::string* response);
 
   const uint32_t timeout_in_seconds_;
-  std::string ca_file_;
-  std::string client_cert_file_;
-  std::string client_cert_private_key_file_;
-  std::string client_cert_private_key_password_;
 
   DISALLOW_COPY_AND_ASSIGN(HttpKeyFetcher);
 };

packager/media/base/playready_key_source.cc

@@ -73,19 +73,6 @@ PlayReadyKeySource::PlayReadyKeySource(const std::string& server_url,
       encryption_key_(new EncryptionKey),
       server_url_(server_url) {}
 
-PlayReadyKeySource::PlayReadyKeySource(
-    const std::string& server_url,
-    const std::string& client_cert_file,
-    const std::string& client_cert_private_key_file,
-    const std::string& client_cert_private_key_password,
-    ProtectionSystem protection_systems)
-    // PlayReady PSSH is retrived from PlayReady server response.
-    : encryption_key_(new EncryptionKey),
-      server_url_(server_url),
-      client_cert_file_(client_cert_file),
-      client_cert_private_key_file_(client_cert_private_key_file),
-      client_cert_private_key_password_(client_cert_private_key_password) {}
-
 PlayReadyKeySource::~PlayReadyKeySource() = default;
 
 Status RetrieveTextInXMLElement(const std::string& element,
@@ -161,14 +148,6 @@ Status PlayReadyKeySource::FetchKeysWithProgramIdentifier(
     const std::string& program_identifier) {
   std::unique_ptr<EncryptionKey> encryption_key(new EncryptionKey);
   HttpKeyFetcher key_fetcher(kHttpFetchTimeout);
-  if (!client_cert_file_.empty() && !client_cert_private_key_file_.empty()) {
-    key_fetcher.SetClientCertInfo(client_cert_file_,
-                                  client_cert_private_key_file_,
-                                  client_cert_private_key_password_);
-  }
-  if (!ca_file_.empty()) {
-    key_fetcher.SetCaFile(ca_file_);
-  }
 
   std::string acquire_license_request = kAcquireLicenseRequest;
   base::ReplaceFirstSubstringAfterOffset(

packager/media/base/playready_key_source.h

@@ -25,19 +25,6 @@ class PlayReadyKeySource : public KeySource {
   ///        be included.
   PlayReadyKeySource(const std::string& server_url,
                      ProtectionSystem protection_systems);
-  /// Creates a new PlayReadyKeySource from the given packaging information.
-  /// @param server_url PlayReady packaging server url.
-  /// @param client_cert_file absolute path to a client certificate.
-  /// @param client_cert_private_key_file absolute path to the private file
-  ///     for the client certificate.
-  /// @param client_cert_private_key_password password for the private key.
-  /// @param protection_systems is the enum indicating which PSSH should
-  ///        be included.
-  PlayReadyKeySource(const std::string& server_url,
-                     const std::string& client_cert_file,
-                     const std::string& client_cert_private_key_file,
-                     const std::string& client_cert_private_key_password,
-                     ProtectionSystem protection_systems);
   ~PlayReadyKeySource() override;
 
   /// @name KeySource implementation overrides.
@@ -61,10 +48,6 @@ class PlayReadyKeySource : public KeySource {
   static std::unique_ptr<PlayReadyKeySource> CreateFromKeyAndKeyId(
       const std::vector<uint8_t>& key_id,
       const std::vector<uint8_t>& key);
-  /// Sets the Certificate Authority file for validating self-signed certificates.
-  void SetCaFile(const std::string& ca_file) {
-    ca_file_ = ca_file;
-  }
 
  private:
   Status GetKeyInternal();
@@ -75,10 +58,6 @@ class PlayReadyKeySource : public KeySource {
 
   std::unique_ptr<EncryptionKey> encryption_key_;
   std::string server_url_;
-  std::string ca_file_;
-  std::string client_cert_file_;
-  std::string client_cert_private_key_file_;
-  std::string client_cert_private_key_password_;
 
   DISALLOW_COPY_AND_ASSIGN(PlayReadyKeySource);
 };

packager/media/formats/packed_audio/packed_audio_writer.cc

@@ -107,7 +107,6 @@ Status PackedAudioWriter::WriteSegment(const std::string& segment_path,
     range.end = range.start + segment_buffer->Size() - 1;
     media_ranges_.subsegment_ranges.push_back(range);
   } else {
-    VLOG(2) << "PackedAudioWriter::WriteSegment: File::Open(" << segment_path << ")";
     file.reset(File::Open(segment_path.c_str(), "w"));
     if (!file) {
       return Status(error::FILE_FAILURE,