From 419d463eaab8298cbf90e01718615a8f2d3015f8 Mon Sep 17 00:00:00 2001 From: KongQun Yang Date: Tue, 17 Jun 2014 18:33:07 -0700 Subject: [PATCH] Modify HttpFetcher to use libcurl With libcurl, HttpFetcher now supports timeout and https. Two additional changes: 1. Remove happyhttp which is no longer needed; 2. Add README.packager for curl Bug: 13658515 Bug: 14301830 Change-Id: I13c2835e7feca9abf36e5bb8f7bc35a7db9ec94c --- .gitignore | 1 - DEPS | 6 - media/base/http_fetcher.cc | 209 ++++++++------------------ media/base/http_fetcher.h | 27 +++- media/base/http_fetcher_unittest.cc | 32 +++- media/base/media_base.gyp | 40 +---- third_party/curl/README.packager | 38 +++++ third_party/happyhttp/README.packager | 11 -- third_party/happyhttp/happyhttp.gyp | 37 ----- 9 files changed, 153 insertions(+), 248 deletions(-) create mode 100644 third_party/curl/README.packager delete mode 100644 third_party/happyhttp/README.packager delete mode 100644 third_party/happyhttp/happyhttp.gyp diff --git a/.gitignore b/.gitignore index a061b8bcf7..209d445489 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,6 @@ /third_party/curl/source/ /third_party/gflags/ /third_party/gold/ -/third_party/happyhttp/src/ /third_party/icu/ /third_party/libevent/ /third_party/libxml/ diff --git a/DEPS b/DEPS index b2abebe732..fb04d317a8 100644 --- a/DEPS +++ b/DEPS @@ -19,9 +19,6 @@ vars = { "curl_url": "https://github.com/bagder/curl.git", "curl_rev": "curl-7_37_0", - - "happyhttp_url": "https://github.com/Zintinio/HappyHTTP.git", - "happyhttp_rev": "6b11b3b02cb3c8b649de9fffe8e08ae68c42bfd0", } deps = { @@ -65,9 +62,6 @@ deps = { "src/third_party/gflags/src": (Var("googlecode_url") % "gflags") + "/trunk/src@" + Var("gflags_rev"), - "src/third_party/happyhttp/src": - Var("happyhttp_url") + "@" + Var("happyhttp_rev"), - # Required by libxml. "src/third_party/icu": Var("chromium_svn") + "/deps/third_party/icu46@" + Var("chromium_rev"), diff --git a/media/base/http_fetcher.cc b/media/base/http_fetcher.cc index ee741f3d23..6bc171516c 100644 --- a/media/base/http_fetcher.cc +++ b/media/base/http_fetcher.cc @@ -6,97 +6,35 @@ #include "media/base/http_fetcher.h" -#ifdef WIN32 -#include -#endif // WIN32 - -#include "base/strings/string_number_conversions.h" -#include "base/strings/string_util.h" -#include "third_party/happyhttp/src/happyhttp.h" +#include +#include "base/strings/stringprintf.h" namespace { +const char kUserAgentString[] = "edash-packager-http_fetcher/1.0"; -struct HttpResult { - int status_code; - std::string status_message; - std::string response; +// Scoped CURL implementation which cleans up itself when goes out of scope. +class ScopedCurl { + public: + ScopedCurl() { ptr_ = curl_easy_init(); } + ~ScopedCurl() { + if (ptr_) + curl_easy_cleanup(ptr_); + } + + CURL* get() { return ptr_; } + + private: + CURL* ptr_; + DISALLOW_COPY_AND_ASSIGN(ScopedCurl); }; -bool ExtractUrlParams(const std::string& url, std::string* host, - std::string* path, int* port) { - DCHECK(host && path && port); - - static const char kHttp[] = "http://"; - // arraysize counts the last null character, which needs to be removed. - const char kHttpSize = arraysize(kHttp) - 1; - static const char kHttps[] = "https://"; - const char kHttpsSize = arraysize(kHttps) - 1; - size_t host_start_pos; - if (StartsWithASCII(url, kHttp, false)) { - host_start_pos = kHttpSize; - } else if (StartsWithASCII(url, kHttps, false)) { - host_start_pos = kHttpsSize; - NOTIMPLEMENTED() << "Secure HTTP is not implemented yet."; - return false; - } else { - host_start_pos = 0; - } - - const size_t npos = std::string::npos; - const size_t port_start_pos = url.find(':', host_start_pos); - const size_t path_start_pos = url.find('/', host_start_pos); - - size_t host_size; - if (port_start_pos == npos) { - const int kStandardHttpPort = 80; - *port = kStandardHttpPort; - - host_size = path_start_pos == npos ? npos : path_start_pos - host_start_pos; - } else { - if (port_start_pos >= path_start_pos) - return false; - const size_t port_size = - path_start_pos == npos ? npos : path_start_pos - port_start_pos - 1; - if (!base::StringToInt(url.substr(port_start_pos + 1, port_size), port)) - return false; - - host_size = port_start_pos - host_start_pos; - } - - *host = url.substr(host_start_pos, host_size); - *path = path_start_pos == npos ? "/" : url.substr(path_start_pos); - return true; +size_t AppendToString(char* ptr, size_t size, size_t nmemb, std::string* response) { + DCHECK(ptr); + DCHECK(response); + const size_t total_size = size * nmemb; + response->append(ptr, total_size); + return total_size; } - -// happyhttp event callbacks. -void OnBegin(const happyhttp::Response* response, void* userdata) { - DCHECK(response && userdata); - DLOG(INFO) << "BEGIN (" << response->getstatus() << ", " - << response->getreason() << ")."; - - HttpResult* result = static_cast(userdata); - result->status_code = response->getstatus(); - result->status_message = response->getreason(); - result->response.clear(); -} - -void OnData(const happyhttp::Response* response, - void* userdata, - const unsigned char* data, - int num_bytes) { - DCHECK(response && userdata && data); - HttpResult* result = static_cast(userdata); - result->response.append(reinterpret_cast(data), num_bytes); -} - -void OnComplete(const happyhttp::Response* response, void* userdata) { - DCHECK(response && userdata); - HttpResult* result = static_cast(userdata); - DLOG(INFO) << "COMPLETE (" << result->response.size() << " bytes)."; -} - -const int kHttpOK = 200; - } // namespace namespace media { @@ -104,86 +42,69 @@ namespace media { HttpFetcher::HttpFetcher() {} HttpFetcher::~HttpFetcher() {} -SimpleHttpFetcher::SimpleHttpFetcher() { -#ifdef WIN32 - WSAData wsa_data; - int code = WSAStartup(MAKEWORD(1, 1), &wsa_data); - wsa_startup_succeeded_ = (code == 0); - if (!wsa_startup_succeeded_) - LOG(ERROR) << "WSAStartup failed with code " << code; -#endif // WIN32 +SimpleHttpFetcher::SimpleHttpFetcher() : timeout_in_seconds_(0) { + curl_global_init(CURL_GLOBAL_DEFAULT); +} + +SimpleHttpFetcher::SimpleHttpFetcher(uint32 timeout_in_seconds) + : timeout_in_seconds_(timeout_in_seconds) { + curl_global_init(CURL_GLOBAL_DEFAULT); } SimpleHttpFetcher::~SimpleHttpFetcher() { -#ifdef WIN32 - if (wsa_startup_succeeded_) - WSACleanup(); -#endif // WIN32 + curl_global_cleanup(); } Status SimpleHttpFetcher::Get(const std::string& path, std::string* response) { - return FetchInternal("GET", path, "", response); + return FetchInternal(GET, path, "", response); } Status SimpleHttpFetcher::Post(const std::string& path, const std::string& data, std::string* response) { - return FetchInternal("POST", path, data, response); + return FetchInternal(POST, path, data, response); } -Status SimpleHttpFetcher::FetchInternal(const std::string& method, - const std::string& url, +Status SimpleHttpFetcher::FetchInternal(HttpMethod method, + const std::string& path, const std::string& data, std::string* response) { - DCHECK(response); + DCHECK(method == GET || method == POST); - int status_code = 0; + ScopedCurl scoped_curl; + CURL* curl = scoped_curl.get(); + if (!curl) { + LOG(ERROR) << "curl_easy_init() failed."; + return Status(error::HTTP_FAILURE, "curl_easy_init() failed."); + } + response->clear(); - std::string host; - std::string path; - int port = 0; - if (!ExtractUrlParams(url, &host, &path, &port)) { - std::string error_message = "Cannot extract url parameters from " + url; - LOG(ERROR) << error_message; - return Status(error::INVALID_ARGUMENT, error_message); + curl_easy_setopt(curl, CURLOPT_URL, path.c_str()); + curl_easy_setopt(curl, CURLOPT_USERAGENT, kUserAgentString); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout_in_seconds_); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, AppendToString); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, response); + if (method == POST) { + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data.size()); } - try { - HttpResult result; - happyhttp::Connection connection(host.data(), port); - connection.setcallbacks(OnBegin, OnData, OnComplete, &result); + CURLcode res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::string error_message = base::StringPrintf( + "curl_easy_perform() failed: %s.", curl_easy_strerror(res)); + if (res == CURLE_HTTP_RETURNED_ERROR) { + long response_code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + error_message += base::StringPrintf(" Response code: %ld.", response_code); + } - VLOG(1) << "Send " << method << " request to " << url << ": " << data; - - static const char* kHeaders[] = { - "Connection", "close", - "Content-type", "application/x-www-form-urlencoded", - "Accept", "text/plain", - 0}; - connection.request( - method.data(), path.data(), kHeaders, - data.empty() ? NULL : reinterpret_cast(data.data()), - data.size()); - - while (connection.outstanding()) - connection.pump(); - - status_code = result.status_code; - *response = result.response; - - VLOG(1) << "Response: " << result.response; - } catch (happyhttp::Wobbly& exception) { - std::string error_message = - std::string("HTTP fetcher failed: ") + exception.what(); LOG(ERROR) << error_message; - return Status(error::HTTP_FAILURE, error_message); - } - - if (status_code != kHttpOK) { - std::string error_message = - "HTTP returns status " + base::IntToString(status_code); - LOG(ERROR) << error_message; - return Status(error::HTTP_FAILURE, error_message); + return Status( + res == CURLE_OPERATION_TIMEDOUT ? error::TIME_OUT : error::HTTP_FAILURE, + error_message); } return Status::OK; } diff --git a/media/base/http_fetcher.h b/media/base/http_fetcher.h index bb57b9b479..fd08835d4d 100644 --- a/media/base/http_fetcher.h +++ b/media/base/http_fetcher.h @@ -38,27 +38,40 @@ class HttpFetcher { DISALLOW_COPY_AND_ASSIGN(HttpFetcher); }; -/// A simple HttpFetcher implementation using happyhttp. +/// A simple HttpFetcher implementation. +/// This class is not fully thread safe. It can be used in multi-thread +/// environment once constructed, but it may not be safe to create a +/// SimpleHttpFetcher object when any other thread is running due to use of +/// curl_global_init. class SimpleHttpFetcher : public HttpFetcher { public: + /// Creates a fetcher with no timeout. SimpleHttpFetcher(); + /// Create a fetcher with timeout. + /// @param timeout_in_seconds specifies the timeout in seconds. + SimpleHttpFetcher(uint32 timeout_in_seconds); virtual ~SimpleHttpFetcher(); - // HttpFetcher implementation overrides. + /// @name HttpFetcher implementation overrides. + /// @{ virtual Status Get(const std::string& url, std::string* response) OVERRIDE; virtual Status Post(const std::string& url, const std::string& data, std::string* response) OVERRIDE; + /// @} private: + enum HttpMethod { + GET, + POST, + PUT + }; + // Internal implementation of HTTP functions, e.g. Get and Post. - Status FetchInternal(const std::string& method, const std::string& url, + Status FetchInternal(HttpMethod method, const std::string& url, const std::string& data, std::string* response); -#ifdef WIN32 - // Track whether WSAStartup executes successfully. - bool wsa_startup_succeeded_; -#endif + const uint32 timeout_in_seconds_; DISALLOW_COPY_AND_ASSIGN(SimpleHttpFetcher); }; diff --git a/media/base/http_fetcher_unittest.cc b/media/base/http_fetcher_unittest.cc index d2b9e46529..f697967b10 100644 --- a/media/base/http_fetcher_unittest.cc +++ b/media/base/http_fetcher_unittest.cc @@ -12,8 +12,6 @@ #include "media/base/status_test_util.h" namespace { -const int kHttpNotFound = 404; - const char kTestUrl[] = "http://packager-test.appspot.com/http_test"; const char kTestUrlWithPort[] = "http://packager-test.appspot.com:80/http_test"; const char kExpectedGetResponse[] = @@ -23,6 +21,7 @@ const char kPostData[] = "foo=62&type=mp4"; const char kExpectedPostResponse[] = "http_test
"
     "Arguments([foo]=>62[type]=>mp4)
"; +const char kDelayTwoSecs[] = "delay=2"; // This causes host to delay 2 seconds. } // namespace namespace media { @@ -46,27 +45,44 @@ static void CheckHttpPost(const std::string& url, const std::string& data, } -TEST(HttpFetcherTest, HttpGet) { +TEST(DISABLED_HttpFetcherTest, HttpGet) { CheckHttpGet(kTestUrl, kExpectedGetResponse); } -TEST(HttpFetcherTest, HttpPost) { +TEST(DISABLED_HttpFetcherTest, HttpPost) { CheckHttpPost(kTestUrl, kPostData, kExpectedPostResponse); } -TEST(HttpFetcherTest, InvalidUrl) { +TEST(DISABLED_HttpFetcherTest, InvalidUrl) { + const char kHttpNotFound[] = "404"; + SimpleHttpFetcher fetcher; std::string response; const std::string invalid_url(kTestUrl, sizeof(kTestUrl) - 2); Status status = fetcher.Get(invalid_url, &response); EXPECT_EQ(error::HTTP_FAILURE, status.error_code()); - EXPECT_TRUE( - EndsWith(status.error_message(), base::IntToString(kHttpNotFound), true)); + EXPECT_NE(std::string::npos, status.error_message().find(kHttpNotFound)); } -TEST(HttpFetcherTest, UrlWithPort) { +TEST(DISABLED_HttpFetcherTest, UrlWithPort) { CheckHttpGet(kTestUrlWithPort, kExpectedGetResponse); } +TEST(DISABLED_HttpFetcherTest, SmallTimeout) { + const uint32 kTimeoutInSeconds = 1; + SimpleHttpFetcher fetcher(kTimeoutInSeconds); + std::string response; + Status status = fetcher.Post(kTestUrl, kDelayTwoSecs, &response); + EXPECT_EQ(error::TIME_OUT, status.error_code()); +} + +TEST(DISABLED_HttpFetcherTest, BigTimeout) { + const uint32 kTimeoutInSeconds = 5; + SimpleHttpFetcher fetcher(kTimeoutInSeconds); + std::string response; + Status status = fetcher.Post(kTestUrl, kDelayTwoSecs, &response); + EXPECT_OK(status); +} + } // namespace media diff --git a/media/base/media_base.gyp b/media/base/media_base.gyp index 8ae9bb6cb3..2f86d44005 100644 --- a/media/base/media_base.gyp +++ b/media/base/media_base.gyp @@ -15,38 +15,6 @@ ], }, 'targets': [ - { - 'target_name': 'status', - 'type': '<(component)', - 'sources': [ - 'status.cc', - 'status.h', - ], - 'dependencies': [ - '../../base/base.gyp:base', - ], - }, - { - 'target_name': 'http_fetcher', - 'type': '<(component)', - 'sources': [ - 'http_fetcher.cc', - 'http_fetcher.h', - ], - 'cflags!': [ '-fno-exceptions' ], - 'cflags_cc!': [ '-fno-exceptions' ], - 'conditions': [ - ['OS=="mac"', { - 'xcode_settings': { - 'GCC_ENABLE_CPP_EXCEPTIONS': 'YES' - } - }] - ], - 'dependencies': [ - '../../third_party/happyhttp/happyhttp.gyp:happyhttp_lib', - 'status', - ], - }, { 'target_name': 'base', 'type': '<(component)', @@ -76,6 +44,8 @@ 'decryptor_source.h', 'encryption_key_source.cc', 'encryption_key_source.h', + 'http_fetcher.cc', + 'http_fetcher.h', 'limits.h', 'media_parser.h', 'media_sample.cc', @@ -95,6 +65,8 @@ 'request_signer.h', 'rsa_key.cc', 'rsa_key.h', + 'status.cc', + 'status.h', 'stream_info.cc', 'stream_info.h', 'text_track.h', @@ -106,9 +78,8 @@ ], 'dependencies': [ '../../base/base.gyp:base', + '../../third_party/curl/curl.gyp:libcurl', '../../third_party/openssl/openssl.gyp:openssl', - 'http_fetcher', - 'status', ], }, { @@ -123,6 +94,7 @@ 'container_names_unittest.cc', 'fake_prng.cc', # For rsa_key_unittest 'fake_prng.h', # For rsa_key_unittest + 'http_fetcher_unittest.cc', 'muxer_util_unittest.cc', 'offset_byte_queue_unittest.cc', 'producer_consumer_queue_unittest.cc', diff --git a/third_party/curl/README.packager b/third_party/curl/README.packager new file mode 100644 index 0000000000..0d808b6139 --- /dev/null +++ b/third_party/curl/README.packager @@ -0,0 +1,38 @@ +Name: curl +URL: http://curl.haxx.se/dev/ +License: MIT/X +License File: source/COPYING +Local Modifications: None + +Description: +libcurl is a free and easy-to-use client-side URL transfer library. libcurl +supports SSL certificates, HTTP GET, HTTP POST, HTTP PUT, and various other +transfer protocols. + +************************************************************************** +Description of source tree. + +1) config/ + Directory containing configuration files, which are required to build + libcurl and curl correctly. On linux platform, an *auto-generated* + configuration file "config/linux/curl_config.h" is used; the library uses + the configuration files coming with libcurl distribution + "source/lib/config-*.h" on other platforms. + + config/curl/curlbuild.h + Curl build file renamed from source/include/curl/curlbuild.h.dist + + config/dummy_tool_hugehelp.c + A dummy manual required to build curl command line tool. + + config/linux/curl_config.h + An *auto-generated* configuration file by running source/buildconf on + linux platform, with a few features disabled to build correctly on a + fresh linux box. + +2) curl.gyp + A gyp build file for the library. Manually maintained. + +3) source/ + Directory containing curl source codes from github without modification. + diff --git a/third_party/happyhttp/README.packager b/third_party/happyhttp/README.packager deleted file mode 100644 index 8bc1dc66e3..0000000000 --- a/third_party/happyhttp/README.packager +++ /dev/null @@ -1,11 +0,0 @@ -Name: happyhttp -URL: https://github.com/Zintinio/HappyHTTP -Version: 0.1 -License: zlib/libpng -License File: http://opensource.org/licenses/zlib-license.php - -Description: -HappyHTTP is a simple C++ library for issuing HTTP requests and processing -responses. - -Local Modifications: None diff --git a/third_party/happyhttp/happyhttp.gyp b/third_party/happyhttp/happyhttp.gyp deleted file mode 100644 index d9491acd5a..0000000000 --- a/third_party/happyhttp/happyhttp.gyp +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file or at -# https://developers.google.com/open-source/licenses/bsd - -{ - 'targets': [ - { - 'target_name': 'happyhttp_lib', - 'type': 'static_library', - 'sources': [ - 'src/happyhttp.cpp', - 'src/happyhttp.h', - ], - 'cflags!': [ '-fno-exceptions' ], - 'cflags_cc!': [ '-fno-exceptions' ], - 'conditions': [ - ['OS=="mac"', { - 'xcode_settings': { - 'GCC_ENABLE_CPP_EXCEPTIONS': 'YES' - } - }] - ], - }, - { - 'target_name': 'happyhttp_lib_test', - 'type': 'executable', - 'sources': [ - 'src/test.cpp', - ], - 'dependencies': [ - 'happyhttp_lib', - ], - }, - ], -}