Mirror-Tools
/
shaka-packager
mirror of https://github.com/shaka-project/shaka-packager.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Modify HttpFetcher to use libcurl 

With libcurl, HttpFetcher now supports timeout and https.

Two additional changes:
1. Remove happyhttp which is no longer needed;
2. Add README.packager for curl

Bug: 13658515
Bug: 14301830

Change-Id: I13c2835e7feca9abf36e5bb8f7bc35a7db9ec94c
This commit is contained in:
KongQun Yang 2014-06-17 18:33:07 -07:00
parent b3f6015ccd
commit 419d463eaa
9 changed files with 153 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -15,7 +15,6 @@
/third_party/curl/source/ /third_party/curl/source/
/third_party/gflags/ /third_party/gflags/
/third_party/gold/ /third_party/gold/
/third_party/happyhttp/src/
/third_party/icu/ /third_party/icu/
/third_party/libevent/ /third_party/libevent/
/third_party/libxml/ /third_party/libxml/

6
DEPS
View File

@ -19,9 +19,6 @@ vars = {
"curl_url": "https://github.com/bagder/curl.git", "curl_url": "https://github.com/bagder/curl.git",
"curl_rev": "curl-7_37_0", "curl_rev": "curl-7_37_0",
"happyhttp_url": "https://github.com/Zintinio/HappyHTTP.git",
"happyhttp_rev": "6b11b3b02cb3c8b649de9fffe8e08ae68c42bfd0",
} }
deps = { deps = {
@ -65,9 +62,6 @@ deps = {
"src/third_party/gflags/src": "src/third_party/gflags/src":
(Var("googlecode_url") % "gflags") + "/trunk/src@" + Var("gflags_rev"), (Var("googlecode_url") % "gflags") + "/trunk/src@" + Var("gflags_rev"),
"src/third_party/happyhttp/src":
Var("happyhttp_url") + "@" + Var("happyhttp_rev"),
# Required by libxml. # Required by libxml.
"src/third_party/icu": "src/third_party/icu":
Var("chromium_svn") + "/deps/third_party/icu46@" + Var("chromium_rev"), Var("chromium_svn") + "/deps/third_party/icu46@" + Var("chromium_rev"),

207
media/base/http_fetcher.cc
View File

@ -6,97 +6,35 @@
#include "media/base/http_fetcher.h" #include "media/base/http_fetcher.h"
#ifdef WIN32 #include <curl/curl.h>
#include <winsock2.h> #include "base/strings/stringprintf.h"
#endif // WIN32
#include "base/strings/string_number_conversions.h"
#include "base/strings/string_util.h"
#include "third_party/happyhttp/src/happyhttp.h"
namespace { namespace {
const char kUserAgentString[] = "edash-packager-http_fetcher/1.0";
struct HttpResult { // Scoped CURL implementation which cleans up itself when goes out of scope.
int status_code; class ScopedCurl {
std::string status_message; public:
std::string response; ScopedCurl() { ptr_ = curl_easy_init(); }
~ScopedCurl() {
if (ptr_)
curl_easy_cleanup(ptr_);
}
CURL* get() { return ptr_; }
private:
CURL* ptr_;
DISALLOW_COPY_AND_ASSIGN(ScopedCurl);
}; };
bool ExtractUrlParams(const std::string& url, std::string* host, size_t AppendToString(char* ptr, size_t size, size_t nmemb, std::string* response) {
std::string* path, int* port) { DCHECK(ptr);
DCHECK(host && path && port); DCHECK(response);
const size_t total_size = size * nmemb;
static const char kHttp[] = "http://"; response->append(ptr, total_size);
// arraysize counts the last null character, which needs to be removed. return total_size;
const char kHttpSize = arraysize(kHttp) - 1;
static const char kHttps[] = "https://";
const char kHttpsSize = arraysize(kHttps) - 1;
size_t host_start_pos;
if (StartsWithASCII(url, kHttp, false)) {
host_start_pos = kHttpSize;
} else if (StartsWithASCII(url, kHttps, false)) {
host_start_pos = kHttpsSize;
NOTIMPLEMENTED() << "Secure HTTP is not implemented yet.";
return false;
} else {
host_start_pos = 0;
}
const size_t npos = std::string::npos;
const size_t port_start_pos = url.find(':', host_start_pos);
const size_t path_start_pos = url.find('/', host_start_pos);
size_t host_size;
if (port_start_pos == npos) {
const int kStandardHttpPort = 80;
*port = kStandardHttpPort;
host_size = path_start_pos == npos ? npos : path_start_pos - host_start_pos;
} else {
if (port_start_pos >= path_start_pos)
return false;
const size_t port_size =
path_start_pos == npos ? npos : path_start_pos - port_start_pos - 1;
if (!base::StringToInt(url.substr(port_start_pos + 1, port_size), port))
return false;
host_size = port_start_pos - host_start_pos;
}
*host = url.substr(host_start_pos, host_size);
*path = path_start_pos == npos ? "/" : url.substr(path_start_pos);
return true;
} }
// happyhttp event callbacks.
void OnBegin(const happyhttp::Response* response, void* userdata) {
DCHECK(response && userdata);
DLOG(INFO) << "BEGIN (" << response->getstatus() << ", "
<< response->getreason() << ").";
HttpResult* result = static_cast<HttpResult*>(userdata);
result->status_code = response->getstatus();
result->status_message = response->getreason();
result->response.clear();
}
void OnData(const happyhttp::Response* response,
void* userdata,
const unsigned char* data,
int num_bytes) {
DCHECK(response && userdata && data);
HttpResult* result = static_cast<HttpResult*>(userdata);
result->response.append(reinterpret_cast<const char*>(data), num_bytes);
}
void OnComplete(const happyhttp::Response* response, void* userdata) {
DCHECK(response && userdata);
HttpResult* result = static_cast<HttpResult*>(userdata);
DLOG(INFO) << "COMPLETE (" << result->response.size() << " bytes).";
}
const int kHttpOK = 200;
} // namespace } // namespace
namespace media { namespace media {
@ -104,86 +42,69 @@ namespace media {
HttpFetcher::HttpFetcher() {} HttpFetcher::HttpFetcher() {}
HttpFetcher::~HttpFetcher() {} HttpFetcher::~HttpFetcher() {}
SimpleHttpFetcher::SimpleHttpFetcher() { SimpleHttpFetcher::SimpleHttpFetcher() : timeout_in_seconds_(0) {
#ifdef WIN32 curl_global_init(CURL_GLOBAL_DEFAULT);
WSAData wsa_data; }
int code = WSAStartup(MAKEWORD(1, 1), &wsa_data);
wsa_startup_succeeded_ = (code == 0); SimpleHttpFetcher::SimpleHttpFetcher(uint32 timeout_in_seconds)
if (!wsa_startup_succeeded_) : timeout_in_seconds_(timeout_in_seconds) {
LOG(ERROR) << "WSAStartup failed with code " << code; curl_global_init(CURL_GLOBAL_DEFAULT);
#endif // WIN32
} }
SimpleHttpFetcher::~SimpleHttpFetcher() { SimpleHttpFetcher::~SimpleHttpFetcher() {
#ifdef WIN32 curl_global_cleanup();
if (wsa_startup_succeeded_)
WSACleanup();
#endif // WIN32
} }
Status SimpleHttpFetcher::Get(const std::string& path, std::string* response) { Status SimpleHttpFetcher::Get(const std::string& path, std::string* response) {
return FetchInternal("GET", path, "", response); return FetchInternal(GET, path, "", response);
} }
Status SimpleHttpFetcher::Post(const std::string& path, Status SimpleHttpFetcher::Post(const std::string& path,
const std::string& data, const std::string& data,
std::string* response) { std::string* response) {
return FetchInternal("POST", path, data, response); return FetchInternal(POST, path, data, response);
} }
Status SimpleHttpFetcher::FetchInternal(const std::string& method, Status SimpleHttpFetcher::FetchInternal(HttpMethod method,
const std::string& url, const std::string& path,
const std::string& data, const std::string& data,
std::string* response) { std::string* response) {
DCHECK(response); DCHECK(method == GET || method == POST);
int status_code = 0; ScopedCurl scoped_curl;
CURL* curl = scoped_curl.get();
if (!curl) {
LOG(ERROR) << "curl_easy_init() failed.";
return Status(error::HTTP_FAILURE, "curl_easy_init() failed.");
}
response->clear();
std::string host; curl_easy_setopt(curl, CURLOPT_URL, path.c_str());
std::string path; curl_easy_setopt(curl, CURLOPT_USERAGENT, kUserAgentString);
int port = 0; curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout_in_seconds_);
if (!ExtractUrlParams(url, &host, &path, &port)) { curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
std::string error_message = "Cannot extract url parameters from " + url; curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
LOG(ERROR) << error_message; curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, AppendToString);
return Status(error::INVALID_ARGUMENT, error_message); curl_easy_setopt(curl, CURLOPT_WRITEDATA, response);
if (method == POST) {
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.c_str());
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data.size());
} }
try { CURLcode res = curl_easy_perform(curl);
HttpResult result; if (res != CURLE_OK) {
happyhttp::Connection connection(host.data(), port); std::string error_message = base::StringPrintf(
connection.setcallbacks(OnBegin, OnData, OnComplete, &result); "curl_easy_perform() failed: %s.", curl_easy_strerror(res));
if (res == CURLE_HTTP_RETURNED_ERROR) {
VLOG(1) << "Send " << method << " request to " << url << ": " << data; long response_code = 0;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
static const char* kHeaders[] = { error_message += base::StringPrintf(" Response code: %ld.", response_code);
"Connection", "close",
"Content-type", "application/x-www-form-urlencoded",
"Accept", "text/plain",
0};
connection.request(
method.data(), path.data(), kHeaders,
data.empty() ? NULL : reinterpret_cast<const uint8*>(data.data()),
data.size());
while (connection.outstanding())
connection.pump();
status_code = result.status_code;
*response = result.response;
VLOG(1) << "Response: " << result.response;
} catch (happyhttp::Wobbly& exception) {
std::string error_message =
std::string("HTTP fetcher failed: ") + exception.what();
LOG(ERROR) << error_message;
return Status(error::HTTP_FAILURE, error_message);
} }
if (status_code != kHttpOK) {
std::string error_message =
"HTTP returns status " + base::IntToString(status_code);
LOG(ERROR) << error_message; LOG(ERROR) << error_message;
return Status(error::HTTP_FAILURE, error_message); return Status(
res == CURLE_OPERATION_TIMEDOUT ? error::TIME_OUT : error::HTTP_FAILURE,
error_message);
} }
return Status::OK; return Status::OK;
} }

27
media/base/http_fetcher.h
View File

@ -38,27 +38,40 @@ class HttpFetcher {
DISALLOW_COPY_AND_ASSIGN(HttpFetcher); DISALLOW_COPY_AND_ASSIGN(HttpFetcher);
}; };
/// A simple HttpFetcher implementation using happyhttp. /// A simple HttpFetcher implementation.
/// This class is not fully thread safe. It can be used in multi-thread
/// environment once constructed, but it may not be safe to create a
/// SimpleHttpFetcher object when any other thread is running due to use of
/// curl_global_init.
class SimpleHttpFetcher : public HttpFetcher { class SimpleHttpFetcher : public HttpFetcher {
public: public:
/// Creates a fetcher with no timeout.
SimpleHttpFetcher(); SimpleHttpFetcher();
/// Create a fetcher with timeout.
/// @param timeout_in_seconds specifies the timeout in seconds.
SimpleHttpFetcher(uint32 timeout_in_seconds);
virtual ~SimpleHttpFetcher(); virtual ~SimpleHttpFetcher();
// HttpFetcher implementation overrides. /// @name HttpFetcher implementation overrides.
/// @{
virtual Status Get(const std::string& url, std::string* response) OVERRIDE; virtual Status Get(const std::string& url, std::string* response) OVERRIDE;
virtual Status Post(const std::string& url, virtual Status Post(const std::string& url,
const std::string& data, const std::string& data,
std::string* response) OVERRIDE; std::string* response) OVERRIDE;
/// @}
private: private:
enum HttpMethod {
GET,
POST,
PUT
};
// Internal implementation of HTTP functions, e.g. Get and Post. // Internal implementation of HTTP functions, e.g. Get and Post.
Status FetchInternal(const std::string& method, const std::string& url, Status FetchInternal(HttpMethod method, const std::string& url,
const std::string& data, std::string* response); const std::string& data, std::string* response);
#ifdef WIN32 const uint32 timeout_in_seconds_;
// Track whether WSAStartup executes successfully.
bool wsa_startup_succeeded_;
#endif
DISALLOW_COPY_AND_ASSIGN(SimpleHttpFetcher); DISALLOW_COPY_AND_ASSIGN(SimpleHttpFetcher);
}; };

32
media/base/http_fetcher_unittest.cc
View File

@ -12,8 +12,6 @@
#include "media/base/status_test_util.h" #include "media/base/status_test_util.h"
namespace { namespace {
const int kHttpNotFound = 404;
const char kTestUrl[] = "http://packager-test.appspot.com/http_test"; const char kTestUrl[] = "http://packager-test.appspot.com/http_test";
const char kTestUrlWithPort[] = "http://packager-test.appspot.com:80/http_test"; const char kTestUrlWithPort[] = "http://packager-test.appspot.com:80/http_test";
const char kExpectedGetResponse[] = const char kExpectedGetResponse[] =
@ -23,6 +21,7 @@ const char kPostData[] = "foo=62&type=mp4";
const char kExpectedPostResponse[] = const char kExpectedPostResponse[] =
"<html><head><title>http_test</title></head><body><pre>" "<html><head><title>http_test</title></head><body><pre>"
"Arguments([foo]=>62[type]=>mp4)</pre></body></html>"; "Arguments([foo]=>62[type]=>mp4)</pre></body></html>";
const char kDelayTwoSecs[] = "delay=2"; // This causes host to delay 2 seconds.
} // namespace } // namespace
namespace media { namespace media {
@ -46,27 +45,44 @@ static void CheckHttpPost(const std::string& url, const std::string& data,
} }
TEST(HttpFetcherTest, HttpGet) { TEST(DISABLED_HttpFetcherTest, HttpGet) {
CheckHttpGet(kTestUrl, kExpectedGetResponse); CheckHttpGet(kTestUrl, kExpectedGetResponse);
} }
TEST(HttpFetcherTest, HttpPost) { TEST(DISABLED_HttpFetcherTest, HttpPost) {
CheckHttpPost(kTestUrl, kPostData, kExpectedPostResponse); CheckHttpPost(kTestUrl, kPostData, kExpectedPostResponse);
} }
TEST(HttpFetcherTest, InvalidUrl) { TEST(DISABLED_HttpFetcherTest, InvalidUrl) {
const char kHttpNotFound[] = "404";
SimpleHttpFetcher fetcher; SimpleHttpFetcher fetcher;
std::string response; std::string response;
const std::string invalid_url(kTestUrl, sizeof(kTestUrl) - 2); const std::string invalid_url(kTestUrl, sizeof(kTestUrl) - 2);
Status status = fetcher.Get(invalid_url, &response); Status status = fetcher.Get(invalid_url, &response);
EXPECT_EQ(error::HTTP_FAILURE, status.error_code()); EXPECT_EQ(error::HTTP_FAILURE, status.error_code());
EXPECT_TRUE( EXPECT_NE(std::string::npos, status.error_message().find(kHttpNotFound));
EndsWith(status.error_message(), base::IntToString(kHttpNotFound), true));
} }
TEST(HttpFetcherTest, UrlWithPort) { TEST(DISABLED_HttpFetcherTest, UrlWithPort) {
CheckHttpGet(kTestUrlWithPort, kExpectedGetResponse); CheckHttpGet(kTestUrlWithPort, kExpectedGetResponse);
} }
TEST(DISABLED_HttpFetcherTest, SmallTimeout) {
const uint32 kTimeoutInSeconds = 1;
SimpleHttpFetcher fetcher(kTimeoutInSeconds);
std::string response;
Status status = fetcher.Post(kTestUrl, kDelayTwoSecs, &response);
EXPECT_EQ(error::TIME_OUT, status.error_code());
}
TEST(DISABLED_HttpFetcherTest, BigTimeout) {
const uint32 kTimeoutInSeconds = 5;
SimpleHttpFetcher fetcher(kTimeoutInSeconds);
std::string response;
Status status = fetcher.Post(kTestUrl, kDelayTwoSecs, &response);
EXPECT_OK(status);
}
} // namespace media } // namespace media

40
media/base/media_base.gyp
View File

@ -15,38 +15,6 @@
], ],
}, },
'targets': [ 'targets': [
{
'target_name': 'status',
'type': '<(component)',
'sources': [
'status.cc',
'status.h',
],
'dependencies': [
'../../base/base.gyp:base',
],
},
{
'target_name': 'http_fetcher',
'type': '<(component)',
'sources': [
'http_fetcher.cc',
'http_fetcher.h',
],
'cflags!': [ '-fno-exceptions' ],
'cflags_cc!': [ '-fno-exceptions' ],
'conditions': [
['OS=="mac"', {
'xcode_settings': {
'GCC_ENABLE_CPP_EXCEPTIONS': 'YES'
}
}]
],
'dependencies': [
'../../third_party/happyhttp/happyhttp.gyp:happyhttp_lib',
'status',
],
},
{ {
'target_name': 'base', 'target_name': 'base',
'type': '<(component)', 'type': '<(component)',
@ -76,6 +44,8 @@
'decryptor_source.h', 'decryptor_source.h',
'encryption_key_source.cc', 'encryption_key_source.cc',
'encryption_key_source.h', 'encryption_key_source.h',
'http_fetcher.cc',
'http_fetcher.h',
'limits.h', 'limits.h',
'media_parser.h', 'media_parser.h',
'media_sample.cc', 'media_sample.cc',
@ -95,6 +65,8 @@
'request_signer.h', 'request_signer.h',
'rsa_key.cc', 'rsa_key.cc',
'rsa_key.h', 'rsa_key.h',
'status.cc',
'status.h',
'stream_info.cc', 'stream_info.cc',
'stream_info.h', 'stream_info.h',
'text_track.h', 'text_track.h',
@ -106,9 +78,8 @@
], ],
'dependencies': [ 'dependencies': [
'../../base/base.gyp:base', '../../base/base.gyp:base',
'../../third_party/curl/curl.gyp:libcurl',
'../../third_party/openssl/openssl.gyp:openssl', '../../third_party/openssl/openssl.gyp:openssl',
'http_fetcher',
'status',
], ],
}, },
{ {
@ -123,6 +94,7 @@
'container_names_unittest.cc', 'container_names_unittest.cc',
'fake_prng.cc', # For rsa_key_unittest 'fake_prng.cc', # For rsa_key_unittest
'fake_prng.h', # For rsa_key_unittest 'fake_prng.h', # For rsa_key_unittest
'http_fetcher_unittest.cc',
'muxer_util_unittest.cc', 'muxer_util_unittest.cc',
'offset_byte_queue_unittest.cc', 'offset_byte_queue_unittest.cc',
'producer_consumer_queue_unittest.cc', 'producer_consumer_queue_unittest.cc',

38
third_party/curl/README.packager vendored Normal file
View File

@ -0,0 +1,38 @@
Name: curl
URL: http://curl.haxx.se/dev/
License: MIT/X
License File: source/COPYING
Local Modifications: None
Description:
libcurl is a free and easy-to-use client-side URL transfer library. libcurl
supports SSL certificates, HTTP GET, HTTP POST, HTTP PUT, and various other
transfer protocols.
**************************************************************************
Description of source tree.
1) config/
Directory containing configuration files, which are required to build
libcurl and curl correctly. On linux platform, an *auto-generated*
configuration file "config/linux/curl_config.h" is used; the library uses
the configuration files coming with libcurl distribution
"source/lib/config-*.h" on other platforms.
config/curl/curlbuild.h
Curl build file renamed from source/include/curl/curlbuild.h.dist
config/dummy_tool_hugehelp.c
A dummy manual required to build curl command line tool.
config/linux/curl_config.h
An *auto-generated* configuration file by running source/buildconf on
linux platform, with a few features disabled to build correctly on a
fresh linux box.
2) curl.gyp
A gyp build file for the library. Manually maintained.
3) source/
Directory containing curl source codes from github without modification.

11
third_party/happyhttp/README.packager vendored
View File

@ -1,11 +0,0 @@
Name: happyhttp
URL: https://github.com/Zintinio/HappyHTTP
Version: 0.1
License: zlib/libpng
License File: http://opensource.org/licenses/zlib-license.php
Description:
HappyHTTP is a simple C++ library for issuing HTTP requests and processing
responses.
Local Modifications: None

37
third_party/happyhttp/happyhttp.gyp vendored
View File

@ -1,37 +0,0 @@
# Copyright 2014 Google Inc. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
{
'targets': [
{
'target_name': 'happyhttp_lib',
'type': 'static_library',
'sources': [
'src/happyhttp.cpp',
'src/happyhttp.h',
],
'cflags!': [ '-fno-exceptions' ],
'cflags_cc!': [ '-fno-exceptions' ],
'conditions': [
['OS=="mac"', {
'xcode_settings': {
'GCC_ENABLE_CPP_EXCEPTIONS': 'YES'
}
}]
],
},
{
'target_name': 'happyhttp_lib_test',
'type': 'executable',
'sources': [
'src/test.cpp',
],
'dependencies': [
'happyhttp_lib',
],
},
],
}