From d4fcfb2f4f8f2328d8077f5099e75f08f7d1495e Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 13 Jul 2023 16:36:42 -0700 Subject: [PATCH] test: Add Mongoose embedded HTTP server (#1231) This adds Mongoose as a third-party library, and builds on top of that an embedded HTTP server for our unit tests. We are using a fork of Mongoose pending the merging of this PR: https://github.com/cesanta/mongoose/pull/2301 The embedded web server will make our HTTP-based tests independent of httpbin.org, which will make them quick and reliable. --- .gitignore | 1 + .gitmodules | 3 + packager/file/CMakeLists.txt | 3 +- packager/media/test/CMakeLists.txt | 10 +- packager/media/test/test_web_server.cc | 259 +++++++++++++++++++ packager/media/test/test_web_server.h | 70 +++++ packager/third_party/CMakeLists.txt | 1 + packager/third_party/mongoose/CMakeLists.txt | 15 ++ packager/third_party/mongoose/source | 1 + 9 files changed, 361 insertions(+), 2 deletions(-) create mode 100644 packager/media/test/test_web_server.cc create mode 100644 packager/media/test/test_web_server.h create mode 100644 packager/third_party/mongoose/CMakeLists.txt create mode 160000 packager/third_party/mongoose/source diff --git a/.gitignore b/.gitignore index 5a7e5d5d9b..1cca919155 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc *.sln +*.swp *.VC.db *.vcxproj* */.vs/* diff --git a/.gitmodules b/.gitmodules index c528969fd4..979da1c90b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -31,3 +31,6 @@ [submodule "packager/third_party/protobuf/source"] path = packager/third_party/protobuf/source url = https://github.com/protocolbuffers/protobuf +[submodule "packager/third_party/mongoose/source"] + path = packager/third_party/mongoose/source + url = https://github.com/joeyparrish/mongoose diff --git a/packager/file/CMakeLists.txt b/packager/file/CMakeLists.txt index 79396d36b0..9fa7de5fcb 100644 --- a/packager/file/CMakeLists.txt +++ b/packager/file/CMakeLists.txt @@ -48,5 +48,6 @@ target_link_libraries(file_unittest gmock gtest gtest_main - nlohmann_json) + nlohmann_json + test_web_server) add_test(NAME file_unittest COMMAND file_unittest) diff --git a/packager/media/test/CMakeLists.txt b/packager/media/test/CMakeLists.txt index 41d890b437..782879531f 100644 --- a/packager/media/test/CMakeLists.txt +++ b/packager/media/test/CMakeLists.txt @@ -6,4 +6,12 @@ add_library(test_data_util STATIC test_data_util.cc) -target_link_libraries(test_data_util glog) +target_link_libraries(test_data_util + glog) +add_library(test_web_server STATIC + test_web_server.cc) +target_link_libraries(test_web_server + absl::str_format + absl::strings + mongoose + nlohmann_json) diff --git a/packager/media/test/test_web_server.cc b/packager/media/test/test_web_server.cc new file mode 100644 index 0000000000..1d4fb9ac0f --- /dev/null +++ b/packager/media/test/test_web_server.cc @@ -0,0 +1,259 @@ +// Copyright 2023 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "packager/media/test/test_web_server.h" + +#include +#include + +#include "absl/strings/numbers.h" +#include "absl/strings/str_format.h" +#include "mongoose.h" +#include "nlohmann/json.hpp" + +// A full replacement for our former use of httpbin.org in tests. This +// embedded web server can: +// +// 1. Reflect the request method, body, and headers +// 2. Return a requested status code +// 3. Delay a response by a requested amount of time + +namespace { + +// Get a string_view on mongoose's mg_string, which may not be nul-terminated. +std::string_view MongooseStringView(const mg_str& mg_string) { + return std::string_view(mg_string.ptr, mg_string.len); +} + +bool IsMongooseStringNull(const mg_str& mg_string) { + return mg_string.ptr == NULL; +} + +bool IsMongooseStringNullOrBlank(const mg_str& mg_string) { + return mg_string.ptr == NULL || mg_string.len == 0; +} + +// Get a string query parameter from a mongoose HTTP message. +bool GetStringQueryParameter(struct mg_http_message* message, + const char* name, + std::string_view* str) { + struct mg_str value_mg_str = mg_http_var(message->query, mg_str(name)); + + if (!IsMongooseStringNull(value_mg_str)) { + *str = MongooseStringView(value_mg_str); + return true; + } + + return false; +} + +// Get an integer query parameter from a mongoose HTTP message. +bool GetIntQueryParameter(struct mg_http_message* message, + const char* name, + int* value) { + std::string_view str; + + if (GetStringQueryParameter(message, name, &str)) { + return absl::SimpleAtoi(str, value); + } + + return false; +} + +} // namespace + +namespace shaka { +namespace media { + +TestWebServer::TestWebServer() : status_(kNew), stopped_(false) {} + +TestWebServer::~TestWebServer() { + { + absl::MutexLock lock(&mutex_); + stop_.Signal(); + stopped_ = true; + } + if (thread_) { + thread_->join(); + } + thread_.reset(); +} + +bool TestWebServer::Start(int port) { + thread_.reset(new std::thread(&TestWebServer::ThreadCallback, this, port)); + + absl::MutexLock lock(&mutex_); + while (status_ == kNew) { + started_.Wait(&mutex_); + } + + return status_ == kStarted; +} + +void TestWebServer::ThreadCallback(int port) { + // Mongoose needs an HTTP server address in string format. + // "127.0.0.1" is "localhost", and is not visible to other machines on the + // network. + std::string http_address = absl::StrFormat("http://127.0.0.1:%d", port); + + // Set up the manager structure to be automatically cleaned up when it leaves + // scope. + std::unique_ptr manager( + new struct mg_mgr, mg_mgr_free); + // Then initialize it. + mg_mgr_init(manager.get()); + + auto connection = + mg_http_listen(manager.get(), http_address.c_str(), + &TestWebServer::HandleEvent, this /* callback_data */); + if (connection == NULL) { + // Failed to listen to the requested port. Mongoose has already printed an + // error message. + absl::MutexLock lock(&mutex_); + status_ = kFailed; + started_.Signal(); + return; + } + + { + absl::MutexLock lock(&mutex_); + status_ = kStarted; + started_.Signal(); + } + + bool stopped = false; + while (!stopped) { + // Let Mongoose poll the sockets for 100ms. + mg_mgr_poll(manager.get(), 100); + + // Check for a stop signal from the test. + { + absl::MutexLock lock(&mutex_); + stopped = stopped_; + if (stopped) + status_ = kStopped; + } + } +} + +// static +void TestWebServer::HandleEvent(struct mg_connection* connection, + int event, + void* event_data, + void* callback_data) { + TestWebServer* instance = static_cast(callback_data); + + if (event == MG_EV_POLL) { + std::vector to_delete; + + // Check if it's time to re-handle any delayed connections. + for (const auto& pair : instance->delayed_connections_) { + const auto delayed_connection = pair.first; + const auto deadline = pair.second; + if (deadline <= absl::Now()) { + to_delete.push_back(delayed_connection); + instance->HandleDelay(NULL, delayed_connection); + } + } + + // Now that we're done iterating the map, delete any connections we are done + // responding to. + for (const auto& delayed_connection : to_delete) { + instance->delayed_connections_.erase(delayed_connection); + } + } else if (event == MG_EV_CLOSE) { + if (instance->delayed_connections_.count(connection)) { + // The client hung up before our delay expired. Remove this from our map. + instance->delayed_connections_.erase(connection); + } + } + + if (event != MG_EV_HTTP_MSG) + return; + + struct mg_http_message* message = + static_cast(event_data); + if (mg_http_match_uri(message, "/reflect")) { + if (instance->HandleReflect(message, connection)) + return; + } else if (mg_http_match_uri(message, "/status")) { + if (instance->HandleStatus(message, connection)) + return; + } else if (mg_http_match_uri(message, "/delay")) { + if (instance->HandleDelay(message, connection)) + return; + } + + mg_http_reply(connection, 400 /* bad request */, NULL /* headers */, + "Bad request!"); +} + +bool TestWebServer::HandleStatus(struct mg_http_message* message, + struct mg_connection* connection) { + int code = 0; + + if (GetIntQueryParameter(message, "code", &code)) { + // Reply with the requested status code. + mg_http_reply(connection, code, NULL /* headers */, "%s", "{}"); + return true; + } + + return false; +} + +bool TestWebServer::HandleDelay(struct mg_http_message* message, + struct mg_connection* connection) { + if (delayed_connections_.count(connection)) { + // We're being called back after a delay has elapsed. + // Respond now. + mg_http_reply(connection, 200 /* OK */, NULL /* headers */, "%s", "{}"); + return true; + } + + int seconds = 0; + // Checking |message| here is a small safety measure, since we call this + // method back a second time with message set to NULL. That is supposed to + // be handled above, but this is defense in depth against a crash. + if (message && GetIntQueryParameter(message, "seconds", &seconds)) { + // We can't block this thread, so compute the deadline and add the + // connection to a map. The main handler will call us back later if the + // client doesn't hang up first. + absl::Time deadline = absl::Now() + absl::Seconds(seconds); + delayed_connections_[connection] = deadline; + return true; + } + + return false; +} + +bool TestWebServer::HandleReflect(struct mg_http_message* message, + struct mg_connection* connection) { + // Serialize a reply in JSON that reflects the request method, body, and + // headers. + nlohmann::json reply; + reply["method"] = MongooseStringView(message->method); + if (!IsMongooseStringNull(message->body)) { + reply["body"] = MongooseStringView(message->body); + } + + nlohmann::json headers; + for (int i = 0; i < MG_MAX_HTTP_HEADERS; ++i) { + struct mg_http_header header = message->headers[i]; + if (IsMongooseStringNullOrBlank(header.name)) { + break; + } + + headers[MongooseStringView(header.name)] = MongooseStringView(header.value); + } + reply["headers"] = headers; + + mg_http_reply(connection, 200 /* OK */, NULL /* headers */, "%s\n", + reply.dump().c_str()); + return true; +} + +} // namespace media +} // namespace shaka diff --git a/packager/media/test/test_web_server.h b/packager/media/test/test_web_server.h new file mode 100644 index 0000000000..d6e0c5f20b --- /dev/null +++ b/packager/media/test/test_web_server.h @@ -0,0 +1,70 @@ +// Copyright 2023 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef PACKAGER_MEDIA_TEST_TEST_WEB_SERVER_H_ +#define PACKAGER_MEDIA_TEST_TEST_WEB_SERVER_H_ + +#include +#include +#include + +#include "absl/synchronization/mutex.h" +#include "absl/time/time.h" + +// Forward declare mongoose struct types, used as pointers below. +struct mg_connection; +struct mg_http_message; + +namespace shaka { +namespace media { + +class TestWebServer { + public: + TestWebServer(); + ~TestWebServer(); + + bool Start(int port); + + private: + enum TestWebServerStatus { + kNew, + kFailed, + kStarted, + kStopped, + }; + + absl::Mutex mutex_; + TestWebServerStatus status_ GUARDED_BY(mutex_); + absl::CondVar started_ GUARDED_BY(mutex_); + absl::CondVar stop_ GUARDED_BY(mutex_); + bool stopped_ GUARDED_BY(mutex_); + + // Connections to be handled again later, mapped to the time at which we + // should handle them again. We can't block the server thread directly to + // simulate delays. Only ever accessed from |thread_|. + std::map delayed_connections_; + + std::unique_ptr thread_; + + void ThreadCallback(int port); + + static void HandleEvent(struct mg_connection* connection, + int event, + void* event_data, + void* callback_data); + + bool HandleStatus(struct mg_http_message* message, + struct mg_connection* connection); + bool HandleDelay(struct mg_http_message* message, + struct mg_connection* connection); + bool HandleReflect(struct mg_http_message* message, + struct mg_connection* connection); +}; + +} // namespace media +} // namespace shaka + +#endif // PACKAGER_MEDIA_TEST_TEST_WEB_SERVER_H_ diff --git a/packager/third_party/CMakeLists.txt b/packager/third_party/CMakeLists.txt index f19b3fcdef..dba432e8a0 100644 --- a/packager/third_party/CMakeLists.txt +++ b/packager/third_party/CMakeLists.txt @@ -35,5 +35,6 @@ add_subdirectory(libpng EXCLUDE_FROM_ALL) add_subdirectory(libwebm EXCLUDE_FROM_ALL) add_subdirectory(libxml2 EXCLUDE_FROM_ALL) add_subdirectory(mbedtls EXCLUDE_FROM_ALL) +add_subdirectory(mongoose EXCLUDE_FROM_ALL) add_subdirectory(protobuf EXCLUDE_FROM_ALL) add_subdirectory(zlib EXCLUDE_FROM_ALL) diff --git a/packager/third_party/mongoose/CMakeLists.txt b/packager/third_party/mongoose/CMakeLists.txt new file mode 100644 index 0000000000..c590633167 --- /dev/null +++ b/packager/third_party/mongoose/CMakeLists.txt @@ -0,0 +1,15 @@ +# Copyright 2023 Google LLC. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +# CMake build file for the mongoose library, which is used as a built-in web +# server for testing certain HTTP client features of Packager. + +# Mongoose does not have its own CMakeLists.txt, but mongoose is very simple. + +add_library(mongoose STATIC + source/mongoose.c) +target_include_directories(mongoose + PUBLIC source/) diff --git a/packager/third_party/mongoose/source b/packager/third_party/mongoose/source new file mode 160000 index 0000000000..25650bd9be --- /dev/null +++ b/packager/third_party/mongoose/source @@ -0,0 +1 @@ +Subproject commit 25650bd9be1578d8cc28881ca1255392a53c6123