440 lines
17 KiB
Python
440 lines
17 KiB
Python
|
# Copyright 2013 The Chromium Authors. All rights reserved.
|
||
|
|
# Use of this source code is governed by a BSD-style license that can be
|
||
|
|
# found in the LICENSE file.
|
||
|
|
|
||
|
|
"""A "Test Server Spawner" that handles killing/stopping per-test test servers.
|
||
|
|
|
||
|
|
It's used to accept requests from the device to spawn and kill instances of the
|
||
|
|
chrome test server on the host.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import BaseHTTPServer
|
||
|
|
import json
|
||
|
|
import logging
|
||
|
|
import os
|
||
|
|
import select
|
||
|
|
import struct
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
import threading
|
||
|
|
import time
|
||
|
|
import urlparse
|
||
|
|
|
||
|
|
import constants
|
||
|
|
import ports
|
||
|
|
|
||
|
|
from pylib.forwarder import Forwarder
|
||
|
|
|
||
|
|
# Path that are needed to import necessary modules when launching a testserver.
|
||
|
|
os.environ['PYTHONPATH'] = os.environ.get('PYTHONPATH', '') + (':%s:%s:%s:%s:%s'
|
||
|
|
% (os.path.join(constants.DIR_SOURCE_ROOT, 'third_party'),
|
||
|
|
os.path.join(constants.DIR_SOURCE_ROOT, 'third_party', 'tlslite'),
|
||
|
|
os.path.join(constants.DIR_SOURCE_ROOT, 'third_party', 'pyftpdlib',
|
||
|
|
'src'),
|
||
|
|
os.path.join(constants.DIR_SOURCE_ROOT, 'net', 'tools', 'testserver'),
|
||
|
|
os.path.join(constants.DIR_SOURCE_ROOT, 'sync', 'tools', 'testserver')))
|
||
|
|
|
||
|
|
|
||
|
|
SERVER_TYPES = {
|
||
|
|
'http': '',
|
||
|
|
'ftp': '-f',
|
||
|
|
'sync': '', # Sync uses its own script, and doesn't take a server type arg.
|
||
|
|
'tcpecho': '--tcp-echo',
|
||
|
|
'udpecho': '--udp-echo',
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
# The timeout (in seconds) of starting up the Python test server.
|
||
|
|
TEST_SERVER_STARTUP_TIMEOUT = 10
|
||
|
|
|
||
|
|
def _WaitUntil(predicate, max_attempts=5):
|
||
|
|
"""Blocks until the provided predicate (function) is true.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
Whether the provided predicate was satisfied once (before the timeout).
|
||
|
|
"""
|
||
|
|
sleep_time_sec = 0.025
|
||
|
|
for attempt in xrange(1, max_attempts):
|
||
|
|
if predicate():
|
||
|
|
return True
|
||
|
|
time.sleep(sleep_time_sec)
|
||
|
|
sleep_time_sec = min(1, sleep_time_sec * 2) # Don't wait more than 1 sec.
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
def _CheckPortStatus(port, expected_status):
|
||
|
|
"""Returns True if port has expected_status.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
port: the port number.
|
||
|
|
expected_status: boolean of expected status.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
Returns True if the status is expected. Otherwise returns False.
|
||
|
|
"""
|
||
|
|
return _WaitUntil(lambda: ports.IsHostPortUsed(port) == expected_status)
|
||
|
|
|
||
|
|
|
||
|
|
def _CheckDevicePortStatus(adb, port):
|
||
|
|
"""Returns whether the provided port is used."""
|
||
|
|
return _WaitUntil(lambda: ports.IsDevicePortUsed(adb, port))
|
||
|
|
|
||
|
|
|
||
|
|
def _GetServerTypeCommandLine(server_type):
|
||
|
|
"""Returns the command-line by the given server type.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
server_type: the server type to be used (e.g. 'http').
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
A string containing the command-line argument.
|
||
|
|
"""
|
||
|
|
if server_type not in SERVER_TYPES:
|
||
|
|
raise NotImplementedError('Unknown server type: %s' % server_type)
|
||
|
|
if server_type == 'udpecho':
|
||
|
|
raise Exception('Please do not run UDP echo tests because we do not have '
|
||
|
|
'a UDP forwarder tool.')
|
||
|
|
return SERVER_TYPES[server_type]
|
||
|
|
|
||
|
|
|
||
|
|
class TestServerThread(threading.Thread):
|
||
|
|
"""A thread to run the test server in a separate process."""
|
||
|
|
|
||
|
|
def __init__(self, ready_event, arguments, adb, tool, build_type):
|
||
|
|
"""Initialize TestServerThread with the following argument.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
ready_event: event which will be set when the test server is ready.
|
||
|
|
arguments: dictionary of arguments to run the test server.
|
||
|
|
adb: instance of AndroidCommands.
|
||
|
|
tool: instance of runtime error detection tool.
|
||
|
|
build_type: 'Release' or 'Debug'.
|
||
|
|
"""
|
||
|
|
threading.Thread.__init__(self)
|
||
|
|
self.wait_event = threading.Event()
|
||
|
|
self.stop_flag = False
|
||
|
|
self.ready_event = ready_event
|
||
|
|
self.ready_event.clear()
|
||
|
|
self.arguments = arguments
|
||
|
|
self.adb = adb
|
||
|
|
self.tool = tool
|
||
|
|
self.test_server_process = None
|
||
|
|
self.is_ready = False
|
||
|
|
self.host_port = self.arguments['port']
|
||
|
|
assert isinstance(self.host_port, int)
|
||
|
|
# The forwarder device port now is dynamically allocated.
|
||
|
|
self.forwarder_device_port = 0
|
||
|
|
# Anonymous pipe in order to get port info from test server.
|
||
|
|
self.pipe_in = None
|
||
|
|
self.pipe_out = None
|
||
|
|
self.command_line = []
|
||
|
|
self.build_type = build_type
|
||
|
|
|
||
|
|
def _WaitToStartAndGetPortFromTestServer(self):
|
||
|
|
"""Waits for the Python test server to start and gets the port it is using.
|
||
|
|
|
||
|
|
The port information is passed by the Python test server with a pipe given
|
||
|
|
by self.pipe_out. It is written as a result to |self.host_port|.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
Whether the port used by the test server was successfully fetched.
|
||
|
|
"""
|
||
|
|
assert self.host_port == 0 and self.pipe_out and self.pipe_in
|
||
|
|
(in_fds, _, _) = select.select([self.pipe_in, ], [], [],
|
||
|
|
TEST_SERVER_STARTUP_TIMEOUT)
|
||
|
|
if len(in_fds) == 0:
|
||
|
|
logging.error('Failed to wait to the Python test server to be started.')
|
||
|
|
return False
|
||
|
|
# First read the data length as an unsigned 4-byte value. This
|
||
|
|
# is _not_ using network byte ordering since the Python test server packs
|
||
|
|
# size as native byte order and all Chromium platforms so far are
|
||
|
|
# configured to use little-endian.
|
||
|
|
# TODO(jnd): Change the Python test server and local_test_server_*.cc to
|
||
|
|
# use a unified byte order (either big-endian or little-endian).
|
||
|
|
data_length = os.read(self.pipe_in, struct.calcsize('=L'))
|
||
|
|
if data_length:
|
||
|
|
(data_length,) = struct.unpack('=L', data_length)
|
||
|
|
assert data_length
|
||
|
|
if not data_length:
|
||
|
|
logging.error('Failed to get length of server data.')
|
||
|
|
return False
|
||
|
|
port_json = os.read(self.pipe_in, data_length)
|
||
|
|
if not port_json:
|
||
|
|
logging.error('Failed to get server data.')
|
||
|
|
return False
|
||
|
|
logging.info('Got port json data: %s', port_json)
|
||
|
|
port_json = json.loads(port_json)
|
||
|
|
if port_json.has_key('port') and isinstance(port_json['port'], int):
|
||
|
|
self.host_port = port_json['port']
|
||
|
|
return _CheckPortStatus(self.host_port, True)
|
||
|
|
logging.error('Failed to get port information from the server data.')
|
||
|
|
return False
|
||
|
|
|
||
|
|
def _GenerateCommandLineArguments(self):
|
||
|
|
"""Generates the command line to run the test server.
|
||
|
|
|
||
|
|
Note that all options are processed by following the definitions in
|
||
|
|
testserver.py.
|
||
|
|
"""
|
||
|
|
if self.command_line:
|
||
|
|
return
|
||
|
|
# The following arguments must exist.
|
||
|
|
type_cmd = _GetServerTypeCommandLine(self.arguments['server-type'])
|
||
|
|
if type_cmd:
|
||
|
|
self.command_line.append(type_cmd)
|
||
|
|
self.command_line.append('--port=%d' % self.host_port)
|
||
|
|
# Use a pipe to get the port given by the instance of Python test server
|
||
|
|
# if the test does not specify the port.
|
||
|
|
if self.host_port == 0:
|
||
|
|
(self.pipe_in, self.pipe_out) = os.pipe()
|
||
|
|
self.command_line.append('--startup-pipe=%d' % self.pipe_out)
|
||
|
|
self.command_line.append('--host=%s' % self.arguments['host'])
|
||
|
|
data_dir = self.arguments['data-dir'] or 'chrome/test/data'
|
||
|
|
if not os.path.isabs(data_dir):
|
||
|
|
data_dir = os.path.join(constants.DIR_SOURCE_ROOT, data_dir)
|
||
|
|
self.command_line.append('--data-dir=%s' % data_dir)
|
||
|
|
# The following arguments are optional depending on the individual test.
|
||
|
|
if self.arguments.has_key('log-to-console'):
|
||
|
|
self.command_line.append('--log-to-console')
|
||
|
|
if self.arguments.has_key('auth-token'):
|
||
|
|
self.command_line.append('--auth-token=%s' % self.arguments['auth-token'])
|
||
|
|
if self.arguments.has_key('https'):
|
||
|
|
self.command_line.append('--https')
|
||
|
|
if self.arguments.has_key('cert-and-key-file'):
|
||
|
|
self.command_line.append('--cert-and-key-file=%s' % os.path.join(
|
||
|
|
constants.DIR_SOURCE_ROOT, self.arguments['cert-and-key-file']))
|
||
|
|
if self.arguments.has_key('ocsp'):
|
||
|
|
self.command_line.append('--ocsp=%s' % self.arguments['ocsp'])
|
||
|
|
if self.arguments.has_key('https-record-resume'):
|
||
|
|
self.command_line.append('--https-record-resume')
|
||
|
|
if self.arguments.has_key('ssl-client-auth'):
|
||
|
|
self.command_line.append('--ssl-client-auth')
|
||
|
|
if self.arguments.has_key('tls-intolerant'):
|
||
|
|
self.command_line.append('--tls-intolerant=%s' %
|
||
|
|
self.arguments['tls-intolerant'])
|
||
|
|
if self.arguments.has_key('ssl-client-ca'):
|
||
|
|
for ca in self.arguments['ssl-client-ca']:
|
||
|
|
self.command_line.append('--ssl-client-ca=%s' %
|
||
|
|
os.path.join(constants.DIR_SOURCE_ROOT, ca))
|
||
|
|
if self.arguments.has_key('ssl-bulk-cipher'):
|
||
|
|
for bulk_cipher in self.arguments['ssl-bulk-cipher']:
|
||
|
|
self.command_line.append('--ssl-bulk-cipher=%s' % bulk_cipher)
|
||
|
|
|
||
|
|
def _CloseUnnecessaryFDsForTestServerProcess(self):
|
||
|
|
# This is required to avoid subtle deadlocks that could be caused by the
|
||
|
|
# test server child process inheriting undesirable file descriptors such as
|
||
|
|
# file lock file descriptors.
|
||
|
|
for fd in xrange(0, 1024):
|
||
|
|
if fd != self.pipe_out:
|
||
|
|
try:
|
||
|
|
os.close(fd)
|
||
|
|
except:
|
||
|
|
pass
|
||
|
|
|
||
|
|
def run(self):
|
||
|
|
logging.info('Start running the thread!')
|
||
|
|
self.wait_event.clear()
|
||
|
|
self._GenerateCommandLineArguments()
|
||
|
|
command = constants.DIR_SOURCE_ROOT
|
||
|
|
if self.arguments['server-type'] == 'sync':
|
||
|
|
command = [os.path.join(command, 'sync', 'tools', 'testserver',
|
||
|
|
'sync_testserver.py')] + self.command_line
|
||
|
|
else:
|
||
|
|
command = [os.path.join(command, 'net', 'tools', 'testserver',
|
||
|
|
'testserver.py')] + self.command_line
|
||
|
|
logging.info('Running: %s', command)
|
||
|
|
self.process = subprocess.Popen(
|
||
|
|
command, preexec_fn=self._CloseUnnecessaryFDsForTestServerProcess)
|
||
|
|
if self.process:
|
||
|
|
if self.pipe_out:
|
||
|
|
self.is_ready = self._WaitToStartAndGetPortFromTestServer()
|
||
|
|
else:
|
||
|
|
self.is_ready = _CheckPortStatus(self.host_port, True)
|
||
|
|
if self.is_ready:
|
||
|
|
Forwarder.Map([(0, self.host_port)], self.adb, self.build_type, self.tool)
|
||
|
|
# Check whether the forwarder is ready on the device.
|
||
|
|
self.is_ready = False
|
||
|
|
device_port = Forwarder.DevicePortForHostPort(self.host_port)
|
||
|
|
if device_port and _CheckDevicePortStatus(self.adb, device_port):
|
||
|
|
self.is_ready = True
|
||
|
|
self.forwarder_device_port = device_port
|
||
|
|
# Wake up the request handler thread.
|
||
|
|
self.ready_event.set()
|
||
|
|
# Keep thread running until Stop() gets called.
|
||
|
|
_WaitUntil(lambda: self.stop_flag, max_attempts=sys.maxint)
|
||
|
|
if self.process.poll() is None:
|
||
|
|
self.process.kill()
|
||
|
|
Forwarder.UnmapDevicePort(self.forwarder_device_port, self.adb)
|
||
|
|
self.process = None
|
||
|
|
self.is_ready = False
|
||
|
|
if self.pipe_out:
|
||
|
|
os.close(self.pipe_in)
|
||
|
|
os.close(self.pipe_out)
|
||
|
|
self.pipe_in = None
|
||
|
|
self.pipe_out = None
|
||
|
|
logging.info('Test-server has died.')
|
||
|
|
self.wait_event.set()
|
||
|
|
|
||
|
|
def Stop(self):
|
||
|
|
"""Blocks until the loop has finished.
|
||
|
|
|
||
|
|
Note that this must be called in another thread.
|
||
|
|
"""
|
||
|
|
if not self.process:
|
||
|
|
return
|
||
|
|
self.stop_flag = True
|
||
|
|
self.wait_event.wait()
|
||
|
|
|
||
|
|
|
||
|
|
class SpawningServerRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
|
||
|
|
"""A handler used to process http GET/POST request."""
|
||
|
|
|
||
|
|
def _SendResponse(self, response_code, response_reason, additional_headers,
|
||
|
|
contents):
|
||
|
|
"""Generates a response sent to the client from the provided parameters.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
response_code: number of the response status.
|
||
|
|
response_reason: string of reason description of the response.
|
||
|
|
additional_headers: dict of additional headers. Each key is the name of
|
||
|
|
the header, each value is the content of the header.
|
||
|
|
contents: string of the contents we want to send to client.
|
||
|
|
"""
|
||
|
|
self.send_response(response_code, response_reason)
|
||
|
|
self.send_header('Content-Type', 'text/html')
|
||
|
|
# Specify the content-length as without it the http(s) response will not
|
||
|
|
# be completed properly (and the browser keeps expecting data).
|
||
|
|
self.send_header('Content-Length', len(contents))
|
||
|
|
for header_name in additional_headers:
|
||
|
|
self.send_header(header_name, additional_headers[header_name])
|
||
|
|
self.end_headers()
|
||
|
|
self.wfile.write(contents)
|
||
|
|
self.wfile.flush()
|
||
|
|
|
||
|
|
def _StartTestServer(self):
|
||
|
|
"""Starts the test server thread."""
|
||
|
|
logging.info('Handling request to spawn a test server.')
|
||
|
|
content_type = self.headers.getheader('content-type')
|
||
|
|
if content_type != 'application/json':
|
||
|
|
raise Exception('Bad content-type for start request.')
|
||
|
|
content_length = self.headers.getheader('content-length')
|
||
|
|
if not content_length:
|
||
|
|
content_length = 0
|
||
|
|
try:
|
||
|
|
content_length = int(content_length)
|
||
|
|
except:
|
||
|
|
raise Exception('Bad content-length for start request.')
|
||
|
|
logging.info(content_length)
|
||
|
|
test_server_argument_json = self.rfile.read(content_length)
|
||
|
|
logging.info(test_server_argument_json)
|
||
|
|
assert not self.server.test_server_instance
|
||
|
|
ready_event = threading.Event()
|
||
|
|
self.server.test_server_instance = TestServerThread(
|
||
|
|
ready_event,
|
||
|
|
json.loads(test_server_argument_json),
|
||
|
|
self.server.adb,
|
||
|
|
self.server.tool,
|
||
|
|
self.server.build_type)
|
||
|
|
self.server.test_server_instance.setDaemon(True)
|
||
|
|
self.server.test_server_instance.start()
|
||
|
|
ready_event.wait()
|
||
|
|
if self.server.test_server_instance.is_ready:
|
||
|
|
self._SendResponse(200, 'OK', {}, json.dumps(
|
||
|
|
{'port': self.server.test_server_instance.forwarder_device_port,
|
||
|
|
'message': 'started'}))
|
||
|
|
logging.info('Test server is running on port: %d.',
|
||
|
|
self.server.test_server_instance.host_port)
|
||
|
|
else:
|
||
|
|
self.server.test_server_instance.Stop()
|
||
|
|
self.server.test_server_instance = None
|
||
|
|
self._SendResponse(500, 'Test Server Error.', {}, '')
|
||
|
|
logging.info('Encounter problem during starting a test server.')
|
||
|
|
|
||
|
|
def _KillTestServer(self):
|
||
|
|
"""Stops the test server instance."""
|
||
|
|
# There should only ever be one test server at a time. This may do the
|
||
|
|
# wrong thing if we try and start multiple test servers.
|
||
|
|
if not self.server.test_server_instance:
|
||
|
|
return
|
||
|
|
port = self.server.test_server_instance.host_port
|
||
|
|
logging.info('Handling request to kill a test server on port: %d.', port)
|
||
|
|
self.server.test_server_instance.Stop()
|
||
|
|
# Make sure the status of test server is correct before sending response.
|
||
|
|
if _CheckPortStatus(port, False):
|
||
|
|
self._SendResponse(200, 'OK', {}, 'killed')
|
||
|
|
logging.info('Test server on port %d is killed', port)
|
||
|
|
else:
|
||
|
|
self._SendResponse(500, 'Test Server Error.', {}, '')
|
||
|
|
logging.info('Encounter problem during killing a test server.')
|
||
|
|
self.server.test_server_instance = None
|
||
|
|
|
||
|
|
def do_POST(self):
|
||
|
|
parsed_path = urlparse.urlparse(self.path)
|
||
|
|
action = parsed_path.path
|
||
|
|
logging.info('Action for POST method is: %s.', action)
|
||
|
|
if action == '/start':
|
||
|
|
self._StartTestServer()
|
||
|
|
else:
|
||
|
|
self._SendResponse(400, 'Unknown request.', {}, '')
|
||
|
|
logging.info('Encounter unknown request: %s.', action)
|
||
|
|
|
||
|
|
def do_GET(self):
|
||
|
|
parsed_path = urlparse.urlparse(self.path)
|
||
|
|
action = parsed_path.path
|
||
|
|
params = urlparse.parse_qs(parsed_path.query, keep_blank_values=1)
|
||
|
|
logging.info('Action for GET method is: %s.', action)
|
||
|
|
for param in params:
|
||
|
|
logging.info('%s=%s', param, params[param][0])
|
||
|
|
if action == '/kill':
|
||
|
|
self._KillTestServer()
|
||
|
|
elif action == '/ping':
|
||
|
|
# The ping handler is used to check whether the spawner server is ready
|
||
|
|
# to serve the requests. We don't need to test the status of the test
|
||
|
|
# server when handling ping request.
|
||
|
|
self._SendResponse(200, 'OK', {}, 'ready')
|
||
|
|
logging.info('Handled ping request and sent response.')
|
||
|
|
else:
|
||
|
|
self._SendResponse(400, 'Unknown request', {}, '')
|
||
|
|
logging.info('Encounter unknown request: %s.', action)
|
||
|
|
|
||
|
|
|
||
|
|
class SpawningServer(object):
|
||
|
|
"""The class used to start/stop a http server."""
|
||
|
|
|
||
|
|
def __init__(self, test_server_spawner_port, adb, tool, build_type):
|
||
|
|
logging.info('Creating new spawner on port: %d.', test_server_spawner_port)
|
||
|
|
self.server = BaseHTTPServer.HTTPServer(('', test_server_spawner_port),
|
||
|
|
SpawningServerRequestHandler)
|
||
|
|
self.server.adb = adb
|
||
|
|
self.server.tool = tool
|
||
|
|
self.server.test_server_instance = None
|
||
|
|
self.server.build_type = build_type
|
||
|
|
|
||
|
|
def _Listen(self):
|
||
|
|
logging.info('Starting test server spawner')
|
||
|
|
self.server.serve_forever()
|
||
|
|
|
||
|
|
def Start(self):
|
||
|
|
"""Starts the test server spawner."""
|
||
|
|
listener_thread = threading.Thread(target=self._Listen)
|
||
|
|
listener_thread.setDaemon(True)
|
||
|
|
listener_thread.start()
|
||
|
|
|
||
|
|
def Stop(self):
|
||
|
|
"""Stops the test server spawner.
|
||
|
|
|
||
|
|
Also cleans the server state.
|
||
|
|
"""
|
||
|
|
self.CleanupState()
|
||
|
|
self.server.shutdown()
|
||
|
|
|
||
|
|
def CleanupState(self):
|
||
|
|
"""Cleans up the spawning server state.
|
||
|
|
|
||
|
|
This should be called if the test server spawner is reused,
|
||
|
|
to avoid sharing the test server instance.
|
||
|
|
"""
|
||
|
|
if self.server.test_server_instance:
|
||
|
|
self.server.test_server_instance.Stop()
|
||
|
|
self.server.test_server_instance = None
|