2457 lines
79 KiB
Python
Executable File
2457 lines
79 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""Performance Test Bisect Tool
|
|
|
|
This script bisects a series of changelists using binary search. It starts at
|
|
a bad revision where a performance metric has regressed, and asks for a last
|
|
known-good revision. It will then binary search across this revision range by
|
|
syncing, building, and running a performance test. If the change is
|
|
suspected to occur as a result of WebKit/V8 changes, the script will
|
|
further bisect changes to those depots and attempt to narrow down the revision
|
|
range.
|
|
|
|
|
|
An example usage (using svn cl's):
|
|
|
|
./tools/bisect-perf-regression.py -c\
|
|
"out/Release/performance_ui_tests --gtest_filter=ShutdownTest.SimpleUserQuit"\
|
|
-g 168222 -b 168232 -m shutdown/simple-user-quit
|
|
|
|
Be aware that if you're using the git workflow and specify an svn revision,
|
|
the script will attempt to find the git SHA1 where svn changes up to that
|
|
revision were merged in.
|
|
|
|
|
|
An example usage (using git hashes):
|
|
|
|
./tools/bisect-perf-regression.py -c\
|
|
"out/Release/performance_ui_tests --gtest_filter=ShutdownTest.SimpleUserQuit"\
|
|
-g 1f6e67861535121c5c819c16a666f2436c207e7b\
|
|
-b b732f23b4f81c382db0b23b9035f3dadc7d925bb\
|
|
-m shutdown/simple-user-quit
|
|
|
|
"""
|
|
|
|
import errno
|
|
import imp
|
|
import math
|
|
import optparse
|
|
import os
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
import bisect_utils
|
|
|
|
|
|
# The additional repositories that might need to be bisected.
|
|
# If the repository has any dependant repositories (such as skia/src needs
|
|
# skia/include and skia/gyp to be updated), specify them in the 'depends'
|
|
# so that they're synced appropriately.
|
|
# Format is:
|
|
# src: path to the working directory.
|
|
# recurse: True if this repositry will get bisected.
|
|
# depends: A list of other repositories that are actually part of the same
|
|
# repository in svn.
|
|
# svn: Needed for git workflow to resolve hashes to svn revisions.
|
|
# from: Parent depot that must be bisected before this is bisected.
|
|
DEPOT_DEPS_NAME = {
|
|
'chromium' : {
|
|
"src" : "src/",
|
|
"recurse" : True,
|
|
"depends" : None,
|
|
"from" : 'cros'
|
|
},
|
|
'webkit' : {
|
|
"src" : "src/third_party/WebKit",
|
|
"recurse" : True,
|
|
"depends" : None,
|
|
"from" : 'chromium'
|
|
},
|
|
'v8' : {
|
|
"src" : "src/v8",
|
|
"recurse" : True,
|
|
"depends" : None,
|
|
"build_with": 'v8_bleeding_edge',
|
|
"from" : 'chromium',
|
|
"custom_deps": bisect_utils.GCLIENT_CUSTOM_DEPS_V8
|
|
},
|
|
'v8_bleeding_edge' : {
|
|
"src" : "src/v8_bleeding_edge",
|
|
"recurse" : False,
|
|
"depends" : None,
|
|
"svn": "https://v8.googlecode.com/svn/branches/bleeding_edge",
|
|
"from" : 'chromium'
|
|
},
|
|
'skia/src' : {
|
|
"src" : "src/third_party/skia/src",
|
|
"recurse" : True,
|
|
"svn" : "http://skia.googlecode.com/svn/trunk/src",
|
|
"depends" : ['skia/include', 'skia/gyp'],
|
|
"from" : 'chromium'
|
|
},
|
|
'skia/include' : {
|
|
"src" : "src/third_party/skia/include",
|
|
"recurse" : False,
|
|
"svn" : "http://skia.googlecode.com/svn/trunk/include",
|
|
"depends" : None,
|
|
"from" : 'chromium'
|
|
},
|
|
'skia/gyp' : {
|
|
"src" : "src/third_party/skia/gyp",
|
|
"recurse" : False,
|
|
"svn" : "http://skia.googlecode.com/svn/trunk/gyp",
|
|
"depends" : None,
|
|
"from" : 'chromium'
|
|
}
|
|
}
|
|
|
|
DEPOT_NAMES = DEPOT_DEPS_NAME.keys()
|
|
CROS_SDK_PATH = os.path.join('..', 'cros', 'chromite', 'bin', 'cros_sdk')
|
|
CROS_VERSION_PATTERN = 'new version number from %s'
|
|
CROS_CHROMEOS_PATTERN = 'chromeos-base/chromeos-chrome'
|
|
CROS_TEST_KEY_PATH = os.path.join('..', 'cros', 'chromite', 'ssh_keys',
|
|
'testing_rsa')
|
|
CROS_SCRIPT_KEY_PATH = os.path.join('..', 'cros', 'src', 'scripts',
|
|
'mod_for_test_scripts', 'ssh_keys',
|
|
'testing_rsa')
|
|
|
|
BUILD_RESULT_SUCCEED = 0
|
|
BUILD_RESULT_FAIL = 1
|
|
BUILD_RESULT_SKIPPED = 2
|
|
|
|
def CalculateTruncatedMean(data_set, truncate_percent):
|
|
"""Calculates the truncated mean of a set of values.
|
|
|
|
Args:
|
|
data_set: Set of values to use in calculation.
|
|
truncate_percent: The % from the upper/lower portions of the data set to
|
|
discard, expressed as a value in [0, 1].
|
|
|
|
Returns:
|
|
The truncated mean as a float.
|
|
"""
|
|
if len(data_set) > 2:
|
|
data_set = sorted(data_set)
|
|
|
|
discard_num_float = len(data_set) * truncate_percent
|
|
discard_num_int = int(math.floor(discard_num_float))
|
|
kept_weight = len(data_set) - discard_num_float * 2
|
|
|
|
data_set = data_set[discard_num_int:len(data_set)-discard_num_int]
|
|
|
|
weight_left = 1.0 - (discard_num_float - discard_num_int)
|
|
|
|
if weight_left < 1:
|
|
# If the % to discard leaves a fractional portion, need to weight those
|
|
# values.
|
|
unweighted_vals = data_set[1:len(data_set)-1]
|
|
weighted_vals = [data_set[0], data_set[len(data_set)-1]]
|
|
weighted_vals = [w * weight_left for w in weighted_vals]
|
|
data_set = weighted_vals + unweighted_vals
|
|
else:
|
|
kept_weight = len(data_set)
|
|
|
|
truncated_mean = reduce(lambda x, y: float(x) + float(y),
|
|
data_set) / kept_weight
|
|
|
|
return truncated_mean
|
|
|
|
|
|
def CalculateStandardDeviation(v):
|
|
if len(v) == 1:
|
|
return 0.0
|
|
|
|
mean = CalculateTruncatedMean(v, 0.0)
|
|
variances = [float(x) - mean for x in v]
|
|
variances = [x * x for x in variances]
|
|
variance = reduce(lambda x, y: float(x) + float(y), variances) / (len(v) - 1)
|
|
std_dev = math.sqrt(variance)
|
|
|
|
return std_dev
|
|
|
|
|
|
def IsStringFloat(string_to_check):
|
|
"""Checks whether or not the given string can be converted to a floating
|
|
point number.
|
|
|
|
Args:
|
|
string_to_check: Input string to check if it can be converted to a float.
|
|
|
|
Returns:
|
|
True if the string can be converted to a float.
|
|
"""
|
|
try:
|
|
float(string_to_check)
|
|
|
|
return True
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def IsStringInt(string_to_check):
|
|
"""Checks whether or not the given string can be converted to a integer.
|
|
|
|
Args:
|
|
string_to_check: Input string to check if it can be converted to an int.
|
|
|
|
Returns:
|
|
True if the string can be converted to an int.
|
|
"""
|
|
try:
|
|
int(string_to_check)
|
|
|
|
return True
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def IsWindows():
|
|
"""Checks whether or not the script is running on Windows.
|
|
|
|
Returns:
|
|
True if running on Windows.
|
|
"""
|
|
return os.name == 'nt'
|
|
|
|
|
|
def RunProcess(command):
|
|
"""Run an arbitrary command. If output from the call is needed, use
|
|
RunProcessAndRetrieveOutput instead.
|
|
|
|
Args:
|
|
command: A list containing the command and args to execute.
|
|
|
|
Returns:
|
|
The return code of the call.
|
|
"""
|
|
# On Windows, use shell=True to get PATH interpretation.
|
|
shell = IsWindows()
|
|
return subprocess.call(command, shell=shell)
|
|
|
|
|
|
def RunProcessAndRetrieveOutput(command):
|
|
"""Run an arbitrary command, returning its output and return code. Since
|
|
output is collected via communicate(), there will be no output until the
|
|
call terminates. If you need output while the program runs (ie. so
|
|
that the buildbot doesn't terminate the script), consider RunProcess().
|
|
|
|
Args:
|
|
command: A list containing the command and args to execute.
|
|
print_output: Optional parameter to write output to stdout as it's
|
|
being collected.
|
|
|
|
Returns:
|
|
A tuple of the output and return code.
|
|
"""
|
|
# On Windows, use shell=True to get PATH interpretation.
|
|
shell = IsWindows()
|
|
proc = subprocess.Popen(command,
|
|
shell=shell,
|
|
stdout=subprocess.PIPE)
|
|
|
|
(output, _) = proc.communicate()
|
|
|
|
return (output, proc.returncode)
|
|
|
|
|
|
def RunGit(command):
|
|
"""Run a git subcommand, returning its output and return code.
|
|
|
|
Args:
|
|
command: A list containing the args to git.
|
|
|
|
Returns:
|
|
A tuple of the output and return code.
|
|
"""
|
|
command = ['git'] + command
|
|
|
|
return RunProcessAndRetrieveOutput(command)
|
|
|
|
|
|
def CheckRunGit(command):
|
|
"""Run a git subcommand, returning its output and return code. Asserts if
|
|
the return code of the call is non-zero.
|
|
|
|
Args:
|
|
command: A list containing the args to git.
|
|
|
|
Returns:
|
|
A tuple of the output and return code.
|
|
"""
|
|
(output, return_code) = RunGit(command)
|
|
|
|
assert not return_code, 'An error occurred while running'\
|
|
' "git %s"' % ' '.join(command)
|
|
return output
|
|
|
|
|
|
def BuildWithMake(threads, targets):
|
|
cmd = ['make', 'BUILDTYPE=Release']
|
|
|
|
if threads:
|
|
cmd.append('-j%d' % threads)
|
|
|
|
cmd += targets
|
|
|
|
return_code = RunProcess(cmd)
|
|
|
|
return not return_code
|
|
|
|
|
|
def BuildWithNinja(threads, targets):
|
|
cmd = ['ninja', '-C', os.path.join('out', 'Release')]
|
|
|
|
if threads:
|
|
cmd.append('-j%d' % threads)
|
|
|
|
cmd += targets
|
|
|
|
return_code = RunProcess(cmd)
|
|
|
|
return not return_code
|
|
|
|
|
|
def BuildWithVisualStudio(targets):
|
|
path_to_devenv = os.path.abspath(
|
|
os.path.join(os.environ['VS100COMNTOOLS'], '..', 'IDE', 'devenv.com'))
|
|
path_to_sln = os.path.join(os.getcwd(), 'chrome', 'chrome.sln')
|
|
cmd = [path_to_devenv, '/build', 'Release', path_to_sln]
|
|
|
|
for t in targets:
|
|
cmd.extend(['/Project', t])
|
|
|
|
return_code = RunProcess(cmd)
|
|
|
|
return not return_code
|
|
|
|
|
|
class Builder(object):
|
|
"""Builder is used by the bisect script to build relevant targets and deploy.
|
|
"""
|
|
def Build(self, depot, opts):
|
|
raise NotImplementedError()
|
|
|
|
|
|
class DesktopBuilder(Builder):
|
|
"""DesktopBuilder is used to build Chromium on linux/mac/windows."""
|
|
def Build(self, depot, opts):
|
|
"""Builds chrome and performance_ui_tests using options passed into
|
|
the script.
|
|
|
|
Args:
|
|
depot: Current depot being bisected.
|
|
opts: The options parsed from the command line.
|
|
|
|
Returns:
|
|
True if build was successful.
|
|
"""
|
|
targets = ['chrome', 'performance_ui_tests']
|
|
|
|
threads = None
|
|
if opts.use_goma:
|
|
threads = 64
|
|
|
|
build_success = False
|
|
if opts.build_preference == 'make':
|
|
build_success = BuildWithMake(threads, targets)
|
|
elif opts.build_preference == 'ninja':
|
|
if IsWindows():
|
|
targets = [t + '.exe' for t in targets]
|
|
build_success = BuildWithNinja(threads, targets)
|
|
elif opts.build_preference == 'msvs':
|
|
assert IsWindows(), 'msvs is only supported on Windows.'
|
|
build_success = BuildWithVisualStudio(targets)
|
|
else:
|
|
assert False, 'No build system defined.'
|
|
return build_success
|
|
|
|
|
|
class AndroidBuilder(Builder):
|
|
"""AndroidBuilder is used to build on android."""
|
|
def InstallAPK(self, opts):
|
|
"""Installs apk to device.
|
|
|
|
Args:
|
|
opts: The options parsed from the command line.
|
|
|
|
Returns:
|
|
True if successful.
|
|
"""
|
|
path_to_tool = os.path.join('build', 'android', 'adb_install_apk.py')
|
|
cmd = [path_to_tool, '--apk', 'ChromiumTestShell.apk', '--apk_package',
|
|
'org.chromium.chrome.testshell', '--release']
|
|
return_code = RunProcess(cmd)
|
|
|
|
return not return_code
|
|
|
|
def Build(self, depot, opts):
|
|
"""Builds the android content shell and other necessary tools using options
|
|
passed into the script.
|
|
|
|
Args:
|
|
depot: Current depot being bisected.
|
|
opts: The options parsed from the command line.
|
|
|
|
Returns:
|
|
True if build was successful.
|
|
"""
|
|
targets = ['chromium_testshell', 'forwarder2', 'md5sum']
|
|
threads = None
|
|
if opts.use_goma:
|
|
threads = 64
|
|
|
|
build_success = False
|
|
if opts.build_preference == 'ninja':
|
|
build_success = BuildWithNinja(threads, targets)
|
|
else:
|
|
assert False, 'No build system defined.'
|
|
|
|
if build_success:
|
|
build_success = self.InstallAPK(opts)
|
|
|
|
return build_success
|
|
|
|
|
|
class CrosBuilder(Builder):
|
|
"""CrosBuilder is used to build and image ChromeOS/Chromium when cros is the
|
|
target platform."""
|
|
def ImageToTarget(self, opts):
|
|
"""Installs latest image to target specified by opts.cros_remote_ip.
|
|
|
|
Args:
|
|
opts: Program options containing cros_board and cros_remote_ip.
|
|
|
|
Returns:
|
|
True if successful.
|
|
"""
|
|
try:
|
|
# Keys will most likely be set to 0640 after wiping the chroot.
|
|
os.chmod(CROS_SCRIPT_KEY_PATH, 0600)
|
|
os.chmod(CROS_TEST_KEY_PATH, 0600)
|
|
cmd = [CROS_SDK_PATH, '--', './bin/cros_image_to_target.py',
|
|
'--remote=%s' % opts.cros_remote_ip,
|
|
'--board=%s' % opts.cros_board, '--test', '--verbose']
|
|
|
|
return_code = RunProcess(cmd)
|
|
return not return_code
|
|
except OSError, e:
|
|
return False
|
|
|
|
def BuildPackages(self, opts, depot):
|
|
"""Builds packages for cros.
|
|
|
|
Args:
|
|
opts: Program options containing cros_board.
|
|
depot: The depot being bisected.
|
|
|
|
Returns:
|
|
True if successful.
|
|
"""
|
|
cmd = [CROS_SDK_PATH]
|
|
|
|
if depot != 'cros':
|
|
path_to_chrome = os.path.join(os.getcwd(), '..')
|
|
cmd += ['--chrome_root=%s' % path_to_chrome]
|
|
|
|
cmd += ['--']
|
|
|
|
if depot != 'cros':
|
|
cmd += ['CHROME_ORIGIN=LOCAL_SOURCE']
|
|
|
|
cmd += ['BUILDTYPE=Release', './build_packages',
|
|
'--board=%s' % opts.cros_board]
|
|
return_code = RunProcess(cmd)
|
|
|
|
return not return_code
|
|
|
|
def BuildImage(self, opts, depot):
|
|
"""Builds test image for cros.
|
|
|
|
Args:
|
|
opts: Program options containing cros_board.
|
|
depot: The depot being bisected.
|
|
|
|
Returns:
|
|
True if successful.
|
|
"""
|
|
cmd = [CROS_SDK_PATH]
|
|
|
|
if depot != 'cros':
|
|
path_to_chrome = os.path.join(os.getcwd(), '..')
|
|
cmd += ['--chrome_root=%s' % path_to_chrome]
|
|
|
|
cmd += ['--']
|
|
|
|
if depot != 'cros':
|
|
cmd += ['CHROME_ORIGIN=LOCAL_SOURCE']
|
|
|
|
cmd += ['BUILDTYPE=Release', '--', './build_image',
|
|
'--board=%s' % opts.cros_board, 'test']
|
|
|
|
return_code = RunProcess(cmd)
|
|
|
|
return not return_code
|
|
|
|
def Build(self, depot, opts):
|
|
"""Builds targets using options passed into the script.
|
|
|
|
Args:
|
|
depot: Current depot being bisected.
|
|
opts: The options parsed from the command line.
|
|
|
|
Returns:
|
|
True if build was successful.
|
|
"""
|
|
if self.BuildPackages(opts, depot):
|
|
if self.BuildImage(opts, depot):
|
|
return self.ImageToTarget(opts)
|
|
return False
|
|
|
|
|
|
class SourceControl(object):
|
|
"""SourceControl is an abstraction over the underlying source control
|
|
system used for chromium. For now only git is supported, but in the
|
|
future, the svn workflow could be added as well."""
|
|
def __init__(self):
|
|
super(SourceControl, self).__init__()
|
|
|
|
def SyncToRevisionWithGClient(self, revision):
|
|
"""Uses gclient to sync to the specified revision.
|
|
|
|
ie. gclient sync --revision <revision>
|
|
|
|
Args:
|
|
revision: The git SHA1 or svn CL (depending on workflow).
|
|
|
|
Returns:
|
|
The return code of the call.
|
|
"""
|
|
return bisect_utils.RunGClient(['sync', '--revision',
|
|
revision, '--verbose', '--nohooks', '--reset', '--force'])
|
|
|
|
def SyncToRevisionWithRepo(self, timestamp):
|
|
"""Uses repo to sync all the underlying git depots to the specified
|
|
time.
|
|
|
|
Args:
|
|
timestamp: The unix timestamp to sync to.
|
|
|
|
Returns:
|
|
The return code of the call.
|
|
"""
|
|
return bisect_utils.RunRepoSyncAtTimestamp(timestamp)
|
|
|
|
|
|
class GitSourceControl(SourceControl):
|
|
"""GitSourceControl is used to query the underlying source control. """
|
|
def __init__(self, opts):
|
|
super(GitSourceControl, self).__init__()
|
|
self.opts = opts
|
|
|
|
def IsGit(self):
|
|
return True
|
|
|
|
def GetRevisionList(self, revision_range_end, revision_range_start):
|
|
"""Retrieves a list of revisions between |revision_range_start| and
|
|
|revision_range_end|.
|
|
|
|
Args:
|
|
revision_range_end: The SHA1 for the end of the range.
|
|
revision_range_start: The SHA1 for the beginning of the range.
|
|
|
|
Returns:
|
|
A list of the revisions between |revision_range_start| and
|
|
|revision_range_end| (inclusive).
|
|
"""
|
|
revision_range = '%s..%s' % (revision_range_start, revision_range_end)
|
|
cmd = ['log', '--format=%H', '-10000', '--first-parent', revision_range]
|
|
log_output = CheckRunGit(cmd)
|
|
|
|
revision_hash_list = log_output.split()
|
|
revision_hash_list.append(revision_range_start)
|
|
|
|
return revision_hash_list
|
|
|
|
def SyncToRevision(self, revision, sync_client=None):
|
|
"""Syncs to the specified revision.
|
|
|
|
Args:
|
|
revision: The revision to sync to.
|
|
use_gclient: Specifies whether or not we should sync using gclient or
|
|
just use source control directly.
|
|
|
|
Returns:
|
|
True if successful.
|
|
"""
|
|
|
|
if not sync_client:
|
|
results = RunGit(['checkout', revision])[1]
|
|
elif sync_client == 'gclient':
|
|
results = self.SyncToRevisionWithGClient(revision)
|
|
elif sync_client == 'repo':
|
|
results = self.SyncToRevisionWithRepo(revision)
|
|
|
|
return not results
|
|
|
|
def ResolveToRevision(self, revision_to_check, depot, search):
|
|
"""If an SVN revision is supplied, try to resolve it to a git SHA1.
|
|
|
|
Args:
|
|
revision_to_check: The user supplied revision string that may need to be
|
|
resolved to a git SHA1.
|
|
depot: The depot the revision_to_check is from.
|
|
search: The number of changelists to try if the first fails to resolve
|
|
to a git hash. If the value is negative, the function will search
|
|
backwards chronologically, otherwise it will search forward.
|
|
|
|
Returns:
|
|
A string containing a git SHA1 hash, otherwise None.
|
|
"""
|
|
if depot != 'cros':
|
|
if not IsStringInt(revision_to_check):
|
|
return revision_to_check
|
|
|
|
depot_svn = 'svn://svn.chromium.org/chrome/trunk/src'
|
|
|
|
if depot != 'chromium':
|
|
depot_svn = DEPOT_DEPS_NAME[depot]['svn']
|
|
|
|
svn_revision = int(revision_to_check)
|
|
git_revision = None
|
|
|
|
if search > 0:
|
|
search_range = xrange(svn_revision, svn_revision + search, 1)
|
|
else:
|
|
search_range = xrange(svn_revision, svn_revision + search, -1)
|
|
|
|
for i in search_range:
|
|
svn_pattern = 'git-svn-id: %s@%d' % (depot_svn, i)
|
|
cmd = ['log', '--format=%H', '-1', '--grep', svn_pattern,
|
|
'origin/master']
|
|
|
|
(log_output, return_code) = RunGit(cmd)
|
|
|
|
assert not return_code, 'An error occurred while running'\
|
|
' "git %s"' % ' '.join(cmd)
|
|
|
|
if not return_code:
|
|
log_output = log_output.strip()
|
|
|
|
if log_output:
|
|
git_revision = log_output
|
|
|
|
break
|
|
|
|
return git_revision
|
|
else:
|
|
if IsStringInt(revision_to_check):
|
|
return int(revision_to_check)
|
|
else:
|
|
cwd = os.getcwd()
|
|
os.chdir(os.path.join(os.getcwd(), 'src', 'third_party',
|
|
'chromiumos-overlay'))
|
|
pattern = CROS_VERSION_PATTERN % revision_to_check
|
|
cmd = ['log', '--format=%ct', '-1', '--grep', pattern]
|
|
|
|
git_revision = None
|
|
|
|
log_output = CheckRunGit(cmd)
|
|
if log_output:
|
|
git_revision = log_output
|
|
git_revision = int(log_output.strip())
|
|
os.chdir(cwd)
|
|
|
|
return git_revision
|
|
|
|
def IsInProperBranch(self):
|
|
"""Confirms they're in the master branch for performing the bisection.
|
|
This is needed or gclient will fail to sync properly.
|
|
|
|
Returns:
|
|
True if the current branch on src is 'master'
|
|
"""
|
|
cmd = ['rev-parse', '--abbrev-ref', 'HEAD']
|
|
log_output = CheckRunGit(cmd)
|
|
log_output = log_output.strip()
|
|
|
|
return log_output == "master"
|
|
|
|
def SVNFindRev(self, revision):
|
|
"""Maps directly to the 'git svn find-rev' command.
|
|
|
|
Args:
|
|
revision: The git SHA1 to use.
|
|
|
|
Returns:
|
|
An integer changelist #, otherwise None.
|
|
"""
|
|
|
|
cmd = ['svn', 'find-rev', revision]
|
|
|
|
output = CheckRunGit(cmd)
|
|
svn_revision = output.strip()
|
|
|
|
if IsStringInt(svn_revision):
|
|
return int(svn_revision)
|
|
|
|
return None
|
|
|
|
def QueryRevisionInfo(self, revision):
|
|
"""Gathers information on a particular revision, such as author's name,
|
|
email, subject, and date.
|
|
|
|
Args:
|
|
revision: Revision you want to gather information on.
|
|
Returns:
|
|
A dict in the following format:
|
|
{
|
|
'author': %s,
|
|
'email': %s,
|
|
'date': %s,
|
|
'subject': %s,
|
|
}
|
|
"""
|
|
commit_info = {}
|
|
|
|
formats = ['%cN', '%cE', '%s', '%cD']
|
|
targets = ['author', 'email', 'subject', 'date']
|
|
|
|
for i in xrange(len(formats)):
|
|
cmd = ['log', '--format=%s' % formats[i], '-1', revision]
|
|
output = CheckRunGit(cmd)
|
|
commit_info[targets[i]] = output.rstrip()
|
|
|
|
return commit_info
|
|
|
|
def CheckoutFileAtRevision(self, file_name, revision):
|
|
"""Performs a checkout on a file at the given revision.
|
|
|
|
Returns:
|
|
True if successful.
|
|
"""
|
|
return not RunGit(['checkout', revision, file_name])[1]
|
|
|
|
def RevertFileToHead(self, file_name):
|
|
"""Unstages a file and returns it to HEAD.
|
|
|
|
Returns:
|
|
True if successful.
|
|
"""
|
|
# Reset doesn't seem to return 0 on success.
|
|
RunGit(['reset', 'HEAD', bisect_utils.FILE_DEPS_GIT])
|
|
|
|
return not RunGit(['checkout', bisect_utils.FILE_DEPS_GIT])[1]
|
|
|
|
def QueryFileRevisionHistory(self, filename, revision_start, revision_end):
|
|
"""Returns a list of commits that modified this file.
|
|
|
|
Args:
|
|
filename: Name of file.
|
|
revision_start: Start of revision range.
|
|
revision_end: End of revision range.
|
|
|
|
Returns:
|
|
Returns a list of commits that touched this file.
|
|
"""
|
|
cmd = ['log', '--format=%H', '%s~1..%s' % (revision_start, revision_end),
|
|
filename]
|
|
output = CheckRunGit(cmd)
|
|
|
|
return [o for o in output.split('\n') if o]
|
|
|
|
class BisectPerformanceMetrics(object):
|
|
"""BisectPerformanceMetrics performs a bisection against a list of range
|
|
of revisions to narrow down where performance regressions may have
|
|
occurred."""
|
|
|
|
def __init__(self, source_control, opts):
|
|
super(BisectPerformanceMetrics, self).__init__()
|
|
|
|
self.opts = opts
|
|
self.source_control = source_control
|
|
self.src_cwd = os.getcwd()
|
|
self.cros_cwd = os.path.join(os.getcwd(), '..', 'cros')
|
|
self.depot_cwd = {}
|
|
self.cleanup_commands = []
|
|
self.warnings = []
|
|
self.builder = None
|
|
|
|
if opts.target_platform == 'cros':
|
|
self.builder = CrosBuilder()
|
|
elif opts.target_platform == 'android':
|
|
self.builder = AndroidBuilder()
|
|
else:
|
|
self.builder = DesktopBuilder()
|
|
|
|
# This always starts true since the script grabs latest first.
|
|
self.was_blink = True
|
|
|
|
for d in DEPOT_NAMES:
|
|
# The working directory of each depot is just the path to the depot, but
|
|
# since we're already in 'src', we can skip that part.
|
|
|
|
self.depot_cwd[d] = self.src_cwd + DEPOT_DEPS_NAME[d]['src'][3:]
|
|
|
|
def PerformCleanup(self):
|
|
"""Performs cleanup when script is finished."""
|
|
os.chdir(self.src_cwd)
|
|
for c in self.cleanup_commands:
|
|
if c[0] == 'mv':
|
|
shutil.move(c[1], c[2])
|
|
else:
|
|
assert False, 'Invalid cleanup command.'
|
|
|
|
def GetRevisionList(self, depot, bad_revision, good_revision):
|
|
"""Retrieves a list of all the commits between the bad revision and
|
|
last known good revision."""
|
|
|
|
revision_work_list = []
|
|
|
|
if depot == 'cros':
|
|
revision_range_start = good_revision
|
|
revision_range_end = bad_revision
|
|
|
|
cwd = os.getcwd()
|
|
self.ChangeToDepotWorkingDirectory('cros')
|
|
|
|
# Print the commit timestamps for every commit in the revision time
|
|
# range. We'll sort them and bisect by that. There is a remote chance that
|
|
# 2 (or more) commits will share the exact same timestamp, but it's
|
|
# probably safe to ignore that case.
|
|
cmd = ['repo', 'forall', '-c',
|
|
'git log --format=%%ct --before=%d --after=%d' % (
|
|
revision_range_end, revision_range_start)]
|
|
(output, return_code) = RunProcessAndRetrieveOutput(cmd)
|
|
|
|
assert not return_code, 'An error occurred while running'\
|
|
' "%s"' % ' '.join(cmd)
|
|
|
|
os.chdir(cwd)
|
|
|
|
revision_work_list = list(set(
|
|
[int(o) for o in output.split('\n') if IsStringInt(o)]))
|
|
revision_work_list = sorted(revision_work_list, reverse=True)
|
|
else:
|
|
revision_work_list = self.source_control.GetRevisionList(bad_revision,
|
|
good_revision)
|
|
|
|
return revision_work_list
|
|
|
|
def Get3rdPartyRevisionsFromCurrentRevision(self, depot):
|
|
"""Parses the DEPS file to determine WebKit/v8/etc... versions.
|
|
|
|
Returns:
|
|
A dict in the format {depot:revision} if successful, otherwise None.
|
|
"""
|
|
|
|
cwd = os.getcwd()
|
|
self.ChangeToDepotWorkingDirectory(depot)
|
|
|
|
results = {}
|
|
|
|
if depot == 'chromium':
|
|
locals = {'Var': lambda _: locals["vars"][_],
|
|
'From': lambda *args: None}
|
|
execfile(bisect_utils.FILE_DEPS_GIT, {}, locals)
|
|
|
|
os.chdir(cwd)
|
|
|
|
rxp = re.compile(".git@(?P<revision>[a-fA-F0-9]+)")
|
|
|
|
for d in DEPOT_NAMES:
|
|
if DEPOT_DEPS_NAME[d]['recurse'] and\
|
|
DEPOT_DEPS_NAME[d]['from'] == depot:
|
|
if locals['deps'].has_key(DEPOT_DEPS_NAME[d]['src']):
|
|
re_results = rxp.search(locals['deps'][DEPOT_DEPS_NAME[d]['src']])
|
|
|
|
if re_results:
|
|
results[d] = re_results.group('revision')
|
|
else:
|
|
return None
|
|
else:
|
|
return None
|
|
elif depot == 'cros':
|
|
cmd = [CROS_SDK_PATH, '--', 'portageq-%s' % self.opts.cros_board,
|
|
'best_visible', '/build/%s' % self.opts.cros_board, 'ebuild',
|
|
CROS_CHROMEOS_PATTERN]
|
|
(output, return_code) = RunProcessAndRetrieveOutput(cmd)
|
|
|
|
assert not return_code, 'An error occurred while running'\
|
|
' "%s"' % ' '.join(cmd)
|
|
|
|
if len(output) > CROS_CHROMEOS_PATTERN:
|
|
output = output[len(CROS_CHROMEOS_PATTERN):]
|
|
|
|
if len(output) > 1:
|
|
output = output.split('_')[0]
|
|
|
|
if len(output) > 3:
|
|
contents = output.split('.')
|
|
|
|
version = contents[2]
|
|
|
|
if contents[3] != '0':
|
|
warningText = 'Chrome version: %s.%s but using %s.0 to bisect.' %\
|
|
(version, contents[3], version)
|
|
if not warningText in self.warnings:
|
|
self.warnings.append(warningText)
|
|
|
|
cwd = os.getcwd()
|
|
self.ChangeToDepotWorkingDirectory('chromium')
|
|
return_code = CheckRunGit(['log', '-1', '--format=%H',
|
|
'--author=chrome-release@google.com', '--grep=to %s' % version,
|
|
'origin/master'])
|
|
os.chdir(cwd)
|
|
|
|
results['chromium'] = output.strip()
|
|
|
|
return results
|
|
|
|
def BuildCurrentRevision(self, depot):
|
|
"""Builds chrome and performance_ui_tests on the current revision.
|
|
|
|
Returns:
|
|
True if the build was successful.
|
|
"""
|
|
if self.opts.debug_ignore_build:
|
|
return True
|
|
|
|
cwd = os.getcwd()
|
|
os.chdir(self.src_cwd)
|
|
|
|
build_success = self.builder.Build(depot, self.opts)
|
|
|
|
os.chdir(cwd)
|
|
|
|
return build_success
|
|
|
|
def RunGClientHooks(self):
|
|
"""Runs gclient with runhooks command.
|
|
|
|
Returns:
|
|
True if gclient reports no errors.
|
|
"""
|
|
|
|
if self.opts.debug_ignore_build:
|
|
return True
|
|
|
|
return not bisect_utils.RunGClient(['runhooks'])
|
|
|
|
def TryParseHistogramValuesFromOutput(self, metric, text):
|
|
"""Attempts to parse a metric in the format HISTOGRAM <graph: <trace>.
|
|
|
|
Args:
|
|
metric: The metric as a list of [<trace>, <value>] strings.
|
|
text: The text to parse the metric values from.
|
|
|
|
Returns:
|
|
A list of floating point numbers found.
|
|
"""
|
|
metric_formatted = 'HISTOGRAM %s: %s= ' % (metric[0], metric[1])
|
|
|
|
text_lines = text.split('\n')
|
|
values_list = []
|
|
|
|
for current_line in text_lines:
|
|
if metric_formatted in current_line:
|
|
current_line = current_line[len(metric_formatted):]
|
|
|
|
try:
|
|
histogram_values = eval(current_line)
|
|
|
|
for b in histogram_values['buckets']:
|
|
average_for_bucket = float(b['high'] + b['low']) * 0.5
|
|
# Extends the list with N-elements with the average for that bucket.
|
|
values_list.extend([average_for_bucket] * b['count'])
|
|
except:
|
|
pass
|
|
|
|
return values_list
|
|
|
|
def TryParseResultValuesFromOutput(self, metric, text):
|
|
"""Attempts to parse a metric in the format RESULT <graph: <trace>.
|
|
|
|
Args:
|
|
metric: The metric as a list of [<trace>, <value>] strings.
|
|
text: The text to parse the metric values from.
|
|
|
|
Returns:
|
|
A list of floating point numbers found.
|
|
"""
|
|
# Format is: RESULT <graph>: <trace>= <value> <units>
|
|
metric_formatted = re.escape('RESULT %s: %s=' % (metric[0], metric[1]))
|
|
|
|
text_lines = text.split('\n')
|
|
values_list = []
|
|
|
|
for current_line in text_lines:
|
|
# Parse the output from the performance test for the metric we're
|
|
# interested in.
|
|
metric_re = metric_formatted +\
|
|
"(\s)*(?P<values>[0-9]+(\.[0-9]*)?)"
|
|
metric_re = re.compile(metric_re)
|
|
regex_results = metric_re.search(current_line)
|
|
|
|
if not regex_results is None:
|
|
values_list += [regex_results.group('values')]
|
|
else:
|
|
metric_re = metric_formatted +\
|
|
"(\s)*\[(\s)*(?P<values>[0-9,.]+)\]"
|
|
metric_re = re.compile(metric_re)
|
|
regex_results = metric_re.search(current_line)
|
|
|
|
if not regex_results is None:
|
|
metric_values = regex_results.group('values')
|
|
|
|
values_list += metric_values.split(',')
|
|
|
|
values_list = [float(v) for v in values_list if IsStringFloat(v)]
|
|
|
|
# If the metric is times/t, we need to sum the timings in order to get
|
|
# similar regression results as the try-bots.
|
|
|
|
if metric == ['times', 't']:
|
|
if values_list:
|
|
values_list = [reduce(lambda x, y: float(x) + float(y), values_list)]
|
|
|
|
return values_list
|
|
|
|
def ParseMetricValuesFromOutput(self, metric, text):
|
|
"""Parses output from performance_ui_tests and retrieves the results for
|
|
a given metric.
|
|
|
|
Args:
|
|
metric: The metric as a list of [<trace>, <value>] strings.
|
|
text: The text to parse the metric values from.
|
|
|
|
Returns:
|
|
A list of floating point numbers found.
|
|
"""
|
|
metric_values = self.TryParseResultValuesFromOutput(metric, text)
|
|
|
|
if not metric_values:
|
|
metric_values = self.TryParseHistogramValuesFromOutput(metric, text)
|
|
|
|
return metric_values
|
|
|
|
def RunPerformanceTestAndParseResults(self, command_to_run, metric):
|
|
"""Runs a performance test on the current revision by executing the
|
|
'command_to_run' and parses the results.
|
|
|
|
Args:
|
|
command_to_run: The command to be run to execute the performance test.
|
|
metric: The metric to parse out from the results of the performance test.
|
|
|
|
Returns:
|
|
On success, it will return a tuple of the average value of the metric,
|
|
and a success code of 0.
|
|
"""
|
|
|
|
if self.opts.debug_ignore_perf_test:
|
|
return ({'mean': 0.0, 'std_dev': 0.0}, 0)
|
|
|
|
if IsWindows():
|
|
command_to_run = command_to_run.replace('/', r'\\')
|
|
|
|
args = shlex.split(command_to_run)
|
|
|
|
# If running a telemetry test for cros, insert the remote ip, and
|
|
# identity parameters.
|
|
if self.opts.target_platform == 'cros':
|
|
if 'tools/perf/run_' in args[0]:
|
|
args.append('--remote=%s' % self.opts.cros_remote_ip)
|
|
args.append('--identity=%s' % CROS_TEST_KEY_PATH)
|
|
|
|
cwd = os.getcwd()
|
|
os.chdir(self.src_cwd)
|
|
|
|
start_time = time.time()
|
|
|
|
metric_values = []
|
|
for i in xrange(self.opts.repeat_test_count):
|
|
# Can ignore the return code since if the tests fail, it won't return 0.
|
|
try:
|
|
(output, return_code) = RunProcessAndRetrieveOutput(args)
|
|
except OSError, e:
|
|
if e.errno == errno.ENOENT:
|
|
err_text = ("Something went wrong running the performance test. "
|
|
"Please review the command line:\n\n")
|
|
if 'src/' in ' '.join(args):
|
|
err_text += ("Check that you haven't accidentally specified a path "
|
|
"with src/ in the command.\n\n")
|
|
err_text += ' '.join(args)
|
|
err_text += '\n'
|
|
|
|
return (err_text, -1)
|
|
raise
|
|
|
|
if self.opts.output_buildbot_annotations:
|
|
print output
|
|
|
|
metric_values += self.ParseMetricValuesFromOutput(metric, output)
|
|
|
|
elapsed_minutes = (time.time() - start_time) / 60.0
|
|
|
|
if elapsed_minutes >= self.opts.repeat_test_max_time or not metric_values:
|
|
break
|
|
|
|
os.chdir(cwd)
|
|
|
|
# Need to get the average value if there were multiple values.
|
|
if metric_values:
|
|
truncated_mean = CalculateTruncatedMean(metric_values,
|
|
self.opts.truncate_percent)
|
|
standard_dev = CalculateStandardDeviation(metric_values)
|
|
|
|
values = {
|
|
'mean': truncated_mean,
|
|
'std_dev': standard_dev,
|
|
}
|
|
|
|
print 'Results of performance test: %12f %12f' % (
|
|
truncated_mean, standard_dev)
|
|
print
|
|
return (values, 0)
|
|
else:
|
|
return ('Invalid metric specified, or no values returned from '
|
|
'performance test.', -1)
|
|
|
|
def FindAllRevisionsToSync(self, revision, depot):
|
|
"""Finds all dependant revisions and depots that need to be synced for a
|
|
given revision. This is only useful in the git workflow, as an svn depot
|
|
may be split into multiple mirrors.
|
|
|
|
ie. skia is broken up into 3 git mirrors over skia/src, skia/gyp, and
|
|
skia/include. To sync skia/src properly, one has to find the proper
|
|
revisions in skia/gyp and skia/include.
|
|
|
|
Args:
|
|
revision: The revision to sync to.
|
|
depot: The depot in use at the moment (probably skia).
|
|
|
|
Returns:
|
|
A list of [depot, revision] pairs that need to be synced.
|
|
"""
|
|
revisions_to_sync = [[depot, revision]]
|
|
|
|
is_base = (depot == 'chromium') or (depot == 'cros')
|
|
|
|
# Some SVN depots were split into multiple git depots, so we need to
|
|
# figure out for each mirror which git revision to grab. There's no
|
|
# guarantee that the SVN revision will exist for each of the dependant
|
|
# depots, so we have to grep the git logs and grab the next earlier one.
|
|
if not is_base and\
|
|
DEPOT_DEPS_NAME[depot]['depends'] and\
|
|
self.source_control.IsGit():
|
|
svn_rev = self.source_control.SVNFindRev(revision)
|
|
|
|
for d in DEPOT_DEPS_NAME[depot]['depends']:
|
|
self.ChangeToDepotWorkingDirectory(d)
|
|
|
|
dependant_rev = self.source_control.ResolveToRevision(svn_rev, d, -1000)
|
|
|
|
if dependant_rev:
|
|
revisions_to_sync.append([d, dependant_rev])
|
|
|
|
num_resolved = len(revisions_to_sync)
|
|
num_needed = len(DEPOT_DEPS_NAME[depot]['depends'])
|
|
|
|
self.ChangeToDepotWorkingDirectory(depot)
|
|
|
|
if not ((num_resolved - 1) == num_needed):
|
|
return None
|
|
|
|
return revisions_to_sync
|
|
|
|
def PerformPreBuildCleanup(self):
|
|
"""Performs necessary cleanup between runs."""
|
|
print 'Cleaning up between runs.'
|
|
print
|
|
|
|
# Having these pyc files around between runs can confuse the
|
|
# perf tests and cause them to crash.
|
|
for (path, dir, files) in os.walk(self.src_cwd):
|
|
for cur_file in files:
|
|
if cur_file.endswith('.pyc'):
|
|
path_to_file = os.path.join(path, cur_file)
|
|
os.remove(path_to_file)
|
|
|
|
def PerformWebkitDirectoryCleanup(self, revision):
|
|
"""If the script is switching between Blink and WebKit during bisect,
|
|
its faster to just delete the directory rather than leave it up to git
|
|
to sync.
|
|
|
|
Returns:
|
|
True if successful.
|
|
"""
|
|
if not self.source_control.CheckoutFileAtRevision(
|
|
bisect_utils.FILE_DEPS_GIT, revision):
|
|
return False
|
|
|
|
cwd = os.getcwd()
|
|
os.chdir(self.src_cwd)
|
|
|
|
is_blink = bisect_utils.IsDepsFileBlink()
|
|
|
|
os.chdir(cwd)
|
|
|
|
if not self.source_control.RevertFileToHead(
|
|
bisect_utils.FILE_DEPS_GIT):
|
|
return False
|
|
|
|
if self.was_blink != is_blink:
|
|
self.was_blink = is_blink
|
|
return bisect_utils.RemoveThirdPartyWebkitDirectory()
|
|
return True
|
|
|
|
def PerformCrosChrootCleanup(self):
|
|
"""Deletes the chroot.
|
|
|
|
Returns:
|
|
True if successful.
|
|
"""
|
|
cwd = os.getcwd()
|
|
self.ChangeToDepotWorkingDirectory('cros')
|
|
cmd = [CROS_SDK_PATH, '--delete']
|
|
return_code = RunProcess(cmd)
|
|
os.chdir(cwd)
|
|
return not return_code
|
|
|
|
def CreateCrosChroot(self):
|
|
"""Creates a new chroot.
|
|
|
|
Returns:
|
|
True if successful.
|
|
"""
|
|
cwd = os.getcwd()
|
|
self.ChangeToDepotWorkingDirectory('cros')
|
|
cmd = [CROS_SDK_PATH, '--create']
|
|
return_code = RunProcess(cmd)
|
|
os.chdir(cwd)
|
|
return not return_code
|
|
|
|
def PerformPreSyncCleanup(self, revision, depot):
|
|
"""Performs any necessary cleanup before syncing.
|
|
|
|
Returns:
|
|
True if successful.
|
|
"""
|
|
if depot == 'chromium':
|
|
return self.PerformWebkitDirectoryCleanup(revision)
|
|
elif depot == 'cros':
|
|
return self.PerformCrosChrootCleanup()
|
|
return True
|
|
|
|
def RunPostSync(self, depot):
|
|
"""Performs any work after syncing.
|
|
|
|
Returns:
|
|
True if successful.
|
|
"""
|
|
if self.opts.target_platform == 'android':
|
|
cwd = os.getcwd()
|
|
os.chdir(os.path.join(self.src_cwd, '..'))
|
|
if not bisect_utils.SetupAndroidBuildEnvironment(self.opts):
|
|
return False
|
|
os.chdir(cwd)
|
|
|
|
if depot == 'cros':
|
|
return self.CreateCrosChroot()
|
|
else:
|
|
return self.RunGClientHooks()
|
|
return True
|
|
|
|
def ShouldSkipRevision(self, depot, revision):
|
|
"""Some commits can be safely skipped (such as a DEPS roll), since the tool
|
|
is git based those changes would have no effect.
|
|
|
|
Args:
|
|
depot: The depot being bisected.
|
|
revision: Current revision we're synced to.
|
|
|
|
Returns:
|
|
True if we should skip building/testing this revision.
|
|
"""
|
|
if depot == 'chromium':
|
|
if self.source_control.IsGit():
|
|
cmd = ['diff-tree', '--no-commit-id', '--name-only', '-r', revision]
|
|
output = CheckRunGit(cmd)
|
|
|
|
files = output.splitlines()
|
|
|
|
if len(files) == 1 and files[0] == 'DEPS':
|
|
return True
|
|
|
|
return False
|
|
|
|
def SyncBuildAndRunRevision(self, revision, depot, command_to_run, metric,
|
|
skippable=False):
|
|
"""Performs a full sync/build/run of the specified revision.
|
|
|
|
Args:
|
|
revision: The revision to sync to.
|
|
depot: The depot that's being used at the moment (src, webkit, etc.)
|
|
command_to_run: The command to execute the performance test.
|
|
metric: The performance metric being tested.
|
|
|
|
Returns:
|
|
On success, a tuple containing the results of the performance test.
|
|
Otherwise, a tuple with the error message.
|
|
"""
|
|
sync_client = None
|
|
if depot == 'chromium':
|
|
sync_client = 'gclient'
|
|
elif depot == 'cros':
|
|
sync_client = 'repo'
|
|
|
|
revisions_to_sync = self.FindAllRevisionsToSync(revision, depot)
|
|
|
|
if not revisions_to_sync:
|
|
return ('Failed to resolve dependant depots.', BUILD_RESULT_FAIL)
|
|
|
|
if not self.PerformPreSyncCleanup(revision, depot):
|
|
return ('Failed to perform pre-sync cleanup.', BUILD_RESULT_FAIL)
|
|
|
|
success = True
|
|
|
|
if not self.opts.debug_ignore_sync:
|
|
for r in revisions_to_sync:
|
|
self.ChangeToDepotWorkingDirectory(r[0])
|
|
|
|
if sync_client:
|
|
self.PerformPreBuildCleanup()
|
|
|
|
if not self.source_control.SyncToRevision(r[1], sync_client):
|
|
success = False
|
|
|
|
break
|
|
|
|
if success:
|
|
success = self.RunPostSync(depot)
|
|
|
|
if success:
|
|
if skippable and self.ShouldSkipRevision(depot, revision):
|
|
return ('Skipped revision: [%s]' % str(revision),
|
|
BUILD_RESULT_SKIPPED)
|
|
|
|
if self.BuildCurrentRevision(depot):
|
|
results = self.RunPerformanceTestAndParseResults(command_to_run,
|
|
metric)
|
|
|
|
if results[1] == 0 and sync_client:
|
|
external_revisions = self.Get3rdPartyRevisionsFromCurrentRevision(
|
|
depot)
|
|
|
|
if external_revisions:
|
|
return (results[0], results[1], external_revisions)
|
|
else:
|
|
return ('Failed to parse DEPS file for external revisions.',
|
|
BUILD_RESULT_FAIL)
|
|
else:
|
|
return results
|
|
else:
|
|
return ('Failed to build revision: [%s]' % (str(revision, )),
|
|
BUILD_RESULT_FAIL)
|
|
else:
|
|
return ('Failed to run [gclient runhooks].', BUILD_RESULT_FAIL)
|
|
else:
|
|
return ('Failed to sync revision: [%s]' % (str(revision, )),
|
|
BUILD_RESULT_FAIL)
|
|
|
|
def CheckIfRunPassed(self, current_value, known_good_value, known_bad_value):
|
|
"""Given known good and bad values, decide if the current_value passed
|
|
or failed.
|
|
|
|
Args:
|
|
current_value: The value of the metric being checked.
|
|
known_bad_value: The reference value for a "failed" run.
|
|
known_good_value: The reference value for a "passed" run.
|
|
|
|
Returns:
|
|
True if the current_value is closer to the known_good_value than the
|
|
known_bad_value.
|
|
"""
|
|
dist_to_good_value = abs(current_value['mean'] - known_good_value['mean'])
|
|
dist_to_bad_value = abs(current_value['mean'] - known_bad_value['mean'])
|
|
|
|
return dist_to_good_value < dist_to_bad_value
|
|
|
|
def ChangeToDepotWorkingDirectory(self, depot_name):
|
|
"""Given a depot, changes to the appropriate working directory.
|
|
|
|
Args:
|
|
depot_name: The name of the depot (see DEPOT_NAMES).
|
|
"""
|
|
if depot_name == 'chromium':
|
|
os.chdir(self.src_cwd)
|
|
elif depot_name == 'cros':
|
|
os.chdir(self.cros_cwd)
|
|
elif depot_name in DEPOT_NAMES:
|
|
os.chdir(self.depot_cwd[depot_name])
|
|
else:
|
|
assert False, 'Unknown depot [ %s ] encountered. Possibly a new one'\
|
|
' was added without proper support?' %\
|
|
(depot_name,)
|
|
|
|
def PrepareToBisectOnDepot(self,
|
|
current_depot,
|
|
end_revision,
|
|
start_revision,
|
|
previous_depot,
|
|
previous_revision):
|
|
"""Changes to the appropriate directory and gathers a list of revisions
|
|
to bisect between |start_revision| and |end_revision|.
|
|
|
|
Args:
|
|
current_depot: The depot we want to bisect.
|
|
end_revision: End of the revision range.
|
|
start_revision: Start of the revision range.
|
|
previous_depot: The depot we were previously bisecting.
|
|
previous_revision: The last revision we synced to on |previous_depot|.
|
|
|
|
Returns:
|
|
A list containing the revisions between |start_revision| and
|
|
|end_revision| inclusive.
|
|
"""
|
|
# Change into working directory of external library to run
|
|
# subsequent commands.
|
|
old_cwd = os.getcwd()
|
|
os.chdir(self.depot_cwd[current_depot])
|
|
|
|
# V8 (and possibly others) is merged in periodically. Bisecting
|
|
# this directory directly won't give much good info.
|
|
if DEPOT_DEPS_NAME[current_depot].has_key('build_with'):
|
|
if (DEPOT_DEPS_NAME[current_depot].has_key('custom_deps') and
|
|
previous_depot == 'chromium'):
|
|
config_path = os.path.join(self.src_cwd, '..')
|
|
if bisect_utils.RunGClientAndCreateConfig(self.opts,
|
|
DEPOT_DEPS_NAME[current_depot]['custom_deps'], cwd=config_path):
|
|
return []
|
|
if bisect_utils.RunGClient(
|
|
['sync', '--revision', previous_revision], cwd=self.src_cwd):
|
|
return []
|
|
|
|
new_depot = DEPOT_DEPS_NAME[current_depot]['build_with']
|
|
|
|
svn_start_revision = self.source_control.SVNFindRev(start_revision)
|
|
svn_end_revision = self.source_control.SVNFindRev(end_revision)
|
|
os.chdir(self.depot_cwd[new_depot])
|
|
|
|
start_revision = self.source_control.ResolveToRevision(
|
|
svn_start_revision, new_depot, -1000)
|
|
end_revision = self.source_control.ResolveToRevision(
|
|
svn_end_revision, new_depot, -1000)
|
|
|
|
old_name = DEPOT_DEPS_NAME[current_depot]['src'][4:]
|
|
new_name = DEPOT_DEPS_NAME[new_depot]['src'][4:]
|
|
|
|
os.chdir(self.src_cwd)
|
|
|
|
shutil.move(old_name, old_name + '.bak')
|
|
shutil.move(new_name, old_name)
|
|
os.chdir(self.depot_cwd[current_depot])
|
|
|
|
self.cleanup_commands.append(['mv', old_name, new_name])
|
|
self.cleanup_commands.append(['mv', old_name + '.bak', old_name])
|
|
|
|
os.chdir(self.depot_cwd[current_depot])
|
|
|
|
depot_revision_list = self.GetRevisionList(current_depot,
|
|
end_revision,
|
|
start_revision)
|
|
|
|
os.chdir(old_cwd)
|
|
|
|
return depot_revision_list
|
|
|
|
def GatherReferenceValues(self, good_rev, bad_rev, cmd, metric, target_depot):
|
|
"""Gathers reference values by running the performance tests on the
|
|
known good and bad revisions.
|
|
|
|
Args:
|
|
good_rev: The last known good revision where the performance regression
|
|
has not occurred yet.
|
|
bad_rev: A revision where the performance regression has already occurred.
|
|
cmd: The command to execute the performance test.
|
|
metric: The metric being tested for regression.
|
|
|
|
Returns:
|
|
A tuple with the results of building and running each revision.
|
|
"""
|
|
bad_run_results = self.SyncBuildAndRunRevision(bad_rev,
|
|
target_depot,
|
|
cmd,
|
|
metric)
|
|
|
|
good_run_results = None
|
|
|
|
if not bad_run_results[1]:
|
|
good_run_results = self.SyncBuildAndRunRevision(good_rev,
|
|
target_depot,
|
|
cmd,
|
|
metric)
|
|
|
|
return (bad_run_results, good_run_results)
|
|
|
|
def AddRevisionsIntoRevisionData(self, revisions, depot, sort, revision_data):
|
|
"""Adds new revisions to the revision_data dict and initializes them.
|
|
|
|
Args:
|
|
revisions: List of revisions to add.
|
|
depot: Depot that's currently in use (src, webkit, etc...)
|
|
sort: Sorting key for displaying revisions.
|
|
revision_data: A dict to add the new revisions into. Existing revisions
|
|
will have their sort keys offset.
|
|
"""
|
|
|
|
num_depot_revisions = len(revisions)
|
|
|
|
for k, v in revision_data.iteritems():
|
|
if v['sort'] > sort:
|
|
v['sort'] += num_depot_revisions
|
|
|
|
for i in xrange(num_depot_revisions):
|
|
r = revisions[i]
|
|
|
|
revision_data[r] = {'revision' : r,
|
|
'depot' : depot,
|
|
'value' : None,
|
|
'passed' : '?',
|
|
'sort' : i + sort + 1}
|
|
|
|
def PrintRevisionsToBisectMessage(self, revision_list, depot):
|
|
if self.opts.output_buildbot_annotations:
|
|
step_name = 'Bisection Range: [%s - %s]' % (
|
|
revision_list[len(revision_list)-1], revision_list[0])
|
|
bisect_utils.OutputAnnotationStepStart(step_name)
|
|
|
|
print
|
|
print 'Revisions to bisect on [%s]:' % depot
|
|
for revision_id in revision_list:
|
|
print ' -> %s' % (revision_id, )
|
|
print
|
|
|
|
if self.opts.output_buildbot_annotations:
|
|
bisect_utils.OutputAnnotationStepClosed()
|
|
|
|
def NudgeRevisionsIfDEPSChange(self, bad_revision, good_revision):
|
|
"""Checks to see if changes to DEPS file occurred, and that the revision
|
|
range also includes the change to .DEPS.git. If it doesn't, attempts to
|
|
expand the revision range to include it.
|
|
|
|
Args:
|
|
bad_rev: First known bad revision.
|
|
good_revision: Last known good revision.
|
|
|
|
Returns:
|
|
A tuple with the new bad and good revisions.
|
|
"""
|
|
if self.source_control.IsGit() and self.opts.target_platform == 'chromium':
|
|
changes_to_deps = self.source_control.QueryFileRevisionHistory(
|
|
'DEPS', good_revision, bad_revision)
|
|
|
|
if changes_to_deps:
|
|
# DEPS file was changed, search from the oldest change to DEPS file to
|
|
# bad_revision to see if there are matching .DEPS.git changes.
|
|
oldest_deps_change = changes_to_deps[-1]
|
|
changes_to_gitdeps = self.source_control.QueryFileRevisionHistory(
|
|
bisect_utils.FILE_DEPS_GIT, oldest_deps_change, bad_revision)
|
|
|
|
if len(changes_to_deps) != len(changes_to_gitdeps):
|
|
# Grab the timestamp of the last DEPS change
|
|
cmd = ['log', '--format=%ct', '-1', changes_to_deps[0]]
|
|
output = CheckRunGit(cmd)
|
|
commit_time = int(output)
|
|
|
|
# Try looking for a commit that touches the .DEPS.git file in the
|
|
# next 15 minutes after the DEPS file change.
|
|
cmd = ['log', '--format=%H', '-1',
|
|
'--before=%d' % (commit_time + 900), '--after=%d' % commit_time,
|
|
'origin/master', bisect_utils.FILE_DEPS_GIT]
|
|
output = CheckRunGit(cmd)
|
|
output = output.strip()
|
|
if output:
|
|
self.warnings.append('Detected change to DEPS and modified '
|
|
'revision range to include change to .DEPS.git')
|
|
return (output, good_revision)
|
|
else:
|
|
self.warnings.append('Detected change to DEPS but couldn\'t find '
|
|
'matching change to .DEPS.git')
|
|
return (bad_revision, good_revision)
|
|
|
|
def CheckIfRevisionsInProperOrder(self,
|
|
target_depot,
|
|
good_revision,
|
|
bad_revision):
|
|
"""Checks that |good_revision| is an earlier revision than |bad_revision|.
|
|
|
|
Args:
|
|
good_revision: Number/tag of the known good revision.
|
|
bad_revision: Number/tag of the known bad revision.
|
|
|
|
Returns:
|
|
True if the revisions are in the proper order (good earlier than bad).
|
|
"""
|
|
if self.source_control.IsGit() and target_depot != 'cros':
|
|
cmd = ['log', '--format=%ct', '-1', good_revision]
|
|
output = CheckRunGit(cmd)
|
|
good_commit_time = int(output)
|
|
|
|
cmd = ['log', '--format=%ct', '-1', bad_revision]
|
|
output = CheckRunGit(cmd)
|
|
bad_commit_time = int(output)
|
|
|
|
return good_commit_time <= bad_commit_time
|
|
else:
|
|
# Cros/svn use integers
|
|
return int(good_revision) <= int(bad_revision)
|
|
|
|
def Run(self, command_to_run, bad_revision_in, good_revision_in, metric):
|
|
"""Given known good and bad revisions, run a binary search on all
|
|
intermediate revisions to determine the CL where the performance regression
|
|
occurred.
|
|
|
|
Args:
|
|
command_to_run: Specify the command to execute the performance test.
|
|
good_revision: Number/tag of the known good revision.
|
|
bad_revision: Number/tag of the known bad revision.
|
|
metric: The performance metric to monitor.
|
|
|
|
Returns:
|
|
A dict with 2 members, 'revision_data' and 'error'. On success,
|
|
'revision_data' will contain a dict mapping revision ids to
|
|
data about that revision. Each piece of revision data consists of a
|
|
dict with the following keys:
|
|
|
|
'passed': Represents whether the performance test was successful at
|
|
that revision. Possible values include: 1 (passed), 0 (failed),
|
|
'?' (skipped), 'F' (build failed).
|
|
'depot': The depot that this revision is from (ie. WebKit)
|
|
'external': If the revision is a 'src' revision, 'external' contains
|
|
the revisions of each of the external libraries.
|
|
'sort': A sort value for sorting the dict in order of commits.
|
|
|
|
For example:
|
|
{
|
|
'error':None,
|
|
'revision_data':
|
|
{
|
|
'CL #1':
|
|
{
|
|
'passed':False,
|
|
'depot':'chromium',
|
|
'external':None,
|
|
'sort':0
|
|
}
|
|
}
|
|
}
|
|
|
|
If an error occurred, the 'error' field will contain the message and
|
|
'revision_data' will be empty.
|
|
"""
|
|
|
|
results = {'revision_data' : {},
|
|
'error' : None}
|
|
|
|
# Choose depot to bisect first
|
|
target_depot = 'chromium'
|
|
if self.opts.target_platform == 'cros':
|
|
target_depot = 'cros'
|
|
|
|
cwd = os.getcwd()
|
|
self.ChangeToDepotWorkingDirectory(target_depot)
|
|
|
|
# If they passed SVN CL's, etc... we can try match them to git SHA1's.
|
|
bad_revision = self.source_control.ResolveToRevision(bad_revision_in,
|
|
target_depot, 100)
|
|
good_revision = self.source_control.ResolveToRevision(good_revision_in,
|
|
target_depot, -100)
|
|
|
|
os.chdir(cwd)
|
|
|
|
|
|
if bad_revision is None:
|
|
results['error'] = 'Could\'t resolve [%s] to SHA1.' % (bad_revision_in,)
|
|
return results
|
|
|
|
if good_revision is None:
|
|
results['error'] = 'Could\'t resolve [%s] to SHA1.' % (good_revision_in,)
|
|
return results
|
|
|
|
# Check that they didn't accidentally swap good and bad revisions.
|
|
if not self.CheckIfRevisionsInProperOrder(
|
|
target_depot, good_revision, bad_revision):
|
|
results['error'] = 'bad_revision < good_revision, did you swap these '\
|
|
'by mistake?'
|
|
return results
|
|
|
|
(bad_revision, good_revision) = self.NudgeRevisionsIfDEPSChange(
|
|
bad_revision, good_revision)
|
|
|
|
if self.opts.output_buildbot_annotations:
|
|
bisect_utils.OutputAnnotationStepStart('Gathering Revisions')
|
|
|
|
print 'Gathering revision range for bisection.'
|
|
|
|
# Retrieve a list of revisions to do bisection on.
|
|
src_revision_list = self.GetRevisionList(target_depot,
|
|
bad_revision,
|
|
good_revision)
|
|
|
|
if self.opts.output_buildbot_annotations:
|
|
bisect_utils.OutputAnnotationStepClosed()
|
|
|
|
if src_revision_list:
|
|
# revision_data will store information about a revision such as the
|
|
# depot it came from, the webkit/V8 revision at that time,
|
|
# performance timing, build state, etc...
|
|
revision_data = results['revision_data']
|
|
|
|
# revision_list is the list we're binary searching through at the moment.
|
|
revision_list = []
|
|
|
|
sort_key_ids = 0
|
|
|
|
for current_revision_id in src_revision_list:
|
|
sort_key_ids += 1
|
|
|
|
revision_data[current_revision_id] = {'value' : None,
|
|
'passed' : '?',
|
|
'depot' : target_depot,
|
|
'external' : None,
|
|
'sort' : sort_key_ids}
|
|
revision_list.append(current_revision_id)
|
|
|
|
min_revision = 0
|
|
max_revision = len(revision_list) - 1
|
|
|
|
self.PrintRevisionsToBisectMessage(revision_list, target_depot)
|
|
|
|
if self.opts.output_buildbot_annotations:
|
|
bisect_utils.OutputAnnotationStepStart('Gathering Reference Values')
|
|
|
|
print 'Gathering reference values for bisection.'
|
|
|
|
# Perform the performance tests on the good and bad revisions, to get
|
|
# reference values.
|
|
(bad_results, good_results) = self.GatherReferenceValues(good_revision,
|
|
bad_revision,
|
|
command_to_run,
|
|
metric,
|
|
target_depot)
|
|
|
|
if self.opts.output_buildbot_annotations:
|
|
bisect_utils.OutputAnnotationStepClosed()
|
|
|
|
if bad_results[1]:
|
|
results['error'] = bad_results[0]
|
|
return results
|
|
|
|
if good_results[1]:
|
|
results['error'] = good_results[0]
|
|
return results
|
|
|
|
|
|
# We need these reference values to determine if later runs should be
|
|
# classified as pass or fail.
|
|
known_bad_value = bad_results[0]
|
|
known_good_value = good_results[0]
|
|
|
|
# Can just mark the good and bad revisions explicitly here since we
|
|
# already know the results.
|
|
bad_revision_data = revision_data[revision_list[0]]
|
|
bad_revision_data['external'] = bad_results[2]
|
|
bad_revision_data['passed'] = 0
|
|
bad_revision_data['value'] = known_bad_value
|
|
|
|
good_revision_data = revision_data[revision_list[max_revision]]
|
|
good_revision_data['external'] = good_results[2]
|
|
good_revision_data['passed'] = 1
|
|
good_revision_data['value'] = known_good_value
|
|
|
|
next_revision_depot = target_depot
|
|
|
|
while True:
|
|
if not revision_list:
|
|
break
|
|
|
|
min_revision_data = revision_data[revision_list[min_revision]]
|
|
max_revision_data = revision_data[revision_list[max_revision]]
|
|
|
|
if max_revision - min_revision <= 1:
|
|
if min_revision_data['passed'] == '?':
|
|
next_revision_index = min_revision
|
|
elif max_revision_data['passed'] == '?':
|
|
next_revision_index = max_revision
|
|
elif min_revision_data['depot'] == 'chromium' or\
|
|
min_revision_data['depot'] == 'cros':
|
|
# If there were changes to any of the external libraries we track,
|
|
# should bisect the changes there as well.
|
|
external_depot = None
|
|
|
|
for current_depot in DEPOT_NAMES:
|
|
if DEPOT_DEPS_NAME[current_depot]["recurse"] and\
|
|
DEPOT_DEPS_NAME[current_depot]['from'] ==\
|
|
min_revision_data['depot']:
|
|
if min_revision_data['external'][current_depot] !=\
|
|
max_revision_data['external'][current_depot]:
|
|
external_depot = current_depot
|
|
break
|
|
|
|
# If there was no change in any of the external depots, the search
|
|
# is over.
|
|
if not external_depot:
|
|
break
|
|
|
|
previous_revision = revision_list[min_revision]
|
|
|
|
earliest_revision = max_revision_data['external'][external_depot]
|
|
latest_revision = min_revision_data['external'][external_depot]
|
|
|
|
new_revision_list = self.PrepareToBisectOnDepot(external_depot,
|
|
latest_revision,
|
|
earliest_revision,
|
|
next_revision_depot,
|
|
previous_revision)
|
|
|
|
if not new_revision_list:
|
|
results['error'] = 'An error occurred attempting to retrieve'\
|
|
' revision range: [%s..%s]' %\
|
|
(depot_rev_range[1], depot_rev_range[0])
|
|
return results
|
|
|
|
self.AddRevisionsIntoRevisionData(new_revision_list,
|
|
external_depot,
|
|
min_revision_data['sort'],
|
|
revision_data)
|
|
|
|
# Reset the bisection and perform it on the newly inserted
|
|
# changelists.
|
|
revision_list = new_revision_list
|
|
min_revision = 0
|
|
max_revision = len(revision_list) - 1
|
|
sort_key_ids += len(revision_list)
|
|
|
|
print 'Regression in metric:%s appears to be the result of changes'\
|
|
' in [%s].' % (metric, external_depot)
|
|
|
|
self.PrintRevisionsToBisectMessage(revision_list, external_depot)
|
|
|
|
continue
|
|
else:
|
|
break
|
|
else:
|
|
next_revision_index = int((max_revision - min_revision) / 2) +\
|
|
min_revision
|
|
|
|
next_revision_id = revision_list[next_revision_index]
|
|
next_revision_data = revision_data[next_revision_id]
|
|
next_revision_depot = next_revision_data['depot']
|
|
|
|
self.ChangeToDepotWorkingDirectory(next_revision_depot)
|
|
|
|
if self.opts.output_buildbot_annotations:
|
|
step_name = 'Working on [%s]' % next_revision_id
|
|
bisect_utils.OutputAnnotationStepStart(step_name)
|
|
|
|
print 'Working on revision: [%s]' % next_revision_id
|
|
|
|
run_results = self.SyncBuildAndRunRevision(next_revision_id,
|
|
next_revision_depot,
|
|
command_to_run,
|
|
metric, skippable=True)
|
|
|
|
# If the build is successful, check whether or not the metric
|
|
# had regressed.
|
|
if not run_results[1]:
|
|
if len(run_results) > 2:
|
|
next_revision_data['external'] = run_results[2]
|
|
|
|
passed_regression = self.CheckIfRunPassed(run_results[0],
|
|
known_good_value,
|
|
known_bad_value)
|
|
|
|
next_revision_data['passed'] = passed_regression
|
|
next_revision_data['value'] = run_results[0]
|
|
|
|
if passed_regression:
|
|
max_revision = next_revision_index
|
|
else:
|
|
min_revision = next_revision_index
|
|
else:
|
|
if run_results[1] == BUILD_RESULT_SKIPPED:
|
|
next_revision_data['passed'] = 'Skipped'
|
|
elif run_results[1] == BUILD_RESULT_FAIL:
|
|
next_revision_data['passed'] = 'Failed'
|
|
|
|
print run_results[0]
|
|
|
|
# If the build is broken, remove it and redo search.
|
|
revision_list.pop(next_revision_index)
|
|
|
|
max_revision -= 1
|
|
|
|
if self.opts.output_buildbot_annotations:
|
|
bisect_utils.OutputAnnotationStepClosed()
|
|
else:
|
|
# Weren't able to sync and retrieve the revision range.
|
|
results['error'] = 'An error occurred attempting to retrieve revision '\
|
|
'range: [%s..%s]' % (good_revision, bad_revision)
|
|
|
|
return results
|
|
|
|
def FormatAndPrintResults(self, bisect_results):
|
|
"""Prints the results from a bisection run in a readable format.
|
|
|
|
Args
|
|
bisect_results: The results from a bisection test run.
|
|
"""
|
|
revision_data = bisect_results['revision_data']
|
|
revision_data_sorted = sorted(revision_data.iteritems(),
|
|
key = lambda x: x[1]['sort'])
|
|
|
|
if self.opts.output_buildbot_annotations:
|
|
bisect_utils.OutputAnnotationStepStart('Results')
|
|
|
|
print
|
|
print 'Full results of bisection:'
|
|
for current_id, current_data in revision_data_sorted:
|
|
build_status = current_data['passed']
|
|
|
|
if type(build_status) is bool:
|
|
build_status = int(build_status)
|
|
|
|
print ' %8s %40s %s' % (current_data['depot'],
|
|
current_id, build_status)
|
|
print
|
|
|
|
print
|
|
print 'Tested commits:'
|
|
for current_id, current_data in revision_data_sorted:
|
|
if current_data['value']:
|
|
print ' %8s %40s %12f %12f' % (
|
|
current_data['depot'], current_id,
|
|
current_data['value']['mean'], current_data['value']['std_dev'])
|
|
print
|
|
|
|
# Find range where it possibly broke.
|
|
first_working_revision = None
|
|
last_broken_revision = None
|
|
last_broken_revision_index = -1
|
|
|
|
for i in xrange(len(revision_data_sorted)):
|
|
k, v = revision_data_sorted[i]
|
|
if v['passed'] == 1:
|
|
if not first_working_revision:
|
|
first_working_revision = k
|
|
|
|
if not v['passed']:
|
|
last_broken_revision = k
|
|
last_broken_revision_index = i
|
|
|
|
if last_broken_revision != None and first_working_revision != None:
|
|
print 'Results: Regression may have occurred in range:'
|
|
print ' -> First Bad Revision: [%40s] [%s]' %\
|
|
(last_broken_revision,
|
|
revision_data[last_broken_revision]['depot'])
|
|
print ' -> Last Good Revision: [%40s] [%s]' %\
|
|
(first_working_revision,
|
|
revision_data[first_working_revision]['depot'])
|
|
|
|
cwd = os.getcwd()
|
|
self.ChangeToDepotWorkingDirectory(
|
|
revision_data[last_broken_revision]['depot'])
|
|
|
|
if revision_data[last_broken_revision]['depot'] == 'cros':
|
|
# Want to get a list of all the commits and what depots they belong
|
|
# to so that we can grab info about each.
|
|
cmd = ['repo', 'forall', '-c',
|
|
'pwd ; git log --pretty=oneline --before=%d --after=%d' % (
|
|
last_broken_revision, first_working_revision + 1)]
|
|
(output, return_code) = RunProcessAndRetrieveOutput(cmd)
|
|
|
|
changes = []
|
|
|
|
assert not return_code, 'An error occurred while running'\
|
|
' "%s"' % ' '.join(cmd)
|
|
|
|
last_depot = None
|
|
cwd = os.getcwd()
|
|
for l in output.split('\n'):
|
|
if l:
|
|
# Output will be in form:
|
|
# /path_to_depot
|
|
# /path_to_other_depot
|
|
# <SHA1>
|
|
# /path_again
|
|
# <SHA1>
|
|
# etc.
|
|
if l[0] == '/':
|
|
last_depot = l
|
|
else:
|
|
contents = l.split(' ')
|
|
if len(contents) > 1:
|
|
changes.append([last_depot, contents[0]])
|
|
|
|
print
|
|
for c in changes:
|
|
os.chdir(c[0])
|
|
info = self.source_control.QueryRevisionInfo(c[1])
|
|
|
|
print
|
|
print 'Commit : %s' % c[1]
|
|
print 'Author : %s' % info['author']
|
|
print 'Email : %s' % info['email']
|
|
print 'Date : %s' % info['date']
|
|
print 'Subject : %s' % info['subject']
|
|
print
|
|
else:
|
|
multiple_commits = 0
|
|
for i in xrange(last_broken_revision_index, len(revision_data_sorted)):
|
|
k, v = revision_data_sorted[i]
|
|
if k == first_working_revision:
|
|
break
|
|
|
|
self.ChangeToDepotWorkingDirectory(v['depot'])
|
|
|
|
info = self.source_control.QueryRevisionInfo(k)
|
|
|
|
print
|
|
print 'Commit : %s' % k
|
|
print 'Author : %s' % info['author']
|
|
print 'Email : %s' % info['email']
|
|
print 'Date : %s' % info['date']
|
|
print 'Subject : %s' % info['subject']
|
|
|
|
multiple_commits += 1
|
|
if multiple_commits > 1:
|
|
self.warnings.append('Due to build errors, regression range could'
|
|
' not be narrowed down to a single commit.')
|
|
print
|
|
os.chdir(cwd)
|
|
|
|
# Give a warning if the values were very close together
|
|
good_std_dev = revision_data[first_working_revision]['value']['std_dev']
|
|
good_mean = revision_data[first_working_revision]['value']['mean']
|
|
bad_mean = revision_data[last_broken_revision]['value']['mean']
|
|
|
|
# A standard deviation of 0 could indicate either insufficient runs
|
|
# or a test that consistently returns the same value.
|
|
if good_std_dev > 0:
|
|
deviations = math.fabs(bad_mean - good_mean) / good_std_dev
|
|
|
|
if deviations < 1.5:
|
|
self.warnings.append('Regression was less than 1.5 standard '
|
|
'deviations from "good" value. Results may not be accurate.')
|
|
elif self.opts.repeat_test_count == 1:
|
|
self.warnings.append('Tests were only set to run once. This '
|
|
'may be insufficient to get meaningful results.')
|
|
|
|
# Check for any other possible regression ranges
|
|
prev_revision_data = revision_data_sorted[0][1]
|
|
prev_revision_id = revision_data_sorted[0][0]
|
|
possible_regressions = []
|
|
for current_id, current_data in revision_data_sorted:
|
|
if current_data['value']:
|
|
prev_mean = prev_revision_data['value']['mean']
|
|
cur_mean = current_data['value']['mean']
|
|
|
|
if good_std_dev:
|
|
deviations = math.fabs(prev_mean - cur_mean) / good_std_dev
|
|
else:
|
|
deviations = None
|
|
|
|
if good_mean:
|
|
percent_change = (prev_mean - cur_mean) / good_mean
|
|
|
|
# If the "good" valuse are supposed to be higher than the "bad"
|
|
# values (ie. scores), flip the sign of the percent change so that
|
|
# a positive value always represents a regression.
|
|
if bad_mean < good_mean:
|
|
percent_change *= -1.0
|
|
else:
|
|
percent_change = None
|
|
|
|
if deviations >= 1.5 or percent_change > 0.01:
|
|
if current_id != first_working_revision:
|
|
possible_regressions.append(
|
|
[current_id, prev_revision_id, percent_change, deviations])
|
|
prev_revision_data = current_data
|
|
prev_revision_id = current_id
|
|
|
|
if possible_regressions:
|
|
print
|
|
print 'Other regressions may have occurred:'
|
|
print
|
|
for p in possible_regressions:
|
|
current_id = p[0]
|
|
percent_change = p[2]
|
|
deviations = p[3]
|
|
current_data = revision_data[current_id]
|
|
previous_id = p[1]
|
|
previous_data = revision_data[previous_id]
|
|
|
|
if deviations is None:
|
|
deviations = 'N/A'
|
|
else:
|
|
deviations = '%.2f' % deviations
|
|
|
|
if percent_change is None:
|
|
percent_change = 0
|
|
|
|
print ' %8s %s [%.2f%%, %s x std.dev]' % (
|
|
previous_data['depot'], previous_id, 100 * percent_change,
|
|
deviations)
|
|
print ' %8s %s' % (
|
|
current_data['depot'], current_id)
|
|
print
|
|
|
|
if self.warnings:
|
|
print
|
|
print 'The following warnings were generated:'
|
|
print
|
|
for w in self.warnings:
|
|
print ' - %s' % w
|
|
print
|
|
|
|
if self.opts.output_buildbot_annotations:
|
|
bisect_utils.OutputAnnotationStepClosed()
|
|
|
|
|
|
def DetermineAndCreateSourceControl(opts):
|
|
"""Attempts to determine the underlying source control workflow and returns
|
|
a SourceControl object.
|
|
|
|
Returns:
|
|
An instance of a SourceControl object, or None if the current workflow
|
|
is unsupported.
|
|
"""
|
|
|
|
(output, return_code) = RunGit(['rev-parse', '--is-inside-work-tree'])
|
|
|
|
if output.strip() == 'true':
|
|
return GitSourceControl(opts)
|
|
|
|
return None
|
|
|
|
|
|
def SetNinjaBuildSystemDefault():
|
|
"""Makes ninja the default build system to be used by
|
|
the bisection script."""
|
|
gyp_var = os.getenv('GYP_GENERATORS')
|
|
|
|
if not gyp_var or not 'ninja' in gyp_var:
|
|
if gyp_var:
|
|
os.environ['GYP_GENERATORS'] = gyp_var + ',ninja'
|
|
else:
|
|
os.environ['GYP_GENERATORS'] = 'ninja'
|
|
|
|
if IsWindows():
|
|
os.environ['GYP_DEFINES'] = 'component=shared_library '\
|
|
'incremental_chrome_dll=1 disable_nacl=1 fastbuild=1 '\
|
|
'chromium_win_pch=0'
|
|
|
|
|
|
def SetMakeBuildSystemDefault():
|
|
"""Makes make the default build system to be used by
|
|
the bisection script."""
|
|
os.environ['GYP_GENERATORS'] = 'make'
|
|
|
|
|
|
def CheckPlatformSupported(opts):
|
|
"""Checks that this platform and build system are supported.
|
|
|
|
Args:
|
|
opts: The options parsed from the command line.
|
|
|
|
Returns:
|
|
True if the platform and build system are supported.
|
|
"""
|
|
# Haven't tested the script out on any other platforms yet.
|
|
supported = ['posix', 'nt']
|
|
if not os.name in supported:
|
|
print "Sorry, this platform isn't supported yet."
|
|
print
|
|
return False
|
|
|
|
if IsWindows():
|
|
if not opts.build_preference:
|
|
opts.build_preference = 'msvs'
|
|
|
|
if opts.build_preference == 'msvs':
|
|
if not os.getenv('VS100COMNTOOLS'):
|
|
print 'Error: Path to visual studio could not be determined.'
|
|
print
|
|
return False
|
|
elif opts.build_preference == 'ninja':
|
|
SetNinjaBuildSystemDefault()
|
|
else:
|
|
assert False, 'Error: %s build not supported' % opts.build_preference
|
|
else:
|
|
if not opts.build_preference:
|
|
if 'ninja' in os.getenv('GYP_GENERATORS'):
|
|
opts.build_preference = 'ninja'
|
|
else:
|
|
opts.build_preference = 'make'
|
|
|
|
if opts.build_preference == 'ninja':
|
|
SetNinjaBuildSystemDefault()
|
|
elif opts.build_preference == 'make':
|
|
SetMakeBuildSystemDefault()
|
|
elif opts.build_preference != 'make':
|
|
assert False, 'Error: %s build not supported' % opts.build_preference
|
|
|
|
bisect_utils.RunGClient(['runhooks'])
|
|
|
|
return True
|
|
|
|
|
|
def RmTreeAndMkDir(path_to_dir):
|
|
"""Removes the directory tree specified, and then creates an empty
|
|
directory in the same location.
|
|
|
|
Args:
|
|
path_to_dir: Path to the directory tree.
|
|
|
|
Returns:
|
|
True if successful, False if an error occurred.
|
|
"""
|
|
try:
|
|
if os.path.exists(path_to_dir):
|
|
shutil.rmtree(path_to_dir)
|
|
except OSError, e:
|
|
if e.errno != errno.ENOENT:
|
|
return False
|
|
|
|
try:
|
|
os.makedirs(path_to_dir)
|
|
except OSError, e:
|
|
if e.errno != errno.EEXIST:
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def RemoveBuildFiles():
|
|
"""Removes build files from previous runs."""
|
|
if RmTreeAndMkDir(os.path.join('out', 'Release')):
|
|
if RmTreeAndMkDir(os.path.join('build', 'Release')):
|
|
return True
|
|
return False
|
|
|
|
|
|
def main():
|
|
|
|
usage = ('%prog [options] [-- chromium-options]\n'
|
|
'Perform binary search on revision history to find a minimal '
|
|
'range of revisions where a peformance metric regressed.\n')
|
|
|
|
parser = optparse.OptionParser(usage=usage)
|
|
|
|
parser.add_option('-c', '--command',
|
|
type='str',
|
|
help='A command to execute your performance test at' +
|
|
' each point in the bisection.')
|
|
parser.add_option('-b', '--bad_revision',
|
|
type='str',
|
|
help='A bad revision to start bisection. ' +
|
|
'Must be later than good revision. May be either a git' +
|
|
' or svn revision.')
|
|
parser.add_option('-g', '--good_revision',
|
|
type='str',
|
|
help='A revision to start bisection where performance' +
|
|
' test is known to pass. Must be earlier than the ' +
|
|
'bad revision. May be either a git or svn revision.')
|
|
parser.add_option('-m', '--metric',
|
|
type='str',
|
|
help='The desired metric to bisect on. For example ' +
|
|
'"vm_rss_final_b/vm_rss_f_b"')
|
|
parser.add_option('-w', '--working_directory',
|
|
type='str',
|
|
help='Path to the working directory where the script will '
|
|
'do an initial checkout of the chromium depot. The '
|
|
'files will be placed in a subdirectory "bisect" under '
|
|
'working_directory and that will be used to perform the '
|
|
'bisection. This parameter is optional, if it is not '
|
|
'supplied, the script will work from the current depot.')
|
|
parser.add_option('-r', '--repeat_test_count',
|
|
type='int',
|
|
default=20,
|
|
help='The number of times to repeat the performance test. '
|
|
'Values will be clamped to range [1, 100]. '
|
|
'Default value is 20.')
|
|
parser.add_option('--repeat_test_max_time',
|
|
type='int',
|
|
default=20,
|
|
help='The maximum time (in minutes) to take running the '
|
|
'performance tests. The script will run the performance '
|
|
'tests according to --repeat_test_count, so long as it '
|
|
'doesn\'t exceed --repeat_test_max_time. Values will be '
|
|
'clamped to range [1, 60].'
|
|
'Default value is 20.')
|
|
parser.add_option('-t', '--truncate_percent',
|
|
type='int',
|
|
default=25,
|
|
help='The highest/lowest % are discarded to form a '
|
|
'truncated mean. Values will be clamped to range [0, 25]. '
|
|
'Default value is 25 (highest/lowest 25% will be '
|
|
'discarded).')
|
|
parser.add_option('--build_preference',
|
|
type='choice',
|
|
choices=['msvs', 'ninja', 'make'],
|
|
help='The preferred build system to use. On linux/mac '
|
|
'the options are make/ninja. On Windows, the options '
|
|
'are msvs/ninja.')
|
|
parser.add_option('--target_platform',
|
|
type='choice',
|
|
choices=['chromium', 'cros', 'android'],
|
|
default='chromium',
|
|
help='The target platform. Choices are "chromium" (current '
|
|
'platform), "cros", or "android". If you specify something '
|
|
'other than "chromium", you must be properly set up to '
|
|
'build that platform.')
|
|
parser.add_option('--cros_board',
|
|
type='str',
|
|
help='The cros board type to build.')
|
|
parser.add_option('--cros_remote_ip',
|
|
type='str',
|
|
help='The remote machine to image to.')
|
|
parser.add_option('--use_goma',
|
|
action="store_true",
|
|
help='Add a bunch of extra threads for goma.')
|
|
parser.add_option('--output_buildbot_annotations',
|
|
action="store_true",
|
|
help='Add extra annotation output for buildbot.')
|
|
parser.add_option('--debug_ignore_build',
|
|
action="store_true",
|
|
help='DEBUG: Don\'t perform builds.')
|
|
parser.add_option('--debug_ignore_sync',
|
|
action="store_true",
|
|
help='DEBUG: Don\'t perform syncs.')
|
|
parser.add_option('--debug_ignore_perf_test',
|
|
action="store_true",
|
|
help='DEBUG: Don\'t perform performance tests.')
|
|
(opts, args) = parser.parse_args()
|
|
|
|
if not opts.command:
|
|
print 'Error: missing required parameter: --command'
|
|
print
|
|
parser.print_help()
|
|
return 1
|
|
|
|
if not opts.good_revision:
|
|
print 'Error: missing required parameter: --good_revision'
|
|
print
|
|
parser.print_help()
|
|
return 1
|
|
|
|
if not opts.bad_revision:
|
|
print 'Error: missing required parameter: --bad_revision'
|
|
print
|
|
parser.print_help()
|
|
return 1
|
|
|
|
if not opts.metric:
|
|
print 'Error: missing required parameter: --metric'
|
|
print
|
|
parser.print_help()
|
|
return 1
|
|
|
|
if opts.target_platform == 'cros':
|
|
# Run sudo up front to make sure credentials are cached for later.
|
|
print 'Sudo is required to build cros:'
|
|
print
|
|
RunProcess(['sudo', 'true'])
|
|
|
|
if not opts.cros_board:
|
|
print 'Error: missing required parameter: --cros_board'
|
|
print
|
|
parser.print_help()
|
|
return 1
|
|
|
|
if not opts.cros_remote_ip:
|
|
print 'Error: missing required parameter: --cros_remote_ip'
|
|
print
|
|
parser.print_help()
|
|
return 1
|
|
|
|
if not opts.working_directory:
|
|
print 'Error: missing required parameter: --working_directory'
|
|
print
|
|
parser.print_help()
|
|
return 1
|
|
|
|
opts.repeat_test_count = min(max(opts.repeat_test_count, 1), 100)
|
|
opts.repeat_test_max_time = min(max(opts.repeat_test_max_time, 1), 60)
|
|
opts.truncate_percent = min(max(opts.truncate_percent, 0), 25)
|
|
opts.truncate_percent = opts.truncate_percent / 100.0
|
|
|
|
metric_values = opts.metric.split('/')
|
|
if len(metric_values) != 2:
|
|
print "Invalid metric specified: [%s]" % (opts.metric,)
|
|
print
|
|
return 1
|
|
|
|
if opts.working_directory:
|
|
if bisect_utils.CreateBisectDirectoryAndSetupDepot(opts):
|
|
return 1
|
|
|
|
if not bisect_utils.SetupPlatformBuildEnvironment(opts):
|
|
print 'Error: Failed to set platform environment.'
|
|
print
|
|
return 1
|
|
|
|
os.chdir(os.path.join(os.getcwd(), 'src'))
|
|
|
|
if not RemoveBuildFiles():
|
|
print "Something went wrong removing the build files."
|
|
print
|
|
return 1
|
|
|
|
if not CheckPlatformSupported(opts):
|
|
return 1
|
|
|
|
# Check what source control method they're using. Only support git workflow
|
|
# at the moment.
|
|
source_control = DetermineAndCreateSourceControl(opts)
|
|
|
|
if not source_control:
|
|
print "Sorry, only the git workflow is supported at the moment."
|
|
print
|
|
return 1
|
|
|
|
# gClient sync seems to fail if you're not in master branch.
|
|
if not source_control.IsInProperBranch() and not opts.debug_ignore_sync:
|
|
print "You must switch to master branch to run bisection."
|
|
print
|
|
return 1
|
|
|
|
bisect_test = BisectPerformanceMetrics(source_control, opts)
|
|
try:
|
|
bisect_results = bisect_test.Run(opts.command,
|
|
opts.bad_revision,
|
|
opts.good_revision,
|
|
metric_values)
|
|
if not(bisect_results['error']):
|
|
bisect_test.FormatAndPrintResults(bisect_results)
|
|
finally:
|
|
bisect_test.PerformCleanup()
|
|
|
|
if not(bisect_results['error']):
|
|
return 0
|
|
else:
|
|
print 'Error: ' + bisect_results['error']
|
|
print
|
|
return 1
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|