264 lines
9.6 KiB
Python
264 lines
9.6 KiB
Python
|
#!/usr/bin/env python
|
||
|
|
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
|
||
|
|
# Use of this source code is governed by a BSD-style license that can be
|
||
|
|
# found in the LICENSE file.
|
||
|
|
|
||
|
|
"""Prepare tests that require re-baselining for input to make_expectations.py.
|
||
|
|
|
||
|
|
The regularly running perf-AV tests require re-baselineing of expectations
|
||
|
|
about once a week. The steps involved in rebaselining are:
|
||
|
|
|
||
|
|
1.) Identify the tests to update, based off reported e-mail results.
|
||
|
|
2.) Figure out reva and revb values, which is the starting and ending revision
|
||
|
|
numbers for the range that we should use to obtain new thresholds.
|
||
|
|
3.) Modify lines in perf_expectations.json referring to the tests to be updated,
|
||
|
|
so that they may be used as input to make_expectations.py.
|
||
|
|
|
||
|
|
This script automates the last step above.
|
||
|
|
|
||
|
|
Here's a sample line from perf_expectations.json:
|
||
|
|
|
||
|
|
"win-release/media_tests_av_perf/fps/tulip2.m4a": {"reva": 163299, \
|
||
|
|
"revb": 164141, "type": "absolute", "better": "higher", "improve": 0, \
|
||
|
|
"regress": 0, "sha1": "54d94538"},
|
||
|
|
|
||
|
|
To get the above test ready for input to make_expectations.py, it should become:
|
||
|
|
|
||
|
|
"win-release/media_tests_av_perf/fps/tulip2.m4a": {"reva": <new reva>, \
|
||
|
|
"revb": <new revb>, "type": "absolute", "better": "higher", "improve": 0, \
|
||
|
|
"regress": 0},
|
||
|
|
|
||
|
|
Examples:
|
||
|
|
|
||
|
|
1.) To update the test specified above and get baseline
|
||
|
|
values using the revision range 12345 and 23456, run this script with a command
|
||
|
|
line like this:
|
||
|
|
python update_perf_expectations.py -f \
|
||
|
|
win-release/media_tests_av_perf/fps/tulip2.m4a --reva 12345 --revb 23456
|
||
|
|
Or, using an input file,
|
||
|
|
where the input file contains a single line with text
|
||
|
|
win-release/media_tests_av_perf/fps/tulip2.m4a
|
||
|
|
run with this command line:
|
||
|
|
python update_perf_expectations.py -i input.txt --reva 12345 --revb 23456
|
||
|
|
|
||
|
|
2.) Let's say you want to update all seek tests on windows, and get baseline
|
||
|
|
values using the revision range 12345 and 23456.
|
||
|
|
Run this script with this command line:
|
||
|
|
python update_perf_expectations.py -f win-release/media_tests_av_perf/seek/ \
|
||
|
|
--reva 12345 --revb 23456
|
||
|
|
Or:
|
||
|
|
python update_perf_expectations.py -f win-release/.*/seek/ --reva 12345 \
|
||
|
|
--revb 23456
|
||
|
|
|
||
|
|
Or, using an input file,
|
||
|
|
where the input file contains a single line with text win-release/.*/seek/:
|
||
|
|
python update_perf_expectations.py -i input.txt --reva 12345 --revb 23456
|
||
|
|
|
||
|
|
3.) Similarly, if you want to update seek tests on all platforms
|
||
|
|
python update_perf_expectations.py -f .*-release/.*/seek/ --reva 12345 \
|
||
|
|
--revb 23456
|
||
|
|
|
||
|
|
"""
|
||
|
|
|
||
|
|
import logging
|
||
|
|
from optparse import OptionParser
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
|
||
|
|
import make_expectations as perf_ex_lib
|
||
|
|
|
||
|
|
# Default logging is INFO. Use --verbose to enable DEBUG logging.
|
||
|
|
_DEFAULT_LOG_LEVEL = logging.INFO
|
||
|
|
|
||
|
|
|
||
|
|
def GetTestsToUpdate(contents, all_test_keys):
|
||
|
|
"""Parses input contents and obtains tests to be re-baselined.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
contents: string containing contents of input file.
|
||
|
|
all_test_keys: list of keys of test dictionary.
|
||
|
|
Returns:
|
||
|
|
A list of keys for tests that should be updated.
|
||
|
|
"""
|
||
|
|
# Each line of the input file specifies a test case to update.
|
||
|
|
tests_list = []
|
||
|
|
for test_case_filter in contents.splitlines():
|
||
|
|
# Skip any empty lines.
|
||
|
|
if test_case_filter:
|
||
|
|
# Sample expected line:
|
||
|
|
# win-release/media_tests_av_perf/seek/\
|
||
|
|
# CACHED_BUFFERED_SEEK_NoConstraints_crowd1080.ogv
|
||
|
|
# Or, if reg-ex, then sample line:
|
||
|
|
# win-release/media-tests_av_perf/seek*
|
||
|
|
# Skip any leading spaces if they exist in the input file.
|
||
|
|
logging.debug('Trying to match %s', test_case_filter)
|
||
|
|
tests_list.extend(GetMatchingTests(test_case_filter.strip(),
|
||
|
|
all_test_keys))
|
||
|
|
return tests_list
|
||
|
|
|
||
|
|
|
||
|
|
def GetMatchingTests(tests_to_update, all_test_keys):
|
||
|
|
"""Parses input reg-ex filter and obtains tests to be re-baselined.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
tests_to_update: reg-ex string specifying tests to be updated.
|
||
|
|
all_test_keys: list of keys of tests dictionary.
|
||
|
|
Returns:
|
||
|
|
A list of keys for tests that should be updated.
|
||
|
|
"""
|
||
|
|
tests_list = []
|
||
|
|
search_string = re.compile(tests_to_update)
|
||
|
|
# Get matching tests from the dictionary of tests
|
||
|
|
for test_key in all_test_keys:
|
||
|
|
if search_string.match(test_key):
|
||
|
|
tests_list.append(test_key)
|
||
|
|
logging.debug('%s will be updated', test_key)
|
||
|
|
logging.info('%s tests found matching reg-ex: %s', len(tests_list),
|
||
|
|
tests_to_update)
|
||
|
|
return tests_list
|
||
|
|
|
||
|
|
|
||
|
|
def PrepareTestsForUpdate(tests_to_update, all_tests, reva, revb):
|
||
|
|
"""Modifies value of tests that are to re-baselined:
|
||
|
|
Set reva and revb values to specified new values. Remove sha1.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
tests_to_update: list of tests to be updated.
|
||
|
|
all_tests: dictionary of all tests.
|
||
|
|
reva: oldest revision in range to use for new values.
|
||
|
|
revb: newest revision in range to use for new values.
|
||
|
|
Raises:
|
||
|
|
ValueError: If reva or revb are not valid ints, or if either
|
||
|
|
of them are negative.
|
||
|
|
"""
|
||
|
|
reva = int(reva)
|
||
|
|
revb = int(revb)
|
||
|
|
|
||
|
|
if reva < 0 or revb < 0:
|
||
|
|
raise ValueError('Revision values should be positive.')
|
||
|
|
# Ensure reva is less than revb.
|
||
|
|
# (this is similar to the check done in make_expectations.py)
|
||
|
|
if revb < reva:
|
||
|
|
temp = revb
|
||
|
|
revb = reva
|
||
|
|
reva = temp
|
||
|
|
for test_key in tests_to_update:
|
||
|
|
# Get original test from the dictionary of tests
|
||
|
|
test_value = all_tests[test_key]
|
||
|
|
if test_value:
|
||
|
|
# Sample line in perf_expectations.json:
|
||
|
|
# "linux-release/media_tests _av_perf/dropped_frames/crowd360.webm":\
|
||
|
|
# {"reva": 155180, "revb": 155280, "type": "absolute", \
|
||
|
|
# "better": "lower", "improve": 0, "regress": 3, "sha1": "276ba29c"},
|
||
|
|
# Set new revision range
|
||
|
|
test_value['reva'] = reva
|
||
|
|
test_value['revb'] = revb
|
||
|
|
# Remove sha1 to indicate this test requires an update
|
||
|
|
# Check first to make sure it exist.
|
||
|
|
if 'sha1' in test_value:
|
||
|
|
del test_value['sha1']
|
||
|
|
else:
|
||
|
|
logging.warning('%s does not exist.', test_key)
|
||
|
|
logging.info('Done preparing tests for update.')
|
||
|
|
|
||
|
|
|
||
|
|
def GetCommandLineOptions():
|
||
|
|
"""Parse command line arguments.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
An options object containing command line arguments and their values.
|
||
|
|
"""
|
||
|
|
parser = OptionParser()
|
||
|
|
|
||
|
|
parser.add_option('--reva', dest='reva', type='int',
|
||
|
|
help='Starting revision of new range.',
|
||
|
|
metavar='START_REVISION')
|
||
|
|
parser.add_option('--revb', dest='revb', type='int',
|
||
|
|
help='Ending revision of new range.',
|
||
|
|
metavar='END_REVISION')
|
||
|
|
parser.add_option('-f', dest='tests_filter',
|
||
|
|
help='Regex to use for filtering tests to be updated. '
|
||
|
|
'At least one of -filter or -input_file must be provided. '
|
||
|
|
'If both are provided, then input-file is used.',
|
||
|
|
metavar='FILTER', default='')
|
||
|
|
parser.add_option('-i', dest='input_file',
|
||
|
|
help='Optional path to file with reg-exes for tests to'
|
||
|
|
' update. If provided, it overrides the filter argument.',
|
||
|
|
metavar='INPUT_FILE', default='')
|
||
|
|
parser.add_option('--config', dest='config_file',
|
||
|
|
default=perf_ex_lib.DEFAULT_CONFIG_FILE,
|
||
|
|
help='Set the config file to FILE.', metavar='FILE')
|
||
|
|
parser.add_option('-v', dest='verbose', action='store_true', default=False,
|
||
|
|
help='Enable verbose output.')
|
||
|
|
options = parser.parse_args()[0]
|
||
|
|
return options
|
||
|
|
|
||
|
|
|
||
|
|
def Main():
|
||
|
|
"""Main driver function."""
|
||
|
|
options = GetCommandLineOptions()
|
||
|
|
|
||
|
|
_SetLogger(options.verbose)
|
||
|
|
# Do some command-line validation
|
||
|
|
if not options.input_file and not options.tests_filter:
|
||
|
|
logging.error('At least one of input-file or test-filter must be provided.')
|
||
|
|
exit(1)
|
||
|
|
if options.input_file and options.tests_filter:
|
||
|
|
logging.error('Specify only one of input file or test-filter.')
|
||
|
|
exit(1)
|
||
|
|
if not options.reva or not options.revb:
|
||
|
|
logging.error('Start and end revision of range must be specified.')
|
||
|
|
exit(1)
|
||
|
|
|
||
|
|
# Load config.
|
||
|
|
config = perf_ex_lib.ConvertJsonIntoDict(
|
||
|
|
perf_ex_lib.ReadFile(options.config_file))
|
||
|
|
|
||
|
|
# Obtain the perf expectations file from the config file.
|
||
|
|
perf_file = os.path.join(
|
||
|
|
os.path.dirname(options.config_file), config['perf_file'])
|
||
|
|
|
||
|
|
# We should have all the information we require now.
|
||
|
|
# On to the real thang.
|
||
|
|
# First, get all the existing tests from the original perf_expectations file.
|
||
|
|
all_tests = perf_ex_lib.ConvertJsonIntoDict(
|
||
|
|
perf_ex_lib.ReadFile(perf_file))
|
||
|
|
all_test_keys = all_tests.keys()
|
||
|
|
# Remove the load key, because we don't want to modify it.
|
||
|
|
all_test_keys.remove('load')
|
||
|
|
# Keep tests sorted, like in the original file.
|
||
|
|
all_test_keys.sort()
|
||
|
|
|
||
|
|
# Next, get all tests that have been identified for an update.
|
||
|
|
tests_to_update = []
|
||
|
|
if options.input_file:
|
||
|
|
# Tests to update have been specified in an input_file.
|
||
|
|
# Get contents of file.
|
||
|
|
tests_filter = perf_ex_lib.ReadFile(options.input_file)
|
||
|
|
elif options.tests_filter:
|
||
|
|
# Tests to update have been specified as a reg-ex filter.
|
||
|
|
tests_filter = options.tests_filter
|
||
|
|
|
||
|
|
# Get tests to update based on filter specified.
|
||
|
|
tests_to_update = GetTestsToUpdate(tests_filter, all_test_keys)
|
||
|
|
logging.info('Done obtaining matching tests.')
|
||
|
|
|
||
|
|
# Now, prepare tests for update.
|
||
|
|
PrepareTestsForUpdate(tests_to_update, all_tests, options.reva, options.revb)
|
||
|
|
|
||
|
|
# Finally, write modified tests back to perf_expectations file.
|
||
|
|
perf_ex_lib.WriteJson(perf_file, all_tests, all_test_keys,
|
||
|
|
calculate_sha1=False)
|
||
|
|
logging.info('Done writing tests for update to %s.', perf_file)
|
||
|
|
|
||
|
|
|
||
|
|
def _SetLogger(verbose):
|
||
|
|
log_level = _DEFAULT_LOG_LEVEL
|
||
|
|
if verbose:
|
||
|
|
log_level = logging.DEBUG
|
||
|
|
logging.basicConfig(level=log_level, format='%(message)s')
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
Main()
|