232 lines
7.6 KiB
Python
Executable File
232 lines
7.6 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""Updates the Chrome reference builds.
|
|
|
|
Usage:
|
|
$ cd /tmp
|
|
$ /path/to/update_reference_build.py -r <revision>
|
|
$ cd reference_builds/reference_builds
|
|
$ gcl change
|
|
$ gcl upload <change>
|
|
$ gcl commit <change>
|
|
"""
|
|
|
|
import errno
|
|
import logging
|
|
import optparse
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib
|
|
import urllib2
|
|
import zipfile
|
|
|
|
|
|
class BuildUpdater(object):
|
|
_PLATFORM_FILES_MAP = {
|
|
'Win': [
|
|
'chrome-win32.zip',
|
|
'chrome-win32-syms.zip',
|
|
'chrome-win32.test/_pyautolib.pyd',
|
|
'chrome-win32.test/pyautolib.py',
|
|
],
|
|
'Mac': [
|
|
'chrome-mac.zip',
|
|
'chrome-mac.test/_pyautolib.so',
|
|
'chrome-mac.test/pyautolib.py',
|
|
],
|
|
'Linux': [
|
|
'chrome-linux.zip',
|
|
],
|
|
'Linux_x64': [
|
|
'chrome-linux.zip',
|
|
],
|
|
}
|
|
|
|
_PLATFORM_DEST_MAP = {
|
|
'Linux': 'chrome_linux',
|
|
'Linux_x64': 'chrome_linux64',
|
|
'Win': 'chrome_win',
|
|
'Mac': 'chrome_mac',
|
|
}
|
|
|
|
def __init__(self, options):
|
|
self._platforms = options.platforms.split(',')
|
|
self._revision = int(options.revision)
|
|
|
|
@staticmethod
|
|
def _GetCmdStatusAndOutput(args, cwd=None, shell=False):
|
|
"""Executes a subprocess and returns its exit code and output.
|
|
|
|
Args:
|
|
args: A string or a sequence of program arguments.
|
|
cwd: If not None, the subprocess's current directory will be changed to
|
|
|cwd| before it's executed.
|
|
shell: Whether to execute args as a shell command.
|
|
|
|
Returns:
|
|
The tuple (exit code, output).
|
|
"""
|
|
logging.info(str(args) + ' ' + (cwd or ''))
|
|
p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, shell=shell)
|
|
stdout, stderr = p.communicate()
|
|
exit_code = p.returncode
|
|
if stderr:
|
|
logging.critical(stderr)
|
|
logging.info(stdout)
|
|
return (exit_code, stdout)
|
|
|
|
def _GetBuildUrl(self, platform, revision, filename):
|
|
URL_FMT = ('http://commondatastorage.googleapis.com/'
|
|
'chromium-browser-snapshots/%s/%s/%s')
|
|
return URL_FMT % (urllib.quote_plus(platform), revision, filename)
|
|
|
|
def _FindBuildRevision(self, platform, revision, filename):
|
|
MAX_REVISIONS_PER_BUILD = 100
|
|
for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD):
|
|
r = urllib2.Request(self._GetBuildUrl(platform, revision_guess, filename))
|
|
r.get_method = lambda: 'HEAD'
|
|
try:
|
|
response = urllib2.urlopen(r)
|
|
return revision_guess
|
|
except urllib2.HTTPError, err:
|
|
if err.code == 404:
|
|
time.sleep(.1)
|
|
continue
|
|
return None
|
|
|
|
def _DownloadBuilds(self):
|
|
for platform in self._platforms:
|
|
for f in BuildUpdater._PLATFORM_FILES_MAP[platform]:
|
|
output = os.path.join('dl', platform,
|
|
'%s_%s_%s' % (platform, self._revision, f))
|
|
if os.path.exists(output):
|
|
logging.info('%s alread exists, skipping download' % output)
|
|
continue
|
|
build_revision = self._FindBuildRevision(platform, self._revision, f)
|
|
if not build_revision:
|
|
logging.critical('Failed to find %s build for r%s\n' % (
|
|
platform, self._revision))
|
|
sys.exit(1)
|
|
dirname = os.path.dirname(output)
|
|
if dirname and not os.path.exists(dirname):
|
|
os.makedirs(dirname)
|
|
url = self._GetBuildUrl(platform, build_revision, f)
|
|
logging.info('Downloading %s, saving to %s' % (url, output))
|
|
r = urllib2.urlopen(url)
|
|
with file(output, 'wb') as f:
|
|
f.write(r.read())
|
|
|
|
def _FetchSvnRepos(self):
|
|
if not os.path.exists('reference_builds'):
|
|
os.makedirs('reference_builds')
|
|
BuildUpdater._GetCmdStatusAndOutput(
|
|
['gclient', 'config',
|
|
'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'],
|
|
'reference_builds')
|
|
BuildUpdater._GetCmdStatusAndOutput(
|
|
['gclient', 'sync'], 'reference_builds')
|
|
|
|
def _UnzipFile(self, dl_file, dest_dir):
|
|
if not zipfile.is_zipfile(dl_file):
|
|
return False
|
|
logging.info('Opening %s' % dl_file)
|
|
with zipfile.ZipFile(dl_file, 'r') as z:
|
|
for content in z.namelist():
|
|
dest = os.path.join(dest_dir, content[content.find('/')+1:])
|
|
if not os.path.basename(dest):
|
|
if not os.path.isdir(dest):
|
|
os.makedirs(dest)
|
|
continue
|
|
with z.open(content) as unzipped_content:
|
|
logging.info('Extracting %s to %s (%s)' % (content, dest, dl_file))
|
|
with file(dest, 'wb') as dest_file:
|
|
dest_file.write(unzipped_content.read())
|
|
permissions = z.getinfo(content).external_attr >> 16
|
|
if permissions:
|
|
os.chmod(dest, permissions)
|
|
return True
|
|
|
|
def _ClearDir(self, dir):
|
|
"""Clears all files in |dir| except for hidden files and folders."""
|
|
for root, dirs, files in os.walk(dir):
|
|
# Skip hidden files and folders (like .svn and .git).
|
|
files = [f for f in files if f[0] != '.']
|
|
dirs[:] = [d for d in dirs if d[0] != '.']
|
|
|
|
for f in files:
|
|
os.remove(os.path.join(root, f))
|
|
|
|
def _ExtractBuilds(self):
|
|
for platform in self._platforms:
|
|
if os.path.exists('tmp_unzip'):
|
|
os.path.unlink('tmp_unzip')
|
|
dest_dir = os.path.join('reference_builds', 'reference_builds',
|
|
BuildUpdater._PLATFORM_DEST_MAP[platform])
|
|
self._ClearDir(dest_dir)
|
|
for root, _, dl_files in os.walk(os.path.join('dl', platform)):
|
|
for dl_file in dl_files:
|
|
dl_file = os.path.join(root, dl_file)
|
|
if not self._UnzipFile(dl_file, dest_dir):
|
|
logging.info('Copying %s to %s' % (dl_file, dest_dir))
|
|
shutil.copy(dl_file, dest_dir)
|
|
|
|
def _SvnAddAndRemove(self):
|
|
svn_dir = os.path.join('reference_builds', 'reference_builds')
|
|
stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat'], svn_dir)[1]
|
|
for line in stat.splitlines():
|
|
action, filename = line.split(None, 1)
|
|
if action == '?':
|
|
BuildUpdater._GetCmdStatusAndOutput(
|
|
['svn', 'add', filename], svn_dir)
|
|
elif action == '!':
|
|
BuildUpdater._GetCmdStatusAndOutput(
|
|
['svn', 'delete', filename], svn_dir)
|
|
filepath = os.path.join(svn_dir, filename)
|
|
if not os.path.isdir(filepath) and os.access(filepath, os.X_OK):
|
|
BuildUpdater._GetCmdStatusAndOutput(
|
|
['svn', 'propset', 'svn:executable', 'true', filename], svn_dir)
|
|
|
|
def DownloadAndUpdateBuilds(self):
|
|
self._DownloadBuilds()
|
|
self._FetchSvnRepos()
|
|
self._ExtractBuilds()
|
|
self._SvnAddAndRemove()
|
|
|
|
|
|
def ParseOptions(argv):
|
|
parser = optparse.OptionParser()
|
|
usage = 'usage: %prog <options>'
|
|
parser.set_usage(usage)
|
|
parser.add_option('-r', dest='revision',
|
|
help='Revision to pickup')
|
|
parser.add_option('-p', dest='platforms',
|
|
default='Win,Mac,Linux,Linux_x64',
|
|
help='Comma separated list of platforms to download '
|
|
'(as defined by the chromium builders).')
|
|
(options, _) = parser.parse_args(argv)
|
|
if not options.revision:
|
|
logging.critical('Must specify -r\n')
|
|
sys.exit(1)
|
|
|
|
return options
|
|
|
|
|
|
def main(argv):
|
|
logging.getLogger().setLevel(logging.DEBUG)
|
|
options = ParseOptions(argv)
|
|
b = BuildUpdater(options)
|
|
b.DownloadAndUpdateBuilds()
|
|
logging.info('Successfully updated reference builds. Move to '
|
|
'reference_builds/reference_builds and make a change with gcl.')
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main(sys.argv))
|