177 lines
6.4 KiB
Python
177 lines
6.4 KiB
Python
|
#!/usr/bin/env python
|
||
|
|
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
|
||
|
|
# Use of this source code is governed by a BSD-style license that can be
|
||
|
|
# found in the LICENSE file.
|
||
|
|
|
||
|
|
"""SiteCompare component to handle bulk scrapes.
|
||
|
|
|
||
|
|
Invokes a list of browsers and sends them to a list of URLs,
|
||
|
|
saving the rendered results to a specified directory, then
|
||
|
|
performs comparison operations on the resulting bitmaps and
|
||
|
|
saves the results
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
# This line is necessary to work around a QEMU bug
|
||
|
|
import _imaging
|
||
|
|
|
||
|
|
import os # Functions for walking the directory tree
|
||
|
|
import types # Runtime type-checking
|
||
|
|
|
||
|
|
import command_line # command-line parsing
|
||
|
|
import drivers # Functions for driving keyboard/mouse/windows, OS-specific
|
||
|
|
import operators # Functions that, given two bitmaps as input, produce
|
||
|
|
# output depending on the performance of an operation
|
||
|
|
import scrapers # Functions that know how to capture a render from
|
||
|
|
# particular browsers
|
||
|
|
|
||
|
|
import commands.compare2 # compare one page in two versions of same browser
|
||
|
|
import commands.maskmaker # generate a mask based on repeated scrapes
|
||
|
|
import commands.measure # measure length of time a page takes to load
|
||
|
|
import commands.scrape # scrape a URL or series of URLs to a bitmap
|
||
|
|
|
||
|
|
# The timeload command is obsolete (too flaky); it may be reinstated
|
||
|
|
# later but for now it's been superceded by "measure"
|
||
|
|
# import commands.timeload # measure length of time a page takes to load
|
||
|
|
|
||
|
|
def Scrape(browsers, urls, window_size=(1024, 768),
|
||
|
|
window_pos=(0, 0), timeout=20, save_path=None, **kwargs):
|
||
|
|
"""Invoke one or more browsers over one or more URLs, scraping renders.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
browsers: browsers to invoke with optional version strings
|
||
|
|
urls: URLs to visit
|
||
|
|
window_size: size of the browser window to display
|
||
|
|
window_pos: location of browser window
|
||
|
|
timeout: time (in seconds) to wait for page to load
|
||
|
|
save_path: root of save path, automatically appended with browser and
|
||
|
|
version
|
||
|
|
kwargs: miscellaneous keyword args, passed to scraper
|
||
|
|
Returns:
|
||
|
|
None
|
||
|
|
|
||
|
|
@TODO(jhaas): more parameters, or perhaps an indefinite dictionary
|
||
|
|
parameter, for things like length of time to wait for timeout, speed
|
||
|
|
of mouse clicks, etc. Possibly on a per-browser, per-URL, or
|
||
|
|
per-browser-per-URL basis
|
||
|
|
"""
|
||
|
|
|
||
|
|
if type(browsers) in types.StringTypes: browsers = [browsers]
|
||
|
|
|
||
|
|
if save_path is None:
|
||
|
|
# default save path is "scrapes" off the current root
|
||
|
|
save_path = os.path.join(os.path.split(__file__)[0], "Scrapes")
|
||
|
|
|
||
|
|
for browser in browsers:
|
||
|
|
# Browsers should be tuples of (browser, version)
|
||
|
|
if type(browser) in types.StringTypes: browser = (browser, None)
|
||
|
|
scraper = scrapers.GetScraper(browser)
|
||
|
|
|
||
|
|
full_path = os.path.join(save_path, browser[0], scraper.version)
|
||
|
|
drivers.windowing.PreparePath(full_path)
|
||
|
|
|
||
|
|
scraper.Scrape(urls, full_path, window_size, window_pos, timeout, kwargs)
|
||
|
|
|
||
|
|
|
||
|
|
def Compare(base, compare, ops, root_path=None, out_path=None):
|
||
|
|
"""Compares a series of scrapes using a series of operators.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
base: (browser, version) tuple of version to consider the baseline
|
||
|
|
compare: (browser, version) tuple of version to compare to
|
||
|
|
ops: list of operators plus operator arguments
|
||
|
|
root_path: root of the scrapes
|
||
|
|
out_path: place to put any output from the operators
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
None
|
||
|
|
|
||
|
|
@TODO(jhaas): this method will likely change, to provide a robust and
|
||
|
|
well-defined way of chaining operators, applying operators conditionally,
|
||
|
|
and full-featured scripting of the operator chain. There also needs
|
||
|
|
to be better definition of the output; right now it's to stdout and
|
||
|
|
a log.txt file, with operator-dependent images saved for error output
|
||
|
|
"""
|
||
|
|
if root_path is None:
|
||
|
|
# default save path is "scrapes" off the current root
|
||
|
|
root_path = os.path.join(os.path.split(__file__)[0], "Scrapes")
|
||
|
|
|
||
|
|
if out_path is None:
|
||
|
|
out_path = os.path.join(os.path.split(__file__)[0], "Compares")
|
||
|
|
|
||
|
|
if type(base) in types.StringTypes: base = (base, None)
|
||
|
|
if type(compare) in types.StringTypes: compare = (compare, None)
|
||
|
|
if type(ops) in types.StringTypes: ops = [ops]
|
||
|
|
|
||
|
|
base_dir = os.path.join(root_path, base[0])
|
||
|
|
compare_dir = os.path.join(root_path, compare[0])
|
||
|
|
|
||
|
|
if base[1] is None:
|
||
|
|
# base defaults to earliest capture
|
||
|
|
base = (base[0], max(os.listdir(base_dir)))
|
||
|
|
|
||
|
|
if compare[1] is None:
|
||
|
|
# compare defaults to latest capture
|
||
|
|
compare = (compare[0], min(os.listdir(compare_dir)))
|
||
|
|
|
||
|
|
out_path = os.path.join(out_path, base[0], base[1], compare[0], compare[1])
|
||
|
|
drivers.windowing.PreparePath(out_path)
|
||
|
|
|
||
|
|
# TODO(jhaas): right now we're just dumping output to a log file
|
||
|
|
# (and the console), which works as far as it goes but isn't nearly
|
||
|
|
# robust enough. Change this after deciding exactly what we want to
|
||
|
|
# change it to.
|
||
|
|
out_file = open(os.path.join(out_path, "log.txt"), "w")
|
||
|
|
description_string = ("Comparing %s %s to %s %s" %
|
||
|
|
(base[0], base[1], compare[0], compare[1]))
|
||
|
|
out_file.write(description_string)
|
||
|
|
print description_string
|
||
|
|
|
||
|
|
base_dir = os.path.join(base_dir, base[1])
|
||
|
|
compare_dir = os.path.join(compare_dir, compare[1])
|
||
|
|
|
||
|
|
for filename in os.listdir(base_dir):
|
||
|
|
out_file.write("%s: " % filename)
|
||
|
|
|
||
|
|
if not os.path.isfile(os.path.join(compare_dir, filename)):
|
||
|
|
out_file.write("Does not exist in target directory\n")
|
||
|
|
print "File %s does not exist in target directory" % filename
|
||
|
|
continue
|
||
|
|
|
||
|
|
base_filename = os.path.join(base_dir, filename)
|
||
|
|
compare_filename = os.path.join(compare_dir, filename)
|
||
|
|
|
||
|
|
for op in ops:
|
||
|
|
if type(op) in types.StringTypes: op = (op, None)
|
||
|
|
|
||
|
|
module = operators.GetOperator(op[0])
|
||
|
|
|
||
|
|
ret = module.Compare(base_filename, compare_filename)
|
||
|
|
if ret is None:
|
||
|
|
print "%s: OK" % (filename,)
|
||
|
|
out_file.write("OK\n")
|
||
|
|
else:
|
||
|
|
print "%s: %s" % (filename, ret[0])
|
||
|
|
out_file.write("%s\n" % (ret[0]))
|
||
|
|
ret[1].save(os.path.join(out_path, filename))
|
||
|
|
|
||
|
|
out_file.close()
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
"""Main executable. Parse the command line and invoke the command."""
|
||
|
|
cmdline = command_line.CommandLine()
|
||
|
|
|
||
|
|
# The below two commands are currently unstable so have been disabled
|
||
|
|
# commands.compare2.CreateCommand(cmdline)
|
||
|
|
# commands.maskmaker.CreateCommand(cmdline)
|
||
|
|
commands.measure.CreateCommand(cmdline)
|
||
|
|
commands.scrape.CreateCommand(cmdline)
|
||
|
|
|
||
|
|
cmdline.ParseCommandLine()
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
sys.exit(main())
|