478 lines
16 KiB
Python
478 lines
16 KiB
Python
|
#!/usr/bin/env python
|
||
|
|
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
|
||
|
|
# Use of this source code is governed by a BSD-style license that can be
|
||
|
|
# found in the LICENSE file.
|
||
|
|
|
||
|
|
"""Utility for checking and processing licensing information in third_party
|
||
|
|
directories.
|
||
|
|
|
||
|
|
Usage: licenses.py <command>
|
||
|
|
|
||
|
|
Commands:
|
||
|
|
scan scan third_party directories, verifying that we have licensing info
|
||
|
|
credits generate about:credits on stdout
|
||
|
|
|
||
|
|
(You can also import this as a module.)
|
||
|
|
"""
|
||
|
|
|
||
|
|
import cgi
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
|
||
|
|
# Paths from the root of the tree to directories to skip.
|
||
|
|
PRUNE_PATHS = set([
|
||
|
|
# Same module occurs in crypto/third_party/nss and net/third_party/nss, so
|
||
|
|
# skip this one.
|
||
|
|
os.path.join('third_party','nss'),
|
||
|
|
|
||
|
|
# Placeholder directory only, not third-party code.
|
||
|
|
os.path.join('third_party','adobe'),
|
||
|
|
|
||
|
|
# Build files only, not third-party code.
|
||
|
|
os.path.join('third_party','widevine'),
|
||
|
|
|
||
|
|
# Only binaries, used during development.
|
||
|
|
os.path.join('third_party','valgrind'),
|
||
|
|
|
||
|
|
# Used for development and test, not in the shipping product.
|
||
|
|
os.path.join('third_party','bison'),
|
||
|
|
os.path.join('third_party','cygwin'),
|
||
|
|
os.path.join('third_party','gnu_binutils'),
|
||
|
|
os.path.join('third_party','gold'),
|
||
|
|
os.path.join('third_party','gperf'),
|
||
|
|
os.path.join('third_party','lighttpd'),
|
||
|
|
os.path.join('third_party','llvm'),
|
||
|
|
os.path.join('third_party','llvm-build'),
|
||
|
|
os.path.join('third_party','mingw-w64'),
|
||
|
|
os.path.join('third_party','nacl_sdk_binaries'),
|
||
|
|
os.path.join('third_party','pefile'),
|
||
|
|
os.path.join('third_party','perl'),
|
||
|
|
os.path.join('third_party','psyco_win32'),
|
||
|
|
os.path.join('third_party','pylib'),
|
||
|
|
os.path.join('third_party','python_26'),
|
||
|
|
os.path.join('third_party','pywebsocket'),
|
||
|
|
os.path.join('third_party','syzygy'),
|
||
|
|
os.path.join('tools','gn'),
|
||
|
|
|
||
|
|
# Chromium code in third_party.
|
||
|
|
os.path.join('third_party','fuzzymatch'),
|
||
|
|
|
||
|
|
# Stuff pulled in from chrome-internal for official builds/tools.
|
||
|
|
os.path.join('third_party', 'clear_cache'),
|
||
|
|
os.path.join('third_party', 'gnu'),
|
||
|
|
os.path.join('third_party', 'googlemac'),
|
||
|
|
os.path.join('third_party', 'pcre'),
|
||
|
|
os.path.join('third_party', 'psutils'),
|
||
|
|
os.path.join('third_party', 'sawbuck'),
|
||
|
|
|
||
|
|
# Redistribution does not require attribution in documentation.
|
||
|
|
os.path.join('third_party','directxsdk'),
|
||
|
|
os.path.join('third_party','platformsdk_win2008_6_1'),
|
||
|
|
os.path.join('third_party','platformsdk_win7'),
|
||
|
|
])
|
||
|
|
|
||
|
|
# Directories we don't scan through.
|
||
|
|
VCS_METADATA_DIRS = ('.svn', '.git')
|
||
|
|
PRUNE_DIRS = (VCS_METADATA_DIRS +
|
||
|
|
('out', 'Debug', 'Release', # build files
|
||
|
|
'layout_tests')) # lots of subdirs
|
||
|
|
|
||
|
|
ADDITIONAL_PATHS = (
|
||
|
|
os.path.join('breakpad'),
|
||
|
|
os.path.join('chrome', 'common', 'extensions', 'docs', 'examples'),
|
||
|
|
os.path.join('chrome', 'test', 'chromeos', 'autotest'),
|
||
|
|
os.path.join('chrome', 'test', 'data'),
|
||
|
|
os.path.join('native_client'),
|
||
|
|
os.path.join('native_client_sdk'),
|
||
|
|
os.path.join('net', 'tools', 'spdyshark'),
|
||
|
|
os.path.join('ppapi'),
|
||
|
|
os.path.join('sandbox', 'linux', 'seccomp-legacy'),
|
||
|
|
os.path.join('sdch', 'open-vcdiff'),
|
||
|
|
os.path.join('testing', 'gmock'),
|
||
|
|
os.path.join('testing', 'gtest'),
|
||
|
|
# The directory with the word list for Chinese and Japanese segmentation
|
||
|
|
# with different license terms than ICU.
|
||
|
|
os.path.join('third_party','icu','source','data','brkitr'),
|
||
|
|
os.path.join('tools', 'grit'),
|
||
|
|
os.path.join('tools', 'gyp'),
|
||
|
|
os.path.join('tools', 'page_cycler', 'acid3'),
|
||
|
|
os.path.join('url', 'third_party', 'mozilla'),
|
||
|
|
os.path.join('v8'),
|
||
|
|
# Fake directory so we can include the strongtalk license.
|
||
|
|
os.path.join('v8', 'strongtalk'),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
# Directories where we check out directly from upstream, and therefore
|
||
|
|
# can't provide a README.chromium. Please prefer a README.chromium
|
||
|
|
# wherever possible.
|
||
|
|
SPECIAL_CASES = {
|
||
|
|
os.path.join('native_client'): {
|
||
|
|
"Name": "native client",
|
||
|
|
"URL": "http://code.google.com/p/nativeclient",
|
||
|
|
"License": "BSD",
|
||
|
|
},
|
||
|
|
os.path.join('sandbox', 'linux', 'seccomp-legacy'): {
|
||
|
|
"Name": "seccompsandbox",
|
||
|
|
"URL": "http://code.google.com/p/seccompsandbox",
|
||
|
|
"License": "BSD",
|
||
|
|
},
|
||
|
|
os.path.join('sdch', 'open-vcdiff'): {
|
||
|
|
"Name": "open-vcdiff",
|
||
|
|
"URL": "http://code.google.com/p/open-vcdiff",
|
||
|
|
"License": "Apache 2.0, MIT, GPL v2 and custom licenses",
|
||
|
|
"License Android Compatible": "yes",
|
||
|
|
},
|
||
|
|
os.path.join('testing', 'gmock'): {
|
||
|
|
"Name": "gmock",
|
||
|
|
"URL": "http://code.google.com/p/googlemock",
|
||
|
|
"License": "BSD",
|
||
|
|
"License File": "NOT_SHIPPED",
|
||
|
|
},
|
||
|
|
os.path.join('testing', 'gtest'): {
|
||
|
|
"Name": "gtest",
|
||
|
|
"URL": "http://code.google.com/p/googletest",
|
||
|
|
"License": "BSD",
|
||
|
|
"License File": "NOT_SHIPPED",
|
||
|
|
},
|
||
|
|
os.path.join('third_party', 'angle'): {
|
||
|
|
"Name": "Almost Native Graphics Layer Engine",
|
||
|
|
"URL": "http://code.google.com/p/angleproject/",
|
||
|
|
"License": "BSD",
|
||
|
|
},
|
||
|
|
os.path.join('third_party', 'cros_system_api'): {
|
||
|
|
"Name": "Chromium OS system API",
|
||
|
|
"URL": "http://www.chromium.org/chromium-os",
|
||
|
|
"License": "BSD",
|
||
|
|
# Absolute path here is resolved as relative to the source root.
|
||
|
|
"License File": "/LICENSE.chromium_os",
|
||
|
|
},
|
||
|
|
os.path.join('third_party', 'GTM'): {
|
||
|
|
"Name": "Google Toolbox for Mac",
|
||
|
|
"URL": "http://code.google.com/p/google-toolbox-for-mac/",
|
||
|
|
"License": "Apache 2.0",
|
||
|
|
"License File": "COPYING",
|
||
|
|
},
|
||
|
|
os.path.join('third_party', 'lss'): {
|
||
|
|
"Name": "linux-syscall-support",
|
||
|
|
"URL": "http://code.google.com/p/linux-syscall-support/",
|
||
|
|
"License": "BSD",
|
||
|
|
"License File": "/LICENSE",
|
||
|
|
},
|
||
|
|
os.path.join('third_party', 'ots'): {
|
||
|
|
"Name": "OTS (OpenType Sanitizer)",
|
||
|
|
"URL": "http://code.google.com/p/ots/",
|
||
|
|
"License": "BSD",
|
||
|
|
},
|
||
|
|
os.path.join('third_party', 'pdfsqueeze'): {
|
||
|
|
"Name": "pdfsqueeze",
|
||
|
|
"URL": "http://code.google.com/p/pdfsqueeze/",
|
||
|
|
"License": "Apache 2.0",
|
||
|
|
"License File": "COPYING",
|
||
|
|
},
|
||
|
|
os.path.join('third_party', 'ppapi'): {
|
||
|
|
"Name": "ppapi",
|
||
|
|
"URL": "http://code.google.com/p/ppapi/",
|
||
|
|
},
|
||
|
|
os.path.join('third_party', 'scons-2.0.1'): {
|
||
|
|
"Name": "scons-2.0.1",
|
||
|
|
"URL": "http://www.scons.org",
|
||
|
|
"License": "MIT",
|
||
|
|
"License File": "NOT_SHIPPED",
|
||
|
|
},
|
||
|
|
os.path.join('third_party', 'trace-viewer'): {
|
||
|
|
"Name": "trace-viewer",
|
||
|
|
"URL": "http://code.google.com/p/trace-viewer",
|
||
|
|
"License": "BSD",
|
||
|
|
"License File": "NOT_SHIPPED",
|
||
|
|
},
|
||
|
|
os.path.join('third_party', 'v8-i18n'): {
|
||
|
|
"Name": "Internationalization Library for v8",
|
||
|
|
"URL": "http://code.google.com/p/v8-i18n/",
|
||
|
|
"License": "Apache 2.0",
|
||
|
|
},
|
||
|
|
os.path.join('third_party', 'WebKit'): {
|
||
|
|
"Name": "WebKit",
|
||
|
|
"URL": "http://webkit.org/",
|
||
|
|
"License": "BSD and GPL v2",
|
||
|
|
# Absolute path here is resolved as relative to the source root.
|
||
|
|
"License File": "/webkit/LICENSE",
|
||
|
|
},
|
||
|
|
os.path.join('third_party', 'webpagereplay'): {
|
||
|
|
"Name": "webpagereplay",
|
||
|
|
"URL": "http://code.google.com/p/web-page-replay",
|
||
|
|
"License": "Apache 2.0",
|
||
|
|
"License File": "NOT_SHIPPED",
|
||
|
|
},
|
||
|
|
os.path.join('tools', 'grit'): {
|
||
|
|
"Name": "grit",
|
||
|
|
"URL": "http://code.google.com/p/grit-i18n",
|
||
|
|
"License": "BSD",
|
||
|
|
"License File": "NOT_SHIPPED",
|
||
|
|
},
|
||
|
|
os.path.join('tools', 'gyp'): {
|
||
|
|
"Name": "gyp",
|
||
|
|
"URL": "http://code.google.com/p/gyp",
|
||
|
|
"License": "BSD",
|
||
|
|
"License File": "NOT_SHIPPED",
|
||
|
|
},
|
||
|
|
os.path.join('v8'): {
|
||
|
|
"Name": "V8 JavaScript Engine",
|
||
|
|
"URL": "http://code.google.com/p/v8",
|
||
|
|
"License": "BSD",
|
||
|
|
},
|
||
|
|
os.path.join('v8', 'strongtalk'): {
|
||
|
|
"Name": "Strongtalk",
|
||
|
|
"URL": "http://www.strongtalk.org/",
|
||
|
|
"License": "BSD",
|
||
|
|
# Absolute path here is resolved as relative to the source root.
|
||
|
|
"License File": "/v8/LICENSE.strongtalk",
|
||
|
|
},
|
||
|
|
}
|
||
|
|
|
||
|
|
# Special value for 'License File' field used to indicate that the license file
|
||
|
|
# should not be used in about:credits.
|
||
|
|
NOT_SHIPPED = "NOT_SHIPPED"
|
||
|
|
|
||
|
|
|
||
|
|
class LicenseError(Exception):
|
||
|
|
"""We raise this exception when a directory's licensing info isn't
|
||
|
|
fully filled out."""
|
||
|
|
pass
|
||
|
|
|
||
|
|
def AbsolutePath(path, filename, root):
|
||
|
|
"""Convert a path in README.chromium to be absolute based on the source
|
||
|
|
root."""
|
||
|
|
if filename.startswith('/'):
|
||
|
|
# Absolute-looking paths are relative to the source root
|
||
|
|
# (which is the directory we're run from).
|
||
|
|
absolute_path = os.path.join(root, filename[1:])
|
||
|
|
else:
|
||
|
|
absolute_path = os.path.join(root, path, filename)
|
||
|
|
if os.path.exists(absolute_path):
|
||
|
|
return absolute_path
|
||
|
|
return None
|
||
|
|
|
||
|
|
def ParseDir(path, root, require_license_file=True):
|
||
|
|
"""Examine a third_party/foo component and extract its metadata."""
|
||
|
|
|
||
|
|
# Parse metadata fields out of README.chromium.
|
||
|
|
# We examine "LICENSE" for the license file by default.
|
||
|
|
metadata = {
|
||
|
|
"License File": "LICENSE", # Relative path to license text.
|
||
|
|
"Name": None, # Short name (for header on about:credits).
|
||
|
|
"URL": None, # Project home page.
|
||
|
|
"License": None, # Software license.
|
||
|
|
}
|
||
|
|
|
||
|
|
# Relative path to a file containing some html we're required to place in
|
||
|
|
# about:credits.
|
||
|
|
optional_keys = ["Required Text", "License Android Compatible"]
|
||
|
|
|
||
|
|
if path in SPECIAL_CASES:
|
||
|
|
metadata.update(SPECIAL_CASES[path])
|
||
|
|
else:
|
||
|
|
# Try to find README.chromium.
|
||
|
|
readme_path = os.path.join(root, path, 'README.chromium')
|
||
|
|
if not os.path.exists(readme_path):
|
||
|
|
raise LicenseError("missing README.chromium or licenses.py "
|
||
|
|
"SPECIAL_CASES entry")
|
||
|
|
|
||
|
|
for line in open(readme_path):
|
||
|
|
line = line.strip()
|
||
|
|
if not line:
|
||
|
|
break
|
||
|
|
for key in metadata.keys() + optional_keys:
|
||
|
|
field = key + ": "
|
||
|
|
if line.startswith(field):
|
||
|
|
metadata[key] = line[len(field):]
|
||
|
|
|
||
|
|
# Check that all expected metadata is present.
|
||
|
|
for key, value in metadata.iteritems():
|
||
|
|
if not value:
|
||
|
|
raise LicenseError("couldn't find '" + key + "' line "
|
||
|
|
"in README.chromium or licences.py "
|
||
|
|
"SPECIAL_CASES")
|
||
|
|
|
||
|
|
# Special-case modules that aren't in the shipping product, so don't need
|
||
|
|
# their license in about:credits.
|
||
|
|
if metadata["License File"] != NOT_SHIPPED:
|
||
|
|
# Check that the license file exists.
|
||
|
|
for filename in (metadata["License File"], "COPYING"):
|
||
|
|
license_path = AbsolutePath(path, filename, root)
|
||
|
|
if license_path is not None:
|
||
|
|
break
|
||
|
|
|
||
|
|
if require_license_file and not license_path:
|
||
|
|
raise LicenseError("License file not found. "
|
||
|
|
"Either add a file named LICENSE, "
|
||
|
|
"import upstream's COPYING if available, "
|
||
|
|
"or add a 'License File:' line to "
|
||
|
|
"README.chromium with the appropriate path.")
|
||
|
|
metadata["License File"] = license_path
|
||
|
|
|
||
|
|
if "Required Text" in metadata:
|
||
|
|
required_path = AbsolutePath(path, metadata["Required Text"], root)
|
||
|
|
if required_path is not None:
|
||
|
|
metadata["Required Text"] = required_path
|
||
|
|
else:
|
||
|
|
raise LicenseError("Required text file listed but not found.")
|
||
|
|
|
||
|
|
return metadata
|
||
|
|
|
||
|
|
|
||
|
|
def ContainsFiles(path, root):
|
||
|
|
"""Determines whether any files exist in a directory or in any of its
|
||
|
|
subdirectories."""
|
||
|
|
for _, dirs, files in os.walk(os.path.join(root, path)):
|
||
|
|
if files:
|
||
|
|
return True
|
||
|
|
for vcs_metadata in VCS_METADATA_DIRS:
|
||
|
|
if vcs_metadata in dirs:
|
||
|
|
dirs.remove(vcs_metadata)
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
def FilterDirsWithFiles(dirs_list, root):
|
||
|
|
# If a directory contains no files, assume it's a DEPS directory for a
|
||
|
|
# project not used by our current configuration and skip it.
|
||
|
|
return [x for x in dirs_list if ContainsFiles(x, root)]
|
||
|
|
|
||
|
|
|
||
|
|
def FindThirdPartyDirs(prune_paths, root):
|
||
|
|
"""Find all third_party directories underneath the source root."""
|
||
|
|
third_party_dirs = []
|
||
|
|
for path, dirs, files in os.walk(root):
|
||
|
|
path = path[len(root)+1:] # Pretty up the path.
|
||
|
|
|
||
|
|
if path in prune_paths:
|
||
|
|
dirs[:] = []
|
||
|
|
continue
|
||
|
|
|
||
|
|
# Prune out directories we want to skip.
|
||
|
|
# (Note that we loop over PRUNE_DIRS so we're not iterating over a
|
||
|
|
# list that we're simultaneously mutating.)
|
||
|
|
for skip in PRUNE_DIRS:
|
||
|
|
if skip in dirs:
|
||
|
|
dirs.remove(skip)
|
||
|
|
|
||
|
|
if os.path.basename(path) == 'third_party':
|
||
|
|
# Add all subdirectories that are not marked for skipping.
|
||
|
|
for dir in dirs:
|
||
|
|
dirpath = os.path.join(path, dir)
|
||
|
|
if dirpath not in prune_paths:
|
||
|
|
third_party_dirs.append(dirpath)
|
||
|
|
|
||
|
|
# Don't recurse into any subdirs from here.
|
||
|
|
dirs[:] = []
|
||
|
|
continue
|
||
|
|
|
||
|
|
# Don't recurse into paths in ADDITIONAL_PATHS, like we do with regular
|
||
|
|
# third_party/foo paths.
|
||
|
|
if path in ADDITIONAL_PATHS:
|
||
|
|
dirs[:] = []
|
||
|
|
|
||
|
|
for dir in ADDITIONAL_PATHS:
|
||
|
|
if dir not in prune_paths:
|
||
|
|
third_party_dirs.append(dir)
|
||
|
|
|
||
|
|
return third_party_dirs
|
||
|
|
|
||
|
|
|
||
|
|
def ScanThirdPartyDirs(root=None):
|
||
|
|
"""Scan a list of directories and report on any problems we find."""
|
||
|
|
if root is None:
|
||
|
|
root = os.getcwd()
|
||
|
|
third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS, root)
|
||
|
|
third_party_dirs = FilterDirsWithFiles(third_party_dirs, root)
|
||
|
|
|
||
|
|
errors = []
|
||
|
|
for path in sorted(third_party_dirs):
|
||
|
|
try:
|
||
|
|
metadata = ParseDir(path, root)
|
||
|
|
except LicenseError, e:
|
||
|
|
errors.append((path, e.args[0]))
|
||
|
|
continue
|
||
|
|
|
||
|
|
for path, error in sorted(errors):
|
||
|
|
print path + ": " + error
|
||
|
|
|
||
|
|
return len(errors) == 0
|
||
|
|
|
||
|
|
|
||
|
|
def GenerateCredits():
|
||
|
|
"""Generate about:credits."""
|
||
|
|
|
||
|
|
if len(sys.argv) not in (2, 3):
|
||
|
|
print 'usage: licenses.py credits [output_file]'
|
||
|
|
return False
|
||
|
|
|
||
|
|
def EvaluateTemplate(template, env, escape=True):
|
||
|
|
"""Expand a template with variables like {{foo}} using a
|
||
|
|
dictionary of expansions."""
|
||
|
|
for key, val in env.items():
|
||
|
|
if escape and not key.endswith("_unescaped"):
|
||
|
|
val = cgi.escape(val)
|
||
|
|
template = template.replace('{{%s}}' % key, val)
|
||
|
|
return template
|
||
|
|
|
||
|
|
root = os.path.join(os.path.dirname(__file__), '..')
|
||
|
|
third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS, root)
|
||
|
|
|
||
|
|
entry_template = open(os.path.join(root, 'chrome', 'browser', 'resources',
|
||
|
|
'about_credits_entry.tmpl'), 'rb').read()
|
||
|
|
entries = []
|
||
|
|
for path in sorted(third_party_dirs):
|
||
|
|
try:
|
||
|
|
metadata = ParseDir(path, root)
|
||
|
|
except LicenseError:
|
||
|
|
# TODO(phajdan.jr): Convert to fatal error (http://crbug.com/39240).
|
||
|
|
continue
|
||
|
|
if metadata['License File'] == NOT_SHIPPED:
|
||
|
|
continue
|
||
|
|
env = {
|
||
|
|
'name': metadata['Name'],
|
||
|
|
'url': metadata['URL'],
|
||
|
|
'license': open(metadata['License File'], 'rb').read(),
|
||
|
|
'license_unescaped': '',
|
||
|
|
}
|
||
|
|
if 'Required Text' in metadata:
|
||
|
|
required_text = open(metadata['Required Text'], 'rb').read()
|
||
|
|
env["license_unescaped"] = required_text
|
||
|
|
entries.append(EvaluateTemplate(entry_template, env))
|
||
|
|
|
||
|
|
file_template = open(os.path.join(root, 'chrome', 'browser', 'resources',
|
||
|
|
'about_credits.tmpl'), 'rb').read()
|
||
|
|
template_contents = "<!-- Generated by licenses.py; do not edit. -->"
|
||
|
|
template_contents += EvaluateTemplate(file_template,
|
||
|
|
{'entries': '\n'.join(entries)},
|
||
|
|
escape=False)
|
||
|
|
|
||
|
|
if len(sys.argv) == 3:
|
||
|
|
with open(sys.argv[2], 'w') as output_file:
|
||
|
|
output_file.write(template_contents)
|
||
|
|
elif len(sys.argv) == 2:
|
||
|
|
print template_contents
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
command = 'help'
|
||
|
|
if len(sys.argv) > 1:
|
||
|
|
command = sys.argv[1]
|
||
|
|
|
||
|
|
if command == 'scan':
|
||
|
|
if not ScanThirdPartyDirs():
|
||
|
|
return 1
|
||
|
|
elif command == 'credits':
|
||
|
|
if not GenerateCredits():
|
||
|
|
return 1
|
||
|
|
else:
|
||
|
|
print __doc__
|
||
|
|
return 1
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
sys.exit(main())
|