421 lines
14 KiB
Python
421 lines
14 KiB
Python
|
# Copyright 2013 The Chromium Authors. All rights reserved.
|
||
|
|
# Use of this source code is governed by a BSD-style license that can be
|
||
|
|
# found in the LICENSE file.
|
||
|
|
|
||
|
|
"""Takes the same arguments as Windows link.exe, and a definition of libraries
|
||
|
|
to split into subcomponents. Does multiple passes of link.exe invocation to
|
||
|
|
determine exports between parts and generates .def and import libraries to
|
||
|
|
cause symbols to be available to other parts."""
|
||
|
|
|
||
|
|
import _winreg
|
||
|
|
import ctypes
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import shutil
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
import tempfile
|
||
|
|
|
||
|
|
|
||
|
|
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||
|
|
|
||
|
|
|
||
|
|
# This can be set to ignore data exports. The resulting DLLs will probably not
|
||
|
|
# run, but at least they can be generated. The log of data exports will still
|
||
|
|
# be output.
|
||
|
|
IGNORE_DATA = 0
|
||
|
|
|
||
|
|
|
||
|
|
def Log(message):
|
||
|
|
print 'split_link:', message
|
||
|
|
|
||
|
|
|
||
|
|
def GetFlagsAndInputs(argv):
|
||
|
|
"""Parses the command line intended for link.exe and return the flags and
|
||
|
|
input files."""
|
||
|
|
rsp_expanded = []
|
||
|
|
for arg in argv:
|
||
|
|
if arg[0] == '@':
|
||
|
|
with open(arg[1:]) as rsp:
|
||
|
|
rsp_expanded.extend(rsp.read().splitlines())
|
||
|
|
else:
|
||
|
|
rsp_expanded.append(arg)
|
||
|
|
|
||
|
|
# Use CommandLineToArgvW so we match link.exe parsing.
|
||
|
|
try:
|
||
|
|
size = ctypes.c_int()
|
||
|
|
ptr = ctypes.windll.shell32.CommandLineToArgvW(
|
||
|
|
ctypes.create_unicode_buffer(' '.join(rsp_expanded)),
|
||
|
|
ctypes.byref(size))
|
||
|
|
ref = ctypes.c_wchar_p * size.value
|
||
|
|
raw = ref.from_address(ptr)
|
||
|
|
args = [arg for arg in raw]
|
||
|
|
finally:
|
||
|
|
ctypes.windll.kernel32.LocalFree(ptr)
|
||
|
|
|
||
|
|
inputs = []
|
||
|
|
flags = []
|
||
|
|
intermediate_manifest = ''
|
||
|
|
for arg in args:
|
||
|
|
lower_arg = arg.lower()
|
||
|
|
# We'll be replacing these ourselves.
|
||
|
|
if lower_arg.startswith('/out:'):
|
||
|
|
continue
|
||
|
|
if lower_arg.startswith('/manifestfile:'):
|
||
|
|
intermediate_manifest = arg[arg.index(':')+1:]
|
||
|
|
continue
|
||
|
|
if lower_arg.startswith('/pdb:'):
|
||
|
|
continue
|
||
|
|
if (not lower_arg.startswith('/') and
|
||
|
|
lower_arg.endswith(('.obj', '.lib', '.res'))):
|
||
|
|
inputs.append(arg)
|
||
|
|
else:
|
||
|
|
flags.append(arg)
|
||
|
|
|
||
|
|
return flags, inputs, intermediate_manifest
|
||
|
|
|
||
|
|
|
||
|
|
def GetRegistryValue(subkey):
|
||
|
|
try:
|
||
|
|
val = _winreg.QueryValue(_winreg.HKEY_CURRENT_USER,
|
||
|
|
'Software\\Chromium\\' + subkey)
|
||
|
|
if os.path.exists(val):
|
||
|
|
return val
|
||
|
|
except WindowsError:
|
||
|
|
pass
|
||
|
|
|
||
|
|
raise SystemExit("Couldn't read from registry")
|
||
|
|
|
||
|
|
|
||
|
|
def GetOriginalLinkerPath():
|
||
|
|
return GetRegistryValue('split_link_installed')
|
||
|
|
|
||
|
|
|
||
|
|
def GetMtPath():
|
||
|
|
return GetRegistryValue('split_link_mt_path')
|
||
|
|
|
||
|
|
|
||
|
|
def PartFor(input_file, description_parts, description_all):
|
||
|
|
"""Determines which part a given link input should be put into (or all)."""
|
||
|
|
# Check if it should go in all parts.
|
||
|
|
input_file = input_file.lower()
|
||
|
|
if any(re.search(spec, input_file) for spec in description_all):
|
||
|
|
return -1
|
||
|
|
# Or pick which particular one it belongs in.
|
||
|
|
for i, spec_list in enumerate(description_parts):
|
||
|
|
if any(re.search(spec, input_file) for spec in spec_list):
|
||
|
|
return i
|
||
|
|
raise ValueError("couldn't find location for %s" % input_file)
|
||
|
|
|
||
|
|
|
||
|
|
def ParseOutExternals(output):
|
||
|
|
"""Given the stdout of link.exe, parses the error messages to retrieve all
|
||
|
|
symbols that are unresolved."""
|
||
|
|
result = set()
|
||
|
|
# Styles of messages for unresolved externals, and a boolean to indicate
|
||
|
|
# whether the error message emits the symbols with or without a leading
|
||
|
|
# underscore.
|
||
|
|
unresolved_regexes = [
|
||
|
|
(re.compile(r' : error LNK2019: unresolved external symbol ".*" \((.*)\)'
|
||
|
|
r' referenced in function'),
|
||
|
|
False),
|
||
|
|
(re.compile(r' : error LNK2001: unresolved external symbol ".*" \((.*)\)$'),
|
||
|
|
False),
|
||
|
|
(re.compile(r' : error LNK2019: unresolved external symbol (.*)'
|
||
|
|
r' referenced in function '),
|
||
|
|
True),
|
||
|
|
(re.compile(r' : error LNK2001: unresolved external symbol (.*)$'),
|
||
|
|
True),
|
||
|
|
]
|
||
|
|
for line in output.splitlines():
|
||
|
|
line = line.strip()
|
||
|
|
for regex, strip_leading_underscore in unresolved_regexes:
|
||
|
|
mo = regex.search(line)
|
||
|
|
if mo:
|
||
|
|
if strip_leading_underscore:
|
||
|
|
result.add(mo.group(1)[1:])
|
||
|
|
else:
|
||
|
|
result.add(mo.group(1))
|
||
|
|
break
|
||
|
|
|
||
|
|
mo = re.search(r'fatal error LNK1120: (\d+) unresolved externals', output)
|
||
|
|
# Make sure we have the same number that the linker thinks we have.
|
||
|
|
if mo is None and result:
|
||
|
|
raise SystemExit(output)
|
||
|
|
if len(result) != int(mo.group(1)):
|
||
|
|
print output
|
||
|
|
print 'Expecting %d, got %d' % (int(mo.group(1)), len(result))
|
||
|
|
assert len(result) == int(mo.group(1))
|
||
|
|
return sorted(result)
|
||
|
|
|
||
|
|
|
||
|
|
def AsCommandLineArgs(items):
|
||
|
|
"""Intended for output to a response file. Quotes all arguments."""
|
||
|
|
return '\n'.join('"' + x + '"' for x in items)
|
||
|
|
|
||
|
|
|
||
|
|
def OutputNameForIndex(index):
|
||
|
|
"""Gets the final output DLL name, given a zero-based index."""
|
||
|
|
if index == 0:
|
||
|
|
return "chrome.dll"
|
||
|
|
else:
|
||
|
|
return 'chrome%d.dll' % index
|
||
|
|
|
||
|
|
|
||
|
|
def ManifestNameForIndex(index):
|
||
|
|
return OutputNameForIndex(index) + '.intermediate.manifest'
|
||
|
|
|
||
|
|
|
||
|
|
def PdbNameForIndex(index):
|
||
|
|
return OutputNameForIndex(index) + '.pdb'
|
||
|
|
|
||
|
|
|
||
|
|
def RunLinker(flags, index, inputs, phase, intermediate_manifest):
|
||
|
|
"""Invokes the linker and returns the stdout, returncode and target name."""
|
||
|
|
rspfile = 'part%d_%s.rsp' % (index, phase)
|
||
|
|
with open(rspfile, 'w') as f:
|
||
|
|
print >> f, AsCommandLineArgs(inputs)
|
||
|
|
print >> f, AsCommandLineArgs(flags)
|
||
|
|
output_name = OutputNameForIndex(index)
|
||
|
|
manifest_name = ManifestNameForIndex(index)
|
||
|
|
print >> f, '/ENTRY:ChromeEmptyEntry@12'
|
||
|
|
print >> f, '/OUT:' + output_name
|
||
|
|
print >> f, '/MANIFESTFILE:' + manifest_name
|
||
|
|
print >> f, '/PDB:' + PdbNameForIndex(index)
|
||
|
|
# Log('[[[\n' + open(rspfile).read() + '\n]]]')
|
||
|
|
link_exe = GetOriginalLinkerPath()
|
||
|
|
popen = subprocess.Popen([link_exe, '@' + rspfile], stdout=subprocess.PIPE)
|
||
|
|
stdout, _ = popen.communicate()
|
||
|
|
if index == 0 and popen.returncode == 0 and intermediate_manifest:
|
||
|
|
# Hack for ninja build. After the linker runs, it does some manifest
|
||
|
|
# things and expects there to be a file in this location. We just put it
|
||
|
|
# there so it's happy. This is a no-op.
|
||
|
|
if os.path.isdir(os.path.dirname(intermediate_manifest)):
|
||
|
|
shutil.copyfile(manifest_name, intermediate_manifest)
|
||
|
|
return stdout, popen.returncode, output_name
|
||
|
|
|
||
|
|
|
||
|
|
def GetLibObjList(lib):
|
||
|
|
"""Gets the list of object files contained in a .lib."""
|
||
|
|
link_exe = GetOriginalLinkerPath()
|
||
|
|
popen = subprocess.Popen(
|
||
|
|
[link_exe, '/lib', '/nologo', '/list', lib], stdout=subprocess.PIPE)
|
||
|
|
stdout, _ = popen.communicate()
|
||
|
|
return stdout.splitlines()
|
||
|
|
|
||
|
|
|
||
|
|
def ExtractObjFromLib(lib, obj):
|
||
|
|
"""Extracts a .obj file contained in a .lib file. Returns the absolute path
|
||
|
|
a temp file."""
|
||
|
|
link_exe = GetOriginalLinkerPath()
|
||
|
|
temp = tempfile.NamedTemporaryFile(
|
||
|
|
prefix='split_link_', suffix='.obj', delete=False)
|
||
|
|
temp.close()
|
||
|
|
subprocess.check_call([
|
||
|
|
link_exe, '/lib', '/nologo', '/extract:' + obj, lib, '/out:' + temp.name])
|
||
|
|
return temp.name
|
||
|
|
|
||
|
|
|
||
|
|
def Unmangle(export):
|
||
|
|
"Returns the human-presentable name of a mangled symbol."""
|
||
|
|
# Use dbghelp.dll to demangle the name.
|
||
|
|
# TODO(scottmg): Perhaps a simple cache? Seems pretty fast though.
|
||
|
|
UnDecorateSymbolName = ctypes.windll.dbghelp.UnDecorateSymbolName
|
||
|
|
buffer_size = 2048
|
||
|
|
output_string = ctypes.create_string_buffer(buffer_size)
|
||
|
|
if not UnDecorateSymbolName(
|
||
|
|
export, ctypes.byref(output_string), buffer_size, 0):
|
||
|
|
raise ctypes.WinError()
|
||
|
|
return output_string.value
|
||
|
|
|
||
|
|
|
||
|
|
def IsDataDefinition(export):
|
||
|
|
"""Determines if a given name is data rather than a function. Always returns
|
||
|
|
False for C-style (as opposed to C++-style names)."""
|
||
|
|
if export[0] != '?':
|
||
|
|
return False
|
||
|
|
|
||
|
|
# If it contains a '(' we assume it's a function.
|
||
|
|
return '(' not in Unmangle(export)
|
||
|
|
|
||
|
|
|
||
|
|
def GenerateDefFiles(unresolved_by_part):
|
||
|
|
"""Given a list of unresolved externals, generates a .def file that will
|
||
|
|
cause all those symbols to be exported."""
|
||
|
|
deffiles = []
|
||
|
|
Log('generating .def files')
|
||
|
|
for i, part in enumerate(unresolved_by_part):
|
||
|
|
deffile = 'part%d.def' % i
|
||
|
|
with open(deffile, 'w') as f:
|
||
|
|
print >> f, 'LIBRARY %s' % OutputNameForIndex(i)
|
||
|
|
print >> f, 'EXPORTS'
|
||
|
|
for j, part in enumerate(unresolved_by_part):
|
||
|
|
if i == j:
|
||
|
|
continue
|
||
|
|
is_data = \
|
||
|
|
[' DATA' if IsDataDefinition(export) and not IGNORE_DATA else ''
|
||
|
|
for export in part]
|
||
|
|
print >> f, '\n'.join(' ' + export + data
|
||
|
|
for export, data in zip(part, is_data))
|
||
|
|
deffiles.append(deffile)
|
||
|
|
return deffiles
|
||
|
|
|
||
|
|
|
||
|
|
def BuildImportLibs(flags, inputs_by_part, deffiles):
|
||
|
|
"""Runs the linker to generate an import library."""
|
||
|
|
import_libs = []
|
||
|
|
Log('building import libs')
|
||
|
|
for i, (inputs, deffile) in enumerate(zip(inputs_by_part, deffiles)):
|
||
|
|
libfile = 'part%d.lib' % i
|
||
|
|
flags_with_implib_and_deffile = flags + ['/IMPLIB:%s' % libfile,
|
||
|
|
'/DEF:%s' % deffile]
|
||
|
|
RunLinker(flags_with_implib_and_deffile, i, inputs, 'implib', None)
|
||
|
|
import_libs.append(libfile)
|
||
|
|
return import_libs
|
||
|
|
|
||
|
|
|
||
|
|
def AttemptLink(flags, inputs_by_part, unresolved_by_part, deffiles,
|
||
|
|
import_libs, intermediate_manifest):
|
||
|
|
"""Tries to run the linker for all parts using the current round of
|
||
|
|
generated import libs and .def files. If the link fails, updates the
|
||
|
|
unresolved externals list per part."""
|
||
|
|
dlls = []
|
||
|
|
all_succeeded = True
|
||
|
|
new_externals = []
|
||
|
|
Log('unresolveds now: %r' % [len(part) for part in unresolved_by_part])
|
||
|
|
for i, (inputs, deffile) in enumerate(zip(inputs_by_part, deffiles)):
|
||
|
|
Log('running link, part %d' % i)
|
||
|
|
others_implibs = import_libs[:]
|
||
|
|
others_implibs.pop(i)
|
||
|
|
inputs_with_implib = inputs + filter(lambda x: x, others_implibs)
|
||
|
|
if deffile:
|
||
|
|
flags = flags + ['/DEF:%s' % deffile, '/LTCG']
|
||
|
|
stdout, rc, output = RunLinker(
|
||
|
|
flags, i, inputs_with_implib, 'final', intermediate_manifest)
|
||
|
|
if rc != 0:
|
||
|
|
all_succeeded = False
|
||
|
|
new_externals.append(ParseOutExternals(stdout))
|
||
|
|
else:
|
||
|
|
new_externals.append([])
|
||
|
|
dlls.append(output)
|
||
|
|
combined_externals = [sorted(set(prev) | set(new))
|
||
|
|
for prev, new in zip(unresolved_by_part, new_externals)]
|
||
|
|
return all_succeeded, dlls, combined_externals
|
||
|
|
|
||
|
|
|
||
|
|
def ExtractSubObjsTargetedAtAll(
|
||
|
|
inputs,
|
||
|
|
num_parts,
|
||
|
|
description_parts,
|
||
|
|
description_all,
|
||
|
|
description_all_from_libs):
|
||
|
|
"""For (lib, obj) tuples in the all_from_libs section, extract the obj out of
|
||
|
|
the lib and added it to inputs. Returns a list of lists for which part the
|
||
|
|
extracted obj belongs in (which is whichever the .lib isn't in)."""
|
||
|
|
by_parts = [[] for _ in range(num_parts)]
|
||
|
|
for lib_spec, obj_spec in description_all_from_libs:
|
||
|
|
for input_file in inputs:
|
||
|
|
if re.search(lib_spec, input_file):
|
||
|
|
objs = GetLibObjList(input_file)
|
||
|
|
match_count = 0
|
||
|
|
for obj in objs:
|
||
|
|
if re.search(obj_spec, obj, re.I):
|
||
|
|
extracted_obj = ExtractObjFromLib(input_file, obj)
|
||
|
|
#Log('extracted %s (%s %s)' % (extracted_obj, input_file, obj))
|
||
|
|
i = PartFor(input_file, description_parts, description_all)
|
||
|
|
if i == -1:
|
||
|
|
raise SystemExit(
|
||
|
|
'%s is already in all parts, but matched '
|
||
|
|
'%s in all_from_libs' % (input_file, obj))
|
||
|
|
# See note in main().
|
||
|
|
assert num_parts == 2, "Can't handle > 2 dlls currently"
|
||
|
|
by_parts[1 - i].append(obj)
|
||
|
|
match_count += 1
|
||
|
|
if match_count == 0:
|
||
|
|
raise SystemExit(
|
||
|
|
'%s, %s matched a lib, but no objs' % (lib_spec, obj_spec))
|
||
|
|
return by_parts
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
flags, inputs, intermediate_manifest = GetFlagsAndInputs(sys.argv[1:])
|
||
|
|
partition_file = os.path.normpath(
|
||
|
|
os.path.join(BASE_DIR, '../../../build/split_link_partition.py'))
|
||
|
|
with open(partition_file) as partition:
|
||
|
|
description = eval(partition.read())
|
||
|
|
inputs_by_part = []
|
||
|
|
description_parts = description['parts']
|
||
|
|
# We currently assume that if a symbol isn't in dll 0, then it's in dll 1
|
||
|
|
# when generating def files. Otherwise, we'd need to do more complex things
|
||
|
|
# to figure out where each symbol actually is to assign it to the correct
|
||
|
|
# .def file.
|
||
|
|
num_parts = len(description_parts)
|
||
|
|
assert num_parts == 2, "Can't handle > 2 dlls currently"
|
||
|
|
description_parts.reverse()
|
||
|
|
objs_from_libs = ExtractSubObjsTargetedAtAll(
|
||
|
|
inputs,
|
||
|
|
num_parts,
|
||
|
|
description_parts,
|
||
|
|
description['all'],
|
||
|
|
description['all_from_libs'])
|
||
|
|
objs_from_libs.reverse()
|
||
|
|
inputs_by_part = [[] for _ in range(num_parts)]
|
||
|
|
for input_file in inputs:
|
||
|
|
i = PartFor(input_file, description_parts, description['all'])
|
||
|
|
if i == -1:
|
||
|
|
for part in inputs_by_part:
|
||
|
|
part.append(input_file)
|
||
|
|
else:
|
||
|
|
inputs_by_part[i].append(input_file)
|
||
|
|
inputs_by_part.reverse()
|
||
|
|
|
||
|
|
# Put the subobjs on to the main list.
|
||
|
|
for i, part in enumerate(objs_from_libs):
|
||
|
|
Log('%d sub .objs added to part %d' % (len(part), i))
|
||
|
|
inputs_by_part[i].extend(part)
|
||
|
|
|
||
|
|
unresolved_by_part = [[] for _ in range(num_parts)]
|
||
|
|
import_libs = [None] * num_parts
|
||
|
|
deffiles = [None] * num_parts
|
||
|
|
|
||
|
|
data_exports = 0
|
||
|
|
for i in range(5):
|
||
|
|
Log('--- starting pass %d' % i)
|
||
|
|
ok, dlls, unresolved_by_part = AttemptLink(
|
||
|
|
flags, inputs_by_part, unresolved_by_part, deffiles, import_libs,
|
||
|
|
intermediate_manifest)
|
||
|
|
if ok:
|
||
|
|
break
|
||
|
|
data_exports = 0
|
||
|
|
for i, part in enumerate(unresolved_by_part):
|
||
|
|
for export in part:
|
||
|
|
if IsDataDefinition(export):
|
||
|
|
print 'part %d contains data export: %s (aka %s)' % (
|
||
|
|
i, Unmangle(export), export)
|
||
|
|
data_exports += 1
|
||
|
|
deffiles = GenerateDefFiles(unresolved_by_part)
|
||
|
|
import_libs = BuildImportLibs(flags, inputs_by_part, deffiles)
|
||
|
|
else:
|
||
|
|
if data_exports and not IGNORE_DATA:
|
||
|
|
print '%d data exports found, see report above.' % data_exports
|
||
|
|
print('These cannot be exported, and must be either duplicated to the '
|
||
|
|
'target DLL (if constant), or wrapped in a function.')
|
||
|
|
return 1
|
||
|
|
|
||
|
|
mt_exe = GetMtPath()
|
||
|
|
for i, dll in enumerate(dlls):
|
||
|
|
Log('embedding manifest in %s' % dll)
|
||
|
|
args = [mt_exe, '-nologo', '-manifest']
|
||
|
|
args.append(ManifestNameForIndex(i))
|
||
|
|
args.append(description['manifest'])
|
||
|
|
args.append('-outputresource:%s;2' % dll)
|
||
|
|
subprocess.check_call(args)
|
||
|
|
|
||
|
|
Log('built %r' % dlls)
|
||
|
|
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
sys.exit(main())
|