Merge pull request #1171 from centricular/fix-extracted-generated-prebuilt-object-targets-linking

Several fixes to how we handle objects in build targets
pull/1188/head
Jussi Pakkanen 8 years ago committed by GitHub
commit de83e94b5a
  1. 40
      mesonbuild/backend/backends.py
  2. 40
      mesonbuild/backend/ninjabackend.py
  3. 144
      mesonbuild/build.py
  4. 11
      mesonbuild/compilers.py
  5. 12
      mesonbuild/interpreter.py
  6. 2
      mesonbuild/interpreterbase.py
  7. 10
      mesonbuild/modules/gnome.py
  8. 2
      mesonbuild/modules/rpm.py
  9. 6
      mesonbuild/modules/windows.py
  10. 6
      test cases/common/128 extract all shared library/extractor.h
  11. 5
      test cases/common/128 extract all shared library/four.c
  12. 5
      test cases/common/128 extract all shared library/func1234.def
  13. 10
      test cases/common/128 extract all shared library/meson.build
  14. 5
      test cases/common/128 extract all shared library/one.c
  15. 10
      test cases/common/128 extract all shared library/prog.c
  16. 5
      test cases/common/128 extract all shared library/three.c
  17. 5
      test cases/common/128 extract all shared library/two.c
  18. 1
      test cases/common/129 object only target/installed_files.txt
  19. 45
      test cases/common/129 object only target/meson.build
  20. 18
      test cases/common/129 object only target/obj_generator.py
  21. 7
      test cases/common/129 object only target/prog.c
  22. 3
      test cases/common/129 object only target/source.c
  23. 3
      test cases/common/129 object only target/source2.c
  24. 2
      test cases/common/129 object only target/source2.def
  25. 3
      test cases/common/129 object only target/source3.c

@ -178,6 +178,8 @@ class Backend():
o = os.path.join(proj_dir_to_build_root,
self.build_to_src, target.get_subdir(), obj)
obj_list.append(o)
elif isinstance(obj, mesonlib.File):
obj_list.append(obj.rel_to_builddir(self.build_to_src))
elif isinstance(obj, build.ExtractedObjects):
obj_list += self.determine_ext_objs(obj, proj_dir_to_build_root)
else:
@ -233,39 +235,21 @@ class Backend():
self.write_benchmark_file(datafile)
return (test_data, benchmark_data)
def determine_linker(self, target, src):
def determine_linker(self, target):
'''
If we're building a static library, there is only one static linker.
Otherwise, we query the target for the dynamic linker.
'''
if isinstance(target, build.StaticLibrary):
if target.is_cross:
return self.build.static_cross_linker
else:
return self.build.static_linker
if target.is_cross:
compilers = self.build.cross_compilers
else:
compilers = self.build.compilers
if len(compilers) == 1:
return compilers[0]
# Currently a bit naive. C++ must
# be linked with a C++ compiler, but
# otherwise we don't care. This will
# become trickier if and when Fortran
# and the like become supported.
cpp = None
for c in compilers:
if c.get_language() == 'cpp':
cpp = c
break
if cpp is not None:
for s in src:
if c.can_compile(s):
return cpp
for c in compilers:
if c.get_language() == 'vala':
continue
for s in src:
if c.can_compile(s):
return c
raise AssertionError("BUG: Couldn't determine linker for sources {!r}".format(src))
l = target.get_clike_dynamic_linker()
if not l:
m = "Couldn't determine linker for target {!r}"
raise MesonException(m.format(target.name))
return l
def object_filename_from_source(self, target, source):
if isinstance(source, mesonlib.File):

@ -299,6 +299,9 @@ int dummy;
self.generate_swift_target(target, outfile)
return
# Now we handle the following languages:
# ObjC++, ObjC, C++, C, D, Fortran, Vala
# Pre-existing target C/C++ sources to be built; dict of full path to
# source relative to build root and the original File object.
target_sources = OrderedDict()
@ -332,7 +335,6 @@ int dummy;
unity_src = []
unity_deps = [] # Generated sources that must be built before compiling a Unity target.
header_deps += self.get_generated_headers(target)
src_list = []
if is_unity:
# Warn about incompatible sources if a unity build is enabled
@ -346,16 +348,12 @@ int dummy;
''.format(langs_are, langs, target.name)
mlog.log(mlog.red('FIXME'), msg)
# Get a list of all generated *sources* (sources files, headers,
# objects, etc). Needed to determine the linker.
generated_output_sources = []
# Get a list of all generated headers that will be needed while building
# this target's sources (generated sources and pre-existing sources).
# This will be set as dependencies of all the target's sources. At the
# same time, also deal with generated sources that need to be compiled.
generated_source_files = []
for rel_src, gensrc in generated_sources.items():
generated_output_sources.append(rel_src)
raw_src = RawFilename(rel_src)
if self.environment.is_source(rel_src) and not self.environment.is_header(rel_src):
if is_unity and self.get_target_source_can_unity(target, rel_src):
@ -377,12 +375,12 @@ int dummy;
# this target. We create the Ninja build file elements for this here
# because we need `header_deps` to be fully generated in the above loop.
for src in generated_source_files:
src_list.append(src)
if self.environment.is_llvm_ir(src):
obj_list.append(self.generate_llvm_ir_compile(target, outfile, src))
continue
obj_list.append(self.generate_single_compile(target, outfile, src, True,
header_deps=header_deps))
o = self.generate_llvm_ir_compile(target, outfile, src)
else:
o = self.generate_single_compile(target, outfile, src, True,
header_deps=header_deps)
obj_list.append(o)
# Generate compilation targets for C sources generated from Vala
# sources. This can be extended to other $LANG->C compilers later if
@ -390,7 +388,6 @@ int dummy;
vala_generated_source_files = []
for src in vala_generated_sources:
raw_src = RawFilename(src)
src_list.append(src)
if is_unity:
unity_src.append(os.path.join(self.environment.get_build_dir(), src))
header_deps.append(raw_src)
@ -415,7 +412,6 @@ int dummy;
# Generate compile targets for all the pre-existing sources for this target
for f, src in target_sources.items():
if not self.environment.is_header(src):
src_list.append(src)
if self.environment.is_llvm_ir(src):
obj_list.append(self.generate_llvm_ir_compile(target, outfile, src))
elif is_unity and self.get_target_source_can_unity(target, src):
@ -428,7 +424,7 @@ int dummy;
if is_unity:
for src in self.generate_unity_files(target, unity_src):
obj_list.append(self.generate_single_compile(target, outfile, RawFilename(src), True, unity_deps + header_deps))
linker = self.determine_linker(target, src_list + generated_output_sources)
linker = self.determine_linker(target)
elem = self.generate_link(target, outfile, outname, obj_list, linker, pch_objects)
self.generate_shlib_aliases(target, self.get_target_dir(target))
elem.write(outfile)
@ -970,7 +966,8 @@ int dummy;
(vala_src, vapi_src, other_src) = self.split_vala_sources(target)
extra_dep_files = []
if len(vala_src) == 0:
raise InvalidArguments('Vala library has no Vala source files.')
msg = 'Vala library {!r} has no Vala source files.'
raise InvalidArguments(msg.format(target.name))
valac = target.compilers['vala']
c_out_dir = self.get_target_private_dir(target)
@ -1210,7 +1207,7 @@ int dummy;
raise MesonException('Swift supports only executable and static library targets.')
def generate_static_link_rules(self, is_cross, outfile):
if self.build.has_language('java'):
if 'java' in self.build.compilers:
if not is_cross:
self.generate_java_link(outfile)
if is_cross:
@ -1251,8 +1248,7 @@ int dummy;
else:
ctypes.append((self.build.cross_compilers, True))
for (complist, is_cross) in ctypes:
for compiler in complist:
langname = compiler.get_language()
for langname, compiler in complist.items():
if langname == 'java' or langname == 'vala' or\
langname == 'rust' or langname == 'cs':
continue
@ -1511,8 +1507,7 @@ rule FORTRAN_DEP_HACK
def generate_compile_rules(self, outfile):
qstr = quote_char + "%s" + quote_char
for compiler in self.build.compilers:
langname = compiler.get_language()
for langname, compiler in self.build.compilers.items():
if compiler.get_id() == 'clang':
self.generate_llvm_ir_compile_rule(compiler, False, outfile)
self.generate_compile_rule_for(langname, compiler, qstr, False, outfile)
@ -1524,8 +1519,7 @@ rule FORTRAN_DEP_HACK
cclist = self.build.cross_compilers
else:
cclist = self.build.compilers
for compiler in cclist:
langname = compiler.get_language()
for langname, compiler in cclist.items():
if compiler.get_id() == 'clang':
self.generate_llvm_ir_compile_rule(compiler, True, outfile)
self.generate_compile_rule_for(langname, compiler, qstr, True, outfile)
@ -1588,8 +1582,8 @@ rule FORTRAN_DEP_HACK
def scan_fortran_module_outputs(self, target):
compiler = None
for c in self.build.compilers:
if c.get_language() == 'fortran':
for lang, c in self.build.compilers.items():
if lang == 'fortran':
compiler = c
break
if compiler is None:

@ -19,6 +19,7 @@ from . import mlog
import copy, os, re
from .mesonlib import File, flatten, MesonException, stringlistify, classify_unity_sources
from .environment import for_windows, for_darwin
from .compilers import is_object, clike_langs, lang_suffixes
known_basic_kwargs = {'install' : True,
'c_pch' : True,
@ -62,17 +63,6 @@ known_lib_kwargs.update({'version' : True, # Only for shared libs
'pic' : True, # Only for static libs
})
def compilers_are_msvc(compilers):
"""
Check if all the listed compilers are MSVC. Used by Executable,
StaticLibrary, and SharedLibrary for deciding when to use MSVC-specific
file naming.
"""
for compiler in compilers.values():
if compiler.get_id() != 'msvc':
return False
return True
class InvalidArguments(MesonException):
pass
@ -88,8 +78,8 @@ class Build:
self.environment = environment
self.projects = {}
self.targets = {}
self.compilers = []
self.cross_compilers = []
self.compilers = {}
self.cross_compilers = {}
self.global_args = {}
self.projects_args = {}
self.global_link_args = {}
@ -109,26 +99,19 @@ class Build:
self.dep_manifest = {}
self.cross_stdlibs = {}
def has_language(self, language):
for i in self.compilers:
if i.get_language() == language:
return True
return False
def add_compiler(self, compiler):
if self.static_linker is None and compiler.needs_static_linker():
self.static_linker = self.environment.detect_static_linker(compiler)
if self.has_language(compiler.get_language()):
return
self.compilers.append(compiler)
lang = compiler.get_language()
if lang not in self.compilers:
self.compilers[lang] = compiler
def add_cross_compiler(self, compiler):
if len(self.cross_compilers) == 0:
self.static_cross_linker = self.environment.detect_static_linker(compiler)
for i in self.cross_compilers:
if i.get_language() == compiler.get_language():
return
self.cross_compilers.append(compiler)
lang = compiler.get_language()
if lang not in self.cross_compilers:
self.cross_compilers[lang] = compiler
def get_project(self):
return self.projects['']
@ -204,6 +187,10 @@ class ExtractedObjects():
if is_unity:
self.check_unity_compatible()
def __repr__(self):
r = '<{0} {1!r}: {2}>'
return r.format(self.__class__.__name__, self.target.name, self.srclist)
def check_unity_compatible(self):
# Figure out if the extracted object list is compatible with a Unity
# build. When we're doing a Unified build, we go through the sources,
@ -290,7 +277,14 @@ class BuildTarget():
self.extra_args = {}
self.generated = []
self.extra_files = []
# Sources can be:
# 1. Pre-existing source files in the source tree
# 2. Pre-existing sources generated by configure_file in the build tree
# 3. Sources files generated by another target or a Generator
self.process_sourcelist(sources)
# Objects can be:
# 1. Pre-existing objects provided by the user with the `objects:` kwarg
# 2. Compiled objects created by and extracted from another target
self.process_objectlist(objects)
self.process_kwargs(kwargs, environment)
self.check_unknown_kwargs(kwargs)
@ -333,7 +327,7 @@ class BuildTarget():
for s in objects:
if hasattr(s, 'held_object'):
s = s.held_object
if isinstance(s, (str, ExtractedObjects)):
if isinstance(s, (str, File, ExtractedObjects)):
self.objects.append(s)
elif isinstance(s, (GeneratedList, CustomTarget)):
msg = 'Generated files are not allowed in the \'objects\' kwarg ' + \
@ -380,19 +374,56 @@ class BuildTarget():
return removed
def process_compilers(self):
if len(self.sources) + len(self.generated) == 0:
'''
Populate self.compilers, which is the list of compilers that this
target will use for compiling all its sources.
We also add compilers that were used by extracted objects to simplify
dynamic linker determination.
'''
if len(self.sources) + len(self.generated) + len(self.objects) == 0:
return
sources = list(self.sources)
for gensrc in self.generated:
sources += gensrc.get_outputs()
# Populate list of compilers
if self.is_cross:
compilers = self.environment.coredata.cross_compilers
else:
compilers = self.environment.coredata.compilers
for lang, compiler in compilers.items():
if self.can_compile_sources(compiler, sources):
self.compilers[lang] = compiler
# Pre-existing sources
sources = list(self.sources)
# All generated sources
for gensrc in self.generated:
for s in gensrc.get_outputs():
# Generated objects can't be compiled, so don't use them for
# compiler detection. If our target only has generated objects,
# we will fall back to using the first c-like compiler we find,
# which is what we need.
if not is_object(s):
sources.append(s)
# Sources that were used to create our extracted objects
for o in self.objects:
if not isinstance(o, ExtractedObjects):
continue
for s in o.srclist:
# Don't add Vala sources since that will pull in the Vala
# compiler even though we will never use it since we are
# dealing with compiled C code.
if not s.endswith(lang_suffixes['vala']):
sources.append(s)
if sources:
# Add compilers based on the above sources
for lang, compiler in compilers.items():
# We try to be conservative because sometimes people add files
# in the list of sources that we can't determine the type based
# just on the suffix.
if self.can_compile_sources(compiler, sources):
self.compilers[lang] = compiler
else:
# No source files, target consists of only object files of unknown
# origin. Just add the first clike compiler that we have and hope
# that it can link these objects
for lang in clike_langs:
if lang in compilers:
self.compilers[lang] = compilers[lang]
break
# If all our sources are Vala, our target also needs the C compiler but
# it won't get added above.
if 'vala' in self.compilers and 'c' not in self.compilers:
@ -766,6 +797,43 @@ class BuildTarget():
def get_aliaslist(self):
return []
def get_clike_dynamic_linker(self):
'''
We use the order of languages in `clike_langs` to determine which
linker to use in case the target has sources compiled with multiple
compilers. All languages other than those in this list have their own
linker.
Note that Vala outputs C code, so Vala sources can use any linker
that can link compiled C. We don't actually need to add an exception
for Vala here because of that.
'''
for l in clike_langs:
if l in self.compilers:
return self.compilers[l]
def get_using_msvc(self):
'''
Check if the dynamic linker is MSVC. Used by Executable, StaticLibrary,
and SharedLibrary for deciding when to use MSVC-specific file naming
and debug filenames.
If at least some code is built with MSVC and the final library is
linked with MSVC, we can be sure that some debug info will be
generated. We only check the dynamic linker here because the static
linker is guaranteed to be of the same type.
Interesting cases:
1. The Vala compiler outputs C code to be compiled by whatever
C compiler we're using, so all objects will still be created by the
MSVC compiler.
2. If the target contains only objects, process_compilers guesses and
picks the first compiler that smells right.
'''
linker = self.get_clike_dynamic_linker()
if linker and linker.get_id() == 'msvc':
return True
return False
class Generator():
def __init__(self, args, kwargs):
@ -890,7 +958,7 @@ class Executable(BuildTarget):
self.filename += '.' + self.suffix
# See determine_debug_filenames() in build.SharedLibrary
buildtype = environment.coredata.get_builtin_option('buildtype')
if compilers_are_msvc(self.compilers) and buildtype.startswith('debug'):
if self.get_using_msvc() and buildtype.startswith('debug'):
self.debug_filename = self.prefix + self.name + '.pdb'
def type_suffix(self):
@ -921,7 +989,7 @@ class StaticLibrary(BuildTarget):
self.filename = self.prefix + self.name + '.' + self.suffix
# See determine_debug_filenames() in build.SharedLibrary
buildtype = environment.coredata.get_builtin_option('buildtype')
if compilers_are_msvc(self.compilers) and buildtype.startswith('debug'):
if self.get_using_msvc() and buildtype.startswith('debug'):
self.debug_filename = self.prefix + self.name + '.pdb'
def type_suffix(self):
@ -997,7 +1065,7 @@ class SharedLibrary(BuildTarget):
suffix = 'dll'
self.vs_import_filename = '{0}.lib'.format(self.name)
self.gcc_import_filename = 'lib{0}.dll.a'.format(self.name)
if compilers_are_msvc(self.compilers):
if self.get_using_msvc():
# Shared library is of the form foo.dll
prefix = ''
# Import library is called foo.lib
@ -1044,7 +1112,7 @@ class SharedLibrary(BuildTarget):
determine_filenames() above.
"""
buildtype = env.coredata.get_builtin_option('buildtype')
if compilers_are_msvc(self.compilers) and buildtype.startswith('debug'):
if self.get_using_msvc() and buildtype.startswith('debug'):
# Currently we only implement separate debug symbol files for MSVC
# since the toolchain does it for us. Other toolchains embed the
# debugging symbols in the file itself by default.

@ -45,8 +45,15 @@ lang_suffixes = {
}
cpp_suffixes = lang_suffixes['cpp'] + ('h',)
c_suffixes = lang_suffixes['c'] + ('h',)
clike_suffixes = lang_suffixes['c'] + lang_suffixes['cpp'] + ('h',)
# List of languages that can be linked with C code directly by the linker
# used in build.py:process_compilers() and build.py:get_dynamic_linker()
clike_langs = ('objcpp', 'objc', 'd', 'cpp', 'c', 'fortran',)
clike_suffixes = ()
for l in clike_langs:
clike_suffixes += lang_suffixes[l]
clike_suffixes += ('h',)
# These are used in backend/backends.py:generated_target()
def is_header(fname):
if hasattr(fname, 'fname'):
fname = fname.fname

@ -1082,9 +1082,8 @@ class MesonMain(InterpreterObject):
clist = self.build.compilers
else:
clist = self.build.cross_compilers
for c in clist:
if c.get_language() == cname:
return CompilerHolder(c, self.build.environment)
if cname in clist:
return CompilerHolder(clist[cname], self.build.environment)
raise InterpreterException('Tried to access compiler for unspecified language "%s".' % cname)
def is_unity_method(self, args, kwargs):
@ -1262,8 +1261,7 @@ class Interpreter(InterpreterBase):
def check_cross_stdlibs(self):
if self.build.environment.is_cross_build():
cross_info = self.build.environment.cross_info
for c in self.build.cross_compilers:
l = c.language
for l, c in self.build.cross_compilers.items():
try:
di = mesonlib.stringlistify(cross_info.get_stdlib(l))
if len(di) != 2:
@ -2295,9 +2293,9 @@ requirements use the version keyword argument instead.''')
def get_used_languages(self, target):
result = {}
for i in target.sources:
for c in self.build.compilers:
for lang, c in self.build.compilers.items():
if c.can_compile(i):
result[c.language] = True
result[lang] = True
break
return result

@ -526,7 +526,7 @@ class InterpreterBase:
def flatten(self, args):
if isinstance(args, mparser.StringNode):
return args.value
if isinstance(args, (int, str, InterpreterObject)):
if isinstance(args, (int, str, mesonlib.File, InterpreterObject)):
return args
result = []
for a in args:

@ -433,11 +433,11 @@ can not be used with the current version of glib-compiled-resources, due to
cflags += state.global_args['c']
if state.project_args.get('c'):
cflags += state.project_args['c']
for compiler in state.compilers:
if compiler.get_language() == 'c':
sanitize = compiler.get_options().get('b_sanitize')
if sanitize:
cflags += compilers.sanitizer_compile_args(sanitize)
if 'c' in state.compilers:
compiler = state.compilers['c']
sanitize = compiler.get_options().get('b_sanitize')
if sanitize:
cflags += compilers.sanitizer_compile_args(sanitize)
if cflags:
scan_command += ['--cflags-begin']
scan_command += cflags

@ -27,7 +27,7 @@ class RPMModule:
def generate_spec_template(self, state, args, kwargs):
compiler_deps = set()
for compiler in state.compilers:
for compiler in state.compilers.values():
if isinstance(compiler, compilers.GnuCCompiler):
compiler_deps.add('gcc')
elif isinstance(compiler, compilers.GnuCPPCompiler):

@ -19,9 +19,9 @@ import os
class WindowsModule:
def detect_compiler(self, compilers):
for c in compilers:
if c.language == 'c' or c.language == 'cpp':
return c
for l in ('c', 'cpp'):
if l in compilers:
return compilers[l]
raise MesonException('Resource compilation requires a C or C++ compiler.')
def compile_resources(self, state, args, kwargs):

@ -0,0 +1,6 @@
#pragma once
int func1();
int func2();
int func3();
int func4();

@ -0,0 +1,5 @@
#include"extractor.h"
int func4() {
return 4;
}

@ -0,0 +1,10 @@
project('extract all', 'c', 'cpp')
a = static_library('a', 'one.c', 'two.c')
b = static_library('b', 'three.c', 'four.c')
c = shared_library('c',
objects : [a.extract_all_objects(), b.extract_all_objects()],
vs_module_defs : 'func1234.def')
e = executable('proggie', 'prog.c', link_with : c)
test('extall', e)

@ -0,0 +1,5 @@
#include"extractor.h"
int func1() {
return 1;
}

@ -0,0 +1,10 @@
#include"extractor.h"
#include<stdio.h>
int main(int argc, char **argv) {
if((1+2+3+4) != (func1() + func2() + func3() + func4())) {
printf("Arithmetic is fail.\n");
return 1;
}
return 0;
}

@ -0,0 +1,5 @@
#include"extractor.h"
int func3() {
return 3;
}

@ -0,0 +1,5 @@
#include"extractor.h"
int func2() {
return 2;
}

@ -0,0 +1,45 @@
project('object generator', 'c')
# FIXME: Note that this will not add a dependency to the compiler executable.
# Code will not be rebuilt if it changes.
comp = find_program('obj_generator.py')
if host_machine.system() == 'windows'
ext = '.obj'
else
ext = '.o'
endif
cc = meson.get_compiler('c').cmd_array().get(-1)
# Generate an object file with configure_file to mimic prebuilt objects
# provided by the source tree
source1 = configure_file(input : 'source.c',
output : 'source' + ext,
command : [comp, cc, 'source.c',
join_paths(meson.current_build_dir(), 'source' + ext)])
obj = static_library('obj', objects : source1)
# Generate an object file manually.
gen = generator(comp,
output : '@BASENAME@' + ext,
arguments : [cc, '@INPUT@', '@OUTPUT@'])
generated = gen.process(['source2.c'])
shr = shared_library('shr', generated,
vs_module_defs : 'source2.def')
# Generate an object file with indexed OUTPUT replacement.
gen2 = generator(comp,
output : '@BASENAME@' + ext,
arguments : [cc, '@INPUT@', '@OUTPUT0@'])
generated2 = gen2.process(['source3.c'])
stc = static_library('stc', generated2)
e = executable('prog', 'prog.c', link_with : [obj, shr, stc],
install : true)
test('objgen', e)

@ -0,0 +1,18 @@
#!/usr/bin/env python
# Mimic a binary that generates an object file (e.g. windres).
import sys, shutil, subprocess
if __name__ == '__main__':
if len(sys.argv) != 4:
print(sys.argv[0], 'compiler input_file output_file')
sys.exit(1)
compiler = sys.argv[1]
ifile = sys.argv[2]
ofile = sys.argv[3]
if compiler.endswith('cl'):
cmd = [compiler, '/nologo', '/MDd', '/Fo'+ofile, '/c', ifile]
else:
cmd = [compiler, '-c', ifile, '-o', ofile]
sys.exit(subprocess.call(cmd))

@ -0,0 +1,7 @@
int func1_in_obj();
int func2_in_obj();
int func3_in_obj();
int main(int argc, char **argv) {
return func1_in_obj() + func2_in_obj() + func3_in_obj();
}

@ -0,0 +1,3 @@
int func1_in_obj() {
return 0;
}

@ -0,0 +1,3 @@
int func2_in_obj() {
return 0;
}

@ -0,0 +1,3 @@
int func3_in_obj() {
return 0;
}
Loading…
Cancel
Save