Chiebot-Mirror
/
meson
mirror of https://github.com/mesonbuild/meson.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Extend the C++ module scanner to handle Fortran, too.

Browse Source
pull/8124/merge
Jussi Pakkanen 4 years ago
parent cb10ba75d4
commit 2f836e3acc
2 changed files with 177 additions and 54 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 83
      mesonbuild/backend/ninjabackend.py
  2. 148
      mesonbuild/scripts/depscan.py

83
mesonbuild/backend/ninjabackend.py
Unescape View File

@ -117,7 +117,7 @@ raw_names = {'DEPFILE_UNQUOTED', 'DESC', 'pool', 'description', 'targetdep', 'dy
NINJA_QUOTE_BUILD_PAT = re.compile(r"[$ :\n]") NINJA_QUOTE_BUILD_PAT = re.compile(r"[$ :\n]")
NINJA_QUOTE_VAR_PAT = re.compile(r"[$ \n]") NINJA_QUOTE_VAR_PAT = re.compile(r"[$ \n]")
def ninja_quote(text, is_build_line=False): def ninja_quote(text: str, is_build_line=False) -> str:
if is_build_line: if is_build_line:
quote_re = NINJA_QUOTE_BUILD_PAT quote_re = NINJA_QUOTE_BUILD_PAT
else: else:
@ -872,7 +872,11 @@ int dummy;
self.generate_shlib_aliases(target, self.get_target_dir(target)) self.generate_shlib_aliases(target, self.get_target_dir(target))
self.add_build(elem) self.add_build(elem)
def should_scan_target(self, target): def should_use_dyndeps_for_target(self, target: 'build.BuildTarget') -> bool:
if mesonlib.version_compare(self.ninja_version, '<1.10.0'):
return False
if 'fortran' in target.compilers:
return True
if 'cpp' not in target.compilers: if 'cpp' not in target.compilers:
return False return False
# Currently only the preview version of Visual Studio is supported. # Currently only the preview version of Visual Studio is supported.
@ -883,18 +887,16 @@ int dummy;
return False return False
if mesonlib.version_compare(cpp.version, '<19.28.28617'): if mesonlib.version_compare(cpp.version, '<19.28.28617'):
return False return False
if mesonlib.version_compare(self.ninja_version, '<1.10.0'):
return False
return True return True
def generate_dependency_scan_target(self, target, compiled_sources, source2object): def generate_dependency_scan_target(self, target, compiled_sources, source2object):
if not self.should_scan_target(target): if not self.should_use_dyndeps_for_target(target):
return return
depscan_file = self.get_dep_scan_file_for(target) depscan_file = self.get_dep_scan_file_for(target)
pickle_base = target.name + '.dat' pickle_base = target.name + '.dat'
pickle_file = os.path.join(self.get_target_private_dir(target), pickle_base).replace('\\', '/') pickle_file = os.path.join(self.get_target_private_dir(target), pickle_base).replace('\\', '/')
pickle_abs = os.path.join(self.get_target_private_dir_abs(target), pickle_base).replace('\\', '/') pickle_abs = os.path.join(self.get_target_private_dir_abs(target), pickle_base).replace('\\', '/')
rule_name = 'cppscan' rule_name = 'depscan'
scan_sources = self.select_sources_to_scan(compiled_sources) scan_sources = self.select_sources_to_scan(compiled_sources)
elem = NinjaBuildElement(self.all_outputs, depscan_file, rule_name, scan_sources) elem = NinjaBuildElement(self.all_outputs, depscan_file, rule_name, scan_sources)
elem.add_item('picklefile', pickle_file) elem.add_item('picklefile', pickle_file)
@ -907,10 +909,11 @@ int dummy;
# in practice pick up C++ and Fortran files. If some other language # in practice pick up C++ and Fortran files. If some other language
# requires scanning (possibly Java to deal with inner class files) # requires scanning (possibly Java to deal with inner class files)
# then add them here. # then add them here.
all_suffixes = set(compilers.lang_suffixes['cpp']) | set(compilers.lang_suffixes['fortran'])
selected_sources = [] selected_sources = []
for source in compiled_sources: for source in compiled_sources:
ext = os.path.splitext(source)[1][1:] ext = os.path.splitext(source)[1][1:]
if ext in compilers.lang_suffixes['cpp']: if ext in all_suffixes:
selected_sources.append(source) selected_sources.append(source)
return selected_sources return selected_sources
@ -1945,7 +1948,15 @@ int dummy;
description = 'Compiling Swift source $in' description = 'Compiling Swift source $in'
self.add_rule(NinjaRule(rule, command, [], description)) self.add_rule(NinjaRule(rule, command, [], description))
def generate_fortran_dep_hack(self, crstr): def use_dyndeps_for_fortran(self) -> bool:
'''Use the new Ninja feature for scanning dependencies during build,
rather than up front. Remove this and all old scanning code once Ninja
minimum version is bumped to 1.10.'''
return mesonlib.version_compare(self.ninja_version, '>=1.10.0')
def generate_fortran_dep_hack(self, crstr: str) -> None:
if self.use_dyndeps_for_fortran():
return
rule = 'FORTRAN_DEP_HACK{}'.format(crstr) rule = 'FORTRAN_DEP_HACK{}'.format(crstr)
if mesonlib.is_windows(): if mesonlib.is_windows():
cmd = ['cmd', '/C'] cmd = ['cmd', '/C']
@ -2029,22 +2040,16 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
def generate_scanner_rules(self): def generate_scanner_rules(self):
scanner_languages = {'cpp'} # Fixme, add Fortran. rulename = 'depscan'
for for_machine in MachineChoice: if rulename in self.ruledict:
clist = self.environment.coredata.compilers[for_machine] # Scanning command is the same for native and cross compilation.
for langname, compiler in clist.items(): return
if langname not in scanner_languages: command = self.environment.get_build_command() + \
continue ['--internal', 'depscan']
rulename = '{}scan'.format(langname) args = ['$picklefile', '$out', '$in']
if rulename in self.ruledict: description = 'Module scanner.'
# Scanning command is the same for native and cross compilation. rule = NinjaRule(rulename, command, args, description)
continue self.add_rule(rule)
command = self.environment.get_build_command() + \
['--internal', 'depscan']
args = ['$picklefile', '$out', '$in']
description = 'Module scanner for {}.'.format(langname)
rule = NinjaRule(rulename, command, args, description)
self.add_rule(rule)
def generate_compile_rules(self): def generate_compile_rules(self):
@ -2146,6 +2151,8 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
""" """
Find all module and submodule made available in a Fortran code file. Find all module and submodule made available in a Fortran code file.
""" """
if self.use_dyndeps_for_fortran():
return
compiler = None compiler = None
# TODO other compilers # TODO other compilers
for lang, c in self.environment.coredata.compilers.host.items(): for lang, c in self.environment.coredata.compilers.host.items():
@ -2198,6 +2205,8 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
""" """
Find all module and submodule needed by a Fortran target Find all module and submodule needed by a Fortran target
""" """
if self.use_dyndeps_for_fortran():
return []
dirname = Path(self.get_target_private_dir(target)) dirname = Path(self.get_target_private_dir(target))
tdeps = self.fortran_deps[target.get_basename()] tdeps = self.fortran_deps[target.get_basename()]
@ -2502,16 +2511,20 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
if not is_generated: if not is_generated:
abs_src = Path(build_dir) / rel_src abs_src = Path(build_dir) / rel_src
extra_deps += self.get_fortran_deps(compiler, abs_src, target) extra_deps += self.get_fortran_deps(compiler, abs_src, target)
# Dependency hack. Remove once multiple outputs in Ninja is fixed: if not self.use_dyndeps_for_fortran():
# https://groups.google.com/forum/#!topic/ninja-build/j-2RfBIOd_8 # Dependency hack. Remove once multiple outputs in Ninja is fixed:
for modname, srcfile in self.fortran_deps[target.get_basename()].items(): # https://groups.google.com/forum/#!topic/ninja-build/j-2RfBIOd_8
modfile = os.path.join(self.get_target_private_dir(target), for modname, srcfile in self.fortran_deps[target.get_basename()].items():
compiler.module_name_to_filename(modname)) modfile = os.path.join(self.get_target_private_dir(target),
compiler.module_name_to_filename(modname))
if srcfile == src:
crstr = self.get_rule_suffix(target.for_machine) if srcfile == src:
depelem = NinjaBuildElement(self.all_outputs, modfile, 'FORTRAN_DEP_HACK' + crstr, rel_obj) crstr = self.get_rule_suffix(target.for_machine)
self.add_build(depelem) depelem = NinjaBuildElement(self.all_outputs,
modfile,
'FORTRAN_DEP_HACK' + crstr,
rel_obj)
self.add_build(depelem)
commands += compiler.get_module_outdir_args(self.get_target_private_dir(target)) commands += compiler.get_module_outdir_args(self.get_target_private_dir(target))
element = NinjaBuildElement(self.all_outputs, rel_obj, compiler_name, rel_src) element = NinjaBuildElement(self.all_outputs, rel_obj, compiler_name, rel_src)
@ -2537,7 +2550,7 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
return (rel_obj, rel_src.replace('\\', '/')) return (rel_obj, rel_src.replace('\\', '/'))
def add_dependency_scanner_entries_to_element(self, target, compiler, element): def add_dependency_scanner_entries_to_element(self, target, compiler, element):
if not self.should_scan_target(target): if not self.should_use_dyndeps_for_target(target):
return return
dep_scan_file = self.get_dep_scan_file_for(target) dep_scan_file = self.get_dep_scan_file_for(target)
element.add_item('dyndep', dep_scan_file) element.add_item('dyndep', dep_scan_file)

148
mesonbuild/scripts/depscan.py
Unescape View File

@ -15,12 +15,24 @@
import pathlib import pathlib
import pickle import pickle
import re import re
import os
import sys
import typing as T import typing as T
from ..backend.ninjabackend import TargetDependencyScannerInfo from ..backend.ninjabackend import TargetDependencyScannerInfo, ninja_quote
from ..compilers.compilers import lang_suffixes
import_re = re.compile('\w*import ([a-zA-Z0-9]+);') CPP_IMPORT_RE = re.compile('\w*import ([a-zA-Z0-9]+);')
export_re = re.compile('\w*export module ([a-zA-Z0-9]+);') CPP_EXPORT_RE = re.compile('\w*export module ([a-zA-Z0-9]+);')
FORTRAN_INCLUDE_PAT = r"^\s*include\s*['\"](\w+\.\w+)['\"]"
FORTRAN_MODULE_PAT = r"^\s*\bmodule\b\s+(\w+)\s*(?:!+.*)*$"
FORTRAN_SUBMOD_PAT = r"^\s*\bsubmodule\b\s*\((\w+:?\w+)\)\s*(\w+)"
FORTRAN_USE_PAT = r"^\s*use,?\s*(?:non_intrinsic)?\s*(?:::)?\s*(\w+)"
FORTRAN_MODULE_RE = re.compile(FORTRAN_MODULE_PAT, re.IGNORECASE)
FORTRAN_SUBMOD_RE = re.compile(FORTRAN_SUBMOD_PAT, re.IGNORECASE)
FORTRAN_USE_RE = re.compile(FORTRAN_USE_PAT, re.IGNORECASE)
class DependencyScanner: class DependencyScanner:
def __init__(self, pickle_file: str, outfile: str, sources: T.List[str]): def __init__(self, pickle_file: str, outfile: str, sources: T.List[str]):
@ -32,11 +44,73 @@ class DependencyScanner:
self.exports = {} # type: T.Dict[str, str] self.exports = {} # type: T.Dict[str, str]
self.needs = {} # type: T.Dict[str, T.List[str]] self.needs = {} # type: T.Dict[str, T.List[str]]
self.sources_with_exports = [] # type: T.List[str] self.sources_with_exports = [] # type: T.List[str]
def scan_file(self, fname: str) -> None: def scan_file(self, fname: str) -> None:
for line in pathlib.Path(fname).read_text().split('\n'): suffix = os.path.splitext(fname)[1][1:]
import_match = import_re.match(line) if suffix in lang_suffixes['fortran']:
export_match = export_re.match(line) self.scan_fortran_file(fname)
elif suffix in lang_suffixes['cpp']:
self.scan_cpp_file(fname)
else:
sys.exit('Can not scan files with suffix .{}.'.format(suffix))
def scan_fortran_file(self, fname: str) -> None:
fpath = pathlib.Path(fname)
modules_in_this_file = set()
for line in fpath.read_text().split('\n'):
import_match = FORTRAN_USE_RE.match(line)
export_match = FORTRAN_MODULE_RE.match(line)
submodule_export_match = FORTRAN_SUBMOD_RE.match(line)
if import_match:
needed = import_match.group(1).lower()
# In Fortran you have an using declaration also for the module
# you define in the same file. Prevent circular dependencies.
if needed not in modules_in_this_file:
if fname in self.needs:
self.needs[fname].append(needed)
else:
self.needs[fname] = [needed]
if export_match:
exported_module = export_match.group(1).lower()
assert(exported_module not in modules_in_this_file)
modules_in_this_file.add(exported_module)
if exported_module in self.provided_by:
raise RuntimeError('Multiple files provide module {}.'.format(exported_module))
self.sources_with_exports.append(fname)
self.provided_by[exported_module] = fname
self.exports[fname] = exported_module
if submodule_export_match:
# Store submodule "Foo" "Bar" as "foo:bar".
# A submodule declaration can be both an import and an export declaration:
#
# submodule (a1:a2) a3
# - requires a1@a2.smod
# - produces a1@a3.smod
parent_module_name_full = submodule_export_match.group(1).lower()
parent_module_name = parent_module_name_full.split(':')[0]
submodule_name = submodule_export_match.group(2).lower()
concat_name = '{}:{}'.format(parent_module_name, submodule_name)
self.sources_with_exports.append(fname)
self.provided_by[concat_name] = fname
self.exports[fname] = concat_name
# Fortran requires that the immediate parent module must be built
# before the current one. Thus:
#
# submodule (parent) parent <- requires parent.mod (really parent.smod, but they are created at the same time)
# submodule (a1:a2) a3 <- requires a1@a2.smod
#
# a3 does not depend on the a1 parent module directly, only transitively.
if fname in self.needs:
self.needs[fname].append(parent_module_name_full)
else:
self.needs[fname] = [parent_module_name_full]
def scan_cpp_file(self, fname: str) -> None:
fpath = pathlib.Path(fname)
for line in fpath.read_text().split('\n'):
import_match = CPP_IMPORT_RE.match(line)
export_match = CPP_EXPORT_RE.match(line)
if import_match: if import_match:
needed = import_match.group(1) needed = import_match.group(1)
if fname in self.needs: if fname in self.needs:
@ -56,8 +130,22 @@ class DependencyScanner:
assert(isinstance(objname, str)) assert(isinstance(objname, str))
return objname return objname
def ifcname_for(self, src: str) -> str: def module_name_for(self, src: str) -> str:
return '{}.ifc'.format(self.exports[src]) suffix= os.path.splitext(src)[1][1:]
if suffix in lang_suffixes['fortran']:
exported = self.exports[src]
# Module foo:bar goes to a file name foo@bar.smod
# Module Foo goes to a file name foo.mod
namebase = exported.replace(':', '@')
if ':' in exported:
extension = 'smod'
else:
extension = 'mod'
return os.path.join(self.target_data.private_dir, '{}.{}'.format(namebase, extension))
elif suffix in lang_suffixes['cpp']:
return '{}.ifc'.format(self.exports[src])
else:
raise RuntimeError('Unreachable code.')
def scan(self) -> int: def scan(self) -> int:
for s in self.sources: for s in self.sources:
@ -66,21 +154,43 @@ class DependencyScanner:
ofile.write('ninja_dyndep_version = 1\n') ofile.write('ninja_dyndep_version = 1\n')
for src in self.sources: for src in self.sources:
objfilename = self.objname_for(src) objfilename = self.objname_for(src)
mods_and_submods_needed = []
module_files_generated = []
module_files_needed = []
if src in self.sources_with_exports: if src in self.sources_with_exports:
ifc_entry = '| ' + self.ifcname_for(src) module_files_generated.append(self.module_name_for(src))
else:
ifc_entry = ''
if src in self.needs: if src in self.needs:
# FIXME, handle all sources, not just the first one for modname in self.needs[src]:
modname = self.needs[src][0] if modname not in self.provided_by:
# Nothing provides this module, we assume that it
# comes from a dependency library somewhere and is
# already built by the time this complation starts.
pass
else:
mods_and_submods_needed.append(modname)
for modname in mods_and_submods_needed:
provider_src = self.provided_by[modname] provider_src = self.provided_by[modname]
provider_ifc = self.ifcname_for(provider_src) provider_modfile = self.module_name_for(provider_src)
mod_dep = '| ' + provider_ifc # Prune self-dependencies
if provider_src != src:
module_files_needed.append(provider_modfile)
quoted_objfilename = ninja_quote(objfilename, True)
quoted_module_files_generated = [ninja_quote(x, True) for x in module_files_generated]
quoted_module_files_needed = [ninja_quote(x, True) for x in module_files_needed]
if quoted_module_files_generated:
mod_gen = '| ' + ' '.join(quoted_module_files_generated)
else:
mod_gen = ''
if quoted_module_files_needed:
mod_dep = '| ' + ' '.join(quoted_module_files_needed)
else: else:
mod_dep = '' mod_dep = ''
ofile.write('build {} {}: dyndep {}\n'.format(objfilename, build_line = 'build {} {}: dyndep {}'.format(quoted_objfilename,
ifc_entry, mod_gen,
mod_dep)) mod_dep)
ofile.write(build_line + '\n')
return 0 return 0
def run(args: T.List[str]) -> int: def run(args: T.List[str]) -> int:

Write Preview
Loading…
Cancel
Save