From 2f836e3accd112a52bb8d3ef5d44b7e0fb692fcf Mon Sep 17 00:00:00 2001 From: Jussi Pakkanen Date: Mon, 14 Dec 2020 18:56:30 +0200 Subject: [PATCH] Extend the C++ module scanner to handle Fortran, too. --- mesonbuild/backend/ninjabackend.py | 83 +++++++++------- mesonbuild/scripts/depscan.py | 148 +++++++++++++++++++++++++---- 2 files changed, 177 insertions(+), 54 deletions(-) diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index e64375474..40d4ae3e5 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -117,7 +117,7 @@ raw_names = {'DEPFILE_UNQUOTED', 'DESC', 'pool', 'description', 'targetdep', 'dy NINJA_QUOTE_BUILD_PAT = re.compile(r"[$ :\n]") NINJA_QUOTE_VAR_PAT = re.compile(r"[$ \n]") -def ninja_quote(text, is_build_line=False): +def ninja_quote(text: str, is_build_line=False) -> str: if is_build_line: quote_re = NINJA_QUOTE_BUILD_PAT else: @@ -872,7 +872,11 @@ int dummy; self.generate_shlib_aliases(target, self.get_target_dir(target)) self.add_build(elem) - def should_scan_target(self, target): + def should_use_dyndeps_for_target(self, target: 'build.BuildTarget') -> bool: + if mesonlib.version_compare(self.ninja_version, '<1.10.0'): + return False + if 'fortran' in target.compilers: + return True if 'cpp' not in target.compilers: return False # Currently only the preview version of Visual Studio is supported. @@ -883,18 +887,16 @@ int dummy; return False if mesonlib.version_compare(cpp.version, '<19.28.28617'): return False - if mesonlib.version_compare(self.ninja_version, '<1.10.0'): - return False return True def generate_dependency_scan_target(self, target, compiled_sources, source2object): - if not self.should_scan_target(target): + if not self.should_use_dyndeps_for_target(target): return depscan_file = self.get_dep_scan_file_for(target) pickle_base = target.name + '.dat' pickle_file = os.path.join(self.get_target_private_dir(target), pickle_base).replace('\\', '/') pickle_abs = os.path.join(self.get_target_private_dir_abs(target), pickle_base).replace('\\', '/') - rule_name = 'cppscan' + rule_name = 'depscan' scan_sources = self.select_sources_to_scan(compiled_sources) elem = NinjaBuildElement(self.all_outputs, depscan_file, rule_name, scan_sources) elem.add_item('picklefile', pickle_file) @@ -907,10 +909,11 @@ int dummy; # in practice pick up C++ and Fortran files. If some other language # requires scanning (possibly Java to deal with inner class files) # then add them here. + all_suffixes = set(compilers.lang_suffixes['cpp']) | set(compilers.lang_suffixes['fortran']) selected_sources = [] for source in compiled_sources: ext = os.path.splitext(source)[1][1:] - if ext in compilers.lang_suffixes['cpp']: + if ext in all_suffixes: selected_sources.append(source) return selected_sources @@ -1945,7 +1948,15 @@ int dummy; description = 'Compiling Swift source $in' self.add_rule(NinjaRule(rule, command, [], description)) - def generate_fortran_dep_hack(self, crstr): + def use_dyndeps_for_fortran(self) -> bool: + '''Use the new Ninja feature for scanning dependencies during build, + rather than up front. Remove this and all old scanning code once Ninja + minimum version is bumped to 1.10.''' + return mesonlib.version_compare(self.ninja_version, '>=1.10.0') + + def generate_fortran_dep_hack(self, crstr: str) -> None: + if self.use_dyndeps_for_fortran(): + return rule = 'FORTRAN_DEP_HACK{}'.format(crstr) if mesonlib.is_windows(): cmd = ['cmd', '/C'] @@ -2029,22 +2040,16 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) def generate_scanner_rules(self): - scanner_languages = {'cpp'} # Fixme, add Fortran. - for for_machine in MachineChoice: - clist = self.environment.coredata.compilers[for_machine] - for langname, compiler in clist.items(): - if langname not in scanner_languages: - continue - rulename = '{}scan'.format(langname) - if rulename in self.ruledict: - # Scanning command is the same for native and cross compilation. - continue - command = self.environment.get_build_command() + \ - ['--internal', 'depscan'] - args = ['$picklefile', '$out', '$in'] - description = 'Module scanner for {}.'.format(langname) - rule = NinjaRule(rulename, command, args, description) - self.add_rule(rule) + rulename = 'depscan' + if rulename in self.ruledict: + # Scanning command is the same for native and cross compilation. + return + command = self.environment.get_build_command() + \ + ['--internal', 'depscan'] + args = ['$picklefile', '$out', '$in'] + description = 'Module scanner.' + rule = NinjaRule(rulename, command, args, description) + self.add_rule(rule) def generate_compile_rules(self): @@ -2146,6 +2151,8 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) """ Find all module and submodule made available in a Fortran code file. """ + if self.use_dyndeps_for_fortran(): + return compiler = None # TODO other compilers for lang, c in self.environment.coredata.compilers.host.items(): @@ -2198,6 +2205,8 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) """ Find all module and submodule needed by a Fortran target """ + if self.use_dyndeps_for_fortran(): + return [] dirname = Path(self.get_target_private_dir(target)) tdeps = self.fortran_deps[target.get_basename()] @@ -2502,16 +2511,20 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) if not is_generated: abs_src = Path(build_dir) / rel_src extra_deps += self.get_fortran_deps(compiler, abs_src, target) - # Dependency hack. Remove once multiple outputs in Ninja is fixed: - # https://groups.google.com/forum/#!topic/ninja-build/j-2RfBIOd_8 - for modname, srcfile in self.fortran_deps[target.get_basename()].items(): - modfile = os.path.join(self.get_target_private_dir(target), - compiler.module_name_to_filename(modname)) - - if srcfile == src: - crstr = self.get_rule_suffix(target.for_machine) - depelem = NinjaBuildElement(self.all_outputs, modfile, 'FORTRAN_DEP_HACK' + crstr, rel_obj) - self.add_build(depelem) + if not self.use_dyndeps_for_fortran(): + # Dependency hack. Remove once multiple outputs in Ninja is fixed: + # https://groups.google.com/forum/#!topic/ninja-build/j-2RfBIOd_8 + for modname, srcfile in self.fortran_deps[target.get_basename()].items(): + modfile = os.path.join(self.get_target_private_dir(target), + compiler.module_name_to_filename(modname)) + + if srcfile == src: + crstr = self.get_rule_suffix(target.for_machine) + depelem = NinjaBuildElement(self.all_outputs, + modfile, + 'FORTRAN_DEP_HACK' + crstr, + rel_obj) + self.add_build(depelem) commands += compiler.get_module_outdir_args(self.get_target_private_dir(target)) element = NinjaBuildElement(self.all_outputs, rel_obj, compiler_name, rel_src) @@ -2537,7 +2550,7 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) return (rel_obj, rel_src.replace('\\', '/')) def add_dependency_scanner_entries_to_element(self, target, compiler, element): - if not self.should_scan_target(target): + if not self.should_use_dyndeps_for_target(target): return dep_scan_file = self.get_dep_scan_file_for(target) element.add_item('dyndep', dep_scan_file) diff --git a/mesonbuild/scripts/depscan.py b/mesonbuild/scripts/depscan.py index 6eebbfe6a..9fffb118d 100644 --- a/mesonbuild/scripts/depscan.py +++ b/mesonbuild/scripts/depscan.py @@ -15,12 +15,24 @@ import pathlib import pickle import re +import os +import sys import typing as T -from ..backend.ninjabackend import TargetDependencyScannerInfo +from ..backend.ninjabackend import TargetDependencyScannerInfo, ninja_quote +from ..compilers.compilers import lang_suffixes -import_re = re.compile('\w*import ([a-zA-Z0-9]+);') -export_re = re.compile('\w*export module ([a-zA-Z0-9]+);') +CPP_IMPORT_RE = re.compile('\w*import ([a-zA-Z0-9]+);') +CPP_EXPORT_RE = re.compile('\w*export module ([a-zA-Z0-9]+);') + +FORTRAN_INCLUDE_PAT = r"^\s*include\s*['\"](\w+\.\w+)['\"]" +FORTRAN_MODULE_PAT = r"^\s*\bmodule\b\s+(\w+)\s*(?:!+.*)*$" +FORTRAN_SUBMOD_PAT = r"^\s*\bsubmodule\b\s*\((\w+:?\w+)\)\s*(\w+)" +FORTRAN_USE_PAT = r"^\s*use,?\s*(?:non_intrinsic)?\s*(?:::)?\s*(\w+)" + +FORTRAN_MODULE_RE = re.compile(FORTRAN_MODULE_PAT, re.IGNORECASE) +FORTRAN_SUBMOD_RE = re.compile(FORTRAN_SUBMOD_PAT, re.IGNORECASE) +FORTRAN_USE_RE = re.compile(FORTRAN_USE_PAT, re.IGNORECASE) class DependencyScanner: def __init__(self, pickle_file: str, outfile: str, sources: T.List[str]): @@ -32,11 +44,73 @@ class DependencyScanner: self.exports = {} # type: T.Dict[str, str] self.needs = {} # type: T.Dict[str, T.List[str]] self.sources_with_exports = [] # type: T.List[str] - + def scan_file(self, fname: str) -> None: - for line in pathlib.Path(fname).read_text().split('\n'): - import_match = import_re.match(line) - export_match = export_re.match(line) + suffix = os.path.splitext(fname)[1][1:] + if suffix in lang_suffixes['fortran']: + self.scan_fortran_file(fname) + elif suffix in lang_suffixes['cpp']: + self.scan_cpp_file(fname) + else: + sys.exit('Can not scan files with suffix .{}.'.format(suffix)) + + def scan_fortran_file(self, fname: str) -> None: + fpath = pathlib.Path(fname) + modules_in_this_file = set() + for line in fpath.read_text().split('\n'): + import_match = FORTRAN_USE_RE.match(line) + export_match = FORTRAN_MODULE_RE.match(line) + submodule_export_match = FORTRAN_SUBMOD_RE.match(line) + if import_match: + needed = import_match.group(1).lower() + # In Fortran you have an using declaration also for the module + # you define in the same file. Prevent circular dependencies. + if needed not in modules_in_this_file: + if fname in self.needs: + self.needs[fname].append(needed) + else: + self.needs[fname] = [needed] + if export_match: + exported_module = export_match.group(1).lower() + assert(exported_module not in modules_in_this_file) + modules_in_this_file.add(exported_module) + if exported_module in self.provided_by: + raise RuntimeError('Multiple files provide module {}.'.format(exported_module)) + self.sources_with_exports.append(fname) + self.provided_by[exported_module] = fname + self.exports[fname] = exported_module + if submodule_export_match: + # Store submodule "Foo" "Bar" as "foo:bar". + # A submodule declaration can be both an import and an export declaration: + # + # submodule (a1:a2) a3 + # - requires a1@a2.smod + # - produces a1@a3.smod + parent_module_name_full = submodule_export_match.group(1).lower() + parent_module_name = parent_module_name_full.split(':')[0] + submodule_name = submodule_export_match.group(2).lower() + concat_name = '{}:{}'.format(parent_module_name, submodule_name) + self.sources_with_exports.append(fname) + self.provided_by[concat_name] = fname + self.exports[fname] = concat_name + # Fortran requires that the immediate parent module must be built + # before the current one. Thus: + # + # submodule (parent) parent <- requires parent.mod (really parent.smod, but they are created at the same time) + # submodule (a1:a2) a3 <- requires a1@a2.smod + # + # a3 does not depend on the a1 parent module directly, only transitively. + if fname in self.needs: + self.needs[fname].append(parent_module_name_full) + else: + self.needs[fname] = [parent_module_name_full] + + + def scan_cpp_file(self, fname: str) -> None: + fpath = pathlib.Path(fname) + for line in fpath.read_text().split('\n'): + import_match = CPP_IMPORT_RE.match(line) + export_match = CPP_EXPORT_RE.match(line) if import_match: needed = import_match.group(1) if fname in self.needs: @@ -56,8 +130,22 @@ class DependencyScanner: assert(isinstance(objname, str)) return objname - def ifcname_for(self, src: str) -> str: - return '{}.ifc'.format(self.exports[src]) + def module_name_for(self, src: str) -> str: + suffix= os.path.splitext(src)[1][1:] + if suffix in lang_suffixes['fortran']: + exported = self.exports[src] + # Module foo:bar goes to a file name foo@bar.smod + # Module Foo goes to a file name foo.mod + namebase = exported.replace(':', '@') + if ':' in exported: + extension = 'smod' + else: + extension = 'mod' + return os.path.join(self.target_data.private_dir, '{}.{}'.format(namebase, extension)) + elif suffix in lang_suffixes['cpp']: + return '{}.ifc'.format(self.exports[src]) + else: + raise RuntimeError('Unreachable code.') def scan(self) -> int: for s in self.sources: @@ -66,21 +154,43 @@ class DependencyScanner: ofile.write('ninja_dyndep_version = 1\n') for src in self.sources: objfilename = self.objname_for(src) + mods_and_submods_needed = [] + module_files_generated = [] + module_files_needed = [] if src in self.sources_with_exports: - ifc_entry = '| ' + self.ifcname_for(src) - else: - ifc_entry = '' + module_files_generated.append(self.module_name_for(src)) if src in self.needs: - # FIXME, handle all sources, not just the first one - modname = self.needs[src][0] + for modname in self.needs[src]: + if modname not in self.provided_by: + # Nothing provides this module, we assume that it + # comes from a dependency library somewhere and is + # already built by the time this complation starts. + pass + else: + mods_and_submods_needed.append(modname) + + for modname in mods_and_submods_needed: provider_src = self.provided_by[modname] - provider_ifc = self.ifcname_for(provider_src) - mod_dep = '| ' + provider_ifc + provider_modfile = self.module_name_for(provider_src) + # Prune self-dependencies + if provider_src != src: + module_files_needed.append(provider_modfile) + + quoted_objfilename = ninja_quote(objfilename, True) + quoted_module_files_generated = [ninja_quote(x, True) for x in module_files_generated] + quoted_module_files_needed = [ninja_quote(x, True) for x in module_files_needed] + if quoted_module_files_generated: + mod_gen = '| ' + ' '.join(quoted_module_files_generated) + else: + mod_gen = '' + if quoted_module_files_needed: + mod_dep = '| ' + ' '.join(quoted_module_files_needed) else: mod_dep = '' - ofile.write('build {} {}: dyndep {}\n'.format(objfilename, - ifc_entry, - mod_dep)) + build_line = 'build {} {}: dyndep {}'.format(quoted_objfilename, + mod_gen, + mod_dep) + ofile.write(build_line + '\n') return 0 def run(args: T.List[str]) -> int: