From 30c8a5713409cfd2ffd2f0197b94d8231a75b9a0 Mon Sep 17 00:00:00 2001 From: Jussi Pakkanen Date: Thu, 3 Aug 2017 14:34:24 +0300 Subject: [PATCH] Add option to limit maximum number of concurrent link processes. --- docs/markdown/Release-notes-for-0.42.0.md | 7 +++++ mesonbuild/backend/ninjabackend.py | 14 ++++++++- mesonbuild/coredata.py | 38 +++++++++++++++++++++++ mesonbuild/mconf.py | 13 ++++++++ mesonbuild/optinterpreter.py | 23 +++++++------- 5 files changed, 83 insertions(+), 12 deletions(-) diff --git a/docs/markdown/Release-notes-for-0.42.0.md b/docs/markdown/Release-notes-for-0.42.0.md index f0ea534c1..f093ac641 100644 --- a/docs/markdown/Release-notes-for-0.42.0.md +++ b/docs/markdown/Release-notes-for-0.42.0.md @@ -83,3 +83,10 @@ flag manually, e.g. via `link_args` to a target. This is not recommended because having multiple rpath causes them to stomp on each other. This warning will become a hard error in some future release. +## Limiting the maximum number of linker processes + +With the Ninja backend it is now possible to limit the maximum number of +concurrent linker processes. This is usually only needed for projects +that have many large link steps that cause the system to run out of +memory if they are run in parallel. This limit can be set with the +new `backend_max_links` option. diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 287b6836e..9137a7b45 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -842,6 +842,12 @@ int dummy; outfile.write('# Rules for compiling.\n\n') self.generate_compile_rules(outfile) outfile.write('# Rules for linking.\n\n') + num_pools = self.environment.coredata.backend_options['backend_max_links'].value + if num_pools > 0: + outfile.write('''pool link_pool + depth = %d + +''' % num_pools) if self.environment.is_cross_build(): self.generate_static_link_rules(True, outfile) self.generate_static_link_rules(False, outfile) @@ -1369,6 +1375,7 @@ int dummy; raise MesonException('Swift supports only executable and static library targets.') def generate_static_link_rules(self, is_cross, outfile): + num_pools = self.environment.coredata.backend_options['backend_max_links'].value if 'java' in self.build.compilers: if not is_cross: self.generate_java_link(outfile) @@ -1412,9 +1419,12 @@ int dummy; description = ' description = Linking static target $out.\n\n' outfile.write(rule) outfile.write(command) + if num_pools > 0: + outfile.write(' pool = link_pool\n') outfile.write(description) def generate_dynamic_link_rules(self, outfile): + num_pools = self.environment.coredata.backend_options['backend_max_links'].value ctypes = [(self.build.compilers, False)] if self.environment.is_cross_build(): if self.environment.cross_info.need_cross_compiler(): @@ -1452,9 +1462,11 @@ int dummy; cross_args=' '.join(cross_args), output_args=' '.join(compiler.get_linker_output_args('$out')) ) - description = ' description = Linking target $out.' + description = ' description = Linking target $out.\n' outfile.write(rule) outfile.write(command) + if num_pools > 0: + outfile.write(' pool = link_pool\n') outfile.write(description) outfile.write('\n') outfile.write('\n') diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index 80ad94f85..d39270b96 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -85,6 +85,36 @@ class UserBooleanOption(UserOption): def validate_value(self, value): return self.tobool(value) +class UserIntegerOption(UserOption): + def __init__(self, name, description, min_value, max_value, value): + super().__init__(name, description, [True, False]) + self.min_value = min_value + self.max_value = max_value + self.set_value(value) + + def set_value(self, newvalue): + if isinstance(newvalue, str): + newvalue = self.toint(newvalue) + if not isinstance(newvalue, int): + raise MesonException('New value for integer option is not an integer.') + if self.min_value is not None and newvalue < self.min_value: + raise MesonException('New value %d is less than minimum value %d.' % (newvalue, self.min_value)) + if self.max_value is not None and newvalue > self.max_value: + raise MesonException('New value %d is more than maximum value %d.' % (newvalue, self.max_value)) + self.value = newvalue + + def toint(self, valuestring): + try: + return int(valuestring) + except: + raise MesonException('Value string "%s" is not convertable to an integer.' % valuestring) + + def parse_string(self, valuestring): + return self.toint(valuestring) + + def validate_value(self, value): + return self.toint(value) + class UserComboOption(UserOption): def __init__(self, name, description, choices, value): super().__init__(name, description, choices) @@ -145,6 +175,7 @@ class CoreData: self.target_guids = {} self.version = version self.init_builtins(options) + self.init_backend_options(self.builtins['backend'].value) self.user_options = {} self.compiler_options = {} self.base_options = {} @@ -218,6 +249,13 @@ class CoreData: args = [key] + builtin_options[key][1:-1] + [value] self.builtins[key] = builtin_options[key][0](*args) + def init_backend_options(self, backend_name): + self.backend_options = {} + if backend_name == 'ninja': + self.backend_options['backend_max_links'] = UserIntegerOption('backend_max_links', + 'Maximum number of linker processes to run or 0 for no limit', + 0, None, 0) + def get_builtin_option(self, optname): if optname in self.builtins: return self.builtins[optname].value diff --git a/mesonbuild/mconf.py b/mesonbuild/mconf.py index 14eddf5e2..ee4dbf8fc 100644 --- a/mesonbuild/mconf.py +++ b/mesonbuild/mconf.py @@ -122,6 +122,9 @@ class Conf: (k, v) = o.split('=', 1) if coredata.is_builtin_option(k): self.coredata.set_builtin_option(k, v) + elif k in self.coredata.backend_options: + tgt = self.coredata.backend_options[k] + tgt.set_value(v) elif k in self.coredata.user_options: tgt = self.coredata.user_options[k] tgt.set_value(v) @@ -163,6 +166,16 @@ class Conf: 'choices': coredata.get_builtin_option_choices(key)}) self.print_aligned(carr) print('') + bekeys = sorted(self.coredata.backend_options.keys()) + if not bekeys: + print(' No backend options\n') + else: + bearr = [] + for k in bekeys: + o = self.coredata.backend_options[k] + bearr.append({'name': k, 'descr': o.description, 'value': o.value, 'choices': ''}) + self.print_aligned(bearr) + print('') print('Base options:') okeys = sorted(self.coredata.base_options.keys()) if not okeys: diff --git a/mesonbuild/optinterpreter.py b/mesonbuild/optinterpreter.py index f9e7f26e7..ac2726959 100644 --- a/mesonbuild/optinterpreter.py +++ b/mesonbuild/optinterpreter.py @@ -18,17 +18,18 @@ from . import mesonlib import os, re forbidden_option_names = coredata.get_builtin_options() -forbidden_prefixes = {'c_': True, - 'cpp_': True, - 'd_': True, - 'rust_': True, - 'fortran_': True, - 'objc_': True, - 'objcpp_': True, - 'vala_': True, - 'csharp_': True, - 'swift_': True, - 'b_': True, +forbidden_prefixes = {'c_', + 'cpp_', + 'd_', + 'rust_', + 'fortran_', + 'objc_', + 'objcpp_', + 'vala_', + 'csharp_', + 'swift_', + 'b_', + 'backend_', } def is_invalid_name(name):