diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index e11491fef..c95ee18ea 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -18,7 +18,7 @@ from .. import build from .. import mlog from .. import dependencies from .. import compilers -from ..mesonlib import File, MesonException, get_compiler_for_source +from ..mesonlib import File, MesonException, get_compiler_for_source, Popen_safe from .backends import InstallData from ..build import InvalidArguments import os, sys, pickle, re @@ -159,18 +159,14 @@ class NinjaBackend(backends.Backend): int dummy; ''') - pc = subprocess.Popen(['cl', '/showIncludes', '/c', 'incdetect.c'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=self.environment.get_scratch_dir()) + pc, stdo = Popen_safe(['cl', '/showIncludes', '/c', 'incdetect.c'], + cwd=self.environment.get_scratch_dir())[0:2] - (stdo, _) = pc.communicate() - - for line in stdo.split(b'\r\n'): - if line.endswith(b'stdio.h'): - matchstr = b':'.join(line.split(b':')[0:2]) + b':' - with open(tempfilename, 'ab') as binfile: - binfile.write(b'msvc_deps_prefix = ' + matchstr + b'\r\n') + for line in stdo.split('\n'): + if line.endswith('stdio.h'): + matchstr = ':'.join(line.split(':')[0:2]) + ':' + with open(tempfilename, 'a') as binfile: + binfile.write('msvc_deps_prefix = ' + matchstr + '\n') return open(tempfilename, 'a') raise MesonException('Could not determine vs dep dependency prefix string.') diff --git a/mesonbuild/compilers.py b/mesonbuild/compilers.py index 7b9addee2..2ee4aac62 100644 --- a/mesonbuild/compilers.py +++ b/mesonbuild/compilers.py @@ -18,7 +18,7 @@ import subprocess, os.path import tempfile from .import mesonlib from . import mlog -from .mesonlib import MesonException, version_compare +from .mesonlib import MesonException, version_compare, Popen_safe from . import coredata """This file contains the data files of all compilers Meson knows @@ -457,12 +457,7 @@ class Compiler(): mlog.debug('Working directory: ', tmpdirname) mlog.debug('Command line: ', ' '.join(commands), '\n') mlog.debug('Code:\n', code) - p = subprocess.Popen(commands, cwd=tmpdirname, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - (stde, stdo) = p.communicate() - stde = stde.decode() - stdo = stdo.decode() + p, stdo, stde = Popen_safe(commands, cwd=tmpdirname) mlog.debug('Compiler stdout:\n', stdo) mlog.debug('Compiler stderr:\n', stde) @@ -600,9 +595,7 @@ class CCompiler(Compiler): return ['-shared'] def get_library_dirs(self): - output = subprocess.Popen(self.exelist + ['--print-search-dirs'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) - (stdo, _) = output.communicate() - stdo = stdo.decode('utf-8') + stdo = Popen_safe(self.exelist + ['--print-search-dirs'])[1] for line in stdo.split('\n'): if line.startswith('libraries:'): libstr = line.split('=', 1)[1] @@ -659,10 +652,7 @@ class CCompiler(Compiler): ofile.write(code) # Compile sanity check cmdlist = self.exelist + extra_flags + [source_name] + self.get_output_args(binary_name) - pc = subprocess.Popen(cmdlist, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=work_dir) - (stdo, stde) = pc.communicate() - stdo = stdo.decode() - stde = stde.decode() + pc, stdo, stde = Popen_safe(cmdlist, cwd=work_dir) mlog.debug('Sanity check compiler command line:', ' '.join(cmdlist)) mlog.debug('Sanity check compile stdout:') mlog.debug(stdo) @@ -806,15 +796,11 @@ int main () {{ else: cmdlist = p.output_name try: - pe = subprocess.Popen(cmdlist, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + pe, so, se = Popen_safe(cmdlist) except Exception as e: mlog.debug('Could not run: %s (error: %s)\n' % (cmdlist, e)) return RunResult(False) - (so, se) = pe.communicate() - so = so.decode() - se = se.decode() mlog.debug('Program stdout:\n') mlog.debug(so) mlog.debug('Program stderr:\n') @@ -1931,7 +1917,7 @@ class VisualStudioCCompiler(CCompiler): # understand and you can't tell it to error out on those. # http://stackoverflow.com/questions/15259720/how-can-i-make-the-microsoft-c-compiler-treat-unknown-flags-as-errors-rather-t def has_argument(self, arg, env): - warning_text = b'9002' + warning_text = '9002' code = 'int i;\n' (fd, srcname) = tempfile.mkstemp(suffix='.'+self.default_suffix) os.close(fd) @@ -1944,8 +1930,7 @@ class VisualStudioCCompiler(CCompiler): mlog.debug('Running VS compile:') mlog.debug('Command line: ', ' '.join(commands)) mlog.debug('Code:\n', code) - p = subprocess.Popen(commands, cwd=os.path.split(srcname)[0], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stde, stdo) = p.communicate() + p, stdo, stde = Popen_safe(commands, cwd=os.path.split(srcname)[0]) if p.returncode != 0: raise MesonException('Compiling test app failed.') return not(warning_text in stde or warning_text in stdo) @@ -2614,10 +2599,9 @@ class ArLinker(): def __init__(self, exelist): self.exelist = exelist self.id = 'ar' - pc = subprocess.Popen(self.exelist + ['-h'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) - (stdo, _) = pc.communicate() + pc, stdo = Popen_safe(self.exelist + ['-h'])[0:2] # Enable deterministic builds if they are available. - if b'[D]' in stdo: + if '[D]' in stdo: self.std_args = ['csrD'] else: self.std_args = ['csr'] diff --git a/mesonbuild/dependencies.py b/mesonbuild/dependencies.py index 82d1b750d..606719c42 100644 --- a/mesonbuild/dependencies.py +++ b/mesonbuild/dependencies.py @@ -23,7 +23,7 @@ import re import os, stat, glob, subprocess, shutil import sysconfig from collections import OrderedDict -from . mesonlib import MesonException, version_compare, version_compare_many +from . mesonlib import MesonException, version_compare, version_compare_many, Popen_safe from . import mlog from . import mesonlib from .environment import detect_cpu_family, for_windows @@ -170,17 +170,14 @@ class PkgConfigDependency(Dependency): self._set_libs() def _call_pkgbin(self, args): - p = subprocess.Popen([self.pkgbin] + args, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=os.environ, universal_newlines=True) - out = p.communicate()[0] + p, out = Popen_safe([self.pkgbin] + args, env=os.environ)[0:2] return (p.returncode, out.strip()) def _set_cargs(self): ret, out = self._call_pkgbin(['--cflags', self.name]) if ret != 0: raise DependencyException('Could not generate cargs for %s:\n\n%s' % \ - (self.name, out.decode(errors='ignore'))) + (self.name, out)) self.cargs = out.split() def _set_libs(self): @@ -190,7 +187,7 @@ class PkgConfigDependency(Dependency): ret, out = self._call_pkgbin(libcmd) if ret != 0: raise DependencyException('Could not generate libs for %s:\n\n%s' % \ - (self.name, out.decode(errors='ignore'))) + (self.name, out)) self.libs = [] for lib in out.split(): if lib.endswith(".la"): @@ -238,13 +235,11 @@ class PkgConfigDependency(Dependency): pkgbin = os.environ[evar].strip() else: pkgbin = 'pkg-config' - p = subprocess.Popen([pkgbin, '--version'], stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out = p.communicate()[0] + p, out = Popen_safe([pkgbin, '--version'])[0:2] if p.returncode == 0: if not self.silent: mlog.log('Found pkg-config:', mlog.bold(shutil.which(pkgbin)), - '(%s)' % out.decode().strip()) + '(%s)' % out.strip()) PkgConfigDependency.pkgconfig_found = True return except (FileNotFoundError, PermissionError): @@ -303,16 +298,13 @@ class WxDependency(Dependency): mlog.log("Neither wx-config-3.0 nor wx-config found; can't detect dependency") return - p = subprocess.Popen([self.wxc, '--version'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out = p.communicate()[0] + p, out = Popen_safe([self.wxc, '--version'])[0:2] if p.returncode != 0: mlog.log('Dependency wxwidgets found:', mlog.red('NO')) self.cargs = [] self.libs = [] else: - self.modversion = out.decode().strip() + self.modversion = out.strip() version_req = kwargs.get('version', None) if version_req is not None: if not version_compare(self.modversion, version_req, strict=True): @@ -324,20 +316,15 @@ class WxDependency(Dependency): self.requested_modules = self.get_requested(kwargs) # wx-config seems to have a cflags as well but since it requires C++, # this should be good, at least for now. - p = subprocess.Popen([self.wxc, '--cxxflags'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out = p.communicate()[0] + p, out = Popen_safe([self.wxc, '--cxxflags'])[0:2] if p.returncode != 0: raise DependencyException('Could not generate cargs for wxwidgets.') - self.cargs = out.decode().split() + self.cargs = out.split() - p = subprocess.Popen([self.wxc, '--libs'] + self.requested_modules, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out = p.communicate()[0] + p, out = Popen_safe([self.wxc, '--libs'] + self.requested_modules)[0:2] if p.returncode != 0: raise DependencyException('Could not generate libs for wxwidgets.') - self.libs = out.decode().split() + self.libs = out.split() def get_requested(self, kwargs): modules = 'modules' @@ -363,12 +350,10 @@ class WxDependency(Dependency): def check_wxconfig(self): for wxc in ['wx-config-3.0', 'wx-config']: try: - p = subprocess.Popen([wxc, '--version'], stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out = p.communicate()[0] + p, out = Popen_safe([wxc, '--version'])[0:2] if p.returncode == 0: mlog.log('Found wx-config:', mlog.bold(shutil.which(wxc)), - '(%s)' % out.decode().strip()) + '(%s)' % out.strip()) self.wxc = wxc WxDependency.wx_found = True return @@ -943,10 +928,7 @@ class QtBaseDependency(Dependency): if not self.qmake.found(): continue # Check that the qmake is for qt5 - pc = subprocess.Popen(self.qmake.fullpath + ['-v'], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True) - stdo = pc.communicate()[0] + pc, stdo = Popen_safe(self.qmake.fullpath + ['-v'])[0:2] if pc.returncode != 0: continue if not 'Qt version ' + self.qtver in stdo: @@ -959,9 +941,7 @@ class QtBaseDependency(Dependency): return self.version = re.search(self.qtver + '(\.\d+)+', stdo).group(0) # Query library path, header path, and binary path - stdo = subprocess.Popen(self.qmake.fullpath + ['-query'], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True).communicate()[0] + stdo = Popen_safe(self.qmake.fullpath + ['-query'])[1] qvars = {} for line in stdo.split('\n'): line = line.strip() @@ -1051,9 +1031,7 @@ class GnuStepDependency(Dependency): def detect(self): confprog = 'gnustep-config' try: - gp = subprocess.Popen([confprog, '--help'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - gp.communicate() + gp = Popen_safe([confprog, '--help'])[0] except (FileNotFoundError, PermissionError): self.args = None mlog.log('Dependency GnuStep found:', mlog.red('NO'), '(no gnustep-config)') @@ -1066,20 +1044,12 @@ class GnuStepDependency(Dependency): arg = '--gui-libs' else: arg = '--base-libs' - fp = subprocess.Popen([confprog, '--objc-flags'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (flagtxt, flagerr) = fp.communicate() - flagtxt = flagtxt.decode() - flagerr = flagerr.decode() + fp, flagtxt, flagerr = Popen_safe([confprog, '--objc-flags']) if fp.returncode != 0: raise DependencyException('Error getting objc-args: %s %s' % (flagtxt, flagerr)) args = flagtxt.split() self.args = self.filter_arsg(args) - fp = subprocess.Popen([confprog, arg], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (libtxt, liberr) = fp.communicate() - libtxt = libtxt.decode() - liberr = liberr.decode() + fp, libtxt, liberr = Popen_safe([confprog, arg]) if fp.returncode != 0: raise DependencyException('Error getting objc-lib args: %s %s' % (libtxt, liberr)) self.libs = self.weird_filter(libtxt.split()) @@ -1184,16 +1154,10 @@ class SDL2Dependency(Dependency): pass sdlconf = shutil.which('sdl2-config') if sdlconf: - pc = subprocess.Popen(['sdl2-config', '--cflags'], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL) - (stdo, _) = pc.communicate() - self.cargs = stdo.decode().strip().split() - pc = subprocess.Popen(['sdl2-config', '--libs'], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL) - (stdo, _) = pc.communicate() - self.linkargs = stdo.decode().strip().split() + pc, stdo = Popen_safe(['sdl2-config', '--cflags'])[0:2] + self.cargs = stdo.strip().split() + pc, stdo = Popen_safe(['sdl2-config', '--libs'])[0:2] + self.linkargs = stdo.strip().split() self.is_found = True mlog.log('Dependency', mlog.bold('sdl2'), 'found:', mlog.green('YES'), '(%s)' % sdlconf) self.version = '2' # FIXME diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 098f8ca2f..e673d506f 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -17,6 +17,7 @@ from . import coredata from . import mesonlib from . import mlog from .compilers import * +from .mesonlib import Popen_safe import configparser import shutil @@ -42,11 +43,10 @@ def find_coverage_tools(): def detect_ninja(): for n in ['ninja', 'ninja-build']: try: - p = subprocess.Popen([n, '--version'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + p, version = Popen_safe([n, '--version'])[0:2] except (FileNotFoundError, PermissionError): # Doesn't exist in PATH or isn't executable continue - version = p.communicate()[0].decode(errors='ignore') # Perhaps we should add a way for the caller to know the failure mode # (not found or too old) if p.returncode == 0 and mesonlib.version_compare(version, ">=1.6"): @@ -306,9 +306,7 @@ class Environment(): # Arguments to output compiler pre-processor defines to stdout # gcc, g++, and gfortran all support these arguments args = compiler + ['-E', '-dM', '-'] - p = subprocess.Popen(args, universal_newlines=True, - stdin=subprocess.PIPE, stdout=subprocess.PIPE) - output = p.communicate('')[0] + p, output = Popen_safe(args, write='', stdin=subprocess.PIPE)[0:2] if p.returncode != 0: raise EnvironmentException('Unable to detect GNU compiler type:\n' + output) # Parse several lines of the type: @@ -372,14 +370,10 @@ class Environment(): arg = '/?' else: arg = '--version' - p = subprocess.Popen([compiler, arg], stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p, out, err = Popen_safe([compiler, arg]) except OSError as e: popen_exceptions[' '.join([compiler, arg])] = e continue - (out, err) = p.communicate() - out = out.decode(errors='ignore') - err = err.decode(errors='ignore') version = search_version(out) if 'Free Software Foundation' in out: defines = self.get_gnu_compiler_defines([compiler]) @@ -428,15 +422,10 @@ class Environment(): for compiler in compilers: for arg in ['--version', '-V']: try: - p = subprocess.Popen([compiler, arg], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p, out, err = Popen_safe([compiler, arg]) except OSError as e: popen_exceptions[' '.join([compiler, arg])] = e continue - (out, err) = p.communicate() - out = out.decode(errors='ignore') - err = err.decode(errors='ignore') version = search_version(out) @@ -512,15 +501,10 @@ class Environment(): else: arg = '--version' try: - p = subprocess.Popen([compiler, arg], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p, out, err = Popen_safe([compiler, arg]) except OSError as e: popen_exceptions[' '.join([compiler, arg])] = e continue - (out, err) = p.communicate() - out = out.decode(errors='ignore') - err = err.decode(errors='ignore') version = search_version(out) if 'Free Software Foundation' in out: defines = self.get_gnu_compiler_defines([compiler]) @@ -559,12 +543,9 @@ class Environment(): is_cross = False exe_wrap = None try: - p = subprocess.Popen(exelist + ['--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p, out, err = Popen_safe(exelist + ['--version']) except OSError: raise EnvironmentException('Could not execute ObjC compiler "%s"' % ' '.join(exelist)) - (out, err) = p.communicate() - out = out.decode(errors='ignore') - err = err.decode(errors='ignore') version = search_version(out) if 'Free Software Foundation' in out: defines = self.get_gnu_compiler_defines(exelist) @@ -587,12 +568,9 @@ class Environment(): is_cross = False exe_wrap = None try: - p = subprocess.Popen(exelist + ['--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p, out, err = Popen_safe(exelist + ['--version']) except OSError: raise EnvironmentException('Could not execute ObjC++ compiler "%s"' % ' '.join(exelist)) - (out, err) = p.communicate() - out = out.decode(errors='ignore') - err = err.decode(errors='ignore') version = search_version(out) if 'Free Software Foundation' in out: defines = self.get_gnu_compiler_defines(exelist) @@ -604,12 +582,9 @@ class Environment(): def detect_java_compiler(self): exelist = ['javac'] try: - p = subprocess.Popen(exelist + ['-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p, out, err = Popen_safe(exelist + ['-version']) except OSError: raise EnvironmentException('Could not execute Java compiler "%s"' % ' '.join(exelist)) - (out, err) = p.communicate() - out = out.decode(errors='ignore') - err = err.decode(errors='ignore') version = search_version(err) if 'javac' in err: return JavaCompiler(exelist, version) @@ -618,12 +593,9 @@ class Environment(): def detect_cs_compiler(self): exelist = ['mcs'] try: - p = subprocess.Popen(exelist + ['--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p, out, err = Popen_safe(exelist + ['--version']) except OSError: raise EnvironmentException('Could not execute C# compiler "%s"' % ' '.join(exelist)) - (out, err) = p.communicate() - out = out.decode(errors='ignore') - err = err.decode(errors='ignore') version = search_version(out) if 'Mono' in out: return MonoCompiler(exelist, version) @@ -632,11 +604,9 @@ class Environment(): def detect_vala_compiler(self): exelist = ['valac'] try: - p = subprocess.Popen(exelist + ['--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p, out = Popen_safe(exelist + ['--version'])[0:2] except OSError: raise EnvironmentException('Could not execute Vala compiler "%s"' % ' '.join(exelist)) - (out, _) = p.communicate() - out = out.decode(errors='ignore') version = search_version(out) if 'Vala' in out: return ValaCompiler(exelist, version) @@ -645,11 +615,9 @@ class Environment(): def detect_rust_compiler(self): exelist = ['rustc'] try: - p = subprocess.Popen(exelist + ['--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p, out = Popen_safe(exelist + ['--version'])[0:2] except OSError: raise EnvironmentException('Could not execute Rust compiler "%s"' % ' '.join(exelist)) - (out, _) = p.communicate() - out = out.decode(errors='ignore') version = search_version(out) if 'rustc' in out: return RustCompiler(exelist, version) @@ -679,11 +647,9 @@ class Environment(): raise EnvironmentException('Could not find any supported D compiler.') try: - p = subprocess.Popen(exelist + ['--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p, out = Popen_safe(exelist + ['--version'])[0:2] except OSError: raise EnvironmentException('Could not execute D compiler "%s"' % ' '.join(exelist)) - (out, _) = p.communicate() - out = out.decode(errors='ignore') version = search_version(out) if 'LLVM D compiler' in out: return LLVMDCompiler(exelist, version, is_cross) @@ -696,11 +662,9 @@ class Environment(): def detect_swift_compiler(self): exelist = ['swiftc'] try: - p = subprocess.Popen(exelist + ['-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p, _, err = Popen_safe(exelist + ['-v']) except OSError: raise EnvironmentException('Could not execute Swift compiler "%s"' % ' '.join(exelist)) - (_, err) = p.communicate() - err = err.decode(errors='ignore') version = search_version(err) if 'Swift' in err: return SwiftCompiler(exelist, version) @@ -723,12 +687,9 @@ class Environment(): else: arg = '--version' try: - p = subprocess.Popen([linker, arg], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p, out, err = Popen_safe([linker, arg]) except OSError: raise EnvironmentException('Could not execute static linker "%s".' % linker) - (out, err) = p.communicate() - out = out.decode(errors='ignore') - err = err.decode(errors='ignore') if '/OUT:' in out or '/OUT:' in err: return VisualStudioLinker([linker]) if p.returncode == 0: diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index 3d4f092a3..945430259 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -22,6 +22,7 @@ from . import optinterpreter from . import compilers from .wrap import wrap from . import mesonlib +from .mesonlib import Popen_safe from .dependencies import InternalDependency, Dependency from .interpreterbase import InterpreterBase from .interpreterbase import check_stringlist, noPosargs, noKwargs, stringArgs @@ -70,17 +71,8 @@ class RunProcess(InterpreterObject): def __init__(self, command_array, source_dir, build_dir, subdir, in_builddir=False): super().__init__() - pc = self.run_command(command_array, source_dir, build_dir, subdir, in_builddir) - (stdout, stderr) = pc.communicate() + pc, self.stdout, self.stderr = self.run_command(command_array, source_dir, build_dir, subdir, in_builddir) self.returncode = pc.returncode - if sys.stdout.encoding: - self.stdout = stdout.decode(encoding=sys.stdout.encoding, errors='ignore').replace('\r\n', '\n') - else: - self.stdout = stdout.decode(errors='ignore').replace('\r\n', '\n') - if sys.stderr.encoding: - self.stderr = stderr.decode(encoding=sys.stderr.encoding, errors='ignore').replace('\r\n', '\n') - else: - self.stderr = stderr.decode(errors='ignore').replace('\r\n', '\n') self.methods.update({'returncode' : self.returncode_method, 'stdout' : self.stdout_method, 'stderr' : self.stderr_method, @@ -99,22 +91,19 @@ class RunProcess(InterpreterObject): child_env.update(env) mlog.debug('Running command:', ' '.join(command_array)) try: - return subprocess.Popen(command_array, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=child_env, cwd=cwd) + return Popen_safe(command_array, env=child_env, cwd=cwd) except FileNotFoundError: pass # Was not a command, is a program in path? exe = shutil.which(cmd_name) if exe is not None: command_array = [exe] + command_array[1:] - return subprocess.Popen(command_array, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=child_env, cwd=cwd) + return Popen_safe(command_array, env=child_env, cwd=cwd) # No? Maybe it is a script in the source tree. fullpath = os.path.join(source_dir, subdir, cmd_name) command_array = [fullpath] + command_array[1:] try: - return subprocess.Popen(command_array, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=child_env, cwd=cwd) + return Popen_safe(command_array, env=child_env, cwd=cwd) except FileNotFoundError: raise InterpreterException('Could not execute command "%s".' % cmd_name) diff --git a/mesonbuild/mesonlib.py b/mesonbuild/mesonlib.py index 4670685f3..622cd04bf 100644 --- a/mesonbuild/mesonlib.py +++ b/mesonbuild/mesonlib.py @@ -385,3 +385,10 @@ def expand_arguments(args): print(e) return None return expended_args + +def Popen_safe(args, write=None, stderr=subprocess.PIPE, **kwargs): + p = subprocess.Popen(args, universal_newlines=True, + stdout=subprocess.PIPE, + stderr=stderr, **kwargs) + o, e = p.communicate(write) + return (p, o, e) diff --git a/mesonbuild/modules/gnome.py b/mesonbuild/modules/gnome.py index 5751406ee..caf162fc1 100644 --- a/mesonbuild/modules/gnome.py +++ b/mesonbuild/modules/gnome.py @@ -20,7 +20,7 @@ import os import sys import copy import subprocess -from ..mesonlib import MesonException +from ..mesonlib import MesonException, Popen_safe from .. import dependencies from .. import mlog from .. import mesonlib @@ -197,9 +197,7 @@ can not be used with the current version of glib-compiled-resources, due to cmd += ['--sourcedir', os.path.join(state.subdir, source_dir)] cmd += ['--sourcedir', state.subdir] # Current dir - pc = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True, - cwd=state.environment.get_source_dir()) - (stdout, _) = pc.communicate() + pc, stdout = Popen_safe(cmd, cwd=state.environment.get_source_dir())[0:2] if pc.returncode != 0: mlog.warning('glib-compile-resources has failed to get the dependencies for {}'.format(cmd[1])) raise subprocess.CalledProcessError(pc.returncode, cmd) diff --git a/mesonbuild/modules/qt4.py b/mesonbuild/modules/qt4.py index 5108baae7..ab285fb6b 100644 --- a/mesonbuild/modules/qt4.py +++ b/mesonbuild/modules/qt4.py @@ -15,7 +15,7 @@ import os, subprocess from .. import mlog from .. import build -from ..mesonlib import MesonException +from ..mesonlib import MesonException, Popen_safe from ..dependencies import Qt4Dependency import xml.etree.ElementTree as ET @@ -37,11 +37,9 @@ class Qt4Module(): # Moc and rcc return a non-zero result when doing so. # What kind of an idiot thought that was a good idea? if self.moc.found(): - mp = subprocess.Popen(self.moc.get_command() + ['-v'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = mp.communicate() - stdout = stdout.decode().strip() - stderr = stderr.decode().strip() + stdout, stderr = Popen_safe(self.moc.get_command() + ['-v'])[1:3] + stdout = stdout.strip() + stderr = stderr.strip() if 'Qt Meta' in stderr: moc_ver = stderr else: @@ -52,11 +50,9 @@ class Qt4Module(): else: mlog.log(' moc:', mlog.red('NO')) if self.uic.found(): - up = subprocess.Popen(self.uic.get_command() + ['-v'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = up.communicate() - stdout = stdout.decode().strip() - stderr = stderr.decode().strip() + stdout, stderr = Popen_safe(self.uic.get_command() + ['-v'])[1:3] + stdout = stdout.strip() + stderr = stderr.strip() if 'version 4.' in stderr: uic_ver = stderr else: @@ -67,11 +63,9 @@ class Qt4Module(): else: mlog.log(' uic:', mlog.red('NO')) if self.rcc.found(): - rp = subprocess.Popen(self.rcc.get_command() + ['-v'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = rp.communicate() - stdout = stdout.decode().strip() - stderr = stderr.decode().strip() + stdout, stderr = Popen_safe(self.rcc.get_command() + ['-v'])[1:3] + stdout = stdout.strip() + stderr = stderr.strip() if 'version 4.' in stderr: rcc_ver = stderr else: diff --git a/mesonbuild/modules/qt5.py b/mesonbuild/modules/qt5.py index 52ad155a4..56c626964 100644 --- a/mesonbuild/modules/qt5.py +++ b/mesonbuild/modules/qt5.py @@ -15,7 +15,7 @@ import os, subprocess from .. import mlog from .. import build -from ..mesonlib import MesonException +from ..mesonlib import MesonException, Popen_safe from ..dependencies import Qt5Dependency import xml.etree.ElementTree as ET @@ -37,11 +37,9 @@ class Qt5Module(): # Moc and rcc return a non-zero result when doing so. # What kind of an idiot thought that was a good idea? if self.moc.found(): - mp = subprocess.Popen(self.moc.get_command() + ['-v'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = mp.communicate() - stdout = stdout.decode().strip() - stderr = stderr.decode().strip() + stdout, stderr = Popen_safe(self.moc.get_command() + ['-v'])[1:3] + stdout = stdout.strip() + stderr = stderr.strip() if 'Qt 5' in stderr: moc_ver = stderr elif '5.' in stdout: @@ -54,11 +52,9 @@ class Qt5Module(): else: mlog.log(' moc:', mlog.red('NO')) if self.uic.found(): - up = subprocess.Popen(self.uic.get_command() + ['-v'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = up.communicate() - stdout = stdout.decode().strip() - stderr = stderr.decode().strip() + stdout, stderr = Popen_safe(self.uic.get_command() + ['-v'])[1:3] + stdout = stdout.strip() + stderr = stderr.strip() if 'version 5.' in stderr: uic_ver = stderr elif '5.' in stdout: @@ -71,11 +67,9 @@ class Qt5Module(): else: mlog.log(' uic:', mlog.red('NO')) if self.rcc.found(): - rp = subprocess.Popen(self.rcc.get_command() + ['-v'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = rp.communicate() - stdout = stdout.decode().strip() - stderr = stderr.decode().strip() + stdout, stderr = Popen_safe(self.rcc.get_command() + ['-v'])[1:3] + stdout = stdout.strip() + stderr = stderr.strip() if 'version 5.' in stderr: rcc_ver = stderr elif '5.' in stdout: diff --git a/mesonbuild/scripts/gtkdochelper.py b/mesonbuild/scripts/gtkdochelper.py index 14de48615..0cfd6446a 100755 --- a/mesonbuild/scripts/gtkdochelper.py +++ b/mesonbuild/scripts/gtkdochelper.py @@ -17,7 +17,7 @@ import sys, os import subprocess import shutil import argparse -from ..mesonlib import MesonException +from ..mesonlib import MesonException, Popen_safe from . import destdir_join parser = argparse.ArgumentParser() @@ -46,15 +46,13 @@ parser.add_argument('--mode', dest='mode', default='') parser.add_argument('--installdir', dest='install_dir') def gtkdoc_run_check(cmd, cwd): - p = subprocess.Popen(cmd, cwd=cwd, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stde, stdo) = p.communicate() + # Put stderr into stdout since we want to print it out anyway. + # This preserves the order of messages. + p, out = Popen_safe(cmd, cwd=cwd, stderr=subprocess.STDOUT)[0:2] if p.returncode != 0: err_msg = ["{!r} failed with status {:d}".format(cmd[0], p.returncode)] - if stde: - err_msg.append(stde.decode(errors='ignore')) - if stdo: - err_msg.append(stdo.decode(errors='ignore')) + if out: + err_msg.append(out) raise MesonException('\n'.join(err_msg)) def build_gtkdoc(source_root, build_root, doc_subdir, src_subdirs, diff --git a/mesonbuild/scripts/meson_exe.py b/mesonbuild/scripts/meson_exe.py index 3ea392690..d412e010a 100755 --- a/mesonbuild/scripts/meson_exe.py +++ b/mesonbuild/scripts/meson_exe.py @@ -21,7 +21,7 @@ import pickle import platform import subprocess -import mesonbuild +from ..mesonlib import MesonException, Popen_safe options = None @@ -45,7 +45,7 @@ def run_exe(exe): else: if exe.is_cross: if exe.exe_runner is None: - raise Exception('BUG: Trying to run cross-compiled exes with no wrapper') + raise AssertionError('BUG: Trying to run cross-compiled exes with no wrapper') else: cmd = [exe.exe_runner] + exe.fname else: @@ -55,17 +55,12 @@ def run_exe(exe): if len(exe.extra_paths) > 0: child_env['PATH'] = (os.pathsep.join(exe.extra_paths + ['']) + child_env['PATH']) - p = subprocess.Popen(cmd + exe.cmd_args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=child_env, - cwd=exe.workdir) - stdout, stderr = p.communicate() + p, stdout, stderr = Popen_safe(cmd + exe.cmd_args, env=child_env, cwd=exe.workdir) if exe.capture and p.returncode == 0: - with open(exe.capture, 'wb') as output: + with open(exe.capture, 'w') as output: output.write(stdout) if stderr: - sys.stderr.buffer.write(stderr) + sys.stderr.write(stderr) return p.returncode def run(args): diff --git a/mesonbuild/scripts/meson_install.py b/mesonbuild/scripts/meson_install.py index 14539e090..3d220223a 100755 --- a/mesonbuild/scripts/meson_install.py +++ b/mesonbuild/scripts/meson_install.py @@ -18,6 +18,7 @@ import sys, pickle, os, shutil, subprocess, gzip, platform from glob import glob from . import depfixer from . import destdir_join +from ..mesonlib import MesonException, Popen_safe install_log_file = None @@ -205,12 +206,11 @@ def install_targets(d): do_copy(fname, outname) if should_strip: print('Stripping target') - ps = subprocess.Popen(['strip', outname], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdo, stde) = ps.communicate() + ps, stdo, stde = Popen_safe(['strip', outname]) if ps.returncode != 0: print('Could not strip file.\n') - print('Stdout:\n%s\n' % stdo.decode()) - print('Stderr:\n%s\n' % stde.decode()) + print('Stdout:\n%s\n' % stdo) + print('Stderr:\n%s\n' % stde) sys.exit(1) printed_symlink_error = False for alias in aliases: diff --git a/mesonbuild/scripts/symbolextractor.py b/mesonbuild/scripts/symbolextractor.py index 9d2802867..2df3a0c4d 100755 --- a/mesonbuild/scripts/symbolextractor.py +++ b/mesonbuild/scripts/symbolextractor.py @@ -23,7 +23,8 @@ # http://cgit.freedesktop.org/libreoffice/core/commit/?id=3213cd54b76bc80a6f0516aac75a48ff3b2ad67c import os, sys, subprocess -from mesonbuild import mesonlib +from .. import mesonlib +from ..mesonlib import MesonException, Popen_safe import argparse parser = argparse.ArgumentParser() @@ -59,23 +60,21 @@ def linux_syms(libfilename, outfilename): nmbin = os.environ[evar].strip() else: nmbin = 'nm' - pe = subprocess.Popen([readelfbin, '-d', libfilename], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output = pe.communicate()[0].decode() + pe, output = Popen_safe([readelfbin, '-d', libfilename])[0:2] if pe.returncode != 0: raise RuntimeError('Readelf does not work') result = [x for x in output.split('\n') if 'SONAME' in x] assert(len(result) <= 1) - pnm = subprocess.Popen([nmbin, '--dynamic', '--extern-only', '--defined-only', '--format=posix', libfilename], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output = pnm.communicate()[0].decode() + pnm, output = Popen_safe([nmbin, '--dynamic', '--extern-only', + '--defined-only', '--format=posix', + libfilename])[0:2] if pnm.returncode != 0: raise RuntimeError('nm does not work.') result += [' '.join(x.split()[0:2]) for x in output.split('\n') if len(x) > 0] write_if_changed('\n'.join(result) + '\n', outfilename) def osx_syms(libfilename, outfilename): - pe = subprocess.Popen(['otool', '-l', libfilename], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output = pe.communicate()[0].decode() + pe, output = Popen_safe(['otool', '-l', libfilename])[0:2] if pe.returncode != 0: raise RuntimeError('Otool does not work.') arr = output.split('\n') @@ -84,8 +83,7 @@ def osx_syms(libfilename, outfilename): match = i break result = [arr[match+2], arr[match+5]] # Libreoffice stores all 5 lines but the others seem irrelevant. - pnm = subprocess.Popen(['nm', '-g', '-P', libfilename], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output = pnm.communicate()[0].decode() + pnm, output = Popen_safe(['nm', '-g', '-P', libfilename])[0:2] if pnm.returncode != 0: raise RuntimeError('nm does not work.') result += [' '.join(x.split()[0:2]) for x in output.split('\n') if len(x) > 0 and not x.endswith('U')]