From 8ba88892d7cb4341de80c25884da6bc7c749faaa Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Wed, 17 Jan 2018 16:35:44 +0300 Subject: [PATCH] run.py: simplified scripts, fixed most of PEP8 warnings --- modules/ts/misc/run.py | 81 ++++---- modules/ts/misc/run_android.py | 67 ++----- modules/ts/misc/run_long.py | 9 +- modules/ts/misc/run_suite.py | 69 ++----- modules/ts/misc/run_utils.py | 342 ++++----------------------------- 5 files changed, 121 insertions(+), 447 deletions(-) diff --git a/modules/ts/misc/run.py b/modules/ts/misc/run.py index 8f5f449e94..d25704bfed 100755 --- a/modules/ts/misc/run.py +++ b/modules/ts/misc/run.py @@ -1,8 +1,9 @@ #!/usr/bin/env python -import os, sys +import os import argparse import logging +import datetime from run_utils import Err, CMakeCache, log, execute from run_suite import TestSuite from run_android import AndroidTestSuite @@ -21,13 +22,12 @@ if __name__ == "__main__": description='OpenCV test runner script', epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("build_path", nargs = "*", default = ["."], help="Path to build directory (should contain CMakeCache.txt, default is current) or to directory with tests (all platform checks will be disabled in this case)") + parser.add_argument("build_path", nargs='?', default=".", help="Path to build directory (should contain CMakeCache.txt, default is current) or to directory with tests (all platform checks will be disabled in this case)") parser.add_argument("-t", "--tests", metavar="MODULES", default="", help="Comma-separated list of modules to test (example: -t core,imgproc,java)") parser.add_argument("-b", "--blacklist", metavar="MODULES", default="", help="Comma-separated list of modules to exclude from test (example: -b java)") parser.add_argument("-a", "--accuracy", action="store_true", default=False, help="Look for accuracy tests instead of performance tests") parser.add_argument("--check", action="store_true", default=False, help="Shortcut for '--perf_min_samples=1 --perf_force_samples=1'") parser.add_argument("-w", "--cwd", metavar="PATH", default=".", help="Working directory for tests (default is current)") - parser.add_argument("-l", "--longname", action="store_true", default=False, help="Generate log files with long names") parser.add_argument("--list", action="store_true", default=False, help="List available tests (executables)") parser.add_argument("--list_short", action="store_true", default=False, help="List available tests (aliases)") parser.add_argument("--list_short_main", action="store_true", default=False, help="List available tests (main repository, aliases)") @@ -85,11 +85,11 @@ if __name__ == "__main__": test_args = [a for a in test_args if not a.startswith("--gtest_output=")] if args.check: - if not [a for a in test_args if a.startswith("--perf_min_samples=")] : + if not [a for a in test_args if a.startswith("--perf_min_samples=")]: test_args.extend(["--perf_min_samples=1"]) - if not [a for a in test_args if a.startswith("--perf_force_samples=")] : + if not [a for a in test_args if a.startswith("--perf_force_samples=")]: test_args.extend(["--perf_force_samples=1"]) - if not [a for a in test_args if a.startswith("--perf_verify_sanity")] : + if not [a for a in test_args if a.startswith("--perf_verify_sanity")]: test_args.extend(["--perf_verify_sanity"]) if bool(os.environ.get('BUILD_PRECOMMIT', None)): @@ -97,43 +97,42 @@ if __name__ == "__main__": ret = 0 logs = [] - for path in args.build_path: - try: - if not os.path.isdir(path): - raise Err("Not a directory (should contain CMakeCache.txt ot test executables)") - cache = CMakeCache(args.configuration) - fname = os.path.join(path, "CMakeCache.txt") - - if os.path.isfile(fname): - log.debug("Reading cmake cache file: %s", fname) - cache.read(path, fname) - else: - log.debug("Assuming folder contains tests: %s", path) - cache.setDummy(path) - - if args.android or cache.getOS() == "android": - log.debug("Creating Android test runner") - suite = AndroidTestSuite(args, cache, android_env) - else: - log.debug("Creating native test runner") - suite = TestSuite(args, cache) - - if args.list or args.list_short or args.list_short_main: - suite.listTests(args.list_short or args.list_short_main, args.list_short_main) - else: - log.debug("Running tests in '%s', working dir: '%s'", path, args.cwd) - def parseTests(s): - return [o.strip() for o in s.split(",") if o] - l, r = suite.runTests(parseTests(args.tests), parseTests(args.blacklist), args.cwd, test_args) - logs.extend(l) - if r != 0: - ret = r - except Err as e: - log.error("ERROR: test path '%s' ==> %s", path, e.msg) - ret = -1 + stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + path = args.build_path + try: + if not os.path.isdir(path): + raise Err("Not a directory (should contain CMakeCache.txt ot test executables)") + cache = CMakeCache(args.configuration) + fname = os.path.join(path, "CMakeCache.txt") + + if os.path.isfile(fname): + log.debug("Reading cmake cache file: %s", fname) + cache.read(path, fname) + else: + log.debug("Assuming folder contains tests: %s", path) + cache.setDummy(path) + + if args.android or cache.getOS() == "android": + log.debug("Creating Android test runner") + suite = AndroidTestSuite(args, cache, stamp, android_env) + else: + log.debug("Creating native test runner") + suite = TestSuite(args, cache, stamp) + + if args.list or args.list_short or args.list_short_main: + suite.listTests(args.list_short or args.list_short_main, args.list_short_main) + else: + log.debug("Running tests in '%s', working dir: '%s'", path, args.cwd) + + def parseTests(s): + return [o.strip() for o in s.split(",") if o] + logs, ret = suite.runTests(parseTests(args.tests), parseTests(args.blacklist), args.cwd, test_args) + except Err as e: + log.error("ERROR: test path '%s' ==> %s", path, e.msg) + ret = -1 if logs: - log.warning("Collected: %s", ", ".join(logs)) + log.warning("Collected: %s", logs) if ret != 0: log.error("ERROR: some tests have failed") diff --git a/modules/ts/misc/run_android.py b/modules/ts/misc/run_android.py index d03721774d..5823c34096 100644 --- a/modules/ts/misc/run_android.py +++ b/modules/ts/misc/run_android.py @@ -1,12 +1,15 @@ #!/usr/bin/env python - -import sys -from run_utils import * +import os +import re +import getpass +from run_utils import Err, log, execute, isColorEnabled, hostos from run_suite import TestSuite + def exe(program): return program + ".exe" if hostos == 'nt' else program + class ApkInfo: def __init__(self): self.pkg_name = None @@ -20,20 +23,17 @@ class ApkInfo: else: self.pkg_target = package -#============================================================================== class Tool: def __init__(self): self.cmd = [] - def run(self, args = [], silent = False): + def run(self, args=[], silent=False): cmd = self.cmd[:] cmd.extend(args) return execute(self.cmd + args, silent) -#============================================================================== - class Adb(Tool): def __init__(self, sdk_dir): Tool.__init__(self) @@ -41,12 +41,9 @@ class Adb(Tool): if not os.path.isfile(exe_path) or not os.access(exe_path, os.X_OK): exe_path = None # fix adb tool location - if not exe_path: - exe_path = getRunningProcessExePathByName("adb") if not exe_path: exe_path = "adb" self.cmd = [exe_path] - self.cpuinfo = "" def init(self, serial): # remember current device serial. Needed if another device is connected while this script runs @@ -54,13 +51,9 @@ class Adb(Tool): serial = self.detectSerial() if serial: self.cmd.extend(["-s", serial]) - # read device cpuinfo - self.cpuinfo = self.run(["shell", "cat /proc/cpuinfo"], silent = True) - if not self.cpuinfo: - raise Err("Can not get cpuinfo from Android device") def detectSerial(self): - adb_res = self.run(["devices"], silent = True) + adb_res = self.run(["devices"], silent=True) # assume here that device name may consists of any characters except newline connected_devices = re.findall(r"^[^\n]+[ \t]+device\r?$", adb_res, re.MULTILINE) if not connected_devices: @@ -71,29 +64,15 @@ class Adb(Tool): return connected_devices[0].split("\t")[0] def getOSIdentifier(self): - return "Android" + self.run(["shell", "getprop ro.build.version.release"], silent = True).strip() + return "Android" + self.run(["shell", "getprop ro.build.version.release"], silent=True).strip() - def getHardware(self): - hw = re.search(r"^Hardware[ \t]*:[ \t]*(.*?)$", self.cpuinfo, re.MULTILINE) - if hw: - return hw.group(1).strip() - - def checkArmHardware(self, expected_abi): - if expected_abi and "armeabi-v7a" in expected_abi: - if "ARMv7" not in self.cpuinfo: - raise Err("Android device does not support ARMv7 commands, but tests are built for armeabi-v7a") - if "NEON" in expected_abi and "neon" not in self.cpuinfo: - raise Err("Android device has no NEON, but tests are built for %s", expected_abi) - - -#============================================================================== class Aapt(Tool): def __init__(self, sdk_dir): Tool.__init__(self) aapt_fn = exe("aapt") aapt = None - for r, ds, fs in os.walk( os.path.join(sdk_dir, 'build-tools') ): + for r, ds, fs in os.walk(os.path.join(sdk_dir, 'build-tools')): if aapt_fn in fs: aapt = os.path.join(r, aapt_fn) break @@ -103,7 +82,7 @@ class Aapt(Tool): def dump(self, exe): res = ApkInfo() - output = self.run(["dump", "xmltree", exe, "AndroidManifest.xml"], silent = True) + output = self.run(["dump", "xmltree", exe, "AndroidManifest.xml"], silent=True) if not output: raise Err("Can not dump manifest from %s", exe) tags = re.split(r"[ ]+E: ", output) @@ -111,22 +90,21 @@ class Aapt(Tool): manifest_tag = [t for t in tags if t.startswith("manifest ")] if not manifest_tag: raise Err("Can not read package name from: %s", exe) - res.pkg_name = re.search(r"^[ ]+A: package=\"(?P.*?)\" \(Raw: \"(?P=pkg)\"\)\r?$", manifest_tag[0], flags=re.MULTILINE).group("pkg") + res.pkg_name = re.search(r"^[ ]+A: package=\"(?P.*?)\" \(Raw: \"(?P=pkg)\"\)\r?$", manifest_tag[0], flags=re.MULTILINE).group("pkg") # get test instrumentation info instrumentation_tag = [t for t in tags if t.startswith("instrumentation ")] if not instrumentation_tag: raise Err("Can not find instrumentation detials in: %s", exe) res.pkg_runner = re.search(r"^[ ]+A: android:name\(0x[0-9a-f]{8}\)=\"(?P.*?)\" \(Raw: \"(?P=runner)\"\)\r?$", instrumentation_tag[0], flags=re.MULTILINE).group("runner") - res.pkg_target = re.search(r"^[ ]+A: android:targetPackage\(0x[0-9a-f]{8}\)=\"(?P.*?)\" \(Raw: \"(?P=pkg)\"\)\r?$", instrumentation_tag[0], flags=re.MULTILINE).group("pkg") + res.pkg_target = re.search(r"^[ ]+A: android:targetPackage\(0x[0-9a-f]{8}\)=\"(?P.*?)\" \(Raw: \"(?P=pkg)\"\)\r?$", instrumentation_tag[0], flags=re.MULTILINE).group("pkg") if not res.pkg_name or not res.pkg_runner or not res.pkg_target: raise Err("Can not find instrumentation detials in: %s", exe) return res -#=================================================================================================== class AndroidTestSuite(TestSuite): - def __init__(self, options, cache, android_env = {}): - TestSuite.__init__(self, options, cache) + def __init__(self, options, cache, id, android_env={}): + TestSuite.__init__(self, options, cache, id) sdk_dir = options.android_sdk or os.environ.get("ANDROID_SDK", False) or os.path.dirname(os.path.dirname(self.cache.android_executable)) log.debug("Detecting Android tools in directory: %s", sdk_dir) self.adb = Adb(sdk_dir) @@ -142,14 +120,10 @@ class AndroidTestSuite(TestSuite): def getOS(self): return self.adb.getOSIdentifier() - def getHardware(self): - return [self.adb.getHardware()] - def checkPrerequisites(self): self.adb.init(self.options.serial) - self.adb.checkArmHardware(self.cache.android_abi) - def runTest(self, path, logfile, workingDir, args = []): + def runTest(self, path, logfile, workingDir, args=[]): args = args[:] exe = os.path.abspath(path) @@ -160,7 +134,7 @@ class AndroidTestSuite(TestSuite): info.forcePackage(self.options.package) self.adb.run(["uninstall", info.pkg_name]) - output = self.adb.run(["install", exe], silent = True) + output = self.adb.run(["install", exe], silent=True) if not (output and "Success" in output): raise Err("Can not install package: %s", exe) @@ -168,7 +142,7 @@ class AndroidTestSuite(TestSuite): ret = self.adb.run(["shell", "am instrument -w %s %s/%s" % (" ".join(params), info.pkg_name, info.pkg_runner)]) return None, ret else: - device_dir = getpass.getuser().replace(" ","") + "_" + self.options.mode +"/" + device_dir = getpass.getuser().replace(" ", "") + "_" + self.options.mode + "/" if isColorEnabled(args): args.append("--gtest_color=yes") tempdir = "/data/local/tmp/" @@ -177,7 +151,7 @@ class AndroidTestSuite(TestSuite): android_exe = android_dir + exename self.adb.run(["push", exe, android_exe]) self.adb.run(["shell", "chmod 777 " + android_exe]) - env_pieces = ["export %s=%s" % (a,b) for a,b in self.env.items()] + env_pieces = ["export %s=%s" % (a, b) for a, b in self.env.items()] pieces = ["cd %s" % android_dir, "./%s %s" % (exename, " ".join(args))] log.warning("Run: %s" % " && ".join(pieces)) ret = self.adb.run(["shell", " && ".join(env_pieces + pieces)]) @@ -186,12 +160,11 @@ class AndroidTestSuite(TestSuite): self.adb.run(["pull", android_dir + logfile, hostlogpath]) # cleanup self.adb.run(["shell", "rm " + android_dir + logfile]) - self.adb.run(["shell", "rm " + tempdir + "__opencv_temp.*"], silent = True) + self.adb.run(["shell", "rm " + tempdir + "__opencv_temp.*"], silent=True) if os.path.isfile(hostlogpath): return hostlogpath, ret return None, ret -#=================================================================================================== if __name__ == "__main__": log.error("This is utility file, please execute run.py script") diff --git a/modules/ts/misc/run_long.py b/modules/ts/misc/run_long.py index 5640ea3a58..49ccb7b8c9 100644 --- a/modules/ts/misc/run_long.py +++ b/modules/ts/misc/run_long.py @@ -1,5 +1,4 @@ #!/usr/bin/env python - from __future__ import print_function import xml.etree.ElementTree as ET from glob import glob @@ -7,14 +6,14 @@ from pprint import PrettyPrinter as PP LONG_TESTS_DEBUG_VALGRIND = [ ('calib3d', 'Calib3d_InitUndistortRectifyMap.accuracy', 2017.22), - ('dnn', 'Reproducibility*', 1000), # large DNN models + ('dnn', 'Reproducibility*', 1000), # large DNN models ('features2d', 'Features2d/DescriptorImage.no_crash/3', 1000), ('features2d', 'Features2d/DescriptorImage.no_crash/4', 1000), ('features2d', 'Features2d/DescriptorImage.no_crash/5', 1000), ('features2d', 'Features2d/DescriptorImage.no_crash/6', 1000), ('features2d', 'Features2d/DescriptorImage.no_crash/7', 1000), - ('imgcodecs', 'Imgcodecs_Png.write_big', 1000), # memory limit - ('imgcodecs', 'Imgcodecs_Tiff.decode_tile16384x16384', 1000), # memory limit + ('imgcodecs', 'Imgcodecs_Png.write_big', 1000), # memory limit + ('imgcodecs', 'Imgcodecs_Tiff.decode_tile16384x16384', 1000), # memory limit ('ml', 'ML_RTrees.regression', 1423.47), ('optflow', 'DenseOpticalFlow_DeepFlow.ReferenceAccuracy', 1360.95), ('optflow', 'DenseOpticalFlow_DeepFlow_perf.perf/0', 1881.59), @@ -51,7 +50,7 @@ LONG_TESTS_DEBUG_VALGRIND = [ ] -def longTestFilter(data, module = None): +def longTestFilter(data, module=None): res = ['*', '-'] + [v for _, v, m in data if module is None or m == module] return '--gtest_filter={}'.format(':'.join(res)) diff --git a/modules/ts/misc/run_suite.py b/modules/ts/misc/run_suite.py index 64f4d7e418..947db1e2b5 100644 --- a/modules/ts/misc/run_suite.py +++ b/modules/ts/misc/run_suite.py @@ -1,48 +1,25 @@ #!/usr/bin/env python - -import datetime -from run_utils import * +import os +import re +from run_utils import Err, log, execute, getPlatformVersion, isColorEnabled, TempEnvDir from run_long import LONG_TESTS_DEBUG_VALGRIND, longTestFilter -timestamp = datetime.datetime.now() class TestSuite(object): - def __init__(self, options, cache): + def __init__(self, options, cache, id): self.options = options self.cache = cache self.nameprefix = "opencv_" + self.options.mode + "_" self.tests = self.cache.gatherTests(self.nameprefix + "*", self.isTest) + self.id = id def getOS(self): return getPlatformVersion() or self.cache.getOS() - def getHardware(self): - res = [] - if self.cache.getArch() in ["x86", "x64"] and self.cache.withCuda(): - res.append("CUDA") - return res - - def getLogBaseName(self, app): - global timestamp - app = self.getAlias(app) - rev = self.cache.getGitVersion() - if isinstance(timestamp, datetime.datetime): - timestamp = timestamp.strftime("%Y%m%d-%H%M%S") - if self.options.longname: - small_pieces = [self.getOS(), self.cache.getArch()] + self.cache.getDependencies() + self.getHardware() + [self.cache.getSIMDFeatures()] - big_pieces = [app, str(rev), timestamp, "_".join([p for p in small_pieces if p])] - l = "__".join(big_pieces) - else: - pieces = [app, self.cache.getOS(), self.cache.getArch()] + self.getHardware() + [rev, timestamp] - lname = "_".join([p for p in pieces if p]) - lname = re.sub(r'[\(\)\[\]\s,]', '_', lname) - l = re.sub(r'_+', '_', lname) - return l - def getLogName(self, app): - return self.getLogBaseName(app) + '.xml' + return self.getAlias(app) + '_' + str(self.id) + '.xml' - def listTests(self, short = False, main = False): + def listTests(self, short=False, main=False): if len(self.tests) == 0: raise Err("No tests found") for t in self.tests: @@ -52,7 +29,7 @@ class TestSuite(object): log.info("%s", t) def getAlias(self, fname): - return sorted(self.getAliases(fname), key = len)[0] + return sorted(self.getAliases(fname), key=len)[0] def getAliases(self, fname): def getCuts(fname, prefix): @@ -70,11 +47,11 @@ class TestSuite(object): # input is full path ('/home/.../bin/opencv_test_core') or 'java' res = [fname] fname = os.path.basename(fname) - res.append(fname) # filename (opencv_test_core.exe) + res.append(fname) # filename (opencv_test_core.exe) for s in getCuts(fname, self.nameprefix): res.append(s) if self.cache.build_type == "Debug" and "Visual Studio" in self.cache.cmake_generator: - res.append(re.sub(r"d$", '', s)) # MSVC debug config, remove 'd' suffix + res.append(re.sub(r"d$", '', s)) # MSVC debug config, remove 'd' suffix log.debug("Aliases: %s", set(res)) return set(res) @@ -100,7 +77,7 @@ class TestSuite(object): return False return os.access(fullpath, os.X_OK) - def wrapInValgrind(self, cmd = []): + def wrapInValgrind(self, cmd=[]): if self.options.valgrind: res = ['valgrind'] supp = self.options.valgrind_supp or [] @@ -114,36 +91,36 @@ class TestSuite(object): return res + cmd + ([longTestFilter(LONG_TESTS_DEBUG_VALGRIND)] if not has_gtest_filter else []) return cmd - def tryCommand(self, cmd): + def tryCommand(self, cmd, workingDir): try: - if 0 == execute(cmd, cwd = workingDir): + if 0 == execute(cmd, cwd=workingDir): return True except: pass return False - def runTest(self, path, logfile, workingDir, args = []): + def runTest(self, path, logfile, workingDir, args=[]): args = args[:] exe = os.path.abspath(path) if path == "java": cmd = [self.cache.ant_executable, "-Dopencv.build.type=%s" % self.cache.build_type, "buildAndTest"] - ret = execute(cmd, cwd = self.cache.java_test_dir) + ret = execute(cmd, cwd=self.cache.java_test_dir) return None, ret elif path in ['python2', 'python3']: executable = os.getenv('OPENCV_PYTHON_BINARY', None) if executable is None: executable = path - if not self.tryCommand([executable, '--version']): + if not self.tryCommand([executable, '--version'], workingDir): executable = 'python' cmd = [executable, self.cache.opencv_home + '/modules/python/test/test.py', '--repo', self.cache.opencv_home, '-v'] + args - module_suffix = '' if not 'Visual Studio' in self.cache.cmake_generator else '/' + self.cache.build_type + module_suffix = '' if 'Visual Studio' not in self.cache.cmake_generator else '/' + self.cache.build_type env = {} env['PYTHONPATH'] = self.cache.opencv_build + '/lib' + module_suffix + os.pathsep + os.getenv('PYTHONPATH', '') if self.cache.getOS() == 'nt': env['PATH'] = self.cache.opencv_build + '/bin' + module_suffix + os.pathsep + os.getenv('PATH', '') else: env['LD_LIBRARY_PATH'] = self.cache.opencv_build + '/bin' + os.pathsep + os.getenv('LD_LIBRARY_PATH', '') - ret = execute(cmd, cwd = workingDir, env = env) + ret = execute(cmd, cwd=workingDir, env=env) return None, ret else: if isColorEnabled(args): @@ -157,7 +134,7 @@ class TestSuite(object): tempDir = TempEnvDir('OPENCV_TEMP_PATH', "__opencv_temp.") tempDir.init() log.warning("Run: %s" % " ".join(cmd)) - ret = execute(cmd, cwd = workingDir, env=env) + ret = execute(cmd, cwd=workingDir, env=env) try: if not self.options.valgrind and self.options.trace and int(self.options.trace_dump) >= 0: import trace_profiler @@ -174,12 +151,7 @@ class TestSuite(object): return hostlogpath, ret return None, ret - def checkPrerequisites(self): - if self.cache.getArch() == "x64" and hostmachine == "x86": - raise Err("Target architecture is incompatible with current platform") - - def runTests(self, tests, black, workingDir, args = []): - self.checkPrerequisites() + def runTests(self, tests, black, workingDir, args=[]): args = args[:] logs = [] test_list = self.getTestList(tests, black) @@ -213,7 +185,6 @@ class TestSuite(object): logs.append(os.path.relpath(logfile, workingDir)) return logs, ret -#=================================================================================================== if __name__ == "__main__": log.error("This is utility file, please execute run.py script") diff --git a/modules/ts/misc/run_utils.py b/modules/ts/misc/run_utils.py index d7dbcf4f4e..4fc84cc8ef 100644 --- a/modules/ts/misc/run_utils.py +++ b/modules/ts/misc/run_utils.py @@ -1,28 +1,34 @@ #!/usr/bin/env python - -import sys, os, platform, re, tempfile, glob, getpass, logging +import sys +import os +import platform +import re +import tempfile +import glob +import logging +import shutil from subprocess import check_call, check_output, CalledProcessError, STDOUT -hostos = os.name # 'nt', 'posix' -hostmachine = platform.machine() # 'x86', 'AMD64', 'x86_64' def initLogger(): - l = logging.getLogger("run.py") - l.setLevel(logging.DEBUG) + logger = logging.getLogger("run.py") + logger.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stderr) ch.setFormatter(logging.Formatter("%(message)s")) - l.addHandler(ch) - return l + logger.addHandler(ch) + return logger + log = initLogger() +hostos = os.name # 'nt', 'posix' -#=================================================================================================== class Err(Exception): def __init__(self, msg, *args): self.msg = msg % args -def execute(cmd, silent = False, cwd = ".", env = None): + +def execute(cmd, silent=False, cwd=".", env=None): try: log.debug("Run: %s", cmd) if env is not None: @@ -32,9 +38,9 @@ def execute(cmd, silent = False, cwd = ".", env = None): new_env.update(env) env = new_env if silent: - return check_output(cmd, stderr = STDOUT, cwd = cwd, env = env).decode("latin-1") + return check_output(cmd, stderr=STDOUT, cwd=cwd, env=env).decode("latin-1") else: - return check_call(cmd, cwd = cwd, env = env) + return check_call(cmd, cwd=cwd, env=env) except CalledProcessError as e: if silent: log.debug("Process returned: %d", e.returncode) @@ -43,11 +49,11 @@ def execute(cmd, silent = False, cwd = ".", env = None): log.error("Process returned: %d", e.returncode) return e.returncode + def isColorEnabled(args): usercolor = [a for a in args if a.startswith("--gtest_color=")] return len(usercolor) == 0 and sys.stdout.isatty() and hostos != "nt" -#=================================================================================================== def getPlatformVersion(): mv = platform.mac_ver() @@ -63,130 +69,27 @@ def getPlatformVersion(): return lv[0] + lv[1] return None -def readGitVersion(git, path): - if not path or not git or not os.path.isdir(os.path.join(path, ".git")): - return None - try: - output = execute([git, "-C", path, "rev-parse", "--short", "HEAD"], silent = True) - return output.strip() - except OSError: - log.warning("Git version read failed") - return None - -SIMD_DETECTION_PROGRAM=""" -#if __SSE5__ -# error SSE5 -#endif -#if __AVX2__ -# error AVX2 -#endif -#if __AVX__ -# error AVX -#endif -#if __SSE4_2__ -# error SSE4.2 -#endif -#if __SSE4_1__ -# error SSE4.1 -#endif -#if __SSSE3__ -# error SSSE3 -#endif -#if __SSE3__ -# error SSE3 -#endif -#if __AES__ -# error AES -#endif -#if __SSE2__ -# error SSE2 -#endif -#if __SSE__ -# error SSE -#endif -#if __3dNOW__ -# error 3dNOW -#endif -#if __MMX__ -# error MMX -#endif -#if __ARM_NEON__ -# error NEON -#endif -#error NOSIMD -""" - -def testSIMD(compiler, cxx_flags, compiler_arg = None): - if not compiler: - return None - compiler_output = "" - try: - _, tmpfile = tempfile.mkstemp(suffix=".cpp", text = True) - with open(tmpfile, "w+") as fd: - fd.write(SIMD_DETECTION_PROGRAM) - options = [compiler] - if compiler_arg: - options.append(compiler_arg) - - prev_option = None - for opt in " ".join(cxx_flags).split(): - if opt.count('\"') % 2 == 1: - if prev_option is None: - prev_option = opt - else: - options.append(prev_option + " " + opt) - prev_option = None - elif prev_option is None: - options.append(opt) - else: - prev_option = prev_option + " " + opt - options.append(tmpfile) - compiler_output = execute(options, silent = True) - os.remove(tmpfile) - m = re.search("#error\W+(\w+)", compiler_output) - if m: - return m.group(1) - except OSError: - pass - log.debug("SIMD detection failed") - return None - -#============================================================================== parse_patterns = ( - {'name': "cmake_home", 'default': None, 'pattern': re.compile(r"^CMAKE_HOME_DIRECTORY:INTERNAL=(.+)$")}, - {'name': "opencv_home", 'default': None, 'pattern': re.compile(r"^OpenCV_SOURCE_DIR:STATIC=(.+)$")}, - {'name': "opencv_build", 'default': None, 'pattern': re.compile(r"^OpenCV_BINARY_DIR:STATIC=(.+)$")}, - {'name': "tests_dir", 'default': None, 'pattern': re.compile(r"^EXECUTABLE_OUTPUT_PATH:PATH=(.+)$")}, + {'name': "cmake_home", 'default': None, 'pattern': re.compile(r"^CMAKE_HOME_DIRECTORY:\w+=(.+)$")}, + {'name': "opencv_home", 'default': None, 'pattern': re.compile(r"^OpenCV_SOURCE_DIR:\w+=(.+)$")}, + {'name': "opencv_build", 'default': None, 'pattern': re.compile(r"^OpenCV_BINARY_DIR:\w+=(.+)$")}, + {'name': "tests_dir", 'default': None, 'pattern': re.compile(r"^EXECUTABLE_OUTPUT_PATH:\w+=(.+)$")}, {'name': "build_type", 'default': "Release", 'pattern': re.compile(r"^CMAKE_BUILD_TYPE:\w+=(.*)$")}, - {'name': "git_executable", 'default': None, 'pattern': re.compile(r"^GIT_EXECUTABLE:FILEPATH=(.*)$")}, - {'name': "cxx_flags", 'default': "", 'pattern': re.compile(r"^CMAKE_CXX_FLAGS:STRING=(.*)$")}, - {'name': "cxx_flags_debug", 'default': "", 'pattern': re.compile(r"^CMAKE_CXX_FLAGS_DEBUG:STRING=(.*)$")}, - {'name': "cxx_flags_release", 'default': "", 'pattern': re.compile(r"^CMAKE_CXX_FLAGS_RELEASE:STRING=(.*)$")}, - {'name': "opencv_cxx_flags", 'default': "", 'pattern': re.compile(r"^OPENCV_EXTRA_C_FLAGS:INTERNAL=(.*)$")}, - {'name': "cxx_flags_android", 'default': None, 'pattern': re.compile(r"^ANDROID_CXX_FLAGS:INTERNAL=(.*)$")}, - {'name': "android_abi", 'default': None, 'pattern': re.compile(r"^ANDROID_ABI:STRING=(.*)$")}, - {'name': "android_executable", 'default': None, 'pattern': re.compile(r"^ANDROID_EXECUTABLE:FILEPATH=(.*android.*)$")}, - {'name': "ant_executable", 'default': None, 'pattern': re.compile(r"^ANT_EXECUTABLE:FILEPATH=(.*ant.*)$")}, + {'name': "android_abi", 'default': None, 'pattern': re.compile(r"^ANDROID_ABI:\w+=(.*)$")}, + {'name': "android_executable", 'default': None, 'pattern': re.compile(r"^ANDROID_EXECUTABLE:\w+=(.*android.*)$")}, + {'name': "ant_executable", 'default': None, 'pattern': re.compile(r"^ANT_EXECUTABLE:\w+=(.*ant.*)$")}, {'name': "java_test_dir", 'default': None, 'pattern': re.compile(r"^OPENCV_JAVA_TEST_DIR:\w+=(.*)$")}, - {'name': "is_x64", 'default': "OFF", 'pattern': re.compile(r"^CUDA_64_BIT_DEVICE_CODE:BOOL=(ON)$")},#ugly( - {'name': "cmake_generator", 'default': None, 'pattern': re.compile(r"^CMAKE_GENERATOR:INTERNAL=(.+)$")}, - {'name': "cxx_compiler", 'default': None, 'pattern': re.compile(r"^CMAKE_CXX_COMPILER:\w*PATH=(.+)$")}, - {'name': "cxx_compiler_arg1", 'default': None, 'pattern': re.compile(r"^CMAKE_CXX_COMPILER_ARG1:[A-Z]+=(.+)$")}, - {'name': "with_cuda", 'default': "OFF", 'pattern': re.compile(r"^WITH_CUDA:BOOL=(ON)$")}, - {'name': "cuda_library", 'default': None, 'pattern': re.compile(r"^CUDA_CUDA_LIBRARY:FILEPATH=(.+)$")}, - {'name': "cuda_version", 'default': None, 'pattern': re.compile(r"^CUDA_VERSION:STRING=(.+)$")}, - {'name': "core_dependencies", 'default': None, 'pattern': re.compile(r"^opencv_core_LIB_DEPENDS:STATIC=(.+)$")}, - {'name': "python2", 'default': None, 'pattern': re.compile(r"^BUILD_opencv_python2:BOOL=(.*)$")}, - {'name': "python3", 'default': None, 'pattern': re.compile(r"^BUILD_opencv_python3:BOOL=(.*)$")}, + {'name': "is_x64", 'default': "OFF", 'pattern': re.compile(r"^CUDA_64_BIT_DEVICE_CODE:\w+=(ON)$")}, + {'name': "cmake_generator", 'default': None, 'pattern': re.compile(r"^CMAKE_GENERATOR:\w+=(.+)$")}, + {'name': "python2", 'default': None, 'pattern': re.compile(r"^BUILD_opencv_python2:\w+=(.*)$")}, + {'name': "python3", 'default': None, 'pattern': re.compile(r"^BUILD_opencv_python3:\w+=(.*)$")}, ) + class CMakeCache: - def __init__(self, cfg = None): + def __init__(self, cfg=None): self.setDefaultAttrs() - self.cmake_home_vcver = None - self.opencv_home_vcver = None - self.featuresSIMD = None self.main_modules = [] if cfg: self.build_type = cfg @@ -196,7 +99,7 @@ class CMakeCache: def read(self, path, fname): rx = re.compile(r'^OPENCV_MODULE_opencv_(\w+)_LOCATION:INTERNAL=(.*)$') - module_paths = {} # name -> path + module_paths = {} # name -> path with open(fname, "rt") as cachefile: for l in cachefile.readlines(): ll = l.strip() @@ -225,31 +128,16 @@ class CMakeCache: if "Visual Studio" in self.cmake_generator: self.tests_dir = os.path.join(self.tests_dir, self.build_type) - self.cmake_home_vcver = readGitVersion(self.git_executable, self.cmake_home) - if self.opencv_home == self.cmake_home: - self.opencv_home_vcver = self.cmake_home_vcver - else: - self.opencv_home_vcver = readGitVersion(self.git_executable, self.opencv_home) - - for module,path in module_paths.items(): + for module, path in module_paths.items(): rel = os.path.relpath(path, self.opencv_home) - if not ".." in rel: + if ".." not in rel: self.main_modules.append(module) - self.flags = [ - self.cxx_flags_android, - self.cxx_flags, - self.cxx_flags_release, - self.opencv_cxx_flags, - self.cxx_flags_release] - self.flags = [f for f in self.flags if f] - self.featuresSIMD = testSIMD(self.cxx_compiler, self.flags, self.cxx_compiler_arg1) - def setDefaultAttrs(self): for p in parse_patterns: setattr(self, p["name"], p["default"]) - def gatherTests(self, mask, isGood = None): + def gatherTests(self, mask, isGood=None): if self.tests_dir and os.path.isdir(self.tests_dir): d = os.path.abspath(self.tests_dir) files = glob.glob(os.path.join(d, mask)) @@ -265,11 +153,8 @@ class CMakeCache: def isMainModule(self, name): return name in self.main_modules + ['python2', 'python3'] - def withCuda(self): - return self.cuda_version and self.with_cuda == "ON" and self.cuda_library and not self.cuda_library.endswith("-NOTFOUND") - def withJava(self): - return self.ant_executable and os.path.exists(self.java_test_dir) + return self.ant_executable and self.java_test_dir and os.path.exists(self.java_test_dir) def withPython2(self): return self.python2 == 'ON' @@ -277,164 +162,12 @@ class CMakeCache: def withPython3(self): return self.python3 == 'ON' - def getGitVersion(self): - if self.cmake_home_vcver: - if self.cmake_home_vcver == self.opencv_home_vcver: - rev = self.cmake_home_vcver - elif self.opencv_home_vcver: - rev = self.cmake_home_vcver + "-" + self.opencv_home_vcver - else: - rev = self.cmake_home_vcver - else: - rev = None - if rev: - rev = rev.replace(":","to") - else: - rev = "" - return rev - - def getTestFullName(self, shortname): - return os.path.join(self.tests_dir, shortname) - - def getSIMDFeatures(self): - return self.featuresSIMD - def getOS(self): if self.android_executable: return "android" else: return hostos - def getArch(self): - arch = "unknown" - if self.getOS() == "android": - if "armeabi-v7a" in self.android_abi: - arch = "armv7a" - elif "armeabi-v6" in self.android_abi: - arch = "armv6" - elif "armeabi" in self.android_abi: - arch = "armv5te" - elif "x86" in self.android_abi: - arch = "x86" - elif "mips" in self.android_abi: - arch = "mips" - else: - arch = "ARM" - elif self.is_x64 and hostmachine in ["AMD64", "x86_64"]: - arch = "x64" - elif hostmachine in ["x86", "AMD64", "x86_64"]: - arch = "x86" - return arch - - def getDependencies(self): - if self.core_dependencies: - candidates = ["tbb", "ippicv", "ipp", "pthreads"] - return [a for a in self.core_dependencies.split(";") if a and a in candidates] - return [] - - -#============================================================================== - -def getRunningProcessExePathByName_win32(name): - from ctypes import windll, POINTER, pointer, Structure, sizeof - from ctypes import c_long , c_int , c_uint , c_char , c_ubyte , c_char_p , c_void_p - - class PROCESSENTRY32(Structure): - _fields_ = [ ( 'dwSize' , c_uint ) , - ( 'cntUsage' , c_uint) , - ( 'th32ProcessID' , c_uint) , - ( 'th32DefaultHeapID' , c_uint) , - ( 'th32ModuleID' , c_uint) , - ( 'cntThreads' , c_uint) , - ( 'th32ParentProcessID' , c_uint) , - ( 'pcPriClassBase' , c_long) , - ( 'dwFlags' , c_uint) , - ( 'szExeFile' , c_char * 260 ) , - ( 'th32MemoryBase' , c_long) , - ( 'th32AccessKey' , c_long ) ] - - class MODULEENTRY32(Structure): - _fields_ = [ ( 'dwSize' , c_long ) , - ( 'th32ModuleID' , c_long ), - ( 'th32ProcessID' , c_long ), - ( 'GlblcntUsage' , c_long ), - ( 'ProccntUsage' , c_long ) , - ( 'modBaseAddr' , c_long ) , - ( 'modBaseSize' , c_long ) , - ( 'hModule' , c_void_p ) , - ( 'szModule' , c_char * 256 ), - ( 'szExePath' , c_char * 260 ) ] - - TH32CS_SNAPPROCESS = 2 - TH32CS_SNAPMODULE = 0x00000008 - - ## CreateToolhelp32Snapshot - CreateToolhelp32Snapshot= windll.kernel32.CreateToolhelp32Snapshot - CreateToolhelp32Snapshot.reltype = c_long - CreateToolhelp32Snapshot.argtypes = [ c_int , c_int ] - ## Process32First - Process32First = windll.kernel32.Process32First - Process32First.argtypes = [ c_void_p , POINTER( PROCESSENTRY32 ) ] - Process32First.rettype = c_int - ## Process32Next - Process32Next = windll.kernel32.Process32Next - Process32Next.argtypes = [ c_void_p , POINTER(PROCESSENTRY32) ] - Process32Next.rettype = c_int - ## CloseHandle - CloseHandle = windll.kernel32.CloseHandle - CloseHandle.argtypes = [ c_void_p ] - CloseHandle.rettype = c_int - ## Module32First - Module32First = windll.kernel32.Module32First - Module32First.argtypes = [ c_void_p , POINTER(MODULEENTRY32) ] - Module32First.rettype = c_int - - hProcessSnap = c_void_p(0) - hProcessSnap = CreateToolhelp32Snapshot( TH32CS_SNAPPROCESS , 0 ) - - pe32 = PROCESSENTRY32() - pe32.dwSize = sizeof( PROCESSENTRY32 ) - ret = Process32First( hProcessSnap , pointer( pe32 ) ) - path = None - - while ret : - if name + ".exe" == pe32.szExeFile: - hModuleSnap = c_void_p(0) - me32 = MODULEENTRY32() - me32.dwSize = sizeof( MODULEENTRY32 ) - hModuleSnap = CreateToolhelp32Snapshot( TH32CS_SNAPMODULE, pe32.th32ProcessID ) - - ret = Module32First( hModuleSnap, pointer(me32) ) - path = me32.szExePath - CloseHandle( hModuleSnap ) - if path: - break - ret = Process32Next( hProcessSnap, pointer(pe32) ) - CloseHandle( hProcessSnap ) - return path - - -def getRunningProcessExePathByName_posix(name): - pids= [pid for pid in os.listdir('/proc') if pid.isdigit()] - for pid in pids: - try: - path = os.readlink(os.path.join('/proc', pid, 'exe')) - if path and path.endswith(name): - return path - except: - pass - -def getRunningProcessExePathByName(name): - try: - if hostos == "nt": - return getRunningProcessExePathByName_win32(name) - elif hostos == "posix": - return getRunningProcessExePathByName_posix(name) - else: - return None - except: - return None - class TempEnvDir: def __init__(self, envname, prefix): @@ -458,7 +191,6 @@ class TempEnvDir: except: pass -#=================================================================================================== if __name__ == "__main__": log.error("This is utility file, please execute run.py script")