#!/usr/bin/env python3 # Copyright 2015 The Meson development team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import subprocess, sys, os, argparse import pickle, statistics, json from . import meson_test parser = argparse.ArgumentParser() parser.add_argument('--wd', default=None, dest='wd', help='directory to cd into before running') parser.add_argument('args', nargs='+') def print_stats(numlen, num_tests, name, res, i, duration, stdev): startpad = ' '*(numlen - len('%d' % (i+1))) num = '%s%d/%d' % (startpad, i+1, num_tests) padding1 = ' '*(38-len(name)) padding2 = ' '*(8-len(res)) result_str = '%s %s %s%s%s%5.5f s +- %5.5f s' % \ (num, name, padding1, res, padding2, duration, stdev) print(result_str) # write_json_log(jsonlogfile, name, result) def print_json_log(jsonlogfile, rawruns, test_name, i): jsonobj = {'name' : test_name} runs = [] for r in rawruns: runobj = {'duration': r.duration, 'stdout': r.stdo, 'returncode' : r.returncode, 'duration' : r.duration} if r.stde: runobj['stderr'] = r.stde runs.append(runobj) jsonobj['runs'] = runs jsonlogfile.write(json.dumps(jsonobj) + '\n') jsonlogfile.flush() def run_benchmarks(options, datafile): failed_tests = 0 logfile_base = 'meson-logs/benchmarklog' jsonlogfilename = logfile_base+ '.json' jsonlogfile = open(jsonlogfilename, 'w') tests = pickle.load(open(datafile, 'rb')) num_tests = len(tests) if num_tests == 0: print('No benchmarks defined.') return 0 iteration_count = 5 wrap = [] # Benchmarks on cross builds are pointless so don't support them. for i, test in enumerate(tests): runs = [] durations = [] failed = False for _ in range(iteration_count): res = meson_test.run_single_test(wrap, test) runs.append(res) durations.append(res.duration) if res.returncode != 0: failed = True mean = statistics.mean(durations) stddev = statistics.stdev(durations) if failed: resultstr = 'FAIL' failed_tests += 1 else: resultstr = 'OK' print_stats(3, num_tests, test.name, resultstr, i, mean, stddev) print_json_log(jsonlogfile, runs, test.name, i) print('\nFull log written to meson-logs/benchmarklog.json.') return failed_tests def run(args): global failed_tests options = parser.parse_args(args) if len(options.args) != 1: print('Benchmark runner for Meson. Do not run on your own, mmm\'kay?') print('%s [data file]' % sys.argv[0]) if options.wd is not None: os.chdir(options.wd) datafile = options.args[0] returncode = run_benchmarks(options, datafile) return returncode if __name__ == '__main__': sys.exit(run(sys.argv[1:]))