diff --git a/modules/ts/include/opencv2/ts/ts.hpp b/modules/ts/include/opencv2/ts/ts.hpp index fcef5896c6..ea0c979115 100644 --- a/modules/ts/include/opencv2/ts/ts.hpp +++ b/modules/ts/include/opencv2/ts/ts.hpp @@ -578,6 +578,13 @@ int main(int argc, char **argv) \ return RUN_ALL_TESTS(); \ } +// This usually only makes sense in perf tests with several implementations, +// some of which are not available. +#define CV_TEST_FAIL_NO_IMPL() do { \ + ::testing::Test::RecordProperty("custom_status", "noimpl"); \ + FAIL() << "No equivalent implementation."; \ +} while (0) + #endif #include "ts_perf.hpp" diff --git a/modules/ts/misc/testlog_parser.py b/modules/ts/misc/testlog_parser.py index 5d478645b2..4ab0a3ef2f 100755 --- a/modules/ts/misc/testlog_parser.py +++ b/modules/ts/misc/testlog_parser.py @@ -13,10 +13,17 @@ class TestInfo(object): self.name = xmlnode.getAttribute("name") self.value_param = xmlnode.getAttribute("value_param") self.type_param = xmlnode.getAttribute("type_param") - if xmlnode.getElementsByTagName("failure"): + + custom_status = xmlnode.getAttribute("custom_status") + failures = xmlnode.getElementsByTagName("failure") + + if len(custom_status) > 0: + self.status = custom_status + elif len(failures) > 0: self.status = "failed" else: self.status = xmlnode.getAttribute("status") + if self.name.startswith("DISABLED_"): self.status = "disabled" self.fixture = self.fixture.replace("DISABLED_", "") diff --git a/modules/ts/misc/xls-report.py b/modules/ts/misc/xls-report.py index 131f3fab59..d5db73d482 100755 --- a/modules/ts/misc/xls-report.py +++ b/modules/ts/misc/xls-report.py @@ -64,6 +64,10 @@ Name for the sheet. If this parameter is missing, the name of sheet's directory will be used. + * 'sheet_properties': [(string, string)] + List of arbitrary (key, value) pairs that somehow describe the sheet. Will be + dumped into the first row of the sheet in string form. + Note that all keys are optional, although to get useful results, you'll want to specify at least 'configurations' and 'configuration_matchers'. @@ -100,6 +104,7 @@ bad_speedup_style = xlwt.easyxf('font: color red', num_format_str='#0.00') no_speedup_style = no_time_style error_speedup_style = xlwt.easyxf('pattern: pattern solid, fore_color orange') header_style = xlwt.easyxf('font: bold true; alignment: horizontal centre, vertical top, wrap True') +subheader_style = xlwt.easyxf('alignment: horizontal centre, vertical top') class Collector(object): def __init__(self, config_match_func, include_unmatched): @@ -189,6 +194,8 @@ def main(): arg_parser.add_argument('-c', '--config', metavar='CONF', help='global configuration file') arg_parser.add_argument('--include-unmatched', action='store_true', help='include results from XML files that were not recognized by configuration matchers') + arg_parser.add_argument('--show-times-per-pixel', action='store_true', + help='for tests that have an image size parameter, show per-pixel time, as well as total time') args = arg_parser.parse_args() @@ -231,24 +238,64 @@ def main(): sheet = wb.add_sheet(sheet_conf.get('sheet_name', os.path.basename(os.path.abspath(sheet_path)))) - sheet.row(0).height = 800 + sheet_properties = sheet_conf.get('sheet_properties', []) + + sheet.write(0, 0, 'Properties:') + + sheet.write(0, 1, + 'N/A' if len(sheet_properties) == 0 else + ' '.join(str(k) + '=' + repr(v) for (k, v) in sheet_properties)) + + sheet.row(2).height = 800 sheet.panes_frozen = True sheet.remove_splits = True - sheet.horz_split_pos = 1 - sheet.horz_split_first_visible = 1 sheet_comparisons = sheet_conf.get('comparisons', []) - for i, w in enumerate([2000, 15000, 2500, 2000, 15000] - + (len(config_names) + 1 + len(sheet_comparisons)) * [4000]): - sheet.col(i).width = w + row = 2 - for i, caption in enumerate(['Module', 'Test', 'Image\nsize', 'Data\ntype', 'Parameters'] - + config_names + [None] - + [comp['to'] + '\nvs\n' + comp['from'] for comp in sheet_comparisons]): - sheet.row(0).write(i, caption, header_style) + col = 0 - row = 1 + for (w, caption) in [ + (2500, 'Module'), + (10000, 'Test'), + (2500, 'Image\nsize'), + (2000, 'Data\ntype'), + (7500, 'Other parameters')]: + sheet.col(col).width = w + if args.show_times_per_pixel: + sheet.write_merge(row, row + 1, col, col, caption, header_style) + else: + sheet.write(row, col, caption, header_style) + col += 1 + + for config_name in config_names: + if args.show_times_per_pixel: + sheet.col(col).width = 3000 + sheet.col(col + 1).width = 3000 + sheet.write_merge(row, row, col, col + 1, config_name, header_style) + sheet.write(row + 1, col, 'total, ms', subheader_style) + sheet.write(row + 1, col + 1, 'per pixel, ns', subheader_style) + col += 2 + else: + sheet.col(col).width = 4000 + sheet.write(row, col, config_name, header_style) + col += 1 + + col += 1 # blank column between configurations and comparisons + + for comp in sheet_comparisons: + sheet.col(col).width = 4000 + caption = comp['to'] + '\nvs\n' + comp['from'] + if args.show_times_per_pixel: + sheet.write_merge(row, row + 1, col, col, caption, header_style) + else: + sheet.write(row, col, caption, header_style) + + row += 2 if args.show_times_per_pixel else 1 + + sheet.horz_split_pos = row + sheet.horz_split_first_visible = row module_colors = sheet_conf.get('module_colors', {}) module_styles = {module: xlwt.easyxf('pattern: pattern solid, fore_color {}'.format(color)) @@ -259,21 +306,49 @@ def main(): sheet.write(row, 0, module, module_styles.get(module, xlwt.Style.default_style)) sheet.write(row, 1, test) - param_list = param[1:-1].split(", ") - sheet.write(row, 2, next(ifilter(re_image_size.match, param_list), None)) - sheet.write(row, 3, next(ifilter(re_data_type.match, param_list), None)) + param_list = param[1:-1].split(', ') if param.startswith('(') and param.endswith(')') else [param] + + image_size = next(ifilter(re_image_size.match, param_list), None) + if image_size is not None: + sheet.write(row, 2, image_size) + del param_list[param_list.index(image_size)] + + data_type = next(ifilter(re_data_type.match, param_list), None) + if data_type is not None: + sheet.write(row, 3, data_type) + del param_list[param_list.index(data_type)] - sheet.row(row).write(4, param) - for i, c in enumerate(config_names): + sheet.row(row).write(4, ' | '.join(param_list)) + + col = 5 + + for c in config_names: if c in configs: - sheet.write(row, 5 + i, configs[c], time_style) + sheet.write(row, col, configs[c], time_style) else: - sheet.write(row, 5 + i, None, no_time_style) - - for i, comp in enumerate(sheet_comparisons): + sheet.write(row, col, None, no_time_style) + col += 1 + if args.show_times_per_pixel: + sheet.write(row, col, + xlwt.Formula( + ''' + {0} * 1000000 / ( + VALUE(MID({1}; 1; SEARCH("x"; {1}) - 1)) + * VALUE(MID({1}; SEARCH("x"; {1}) + 1; LEN({1}))) + ) + '''.replace('\n', '').replace(' ', '').format( + xlwt.Utils.rowcol_to_cell(row, col - 1), + xlwt.Utils.rowcol_to_cell(row, 2) + ) + ), + time_style) + col += 1 + + col += 1 # blank column + + for comp in sheet_comparisons: cmp_from = configs.get(comp["from"]) cmp_to = configs.get(comp["to"]) - col = 5 + len(config_names) + 1 + i if isinstance(cmp_from, numbers.Number) and isinstance(cmp_to, numbers.Number): try: @@ -286,6 +361,8 @@ def main(): else: sheet.write(row, col, None, no_speedup_style) + col += 1 + row += 1 if row % 1000 == 0: sheet.flush_row_data()