From 63d151a29c08d3fa0027224b197a2ec53edb4137 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Fri, 5 Jul 2013 16:10:28 +0400 Subject: [PATCH 1/5] xls-report.py: add ability to specify arbitrary sheet properties --- modules/ts/misc/xls-report.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/modules/ts/misc/xls-report.py b/modules/ts/misc/xls-report.py index 131f3fab59..b0839581c4 100755 --- a/modules/ts/misc/xls-report.py +++ b/modules/ts/misc/xls-report.py @@ -64,6 +64,10 @@ Name for the sheet. If this parameter is missing, the name of sheet's directory will be used. + * 'sheet_properties': [(string, string)] + List of arbitrary (key, value) pairs that somehow describe the sheet. Will be + dumped into the first row of the sheet in string form. + Note that all keys are optional, although to get useful results, you'll want to specify at least 'configurations' and 'configuration_matchers'. @@ -231,24 +235,32 @@ def main(): sheet = wb.add_sheet(sheet_conf.get('sheet_name', os.path.basename(os.path.abspath(sheet_path)))) - sheet.row(0).height = 800 + sheet_properties = sheet_conf.get('sheet_properties', []) + + sheet.write(0, 0, 'Properties:') + + sheet.write(0, 1, + 'N/A' if len(sheet_properties) == 0 else + ' '.join(str(k) + '=' + repr(v) for (k, v) in sheet_properties)) + + sheet.row(2).height = 800 sheet.panes_frozen = True sheet.remove_splits = True - sheet.horz_split_pos = 1 - sheet.horz_split_first_visible = 1 + sheet.horz_split_pos = 3 + sheet.horz_split_first_visible = 3 sheet_comparisons = sheet_conf.get('comparisons', []) - for i, w in enumerate([2000, 15000, 2500, 2000, 15000] + for i, w in enumerate([2500, 15000, 2500, 2000, 15000] + (len(config_names) + 1 + len(sheet_comparisons)) * [4000]): sheet.col(i).width = w for i, caption in enumerate(['Module', 'Test', 'Image\nsize', 'Data\ntype', 'Parameters'] + config_names + [None] + [comp['to'] + '\nvs\n' + comp['from'] for comp in sheet_comparisons]): - sheet.row(0).write(i, caption, header_style) + sheet.row(2).write(i, caption, header_style) - row = 1 + row = 3 module_colors = sheet_conf.get('module_colors', {}) module_styles = {module: xlwt.easyxf('pattern: pattern solid, fore_color {}'.format(color)) From 1080c4295ae74dfcd3a05f82e8a64104e4f9d80d Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Fri, 5 Jul 2013 18:41:03 +0400 Subject: [PATCH 2/5] xls-report.py: removed image size and type from the list of other parameters. Also, shrunk the corresponding column and the test name column. --- modules/ts/misc/xls-report.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/modules/ts/misc/xls-report.py b/modules/ts/misc/xls-report.py index b0839581c4..b120484317 100755 --- a/modules/ts/misc/xls-report.py +++ b/modules/ts/misc/xls-report.py @@ -251,11 +251,11 @@ def main(): sheet_comparisons = sheet_conf.get('comparisons', []) - for i, w in enumerate([2500, 15000, 2500, 2000, 15000] + for i, w in enumerate([2500, 10000, 2500, 2000, 7500] + (len(config_names) + 1 + len(sheet_comparisons)) * [4000]): sheet.col(i).width = w - for i, caption in enumerate(['Module', 'Test', 'Image\nsize', 'Data\ntype', 'Parameters'] + for i, caption in enumerate(['Module', 'Test', 'Image\nsize', 'Data\ntype', 'Other parameters'] + config_names + [None] + [comp['to'] + '\nvs\n' + comp['from'] for comp in sheet_comparisons]): sheet.row(2).write(i, caption, header_style) @@ -271,11 +271,19 @@ def main(): sheet.write(row, 0, module, module_styles.get(module, xlwt.Style.default_style)) sheet.write(row, 1, test) - param_list = param[1:-1].split(", ") - sheet.write(row, 2, next(ifilter(re_image_size.match, param_list), None)) - sheet.write(row, 3, next(ifilter(re_data_type.match, param_list), None)) + param_list = param[1:-1].split(', ') if param.startswith('(') and param.endswith(')') else [param] - sheet.row(row).write(4, param) + image_size = next(ifilter(re_image_size.match, param_list), None) + if image_size is not None: + sheet.write(row, 2, image_size) + del param_list[param_list.index(image_size)] + + data_type = next(ifilter(re_data_type.match, param_list), None) + if data_type is not None: + sheet.write(row, 3, data_type) + del param_list[param_list.index(data_type)] + + sheet.row(row).write(4, ' | '.join(param_list)) for i, c in enumerate(config_names): if c in configs: sheet.write(row, 5 + i, configs[c], time_style) From 215b3e749fe35577bfd5789fcb1977e681cde8a4 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Fri, 5 Jul 2013 19:05:42 +0400 Subject: [PATCH 3/5] Added to the test log parser a crude ability to detect non-implemented tests. --- modules/ts/misc/testlog_parser.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/ts/misc/testlog_parser.py b/modules/ts/misc/testlog_parser.py index 5d478645b2..841ad2e130 100755 --- a/modules/ts/misc/testlog_parser.py +++ b/modules/ts/misc/testlog_parser.py @@ -13,10 +13,17 @@ class TestInfo(object): self.name = xmlnode.getAttribute("name") self.value_param = xmlnode.getAttribute("value_param") self.type_param = xmlnode.getAttribute("type_param") - if xmlnode.getElementsByTagName("failure"): - self.status = "failed" + + failures = xmlnode.getElementsByTagName("failure") + if len(failures) > 0: + if any("No equivalent implementation" in f.getAttribute("message") + for f in failures): + self.status = "notimpl" + else: + self.status = "failed" else: self.status = xmlnode.getAttribute("status") + if self.name.startswith("DISABLED_"): self.status = "disabled" self.fixture = self.fixture.replace("DISABLED_", "") From 5b2dc26f2ce9be1227910cfa45292fbfead73ab8 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Tue, 9 Jul 2013 18:57:22 +0400 Subject: [PATCH 4/5] Made the crude ability less crude. --- modules/ts/include/opencv2/ts/ts.hpp | 7 +++++++ modules/ts/misc/testlog_parser.py | 12 ++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/modules/ts/include/opencv2/ts/ts.hpp b/modules/ts/include/opencv2/ts/ts.hpp index fcef5896c6..ea0c979115 100644 --- a/modules/ts/include/opencv2/ts/ts.hpp +++ b/modules/ts/include/opencv2/ts/ts.hpp @@ -578,6 +578,13 @@ int main(int argc, char **argv) \ return RUN_ALL_TESTS(); \ } +// This usually only makes sense in perf tests with several implementations, +// some of which are not available. +#define CV_TEST_FAIL_NO_IMPL() do { \ + ::testing::Test::RecordProperty("custom_status", "noimpl"); \ + FAIL() << "No equivalent implementation."; \ +} while (0) + #endif #include "ts_perf.hpp" diff --git a/modules/ts/misc/testlog_parser.py b/modules/ts/misc/testlog_parser.py index 841ad2e130..4ab0a3ef2f 100755 --- a/modules/ts/misc/testlog_parser.py +++ b/modules/ts/misc/testlog_parser.py @@ -14,13 +14,13 @@ class TestInfo(object): self.value_param = xmlnode.getAttribute("value_param") self.type_param = xmlnode.getAttribute("type_param") + custom_status = xmlnode.getAttribute("custom_status") failures = xmlnode.getElementsByTagName("failure") - if len(failures) > 0: - if any("No equivalent implementation" in f.getAttribute("message") - for f in failures): - self.status = "notimpl" - else: - self.status = "failed" + + if len(custom_status) > 0: + self.status = custom_status + elif len(failures) > 0: + self.status = "failed" else: self.status = xmlnode.getAttribute("status") From ea3239a00efa23e2017c936d0219cfa2708ff616 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Wed, 10 Jul 2013 14:50:51 +0400 Subject: [PATCH 5/5] xls-report.py: Added an option to show per-pixel times --- modules/ts/misc/xls-report.py | 89 ++++++++++++++++++++++++++++------- 1 file changed, 73 insertions(+), 16 deletions(-) diff --git a/modules/ts/misc/xls-report.py b/modules/ts/misc/xls-report.py index b120484317..d5db73d482 100755 --- a/modules/ts/misc/xls-report.py +++ b/modules/ts/misc/xls-report.py @@ -104,6 +104,7 @@ bad_speedup_style = xlwt.easyxf('font: color red', num_format_str='#0.00') no_speedup_style = no_time_style error_speedup_style = xlwt.easyxf('pattern: pattern solid, fore_color orange') header_style = xlwt.easyxf('font: bold true; alignment: horizontal centre, vertical top, wrap True') +subheader_style = xlwt.easyxf('alignment: horizontal centre, vertical top') class Collector(object): def __init__(self, config_match_func, include_unmatched): @@ -193,6 +194,8 @@ def main(): arg_parser.add_argument('-c', '--config', metavar='CONF', help='global configuration file') arg_parser.add_argument('--include-unmatched', action='store_true', help='include results from XML files that were not recognized by configuration matchers') + arg_parser.add_argument('--show-times-per-pixel', action='store_true', + help='for tests that have an image size parameter, show per-pixel time, as well as total time') args = arg_parser.parse_args() @@ -246,21 +249,53 @@ def main(): sheet.row(2).height = 800 sheet.panes_frozen = True sheet.remove_splits = True - sheet.horz_split_pos = 3 - sheet.horz_split_first_visible = 3 sheet_comparisons = sheet_conf.get('comparisons', []) - for i, w in enumerate([2500, 10000, 2500, 2000, 7500] - + (len(config_names) + 1 + len(sheet_comparisons)) * [4000]): - sheet.col(i).width = w + row = 2 - for i, caption in enumerate(['Module', 'Test', 'Image\nsize', 'Data\ntype', 'Other parameters'] - + config_names + [None] - + [comp['to'] + '\nvs\n' + comp['from'] for comp in sheet_comparisons]): - sheet.row(2).write(i, caption, header_style) + col = 0 - row = 3 + for (w, caption) in [ + (2500, 'Module'), + (10000, 'Test'), + (2500, 'Image\nsize'), + (2000, 'Data\ntype'), + (7500, 'Other parameters')]: + sheet.col(col).width = w + if args.show_times_per_pixel: + sheet.write_merge(row, row + 1, col, col, caption, header_style) + else: + sheet.write(row, col, caption, header_style) + col += 1 + + for config_name in config_names: + if args.show_times_per_pixel: + sheet.col(col).width = 3000 + sheet.col(col + 1).width = 3000 + sheet.write_merge(row, row, col, col + 1, config_name, header_style) + sheet.write(row + 1, col, 'total, ms', subheader_style) + sheet.write(row + 1, col + 1, 'per pixel, ns', subheader_style) + col += 2 + else: + sheet.col(col).width = 4000 + sheet.write(row, col, config_name, header_style) + col += 1 + + col += 1 # blank column between configurations and comparisons + + for comp in sheet_comparisons: + sheet.col(col).width = 4000 + caption = comp['to'] + '\nvs\n' + comp['from'] + if args.show_times_per_pixel: + sheet.write_merge(row, row + 1, col, col, caption, header_style) + else: + sheet.write(row, col, caption, header_style) + + row += 2 if args.show_times_per_pixel else 1 + + sheet.horz_split_pos = row + sheet.horz_split_first_visible = row module_colors = sheet_conf.get('module_colors', {}) module_styles = {module: xlwt.easyxf('pattern: pattern solid, fore_color {}'.format(color)) @@ -284,16 +319,36 @@ def main(): del param_list[param_list.index(data_type)] sheet.row(row).write(4, ' | '.join(param_list)) - for i, c in enumerate(config_names): + + col = 5 + + for c in config_names: if c in configs: - sheet.write(row, 5 + i, configs[c], time_style) + sheet.write(row, col, configs[c], time_style) else: - sheet.write(row, 5 + i, None, no_time_style) - - for i, comp in enumerate(sheet_comparisons): + sheet.write(row, col, None, no_time_style) + col += 1 + if args.show_times_per_pixel: + sheet.write(row, col, + xlwt.Formula( + ''' + {0} * 1000000 / ( + VALUE(MID({1}; 1; SEARCH("x"; {1}) - 1)) + * VALUE(MID({1}; SEARCH("x"; {1}) + 1; LEN({1}))) + ) + '''.replace('\n', '').replace(' ', '').format( + xlwt.Utils.rowcol_to_cell(row, col - 1), + xlwt.Utils.rowcol_to_cell(row, 2) + ) + ), + time_style) + col += 1 + + col += 1 # blank column + + for comp in sheet_comparisons: cmp_from = configs.get(comp["from"]) cmp_to = configs.get(comp["to"]) - col = 5 + len(config_names) + 1 + i if isinstance(cmp_from, numbers.Number) and isinstance(cmp_to, numbers.Number): try: @@ -306,6 +361,8 @@ def main(): else: sheet.write(row, col, None, no_speedup_style) + col += 1 + row += 1 if row % 1000 == 0: sheet.flush_row_data()