[Enhance] Accelerate analyze result (#7891)

* analyze_time

* accelerate

* update

* fix bug

* update

* unmerge

* update comments

* add unit test

mmdet/core/evaluation/mean_ap.py

@@ -62,7 +62,8 @@ def tpfp_imagenet(det_bboxes,
                   gt_bboxes_ignore=None,
                   default_iou_thr=0.5,
                   area_ranges=None,
-                  use_legacy_coordinate=False):
+                  use_legacy_coordinate=False,
+                  **kwargs):
     """Check if detected bboxes are true positive or false positive.
 
     Args:
@@ -170,7 +171,8 @@ def tpfp_default(det_bboxes,
                  gt_bboxes_ignore=None,
                  iou_thr=0.5,
                  area_ranges=None,
-                 use_legacy_coordinate=False):
+                 use_legacy_coordinate=False,
+                 **kwargs):
     """Check if detected bboxes are true positive or false positive.
 
     Args:
@@ -275,7 +277,8 @@ def tpfp_openimages(det_bboxes,
                     use_legacy_coordinate=False,
                     gt_bboxes_group_of=None,
                     use_group_of=True,
-                    ioa_thr=0.5):
+                    ioa_thr=0.5,
+                    **kwargs):
     """Check if detected bboxes are true positive or false positive.
 
     Args:
@@ -585,7 +588,13 @@ def eval_map(det_results,
     area_ranges = ([(rg[0]**2, rg[1]**2) for rg in scale_ranges]
                    if scale_ranges is not None else None)
 
+    # There is no need to use multi processes to process
+    # when num_imgs = 1 .
+    if num_imgs > 1:
+        assert nproc > 0, 'nproc must be at least one.'
+        nproc = min(nproc, num_imgs)
         pool = Pool(nproc)
+
     eval_results = []
     for i in range(num_classes):
         # get gt and det bboxes of this class
@@ -603,6 +612,9 @@ def eval_map(det_results,
         if not callable(tpfp_fn):
             raise ValueError(
                 f'tpfp_fn has to be a function or None, but got {tpfp_fn}')
+
+        if num_imgs > 1:
+            # compute tp and fp for each image with multiple processes
             args = []
             if use_group_of:
                 # used in Open Images Dataset evaluation
@@ -611,13 +623,27 @@ def eval_map(det_results,
                 args.append([use_group_of for _ in range(num_imgs)])
             if ioa_thr is not None:
                 args.append([ioa_thr for _ in range(num_imgs)])
-        # compute tp and fp for each image with multiple processes
+
             tpfp = pool.starmap(
                 tpfp_fn,
                 zip(cls_dets, cls_gts, cls_gts_ignore,
                     [iou_thr for _ in range(num_imgs)],
                     [area_ranges for _ in range(num_imgs)],
                     [use_legacy_coordinate for _ in range(num_imgs)], *args))
+        else:
+            tpfp = tpfp_fn(
+                cls_dets[0],
+                cls_gts[0],
+                cls_gts_ignore[0],
+                iou_thr,
+                area_ranges,
+                use_legacy_coordinate,
+                gt_bboxes_group_of=(get_cls_group_ofs(annotations, i)[0]
+                                    if use_group_of else None),
+                use_group_of=use_group_of,
+                ioa_thr=ioa_thr)
+            tpfp = [tpfp]
+
         if use_group_of:
             tp, fp, cls_dets = tuple(zip(*tpfp))
         else:
@@ -660,7 +686,10 @@ def eval_map(det_results,
             'precision': precisions,
             'ap': ap
         })
+
+    if num_imgs > 1:
         pool.close()
+
     if scale_ranges is not None:
         # shape (num_classes, num_scales)
         all_ap = np.vstack([cls_result['ap'] for cls_result in eval_results])

tests/test_metrics/test_mean_ap.py

@@ -84,7 +84,27 @@ def test_eval_map():
     mean_ap, eval_results = eval_map(
         det_results, annotations, use_legacy_coordinate=True)
     assert 0.291 < mean_ap < 0.293
-    eval_map(det_results, annotations, use_legacy_coordinate=False)
+    mean_ap, eval_results = eval_map(
+        det_results, annotations, use_legacy_coordinate=False)
+    assert 0.291 < mean_ap < 0.293
+
+    # 1 image and 2 classes
+    det_results = [[det_bboxes, det_bboxes]]
+
+    labels = np.array([0, 1, 1])
+    labels_ignore = np.array([0, 1])
+    gt_info = {
+        'bboxes': gt_bboxes,
+        'bboxes_ignore': gt_ignore,
+        'labels': labels,
+        'labels_ignore': labels_ignore
+    }
+    annotations = [gt_info]
+    mean_ap, eval_results = eval_map(
+        det_results, annotations, use_legacy_coordinate=True)
+    assert 0.291 < mean_ap < 0.293
+    mean_ap, eval_results = eval_map(
+        det_results, annotations, use_legacy_coordinate=False)
     assert 0.291 < mean_ap < 0.293
 
 

tools/analysis_tools/analyze_results.py

@@ -1,6 +1,7 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import argparse
 import os.path as osp
+from multiprocessing import Pool
 
 import mmcv
 import numpy as np
@@ -12,7 +13,7 @@ from mmdet.datasets import build_dataset, get_loading_pipeline
 from mmdet.utils import replace_cfg_vals, update_data_root
 
 
-def bbox_map_eval(det_result, annotation):
+def bbox_map_eval(det_result, annotation, nproc=4):
     """Evaluate mAP of single image det result.
 
     Args:
@@ -27,6 +28,9 @@ def bbox_map_eval(det_result, annotation):
             - bboxes_ignore (optional): numpy array of shape (k, 4)
             - labels_ignore (optional): numpy array of shape (k, )
 
+        nproc (int): Processes used for computing mAP.
+            Default: 4.
+
     Returns:
         float: mAP
     """
@@ -39,11 +43,24 @@ def bbox_map_eval(det_result, annotation):
     # mAP
     iou_thrs = np.linspace(
         .5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True)
-    mean_aps = []
+
+    processes = []
+    workers = Pool(processes=nproc)
     for thr in iou_thrs:
-        mean_ap, _ = eval_map(
-            bbox_det_result, [annotation], iou_thr=thr, logger='silent')
-        mean_aps.append(mean_ap)
+        p = workers.apply_async(eval_map, (bbox_det_result, [annotation]), {
+            'iou_thr': thr,
+            'logger': 'silent',
+            'nproc': 1
+        })
+        processes.append(p)
+
+    workers.close()
+    workers.join()
+
+    mean_aps = []
+    for p in processes:
+        mean_aps.append(p.get()[0])
+
     return sum(mean_aps) / len(mean_aps)
 
 
@@ -130,7 +147,6 @@ class ResultVisualizer:
             mAP = eval_fn(result, data_info['ann_info'])
             _mAPs[i] = mAP
             prog_bar.update()
-
         # descending select topk image
         _mAPs = list(sorted(_mAPs.items(), key=lambda kv: kv[1]))
         good_mAPs = _mAPs[-topk:]