From 8ba124d82bdd48abeb85dc7a86bda27cfa4d6847 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Thu, 14 Jul 2022 21:39:55 +0800 Subject: [PATCH] Init unittest --- paddlers/custom_models/seg/farseg.py | 2 +- paddlers/tasks/slim/prune.py | 2 +- .../tasks/utils/det_metrics/coco_utils.py | 79 ++++---- paddlers/tasks/utils/visualize.py | 11 +- tests/check_coverage.sh | 5 + tests/components/__init__.py | 13 ++ tests/datasets/__init__.py | 13 ++ tests/deploy/__init__.py | 13 ++ tests/rs_models/__init__.py | 13 ++ tests/run_tests.sh | 3 + tests/tasks/__init__.py | 13 ++ tests/test_examples.py | 13 ++ tests/test_tutorials.py | 13 ++ tests/test_utils.py | 183 ++++++++++++++++++ tests/tools/__init__.py | 13 ++ tests/transforms/__init__.py | 13 ++ 16 files changed, 359 insertions(+), 43 deletions(-) create mode 100644 tests/check_coverage.sh create mode 100644 tests/components/__init__.py create mode 100644 tests/datasets/__init__.py create mode 100644 tests/deploy/__init__.py create mode 100644 tests/rs_models/__init__.py create mode 100644 tests/run_tests.sh create mode 100644 tests/tasks/__init__.py create mode 100644 tests/test_examples.py create mode 100644 tests/test_tutorials.py create mode 100644 tests/test_utils.py create mode 100644 tests/tools/__init__.py create mode 100644 tests/transforms/__init__.py diff --git a/paddlers/custom_models/seg/farseg.py b/paddlers/custom_models/seg/farseg.py index 2e0161f..ad84813 100644 --- a/paddlers/custom_models/seg/farseg.py +++ b/paddlers/custom_models/seg/farseg.py @@ -32,7 +32,7 @@ class FPN(nn.Layer): """ Module that adds FPN on top of a list of feature maps. The feature maps are currently supposed to be in increasing depth - order, and must be consecutive + order, and must be consecutive """ def __init__(self, diff --git a/paddlers/tasks/slim/prune.py b/paddlers/tasks/slim/prune.py index 9792d0f..10df7fe 100644 --- a/paddlers/tasks/slim/prune.py +++ b/paddlers/tasks/slim/prune.py @@ -41,7 +41,7 @@ def _pruner_template_input(sample, model_type): def sensitive_prune(pruner, pruned_flops, skip_vars=[], align=None): - # skip depthwise convolutions + # Skip depthwise convolutions for layer in pruner.model.sublayers(): if isinstance(layer, paddle.nn.layer.conv.Conv2D) and layer._groups > 1: for param in layer.parameters(include_sublayers=False): diff --git a/paddlers/tasks/utils/det_metrics/coco_utils.py b/paddlers/tasks/utils/det_metrics/coco_utils.py index c4a024f..f62e8bf 100644 --- a/paddlers/tasks/utils/det_metrics/coco_utils.py +++ b/paddlers/tasks/utils/det_metrics/coco_utils.py @@ -35,6 +35,7 @@ def get_infer_results(outs, catid, bias=0): For example, bbox result is a list and each element contains image_id, category_id, bbox and score. """ + if outs is None or len(outs) == 0: raise ValueError( 'The number of valid detection result if zero. Please use reasonable model and check input data.' @@ -78,6 +79,7 @@ def cocoapi_eval(anns, max_dets (tuple): COCO evaluation maxDets. classwise (bool): Whether per-category AP and draw P-R Curve or not. """ + assert coco_gt is not None or anno_file is not None from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval @@ -220,19 +222,19 @@ def loadRes(coco_obj, anns): def makeplot(rs, ps, outDir, class_name, iou_type): - """针对某个特定类别，绘制不同评估要求下的准确率和召回率。 - 绘制结果说明参考COCODataset官网给出分析工具说明https://cocodataset.org/#detection-eval。 - - Refer to https://github.com/open-mmlab/mmdetection/blob/master/tools/analysis_tools/coco_error_analysis.py#L13 + """ + 针对某个特定类别，绘制不同评估要求下的准确率和召回率。 + 绘制结果说明参考COCODataset官网给出分析工具说明https://cocodataset.org/#detection-eval。 - Args: - rs (np.array): 在不同置信度阈值下计算得到的召回率。 - ps (np.array): 在不同置信度阈值下计算得到的准确率。ps与rs相同位置下的数值为同一个置信度阈值 - 计算得到的准确率与召回率。 - outDir (str): 图表保存的路径。 - class_name (str): 类别名。 - iou_type (str): iou计算方式，若为检测框，则设置为'bbox'，若为像素级分割结果，则设置为'segm'。 + Refer to https://github.com/open-mmlab/mmdetection/blob/master/tools/analysis_tools/coco_error_analysis.py#L13 + Args: + rs (np.array): 在不同置信度阈值下计算得到的召回率。 + ps (np.array): 在不同置信度阈值下计算得到的准确率。ps与rs相同位置下的数值为同一个置信度阈值 + 计算得到的准确率与召回率。 + outDir (str): 图表保存的路径。 + class_name (str): 类别名。 + iou_type (str): iou计算方式，若为检测框，则设置为'bbox'，若为像素级分割结果，则设置为'segm'。 """ import matplotlib.pyplot as plt @@ -276,21 +278,22 @@ def makeplot(rs, ps, outDir, class_name, iou_type): def analyze_individual_category(k, cocoDt, cocoGt, catId, iou_type, areas=None): - """针对某个特定类别，分析忽略亚类混淆和类别混淆时的准确率。 + """ + 针对某个特定类别，分析忽略亚类混淆和类别混淆时的准确率。 - Refer to https://github.com/open-mmlab/mmdetection/blob/master/tools/analysis_tools/coco_error_analysis.py#L174 + Refer to https://github.com/open-mmlab/mmdetection/blob/master/tools/analysis_tools/coco_error_analysis.py#L174 - Args: - k (int): 待分析类别的序号。 - cocoDt (pycocotols.coco.COCO): 按COCO类存放的预测结果。 - cocoGt (pycocotols.coco.COCO): 按COCO类存放的真值。 - catId (int): 待分析类别在数据集中的类别id。 - iou_type (str): iou计算方式，若为检测框，则设置为'bbox'，若为像素级分割结果，则设置为'segm'。 + Args: + k (int): 待分析类别的序号。 + cocoDt (pycocotols.coco.COCO): 按COCO类存放的预测结果。 + cocoGt (pycocotols.coco.COCO): 按COCO类存放的真值。 + catId (int): 待分析类别在数据集中的类别id。 + iou_type (str): iou计算方式，若为检测框，则设置为'bbox'，若为像素级分割结果，则设置为'segm'。 - Returns: - int: - dict: 有关键字'ps_supercategory'和'ps_allcategory'。关键字'ps_supercategory'的键值是忽略亚类间 - 混淆时的准确率，关键字'ps_allcategory'的键值是忽略类别间混淆时的准确率。 + Returns: + int: + dict: 有关键字'ps_supercategory'和'ps_allcategory'。关键字'ps_supercategory'的键值是忽略亚类间 + 混淆时的准确率，关键字'ps_allcategory'的键值是忽略类别间混淆时的准确率。 """ @@ -362,23 +365,23 @@ def coco_error_analysis(eval_details_file=None, pred_bbox=None, pred_mask=None, save_dir='./output'): - """逐个分析模型预测错误的原因，并将分析结果以图表的形式展示。 - 分析结果说明参考COCODataset官网给出分析工具说明https://cocodataset.org/#detection-eval。 - - Refer to https://github.com/open-mmlab/mmdetection/blob/master/tools/analysis_tools/coco_error_analysis.py - - Args: - eval_details_file (str): 模型评估结果的保存路径，包含真值信息和预测结果。 - gt (list): 数据集的真值信息。默认值为None。 - pred_bbox (list): 模型在数据集上的预测框。默认值为None。 - pred_mask (list): 模型在数据集上的预测mask。默认值为None。 - save_dir (str): 可视化结果保存路径。默认值为'./output'。 + """ + 逐个分析模型预测错误的原因，并将分析结果以图表的形式展示。 + 分析结果说明参考COCODataset官网给出分析工具说明https://cocodataset.org/#detection-eval。 - Note: - eval_details_file的优先级更高，只要eval_details_file不为None， - 就会从eval_details_file提取真值信息和预测结果做分析。 - 当eval_details_file为None时，则用gt、pred_mask、pred_mask做分析。 + Refer to https://github.com/open-mmlab/mmdetection/blob/master/tools/analysis_tools/coco_error_analysis.py + Args: + eval_details_file (str): 模型评估结果的保存路径，包含真值信息和预测结果。 + gt (list): 数据集的真值信息。默认值为None。 + pred_bbox (list): 模型在数据集上的预测框。默认值为None。 + pred_mask (list): 模型在数据集上的预测mask。默认值为None。 + save_dir (str): 可视化结果保存路径。默认值为'./output'。 + + Note: + eval_details_file的优先级更高，只要eval_details_file不为None， + 就会从eval_details_file提取真值信息和预测结果做分析。 + 当eval_details_file为None时，则用gt、pred_mask、pred_mask做分析。 """ import multiprocessing as mp diff --git a/paddlers/tasks/utils/visualize.py b/paddlers/tasks/utils/visualize.py index fb76f96..2c76995 100644 --- a/paddlers/tasks/utils/visualize.py +++ b/paddlers/tasks/utils/visualize.py @@ -25,7 +25,7 @@ from .det_metrics.coco_utils import loadRes def visualize_detection(image, result, threshold=0.5, save_dir='./', color=None): """ - Visualize bbox and mask results + Visualize bbox and mask results """ if isinstance(image, np.ndarray): @@ -48,6 +48,7 @@ def visualize_segmentation(image, result, weight=0.6, save_dir='./', color=None): """ Convert segment result to color image, and save added image. + Args: image: the path of origin image result: the predict result of image @@ -55,6 +56,7 @@ def visualize_segmentation(image, result, weight=0.6, save_dir='./', save_dir: the directory for saving visual image color: the list of a BGR-mode color for each label. """ + label_map = result['label_map'].astype("uint8") color_map = get_color_map_list(256) if color is not None: @@ -104,13 +106,16 @@ def visualize_segmentation(image, result, weight=0.6, save_dir='./', def get_color_map_list(num_classes): - """ Returns the color map for visualizing the segmentation mask, - which can support arbitrary number of classes. + """ + Returns the color map for visualizing the segmentation mask, which can support arbitrary number of classes. + Args: num_classes: Number of classes + Returns: The color map """ + color_map = num_classes * [0, 0, 0] for i in range(0, num_classes): j = 0 diff --git a/tests/check_coverage.sh b/tests/check_coverage.sh new file mode 100644 index 0000000..7da4808 --- /dev/null +++ b/tests/check_coverage.sh @@ -0,0 +1,5 @@ +#!/usr/bin/bash + +coverage run -m unittest discover +coverage report +coverage html -d coverage_html \ No newline at end of file diff --git a/tests/components/__init__.py b/tests/components/__init__.py new file mode 100644 index 0000000..29c8b7d --- /dev/null +++ b/tests/components/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/datasets/__init__.py b/tests/datasets/__init__.py new file mode 100644 index 0000000..29c8b7d --- /dev/null +++ b/tests/datasets/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/deploy/__init__.py b/tests/deploy/__init__.py new file mode 100644 index 0000000..29c8b7d --- /dev/null +++ b/tests/deploy/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/rs_models/__init__.py b/tests/rs_models/__init__.py new file mode 100644 index 0000000..29c8b7d --- /dev/null +++ b/tests/rs_models/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/run_tests.sh b/tests/run_tests.sh new file mode 100644 index 0000000..204fd58 --- /dev/null +++ b/tests/run_tests.sh @@ -0,0 +1,3 @@ +#!/usr/bin/bash + +python -m unittest discover -v \ No newline at end of file diff --git a/tests/tasks/__init__.py b/tests/tasks/__init__.py new file mode 100644 index 0000000..29c8b7d --- /dev/null +++ b/tests/tasks/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/test_examples.py b/tests/test_examples.py new file mode 100644 index 0000000..eeae9aa --- /dev/null +++ b/tests/test_examples.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/tests/test_tutorials.py b/tests/test_tutorials.py new file mode 100644 index 0000000..eeae9aa --- /dev/null +++ b/tests/test_tutorials.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..5e5dafb --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,183 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Based on https://github.com/PaddlePaddle/PaddleNLP/blob/develop/tests/common_test.py + +import unittest +import warnings + +import numpy as np +import paddle + +__all__ = ['CommonTest', 'CpuCommonTest'] + + +# Assume all elements has same data type +def get_container_type(container): + container_t = type(container) + if container_t in [list, tuple]: + if len(container) == 0: + return container_t + return get_container_type(container[0]) + return container_t + + +class _CommonTestNamespace: + # Wrap the subclasses of unittest.TestCase that are expected to be inherited from. + class CommonTest(unittest.TestCase): + CATCH_WARNINGS = False + + def __init__(self, methodName='runTest'): + super(CommonTest, self).__init__(methodName=methodName) + self.config = {} + self.places = ['cpu'] + if paddle.is_compiled_with_cuda(): + self.places.append('gpu') + + @classmethod + def setUpClass(cls): + ''' + Set the decorators for all test function + ''' + for key, value in cls.__dict__.items(): + if key.startswith('test'): + decorator_func_list = ["_test_places"] + if cls.CATCH_WARNINGS: + decorator_func_list.append("_catch_warnings") + for decorator_func in decorator_func_list: + decorator_func = getattr(CommonTest, decorator_func) + value = decorator_func(value) + setattr(cls, key, value) + + def _catch_warnings(func): + ''' + Catch the warnings and treat them as errors for each test. + ''' + + def wrapper(self, *args, **kwargs): + with warnings.catch_warnings(record=True) as w: + warnings.resetwarnings() + # ignore specified warnings + warning_white_list = [UserWarning] + for warning in warning_white_list: + warnings.simplefilter("ignore", warning) + func(self, *args, **kwargs) + msg = None if len(w) == 0 else w[0].message + self.assertFalse(len(w) > 0, msg) + + return wrapper + + def _test_places(func): + ''' + Setting the running place for each test. + ''' + + def wrapper(self, *args, **kwargs): + places = self.places + for place in places: + paddle.set_device(place) + func(self, *args, **kwargs) + + return wrapper + + def _check_output_impl(self, + result, + expected_result, + rtol, + atol, + equal=True): + assertForNormalType = self.assertNotEqual + assertForFloat = self.assertFalse + if equal: + assertForNormalType = self.assertEqual + assertForFloat = self.assertTrue + + result_t = type(result) + error_msg = 'Output has diff at place:{}. \nExpect: {} \nBut Got: {} in class {}' + if result_t in [list, tuple]: + result_t = get_container_type(result) + if result_t in [ + str, int, bool, set, np.bool, np.int32, np.int64, np.str + ]: + assertForNormalType( + result, + expected_result, + msg=error_msg.format(paddle.get_device(), expected_result, + result, self.__class__.__name__)) + elif result_t in [float, np.ndarray, np.float32, np.float64]: + assertForFloat( + np.allclose( + result, expected_result, rtol=rtol, atol=atol), + msg=error_msg.format(paddle.get_device(), expected_result, + result, self.__class__.__name__)) + if result_t == np.ndarray: + assertForNormalType( + result.shape, + expected_result.shape, + msg=error_msg.format( + paddle.get_device(), expected_result.shape, + result.shape, self.__class__.__name__)) + else: + raise ValueError( + 'result type must be str, int, bool, set, np.bool, np.int32, ' + 'np.int64, np.str, float, np.ndarray, np.float32, np.float64' + ) + + def check_output_equal(self, + result, + expected_result, + rtol=1.e-5, + atol=1.e-8): + ''' + Check whether result and expected result are equal, including shape. + Args: + result: str, int, bool, set, np.ndarray. + The result needs to be checked. + expected_result: str, int, bool, set, np.ndarray. The type has to be same as result's. + Use the expected result to check result. + rtol: float + relative tolerance, default 1.e-5. + atol: float + absolute tolerance, default 1.e-8 + ''' + self._check_output_impl(result, expected_result, rtol, atol) + + def check_output_not_equal(self, + result, + expected_result, + rtol=1.e-5, + atol=1.e-8): + ''' + Check whether result and expected result are not equal, including shape. + Args: + result: str, int, bool, set, np.ndarray. + The result needs to be checked. + expected_result: str, int, bool, set, np.ndarray. The type has to be same as result's. + Use the expected result to check result. + rtol: float + relative tolerance, default 1.e-5. + atol: float + absolute tolerance, default 1.e-8 + ''' + self._check_output_impl( + result, expected_result, rtol, atol, equal=False) + + class CpuCommonTest(CommonTest): + def __init__(self, methodName='runTest'): + super(CpuCommonTest, self).__init__(methodName=methodName) + self.places = ['cpu'] + + +CommonTest = _CommonTestNamespace.CommonTest +CpuCommonTest = _CommonTestNamespace.CpuCommonTest diff --git a/tests/tools/__init__.py b/tests/tools/__init__.py new file mode 100644 index 0000000..29c8b7d --- /dev/null +++ b/tests/tools/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/transforms/__init__.py b/tests/transforms/__init__.py new file mode 100644 index 0000000..29c8b7d --- /dev/null +++ b/tests/transforms/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.