You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

352 lines
16 KiB

# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os.path as osp
import tempfile
import unittest.mock as mock
import paddle
import paddlers as pdrs
2 years ago
from paddlers.transforms import decode_image
from testing_utils import CommonTest, run_script
Squashed commit of the following: commit de5651935774034c31be300c421529aafa54b43f Author: Bobholamovic <bob1998425@hotmail.com> Date: Mon Jul 18 02:14:40 2022 +0800 Update workflow commit 25d7f0cdd2fee5e01eaf4c222baec14043c718ad Author: Bobholamovic <bob1998425@hotmail.com> Date: Mon Jul 18 01:57:45 2022 +0800 Test cuda commit e908e613797cf4b8d60bee012b54ebfc5888f021 Author: Bobholamovic <bob1998425@hotmail.com> Date: Mon Jul 18 01:39:43 2022 +0800 Fix bug commit 563fb86d03fd509283f626443029e82e3cfcae48 Author: Bobholamovic <bob1998425@hotmail.com> Date: Mon Jul 18 01:33:05 2022 +0800 Test deploy commit 8d285f94604552bc5944768bb89251ab1a53c1eb Author: Bobholamovic <bob1998425@hotmail.com> Date: Mon Jul 18 01:08:12 2022 +0800 Do not check tools commit 227d98eb7967f0444f5bf7c04fc17d98f6f6dfc1 Author: Bobholamovic <bob1998425@hotmail.com> Date: Mon Jul 18 00:36:51 2022 +0800 test_static_forward->test_to_static commit 91a3cfafb22a645eebaaaff887ca7c7ace2ccb44 Author: Bobholamovic <bob1998425@hotmail.com> Date: Mon Jul 18 00:13:04 2022 +0800 Fix bugs in test_models commit c0bd01ef9d9493f1fe8dc4301a51213cd0814ced Author: Bobholamovic <bob1998425@hotmail.com> Date: Sun Jul 17 23:45:17 2022 +0800 Update URL commit b2718b73a9d93f4c27bc0957b7b9349d4be028ba Author: Bobholamovic <bob1998425@hotmail.com> Date: Sun Jul 17 23:36:32 2022 +0800 Update workflow commit 25ad5ae1477719c1e90e0b1005667c46508e4d9e Author: Bobholamovic <bob1998425@hotmail.com> Date: Sun Jul 17 23:31:31 2022 +0800 Fix bugs commit 4780b365a73557d71f2a1156657cadb2e235e9d1 Author: Bobholamovic <bob1998425@hotmail.com> Date: Sun Jul 17 23:05:00 2022 +0800 Update workflow commit bb79a83447e2efc12d0e1c3a7ed52effcac4e635 Author: Bobholamovic <bob1998425@hotmail.com> Date: Sun Jul 17 22:48:31 2022 +0800 Update workflow commit 1dec49161a7c03c651910772064149f86a2545c0 Author: Bobholamovic <bob1998425@hotmail.com> Date: Sun Jul 17 22:39:17 2022 +0800 Update workflow commit 9df503913be03f147ed4d7d01a4b765f0a03cc1e Author: Bobholamovic <bob1998425@hotmail.com> Date: Sun Jul 17 22:26:07 2022 +0800 Fast tests commit 665a872e6dbf0c326d12c4a86d99315bed6be417 Author: Bobholamovic <bob1998425@hotmail.com> Date: Sun Jul 17 21:16:18 2022 +0800 Update workflows commit d8978cbb2c4638a6cba9bc1378eedeefa36bee60 Author: Bobholamovic <bob1998425@hotmail.com> Date: Sun Jul 17 20:52:30 2022 +0800 Add empty file commit d704501b570a2b9314b3b0974e775a58e2cccfaf Author: Bobholamovic <bob1998425@hotmail.com> Date: Sun Jul 17 20:47:11 2022 +0800 Add install coverage commit dd7c2c4bfd89eb0236ba8f39155a72ba04bf9bce Author: Bobholamovic <bob1998425@hotmail.com> Date: Sun Jul 17 20:39:45 2022 +0800 Add yapf disable commit 1bdebad783ddeab165acafbadd69b42f587bb742 Author: Bobholamovic <bob1998425@hotmail.com> Date: Sun Jul 17 20:19:57 2022 +0800 Apply pre-commit check commit 2de597a5d67bbe2c9bd8ac3f6f3b27b886e9b7d4 Merge: 5f06a86 dfb6978 Author: Bobholamovic <bob1998425@hotmail.com> Date: Sun Jul 17 19:52:54 2022 +0800 Merge branch 'unittest' into develop commit 5f06a861735bb80cde0bca63889ac647664958ad Author: Lin Manhui <mhlin425@whu.edu.cn> Date: Sun Jul 17 19:23:06 2022 +0800 Add unittests (#2) * Init unittest * Add rs_model unittests * Refactor tools * Add data for unittests * Add transforms unittests * test_utils->testing_utils * Add tool unittests * Add scripts for unittests * Rename and polish doc/comments * Update test_operators.py * Add coco det data * Add tools unittests * Update style * Update rs_models unittests * Add predictor unittests * Add tutorial tests * Add github workflow
2 years ago
__all__ = [
'TestCDPredictor', 'TestClasPredictor', 'TestDetPredictor',
'TestSegPredictor'
]
class TestPredictor(CommonTest):
MODULE = pdrs.tasks
TRAINER_NAME_TO_EXPORT_OPTS = {}
2 years ago
WHITE_LIST = []
@staticmethod
def add_tests(cls):
"""
Automatically patch testing functions to cls.
"""
def _test_predictor(trainer_name):
def _test_predictor_impl(self):
trainer_class = getattr(self.MODULE, trainer_name)
# Construct trainer with default parameters
2 years ago
# TODO: Load pretrained weights to avoid numeric problems
trainer = trainer_class()
with tempfile.TemporaryDirectory() as td:
dynamic_model_dir = osp.join(td, "dynamic")
static_model_dir = osp.join(td, "static")
# HACK: BaseModel.save_model() requires BaseModel().optimizer to be set
optimizer = mock.Mock()
optimizer.state_dict.return_value = {'foo': 'bar'}
trainer.optimizer = optimizer
trainer.save_model(dynamic_model_dir)
export_cmd = f"python export_model.py --model_dir {dynamic_model_dir} --save_dir {static_model_dir} "
if trainer_name in self.TRAINER_NAME_TO_EXPORT_OPTS:
export_cmd += self.TRAINER_NAME_TO_EXPORT_OPTS[
trainer_name]
elif '_default' in self.TRAINER_NAME_TO_EXPORT_OPTS:
export_cmd += self.TRAINER_NAME_TO_EXPORT_OPTS[
'_default']
run_script(export_cmd, wd="../deploy/export")
# Construct predictor
# TODO: Test trt and mkl
predictor = pdrs.deploy.Predictor(
static_model_dir,
use_gpu=paddle.device.get_device().startswith('gpu'))
self.check_predictor(predictor, trainer)
return _test_predictor_impl
for trainer_name in cls.MODULE.__all__:
2 years ago
if trainer_name in cls.WHITE_LIST:
continue
setattr(cls, 'test_' + trainer_name, _test_predictor(trainer_name))
return cls
def check_predictor(self, predictor, trainer):
raise NotImplementedError
2 years ago
def check_dict_equal(
self,
dict_,
expected_dict,
ignore_keys=('label_map', 'mask', 'category', 'category_id')):
# By default do not compare label_maps, masks, or categories,
# because numeric errors could result in large difference in labels.
if isinstance(dict_, list):
self.assertIsInstance(expected_dict, list)
self.assertEqual(len(dict_), len(expected_dict))
for d1, d2 in zip(dict_, expected_dict):
2 years ago
self.check_dict_equal(d1, d2, ignore_keys=ignore_keys)
else:
assert isinstance(dict_, dict)
assert isinstance(expected_dict, dict)
self.assertEqual(dict_.keys(), expected_dict.keys())
2 years ago
ignore_keys = set() if ignore_keys is None else set(ignore_keys)
for key in dict_.keys():
2 years ago
if key in ignore_keys:
continue
2 years ago
# Use higher tolerance
self.check_output_equal(
dict_[key], expected_dict[key], rtol=1.e-4, atol=1.e-6)
@TestPredictor.add_tests
class TestCDPredictor(TestPredictor):
MODULE = pdrs.tasks.change_detector
TRAINER_NAME_TO_EXPORT_OPTS = {
'_default': "--fixed_input_shape [-1,3,256,256]"
}
2 years ago
# HACK: Skip CDNet.
# These models are heavily affected by numeric errors.
WHITE_LIST = ['CDNet']
def check_predictor(self, predictor, trainer):
t1_path = "data/ssmt/optical_t1.bmp"
t2_path = "data/ssmt/optical_t2.bmp"
single_input = (t1_path, t2_path)
num_inputs = 2
transforms = pdrs.transforms.Compose([pdrs.transforms.Normalize()])
# Expected failure
with self.assertRaises(ValueError):
predictor.predict(t1_path, transforms=transforms)
# Single input (file paths)
input_ = single_input
out_single_file_p = predictor.predict(input_, transforms=transforms)
out_single_file_t = trainer.predict(input_, transforms=transforms)
self.check_dict_equal(out_single_file_p, out_single_file_t)
out_single_file_list_p = predictor.predict(
[input_], transforms=transforms)
self.assertEqual(len(out_single_file_list_p), 1)
self.check_dict_equal(out_single_file_list_p[0], out_single_file_p)
out_single_file_list_t = trainer.predict(
[input_], transforms=transforms)
self.check_dict_equal(out_single_file_list_p[0],
out_single_file_list_t[0])
# Single input (ndarrays)
2 years ago
input_ = (decode_image(
t1_path, to_rgb=False), decode_image(
t2_path, to_rgb=False)) # Reuse the name `input_`
out_single_array_p = predictor.predict(input_, transforms=transforms)
self.check_dict_equal(out_single_array_p, out_single_file_p)
out_single_array_t = trainer.predict(input_, transforms=transforms)
self.check_dict_equal(out_single_array_p, out_single_array_t)
out_single_array_list_p = predictor.predict(
[input_], transforms=transforms)
self.assertEqual(len(out_single_array_list_p), 1)
self.check_dict_equal(out_single_array_list_p[0], out_single_array_p)
out_single_array_list_t = trainer.predict(
[input_], transforms=transforms)
self.check_dict_equal(out_single_array_list_p[0],
out_single_array_list_t[0])
# Multiple inputs (file paths)
input_ = [single_input] * num_inputs # Reuse the name `input_`
out_multi_file_p = predictor.predict(input_, transforms=transforms)
self.assertEqual(len(out_multi_file_p), num_inputs)
out_multi_file_t = trainer.predict(input_, transforms=transforms)
2 years ago
self.assertEqual(len(out_multi_file_t), num_inputs)
# Multiple inputs (ndarrays)
2 years ago
input_ = [(decode_image(
t1_path, to_rgb=False), decode_image(
t2_path, to_rgb=False))] * num_inputs # Reuse the name `input_`
out_multi_array_p = predictor.predict(input_, transforms=transforms)
self.assertEqual(len(out_multi_array_p), num_inputs)
out_multi_array_t = trainer.predict(input_, transforms=transforms)
2 years ago
self.assertEqual(len(out_multi_array_t), num_inputs)
@TestPredictor.add_tests
class TestClasPredictor(TestPredictor):
MODULE = pdrs.tasks.classifier
TRAINER_NAME_TO_EXPORT_OPTS = {
'_default': "--fixed_input_shape [-1,3,256,256]"
}
def check_predictor(self, predictor, trainer):
single_input = "data/ssmt/optical_t1.bmp"
num_inputs = 2
transforms = pdrs.transforms.Compose([pdrs.transforms.Normalize()])
labels = list(range(2))
trainer.labels = labels
predictor._model.labels = labels
# Single input (file path)
input_ = single_input
out_single_file_p = predictor.predict(input_, transforms=transforms)
out_single_file_t = trainer.predict(input_, transforms=transforms)
self.check_dict_equal(out_single_file_p, out_single_file_t)
out_single_file_list_p = predictor.predict(
[input_], transforms=transforms)
self.assertEqual(len(out_single_file_list_p), 1)
self.check_dict_equal(out_single_file_list_p[0], out_single_file_p)
out_single_file_list_t = trainer.predict(
[input_], transforms=transforms)
self.check_dict_equal(out_single_file_list_p[0],
out_single_file_list_t[0])
# Single input (ndarray)
2 years ago
input_ = decode_image(
single_input, to_rgb=False) # Reuse the name `input_`
out_single_array_p = predictor.predict(input_, transforms=transforms)
self.check_dict_equal(out_single_array_p, out_single_file_p)
out_single_array_t = trainer.predict(input_, transforms=transforms)
self.check_dict_equal(out_single_array_p, out_single_array_t)
out_single_array_list_p = predictor.predict(
[input_], transforms=transforms)
self.assertEqual(len(out_single_array_list_p), 1)
self.check_dict_equal(out_single_array_list_p[0], out_single_array_p)
out_single_array_list_t = trainer.predict(
[input_], transforms=transforms)
self.check_dict_equal(out_single_array_list_p[0],
out_single_array_list_t[0])
# Multiple inputs (file paths)
input_ = [single_input] * num_inputs # Reuse the name `input_`
out_multi_file_p = predictor.predict(input_, transforms=transforms)
self.assertEqual(len(out_multi_file_p), num_inputs)
out_multi_file_t = trainer.predict(input_, transforms=transforms)
2 years ago
# Check value consistence
self.check_dict_equal(out_multi_file_p, out_multi_file_t)
# Multiple inputs (ndarrays)
2 years ago
input_ = [decode_image(
single_input, to_rgb=False)] * num_inputs # Reuse the name `input_`
out_multi_array_p = predictor.predict(input_, transforms=transforms)
self.assertEqual(len(out_multi_array_p), num_inputs)
out_multi_array_t = trainer.predict(input_, transforms=transforms)
self.check_dict_equal(out_multi_array_p, out_multi_array_t)
@TestPredictor.add_tests
class TestDetPredictor(TestPredictor):
MODULE = pdrs.tasks.object_detector
TRAINER_NAME_TO_EXPORT_OPTS = {
'_default': "--fixed_input_shape [-1,3,256,256]"
}
def check_predictor(self, predictor, trainer):
2 years ago
# For detection tasks, do NOT ensure the consistence of bboxes.
# This is because the coordinates of bboxes were observed to be very sensitive to numeric errors,
# given that the network is (partially?) randomly initialized.
single_input = "data/ssmt/optical_t1.bmp"
num_inputs = 2
transforms = pdrs.transforms.Compose([pdrs.transforms.Normalize()])
labels = list(range(80))
trainer.labels = labels
predictor._model.labels = labels
# Single input (file path)
input_ = single_input
2 years ago
predictor.predict(input_, transforms=transforms)
trainer.predict(input_, transforms=transforms)
out_single_file_list_p = predictor.predict(
[input_], transforms=transforms)
self.assertEqual(len(out_single_file_list_p), 1)
out_single_file_list_t = trainer.predict(
[input_], transforms=transforms)
2 years ago
self.assertEqual(len(out_single_file_list_t), 1)
# Single input (ndarray)
2 years ago
input_ = decode_image(
single_input, to_rgb=False) # Reuse the name `input_`
predictor.predict(input_, transforms=transforms)
trainer.predict(input_, transforms=transforms)
out_single_array_list_p = predictor.predict(
[input_], transforms=transforms)
self.assertEqual(len(out_single_array_list_p), 1)
out_single_array_list_t = trainer.predict(
[input_], transforms=transforms)
2 years ago
self.assertEqual(len(out_single_array_list_t), 1)
# Multiple inputs (file paths)
input_ = [single_input] * num_inputs # Reuse the name `input_`
out_multi_file_p = predictor.predict(input_, transforms=transforms)
self.assertEqual(len(out_multi_file_p), num_inputs)
out_multi_file_t = trainer.predict(input_, transforms=transforms)
2 years ago
self.assertEqual(len(out_multi_file_t), num_inputs)
# Multiple inputs (ndarrays)
2 years ago
input_ = [decode_image(
single_input, to_rgb=False)] * num_inputs # Reuse the name `input_`
out_multi_array_p = predictor.predict(input_, transforms=transforms)
self.assertEqual(len(out_multi_array_p), num_inputs)
out_multi_array_t = trainer.predict(input_, transforms=transforms)
2 years ago
self.assertEqual(len(out_multi_array_t), num_inputs)
@TestPredictor.add_tests
class TestSegPredictor(TestPredictor):
MODULE = pdrs.tasks.segmenter
TRAINER_NAME_TO_EXPORT_OPTS = {
'_default': "--fixed_input_shape [-1,3,256,256]"
}
def check_predictor(self, predictor, trainer):
single_input = "data/ssmt/optical_t1.bmp"
num_inputs = 2
transforms = pdrs.transforms.Compose([pdrs.transforms.Normalize()])
# Single input (file path)
input_ = single_input
out_single_file_p = predictor.predict(input_, transforms=transforms)
out_single_file_t = trainer.predict(input_, transforms=transforms)
self.check_dict_equal(out_single_file_p, out_single_file_t)
out_single_file_list_p = predictor.predict(
[input_], transforms=transforms)
self.assertEqual(len(out_single_file_list_p), 1)
self.check_dict_equal(out_single_file_list_p[0], out_single_file_p)
out_single_file_list_t = trainer.predict(
[input_], transforms=transforms)
self.check_dict_equal(out_single_file_list_p[0],
out_single_file_list_t[0])
# Single input (ndarray)
2 years ago
input_ = decode_image(
single_input, to_rgb=False) # Reuse the name `input_`
out_single_array_p = predictor.predict(input_, transforms=transforms)
self.check_dict_equal(out_single_array_p, out_single_file_p)
out_single_array_t = trainer.predict(input_, transforms=transforms)
self.check_dict_equal(out_single_array_p, out_single_array_t)
out_single_array_list_p = predictor.predict(
[input_], transforms=transforms)
self.assertEqual(len(out_single_array_list_p), 1)
self.check_dict_equal(out_single_array_list_p[0], out_single_array_p)
out_single_array_list_t = trainer.predict(
[input_], transforms=transforms)
self.check_dict_equal(out_single_array_list_p[0],
out_single_array_list_t[0])
# Multiple inputs (file paths)
input_ = [single_input] * num_inputs # Reuse the name `input_`
out_multi_file_p = predictor.predict(input_, transforms=transforms)
self.assertEqual(len(out_multi_file_p), num_inputs)
out_multi_file_t = trainer.predict(input_, transforms=transforms)
2 years ago
self.assertEqual(len(out_multi_file_t), num_inputs)
# Multiple inputs (ndarrays)
2 years ago
input_ = [decode_image(
single_input, to_rgb=False)] * num_inputs # Reuse the name `input_`
out_multi_array_p = predictor.predict(input_, transforms=transforms)
self.assertEqual(len(out_multi_array_p), num_inputs)
out_multi_array_t = trainer.predict(input_, transforms=transforms)
2 years ago
self.assertEqual(len(out_multi_array_t), num_inputs)