From 0ddfbdcc1b2363c95239f83c5373b5d050b6af06 Mon Sep 17 00:00:00 2001 From: geoyee Date: Thu, 3 Mar 2022 22:02:34 +0800 Subject: [PATCH] [Feature] Init add change detection (without test task) --- .gitignore | 7 +- paddlers/datasets/__init__.py | 1 + paddlers/datasets/cd_dataset.py | 97 +++ paddlers/datasets/raster.py | 14 +- paddlers/models/ppcd/__init__.py | 1 + paddlers/models/ppcd/cdnet.py | 75 +++ paddlers/models/ppseg/utils/env/__init__.py | 16 + paddlers/models/ppseg/utils/env/seg_env.py | 56 ++ paddlers/models/ppseg/utils/env/sys_env.py | 124 ++++ paddlers/tasks/changedetector.py | 671 ++++++++++++++++++++ paddlers/transforms/__init__.py | 7 + paddlers/transforms/operators.py | 26 + requirements.txt | 2 +- 13 files changed, 1086 insertions(+), 11 deletions(-) create mode 100644 paddlers/datasets/cd_dataset.py create mode 100644 paddlers/models/ppcd/cdnet.py create mode 100644 paddlers/models/ppseg/utils/env/__init__.py create mode 100644 paddlers/models/ppseg/utils/env/seg_env.py create mode 100644 paddlers/models/ppseg/utils/env/sys_env.py create mode 100644 paddlers/tasks/changedetector.py diff --git a/.gitignore b/.gitignore index b6e4761..d1ea072 100644 --- a/.gitignore +++ b/.gitignore @@ -102,11 +102,12 @@ celerybeat.pid *.sage.py # Environments -.env +# don't filter paddleseg's env +# .env .venv -env/ +# env/ venv/ -ENV/ +# ENV/ env.bak/ venv.bak/ diff --git a/paddlers/datasets/__init__.py b/paddlers/datasets/__init__.py index 4e31bee..8c52017 100644 --- a/paddlers/datasets/__init__.py +++ b/paddlers/datasets/__init__.py @@ -1,3 +1,4 @@ from .voc import VOCDetection from .seg_dataset import SegDataset +from .cd_dataset import CDDataset from .raster import Raster \ No newline at end of file diff --git a/paddlers/datasets/cd_dataset.py b/paddlers/datasets/cd_dataset.py new file mode 100644 index 0000000..f5f95fd --- /dev/null +++ b/paddlers/datasets/cd_dataset.py @@ -0,0 +1,97 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os.path as osp +import copy + +from paddle.io import Dataset +from paddlers.utils import logging, get_num_workers, get_encoding, path_normalization, is_pic + + +class CDDataset(Dataset): + """读取变化检测任务数据集，并对样本进行相应的处理（来自SegDataset，图像标签需要两个）。 + + Args: + data_dir (str): 数据集所在的目录路径。 + file_list (str): 描述数据集图片文件和对应标注文件的文件路径（文本内每行路径为相对data_dir的相对路）。 + label_list (str): 描述数据集包含的类别信息文件路径。默认值为None。 + transforms (paddlers.transforms): 数据集中每个样本的预处理/增强算子。 + num_workers (int|str): 数据集中样本在预处理过程中的线程或进程数。默认为'auto'。 + shuffle (bool): 是否需要对数据集中样本打乱顺序。默认为False。 + """ + + def __init__(self, + data_dir, + file_list, + label_list=None, + transforms=None, + num_workers='auto', + shuffle=False): + super(CDDataset, self).__init__() + self.transforms = copy.deepcopy(transforms) + # TODO batch padding + self.batch_transforms = None + self.num_workers = get_num_workers(num_workers) + self.shuffle = shuffle + self.file_list = list() + self.labels = list() + + # TODO：非None时，让用户跳转数据集分析生成label_list + # 不要在此处分析label file + if label_list is not None: + with open(label_list, encoding=get_encoding(label_list)) as f: + for line in f: + item = line.strip() + self.labels.append(item) + with open(file_list, encoding=get_encoding(file_list)) as f: + for line in f: + items = line.strip().split() + if len(items) > 3: + raise Exception( + "A space is defined as the delimiter to separate the image and label path, " \ + "so the space cannot be in the image or label path, but the line[{}] of " \ + " file_list[{}] has a space in the image or label path.".format(line, file_list)) + items[0] = path_normalization(items[0]) + items[1] = path_normalization(items[1]) + items[2] = path_normalization(items[2]) + if not is_pic(items[0]) or not is_pic(items[1]) or not is_pic(items[2]): + continue + full_path_im_t1 = osp.join(data_dir, items[0]) + full_path_im_t2 = osp.join(data_dir, items[1]) + full_path_label = osp.join(data_dir, items[2]) + if not osp.exists(full_path_im_t1): + raise IOError('Image file {} does not exist!'.format( + full_path_im_t1)) + if not osp.exists(full_path_im_t2): + raise IOError('Image file {} does not exist!'.format( + full_path_im_t2)) + if not osp.exists(full_path_label): + raise IOError('Label file {} does not exist!'.format( + full_path_label)) + self.file_list.append({ + 'image_t1': full_path_im_t1, + 'image_t2': full_path_im_t2, + 'mask': full_path_label + }) + self.num_samples = len(self.file_list) + logging.info("{} samples in file {}".format( + len(self.file_list), file_list)) + + def __getitem__(self, idx): + sample = copy.deepcopy(self.file_list[idx]) + outputs = self.transforms(sample) + return outputs + + def __len__(self): + return len(self.file_list) \ No newline at end of file diff --git a/paddlers/datasets/raster.py b/paddlers/datasets/raster.py index ddec49a..0f4c576 100644 --- a/paddlers/datasets/raster.py +++ b/paddlers/datasets/raster.py @@ -42,7 +42,7 @@ class Raster: self.path = path self.__src_data = np.load(path) if path.split(".")[-1] == "npy" \ else gdal.Open(path) - self.__getInfo() + self._getInfo() self.to_uint8 = to_uint8 self.setBands(band_list) else: @@ -78,16 +78,16 @@ class Raster: np.ndarray: data's ndarray. """ if start_loc is None: - return self.__getAarray() + return self._getAarray() else: - return self.__getBlock(start_loc, block_size) + return self._getBlock(start_loc, block_size) - def __getInfo(self) -> None: + def _getInfo(self) -> None: self.bands = self.__src_data.RasterCount self.width = self.__src_data.RasterXSize self.height = self.__src_data.RasterYSize - def __getAarray(self, window: Union[None, List[int], Tuple[int]]=None) -> np.ndarray: + def _getAarray(self, window: Union[None, List[int], Tuple[int]]=None) -> np.ndarray: if window is not None: xoff, yoff, xsize, ysize = window if self.band_list is None: @@ -114,7 +114,7 @@ class Raster: ima = raster2uint8(ima) return ima - def __getBlock(self, + def _getBlock(self, start_loc: Union[List[int], Tuple[int]], block_size: Union[List[int], Tuple[int]]=[512, 512]) -> np.ndarray: if len(start_loc) != 2 or len(block_size) != 2: @@ -128,7 +128,7 @@ class Raster: xsize = self.width - xoff if yoff + ysize > self.height: ysize = self.height - yoff - ima = self.__getAarray([int(xoff), int(yoff), int(xsize), int(ysize)]) + ima = self._getAarray([int(xoff), int(yoff), int(xsize), int(ysize)]) h, w = ima.shape[:2] if len(ima.shape) == 3 else ima.shape if self.bands != 1: tmp = np.zeros((block_size[0], block_size[1], self.bands), dtype=ima.dtype) diff --git a/paddlers/models/ppcd/__init__.py b/paddlers/models/ppcd/__init__.py index e69de29..36fa846 100644 --- a/paddlers/models/ppcd/__init__.py +++ b/paddlers/models/ppcd/__init__.py @@ -0,0 +1 @@ +from .cdnet import CDNet \ No newline at end of file diff --git a/paddlers/models/ppcd/cdnet.py b/paddlers/models/ppcd/cdnet.py new file mode 100644 index 0000000..96394cb --- /dev/null +++ b/paddlers/models/ppcd/cdnet.py @@ -0,0 +1,75 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle +import paddle.nn as nn + + +class CDNet(nn.Layer): + def __init__(self, in_channels=6, num_classes=2): + super(CDNet, self).__init__() + self.conv1 = Conv7x7(in_channels, 64, norm=True, act=True) + self.pool1 = nn.MaxPool2D(2, 2, return_mask=True) + self.conv2 = Conv7x7(64, 64, norm=True, act=True) + self.pool2 = nn.MaxPool2D(2, 2, return_mask=True) + self.conv3 = Conv7x7(64, 64, norm=True, act=True) + self.pool3 = nn.MaxPool2D(2, 2, return_mask=True) + self.conv4 = Conv7x7(64, 64, norm=True, act=True) + self.pool4 = nn.MaxPool2D(2, 2, return_mask=True) + self.conv5 = Conv7x7(64, 64, norm=True, act=True) + self.upool4 = nn.MaxUnPool2D(2, 2) + self.conv6 = Conv7x7(64, 64, norm=True, act=True) + self.upool3 = nn.MaxUnPool2D(2, 2) + self.conv7 = Conv7x7(64, 64, norm=True, act=True) + self.upool2 = nn.MaxUnPool2D(2, 2) + self.conv8 = Conv7x7(64, 64, norm=True, act=True) + self.upool1 = nn.MaxUnPool2D(2, 2) + self.conv_out = Conv7x7(64, num_classes, norm=False, act=False) + + def forward(self, t1, t2): + x = paddle.concat([t1, t2], axis=1) + x, ind1 = self.pool1(self.conv1(x)) + x, ind2 = self.pool2(self.conv2(x)) + x, ind3 = self.pool3(self.conv3(x)) + x, ind4 = self.pool4(self.conv4(x)) + x = self.conv5(self.upool4(x, ind4)) + x = self.conv6(self.upool3(x, ind3)) + x = self.conv7(self.upool2(x, ind2)) + x = self.conv8(self.upool1(x, ind1)) + return [self.conv_out(x)] + + +class Conv7x7(nn.Layer): + def __init__(self, in_ch, out_ch, norm=False, act=False): + super(Conv7x7, self).__init__() + layers = [ + nn.Pad2D(3), + nn.Conv2D(in_ch, out_ch, 7, bias_attr=(False if norm else None)) + ] + if norm: + layers.append(nn.BatchNorm2D(out_ch)) + if act: + layers.append(nn.ReLU()) + self.layers = nn.Sequential(*layers) + + def forward(self, x): + return self.layers(x) + + +if __name__ == "__main__": + t1 = paddle.randn((1, 3, 512, 512), dtype="float32") + t2 = paddle.randn((1, 3, 512, 512), dtype="float32") + model = CDNet(6, 2) + pred = model(t1, t2)[0] + print(pred.shape) \ No newline at end of file diff --git a/paddlers/models/ppseg/utils/env/__init__.py b/paddlers/models/ppseg/utils/env/__init__.py new file mode 100644 index 0000000..5cb0d12 --- /dev/null +++ b/paddlers/models/ppseg/utils/env/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License" +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import seg_env +from .sys_env import get_sys_env \ No newline at end of file diff --git a/paddlers/models/ppseg/utils/env/seg_env.py b/paddlers/models/ppseg/utils/env/seg_env.py new file mode 100644 index 0000000..0c6b9b4 --- /dev/null +++ b/paddlers/models/ppseg/utils/env/seg_env.py @@ -0,0 +1,56 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License" +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This module is used to store environmental parameters in PaddleSeg. + +SEG_HOME : Root directory for storing PaddleSeg related data. Default to ~/.paddleseg. + Users can change the default value through the SEG_HOME environment variable. +DATA_HOME : The directory to store the automatically downloaded dataset, e.g ADE20K. +PRETRAINED_MODEL_HOME : The directory to store the automatically downloaded pretrained model. +""" + +import os + +from paddleseg.utils import logger + + +def _get_user_home(): + return os.path.expanduser('~') + + +def _get_seg_home(): + if 'SEG_HOME' in os.environ: + home_path = os.environ['SEG_HOME'] + if os.path.exists(home_path): + if os.path.isdir(home_path): + return home_path + else: + logger.warning('SEG_HOME {} is a file!'.format(home_path)) + else: + return home_path + return os.path.join(_get_user_home(), '.paddleseg') + + +def _get_sub_home(directory): + home = os.path.join(_get_seg_home(), directory) + if not os.path.exists(home): + os.makedirs(home, exist_ok=True) + return home + + +USER_HOME = _get_user_home() +SEG_HOME = _get_seg_home() +DATA_HOME = _get_sub_home('dataset') +TMP_HOME = _get_sub_home('tmp') +PRETRAINED_MODEL_HOME = _get_sub_home('pretrained_model') \ No newline at end of file diff --git a/paddlers/models/ppseg/utils/env/sys_env.py b/paddlers/models/ppseg/utils/env/sys_env.py new file mode 100644 index 0000000..0b1c131 --- /dev/null +++ b/paddlers/models/ppseg/utils/env/sys_env.py @@ -0,0 +1,124 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import os +import platform +import subprocess +import sys + +import cv2 +import paddle +import paddleseg + +IS_WINDOWS = sys.platform == 'win32' + + +def _find_cuda_home(): + '''Finds the CUDA install path. It refers to the implementation of + pytorch . + ''' + # Guess #1 + cuda_home = os.environ.get('CUDA_HOME') or os.environ.get('CUDA_PATH') + if cuda_home is None: + # Guess #2 + try: + which = 'where' if IS_WINDOWS else 'which' + nvcc = subprocess.check_output([which, + 'nvcc']).decode().rstrip('\r\n') + cuda_home = os.path.dirname(os.path.dirname(nvcc)) + except Exception: + # Guess #3 + if IS_WINDOWS: + cuda_homes = glob.glob( + 'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v*.*') + if len(cuda_homes) == 0: + cuda_home = '' + else: + cuda_home = cuda_homes[0] + else: + cuda_home = '/usr/local/cuda' + if not os.path.exists(cuda_home): + cuda_home = None + return cuda_home + + +def _get_nvcc_info(cuda_home): + if cuda_home is not None and os.path.isdir(cuda_home): + try: + nvcc = os.path.join(cuda_home, 'bin/nvcc') + nvcc = subprocess.check_output( + "{} -V".format(nvcc), shell=True).decode() + nvcc = nvcc.strip().split('\n')[-1] + except subprocess.SubprocessError: + nvcc = "Not Available" + else: + nvcc = "Not Available" + return nvcc + + +def _get_gpu_info(): + try: + gpu_info = subprocess.check_output(['nvidia-smi', + '-L']).decode().strip() + gpu_info = gpu_info.split('\n') + for i in range(len(gpu_info)): + gpu_info[i] = ' '.join(gpu_info[i].split(' ')[:4]) + except: + gpu_info = ' Can not get GPU information. Please make sure CUDA have been installed successfully.' + return gpu_info + + +def get_sys_env(): + """collect environment information""" + env_info = {} + env_info['platform'] = platform.platform() + + env_info['Python'] = sys.version.replace('\n', '') + + # TODO is_compiled_with_cuda() has not been moved + compiled_with_cuda = paddle.is_compiled_with_cuda() + env_info['Paddle compiled with cuda'] = compiled_with_cuda + + if compiled_with_cuda: + cuda_home = _find_cuda_home() + env_info['NVCC'] = _get_nvcc_info(cuda_home) + # refer to https://github.com/PaddlePaddle/Paddle/blob/release/2.0-rc/paddle/fluid/platform/device_context.cc#L327 + v = paddle.get_cudnn_version() + v = str(v // 1000) + '.' + str(v % 1000 // 100) + env_info['cudnn'] = v + if 'gpu' in paddle.get_device(): + gpu_nums = paddle.distributed.ParallelEnv().nranks + else: + gpu_nums = 0 + env_info['GPUs used'] = gpu_nums + + env_info['CUDA_VISIBLE_DEVICES'] = os.environ.get( + 'CUDA_VISIBLE_DEVICES') + if gpu_nums == 0: + os.environ['CUDA_VISIBLE_DEVICES'] = '' + env_info['GPU'] = _get_gpu_info() + + try: + gcc = subprocess.check_output(['gcc', '--version']).decode() + gcc = gcc.strip().split('\n')[0] + env_info['GCC'] = gcc + except: + pass + + env_info['PaddleSeg'] = paddleseg.__version__ + env_info['PaddlePaddle'] = paddle.__version__ + env_info['OpenCV'] = cv2.__version__ + + return env_info \ No newline at end of file diff --git a/paddlers/tasks/changedetector.py b/paddlers/tasks/changedetector.py new file mode 100644 index 0000000..6879443 --- /dev/null +++ b/paddlers/tasks/changedetector.py @@ -0,0 +1,671 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import os.path as osp +import numpy as np +import cv2 +from collections import OrderedDict +import paddle +import paddle.nn.functional as F +from paddle.static import InputSpec +import paddlers.models.ppseg as paddleseg +import paddlers +from paddlers.transforms import arrange_transforms +from paddlers.utils import get_single_card_bs, DisablePrint +import paddlers.utils.logging as logging +from .base import BaseModel +from .utils import seg_metrics as metrics +from paddlers.utils.checkpoint import seg_pretrain_weights_dict +from paddlers.transforms import Decode, Resize +from paddlers.models.ppcd import CDNet + +__all__ = ["CDNet"] + + +class BaseChangeDetector(BaseModel): + def __init__(self, + model_name, + num_classes=2, + use_mixed_loss=False, + **params): + self.init_params = locals() + if 'with_net' in self.init_params: + del self.init_params['with_net'] + super(BaseChangeDetector, self).__init__('changedetector') + if model_name not in __all__: + raise Exception("ERROR: There's no model named {}.".format( + model_name)) + self.model_name = model_name + self.num_classes = num_classes + self.use_mixed_loss = use_mixed_loss + self.losses = None + self.labels = None + if params.get('with_net', True): + params.pop('with_net', None) + self.net = self.build_net(**params) + self.find_unused_parameters = True + + def build_net(self, **params): + # TODO: add other model + net = CDNet(num_classes=self.num_classes, **params) + return net + + def _fix_transforms_shape(self, image_shape): + if hasattr(self, 'test_transforms'): + if self.test_transforms is not None: + has_resize_op = False + resize_op_idx = -1 + normalize_op_idx = len(self.test_transforms.transforms) + for idx, op in enumerate(self.test_transforms.transforms): + name = op.__class__.__name__ + if name == 'Normalize': + normalize_op_idx = idx + if 'Resize' in name: + has_resize_op = True + resize_op_idx = idx + + if not has_resize_op: + self.test_transforms.transforms.insert( + normalize_op_idx, Resize(target_size=image_shape)) + else: + self.test_transforms.transforms[resize_op_idx] = Resize( + target_size=image_shape) + + def _get_test_inputs(self, image_shape): + if image_shape is not None: + if len(image_shape) == 2: + image_shape = [1, 3] + image_shape + self._fix_transforms_shape(image_shape[-2:]) + else: + image_shape = [None, 3, -1, -1] + self.fixed_input_shape = image_shape + input_spec = [ + InputSpec( + shape=image_shape, name='image', dtype='float32') + ] + return input_spec + + def run(self, net, inputs, mode): + net_out = net(inputs[0], inputs[1]) + logit = net_out[0] + outputs = OrderedDict() + if mode == 'test': + origin_shape = inputs[2] + if self.status == 'Infer': + label_map_list, score_map_list = self._postprocess( + net_out, origin_shape, transforms=inputs[3]) + else: + logit_list = self._postprocess( + logit, origin_shape, transforms=inputs[3]) + label_map_list = [] + score_map_list = [] + for logit in logit_list: + logit = paddle.transpose(logit, perm=[0, 2, 3, 1]) # NHWC + label_map_list.append( + paddle.argmax( + logit, axis=-1, keepdim=False, dtype='int32') + .squeeze().numpy()) + score_map_list.append( + F.softmax( + logit, axis=-1).squeeze().numpy().astype( + 'float32')) + outputs['label_map'] = label_map_list + outputs['score_map'] = score_map_list + + if mode == 'eval': + if self.status == 'Infer': + pred = paddle.unsqueeze(net_out[0], axis=1) # NCHW + else: + pred = paddle.argmax( + logit, axis=1, keepdim=True, dtype='int32') + label = inputs[2] + origin_shape = [label.shape[-2:]] + pred = self._postprocess( + pred, origin_shape, transforms=inputs[3])[0] # NCHW + intersect_area, pred_area, label_area = paddleseg.utils.metrics.calculate_area( + pred, label, self.num_classes) + outputs['intersect_area'] = intersect_area + outputs['pred_area'] = pred_area + outputs['label_area'] = label_area + outputs['conf_mat'] = metrics.confusion_matrix(pred, label, + self.num_classes) + if mode == 'train': + loss_list = metrics.loss_computation( + logits_list=net_out, labels=inputs[2], losses=self.losses) + loss = sum(loss_list) + outputs['loss'] = loss + return outputs + + def default_loss(self): + if isinstance(self.use_mixed_loss, bool): + if self.use_mixed_loss: + losses = [ + paddleseg.models.CrossEntropyLoss(), + paddleseg.models.LovaszSoftmaxLoss() + ] + coef = [.8, .2] + loss_type = [ + paddleseg.models.MixedLoss( + losses=losses, coef=coef), + ] + else: + loss_type = [paddleseg.models.CrossEntropyLoss()] + else: + losses, coef = list(zip(*self.use_mixed_loss)) + if not set(losses).issubset( + ['CrossEntropyLoss', 'DiceLoss', 'LovaszSoftmaxLoss']): + raise ValueError( + "Only 'CrossEntropyLoss', 'DiceLoss', 'LovaszSoftmaxLoss' are supported." + ) + losses = [getattr(paddleseg.models, loss)() for loss in losses] + loss_type = [ + paddleseg.models.MixedLoss( + losses=losses, coef=list(coef)) + ] + if self.model_name == 'FastSCNN': + loss_type *= 2 + loss_coef = [1.0, 0.4] + elif self.model_name == 'BiSeNetV2': + loss_type *= 5 + loss_coef = [1.0] * 5 + else: + loss_coef = [1.0] + losses = {'types': loss_type, 'coef': loss_coef} + return losses + + def default_optimizer(self, + parameters, + learning_rate, + num_epochs, + num_steps_each_epoch, + lr_decay_power=0.9): + decay_step = num_epochs * num_steps_each_epoch + lr_scheduler = paddle.optimizer.lr.PolynomialDecay( + learning_rate, decay_step, end_lr=0, power=lr_decay_power) + optimizer = paddle.optimizer.Momentum( + learning_rate=lr_scheduler, + parameters=parameters, + momentum=0.9, + weight_decay=4e-5) + return optimizer + + def train(self, + num_epochs, + train_dataset, + train_batch_size=2, + eval_dataset=None, + optimizer=None, + save_interval_epochs=1, + log_interval_steps=2, + save_dir='output', + pretrain_weights='CITYSCAPES', + learning_rate=0.01, + lr_decay_power=0.9, + early_stop=False, + early_stop_patience=5, + use_vdl=True, + resume_checkpoint=None): + """ + Train the model. + Args: + num_epochs(int): The number of epochs. + train_dataset(paddlers.dataset): Training dataset. + train_batch_size(int, optional): Total batch size among all cards used in training. Defaults to 2. + eval_dataset(paddlers.dataset, optional): + Evaluation dataset. If None, the model will not be evaluated furing training process. Defaults to None. + optimizer(paddle.optimizer.Optimizer or None, optional): + Optimizer used in training. If None, a default optimizer is used. Defaults to None. + save_interval_epochs(int, optional): Epoch interval for saving the model. Defaults to 1. + log_interval_steps(int, optional): Step interval for printing training information. Defaults to 10. + save_dir(str, optional): Directory to save the model. Defaults to 'output'. + pretrain_weights(str or None, optional): + None or name/path of pretrained weights. If None, no pretrained weights will be loaded. Defaults to 'CITYSCAPES'. + learning_rate(float, optional): Learning rate for training. Defaults to .025. + lr_decay_power(float, optional): Learning decay power. Defaults to .9. + early_stop(bool, optional): Whether to adopt early stop strategy. Defaults to False. + early_stop_patience(int, optional): Early stop patience. Defaults to 5. + use_vdl(bool, optional): Whether to use VisualDL to monitor the training process. Defaults to True. + resume_checkpoint(str or None, optional): The path of the checkpoint to resume training from. + If None, no training checkpoint will be resumed. At most one of `resume_checkpoint` and + `pretrain_weights` can be set simultaneously. Defaults to None. + + """ + if self.status == 'Infer': + logging.error( + "Exported inference model does not support training.", + exit=True) + if pretrain_weights is not None and resume_checkpoint is not None: + logging.error( + "pretrain_weights and resume_checkpoint cannot be set simultaneously.", + exit=True) + self.labels = train_dataset.labels + if self.losses is None: + self.losses = self.default_loss() + + if optimizer is None: + num_steps_each_epoch = train_dataset.num_samples // train_batch_size + self.optimizer = self.default_optimizer( + self.net.parameters(), learning_rate, num_epochs, + num_steps_each_epoch, lr_decay_power) + else: + self.optimizer = optimizer + + if pretrain_weights is not None and not osp.exists(pretrain_weights): + if pretrain_weights not in seg_pretrain_weights_dict[ + self.model_name]: + logging.warning( + "Path of pretrain_weights('{}') does not exist!".format( + pretrain_weights)) + logging.warning("Pretrain_weights is forcibly set to '{}'. " + "If don't want to use pretrain weights, " + "set pretrain_weights to be None.".format( + seg_pretrain_weights_dict[self.model_name][ + 0])) + pretrain_weights = seg_pretrain_weights_dict[self.model_name][ + 0] + elif pretrain_weights is not None and osp.exists(pretrain_weights): + if osp.splitext(pretrain_weights)[-1] != '.pdparams': + logging.error( + "Invalid pretrain weights. Please specify a '.pdparams' file.", + exit=True) + pretrained_dir = osp.join(save_dir, 'pretrain') + is_backbone_weights = pretrain_weights == 'IMAGENET' + self.net_initialize( + pretrain_weights=pretrain_weights, + save_dir=pretrained_dir, + resume_checkpoint=resume_checkpoint, + is_backbone_weights=is_backbone_weights) + + self.train_loop( + num_epochs=num_epochs, + train_dataset=train_dataset, + train_batch_size=train_batch_size, + eval_dataset=eval_dataset, + save_interval_epochs=save_interval_epochs, + log_interval_steps=log_interval_steps, + save_dir=save_dir, + early_stop=early_stop, + early_stop_patience=early_stop_patience, + use_vdl=use_vdl) + + def quant_aware_train(self, + num_epochs, + train_dataset, + train_batch_size=2, + eval_dataset=None, + optimizer=None, + save_interval_epochs=1, + log_interval_steps=2, + save_dir='output', + learning_rate=0.0001, + lr_decay_power=0.9, + early_stop=False, + early_stop_patience=5, + use_vdl=True, + resume_checkpoint=None, + quant_config=None): + """ + Quantization-aware training. + Args: + num_epochs(int): The number of epochs. + train_dataset(paddlers.dataset): Training dataset. + train_batch_size(int, optional): Total batch size among all cards used in training. Defaults to 2. + eval_dataset(paddlers.dataset, optional): + Evaluation dataset. If None, the model will not be evaluated furing training process. Defaults to None. + optimizer(paddle.optimizer.Optimizer or None, optional): + Optimizer used in training. If None, a default optimizer is used. Defaults to None. + save_interval_epochs(int, optional): Epoch interval for saving the model. Defaults to 1. + log_interval_steps(int, optional): Step interval for printing training information. Defaults to 10. + save_dir(str, optional): Directory to save the model. Defaults to 'output'. + learning_rate(float, optional): Learning rate for training. Defaults to .025. + lr_decay_power(float, optional): Learning decay power. Defaults to .9. + early_stop(bool, optional): Whether to adopt early stop strategy. Defaults to False. + early_stop_patience(int, optional): Early stop patience. Defaults to 5. + use_vdl(bool, optional): Whether to use VisualDL to monitor the training process. Defaults to True. + quant_config(dict or None, optional): Quantization configuration. If None, a default rule of thumb + configuration will be used. Defaults to None. + resume_checkpoint(str or None, optional): The path of the checkpoint to resume quantization-aware training + from. If None, no training checkpoint will be resumed. Defaults to None. + + """ + self._prepare_qat(quant_config) + self.train( + num_epochs=num_epochs, + train_dataset=train_dataset, + train_batch_size=train_batch_size, + eval_dataset=eval_dataset, + optimizer=optimizer, + save_interval_epochs=save_interval_epochs, + log_interval_steps=log_interval_steps, + save_dir=save_dir, + pretrain_weights=None, + learning_rate=learning_rate, + lr_decay_power=lr_decay_power, + early_stop=early_stop, + early_stop_patience=early_stop_patience, + use_vdl=use_vdl, + resume_checkpoint=resume_checkpoint) + + def evaluate(self, eval_dataset, batch_size=1, return_details=False): + """ + Evaluate the model. + Args: + eval_dataset(paddlers.dataset): Evaluation dataset. + batch_size(int, optional): Total batch size among all cards used for evaluation. Defaults to 1. + return_details(bool, optional): Whether to return evaluation details. Defaults to False. + + Returns: + collections.OrderedDict with key-value pairs: + {"miou": `mean intersection over union`, + "category_iou": `category-wise mean intersection over union`, + "oacc": `overall accuracy`, + "category_acc": `category-wise accuracy`, + "kappa": ` kappa coefficient`, + "category_F1-score": `F1 score`}. + + """ + arrange_transforms( + model_type=self.model_type, + transforms=eval_dataset.transforms, + mode='eval') + + self.net.eval() + nranks = paddle.distributed.get_world_size() + local_rank = paddle.distributed.get_rank() + if nranks > 1: + # Initialize parallel environment if not done. + if not paddle.distributed.parallel.parallel_helper._is_parallel_ctx_initialized( + ): + paddle.distributed.init_parallel_env() + + batch_size_each_card = get_single_card_bs(batch_size) + if batch_size_each_card > 1: + batch_size_each_card = 1 + batch_size = batch_size_each_card * paddlers.env_info['num'] + logging.warning( + "Segmenter only supports batch_size=1 for each gpu/cpu card " \ + "during evaluation, so batch_size " \ + "is forcibly set to {}.".format(batch_size)) + self.eval_data_loader = self.build_data_loader( + eval_dataset, batch_size=batch_size, mode='eval') + + intersect_area_all = 0 + pred_area_all = 0 + label_area_all = 0 + conf_mat_all = [] + logging.info( + "Start to evaluate(total_samples={}, total_steps={})...".format( + eval_dataset.num_samples, + math.ceil(eval_dataset.num_samples * 1.0 / batch_size))) + with paddle.no_grad(): + for step, data in enumerate(self.eval_data_loader): + data.append(eval_dataset.transforms.transforms) + outputs = self.run(self.net, data, 'eval') + pred_area = outputs['pred_area'] + label_area = outputs['label_area'] + intersect_area = outputs['intersect_area'] + conf_mat = outputs['conf_mat'] + + # Gather from all ranks + if nranks > 1: + intersect_area_list = [] + pred_area_list = [] + label_area_list = [] + conf_mat_list = [] + paddle.distributed.all_gather(intersect_area_list, + intersect_area) + paddle.distributed.all_gather(pred_area_list, pred_area) + paddle.distributed.all_gather(label_area_list, label_area) + paddle.distributed.all_gather(conf_mat_list, conf_mat) + + # Some image has been evaluated and should be eliminated in last iter + if (step + 1) * nranks > len(eval_dataset): + valid = len(eval_dataset) - step * nranks + intersect_area_list = intersect_area_list[:valid] + pred_area_list = pred_area_list[:valid] + label_area_list = label_area_list[:valid] + conf_mat_list = conf_mat_list[:valid] + + intersect_area_all += sum(intersect_area_list) + pred_area_all += sum(pred_area_list) + label_area_all += sum(label_area_list) + conf_mat_all.extend(conf_mat_list) + + else: + intersect_area_all = intersect_area_all + intersect_area + pred_area_all = pred_area_all + pred_area + label_area_all = label_area_all + label_area + conf_mat_all.append(conf_mat) + class_iou, miou = paddleseg.utils.metrics.mean_iou( + intersect_area_all, pred_area_all, label_area_all) + # TODO 确认是按oacc还是macc + class_acc, oacc = paddleseg.utils.metrics.accuracy(intersect_area_all, + pred_area_all) + kappa = paddleseg.utils.metrics.kappa(intersect_area_all, + pred_area_all, label_area_all) + category_f1score = metrics.f1_score(intersect_area_all, pred_area_all, + label_area_all) + eval_metrics = OrderedDict( + zip([ + 'miou', 'category_iou', 'oacc', 'category_acc', 'kappa', + 'category_F1-score' + ], [miou, class_iou, oacc, class_acc, kappa, category_f1score])) + + if return_details: + conf_mat = sum(conf_mat_all) + eval_details = {'confusion_matrix': conf_mat.tolist()} + return eval_metrics, eval_details + return eval_metrics + + def predict(self, img_file, transforms=None): + """ + Do inference. + Args: + Args: + img_file(List[np.ndarray or str], str or np.ndarray): + Image path or decoded image data in a BGR format, which also could constitute a list, + meaning all images to be predicted as a mini-batch. + transforms(paddlers.transforms.Compose or None, optional): + Transforms for inputs. If None, the transforms for evaluation process will be used. Defaults to None. + + Returns: + If img_file is a string or np.array, the result is a dict with key-value pairs: + {"label map": `label map`, "score_map": `score map`}. + If img_file is a list, the result is a list composed of dicts with the corresponding fields: + label_map(np.ndarray): the predicted label map (HW) + score_map(np.ndarray): the prediction score map (HWC) + + """ + if transforms is None and not hasattr(self, 'test_transforms'): + raise Exception("transforms need to be defined, now is None.") + if transforms is None: + transforms = self.test_transforms + if isinstance(img_file, (str, np.ndarray)): + images = [img_file] + else: + images = img_file + batch_im, batch_origin_shape = self._preprocess(images, transforms, + self.model_type) + self.net.eval() + data = (batch_im, batch_origin_shape, transforms.transforms) + outputs = self.run(self.net, data, 'test') + label_map_list = outputs['label_map'] + score_map_list = outputs['score_map'] + if isinstance(img_file, list): + prediction = [{ + 'label_map': l, + 'score_map': s + } for l, s in zip(label_map_list, score_map_list)] + else: + prediction = { + 'label_map': label_map_list[0], + 'score_map': score_map_list[0] + } + return prediction + + def _preprocess(self, images, transforms, to_tensor=True): + arrange_transforms( + model_type=self.model_type, transforms=transforms, mode='test') + batch_im = list() + batch_ori_shape = list() + for im in images: + sample = {'image': im} + if isinstance(sample['image'], str): + sample = Decode(to_rgb=False)(sample) + ori_shape = sample['image'].shape[:2] + im = transforms(sample)[0] + batch_im.append(im) + batch_ori_shape.append(ori_shape) + if to_tensor: + batch_im = paddle.to_tensor(batch_im) + else: + batch_im = np.asarray(batch_im) + + return batch_im, batch_ori_shape + + @staticmethod + def get_transforms_shape_info(batch_ori_shape, transforms): + batch_restore_list = list() + for ori_shape in batch_ori_shape: + restore_list = list() + h, w = ori_shape[0], ori_shape[1] + for op in transforms: + if op.__class__.__name__ == 'Resize': + restore_list.append(('resize', (h, w))) + h, w = op.target_size + elif op.__class__.__name__ == 'ResizeByShort': + restore_list.append(('resize', (h, w))) + im_short_size = min(h, w) + im_long_size = max(h, w) + scale = float(op.short_size) / float(im_short_size) + if 0 < op.max_size < np.round(scale * im_long_size): + scale = float(op.max_size) / float(im_long_size) + h = int(round(h * scale)) + w = int(round(w * scale)) + elif op.__class__.__name__ == 'ResizeByLong': + restore_list.append(('resize', (h, w))) + im_long_size = max(h, w) + scale = float(op.long_size) / float(im_long_size) + h = int(round(h * scale)) + w = int(round(w * scale)) + elif op.__class__.__name__ == 'Padding': + if op.target_size: + target_h, target_w = op.target_size + else: + target_h = int( + (np.ceil(h / op.size_divisor) * op.size_divisor)) + target_w = int( + (np.ceil(w / op.size_divisor) * op.size_divisor)) + + if op.pad_mode == -1: + offsets = op.offsets + elif op.pad_mode == 0: + offsets = [0, 0] + elif op.pad_mode == 1: + offsets = [(target_h - h) // 2, (target_w - w) // 2] + else: + offsets = [target_h - h, target_w - w] + restore_list.append(('padding', (h, w), offsets)) + h, w = target_h, target_w + + batch_restore_list.append(restore_list) + return batch_restore_list + + def _postprocess(self, batch_pred, batch_origin_shape, transforms): + batch_restore_list = BaseSegmenter.get_transforms_shape_info( + batch_origin_shape, transforms) + if isinstance(batch_pred, (tuple, list)) and self.status == 'Infer': + return self._infer_postprocess( + batch_label_map=batch_pred[0], + batch_score_map=batch_pred[1], + batch_restore_list=batch_restore_list) + results = [] + if batch_pred.dtype == paddle.float32: + mode = 'bilinear' + else: + mode = 'nearest' + for pred, restore_list in zip(batch_pred, batch_restore_list): + pred = paddle.unsqueeze(pred, axis=0) + for item in restore_list[::-1]: + h, w = item[1][0], item[1][1] + if item[0] == 'resize': + pred = F.interpolate( + pred, (h, w), mode=mode, data_format='NCHW') + elif item[0] == 'padding': + x, y = item[2] + pred = pred[:, :, y:y + h, x:x + w] + else: + pass + results.append(pred) + return results + + def _infer_postprocess(self, batch_label_map, batch_score_map, + batch_restore_list): + label_maps = [] + score_maps = [] + for label_map, score_map, restore_list in zip( + batch_label_map, batch_score_map, batch_restore_list): + if not isinstance(label_map, np.ndarray): + label_map = paddle.unsqueeze(label_map, axis=[0, 3]) + score_map = paddle.unsqueeze(score_map, axis=0) + for item in restore_list[::-1]: + h, w = item[1][0], item[1][1] + if item[0] == 'resize': + if isinstance(label_map, np.ndarray): + label_map = cv2.resize( + label_map, (w, h), interpolation=cv2.INTER_NEAREST) + score_map = cv2.resize( + score_map, (w, h), interpolation=cv2.INTER_LINEAR) + else: + label_map = F.interpolate( + label_map, (h, w), + mode='nearest', + data_format='NHWC') + score_map = F.interpolate( + score_map, (h, w), + mode='bilinear', + data_format='NHWC') + elif item[0] == 'padding': + x, y = item[2] + if isinstance(label_map, np.ndarray): + label_map = label_map[..., y:y + h, x:x + w] + score_map = score_map[..., y:y + h, x:x + w] + else: + label_map = label_map[:, :, y:y + h, x:x + w] + score_map = score_map[:, :, y:y + h, x:x + w] + else: + pass + label_map = label_map.squeeze() + score_map = score_map.squeeze() + if not isinstance(label_map, np.ndarray): + label_map = label_map.numpy() + score_map = score_map.numpy() + label_maps.append(label_map.squeeze()) + score_maps.append(score_map.squeeze()) + return label_maps, score_maps + + +class CDNet(BaseChangeDetector): + def __init__(self, + num_classes=2, + use_mixed_loss=False, + in_channels=6, + **params): + params.update({'in_channels': in_channels}) + super(CDNet, self).__init__( + model_name='UNet', + num_classes=num_classes, + use_mixed_loss=use_mixed_loss, + **params) diff --git a/paddlers/transforms/__init__.py b/paddlers/transforms/__init__.py index 3c6b2a2..fd278ca 100644 --- a/paddlers/transforms/__init__.py +++ b/paddlers/transforms/__init__.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from operator import mod from .operators import * from .batch_operators import BatchRandomResize, BatchRandomResizeByShort, _BatchPadding from paddlers import transforms as T @@ -25,6 +26,12 @@ def arrange_transforms(model_type, transforms, mode='train'): else: transforms.apply_im_only = False arrange_transform = ArrangeSegmenter(mode) + elif model_type == 'changedetctor': + if mode == 'eval': + transforms.apply_im_only = True + else: + transforms.apply_im_only = False + arrange_transform = ArrangeChangeDetector(mode) elif model_type == 'classifier': arrange_transform = ArrangeClassifier(mode) elif model_type == 'detector': diff --git a/paddlers/transforms/operators.py b/paddlers/transforms/operators.py index 5c18672..4faa513 100644 --- a/paddlers/transforms/operators.py +++ b/paddlers/transforms/operators.py @@ -1370,6 +1370,32 @@ class ArrangeSegmenter(Transform): return image, +class ArrangeChangeDetector(Transform): + def __init__(self, mode): + super(ArrangeChangeDetector, self).__init__() + if mode not in ['train', 'eval', 'test', 'quant']: + raise ValueError( + "mode should be defined as one of ['train', 'eval', 'test', 'quant']!" + ) + self.mode = mode + + def apply(self, sample): + if 'mask' in sample: + mask = sample['mask'] + + image_t1 = permute(sample['image_t1'], False) + image_t2 = permute(sample['image_t2'], False) + if self.mode == 'train': + mask = mask.astype('int64') + return image_t1, image_t2, mask + if self.mode == 'eval': + mask = np.asarray(Image.open(mask)) + mask = mask[np.newaxis, :, :].astype('int64') + return image_t1, image_t2, mask + if self.mode == 'test': + return image_t1, image_t2, + + class ArrangeClassifier(Transform): def __init__(self, mode): super(ArrangeClassifier, self).__init__() diff --git a/requirements.txt b/requirements.txt index d08cf53..94e6df2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,4 @@ motmetrics matplotlib chardet openpyxl -GDAL >= 3.2.2 \ No newline at end of file +GDAL >= 3.1.3 \ No newline at end of file