diff --git a/paddlers/custom_models/cd/models/__init__.py b/paddlers/custom_models/cd/models/__init__.py index 7219a26..f7e244b 100644 --- a/paddlers/custom_models/cd/models/__init__.py +++ b/paddlers/custom_models/cd/models/__init__.py @@ -18,6 +18,7 @@ from .dsifn import DSIFN from .stanet import STANet from .snunet import SNUNet from .dsamnet import DSAMNet +from .changestar import ChangeStar from .unet_ef import UNetEarlyFusion from .unet_siamconc import UNetSiamConc from .unet_siamdiff import UNetSiamDiff \ No newline at end of file diff --git a/paddlers/custom_models/cd/models/changestar.py b/paddlers/custom_models/cd/models/changestar.py new file mode 100644 index 0000000..9090e7b --- /dev/null +++ b/paddlers/custom_models/cd/models/changestar.py @@ -0,0 +1,158 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle +import paddle.nn as nn +import paddle.nn.functional as F + +from paddlers.datasets.cd_dataset import MaskType +from paddlers.custom_models.seg.models import FarSeg +from .layers import Conv3x3, Identity + + +class _ChangeStarBase(nn.Layer): + + USE_MULTITASK_DECODER = True + OUT_TYPES = ( + MaskType.CD, + MaskType.CD, + MaskType.SEG_T1, + MaskType.SEG_T2 + ) + + def __init__( + self, + seg_model, + num_classes, + mid_channels, + inner_channels, + num_convs, + scale_factor + ): + super().__init__() + + self.extract = seg_model + self.detect = ChangeMixin( + in_ch=mid_channels*2, + out_ch=num_classes, + mid_ch=inner_channels, + num_convs=num_convs, + scale_factor=scale_factor + ) + self.segment = nn.Sequential( + Conv3x3(mid_channels, 2), + nn.UpsamplingBilinear2D(scale_factor=scale_factor) + ) + + self.init_weight() + + def forward(self, t1, t2): + x1 = self.extract(t1)[0] + x2 = self.extract(t2)[0] + logit12, logit21 = self.detect(x1, x2) + + if not self.training: + logit_list = [logit12] + else: + logit1 = self.segment(x1) + logit2 = self.segment(x2) + logit_list = [logit12, logit21, logit1, logit2] + + return logit_list + + def init_weight(self): + pass + + +class ChangeMixin(nn.Layer): + def __init__(self, in_ch, out_ch, mid_ch, num_convs, scale_factor): + super().__init__() + convs = [Conv3x3(in_ch, mid_ch, norm=True, act=True)] + convs += [ + Conv3x3(mid_ch, mid_ch, norm=True, act=True) + for _ in range(num_convs-1) + ] + self.detect = nn.Sequential( + *convs, + Conv3x3(mid_ch, out_ch), + nn.UpsamplingBilinear2D(scale_factor=scale_factor) + ) + + def forward(self, x1, x2): + pred12 = self.detect(paddle.concat([x1, x2], axis=1)) + pred21 = self.detect(paddle.concat([x2, x1], axis=1)) + return pred12, pred21 + + +class ChangeStar_FarSeg(_ChangeStarBase): + """ + The ChangeStar implementation with a FarSeg encoder based on PaddlePaddle. + + The original article refers to + Z. Zheng, et al., "Change is Everywhere: Single-Temporal Supervised Object Change Detection in Remote Sensing Imagery" + (https://arxiv.org/abs/2108.07002). + + Note that this implementation differs from the original code in two aspects: + 1. The encoder of the FarSeg model is ResNet50. + 2. We use conv-bn-relu instead of conv-relu-bn. + + Args: + num_classes (int): The number of target classes. + mid_channels (int, optional): The number of channels required by the ChangeMixin module. Default: 256. + inner_channels (int, optional): The number of filters used in the convolutional layers in the ChangeMixin module. + Default: 16. + num_convs (int, optional): The number of convolutional layers used in the ChangeMixin module. Default: 4. + scale_factor (float, optional): The scaling factor of the output upsampling layer. Default: 4.0. + """ + + def __init__( + self, + num_classes, + mid_channels=256, + inner_channels=16, + num_convs=4, + scale_factor=4.0, + ): + # TODO: Configurable FarSeg model + class _FarSegWrapper(nn.Layer): + def __init__(self, seg_model): + super().__init__() + self._seg_model = seg_model + self._seg_model.cls_pred_conv = Identity() + + def forward(self, x): + feat_list = self._seg_model.en(x) + fpn_feat_list = self._seg_model.fpn(feat_list) + if self._seg_model.scene_relation: + c5 = feat_list[-1] + c6 = self._seg_model.gap(c5) + refined_fpn_feat_list = self._seg_model.sr(c6, fpn_feat_list) + else: + refined_fpn_feat_list = fpn_feat_list + final_feat = self._seg_model.decoder(refined_fpn_feat_list) + return [final_feat] + + seg_model = FarSeg(out_ch=mid_channels) + + super().__init__( + seg_model=_FarSegWrapper(seg_model), + num_classes=num_classes, + mid_channels=mid_channels, + inner_channels=inner_channels, + num_convs=num_convs, + scale_factor=scale_factor + ) + +# NOTE: Currently, ChangeStar = FarSeg + ChangeMixin + SegHead +ChangeStar = ChangeStar_FarSeg \ No newline at end of file diff --git a/paddlers/tasks/changedetector.py b/paddlers/tasks/changedetector.py index 2ce63e4..feb620e 100644 --- a/paddlers/tasks/changedetector.py +++ b/paddlers/tasks/changedetector.py @@ -44,6 +44,8 @@ __all__ = [ "BIT", "SNUNet", "DSIFN", + "DSAMNet", + "ChangeStar" "DSAMNet" ] @@ -873,4 +875,36 @@ class DSAMNet(BaseChangeDetector): 'coef': [1.0, 0.05, 0.05] } else: - return super().default_loss() \ No newline at end of file + return super().default_loss() + + +class ChangeStar(BaseChangeDetector): + def __init__(self, + num_classes=2, + use_mixed_loss=False, + mid_channels=256, + inner_channels=16, + num_convs=4, + scale_factor=4.0, + **params): + params.update({ + 'mid_channels': mid_channels, + 'inner_channels': inner_channels, + 'num_convs': num_convs, + 'scale_factor': scale_factor + }) + super(ChangeStar, self).__init__( + model_name='ChangeStar', + num_classes=num_classes, + use_mixed_loss=use_mixed_loss, + **params) + + def default_loss(self): + if self.use_mixed_loss is False: + return { + # XXX: make sure the shallow copy works correctly here. + 'types': [paddleseg.models.CrossEntropyLoss()]*4, + 'coef': [1.0]*4 + } + else: + return super().default_loss()