PaddleRS/paddlers/transforms/functions.py

# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy

import cv2
import numpy as np
import shapely.ops
from shapely.geometry import Polygon, MultiPolygon, GeometryCollection
from sklearn.linear_model import LinearRegression
from skimage import exposure
from joblib import load
from PIL import Image


def normalize(im, mean, std, min_value=[0, 0, 0], max_value=[255, 255, 255]):
    # Rescaling (min-max normalization)
    range_value = np.asarray(
        [1. / (max_value[i] - min_value[i]) for i in range(len(max_value))],
        dtype=np.float32)
    im = (im - np.asarray(min_value, dtype=np.float32)) * range_value

    # Standardization (Z-score Normalization)
    im -= mean
    im /= std
    return im


def permute(im, to_bgr=False):
    im = np.swapaxes(im, 1, 2)
    im = np.swapaxes(im, 1, 0)
    if to_bgr:
        im = im[[2, 1, 0], :, :]
    return im


def center_crop(im, crop_size=224):
    height, width = im.shape[:2]
    w_start = (width - crop_size) // 2
    h_start = (height - crop_size) // 2
    w_end = w_start + crop_size
    h_end = h_start + crop_size
    im = im[h_start:h_end, w_start:w_end, ...]
    return im


def img_flip(im, method=0):
    """
    Flip an image. 
    This function provides 5 flipping methods and can be applied to 2D or 3D numpy arrays.

    Args:
        im (np.ndarray): Input image.
        method (int|string): Flipping method. Must be one of [
                                0, 1, 2, 3, 4, 'h', 'v', 'hv', 'rt2lb', 'lt2rb', 
                                'dia', 'adia'].
            0 or 'h': flip the image in horizontal direction, which is the most frequently 
                used method;
            1 or 'v': flip the image in vertical direction;
            2 or 'hv': flip the image in both horizontal diction and vertical direction;
            3 or 'rt2lb' or 'dia': flip the image across the diagonal;
            4 or 'lt2rb' or 'adia': flip the image across the anti-diagonal.

    Returns:
        np.ndarray: Flipped image.

    Raises:
        ValueError: Invalid shape of images.

    Examples:
        Assume an image is like this:

        img:
        / + +
        - / *
        - * /

        We can flip it with following code:

        img_h = img_flip(img, 'h')
        img_v = img_flip(img, 'v')
        img_vh = img_flip(img, 2)
        img_rt2lb = img_flip(img, 3)
        img_lt2rb = img_flip(img, 4)

        Then we get the flipped images:

        img_h, flipped in horizontal direction:
        + + \
        * \ -
        \ * -

        img_v, flipped in vertical direction:
        - * \
        - \ *
        \ + +

        img_vh, flipped in both horizontal diction and vertical direction:
        / * -
        * / -
        + + /

        img_rt2lb, mirrored on the diagonal:
        / | |
        + / *
        + * /

        img_lt2rb, mirrored on the anti-diagonal:
        / * +
        * / +
        | | /
    """

    if not len(im.shape) >= 2:
        raise ValueError("The number of image dimensions is less than 2.")
    if method == 0 or method == 'h':
        return horizontal_flip(im)
    elif method == 1 or method == 'v':
        return vertical_flip(im)
    elif method == 2 or method == 'hv':
        return hv_flip(im)
    elif method == 3 or method == 'rt2lb' or method == 'dia':
        return rt2lb_flip(im)
    elif method == 4 or method == 'lt2rb' or method == 'adia':
        return lt2rb_flip(im)
    else:
        return im


def horizontal_flip(im):
    im = im[:, ::-1, ...]
    return im


def vertical_flip(im):
    im = im[::-1, :, ...]
    return im


def hv_flip(im):
    im = im[::-1, ::-1, ...]
    return im


def rt2lb_flip(im):
    axs_list = list(range(len(im.shape)))
    axs_list[:2] = [1, 0]
    im = im.transpose(axs_list)
    return im


def lt2rb_flip(im):
    axs_list = list(range(len(im.shape)))
    axs_list[:2] = [1, 0]
    im = im[::-1, ::-1, ...].transpose(axs_list)
    return im


def img_simple_rotate(im, method=0):
    """
    Rotate an image. 
    This function provides 3 rotating methods and can be applied to 2D or 3D numpy arrays.

    Args:
        im (np.ndarray): Input image.
        method (int|string): Rotating method, which must be one of [
                                0, 1, 2, 90, 180, 270
                                ].
            0 or 90 : rotate the image by 90 degrees, clockwise;
            1 or 180: rotate the image by 180 degrees, clockwise;
            2 or 270: rotate the image by 270 degrees, clockwise.

    Returns:
        np.ndarray: Rotated image.

    Raises:
        ValueError: Invalid shape of images.

    Examples:
        Assume an image is like this:

        img:
        / + +
        - / *
        - * /

        We can rotate it with following code:

        img_r90 = img_simple_rotate(img, 90)
        img_r180 = img_simple_rotate(img, 1)
        img_r270 = img_simple_rotate(img, 2)

        Then we get the following rotated images:

        img_r90, rotated by 90°:
        | | \
        * \ +
        \ * +

        img_r180, rotated by 180°:
        / * -
        * / -
        + + /

        img_r270, rotated by 270°:
        + * \
        + \ *
        \ | |
    """

    if not len(im.shape) >= 2:
        raise ValueError("The number of image dimensions is less than 2.")
    if method == 0 or method == 90:
        return rot_90(im)
    elif method == 1 or method == 180:
        return rot_180(im)
    elif method == 2 or method == 270:
        return rot_270(im)
    else:
        return im


def rot_90(im):
    axs_list = list(range(len(im.shape)))
    axs_list[:2] = [1, 0]
    im = im[::-1, :, ...].transpose(axs_list)
    return im


def rot_180(im):
    im = im[::-1, ::-1, ...]
    return im


def rot_270(im):
    axs_list = list(range(len(im.shape)))
    axs_list[:2] = [1, 0]
    im = im[:, ::-1, ...].transpose(axs_list)
    return im


def rgb2bgr(im):
    return im[:, :, ::-1]


def is_poly(poly):
    assert isinstance(poly, (list, dict)), \
        "Invalid poly type: {}".format(type(poly))
    return isinstance(poly, list)


def horizontal_flip_poly(poly, width):
    flipped_poly = np.array(poly)
    flipped_poly[0::2] = width - np.array(poly[0::2])
    return flipped_poly.tolist()


def horizontal_flip_rle(rle, height, width):
    import pycocotools.mask as mask_util
    if 'counts' in rle and type(rle['counts']) == list:
        rle = mask_util.frPyObjects(rle, height, width)
    mask = mask_util.decode(rle)
    mask = mask[:, ::-1]
    rle = mask_util.encode(np.array(mask, order='F', dtype=np.uint8))
    return rle


def vertical_flip_poly(poly, height):
    flipped_poly = np.array(poly)
    flipped_poly[1::2] = height - np.array(poly[1::2])
    return flipped_poly.tolist()


def vertical_flip_rle(rle, height, width):
    import pycocotools.mask as mask_util
    if 'counts' in rle and type(rle['counts']) == list:
        rle = mask_util.frPyObjects(rle, height, width)
    mask = mask_util.decode(rle)
    mask = mask[::-1, :]
    rle = mask_util.encode(np.array(mask, order='F', dtype=np.uint8))
    return rle


def crop_poly(segm, crop):
    xmin, ymin, xmax, ymax = crop
    crop_coord = [xmin, ymin, xmin, ymax, xmax, ymax, xmax, ymin]
    crop_p = np.array(crop_coord).reshape(4, 2)
    crop_p = Polygon(crop_p)

    crop_segm = list()
    for poly in segm:
        poly = np.array(poly).reshape(len(poly) // 2, 2)
        polygon = Polygon(poly)
        if not polygon.is_valid:
            exterior = polygon.exterior
            multi_lines = exterior.intersection(exterior)
            polygons = shapely.ops.polygonize(multi_lines)
            polygon = MultiPolygon(polygons)
        multi_polygon = list()
        if isinstance(polygon, MultiPolygon):
            multi_polygon = copy.deepcopy(polygon)
        else:
            multi_polygon.append(copy.deepcopy(polygon))
        for per_polygon in multi_polygon:
            inter = per_polygon.intersection(crop_p)
            if not inter:
                continue
            if isinstance(inter, (MultiPolygon, GeometryCollection)):
                for part in inter:
                    if not isinstance(part, Polygon):
                        continue
                    part = np.squeeze(
                        np.array(part.exterior.coords[:-1]).reshape(1, -1))
                    part[0::2] -= xmin
                    part[1::2] -= ymin
                    crop_segm.append(part.tolist())
            elif isinstance(inter, Polygon):
                crop_poly = np.squeeze(
                    np.array(inter.exterior.coords[:-1]).reshape(1, -1))
                crop_poly[0::2] -= xmin
                crop_poly[1::2] -= ymin
                crop_segm.append(crop_poly.tolist())
            else:
                continue
    return crop_segm


def crop_rle(rle, crop, height, width):
    import pycocotools.mask as mask_util
    if 'counts' in rle and type(rle['counts']) == list:
        rle = mask_util.frPyObjects(rle, height, width)
    mask = mask_util.decode(rle)
    mask = mask[crop[1]:crop[3], crop[0]:crop[2]]
    rle = mask_util.encode(np.array(mask, order='F', dtype=np.uint8))
    return rle


def expand_poly(poly, x, y):
    expanded_poly = np.array(poly)
    expanded_poly[0::2] += x
    expanded_poly[1::2] += y
    return expanded_poly.tolist()


def expand_rle(rle, x, y, height, width, h, w):
    import pycocotools.mask as mask_util
    if 'counts' in rle and type(rle['counts']) == list:
        rle = mask_util.frPyObjects(rle, height, width)
    mask = mask_util.decode(rle)
    expanded_mask = np.full((h, w), 0).astype(mask.dtype)
    expanded_mask[y:y + height, x:x + width] = mask
    rle = mask_util.encode(np.array(expanded_mask, order='F', dtype=np.uint8))
    return rle


def resize_poly(poly, im_scale_x, im_scale_y):
    resized_poly = np.array(poly, dtype=np.float32)
    resized_poly[0::2] *= im_scale_x
    resized_poly[1::2] *= im_scale_y
    return resized_poly.tolist()


def resize_rle(rle, im_h, im_w, im_scale_x, im_scale_y, interp):
    import pycocotools.mask as mask_util
    if 'counts' in rle and type(rle['counts']) == list:
        rle = mask_util.frPyObjects(rle, im_h, im_w)
    mask = mask_util.decode(rle)
    mask = cv2.resize(
        mask, None, None, fx=im_scale_x, fy=im_scale_y, interpolation=interp)
    rle = mask_util.encode(np.array(mask, order='F', dtype=np.uint8))
    return rle


def to_uint8(im, stretch=False):
    """
    Convert raster data to uint8 type.
    
    Args:
        im (np.ndarray): Input raster image.
        stretch (bool, optional): Use 2% linear stretch or not. Default is False.

    Returns:
        np.ndarray: Image data with unit8 type.
    """

    # 2% linear stretch
    def _two_percent_linear(image, max_out=255, min_out=0):
        def _gray_process(gray, maxout=max_out, minout=min_out):
            # Get the corresponding gray level at 98% in the histogram.
            high_value = np.percentile(gray, 98)
            low_value = np.percentile(gray, 2)
            truncated_gray = np.clip(gray, a_min=low_value, a_max=high_value)
            processed_gray = ((truncated_gray - low_value) / (high_value - low_value)) * \
                             (maxout - minout)
            return np.uint8(processed_gray)

        if len(image.shape) == 3:
            processes = []
            for b in range(image.shape[-1]):
                processes.append(_gray_process(image[:, :, b]))
            result = np.stack(processes, axis=2)
        else:  # if len(image.shape) == 2
            result = _gray_process(image)
        return np.uint8(result)

    # Simple image standardization
    def _sample_norm(image):
        stretches = []
        if len(image.shape) == 3:
            for b in range(image.shape[-1]):
                stretched = exposure.equalize_hist(image[:, :, b])
                stretched /= float(np.max(stretched))
                stretches.append(stretched)
            stretched_img = np.stack(stretches, axis=2)
        else:  # if len(image.shape) == 2
            stretched_img = exposure.equalize_hist(image)
        return np.uint8(stretched_img * 255)

    dtype = im.dtype.name
    if dtype != "uint8":
        im = _sample_norm(im)
    if stretch:
        im = _two_percent_linear(im)
    return im


def to_intensity(im):
    """
    Calculate the intensity of SAR data.

    Args:
        im (np.ndarray): SAR image.

    Returns:
        np.ndarray: Intensity image.
    """

    if len(im.shape) != 2:
        raise ValueError("`len(im.shape) must be 2.")
    # If the type is complex, this is SAR data.
    if isinstance(type(im[0, 0]), complex):
        im = abs(im)
    return im


def select_bands(im, band_list=[1, 2, 3]):
    """
    Select bands of a multi-band image.

    Args:
        im (np.ndarray): Input image.
        band_list (list, optional): Bands to select (band index start from 1). 
            Defaults to [1, 2, 3].

    Returns:
        np.ndarray: Image with selected bands.
    """

    if len(im.shape) == 2:  # Image has only one channel
        return im
    if not isinstance(band_list, list) or len(band_list) == 0:
        raise TypeError("band_list must be non empty list.")
    total_band = im.shape[-1]
    result = []
    for band in band_list:
        band = int(band - 1)
        if band < 0 or band >= total_band:
            raise ValueError("The element in band_list must > 1 and <= {}.".
                             format(str(total_band)))
        result.append(im[:, :, band])
    ima = np.stack(result, axis=-1)
    return ima


def dehaze(im, gamma=False):
    """
    Perform single image haze removal using dark channel prior.

    Args:
        im (np.ndarray): Input image.
        gamma (bool, optional): Use gamma correction or not. Defaults to False.

    Returns:
        np.ndarray: Output dehazed image.
    """

    def _guided_filter(I, p, r, eps):
        m_I = cv2.boxFilter(I, -1, (r, r))
        m_p = cv2.boxFilter(p, -1, (r, r))
        m_Ip = cv2.boxFilter(I * p, -1, (r, r))
        cov_Ip = m_Ip - m_I * m_p
        m_II = cv2.boxFilter(I * I, -1, (r, r))
        var_I = m_II - m_I * m_I
        a = cov_Ip / (var_I + eps)
        b = m_p - a * m_I
        m_a = cv2.boxFilter(a, -1, (r, r))
        m_b = cv2.boxFilter(b, -1, (r, r))
        return m_a * I + m_b

    def _dehaze(im, r, w, maxatmo_mask, eps):
        # im is a RGB image and the value ranges in [0, 1].
        atmo_mask = np.min(im, 2)
        dark_channel = cv2.erode(atmo_mask, np.ones((15, 15)))
        atmo_mask = _guided_filter(atmo_mask, dark_channel, r, eps)
        bins = 2000
        ht = np.histogram(atmo_mask, bins)
        d = np.cumsum(ht[0]) / float(atmo_mask.size)
        for lmax in range(bins - 1, 0, -1):
            if d[lmax] <= 0.999:
                break
        atmo_illum = np.mean(im, 2)[atmo_mask >= ht[1][lmax]].max()
        atmo_mask = np.minimum(atmo_mask * w, maxatmo_mask)
        return atmo_mask, atmo_illum

    if np.max(im) > 1:
        im = im / 255.
    result = np.zeros(im.shape)
    mask_img, atmo_illum = _dehaze(
        im, r=81, w=0.95, maxatmo_mask=0.80, eps=1e-8)
    for k in range(3):
        result[:, :, k] = (im[:, :, k] - mask_img) / (1 - mask_img / atmo_illum)
    result = np.clip(result, 0, 1)
    if gamma:
        result = result**(np.log(0.5) / np.log(result.mean()))
    return (result * 255).astype("uint8")


def match_histograms(im, ref):
    """
    Match the cumulative histogram of one image to another.

    Args:
        im (np.ndarray): Input image.
        ref (np.ndarray): Reference image to match histogram of. `ref` must have 
            the same number of channels as `im`.

    Returns:
        np.ndarray: Transformed input image.

    Raises:
        ValueError: When the number of channels of `ref` differs from that of im`.
    """

    # TODO: Check the data types of the inputs to see if they are supported by skimage
    return exposure.match_histograms(
        im, ref, channel_axis=-1 if im.ndim > 2 else None)


def match_by_regression(im, ref, pif_loc=None):
    """
    Match the brightness values of two images using a linear regression method.

    Args:
        im (np.ndarray): Input image.
        ref (np.ndarray): Reference image to match. `ref` must have the same shape 
            as `im`.
        pif_loc (tuple|None, optional): Spatial locations where pseudo-invariant 
            features (PIFs) are obtained. If `pif_loc` is set to None, all pixels in 
            the image will be used as training samples for the regression model. In 
            other cases, `pif_loc` should be a tuple of np.ndarrays. Default: None.

    Returns:
        np.ndarray: Transformed input image.

    Raises:
        ValueError: When the shape of `ref` differs from that of `im`.
    """

    def _linear_regress(im, ref, loc):
        regressor = LinearRegression()
        if loc is not None:
            x, y = im[loc], ref[loc]
        else:
            x, y = im, ref
        x, y = x.reshape(-1, 1), y.ravel()
        regressor.fit(x, y)
        matched = regressor.predict(im.reshape(-1, 1))
        return matched.reshape(im.shape)

    if im.shape != ref.shape:
        raise ValueError("Image and Reference must have the same shape!")

    if im.ndim > 2:
        # Multiple channels
        matched = np.empty(im.shape, dtype=im.dtype)
        for ch in range(im.shape[-1]):
            matched[..., ch] = _linear_regress(im[..., ch], ref[..., ch],
                                               pif_loc)
    else:
        # Single channel
        matched = _linear_regress(im, ref, pif_loc).astype(im.dtype)

    return matched


def match_lf_components(im, ref, lf_ratio=0.01):
    """
    Match the low-frequency components of two images.

    Args:
        im (np.ndarray): Input image.
        ref (np.ndarray): Reference image to match. `ref` must have the same shape 
            as `im`.
        lf_ratio (float, optional): Proportion of frequence components that should
            be recognized as low-frequency components in the frequency domain. 
            Default: 0.01.

    Returns:
        np.ndarray: Transformed input image.

    Raises:
        ValueError: When the shape of `ref` differs from that of `im`.
    """

    def _replace_lf(im, ref, lf_ratio):
        h, w = im.shape
        h_lf, w_lf = int(h // 2 * lf_ratio), int(w // 2 * lf_ratio)
        freq_im = np.fft.fft2(im)
        freq_ref = np.fft.fft2(ref)
        if h_lf > 0:
            freq_im[:h_lf] = freq_ref[:h_lf]
            freq_im[-h_lf:] = freq_ref[-h_lf:]
        if w_lf > 0:
            freq_im[:, :w_lf] = freq_ref[:, :w_lf]
            freq_im[:, -w_lf:] = freq_ref[:, -w_lf:]
        recon_im = np.fft.ifft2(freq_im)
        recon_im = np.abs(recon_im)
        return recon_im

    if im.shape != ref.shape:
        raise ValueError("Image and Reference must have the same shape!")

    if im.ndim > 2:
        # Multiple channels
        matched = np.empty(im.shape, dtype=im.dtype)
        for ch in range(im.shape[-1]):
            matched[..., ch] = _replace_lf(im[..., ch], ref[..., ch], lf_ratio)
    else:
        # Single channel
        matched = _replace_lf(im, ref, lf_ratio).astype(im.dtype)

    return matched


def inv_pca(im, joblib_path):
    """
    Perform inverse PCA transformation.

    Args:
        im (np.ndarray): Input image after performing PCA.
        joblib_path (str): Path of *.joblib file that stores PCA information.

    Returns:
        np.ndarray: Reconstructed input image.
    """

    pca = load(joblib_path)
    H, W, C = im.shape
    n_im = np.reshape(im, (-1, C))
    r_im = pca.inverse_transform(n_im)
    r_im = np.reshape(r_im, (H, W, -1))
    return r_im


def decode_seg_mask(mask_path):
    """
    Decode a segmentation mask image.
    
    Args:
        mask_path (str): Path of the mask image to decode.

    Returns:
        np.ndarray: Decoded mask image.
    """

    mask = np.asarray(Image.open(mask_path))
    mask = mask.astype('int64')
    return mask


def calc_hr_shape(lr_shape, sr_factor):
    return tuple(int(s * sr_factor) for s in lr_shape)