# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import from __future__ import division from __future__ import print_function from sklearn.metrics import hamming_loss from sklearn.metrics import accuracy_score as accuracy_metric from sklearn.metrics import multilabel_confusion_matrix from sklearn.metrics import precision_recall_fscore_support from sklearn.metrics import average_precision_score from sklearn.preprocessing import binarize import numpy as np __all__ = [ "multi_hot_encode", "hamming_distance", "accuracy_score", "precision_recall_fscore", "mean_average_precision" ] def multi_hot_encode(logits, threshold=0.5): """ Encode logits to multi-hot by elementwise for multilabel """ return binarize(logits, threshold=threshold) def hamming_distance(output, target): """ Soft metric based label for multilabel classification Returns: The smaller the return value is, the better model is. """ return hamming_loss(target, output) def accuracy_score(output, target, base="sample"): """ Hard metric for multilabel classification Args: output: target: base: ["sample", "label"], default="sample" if "sample", return metric score based sample, if "label", return metric score based label. Returns: accuracy: """ assert base in ["sample", "label"], 'must be one of ["sample", "label"]' if base == "sample": accuracy = accuracy_metric(target, output) elif base == "label": mcm = multilabel_confusion_matrix(target, output) tns = mcm[:, 0, 0] fns = mcm[:, 1, 0] tps = mcm[:, 1, 1] fps = mcm[:, 0, 1] accuracy = (sum(tps) + sum(tns)) / ( sum(tps) + sum(tns) + sum(fns) + sum(fps)) return accuracy def precision_recall_fscore(output, target): """ Metric based label for multilabel classification Returns: precisions: recalls: fscores: """ precisions, recalls, fscores, _ = precision_recall_fscore_support(target, output) return precisions, recalls, fscores def mean_average_precision(logits, target): """ Calculate average precision Args: logits: probability from network before sigmoid or softmax target: ground truth, 0 or 1 """ if not (isinstance(logits, np.ndarray) and isinstance(target, np.ndarray)): raise TypeError("logits and target should be np.ndarray.") aps = [] for i in range(target.shape[1]): ap = average_precision_score(target[:, i], logits[:, i]) aps.append(ap) return np.mean(aps)