Merge pull request #1211 from VladX:lsbp

pull/1475/head
Maksim Shabunin 7 years ago
commit 6520dbaa22
  1. 2
      modules/bgsegm/CMakeLists.txt
  2. 10
      modules/bgsegm/doc/bgsegm.bib
  3. 129
      modules/bgsegm/include/opencv2/bgsegm.hpp
  4. 134
      modules/bgsegm/samples/evaluation.py
  5. 41
      modules/bgsegm/samples/viz.py
  6. 26
      modules/bgsegm/samples/viz_synthetic_seq.py
  7. 997
      modules/bgsegm/src/bgfg_gsoc.cpp
  8. 221
      modules/bgsegm/src/synthetic_seq.cpp
  9. 137
      modules/bgsegm/test/test_backgroundsubtractor_lsbp.cpp

@ -1,2 +1,2 @@
set(the_description "Background Segmentation Algorithms")
ocv_define_module(bgsegm opencv_core opencv_imgproc opencv_video WRAP python)
ocv_define_module(bgsegm opencv_core opencv_imgproc opencv_video opencv_calib3d WRAP python)

@ -15,3 +15,13 @@
year={2012},
organization={IEEE}
}
@inproceedings{LGuo2016,
author={L. Guo and D. Xu and Z. Qiang},
booktitle={2016 IEEE Conference on Computer Vision and Pattern Recognition Workshops (CVPRW)},
title={Background Subtraction Using Local SVD Binary Pattern},
year={2016},
pages={1159-1167},
doi={10.1109/CVPRW.2016.148},
month={June}
}

@ -242,6 +242,135 @@ createBackgroundSubtractorCNT(int minPixelStability = 15,
int maxPixelStability = 15*60,
bool isParallel = true);
enum LSBPCameraMotionCompensation {
LSBP_CAMERA_MOTION_COMPENSATION_NONE = 0,
LSBP_CAMERA_MOTION_COMPENSATION_LK
};
/** @brief Implementation of the different yet better algorithm which is called GSOC, as it was implemented during GSOC and was not originated from any paper.
This algorithm demonstrates better performance on CDNET 2014 dataset compared to other algorithms in OpenCV.
*/
class CV_EXPORTS_W BackgroundSubtractorGSOC : public BackgroundSubtractor
{
public:
// BackgroundSubtractor interface
CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate=-1) = 0;
CV_WRAP virtual void getBackgroundImage(OutputArray backgroundImage) const = 0;
};
/** @brief Background Subtraction using Local SVD Binary Pattern. More details about the algorithm can be found at @cite LGuo2016
*/
class CV_EXPORTS_W BackgroundSubtractorLSBP : public BackgroundSubtractor
{
public:
// BackgroundSubtractor interface
CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate=-1) = 0;
CV_WRAP virtual void getBackgroundImage(OutputArray backgroundImage) const = 0;
};
/** @brief This is for calculation of the LSBP descriptors.
*/
class CV_EXPORTS_W BackgroundSubtractorLSBPDesc
{
public:
static void calcLocalSVDValues(OutputArray localSVDValues, const Mat& frame);
static void computeFromLocalSVDValues(OutputArray desc, const Mat& localSVDValues, const Point2i* LSBPSamplePoints);
static void compute(OutputArray desc, const Mat& frame, const Point2i* LSBPSamplePoints);
};
/** @brief Creates an instance of BackgroundSubtractorGSOC algorithm.
Implementation of the different yet better algorithm which is called GSOC, as it was implemented during GSOC and was not originated from any paper.
@param mc Whether to use camera motion compensation.
@param nSamples Number of samples to maintain at each point of the frame.
@param replaceRate Probability of replacing the old sample - how fast the model will update itself.
@param propagationRate Probability of propagating to neighbors.
@param hitsThreshold How many positives the sample must get before it will be considered as a possible replacement.
@param alpha Scale coefficient for threshold.
@param beta Bias coefficient for threshold.
@param blinkingSupressionDecay Blinking supression decay factor.
@param blinkingSupressionMultiplier Blinking supression multiplier.
@param noiseRemovalThresholdFacBG Strength of the noise removal for background points.
@param noiseRemovalThresholdFacFG Strength of the noise removal for foreground points.
*/
CV_EXPORTS_W Ptr<BackgroundSubtractorGSOC> createBackgroundSubtractorGSOC(int mc = LSBP_CAMERA_MOTION_COMPENSATION_NONE, int nSamples = 20, float replaceRate = 0.003f, float propagationRate = 0.01f, int hitsThreshold = 32, float alpha = 0.01f, float beta = 0.0022f, float blinkingSupressionDecay = 0.1f, float blinkingSupressionMultiplier = 0.1f, float noiseRemovalThresholdFacBG = 0.0004f, float noiseRemovalThresholdFacFG = 0.0008f);
/** @brief Creates an instance of BackgroundSubtractorLSBP algorithm.
Background Subtraction using Local SVD Binary Pattern. More details about the algorithm can be found at @cite LGuo2016
@param mc Whether to use camera motion compensation.
@param nSamples Number of samples to maintain at each point of the frame.
@param LSBPRadius LSBP descriptor radius.
@param Tlower Lower bound for T-values. See @cite LGuo2016 for details.
@param Tupper Upper bound for T-values. See @cite LGuo2016 for details.
@param Tinc Increase step for T-values. See @cite LGuo2016 for details.
@param Tdec Decrease step for T-values. See @cite LGuo2016 for details.
@param Rscale Scale coefficient for threshold values.
@param Rincdec Increase/Decrease step for threshold values.
@param noiseRemovalThresholdFacBG Strength of the noise removal for background points.
@param noiseRemovalThresholdFacFG Strength of the noise removal for foreground points.
@param LSBPthreshold Threshold for LSBP binary string.
@param minCount Minimal number of matches for sample to be considered as foreground.
*/
CV_EXPORTS_W Ptr<BackgroundSubtractorLSBP> createBackgroundSubtractorLSBP(int mc = LSBP_CAMERA_MOTION_COMPENSATION_NONE, int nSamples = 20, int LSBPRadius = 16, float Tlower = 2.0f, float Tupper = 32.0f, float Tinc = 1.0f, float Tdec = 0.05f, float Rscale = 10.0f, float Rincdec = 0.005f, float noiseRemovalThresholdFacBG = 0.0004f, float noiseRemovalThresholdFacFG = 0.0008f, int LSBPthreshold = 8, int minCount = 2);
/** @brief Synthetic frame sequence generator for testing background subtraction algorithms.
It will generate the moving object on top of the background.
It will apply some distortion to the background to make the test more complex.
*/
class CV_EXPORTS_W SyntheticSequenceGenerator : public Algorithm
{
private:
const double amplitude;
const double wavelength;
const double wavespeed;
const double objspeed;
unsigned timeStep;
Point2d pos;
Point2d dir;
Mat background;
Mat object;
RNG rng;
public:
/** @brief Creates an instance of SyntheticSequenceGenerator.
@param background Background image for object.
@param object Object image which will move slowly over the background.
@param amplitude Amplitude of wave distortion applied to background.
@param wavelength Length of waves in distortion applied to background.
@param wavespeed How fast waves will move.
@param objspeed How fast object will fly over background.
*/
CV_WRAP SyntheticSequenceGenerator(InputArray background, InputArray object, double amplitude, double wavelength, double wavespeed, double objspeed);
/** @brief Obtain the next frame in the sequence.
@param frame Output frame.
@param gtMask Output ground-truth (reference) segmentation mask object/background.
*/
CV_WRAP void getNextFrame(OutputArray frame, OutputArray gtMask);
};
/** @brief Creates an instance of SyntheticSequenceGenerator.
@param background Background image for object.
@param object Object image which will move slowly over the background.
@param amplitude Amplitude of wave distortion applied to background.
@param wavelength Length of waves in distortion applied to background.
@param wavespeed How fast waves will move.
@param objspeed How fast object will fly over background.
*/
CV_EXPORTS_W Ptr<SyntheticSequenceGenerator> createSyntheticSequenceGenerator(InputArray background, InputArray object, double amplitude = 2.0, double wavelength = 20.0, double wavespeed = 0.2, double objspeed = 6.0);
//! @}
}

@ -0,0 +1,134 @@
import argparse
import cv2
import glob
import numpy as np
import os
import time
# This tool is intended for evaluation of different background subtraction algorithms presented in OpenCV.
# Several presets with different settings are available. You can see them below.
# This tool measures quality metrics as well as speed.
ALGORITHMS_TO_EVALUATE = [
(cv2.bgsegm.createBackgroundSubtractorMOG, 'MOG', {}),
(cv2.bgsegm.createBackgroundSubtractorGMG, 'GMG', {}),
(cv2.bgsegm.createBackgroundSubtractorCNT, 'CNT', {}),
(cv2.bgsegm.createBackgroundSubtractorLSBP, 'LSBP-vanilla', {'nSamples': 20, 'LSBPRadius': 4, 'Tlower': 2.0, 'Tupper': 200.0, 'Tinc': 1.0, 'Tdec': 0.05, 'Rscale': 5.0, 'Rincdec': 0.05, 'LSBPthreshold': 8}),
(cv2.bgsegm.createBackgroundSubtractorLSBP, 'LSBP-speed', {'nSamples': 10, 'LSBPRadius': 16, 'Tlower': 2.0, 'Tupper': 32.0, 'Tinc': 1.0, 'Tdec': 0.05, 'Rscale': 10.0, 'Rincdec': 0.005, 'LSBPthreshold': 8}),
(cv2.bgsegm.createBackgroundSubtractorLSBP, 'LSBP-quality', {'nSamples': 20, 'LSBPRadius': 16, 'Tlower': 2.0, 'Tupper': 32.0, 'Tinc': 1.0, 'Tdec': 0.05, 'Rscale': 10.0, 'Rincdec': 0.005, 'LSBPthreshold': 8}),
(cv2.bgsegm.createBackgroundSubtractorLSBP, 'LSBP-camera-motion-compensation', {'mc': 1}),
(cv2.bgsegm.createBackgroundSubtractorGSOC, 'GSOC', {}),
(cv2.bgsegm.createBackgroundSubtractorGSOC, 'GSOC-camera-motion-compensation', {'mc': 1})
]
def contains_relevant_files(root):
return os.path.isdir(os.path.join(root, 'groundtruth')) and os.path.isdir(os.path.join(root, 'input'))
def find_relevant_dirs(root):
relevant_dirs = []
for d in sorted(os.listdir(root)):
d = os.path.join(root, d)
if os.path.isdir(d):
if contains_relevant_files(d):
relevant_dirs += [d]
else:
relevant_dirs += find_relevant_dirs(d)
return relevant_dirs
def load_sequence(root):
gt_dir, frames_dir = os.path.join(root, 'groundtruth'), os.path.join(root, 'input')
gt = sorted(glob.glob(os.path.join(gt_dir, '*.png')))
f = sorted(glob.glob(os.path.join(frames_dir, '*.jpg')))
assert(len(gt) == len(f))
return gt, f
def evaluate_algorithm(gt, frames, algo, algo_arguments):
bgs = algo(**algo_arguments)
mask = []
t_start = time.time()
for i in range(len(gt)):
frame = np.uint8(cv2.imread(frames[i], cv2.IMREAD_COLOR))
mask.append(bgs.apply(frame))
average_duration = (time.time() - t_start) / len(gt)
average_precision, average_recall, average_f1, average_accuracy = [], [], [], []
for i in range(len(gt)):
gt_mask = np.uint8(cv2.imread(gt[i], cv2.IMREAD_GRAYSCALE))
roi = ((gt_mask == 255) | (gt_mask == 0))
if roi.sum() > 0:
gt_answer, answer = gt_mask[roi], mask[i][roi]
tp = ((answer == 255) & (gt_answer == 255)).sum()
tn = ((answer == 0) & (gt_answer == 0)).sum()
fp = ((answer == 255) & (gt_answer == 0)).sum()
fn = ((answer == 0) & (gt_answer == 255)).sum()
if tp + fp > 0:
average_precision.append(float(tp) / (tp + fp))
if tp + fn > 0:
average_recall.append(float(tp) / (tp + fn))
if tp + fn + fp > 0:
average_f1.append(2.0 * tp / (2.0 * tp + fn + fp))
average_accuracy.append(float(tp + tn) / (tp + tn + fp + fn))
return average_duration, np.mean(average_precision), np.mean(average_recall), np.mean(average_f1), np.mean(average_accuracy)
def evaluate_on_sequence(seq, summary):
gt, frames = load_sequence(seq)
category, video_name = os.path.basename(os.path.dirname(seq)), os.path.basename(seq)
print('=== %s:%s ===' % (category, video_name))
for algo, algo_name, algo_arguments in ALGORITHMS_TO_EVALUATE:
print('Algorithm name: %s' % algo_name)
sec_per_step, precision, recall, f1, accuracy = evaluate_algorithm(gt, frames, algo, algo_arguments)
print('Average accuracy: %.3f' % accuracy)
print('Average precision: %.3f' % precision)
print('Average recall: %.3f' % recall)
print('Average F1: %.3f' % f1)
print('Average sec. per step: %.4f' % sec_per_step)
print('')
if category not in summary:
summary[category] = {}
if algo_name not in summary[category]:
summary[category][algo_name] = []
summary[category][algo_name].append((precision, recall, f1, accuracy))
def main():
parser = argparse.ArgumentParser(description='Evaluate all background subtractors using Change Detection 2014 dataset')
parser.add_argument('--dataset_path', help='Path to the directory with dataset. It may contain multiple inner directories. It will be scanned recursively.', required=True)
parser.add_argument('--algorithm', help='Test particular algorithm instead of all.')
args = parser.parse_args()
dataset_dirs = find_relevant_dirs(args.dataset_path)
assert len(dataset_dirs) > 0, ("Passed directory must contain at least one sequence from the Change Detection dataset. There is no relevant directories in %s. Check that this directory is correct." % (args.dataset_path))
if args.algorithm is not None:
global ALGORITHMS_TO_EVALUATE
ALGORITHMS_TO_EVALUATE = filter(lambda a: a[1].lower() == args.algorithm.lower(), ALGORITHMS_TO_EVALUATE)
summary = {}
for seq in dataset_dirs:
evaluate_on_sequence(seq, summary)
for category in summary:
for algo_name in summary[category]:
summary[category][algo_name] = np.mean(summary[category][algo_name], axis=0)
for category in summary:
print('=== SUMMARY for %s (Precision, Recall, F1, Accuracy) ===' % category)
for algo_name in summary[category]:
print('%05s: %.3f %.3f %.3f %.3f' % ((algo_name,) + tuple(summary[category][algo_name])))
if __name__ == '__main__':
main()

@ -0,0 +1,41 @@
import numpy as np
import cv2
import argparse
import os
def main():
argparser = argparse.ArgumentParser(description='Vizualization of the LSBP/GSOC background subtraction algorithm.')
argparser.add_argument('-g', '--gt', help='Directory with ground-truth frames', required=True)
argparser.add_argument('-f', '--frames', help='Directory with input frames', required=True)
argparser.add_argument('-l', '--lsbp', help='Display LSBP instead of GSOC', default=False)
args = argparser.parse_args()
gt = map(lambda x: os.path.join(args.gt, x), os.listdir(args.gt))
gt.sort()
f = map(lambda x: os.path.join(args.frames, x), os.listdir(args.frames))
f.sort()
gt = np.uint8(map(lambda x: cv2.imread(x, cv2.IMREAD_GRAYSCALE), gt))
f = np.uint8(map(lambda x: cv2.imread(x, cv2.IMREAD_COLOR), f))
if not args.lsbp:
bgs = cv2.bgsegm.createBackgroundSubtractorGSOC()
else:
bgs = cv2.bgsegm.createBackgroundSubtractorLSBP()
for i in xrange(f.shape[0]):
cv2.imshow('Frame', f[i])
cv2.imshow('Ground-truth', gt[i])
mask = bgs.apply(f[i])
bg = bgs.getBackgroundImage()
cv2.imshow('BG', bg)
cv2.imshow('Output mask', mask)
k = cv2.waitKey(0)
if k == 27:
break
if __name__ == '__main__':
main()

@ -0,0 +1,26 @@
import cv2
import argparse
def main():
argparser = argparse.ArgumentParser(description='Vizualization of the SyntheticSequenceGenerator.')
argparser.add_argument('-b', '--background', help='Background image.', required=True)
argparser.add_argument('-o', '--obj', help='Object image. It must be strictly smaller than background.', required=True)
args = argparser.parse_args()
bg = cv2.imread(args.background)
obj = cv2.imread(args.obj)
generator = cv2.bgsegm.createSyntheticSequenceGenerator(bg, obj)
while True:
frame, mask = generator.getNextFrame()
cv2.imshow('Generated frame', frame)
cv2.imshow('Generated mask', mask)
k = cv2.waitKey(int(1000.0 / 30))
if k == 27:
break
if __name__ == '__main__':
main()

@ -0,0 +1,997 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000, Intel Corporation, all rights reserved.
// Copyright (C) 2013, OpenCV Foundation, all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
/**
* @file bgfg_gsoc.cpp
* @author Vladislav Samsonov <vvladxx@gmail.com>
* @brief Background Subtraction using Local SVD Binary Pattern. See the following paper:
* http://www.cv-foundation.org/openaccess/content_cvpr_2016_workshops/w24/papers/Guo_Background_Subtraction_Using_CVPR_2016_paper.pdf
* This file also contains implementation of the different yet better algorithm which is called GSOC, as it was implemented during GSOC and was not originated from any paper.
*
*/
#include "precomp.hpp"
#include <opencv2/calib3d.hpp>
#include <iostream>
#include "opencv2/core/cvdef.h"
namespace cv
{
namespace bgsegm
{
namespace
{
const float LSBPtau = 0.05f;
#ifdef _MSC_VER
#include <intrin.h>
#pragma intrinsic(__popcnt)
#endif
inline int LSBPDist32(unsigned n) {
#if defined(__GNUC__) || defined(__clang__)
return __builtin_popcount(n);
#elif defined(_MSC_VER)
return __popcnt(n);
#else
// Taken from http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
n = n - ((n >> 1) & 0x55555555);
n = (n & 0x33333333) + ((n >> 2) & 0x33333333);
return ((n + ((n >> 4) & 0xF0F0F0F)) * 0x1010101) >> 24;
// ---
#endif
}
inline float L2sqdist(const Point3f& a) {
return a.dot(a);
}
inline float L1dist(const Point3f& a) {
return std::abs(a.x) + std::abs(a.y) + std::abs(a.z);
}
inline float det3x3(float a11, float a12, float a13, float a22, float a23, float a33) {
return a11 * (a22 * a33 - a23 * a23) + a12 * (2 * a13 * a23 - a33 * a12) - a13 * a13 * a22;
}
inline float localSVD(float a11, float a12, float a13, float a21, float a22, float a23, float a31, float a32, float a33) {
float b11 = a11 * a11 + a12 * a12 + a13 * a13;
float b12 = a11 * a21 + a12 * a22 + a13 * a23;
float b13 = a11 * a31 + a12 * a32 + a13 * a33;
float b22 = a21 * a21 + a22 * a22 + a23 * a23;
float b23 = a21 * a31 + a22 * a32 + a23 * a33;
float b33 = a31 * a31 + a32 * a32 + a33 * a33;
const float q = (b11 + b22 + b33) / 3;
b11 -= q;
b22 -= q;
b33 -= q;
float p = std::sqrt((b11 * b11 + b22 * b22 + b33 * b33 + 2 * (b12 * b12 + b13 * b13 + b23 * b23)) / 6);
if (p == 0)
return 0;
const float pi = 1 / p;
const float r = det3x3(pi * b11, pi * b12, pi * b13, pi * b22, pi * b23, pi * b33) / 2;
float phi;
if (r <= -1)
phi = float(CV_PI / 3);
else if (r >= 1)
phi = 0;
else
phi = std::acos(r) / 3;
p *= 2;
const float e1 = q + p * std::cos(phi);
float e2, e3;
if (e1 < 3 * q) {
e3 = std::max(q + p * std::cos(phi + float(2 * CV_PI / 3)), 0.0f);
e2 = std::max(3 * q - e1 - e3, 0.0f);
}
else {
e2 = 0;
e3 = 0;
}
return std::sqrt(e2 / e1) + std::sqrt(e3 / e1);
}
void removeNoise(Mat& fgMask, const Mat& compMask, const size_t threshold, const uchar filler) {
const Size sz = fgMask.size();
Mat labels;
const int nComponents = connectedComponents(compMask, labels, 8, CV_32S);
std::vector<size_t> compArea(nComponents, 0);
for (int i = 0; i < sz.height; ++i)
for (int j = 0; j < sz.width; ++j)
++compArea[labels.at<int>(i, j)];
for (int i = 0; i < sz.height; ++i)
for (int j = 0; j < sz.width; ++j)
if (compArea[labels.at<int>(i, j)] < threshold)
fgMask.at<uchar>(i, j) = filler;
}
void FindSparseCorrLK(const Mat& src, const Mat& dst, std::vector<Point2f>& srcPoints, std::vector<Point2f>& dstPoints) {
Size size = src.size();
const unsigned blockSize = 16;
for (int x = blockSize / 2; x < size.width; x += blockSize)
for (int y = blockSize / 2; y < size.height; y += blockSize) {
srcPoints.push_back(Point2f(float(x), float(y)));
dstPoints.push_back(Point2f(float(x), float(y)));
}
std::vector<uchar> predictedStatus;
std::vector<float> predictedError;
Mat srcGr, dstGr;
src.copyTo(srcGr);
dst.copyTo(dstGr);
srcGr *= 255;
dstGr *= 255;
srcGr.convertTo(srcGr, CV_8UC3);
dstGr.convertTo(dstGr, CV_8UC3);
calcOpticalFlowPyrLK(srcGr, dstGr, srcPoints, dstPoints, predictedStatus, predictedError);
size_t j = 0;
for (size_t i = 0; i < srcPoints.size(); ++i) {
if (predictedStatus[i]) {
srcPoints[j] = srcPoints[i];
dstPoints[j] = dstPoints[i];
++j;
}
}
srcPoints.resize(j);
dstPoints.resize(j);
}
class BackgroundSampleGSOC {
public:
Point3f color;
int desc;
uint64 time;
uint64 hits;
BackgroundSampleGSOC(Point3f c = Point3f(), int d = 0, uint64 t = 0, uint64 h = 0) : color(c), desc(d), time(t), hits(h) {}
};
class BackgroundSampleLSBP {
public:
Point3f color;
int desc;
float minDecisionDist;
BackgroundSampleLSBP(Point3f c = Point3f(), int d = 0, float mdd = 1e9f) : color(c), desc(d), minDecisionDist(mdd) {}
};
template<typename BackgroundSampleType>
class BackgroundModel {
protected:
std::vector<BackgroundSampleType> samples;
const Size size;
const int nSamples;
const int stride;
public:
BackgroundModel(Size sz, int S) : size(sz), nSamples(S), stride(sz.width * S) {
samples.resize(sz.area() * S);
}
void swap(BackgroundModel& bm) {
samples.swap(bm.samples);
}
void motionCompensation(const BackgroundModel& bm, const std::vector<Point2f>& points) {
for (int i = 0; i < size.height; ++i)
for (int j = 0; j < size.width; ++j) {
Point2i p = points[j * size.height + i];
if (p.x < 0)
p.x = 0;
if (p.y < 0)
p.y = 0;
if (p.x >= size.width)
p.x = size.width - 1;
if (p.y >= size.height)
p.y = size.height - 1;
memcpy(&samples[i * stride + j * nSamples], &bm.samples[p.y * stride + p.x * nSamples], sizeof(BackgroundSampleType) * nSamples);
}
}
const BackgroundSampleType& operator()(int k) const {
return samples[k];
}
BackgroundSampleType& operator()(int k) {
return samples[k];
}
const BackgroundSampleType& operator()(int i, int j, int k) const {
return samples[i * stride + j * nSamples + k];
}
BackgroundSampleType& operator()(int i, int j, int k) {
return samples[i * stride + j * nSamples + k];
}
Size getSize() const {
return size;
}
};
class BackgroundModelGSOC : public BackgroundModel<BackgroundSampleGSOC> {
public:
BackgroundModelGSOC(Size sz, int S) : BackgroundModel(sz, S) {};
float findClosest(int i, int j, const Point3f& color, int& indOut) const {
const int end = i * stride + (j + 1) * nSamples;
int minInd = i * stride + j * nSamples;
float minDist = L2sqdist(color - samples[minInd].color);
for (int k = minInd + 1; k < end; ++k) {
const float dist = L2sqdist(color - samples[k].color);
if (dist < minDist) {
minInd = k;
minDist = dist;
}
}
indOut = minInd;
return minDist;
}
void replaceOldest(int i, int j, const BackgroundSampleGSOC& sample) {
const int end = i * stride + (j + 1) * nSamples;
int minInd = i * stride + j * nSamples;
for (int k = minInd + 1; k < end; ++k) {
if (samples[k].time < samples[minInd].time)
minInd = k;
}
samples[minInd] = sample;
}
Point3f getMean(int i, int j, uint64 threshold) const {
const int end = i * stride + (j + 1) * nSamples;
Point3f acc(0, 0, 0);
int cnt = 0;
for (int k = i * stride + j * nSamples; k < end; ++k) {
if (samples[k].hits > threshold) {
acc += samples[k].color;
++cnt;
}
}
if (cnt == 0) {
cnt = nSamples;
for (int k = i * stride + j * nSamples; k < end; ++k)
acc += samples[k].color;
}
acc.x /= cnt;
acc.y /= cnt;
acc.z /= cnt;
return acc;
}
};
class BackgroundModelLSBP : public BackgroundModel<BackgroundSampleLSBP> {
public:
BackgroundModelLSBP(Size sz, int S) : BackgroundModel(sz, S) {};
int countMatches(int i, int j, const Point3f& color, int desc, float threshold, int descThreshold, float& minDist) const {
const int end = i * stride + (j + 1) * nSamples;
int count = 0;
minDist = 1e9;
for (int k = i * stride + j * nSamples; k < end; ++k) {
const float dist = L1dist(color - samples[k].color);
if (dist < threshold && LSBPDist32(static_cast<unsigned>(desc ^ samples[k].desc)) < descThreshold)
++count;
if (dist < minDist)
minDist = dist;
}
return count;
}
Point3f getMean(int i, int j) const {
const int end = i * stride + (j + 1) * nSamples;
Point3f acc(0, 0, 0);
for (int k = i * stride + j * nSamples; k < end; ++k) {
acc += samples[k].color;
}
acc.x /= nSamples;
acc.y /= nSamples;
acc.z /= nSamples;
return acc;
}
float getDMean(int i, int j) const {
const int end = i * stride + (j + 1) * nSamples;
float d = 0;
for (int k = i * stride + j * nSamples; k < end; ++k)
d += samples[k].minDecisionDist;
return d / nSamples;
}
};
class ParallelLocalSVDValues : public ParallelLoopBody {
private:
const Size sz;
Mat& localSVDValues;
const Mat& frameGray;
ParallelLocalSVDValues &operator=(const ParallelLocalSVDValues&);
public:
ParallelLocalSVDValues(const Size& _sz, Mat& _localSVDValues, const Mat& _frameGray) : sz(_sz), localSVDValues(_localSVDValues), frameGray(_frameGray) {};
void operator()(const Range &range) const {
for (int i = range.start; i < range.end; ++i)
for (int j = 1; j < sz.width - 1; ++j) {
localSVDValues.at<float>(i, j) = localSVD(
frameGray.at<float>(i - 1, j - 1), frameGray.at<float>(i - 1, j), frameGray.at<float>(i - 1, j + 1),
frameGray.at<float>(i, j - 1), frameGray.at<float>(i, j), frameGray.at<float>(i, j + 1),
frameGray.at<float>(i + 1, j - 1), frameGray.at<float>(i + 1, j), frameGray.at<float>(i + 1, j + 1));
}
}
};
class ParallelFromLocalSVDValues : public ParallelLoopBody {
private:
const Size sz;
Mat& desc;
const Mat& localSVDValues;
const Point2i* LSBPSamplePoints;
ParallelFromLocalSVDValues &operator=(const ParallelFromLocalSVDValues&);
public:
ParallelFromLocalSVDValues(const Size& _sz, Mat& _desc, const Mat& _localSVDValues, const Point2i* _LSBPSamplePoints) : sz(_sz), desc(_desc), localSVDValues(_localSVDValues), LSBPSamplePoints(_LSBPSamplePoints) {};
void operator()(const Range &range) const {
for (int index = range.start; index < range.end; ++index) {
const int i = index / sz.width, j = index % sz.width;
int& descVal = desc.at<int>(i, j);
descVal = 0;
const float centerVal = localSVDValues.at<float>(i, j);
for (int n = 0; n < 32; ++n) {
const int ri = i + LSBPSamplePoints[n].y;
const int rj = j + LSBPSamplePoints[n].x;
if (ri >= 0 && rj >= 0 && ri < sz.height && rj < sz.width && std::abs(localSVDValues.at<float>(ri, rj) - centerVal) > LSBPtau)
descVal |= int(1U) << n;
}
}
}
};
} // namespace
void BackgroundSubtractorLSBPDesc::calcLocalSVDValues(OutputArray _localSVDValues, const Mat& frame) {
Mat frameGray;
const Size sz = frame.size();
_localSVDValues.create(sz, CV_32F);
Mat localSVDValues = _localSVDValues.getMat();
localSVDValues = 0.0f;
cvtColor(frame, frameGray, COLOR_BGR2GRAY);
parallel_for_(Range(1, sz.height - 1), ParallelLocalSVDValues(sz, localSVDValues, frameGray));
for (int i = 1; i < sz.height - 1; ++i) {
localSVDValues.at<float>(i, 0) = localSVD(
frameGray.at<float>(i - 1, 0), frameGray.at<float>(i - 1, 0), frameGray.at<float>(i - 1, 1),
frameGray.at<float>(i, 0), frameGray.at<float>(i, 0), frameGray.at<float>(i, 1),
frameGray.at<float>(i + 1, 0), frameGray.at<float>(i + 1, 0), frameGray.at<float>(i + 1, 1));
localSVDValues.at<float>(i, sz.width - 1) = localSVD(
frameGray.at<float>(i - 1, sz.width - 2), frameGray.at<float>(i - 1, sz.width - 1), frameGray.at<float>(i - 1, sz.width - 1),
frameGray.at<float>(i, sz.width - 2), frameGray.at<float>(i, sz.width - 1), frameGray.at<float>(i, sz.width - 1),
frameGray.at<float>(i + 1, sz.width - 2), frameGray.at<float>(i + 1, sz.width - 1), frameGray.at<float>(i + 1, sz.width - 1));
}
for (int j = 1; j < sz.width - 1; ++j) {
localSVDValues.at<float>(0, j) = localSVD(
frameGray.at<float>(0, j - 1), frameGray.at<float>(0, j), frameGray.at<float>(0, j + 1),
frameGray.at<float>(0, j - 1), frameGray.at<float>(0, j), frameGray.at<float>(0, j + 1),
frameGray.at<float>(1, j - 1), frameGray.at<float>(1, j), frameGray.at<float>(1, j + 1));
localSVDValues.at<float>(sz.height - 1, j) = localSVD(
frameGray.at<float>(sz.height - 2, j - 1), frameGray.at<float>(sz.height - 2, j), frameGray.at<float>(sz.height - 2, j + 1),
frameGray.at<float>(sz.height - 1, j - 1), frameGray.at<float>(sz.height - 1, j), frameGray.at<float>(sz.height - 1, j + 1),
frameGray.at<float>(sz.height - 1, j - 1), frameGray.at<float>(sz.height - 1, j), frameGray.at<float>(sz.height - 1, j + 1));
}
}
void BackgroundSubtractorLSBPDesc::computeFromLocalSVDValues(OutputArray _desc, const Mat& localSVDValues, const Point2i* LSBPSamplePoints) {
const Size sz = localSVDValues.size();
_desc.create(sz, CV_32S);
Mat desc = _desc.getMat();
parallel_for_(Range(0, sz.area()), ParallelFromLocalSVDValues(sz, desc, localSVDValues, LSBPSamplePoints));
}
void BackgroundSubtractorLSBPDesc::compute(OutputArray desc, const Mat& frame, const Point2i* LSBPSamplePoints) {
Mat localSVDValues;
calcLocalSVDValues(localSVDValues, frame);
computeFromLocalSVDValues(desc, localSVDValues, LSBPSamplePoints);
}
class BackgroundSubtractorGSOCImpl : public BackgroundSubtractorGSOC {
private:
Ptr<BackgroundModelGSOC> backgroundModel;
Ptr<BackgroundModelGSOC> backgroundModelPrev;
uint64 currentTime;
const int motionCompensation;
const int nSamples;
const float replaceRate;
const float propagationRate;
const uint64 hitsThreshold;
const float alpha;
const float beta;
const float blinkingSupressionDecay;
const float blinkingSupressionMultiplier;
const float noiseRemovalThresholdFacBG;
const float noiseRemovalThresholdFacFG;
Mat distMovingAvg;
Mat prevFgMask;
Mat prevFrame;
Mat blinkingSupression;
RNG rng;
void postprocessing(Mat& fgMask);
public:
BackgroundSubtractorGSOCImpl(int mc,
int nSamples,
float replaceRate,
float propagationRate,
int hitsThreshold,
float alpha,
float beta,
float blinkingSupressionDecay,
float blinkingSupressionMultiplier,
float noiseRemovalThresholdFacBG,
float noiseRemovalThresholdFacFG);
CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate = -1);
CV_WRAP virtual void getBackgroundImage(OutputArray backgroundImage) const;
friend class ParallelGSOC;
};
class BackgroundSubtractorLSBPImpl : public BackgroundSubtractorLSBP {
private:
Ptr<BackgroundModelLSBP> backgroundModel;
Ptr<BackgroundModelLSBP> backgroundModelPrev;
const int motionCompensation;
const int nSamples;
const int LSBPRadius;
const float Tlower;
const float Tupper;
const float Tinc;
const float Tdec;
const float Rscale;
const float Rincdec;
const float noiseRemovalThresholdFacBG;
const float noiseRemovalThresholdFacFG;
const int LSBPthreshold;
const int minCount;
Mat T;
Mat R;
Mat prevFrame;
RNG rng;
Point2i LSBPSamplePoints[32];
void postprocessing(Mat& fgMask);
public:
BackgroundSubtractorLSBPImpl(int mc,
int nSamples,
int LSBPRadius,
float Tlower,
float Tupper,
float Tinc,
float Tdec,
float Rscale,
float Rincdec,
float noiseRemovalThresholdFacBG,
float noiseRemovalThresholdFacFG,
int LSBPthreshold,
int minCount
);
CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate = -1);
CV_WRAP virtual void getBackgroundImage(OutputArray backgroundImage) const;
friend class ParallelLSBP;
};
class ParallelGSOC : public ParallelLoopBody {
private:
const Size sz;
BackgroundSubtractorGSOCImpl* bgs;
const Mat& frame;
const double learningRate;
Mat& fgMask;
ParallelGSOC &operator=(const ParallelGSOC&);
public:
ParallelGSOC(const Size& _sz, BackgroundSubtractorGSOCImpl* _bgs, const Mat& _frame, double _learningRate, Mat& _fgMask)
: sz(_sz), bgs(_bgs), frame(_frame), learningRate(_learningRate), fgMask(_fgMask) {};
void operator()(const Range &range) const {
BackgroundModelGSOC* backgroundModel = bgs->backgroundModel.get();
Mat& distMovingAvg = bgs->distMovingAvg;
for (int index = range.start; index < range.end; ++index) {
const int i = index / sz.width, j = index % sz.width;
int k;
const float minDist = backgroundModel->findClosest(i, j, frame.at<Point3f>(i, j), k);
distMovingAvg.at<float>(i, j) *= 1 - float(learningRate);
distMovingAvg.at<float>(i, j) += float(learningRate) * minDist;
const float threshold = bgs->alpha * distMovingAvg.at<float>(i, j) + bgs->beta;
BackgroundSampleGSOC& sample = (* backgroundModel)(k);
if (minDist > threshold) {
fgMask.at<uchar>(i, j) = 255;
if (bgs->rng.uniform(0.0f, 1.0f) < bgs->replaceRate)
backgroundModel->replaceOldest(i, j, BackgroundSampleGSOC(frame.at<Point3f>(i, j), 0, bgs->currentTime));
}
else {
sample.color *= 1 - learningRate;
sample.color += learningRate * frame.at<Point3f>(i, j);
sample.time = bgs->currentTime;
++sample.hits;
// Propagation to neighbors
if (sample.hits > bgs->hitsThreshold && bgs->rng.uniform(0.0f, 1.0f) < bgs->propagationRate) {
if (i + 1 < sz.height)
backgroundModel->replaceOldest(i + 1, j, sample);
if (j + 1 < sz.width)
backgroundModel->replaceOldest(i, j + 1, sample);
if (i > 0)
backgroundModel->replaceOldest(i - 1, j, sample);
if (j > 0)
backgroundModel->replaceOldest(i, j - 1, sample);
}
fgMask.at<uchar>(i, j) = 0;
}
}
}
};
class ParallelLSBP : public ParallelLoopBody {
private:
const Size sz;
BackgroundSubtractorLSBPImpl* bgs;
const Mat& frame;
const double learningRate;
const Mat& LSBPDesc;
Mat& fgMask;
ParallelLSBP &operator=(const ParallelLSBP&);
public:
ParallelLSBP(const Size& _sz, BackgroundSubtractorLSBPImpl* _bgs, const Mat& _frame, double _learningRate, const Mat& _LSBPDesc, Mat& _fgMask)
: sz(_sz), bgs(_bgs), frame(_frame), learningRate(_learningRate), LSBPDesc(_LSBPDesc), fgMask(_fgMask) {};
void operator()(const Range &range) const {
BackgroundModelLSBP* backgroundModel = bgs->backgroundModel.get();
Mat& T = bgs->T;
Mat& R = bgs->R;
for (int index = range.start; index < range.end; ++index) {
const int i = index / sz.width, j = index % sz.width;
float minDist = 1e9f;
const float DMean = backgroundModel->getDMean(i, j);
if (R.at<float>(i, j) > DMean * bgs->Rscale)
R.at<float>(i, j) *= 1 - bgs->Rincdec;
else
R.at<float>(i, j) *= 1 + bgs->Rincdec;
if (backgroundModel->countMatches(i, j, frame.at<Point3f>(i, j), LSBPDesc.at<int>(i, j), R.at<float>(i, j), bgs->LSBPthreshold, minDist) < bgs->minCount) {
fgMask.at<uchar>(i, j) = 255;
T.at<float>(i, j) += bgs->Tinc / DMean;
}
else {
fgMask.at<uchar>(i, j) = 0;
T.at<float>(i, j) -= bgs->Tdec / DMean;
if (bgs->rng.uniform(0.0f, 1.0f) < 1 / T.at<float>(i, j))
(* backgroundModel)(i, j, bgs->rng.uniform(0, bgs->nSamples)) = BackgroundSampleLSBP(frame.at<Point3f>(i, j), LSBPDesc.at<int>(i, j), minDist);
if (bgs->rng.uniform(0.0f, 1.0f) < 1 / T.at<float>(i, j)) {
const int oi = i + bgs->rng.uniform(-1, 2);
const int oj = j + bgs->rng.uniform(-1, 2);
if (oi >= 0 && oi < sz.height && oj >= 0 && oj < sz.width)
(* backgroundModel)(oi, oj, bgs->rng.uniform(0, bgs->nSamples)) = BackgroundSampleLSBP(frame.at<Point3f>(oi, oj), LSBPDesc.at<int>(oi, oj), minDist);
}
}
T.at<float>(i, j) = std::min(T.at<float>(i, j), bgs->Tupper);
T.at<float>(i, j) = std::max(T.at<float>(i, j), bgs->Tlower);
}
}
};
BackgroundSubtractorGSOCImpl::BackgroundSubtractorGSOCImpl(int _mc,
int _nSamples,
float _replaceRate,
float _propagationRate,
int _hitsThreshold,
float _alpha,
float _beta,
float _blinkingSupressionDecay,
float _blinkingSupressionMultiplier,
float _noiseRemovalThresholdFacBG,
float _noiseRemovalThresholdFacFG)
: currentTime(0),
motionCompensation(_mc),
nSamples(_nSamples),
replaceRate(_replaceRate),
propagationRate(_propagationRate),
hitsThreshold(_hitsThreshold),
alpha(_alpha),
beta(_beta),
blinkingSupressionDecay(_blinkingSupressionDecay),
blinkingSupressionMultiplier(_blinkingSupressionMultiplier),
noiseRemovalThresholdFacBG(_noiseRemovalThresholdFacBG),
noiseRemovalThresholdFacFG(_noiseRemovalThresholdFacFG) {
CV_Assert(nSamples > 1 && nSamples < 1024);
CV_Assert(replaceRate >= 0 && replaceRate <= 1);
CV_Assert(propagationRate >= 0 && propagationRate <= 1);
CV_Assert(blinkingSupressionDecay > 0 && blinkingSupressionDecay < 1);
CV_Assert(noiseRemovalThresholdFacBG >= 0 && noiseRemovalThresholdFacBG < 0.5);
CV_Assert(noiseRemovalThresholdFacFG >= 0 && noiseRemovalThresholdFacFG < 0.5);
CV_Assert(_hitsThreshold >= 0);
}
void BackgroundSubtractorGSOCImpl::postprocessing(Mat& fgMask) {
removeNoise(fgMask, fgMask, size_t(noiseRemovalThresholdFacBG * fgMask.size().area()), 0);
Mat invFgMask = 255 - fgMask;
removeNoise(fgMask, invFgMask, size_t(noiseRemovalThresholdFacFG * fgMask.size().area()), 255);
GaussianBlur(fgMask, fgMask, Size(5, 5), 0);
fgMask = fgMask > 127;
}
void BackgroundSubtractorGSOCImpl::apply(InputArray _image, OutputArray _fgmask, double learningRate) {
const Size sz = _image.size();
_fgmask.create(sz, CV_8U);
Mat fgMask = _fgmask.getMat();
Mat frame = _image.getMat();
CV_Assert(frame.depth() == CV_8U || frame.depth() == CV_32F);
CV_Assert(frame.channels() == 1 || frame.channels() == 3);
if (frame.channels() != 3)
cvtColor(frame, frame, COLOR_GRAY2BGR);
if (frame.depth() != CV_32F) {
frame.convertTo(frame, CV_32F);
frame /= 255;
}
CV_Assert(frame.channels() == 3);
if (backgroundModel.empty()) {
backgroundModel = makePtr<BackgroundModelGSOC>(sz, nSamples);
backgroundModelPrev = makePtr<BackgroundModelGSOC>(sz, nSamples);
distMovingAvg = Mat(sz, CV_32F, Scalar::all(0.005f));
prevFgMask = Mat(sz, CV_8U, Scalar::all(0));
blinkingSupression = Mat(sz, CV_32F, Scalar::all(0.0f));
for (int i = 0; i < sz.height; ++i)
for (int j = 0; j < sz.width; ++j) {
BackgroundSampleGSOC sample(frame.at<Point3f>(i, j), 0);
for (int k = 0; k < nSamples; ++k) {
(* backgroundModel)(i, j, k) = sample;
(* backgroundModelPrev)(i, j, k) = sample;
}
}
}
CV_Assert(backgroundModel->getSize() == sz);
if (motionCompensation) {
std::vector<Point2f> srcPoints;
std::vector<Point2f> dstPoints;
if (prevFrame.empty())
frame.copyTo(prevFrame);
if (motionCompensation == LSBP_CAMERA_MOTION_COMPENSATION_LK)
FindSparseCorrLK(frame, prevFrame, srcPoints, dstPoints);
if (srcPoints.size()) {
Mat H = findHomography(srcPoints, dstPoints, LMEDS);
srcPoints.clear();
for (int x = 0; x < sz.width; ++x)
for (int y = 0; y < sz.height; ++y)
srcPoints.push_back(Point2f(float(x), float(y)));
dstPoints.resize(srcPoints.size());
perspectiveTransform(srcPoints, dstPoints, H);
backgroundModel->swap(* backgroundModelPrev);
backgroundModel->motionCompensation(* backgroundModelPrev, dstPoints);
}
frame.copyTo(prevFrame);
}
if (learningRate > 1 || learningRate < 0)
learningRate = 0.1;
parallel_for_(Range(0, sz.area()), ParallelGSOC(sz, this, frame, learningRate, fgMask));
++currentTime;
cv::add(blinkingSupression, (fgMask != prevFgMask) / 255, blinkingSupression, cv::noArray(), CV_32F);
blinkingSupression *= blinkingSupressionDecay;
fgMask.copyTo(prevFgMask);
Mat prob = blinkingSupression * (blinkingSupressionMultiplier * (1 - blinkingSupressionDecay) / blinkingSupressionDecay);
for (int i = 0; i < sz.height; ++i)
for (int j = 0; j < sz.width; ++j)
if (rng.uniform(0.0f, 1.0f) < prob.at<float>(i, j))
backgroundModel->replaceOldest(i, j, BackgroundSampleGSOC(frame.at<Point3f>(i, j), 0, currentTime));
this->postprocessing(fgMask);
}
void BackgroundSubtractorGSOCImpl::getBackgroundImage(OutputArray _backgroundImage) const {
CV_Assert(!backgroundModel.empty());
const Size sz = backgroundModel->getSize();
_backgroundImage.create(sz, CV_8UC3);
Mat backgroundImage = _backgroundImage.getMat();
for (int i = 0; i < sz.height; ++i)
for (int j = 0; j < sz.width; ++j)
backgroundImage.at< Point3_<uchar> >(i, j) = backgroundModel->getMean(i, j, hitsThreshold) * 255;
}
BackgroundSubtractorLSBPImpl::BackgroundSubtractorLSBPImpl(int _mc,
int _nSamples,
int _LSBPRadius,
float _Tlower,
float _Tupper,
float _Tinc,
float _Tdec,
float _Rscale,
float _Rincdec,
float _noiseRemovalThresholdFacBG,
float _noiseRemovalThresholdFacFG,
int _LSBPthreshold,
int _minCount
)
: motionCompensation(_mc),
nSamples(_nSamples),
LSBPRadius(_LSBPRadius),
Tlower(_Tlower),
Tupper(_Tupper),
Tinc(_Tinc),
Tdec(_Tdec),
Rscale(_Rscale),
Rincdec(_Rincdec),
noiseRemovalThresholdFacBG(_noiseRemovalThresholdFacBG),
noiseRemovalThresholdFacFG(_noiseRemovalThresholdFacFG),
LSBPthreshold(_LSBPthreshold),
minCount(_minCount) {
CV_Assert(nSamples > 1 && nSamples < 1024);
CV_Assert(LSBPRadius > 0);
CV_Assert(Tlower < Tupper && Tlower > 0);
CV_Assert(noiseRemovalThresholdFacBG >= 0 && noiseRemovalThresholdFacBG < 0.5);
CV_Assert(noiseRemovalThresholdFacFG >= 0 && noiseRemovalThresholdFacFG < 0.5);
for (int i = 0; i < 32; ++i) {
const double phi = i * CV_2PI / 32.0;
LSBPSamplePoints[i] = Point2i(int(LSBPRadius * std::cos(phi)), int(LSBPRadius * std::sin(phi)));
}
}
void BackgroundSubtractorLSBPImpl::postprocessing(Mat& fgMask) {
removeNoise(fgMask, fgMask, size_t(noiseRemovalThresholdFacBG * fgMask.size().area()), 0);
Mat invFgMask = 255 - fgMask;
removeNoise(fgMask, invFgMask, size_t(noiseRemovalThresholdFacFG * fgMask.size().area()), 255);
GaussianBlur(fgMask, fgMask, Size(5, 5), 0);
fgMask = fgMask > 127;
}
void BackgroundSubtractorLSBPImpl::apply(InputArray _image, OutputArray _fgmask, double learningRate) {
const Size sz = _image.size();
_fgmask.create(sz, CV_8U);
Mat fgMask = _fgmask.getMat();
Mat frame = _image.getMat();
CV_Assert(frame.depth() == CV_8U || frame.depth() == CV_32F);
CV_Assert(frame.channels() == 1 || frame.channels() == 3);
if (frame.channels() != 3)
cvtColor(frame, frame, COLOR_GRAY2BGR);
if (frame.depth() != CV_32F) {
frame.convertTo(frame, CV_32F, 1.0/255);
}
CV_Assert(frame.channels() == 3);
Mat LSBPDesc(sz, CV_32S, Scalar::all(0));
BackgroundSubtractorLSBPDesc::compute(LSBPDesc, frame, LSBPSamplePoints);
if (backgroundModel.empty()) {
backgroundModel = makePtr<BackgroundModelLSBP>(sz, nSamples);
backgroundModelPrev = makePtr<BackgroundModelLSBP>(sz, nSamples);
T = Mat(sz, CV_32F);
T = (Tlower + Tupper) * 0.5f;
R = Mat(sz, CV_32F);
R = 0.1f;
for (int i = 0; i < sz.height; ++i)
for (int j = 0; j < sz.width; ++j) {
BackgroundSampleLSBP sample(frame.at<Point3f>(i, j), LSBPDesc.at<int>(i, j));
for (int k = 0; k < nSamples; ++k) {
(* backgroundModel)(i, j, k) = sample;
(* backgroundModelPrev)(i, j, k) = sample;
}
}
}
CV_Assert(backgroundModel->getSize() == sz);
if (motionCompensation) {
std::vector<Point2f> srcPoints;
std::vector<Point2f> dstPoints;
if (prevFrame.empty())
frame.copyTo(prevFrame);
if (motionCompensation == LSBP_CAMERA_MOTION_COMPENSATION_LK)
FindSparseCorrLK(frame, prevFrame, srcPoints, dstPoints);
if (srcPoints.size()) {
Mat H = findHomography(srcPoints, dstPoints, LMEDS);
srcPoints.clear();
for (int x = 0; x < sz.width; ++x)
for (int y = 0; y < sz.height; ++y)
srcPoints.push_back(Point2f(float(x), float(y)));
dstPoints.resize(srcPoints.size());
perspectiveTransform(srcPoints, dstPoints, H);
backgroundModel->swap(* backgroundModelPrev);
backgroundModel->motionCompensation(* backgroundModelPrev, dstPoints);
}
frame.copyTo(prevFrame);
}
if (learningRate > 1 || learningRate < 0)
learningRate = 0.1;
parallel_for_(Range(0, sz.area()), ParallelLSBP(sz, this, frame, learningRate, LSBPDesc, fgMask));
this->postprocessing(fgMask);
}
void BackgroundSubtractorLSBPImpl::getBackgroundImage(OutputArray _backgroundImage) const {
CV_Assert(!backgroundModel.empty());
const Size sz = backgroundModel->getSize();
_backgroundImage.create(sz, CV_8UC3);
Mat backgroundImage = _backgroundImage.getMat();
for (int i = 0; i < sz.height; ++i)
for (int j = 0; j < sz.width; ++j)
backgroundImage.at< Point3_<uchar> >(i, j) = backgroundModel->getMean(i, j) * 255;
}
Ptr<BackgroundSubtractorGSOC> createBackgroundSubtractorGSOC(int mc,
int nSamples,
float replaceRate,
float propagationRate,
int hitsThreshold,
float alpha,
float beta,
float blinkingSupressionDecay,
float blinkingSupressionMultiplier,
float noiseRemovalThresholdFacBG,
float noiseRemovalThresholdFacFG) {
return makePtr<BackgroundSubtractorGSOCImpl>(mc,
nSamples,
replaceRate,
propagationRate,
hitsThreshold,
alpha,
beta,
blinkingSupressionDecay,
blinkingSupressionMultiplier,
noiseRemovalThresholdFacBG,
noiseRemovalThresholdFacFG);
}
Ptr<BackgroundSubtractorLSBP> createBackgroundSubtractorLSBP(int mc,
int nSamples,
int LSBPRadius,
float Tlower,
float Tupper,
float Tinc,
float Tdec,
float Rscale,
float Rincdec,
float noiseRemovalThresholdFacBG,
float noiseRemovalThresholdFacFG,
int LSBPthreshold,
int minCount
) {
return Ptr<BackgroundSubtractorLSBPImpl>(
new BackgroundSubtractorLSBPImpl(
mc,
nSamples,
LSBPRadius,
Tlower,
Tupper,
Tinc,
Tdec,
Rscale,
Rincdec,
noiseRemovalThresholdFacBG,
noiseRemovalThresholdFacFG,
LSBPthreshold,
minCount
));
}
} // namespace bgsegm
} // namespace cv

@ -0,0 +1,221 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000, Intel Corporation, all rights reserved.
// Copyright (C) 2013, OpenCV Foundation, all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
/**
* @file synthetic_seq.cpp
* @author Vladislav Samsonov <vvladxx@gmail.com>
* @brief Synthetic frame sequence generator for testing background subtraction algorithms.
*
*/
#include "precomp.hpp"
namespace cv
{
namespace bgsegm
{
namespace
{
inline int clamp(int x, int l, int u) {
return ((x) < (l)) ? (l) : (((x) > (u)) ? (u) : (x));
}
inline int within(int a, int b, int c) {
return (((a) <= (b)) && ((b) <= (c))) ? 1 : 0;
}
void bilinearInterp(uchar* dest, double x, double y, unsigned bpp, const uchar** values) {
x = std::fmod(x, 1.0);
y = std::fmod(y, 1.0);
if (x < 0.0)
x += 1.0;
if (y < 0.0)
y += 1.0;
for (unsigned i = 0; i < bpp; i++) {
double m0 = (1.0 - x) * values[0][i] + x * values[1][i];
double m1 = (1.0 - x) * values[2][i] + x * values[3][i];
dest[i] = (uchar) ((1.0 - y) * m0 + y * m1);
}
}
// Static background is a way too easy test. We will add distortion to it.
void waveDistortion(const uchar* src, uchar* dst, int width, int height, int bypp, double amplitude, double wavelength, double phase) {
const uchar zeroes[4] = {0, 0, 0, 0};
const long rowsiz = width * bypp;
const double xhsiz = (double) width / 2.0;
const double yhsiz = (double) height / 2.0;
double xscale, yscale;
if (xhsiz < yhsiz) {
xscale = yhsiz / xhsiz;
yscale = 1.0;
}
else if (xhsiz > yhsiz) {
xscale = 1.0;
yscale = xhsiz / yhsiz;
}
else {
xscale = 1.0;
yscale = 1.0;
}
wavelength *= 2;
for (int y = 0; y < height; y++) {
uchar* dest = dst;
for (int x = 0; x < width; x++) {
const double dx = x * xscale;
const double dy = y * yscale;
const double d = sqrt (dx * dx + dy * dy);
const double amnt = amplitude * sin(((d / wavelength) * (2.0 * M_PI) + phase));
const double needx = (amnt + dx) / xscale;
const double needy = (amnt + dy) / yscale;
const int xi = clamp(int(needx), 0, width - 2);
const int yi = clamp(int(needy), 0, height - 2);
const uchar* p = src + rowsiz * yi + xi * bypp;
const int x1_in = within(0, xi, width - 1);
const int y1_in = within(0, yi, height - 1);
const int x2_in = within(0, xi + 1, width - 1);
const int y2_in = within(0, yi + 1, height - 1);
const uchar* values[4];
if (x1_in && y1_in)
values[0] = p;
else
values[0] = zeroes;
if (x2_in && y1_in)
values[1] = p + bypp;
else
values[1] = zeroes;
if (x1_in && y2_in)
values[2] = p + rowsiz;
else
values[2] = zeroes;
if (x2_in && y2_in)
values[3] = p + bypp + rowsiz;
else
values[3] = zeroes;
bilinearInterp(dest, needx, needy, bypp, values);
dest += bypp;
}
dst += rowsiz;
}
}
}
SyntheticSequenceGenerator::SyntheticSequenceGenerator(InputArray _background, InputArray _object, double _amplitude, double _wavelength, double _wavespeed, double _objspeed)
: amplitude(_amplitude), wavelength(_wavelength), wavespeed(_wavespeed), objspeed(_objspeed), timeStep(0) {
_background.getMat().copyTo(background);
_object.getMat().copyTo(object);
if (background.channels() == 1) {
cvtColor(background, background, COLOR_GRAY2BGR);
}
if (object.channels() == 1) {
cvtColor(object, object, COLOR_GRAY2BGR);
}
CV_Assert(background.channels() == 3);
CV_Assert(object.channels() == 3);
CV_Assert(background.size().width > object.size().width);
CV_Assert(background.size().height > object.size().height);
background.convertTo(background, CV_8U);
object.convertTo(object, CV_8U);
pos.x = (background.size().width - object.size().width) / 2;
pos.y = (background.size().height - object.size().height) / 2;
const double phi = rng.uniform(0.0, CV_2PI);
dir.x = std::cos(phi);
dir.y = std::sin(phi);
}
void SyntheticSequenceGenerator::getNextFrame(OutputArray _frame, OutputArray _gtMask) {
CV_Assert(!background.empty() && !object.empty());
const Size sz = background.size();
_frame.create(sz, CV_8UC3);
Mat frame = _frame.getMat();
CV_Assert(background.isContinuous() && frame.isContinuous());
waveDistortion(background.ptr(), frame.ptr(), sz.width, sz.height, 3, amplitude, wavelength, double(timeStep) * wavespeed);
const Size objSz = object.size();
object.copyTo(frame(Rect(Point2i(pos), objSz)));
while (pos.x + dir.x * objspeed < 0 || pos.x + dir.x * objspeed >= sz.width - objSz.width || pos.y + dir.y * objspeed < 0 || pos.y + dir.y * objspeed >= sz.height - objSz.height) {
const double phi = rng.uniform(0.0, CV_2PI);
dir.x = std::cos(phi);
dir.y = std::sin(phi);
}
_gtMask.create(sz, CV_8U);
Mat gtMask = _gtMask.getMat();
gtMask = 0;
gtMask(Rect(Point2i(pos), objSz)) = 255;
pos += dir * objspeed;
++timeStep;
}
Ptr<SyntheticSequenceGenerator> createSyntheticSequenceGenerator(InputArray background, InputArray object, double amplitude, double wavelength, double wavespeed, double objspeed) {
return makePtr<SyntheticSequenceGenerator>(background, object, amplitude, wavelength, wavespeed, objspeed);
}
}
}

@ -0,0 +1,137 @@
#include "test_precomp.hpp"
#include <set>
using namespace std;
using namespace cv;
using namespace cvtest;
static string getDataDir() { return TS::ptr()->get_data_path(); }
static string getLenaImagePath() { return getDataDir() + "shared/lena.png"; }
// Simple synthetic illumination invariance test
TEST(BackgroundSubtractor_LSBP, IlluminationInvariance)
{
RNG rng;
Mat input(100, 100, CV_32FC3);
rng.fill(input, RNG::UNIFORM, 0.0f, 0.1f);
Mat lsv1, lsv2;
cv::bgsegm::BackgroundSubtractorLSBPDesc::calcLocalSVDValues(lsv1, input);
input *= 10;
cv::bgsegm::BackgroundSubtractorLSBPDesc::calcLocalSVDValues(lsv2, input);
ASSERT_LE(cv::norm(lsv1, lsv2), 0.04f);
}
TEST(BackgroundSubtractor_LSBP, Correctness)
{
Mat input(3, 3, CV_32FC3);
float n = 0;
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j) {
input.at<Point3f>(i, j) = Point3f(n, n, n);
++n;
}
Mat lsv;
bgsegm::BackgroundSubtractorLSBPDesc::calcLocalSVDValues(lsv, input);
EXPECT_LE(std::abs(lsv.at<float>(1, 1) - 0.0903614f), 0.001f);
input = 1;
bgsegm::BackgroundSubtractorLSBPDesc::calcLocalSVDValues(lsv, input);
EXPECT_LE(std::abs(lsv.at<float>(1, 1) - 0.0f), 0.001f);
}
TEST(BackgroundSubtractor_LSBP, Discrimination)
{
Point2i LSBPSamplePoints[32];
for (int i = 0; i < 32; ++i) {
const double phi = i * CV_2PI / 32.0;
LSBPSamplePoints[i] = Point2i(int(4 * std::cos(phi)), int(4 * std::sin(phi)));
}
Mat lena = imread(getLenaImagePath());
Mat lsv;
lena.convertTo(lena, CV_32FC3);
bgsegm::BackgroundSubtractorLSBPDesc::calcLocalSVDValues(lsv, lena);
Scalar mean, var;
meanStdDev(lsv, mean, var);
EXPECT_GE(mean[0], 0.02);
EXPECT_LE(mean[0], 0.04);
EXPECT_GE(var[0], 0.03);
Mat desc;
bgsegm::BackgroundSubtractorLSBPDesc::computeFromLocalSVDValues(desc, lsv, LSBPSamplePoints);
Size sz = desc.size();
std::set<int> distinctive_elements;
for (int i = 0; i < sz.height; ++i)
for (int j = 0; j < sz.width; ++j)
distinctive_elements.insert(desc.at<int>(i, j));
EXPECT_GE(distinctive_elements.size(), 35000U);
}
static double scoreBitwiseReduce(const Mat& mask, const Mat& gtMask, uchar v1, uchar v2) {
Mat result;
cv::bitwise_and(mask == v1, gtMask == v2, result);
return cv::countNonZero(result);
}
template<typename T>
static double evaluateBGSAlgorithm(Ptr<T> bgs) {
Mat background = imread(getDataDir() + "shared/fruits.png");
Mat object = imread(getDataDir() + "shared/baboon.png");
cv::resize(object, object, Size(100, 100));
Ptr<bgsegm::SyntheticSequenceGenerator> generator = bgsegm::createSyntheticSequenceGenerator(background, object);
double f1_mean = 0;
unsigned total = 0;
for (int frameNum = 1; frameNum <= 400; ++frameNum) {
Mat frame, gtMask;
generator->getNextFrame(frame, gtMask);
Mat mask;
bgs->apply(frame, mask);
Size sz = frame.size();
EXPECT_EQ(sz, gtMask.size());
EXPECT_EQ(gtMask.size(), mask.size());
EXPECT_EQ(mask.type(), gtMask.type());
EXPECT_EQ(mask.type(), CV_8U);
// We will give the algorithm some time for the proper background model inference.
// Almost all background subtraction algorithms have a problem with cold start and require some time for background model initialization.
// So we will not count first part of the frames in the score.
if (frameNum > 300) {
const double tp = scoreBitwiseReduce(mask, gtMask, 255, 255);
const double fp = scoreBitwiseReduce(mask, gtMask, 255, 0);
const double fn = scoreBitwiseReduce(mask, gtMask, 0, 255);
if (tp + fn + fp > 0) {
const double f1_score = 2.0 * tp / (2.0 * tp + fn + fp);
f1_mean += f1_score;
++total;
}
}
}
f1_mean /= total;
return f1_mean;
}
TEST(BackgroundSubtractor_LSBP, Accuracy)
{
EXPECT_GE(evaluateBGSAlgorithm(bgsegm::createBackgroundSubtractorGSOC()), 0.9);
EXPECT_GE(evaluateBGSAlgorithm(bgsegm::createBackgroundSubtractorLSBP()), 0.25);
}
Loading…
Cancel
Save