From a39e3623aef650d52aada6b7c3f8a216d5aba1d9 Mon Sep 17 00:00:00 2001 From: samontab Date: Mon, 6 Jul 2015 16:00:33 +1000 Subject: [PATCH] Add Fine Grained Saliency algorithm --- modules/saliency/doc/saliency.bib | 9 + .../saliency/saliencySpecializedClasses.hpp | 29 ++ modules/saliency/samples/computeSaliency.cpp | 11 + modules/saliency/src/saliency.cpp | 2 + .../src/staticSaliencyFineGrained.cpp | 310 ++++++++++++++++++ 5 files changed, 361 insertions(+) create mode 100644 modules/saliency/src/staticSaliencyFineGrained.cpp diff --git a/modules/saliency/doc/saliency.bib b/modules/saliency/doc/saliency.bib index 6fd82ea1b..0aa094b93 100644 --- a/modules/saliency/doc/saliency.bib +++ b/modules/saliency/doc/saliency.bib @@ -22,3 +22,12 @@ year={2007}, organization={IEEE} } + +@inproceedings{FGS, + title={Human Detection Using a Mobile Platform and Novel Features Derived from a Visual Saliency Mechanism}, + author={Montabone, Sebastian and Soto, Alvaro}, + booktitle={Image and Vision Computing, Vol. 28 Issue 3}, + pages={391--402}, + year={2010}, + organization={Elsevier} +} diff --git a/modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp b/modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp index 443832f44..0bef69b57 100644 --- a/modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp +++ b/modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp @@ -113,6 +113,35 @@ protected: }; + +/** @brief the Fine Grained Saliency approach from @cite FGS + +This method calculates saliency based on center-surround differences. +High resolution saliency maps are generated in real time by using integral images. + */ +class CV_EXPORTS StaticSaliencyFineGrained : public StaticSaliency +{ +public: + + StaticSaliencyFineGrained(); + virtual ~StaticSaliencyFineGrained(); + +protected: + bool computeSaliencyImpl( InputArray image, OutputArray saliencyMap ); + +private: + void calcIntensityChannel(Mat src, Mat dst); + void copyImage(Mat src, Mat dst); + void getIntensityScaled(Mat integralImage, Mat gray, Mat saliencyOn, Mat saliencyOff, int neighborhood); + float getMean(Mat srcArg, Point2i PixArg, int neighbourhood, int centerVal); + void mixScales(Mat *saliencyOn, Mat intensityOn, Mat *saliencyOff, Mat intensityOff, const int numScales); + void mixOnOff(Mat intensityOn, Mat intensityOff, Mat intensity); + void getIntensity(Mat srcArg, Mat dstArg, Mat dstOnArg, Mat dstOffArg, bool generateOnOff); +}; + + + + /************************************ Specific Motion Saliency Specialized Classes ************************************/ /*! diff --git a/modules/saliency/samples/computeSaliency.cpp b/modules/saliency/samples/computeSaliency.cpp index 38b0d9ad1..58bc9c57d 100644 --- a/modules/saliency/samples/computeSaliency.cpp +++ b/modules/saliency/samples/computeSaliency.cpp @@ -127,6 +127,17 @@ int main( int argc, char** argv ) waitKey( 0 ); } + } + else if( saliency_algorithm.find( "FINE_GRAINED" ) == 0 ) + { + Mat saliencyMap; + if( saliencyAlgorithm->computeSaliency( image, saliencyMap ) ) + { + imshow( "Saliency Map", saliencyMap ); + imshow( "Original Image", image ); + waitKey( 0 ); + } + } else if( saliency_algorithm.find( "BING" ) == 0 ) { diff --git a/modules/saliency/src/saliency.cpp b/modules/saliency/src/saliency.cpp index 1c74f2c4e..ca89b067d 100644 --- a/modules/saliency/src/saliency.cpp +++ b/modules/saliency/src/saliency.cpp @@ -55,6 +55,8 @@ Ptr Saliency::create( const String& saliencyType ) { if (saliencyType == "SPECTRAL_RESIDUAL") return makePtr(); + else if (saliencyType == "FINE_GRAINED") + return makePtr(); else if (saliencyType == "BING") return makePtr(); else if (saliencyType == "BinWangApr2014") diff --git a/modules/saliency/src/staticSaliencyFineGrained.cpp b/modules/saliency/src/staticSaliencyFineGrained.cpp new file mode 100644 index 000000000..ec766b02e --- /dev/null +++ b/modules/saliency/src/staticSaliencyFineGrained.cpp @@ -0,0 +1,310 @@ + /*M/////////////////////////////////////////////////////////////////////////////////////// + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2014, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#include "precomp.hpp" + +namespace cv +{ +namespace saliency +{ + +/** + * Fine Grained Saliency + */ + + +StaticSaliencyFineGrained::StaticSaliencyFineGrained() +{ + className = "FINE_GRAINED"; +} + +StaticSaliencyFineGrained::~StaticSaliencyFineGrained() +{ + +} + + +bool StaticSaliencyFineGrained::computeSaliencyImpl( const InputArray image, OutputArray saliencyMap ) +{ + Mat dst(Size(image.getMat().cols, image.getMat().rows), CV_8UC1); + calcIntensityChannel(image.getMat(), dst); + dst.copyTo(saliencyMap); + + #ifdef SALIENCY_DEBUG + // visualize saliency map + imshow( "Saliency Map Interna", saliencyMap ); + #endif + + return true; +} + +void StaticSaliencyFineGrained::copyImage(Mat srcArg, Mat dstArg) +{ + srcArg.copyTo(dstArg); +} + +void StaticSaliencyFineGrained::calcIntensityChannel(Mat srcArg, Mat dstArg) +{ + if(dstArg.channels() > 1) + { + //("Error: Destiny image must have only one channel.\n"); + return; + } + const int numScales = 6; + Mat intensityScaledOn[numScales]; + Mat intensityScaledOff[numScales]; + Mat gray = Mat::zeros(Size(srcArg.cols, srcArg.rows), CV_8UC1); + Mat integralImage(Size(srcArg.cols + 1, srcArg.rows + 1), CV_32FC1); + Mat intensity(Size(srcArg.cols, srcArg.rows), CV_8UC1); + Mat intensityOn(Size(srcArg.cols, srcArg.rows), CV_8UC1); + Mat intensityOff(Size(srcArg.cols, srcArg.rows), CV_8UC1); + + int i; + int neighborhood; + int neighborhoods[] = {3*4, 3*4*2, 3*4*2*2, 7*4, 7*4*2, 7*4*2*2}; + + for(i=0; i(y, x)); + + meanOn = gray.at(y, x) - value; + meanOff = value - gray.at(y, x); + + if(meanOn > 0) + intensityScaledOn.at(y, x) = (uchar)meanOn; + else + intensityScaledOn.at(y, x) = 0; + + if(meanOff > 0) + intensityScaledOff.at(y, x) = (uchar)meanOff; + else + intensityScaledOff.at(y, x) = 0; + } + } +} + +float StaticSaliencyFineGrained::getMean(Mat srcArg, Point2i PixArg, int neighbourhood, int centerVal) +{ + Point2i P1, P2; + float value; + + P1.x = PixArg.x - neighbourhood + 1; + P1.y = PixArg.y - neighbourhood + 1; + P2.x = PixArg.x + neighbourhood + 1; + P2.y = PixArg.y + neighbourhood + 1; + + if(P1.x < 0) + P1.x = 0; + else if(P1.x > srcArg.cols - 1) + P1.x = srcArg.cols - 1; + if(P2.x < 0) + P2.x = 0; + else if(P2.x > srcArg.cols - 1) + P2.x = srcArg.cols - 1; + if(P1.y < 0) + P1.y = 0; + else if(P1.y > srcArg.rows - 1) + P1.y = srcArg.rows - 1; + if(P2.y < 0) + P2.y = 0; + else if(P2.y > srcArg.rows - 1) + P2.y = srcArg.rows - 1; + + // we use the integral image to compute fast features + value = (float) ( + (srcArg.at(P2.y, P2.x)) + + (srcArg.at(P1.y, P1.x)) - + (srcArg.at(P2.y, P1.x)) - + (srcArg.at(P1.y, P2.x)) + ); + value = (value - centerVal)/ (( (P2.x - P1.x) * (P2.y - P1.y))-1) ; + return value; +} + +void StaticSaliencyFineGrained::mixScales(Mat *intensityScaledOn, Mat intensityOn, Mat *intensityScaledOff, Mat intensityOff, const int numScales) +{ + int i=0, x, y; + int width = intensityScaledOn[0].cols; + int height = intensityScaledOn[0].rows; + short int maxValOn = 0, currValOn=0; + short int maxValOff = 0, currValOff=0; + int maxValSumOff = 0, maxValSumOn=0; + Mat mixedValuesOn(Size(width, height), CV_16UC1); + Mat mixedValuesOff(Size(width, height), CV_16UC1); + + mixedValuesOn.setTo(Scalar::all(0)); + mixedValuesOff.setTo(Scalar::all(0)); + + for(i=0;i(y, x); + if(currValOn > maxValOn) + maxValOn = currValOn; + + currValOff = intensityScaledOff[i].at(y, x); + if(currValOff > maxValOff) + maxValOff = currValOff; + + mixedValuesOn.at(y, x) += currValOn; + mixedValuesOff.at(y, x) += currValOff; + } + } + + for(y=0;y(y, x); + currValOff = mixedValuesOff.at(y, x); + if(currValOff > maxValSumOff) + maxValSumOff = currValOff; + if(currValOn > maxValSumOn) + maxValSumOn = currValOn; + } + + + for(y=0;y(y, x) = (uchar)(255.*((float)(mixedValuesOn.at(y, x) / (float)maxValSumOn))); + intensityOff.at(y, x) = (uchar)(255.*((float)(mixedValuesOff.at(y, x) / (float)maxValSumOff))); + } + +} + +void StaticSaliencyFineGrained::mixOnOff(Mat intensityOn, Mat intensityOff, Mat intensityArg) +{ + int x,y; + int width = intensityOn.cols; + int height= intensityOn.rows; + int maxVal=0; + + int currValOn, currValOff, maxValSumOff, maxValSumOn; + + Mat intensity(Size(width, height), CV_8UC1); + + + maxValSumOff = 0; + maxValSumOn = 0; + + for(y=0;y(y, x); + currValOff = intensityOff.at(y, x); + if(currValOff > maxValSumOff) + maxValSumOff = currValOff; + if(currValOn > maxValSumOn) + maxValSumOn = currValOn; + } + + if(maxValSumOn > maxValSumOff) + maxVal = maxValSumOn; + else + maxVal = maxValSumOff; + + + + for(y=0;y(y, x) = (uchar) (255. * (float) (intensityOn.at(y, x) + intensityOff.at(y, x)) / (float)maxVal); + } + + intensity.copyTo(intensityArg); +} + + +} /* namespace saliency */ +}/* namespace cv */