From 6f1b6eb46940b35026ee51a44ad367e991331525 Mon Sep 17 00:00:00 2001 From: jaco Date: Wed, 11 Jun 2014 19:27:14 +0200 Subject: [PATCH] Objectness porting partially computed --- .../saliency/saliencySpecializedClasses.hpp | 71 ++- modules/saliency/samples/computeSaliency.cpp | 2 +- modules/saliency/src/CmTimer.h | 88 ++++ modules/saliency/src/FilterTIG.cpp | 78 ++++ modules/saliency/src/FilterTIG.h | 45 ++ modules/saliency/src/ValStructVec.h | 72 +++ modules/saliency/src/kyheader.h | 224 ++++++++++ modules/saliency/src/objectnessBING.cpp | 410 +++++++++++++++++- 8 files changed, 969 insertions(+), 21 deletions(-) create mode 100644 modules/saliency/src/CmTimer.h create mode 100644 modules/saliency/src/FilterTIG.cpp create mode 100644 modules/saliency/src/FilterTIG.h create mode 100644 modules/saliency/src/ValStructVec.h create mode 100644 modules/saliency/src/kyheader.h diff --git a/modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp b/modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp index bc23eb1fb..296caa3e8 100644 --- a/modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp +++ b/modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp @@ -43,6 +43,12 @@ #define __OPENCV_SALIENCY_SPECIALIZED_CLASSES_HPP_ #include "saliencyBaseClasses.hpp" +#include "kyheader.h" +#include "ValStructVec.h" +#include "FilterTIG.h" +#include +#include +#include //TODO delete //#define SALIENCY_DEBUG true @@ -134,25 +140,72 @@ class CV_EXPORTS_W MotionSaliencyPBAS : public MotionSaliency class CV_EXPORTS_W ObjectnessBING : public Objectness { public: - /*struct CV_EXPORTS Params - { - Params(); - void read( const FileNode& fn ); - void write( FileStorage& fs ) const; - }; */ - //ObjectnessBING( const ObjectnessBING::Params ¶meters = ObjectnessBING::Params() ); + ObjectnessBING(); ~ObjectnessBING(); void read( const FileNode& fn ); void write( FileStorage& fs ) const; + // Load trained model. + int loadTrainedModel(std::string modelName = ""); // Return -1, 0, or 1 if partial, none, or all loaded + + // Get potential bounding boxes, each of which is represented by a Vec4i for (minX, minY, maxX, maxY). + // The trained model should be prepared before calling this function: loadTrainedModel() or trainStageI() + trainStageII(). + // Use numDet to control the final number of proposed bounding boxes, and number of per size (scale and aspect ratio) + void getObjBndBoxes(CMat &img3u, ValStructVec &valBoxes, int numDetPerSize = 120); + void getObjBndBoxesForImage(Mat img, ValStructVec &boxes, int numDetPerSize); + + void setColorSpace(int clr = MAXBGR); + + // Read matrix from binary file + static bool matRead( const std::string& filename, Mat& M); + + enum {MAXBGR, HSV, G}; + + inline static float LoG(float x, float y, float delta) {float d = -(x*x+y*y)/(2*delta*delta); return -1.0f/((float)(CV_PI)*pow(delta, 4)) * (1+d)*exp(d);} // Laplacian of Gaussian + + protected: bool computeSaliencyImpl( const InputArray& src, OutputArray& dst ); AlgorithmInfo* info() const; //{ return 0; } - private: - //Params params; + private: // Parameters + const double _base, _logBase; // base for window size quantization + const int _W; // As described in the paper: #Size, Size(_W, _H) of feature window. + const int _NSS; // Size for non-maximal suppress + const int _maxT, _minT, _numT; // The minimal and maximal dimensions of the template + + int _Clr; // + static const char* _clrName[3]; + + //DataSetVOC &_voc; // The dataset for training, testing + std:: string _modelName, _trainDirSI, _bbResDir; + + vecI _svmSzIdxs; // Indexes of active size. It's equal to _svmFilters.size() and _svmReW1f.rows + Mat _svmFilter; // Filters learned at stage I, each is a _H by _W CV_32F matrix + FilterTIG _tigF; // TIG filter + Mat _svmReW1f; // Re-weight parameters learned at stage II. + + private: // Help functions + + bool filtersLoaded() {int n = _svmSzIdxs.size(); return n > 0 && _svmReW1f.size() == Size(2, n) && _svmFilter.size() == Size(_W, _W);} + void predictBBoxSI(CMat &mag3u, ValStructVec &valBoxes, vecI &sz, int NUM_WIN_PSZ = 100, bool fast = true); + void predictBBoxSII(ValStructVec &valBoxes, const vecI &sz); + + // Calculate the image gradient: center option as in VLFeat + void gradientMag(CMat &imgBGR3u, Mat &mag1u); + + static void gradientRGB(CMat &bgr3u, Mat &mag1u); + static void gradientGray(CMat &bgr3u, Mat &mag1u); + static void gradientHSV(CMat &bgr3u, Mat &mag1u); + static void gradientXY(CMat &x1i, CMat &y1i, Mat &mag1u); + + static inline int bgrMaxDist(const Vec3b &u, const Vec3b &v) {int b = abs(u[0]-v[0]), g = abs(u[1]-v[1]), r = abs(u[2]-v[2]); b = max(b,g); return max(b,r);} + static inline int vecDist3b(const Vec3b &u, const Vec3b &v) {return abs(u[0]-v[0]) + abs(u[1]-v[1]) + abs(u[2]-v[2]);} + + //Non-maximal suppress + static void nonMaxSup(CMat &matchCost1f, ValStructVec &matchCost, int NSS = 1, int maxPoint = 50, bool fast = true); }; diff --git a/modules/saliency/samples/computeSaliency.cpp b/modules/saliency/samples/computeSaliency.cpp index 7c97a6557..ce0ee33cc 100644 --- a/modules/saliency/samples/computeSaliency.cpp +++ b/modules/saliency/samples/computeSaliency.cpp @@ -15,7 +15,7 @@ static void help() { cout << "\nThis example shows the functionality of \"Saliency \"" "Call:\n" - "./example_saliency_computeSaliency \n" + "./example_saliency_computeSaliency \n" << endl; } diff --git a/modules/saliency/src/CmTimer.h b/modules/saliency/src/CmTimer.h new file mode 100644 index 000000000..27bfbc421 --- /dev/null +++ b/modules/saliency/src/CmTimer.h @@ -0,0 +1,88 @@ +#pragma once +#ifndef CMTIMER_H +#define CMTIMER_H + +#include "kyheader.h" + +class CmTimer +{ +public: + CmTimer(CStr t):title(t) { is_started = false; start_clock = 0; cumulative_clock = 0; n_starts = 0; } + + ~CmTimer(){ if (is_started) printf("CmTimer '%s' is started and is being destroyed.\n", title.c_str()); } + + inline void Start(); + inline void Stop(); + inline void Reset(); + + inline bool Report(); + inline bool StopAndReport() { Stop(); return Report(); } + inline float TimeInSeconds(); + +private: + CStr title; + + bool is_started; + clock_t start_clock; + clock_t cumulative_clock; + unsigned int n_starts; +}; + +/************************************************************************/ +/* Implementations */ +/************************************************************************/ + +void CmTimer::Start() +{ + if (is_started){ + printf("CmTimer '%s' is already started. Nothing done.\n", title.c_str()); + return; + } + + is_started = true; + n_starts++; + start_clock = clock(); +} + +void CmTimer::Stop() +{ + if (!is_started){ + printf("CmTimer '%s' is started. Nothing done\n", title.c_str()); + return; + } + + cumulative_clock += clock() - start_clock; + is_started = false; +} + +void CmTimer::Reset() +{ + if (is_started) { + printf("CmTimer '%s'is started during reset request.\n Only reset cumulative time.\n"); + return; + } + cumulative_clock = 0; +} + +bool CmTimer::Report() +{ + if (is_started){ + printf("CmTimer '%s' is started.\n Cannot provide a time report.", title.c_str()); + return false; + } + + float timeUsed = TimeInSeconds(); + printf("[%s] CumuTime: %gs, #run: %d, AvgTime: %gs\n", title.c_str(), timeUsed, n_starts, timeUsed/n_starts); + return true; +} + +float CmTimer::TimeInSeconds() +{ + if (is_started){ + printf("CmTimer '%s' is started. Nothing done\n", title.c_str()); + return 0; + } + return float(cumulative_clock) / CLOCKS_PER_SEC; +} + +#endif // CMTIMER_H diff --git a/modules/saliency/src/FilterTIG.cpp b/modules/saliency/src/FilterTIG.cpp new file mode 100644 index 000000000..b1769dce7 --- /dev/null +++ b/modules/saliency/src/FilterTIG.cpp @@ -0,0 +1,78 @@ +#include "kyheader.h" +#include "FilterTIG.h" +#include "CmShow.h" + + +void FilterTIG::update(CMat &w1f){ + CV_Assert(w1f.cols * w1f.rows == D && w1f.type() == CV_32F && w1f.isContinuous()); + float b[D], residuals[D]; + memcpy(residuals, w1f.data, sizeof(float)*D); + for (int i = 0; i < NUM_COMP; i++){ + float avg = 0; + for (int j = 0; j < D; j++){ + b[j] = residuals[j] >= 0.0f ? 1.0f : -1.0f; + avg += residuals[j] * b[j]; + } + avg /= D; + _coeffs1[i] = avg, _coeffs2[i] = avg*2, _coeffs4[i] = avg*4, _coeffs8[i] = avg*8; + for (int j = 0; j < D; j++) + residuals[j] -= avg*b[j]; + UINT64 tig = 0; + for (int j = 0; j < D; j++) + tig = (tig << 1) | (b[j] > 0 ? 1 : 0); + _bTIGs[i] = tig; + } +} + +void FilterTIG::reconstruct(Mat &w1f){ + w1f = Mat::zeros(8, 8, CV_32F); + float *weight = (float*)w1f.data; + for (int i = 0; i < NUM_COMP; i++){ + UINT64 tig = _bTIGs[i]; + for (int j = 0; j < D; j++) + weight[j] += _coeffs1[i] * (((tig >> (63-j)) & 1) ? 1 : -1); + } +} + +// For a W by H gradient magnitude map, find a W-7 by H-7 CV_32F matching score map +// Please refer to my paper for definition of the variables used in this function +Mat FilterTIG::matchTemplate(const Mat &mag1u){ + const int H = mag1u.rows, W = mag1u.cols; + const Size sz(W+1, H+1); // Expand original size to avoid dealing with boundary conditions + Mat_ Tig1 = Mat_::zeros(sz), Tig2 = Mat_::zeros(sz); + Mat_ Tig4 = Mat_::zeros(sz), Tig8 = Mat_::zeros(sz); + Mat_ Row1 = Mat_::zeros(sz), Row2 = Mat_::zeros(sz); + Mat_ Row4 = Mat_::zeros(sz), Row8 = Mat_::zeros(sz); + Mat_ scores(sz); + for(int y = 1; y <= H; y++){ + const byte* G = mag1u.ptr(y-1); + INT64* T1 = Tig1.ptr(y); // Binary TIG of current row + INT64* T2 = Tig2.ptr(y); + INT64* T4 = Tig4.ptr(y); + INT64* T8 = Tig8.ptr(y); + INT64* Tu1 = Tig1.ptr(y-1); // Binary TIG of upper row + INT64* Tu2 = Tig2.ptr(y-1); + INT64* Tu4 = Tig4.ptr(y-1); + INT64* Tu8 = Tig8.ptr(y-1); + byte* R1 = Row1.ptr(y); + byte* R2 = Row2.ptr(y); + byte* R4 = Row4.ptr(y); + byte* R8 = Row8.ptr(y); + float *s = scores.ptr(y); + for (int x = 1; x <= W; x++) { + byte g = G[x-1]; + R1[x] = (R1[x-1] << 1) | ((g >> 4) & 1); + R2[x] = (R2[x-1] << 1) | ((g >> 5) & 1); + R4[x] = (R4[x-1] << 1) | ((g >> 6) & 1); + R8[x] = (R8[x-1] << 1) | ((g >> 7) & 1); + T1[x] = (Tu1[x] << 8) | R1[x]; + T2[x] = (Tu2[x] << 8) | R2[x]; + T4[x] = (Tu4[x] << 8) | R4[x]; + T8[x] = (Tu8[x] << 8) | R8[x]; + s[x] = dot(T1[x], T2[x], T4[x], T8[x]); + } + } + Mat matchCost1f; + scores(Rect(8, 8, W-7, H-7)).copyTo(matchCost1f); + return matchCost1f; +} diff --git a/modules/saliency/src/FilterTIG.h b/modules/saliency/src/FilterTIG.h new file mode 100644 index 000000000..820a3a508 --- /dev/null +++ b/modules/saliency/src/FilterTIG.h @@ -0,0 +1,45 @@ +#pragma once +#include "kyheader.h" +class FilterTIG +{ +public: + void update(CMat &w); + + // For a W by H gradient magnitude map, find a W-7 by H-7 CV_32F matching score map + Mat matchTemplate(const Mat &mag1u); + + inline float dot(const INT64 tig1, const INT64 tig2, const INT64 tig4, const INT64 tig8); + +public: + void reconstruct(Mat &w); // For illustration purpose + +private: + static const int NUM_COMP = 2; // Number of components + static const int D = 64; // Dimension of TIG + INT64 _bTIGs[NUM_COMP]; // Binary TIG features + float _coeffs1[NUM_COMP]; // Coefficients of binary TIG features + + // For efficiently deals with different bits in CV_8U gradient map + float _coeffs2[NUM_COMP], _coeffs4[NUM_COMP], _coeffs8[NUM_COMP]; +}; + + +inline float FilterTIG::dot(const INT64 tig1, const INT64 tig2, const INT64 tig4, const INT64 tig8) +{ + INT64 bcT1 = __builtin_popcountll(tig1); + INT64 bcT2 = __builtin_popcountll(tig2); + INT64 bcT4 = __builtin_popcountll(tig4); + INT64 bcT8 = __builtin_popcountll(tig8); + + INT64 bc01 = (__builtin_popcountll(_bTIGs[0] & tig1) << 1) - bcT1; + INT64 bc02 = ((__builtin_popcountll(_bTIGs[0] & tig2) << 1) - bcT2) << 1; + INT64 bc04 = ((__builtin_popcountll(_bTIGs[0] & tig4) << 1) - bcT4) << 2; + INT64 bc08 = ((__builtin_popcountll(_bTIGs[0] & tig8) << 1) - bcT8) << 3; + + INT64 bc11 = (__builtin_popcountll(_bTIGs[1] & tig1) << 1) - bcT1; + INT64 bc12 = ((__builtin_popcountll(_bTIGs[1] & tig2) << 1) - bcT2) << 1; + INT64 bc14 = ((__builtin_popcountll(_bTIGs[1] & tig4) << 1) - bcT4) << 2; + INT64 bc18 = ((__builtin_popcountll(_bTIGs[1] & tig8) << 1) - bcT8) << 3; + + return _coeffs1[0] * (bc01 + bc02 + bc04 + bc08) + _coeffs1[1] * (bc11 + bc12 + bc14 + bc18); +} diff --git a/modules/saliency/src/ValStructVec.h b/modules/saliency/src/ValStructVec.h new file mode 100644 index 000000000..92897ddb7 --- /dev/null +++ b/modules/saliency/src/ValStructVec.h @@ -0,0 +1,72 @@ +#pragma once + +/************************************************************************/ +/* A value struct vector that supports efficient sorting */ +/************************************************************************/ + +template +struct ValStructVec +{ + ValStructVec(){clear();} + inline int size() const {return sz;} + inline void clear() {sz = 0; structVals.clear(); valIdxes.clear();} + inline void reserve(int resSz){clear(); structVals.reserve(resSz); valIdxes.reserve(resSz); } + inline void pushBack(const VT& val, const ST& structVal) {valIdxes.push_back(make_pair(val, sz)); structVals.push_back(structVal); sz++;} + + inline const VT& operator ()(int i) const {return valIdxes[i].first;} // Should be called after sort + inline const ST& operator [](int i) const {return structVals[valIdxes[i].second];} // Should be called after sort + inline VT& operator ()(int i) {return valIdxes[i].first;} // Should be called after sort + inline ST& operator [](int i) {return structVals[valIdxes[i].second];} // Should be called after sort + + void sort(bool descendOrder = true); + const vector &getSortedStructVal(); + void append(const ValStructVec &newVals, int startV = 0); + + vector structVals; // struct values + +private: + int sz; // size of the value struct vector + vector> valIdxes; // Indexes after sort + bool smaller() {return true;}; + vector sortedStructVals; +}; + +template +void ValStructVec::append(const ValStructVec &newVals, int startV) +{ + int sz = newVals.size(); + for (int i = 0; i < sz; i++) + pushBack((float)((i+300)*startV)/*newVals(i)*/, newVals[i]); +} + +template +void ValStructVec::sort(bool descendOrder /* = true */) +{ + if (descendOrder) + std::sort(valIdxes.begin(), valIdxes.end(), std::greater>()); + else + std::sort(valIdxes.begin(), valIdxes.end(), std::less>()); +} + +template +const vector& ValStructVec::getSortedStructVal() +{ + sortedStructVals.resize(sz); + for (int i = 0; i < sz; i++) + sortedStructVals[i] = structVals[valIdxes[i].second]; + return sortedStructVals; +} + +/* +void valStructVecDemo() +{ + ValStructVec sVals; + sVals.pushBack(3, "String 3"); + sVals.pushBack(5, "String 5"); + sVals.pushBack(4, "String 4"); + sVals.pushBack(1, "String 1"); + sVals.sort(false); + for (int i = 0; i < sVals.size(); i++) + printf("%d, %s\n", sVals(i), _S(sVals[i])); +} +*/ diff --git a/modules/saliency/src/kyheader.h b/modules/saliency/src/kyheader.h new file mode 100644 index 000000000..73a6d7683 --- /dev/null +++ b/modules/saliency/src/kyheader.h @@ -0,0 +1,224 @@ +#ifndef KYHEADER_H +#define KYHEADER_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +//#include +//#include +#include +#include + + +// TODO: reference additional headers your program requires here +//#include "LibLinear/linear.h" +//#include +#include "opencv2/core.hpp" + +#define CV_VERSION_ID CVAUX_STR(CV_MAJOR_VERSION) CVAUX_STR(CV_MINOR_VERSION) CVAUX_STR(CV_SUBMINOR_VERSION) +#ifdef _DEBUG +#define cvLIB(name) "opencv_" name CV_VERSION_ID "d" +#else +#define cvLIB(name) "opencv_" name CV_VERSION_ID +#endif + +#pragma comment( lib, cvLIB("core")) +#pragma comment( lib, cvLIB("imgproc")) +#pragma comment( lib, cvLIB("highgui")) +using namespace cv; +using namespace std; +#ifdef WIN32 +/* windows stuff */ +#else +typedef unsigned long DWORD; +typedef unsigned short WORD; +typedef unsigned int UNINT32; +typedef bool BOOL; +typedef void *HANDLE; +typedef unsigned char byte; +#endif + +typedef std::vector vecI; +typedef const std::string CStr; +typedef const Mat CMat; +typedef std::vector vecS; +typedef std::vector vecM; +typedef std::vector vecF; +typedef std::vector vecD; + +enum{CV_FLIP_BOTH = -1, CV_FLIP_VERTICAL = 0, CV_FLIP_HORIZONTAL = 1}; +#define _S(str) ((str).c_str()) +#define CHK_IND(p) ((p).x >= 0 && (p).x < _w && (p).y >= 0 && (p).y < _h) +#define CV_Assert_(expr, args) \ +{\ + if(!(expr)) {\ + String msg = cv::format args; \ + printf("%s in %s:%d\n", msg.c_str(), __FILE__, __LINE__); \ + cv::error(cv::Exception(CV_StsAssert, msg, __FUNCTION__, __FILE__, __LINE__) ); }\ +} + +using namespace std; + +// Return -1 if not in the list +template +static inline int findFromList(const T &word, const vector &strList) { + //TODO delete test code + //cout << "\n\n" << "word" <<" "<< word << endl; + for(int i=0; i +static inline int findFromList(const string &word, const vector &strList) { + //for(int i=0; i inline T sqr(T x) { return x * x; } // out of range risk for T = byte, ... +template inline T vecSqrDist(const Vec &v1, const Vec &v2) {T s = 0; for (int i=0; i inline T vecDist(const Vec &v1, const Vec &v2) { return sqrt(vecSqrDist(v1, v2)); } // out of range risk for T = byte, ... + +inline Rect Vec4i2Rect(Vec4i &v){return Rect(Point(v[0] - 1, v[1] - 1), Point(v[2], v[3])); } + +#ifdef __WIN32 + #define INT64 long long +#else + #define INT64 long + typedef unsigned long UINT64; +#endif + +///// +#if (_MSC_VER >= 1500) +# include +# define POPCNT(x) __popcnt(x) +# define POPCNT64(x) __popcnt64(x) +#endif + +#if defined(__GNUC__) +# define POPCNT(x) __builtin_popcount(x) +# define POPCNT64(x) __builtin_popcountll(x) +#endif + +inline int popcnt64(register uint64_t u) +{ + u = (u & 0x5555555555555555) + ((u >> 1) & 0x5555555555555555); + u = (u & 0x3333333333333333) + ((u >> 2) & 0x3333333333333333); + u = (u & 0x0f0f0f0f0f0f0f0f) + ((u >> 4) & 0x0f0f0f0f0f0f0f0f); + u = (u & 0x00ff00ff00ff00ff) + ((u >> 8) & 0x00ff00ff00ff00ff); + u = (u & 0x0000ffff0000ffff) + ((u >>16) & 0x0000ffff0000ffff); + u = (u & 0x00000000ffffffff) + ((u >>32) & 0x00000000ffffffff); + return u; +} + +inline int popcnt(register uint32_t u) +{ + u = (u & 0x55555555) + ((u >> 1) & 0x55555555); + u = (u & 0x33333333) + ((u >> 2) & 0x33333333); + u = (u & 0x0f0f0f0f) + ((u >> 4) & 0x0f0f0f0f); + u = (u & 0x00ff00ff) + ((u >> 8) & 0x00ff00ff); + u = (u & 0x0000ffff) + ((u >>16) & 0x0000ffff); + return u; +} + +inline int popcnt64_nibble(register uint64_t u) +{ + static const uint8_t Table[] = { + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4 + }; + + int c = 0; + while (u) + { + c += Table[u & 0xf]; + u >>= 4; + } + return c; +} + +inline int popcnt_nibble(register uint32_t u) +{ + static const uint8_t Table[] = { + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4 + }; + + int c = 0; + while (u) + { + c += Table[u & 0xf]; + u >>= 4; + } + return c; +} + +inline int popcnt64_byte(register uint64_t u) +{ +#define B2(k) k, k+1, k+1, k+2 +#define B4(k) B2(k), B2(k+1), B2(k+1), B2(k+2) +#define B6(k) B4(k), B4(k+1), B4(k+1), B4(k+2) + static const uint8_t Table[] = { + B6(0), B6(1), B6(1), B6(2) + }; +#undef B6 +#undef B4 +#undef B2 + + int c = 0; + while (u) + { + c += Table[u & 0xff]; + u >>= 8; + } + return c; +} + +inline int popcnt_byte(register uint32_t u) +{ +#define B2(k) k, k+1, k+1, k+2 +#define B4(k) B2(k), B2(k+1), B2(k+1), B2(k+2) +#define B6(k) B4(k), B4(k+1), B4(k+1), B4(k+2) + static const uint8_t Table[] = { + B6(0), B6(1), B6(1), B6(2) + }; +#undef B6 +#undef B4 +#undef B2 + + int c = 0; + while (u) + { + c += Table[u & 0xff]; + u >>= 8; + } + return c; +} + + +///// +#include "CmTimer.h" +#include "CmFile.h" +#endif // KYHEADER_H diff --git a/modules/saliency/src/objectnessBING.cpp b/modules/saliency/src/objectnessBING.cpp index 031db2206..8b135d0aa 100644 --- a/modules/saliency/src/objectnessBING.cpp +++ b/modules/saliency/src/objectnessBING.cpp @@ -48,33 +48,414 @@ namespace cv * BING Objectness */ -/** - * Parameters +const char* ObjectnessBING::_clrName[3] = +{ "MAXBGR", "HSV", "I" }; -ObjectnessBING::Params::Params() +ObjectnessBING::ObjectnessBING() { + _base = 2; // base for window size quantization + _W = 8; // feature window size (W, W) + _NSS = 2; //non-maximal suppress size NSS + _logBase( log( _base ) ); + _minT( cvCeil( log( 10. ) / _logBase ) ); + _maxT( cvCeil( log( 500. ) / _logBase ) ); + _numT( _maxT - _minT + 1 ); + _Clr( MAXBGR ); + + setColorSpace( _Clr ); + className = "BING"; } -void ObjectnessBING::Params::read( const cv::FileNode& fn ) +ObjectnessBING::~ObjectnessBING() { - //resizedImageSize=Size(fn["resizedImageSize"]); } -void ObjectnessBING::Params::write( cv::FileStorage& fs ) const +void ObjectnessBING::setColorSpace(int clr) { - //fs << "resizedImageSize" << resizedImageSize; -} */ + _Clr = clr; + // _modelName = _voc.resDir + string(format("ObjNessB%gW%d%s", _base, _W, _clrName[_Clr]).c_str()); + // _trainDirSI = _voc.localDir + string(format("TrainS1B%gW%d%s/", _base, _W, _clrName[_Clr]).c_str()); + // _bbResDir = _voc.resDir + string(format("BBoxesB%gW%d%s/", _base, _W, _clrName[_Clr]).c_str()); +} -ObjectnessBING::ObjectnessBING() +int ObjectnessBING::loadTrainedModel(string modelName) // Return -1, 0, or 1 if partial, none, or all loaded { - className = "BING"; + if (modelName.size() == 0) + modelName = _modelName; + CStr s1 = modelName + ".wS1", s2 = modelName + ".wS2", sI = modelName + ".idx"; + Mat filters1f, reW1f, idx1i, show3u; + if (!matRead(s1, filters1f) || !matRead(sI, idx1i)){ + printf("Can't load model: %s or %s\n", _S(s1), _S(sI)); + return 0; + } + + //filters1f = aFilter(0.8f, 8); + //normalize(filters1f, filters1f, p, 1, NORM_MINMAX); + + normalize(filters1f, show3u, 1, 255, NORM_MINMAX, CV_8U); + _tigF.update(filters1f); + //_tigF.reconstruct(filters1f); + + _svmSzIdxs = idx1i; + CV_Assert(_svmSzIdxs.size() > 1 && filters1f.size() == Size(_W, _W) && filters1f.type() == CV_32F); + _svmFilter = filters1f; + + if (!matRead(s2, _svmReW1f) || _svmReW1f.size() != Size(2, _svmSzIdxs.size())){ + _svmReW1f = Mat(); + return -1; + } + return 1; } -ObjectnessBING::~ObjectnessBING() +void ObjectnessBING::predictBBoxSI(CMat &img3u, ValStructVec &valBoxes, vecI &sz, int NUM_WIN_PSZ, bool fast) +{ + const int numSz = _svmSzIdxs.size(); + const int imgW = img3u.cols, imgH = img3u.rows; + valBoxes.reserve(10000); + sz.clear(); sz.reserve(10000); + for (int ir = numSz - 1; ir >= 0; ir--){ + int r = _svmSzIdxs[ir]; + int height = cvRound(pow(_base, r/_numT + _minT)), width = cvRound(pow(_base, r%_numT + _minT)); + if (height > imgH * _base || width > imgW * _base) + continue; + + height = min(height, imgH), width = min(width, imgW); + Mat im3u, matchCost1f, mag1u; + resize(img3u, im3u, Size(cvRound(_W*imgW*1.0/width), cvRound(_W*imgH*1.0/height))); + gradientMag(im3u, mag1u); + + //imwrite(_voc.localDir + format("%d.png", r), mag1u); + //Mat mag1f; + //mag1u.convertTo(mag1f, CV_32F); + //matchTemplate(mag1f, _svmFilter, matchCost1f, CV_TM_CCORR); + + matchCost1f = _tigF.matchTemplate(mag1u); + + ValStructVec matchCost; + nonMaxSup(matchCost1f, matchCost, _NSS, NUM_WIN_PSZ, fast); + + // Find true locations and match values + double ratioX = width/_W, ratioY = height/_W; + int iMax = min(matchCost.size(), NUM_WIN_PSZ); + for (int i = 0; i < iMax; i++){ + float mVal = matchCost(i); + Point pnt = matchCost[i]; + Vec4i box(cvRound(pnt.x * ratioX), cvRound(pnt.y*ratioY)); + box[2] = cvRound(min(box[0] + width, imgW)); + box[3] = cvRound(min(box[1] + height, imgH)); + box[0] ++; + box[1] ++; + valBoxes.pushBack(mVal, box); + sz.push_back(ir); + } + } + +} + +void ObjectnessBING::predictBBoxSII(ValStructVec &valBoxes, const vecI &sz) { + int numI = valBoxes.size(); + for (int i = 0; i < numI; i++){ + const float* svmIIw = _svmReW1f.ptr(sz[i]); + valBoxes(i) = valBoxes(i) * svmIIw[0] + svmIIw[1]; + } + valBoxes.sort(); +} + +// Get potential bounding boxes, each of which is represented by a Vec4i for (minX, minY, maxX, maxY). +// The trained model should be prepared before calling this function: loadTrainedModel() or trainStageI() + trainStageII(). +// Use numDet to control the final number of proposed bounding boxes, and number of per size (scale and aspect ratio) +void ObjectnessBING::getObjBndBoxes(CMat &img3u, ValStructVec &valBoxes, int numDetPerSize) +{ + //CV_Assert_(filtersLoaded() , ("SVM filters should be initialized before getting object proposals\n")); + vecI sz; + predictBBoxSI(img3u, valBoxes, sz, numDetPerSize, false); + predictBBoxSII(valBoxes, sz); + return; +} + +void ObjectnessBING::nonMaxSup(CMat &matchCost1f, ValStructVec &matchCost, int NSS, int maxPoint, bool fast) +{ + const int _h = matchCost1f.rows, _w = matchCost1f.cols; + Mat isMax1u = Mat::ones(_h, _w, CV_8U), costSmooth1f; + ValStructVec valPnt; + matchCost.reserve(_h * _w); + valPnt.reserve(_h * _w); + if (fast){ + blur(matchCost1f, costSmooth1f, Size(3, 3)); + for (int r = 0; r < _h; r++){ + const float* d = matchCost1f.ptr(r); + const float* ds = costSmooth1f.ptr(r); + for (int c = 0; c < _w; c++) + if (d[c] >= ds[c]) + valPnt.pushBack(d[c], Point(c, r)); + } + } + else{ + for (int r = 0; r < _h; r++){ + const float* d = matchCost1f.ptr(r); + for (int c = 0; c < _w; c++) + valPnt.pushBack(d[c], Point(c, r)); + } + } + + valPnt.sort(); + for (int i = 0; i < valPnt.size(); i++){ + Point &pnt = valPnt[i]; + if (isMax1u.at(pnt)){ + matchCost.pushBack(valPnt(i), pnt); + for (int dy = -NSS; dy <= NSS; dy++) for (int dx = -NSS; dx <= NSS; dx++){ + Point neighbor = pnt + Point(dx, dy); + if (!CHK_IND(neighbor)) + continue; + isMax1u.at(neighbor) = false; + } + } + if (matchCost.size() >= maxPoint) + return; + } +} + +void ObjectnessBING::gradientMag(CMat &imgBGR3u, Mat &mag1u) +{ + switch (_Clr){ + case MAXBGR: + gradientRGB(imgBGR3u, mag1u); break; + case G: + gradientGray(imgBGR3u, mag1u); break; + case HSV: + gradientHSV(imgBGR3u, mag1u); break; + default: + printf("Error: not recognized color space\n"); + } +} + +void ObjectnessBING::gradientRGB(CMat &bgr3u, Mat &mag1u) +{ + const int H = bgr3u.rows, W = bgr3u.cols; + Mat Ix(H, W, CV_32S), Iy(H, W, CV_32S); + + // Left/right most column Ix + for (int y = 0; y < H; y++){ + Ix.at(y, 0) = bgrMaxDist(bgr3u.at(y, 1), bgr3u.at(y, 0))*2; + Ix.at(y, W-1) = bgrMaxDist(bgr3u.at(y, W-1), bgr3u.at(y, W-2))*2; + } + + // Top/bottom most column Iy + for (int x = 0; x < W; x++) { + Iy.at(0, x) = bgrMaxDist(bgr3u.at(1, x), bgr3u.at(0, x))*2; + Iy.at(H-1, x) = bgrMaxDist(bgr3u.at(H-1, x), bgr3u.at(H-2, x))*2; + } + + // Find the gradient for inner regions + for (int y = 0; y < H; y++){ + const Vec3b *dataP = bgr3u.ptr(y); + for (int x = 2; x < W; x++) + Ix.at(y, x-1) = bgrMaxDist(dataP[x-2], dataP[x]); // bgr3u.at(y, x+1), bgr3u.at(y, x-1)); + } + for (int y = 1; y < H-1; y++){ + const Vec3b *tP = bgr3u.ptr(y-1); + const Vec3b *bP = bgr3u.ptr(y+1); + for (int x = 0; x < W; x++) + Iy.at(y, x) = bgrMaxDist(tP[x], bP[x]); + } + gradientXY(Ix, Iy, mag1u); +} + +void ObjectnessBING::gradientGray(CMat &bgr3u, Mat &mag1u) +{ + Mat g1u; + cvtColor(bgr3u, g1u, COLOR_BGR2GRAY); + const int H = g1u.rows, W = g1u.cols; + Mat Ix(H, W, CV_32S), Iy(H, W, CV_32S); + + // Left/right most column Ix + for (int y = 0; y < H; y++){ + Ix.at(y, 0) = abs(g1u.at(y, 1) - g1u.at(y, 0)) * 2; + Ix.at(y, W-1) = abs(g1u.at(y, W-1) - g1u.at(y, W-2)) * 2; + } + + // Top/bottom most column Iy + for (int x = 0; x < W; x++) { + Iy.at(0, x) = abs(g1u.at(1, x) - g1u.at(0, x)) * 2; + Iy.at(H-1, x) = abs(g1u.at(H-1, x) - g1u.at(H-2, x)) * 2; + } + + // Find the gradient for inner regions + for (int y = 0; y < H; y++) + for (int x = 1; x < W-1; x++) + Ix.at(y, x) = abs(g1u.at(y, x+1) - g1u.at(y, x-1)); + for (int y = 1; y < H-1; y++) + for (int x = 0; x < W; x++) + Iy.at(y, x) = abs(g1u.at(y+1, x) - g1u.at(y-1, x)); + gradientXY(Ix, Iy, mag1u); +} + + +void ObjectnessBING::gradientHSV(CMat &bgr3u, Mat &mag1u) +{ + Mat hsv3u; + cvtColor(bgr3u, hsv3u, COLOR_BGR2HSV); + const int H = hsv3u.rows, W = hsv3u.cols; + Mat Ix(H, W, CV_32S), Iy(H, W, CV_32S); + + // Left/right most column Ix + for (int y = 0; y < H; y++){ + Ix.at(y, 0) = vecDist3b(hsv3u.at(y, 1), hsv3u.at(y, 0)); + Ix.at(y, W-1) = vecDist3b(hsv3u.at(y, W-1), hsv3u.at(y, W-2)); + } + + // Top/bottom most column Iy + for (int x = 0; x < W; x++) { + Iy.at(0, x) = vecDist3b(hsv3u.at(1, x), hsv3u.at(0, x)); + Iy.at(H-1, x) = vecDist3b(hsv3u.at(H-1, x), hsv3u.at(H-2, x)); + } + + // Find the gradient for inner regions + for (int y = 0; y < H; y++) + for (int x = 1; x < W-1; x++) + Ix.at(y, x) = vecDist3b(hsv3u.at(y, x+1), hsv3u.at(y, x-1))/2; + for (int y = 1; y < H-1; y++) + for (int x = 0; x < W; x++) + Iy.at(y, x) = vecDist3b(hsv3u.at(y+1, x), hsv3u.at(y-1, x))/2; + + gradientXY(Ix, Iy, mag1u); +} + +void ObjectnessBING::gradientXY(CMat &x1i, CMat &y1i, Mat &mag1u) +{ + const int H = x1i.rows, W = x1i.cols; + mag1u.create(H, W, CV_8U); + for (int r = 0; r < H; r++){ + const int *x = x1i.ptr(r), *y = y1i.ptr(r); + byte* m = mag1u.ptr(r); + for (int c = 0; c < W; c++) + m[c] = min(x[c] + y[c], 255); //((int)sqrt(sqr(x[c]) + sqr(y[c])), 255); + } +} + +void ObjectnessBING::getObjBndBoxesForImage(Mat img, ValStructVec &finalBoxes, int numDetPerSize) +{ + const int TestNum = 1; + vecM imgs3u(TestNum); + vector> boxesTests; + boxesTests.resize(TestNum); + + +#pragma omp parallel for + for (int i = 0; i < TestNum; i++){ + imgs3u[i] = img; + boxesTests[i].reserve(10000); + } + + int scales[3] = {1, 3, 5}; + for (int clr = MAXBGR; clr <= G; clr++){ + setColorSpace(clr); + loadTrainedModel(); + CmTimer tm("Predict"); + //tm.Start(); + +#pragma omp parallel for + for (int i = 0; i < TestNum; i++){ + ValStructVec boxes; + getObjBndBoxes(imgs3u[i], boxes, numDetPerSize); + boxesTests[i].append(boxes, scales[clr]); + + //boxes.valIdxes[0].first; + } + + //tm.Stop(); + //printf("Average time for predicting an image (%s) is %gs\n", _clrName[_Clr], tm.TimeInSeconds()/TestNum); + } + +/* _boxesTests.resize(TestNum); + CmFile::MkDir(_bbResDir); +#pragma omp parallel for + for (int i = 0; i < TestNum; i++){ + + CStr fName = _bbResDir + _voc.testSet[i]; + ValStructVec &boxes = boxesTests[i]; + FILE *f = fopen(_S(fName + ".txt"), "w"); + fprintf(f, "%d\n", boxes.size()); + for (size_t k = 0; k < boxes.size(); k++) + //fprintf(f, "%g, %s\n", boxes(k), _S(strVec4i(boxes[k]))); + fclose(f); + + _boxesTests[i].resize(boxesTests[i].size()); + for (int j = 0; j < boxesTests[i].size(); j++) + _boxesTests[i][j] = boxesTests[i][j]; + } */ + + //evaluatePerImgRecall(_boxesTests, "PerImgAllNS.m", 5000); + +#pragma omp parallel for + for (int i = 0; i < TestNum; i++){ + boxesTests[i].sort(false); + //for (int j = 0; j < boxesTests[i].size(); j++) + // _boxesTests[i][j] = boxesTests[i][j]; + finalBoxes=boxesTests[i]; + + } + //evaluatePerImgRecall(_boxesTests, "PerImgAllS.m", 5000); + +} + +struct MatchPathSeparator +{ + bool operator()( char ch ) const + { + return ch == '/'; + } +}; + +std::string inline basename( std::string const& pathname ) +{ + return std::string( + std::find_if( pathname.rbegin(), pathname.rend(), + MatchPathSeparator() ).base(), + pathname.end() ); +} + +std::string inline removeExtension( std::string const& filename ) +{ + std::string::const_reverse_iterator + pivot + = std::find( filename.rbegin(), filename.rend(), '.' ); + return pivot == filename.rend() + ? filename + : std::string( filename.begin(), pivot.base() - 1 ); +} + + +// Read matrix from binary file +bool ObjectnessBING::matRead(const string& filename, Mat& _M){ + + FileStorage fs2(filename+".yml.gz", FileStorage::READ); + String fileNameString(filename.c_str()); + + Mat M; + fs2[removeExtension(basename(fileNameString))]>> M; + + /*FILE* f = fopen(_S(filename), "rb"); + if (f == NULL) + return false; + char buf[8]; + int pre = fread(buf,sizeof(char), 5, f); + if (strncmp(buf, "CmMat", 5) != 0) { + printf("Invalidate CvMat data file %s\n", _S(filename)); + return false; + } + int headData[3]; // Width, height, type + fread(headData, sizeof(int), 3, f); + Mat M(headData[1], headData[0], headData[2]); + fread(M.data, sizeof(char), M.step * M.rows, f); + fclose(f); */ + + M.copyTo(_M); + return true; } void ObjectnessBING::read( const cv::FileNode& fn ) @@ -89,6 +470,13 @@ void ObjectnessBING::write( cv::FileStorage& fs ) const bool ObjectnessBING::computeSaliencyImpl( const InputArray& src, OutputArray& dst ) { + ValStructVec &finalBoxes; + getObjBndBoxesForImage( src.getMat(), finalBoxes, 250 ); + + // list of rectangles returned by objectess function + //std::vector boxesList = finalBoxes.structVals; + dst = finalBoxes.getSortedStructVal(); + //dst = finalBoxes.structVals; return true; }