Objectness porting partially computed

modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp

@@ -43,6 +43,12 @@
 #define __OPENCV_SALIENCY_SPECIALIZED_CLASSES_HPP_
 
 #include "saliencyBaseClasses.hpp"
+#include "kyheader.h"
+#include "ValStructVec.h"
+#include "FilterTIG.h"
+#include <cstdio>
+#include <string>
+#include <iostream>
 
 //TODO delete
 //#define SALIENCY_DEBUG true
@@ -134,25 +140,72 @@ class CV_EXPORTS_W MotionSaliencyPBAS : public MotionSaliency
 class CV_EXPORTS_W ObjectnessBING : public Objectness
 {
  public:
-  /*struct CV_EXPORTS Params
-   {
-   Params();
-   void read( const FileNode& fn );
-   void write( FileStorage& fs ) const;
-   }; */
-  //ObjectnessBING( const ObjectnessBING::Params &parameters = ObjectnessBING::Params() );
+
   ObjectnessBING();
   ~ObjectnessBING();
 
   void read( const FileNode& fn );
   void write( FileStorage& fs ) const;
 
+  // Load trained model.
+     int loadTrainedModel(std::string modelName = ""); // Return -1, 0, or 1 if partial, none, or all loaded
+
+     // Get potential bounding boxes, each of which is represented by a Vec4i for (minX, minY, maxX, maxY).
+     // The trained model should be prepared before calling this function: loadTrainedModel() or trainStageI() + trainStageII().
+     // Use numDet to control the final number of proposed bounding boxes, and number of per size (scale and aspect ratio)
+     void getObjBndBoxes(CMat &img3u, ValStructVec<float, Vec4i> &valBoxes, int numDetPerSize = 120);
+     void getObjBndBoxesForImage(Mat img, ValStructVec<float, Vec4i> &boxes, int numDetPerSize);
+
+     void setColorSpace(int clr = MAXBGR);
+
+     // Read matrix from binary file
+     static bool matRead( const std::string& filename, Mat& M);
+
+     enum {MAXBGR, HSV, G};
+
+     inline static float LoG(float x, float y, float delta) {float d = -(x*x+y*y)/(2*delta*delta);  return -1.0f/((float)(CV_PI)*pow(delta, 4)) * (1+d)*exp(d);} // Laplacian of Gaussian
+
+
  protected:
   bool computeSaliencyImpl( const InputArray& src, OutputArray& dst );
   AlgorithmInfo* info() const;  //{ return 0; }
 
- private:
-  //Params params;
+ private: // Parameters
+     const double _base, _logBase; // base for window size quantization
+     const int _W; // As described in the paper: #Size, Size(_W, _H) of feature window.
+     const int _NSS; // Size for non-maximal suppress
+     const int _maxT, _minT, _numT; // The minimal and maximal dimensions of the template
+
+     int _Clr; //
+     static const char* _clrName[3];
+
+     //DataSetVOC &_voc; // The dataset for training, testing
+     std:: string _modelName, _trainDirSI, _bbResDir;
+
+     vecI _svmSzIdxs; // Indexes of active size. It's equal to _svmFilters.size() and _svmReW1f.rows
+     Mat _svmFilter; // Filters learned at stage I, each is a _H by _W CV_32F matrix
+     FilterTIG _tigF; // TIG filter
+     Mat _svmReW1f; // Re-weight parameters learned at stage II.
+
+ private: // Help functions
+
+     bool filtersLoaded() {int n = _svmSzIdxs.size(); return n > 0 && _svmReW1f.size() == Size(2, n) && _svmFilter.size() == Size(_W, _W);}
+     void predictBBoxSI(CMat &mag3u, ValStructVec<float, Vec4i> &valBoxes, vecI &sz, int NUM_WIN_PSZ = 100, bool fast = true);
+     void predictBBoxSII(ValStructVec<float, Vec4i> &valBoxes, const vecI &sz);
+
+     // Calculate the image gradient: center option as in VLFeat
+     void gradientMag(CMat &imgBGR3u, Mat &mag1u);
+
+     static void gradientRGB(CMat &bgr3u, Mat &mag1u);
+     static void gradientGray(CMat &bgr3u, Mat &mag1u);
+     static void gradientHSV(CMat &bgr3u, Mat &mag1u);
+     static void gradientXY(CMat &x1i, CMat &y1i, Mat &mag1u);
+
+     static inline int bgrMaxDist(const Vec3b &u, const Vec3b &v) {int b = abs(u[0]-v[0]), g = abs(u[1]-v[1]), r = abs(u[2]-v[2]); b = max(b,g);  return max(b,r);}
+     static inline int vecDist3b(const Vec3b &u, const Vec3b &v) {return abs(u[0]-v[0]) + abs(u[1]-v[1]) + abs(u[2]-v[2]);}
+
+     //Non-maximal suppress
+     static void nonMaxSup(CMat &matchCost1f, ValStructVec<float, Point> &matchCost, int NSS = 1, int maxPoint = 50, bool fast = true);
 
 };
 

modules/saliency/samples/computeSaliency.cpp

@@ -15,7 +15,7 @@ static void help()
 {
   cout << "\nThis example shows the functionality of \"Saliency \""
        "Call:\n"
-       "./example_saliency_computeSaliency <saliencyAlgorithmType.[saliencyAlgorithmTypeSubType]> <video_name> <start_frame> \n"
+       "./example_saliency_computeSaliency <SALIENCY.[saliencyAlgorithmSubType]> <video_name> <start_frame> \n"
        << endl;
 }
 

modules/saliency/src/CmTimer.h

@@ -0,0 +1,88 @@
+#pragma once
+#ifndef CMTIMER_H
+#define CMTIMER_H
+
+#include "kyheader.h"
+
+class CmTimer
+{
+public:
+    CmTimer(CStr t):title(t) { is_started = false; start_clock = 0; cumulative_clock = 0; n_starts = 0; }
+
+    ~CmTimer(){	if (is_started) printf("CmTimer '%s' is started and is being destroyed.\n", title.c_str());	}
+
+    inline void Start();
+    inline void Stop();
+    inline void Reset();
+
+    inline bool Report();
+    inline bool StopAndReport() { Stop(); return Report(); }
+    inline float TimeInSeconds();
+
+private:
+    CStr title;
+
+    bool is_started;
+    clock_t start_clock;
+    clock_t cumulative_clock;
+    unsigned int n_starts;
+};
+
+/************************************************************************/
+/*                       Implementations                                */
+/************************************************************************/
+
+void CmTimer::Start()
+{
+    if (is_started){
+        printf("CmTimer '%s' is already started. Nothing done.\n", title.c_str());
+        return;
+    }
+
+    is_started = true;
+    n_starts++;
+    start_clock = clock();
+}
+
+void CmTimer::Stop()
+{
+    if (!is_started){
+        printf("CmTimer '%s' is started. Nothing done\n", title.c_str());
+        return;
+    }
+
+    cumulative_clock += clock() - start_clock;
+    is_started = false;
+}
+
+void CmTimer::Reset()
+{
+    if (is_started)	{
+        printf("CmTimer '%s'is started during reset request.\n Only reset cumulative time.\n");
+        return;
+    }
+    cumulative_clock = 0;
+}
+
+bool CmTimer::Report()
+{
+    if (is_started){
+        printf("CmTimer '%s' is started.\n Cannot provide a time report.", title.c_str());
+        return false;
+    }
+
+    float timeUsed = TimeInSeconds();
+    printf("[%s] CumuTime: %gs, #run: %d, AvgTime: %gs\n", title.c_str(), timeUsed, n_starts, timeUsed/n_starts);
+    return true;
+}
+
+float CmTimer::TimeInSeconds()
+{
+    if (is_started){
+        printf("CmTimer '%s' is started. Nothing done\n", title.c_str());
+        return 0;
+    }
+    return float(cumulative_clock) / CLOCKS_PER_SEC;
+}
+
+#endif // CMTIMER_H

modules/saliency/src/FilterTIG.cpp

@@ -0,0 +1,78 @@
+#include "kyheader.h"
+#include "FilterTIG.h"
+#include "CmShow.h"
+
+
+void FilterTIG::update(CMat &w1f){
+    CV_Assert(w1f.cols * w1f.rows == D && w1f.type() == CV_32F && w1f.isContinuous());
+    float b[D], residuals[D];
+    memcpy(residuals, w1f.data, sizeof(float)*D);
+    for (int i = 0; i < NUM_COMP; i++){
+        float avg = 0;
+        for (int j = 0; j < D; j++){
+            b[j] = residuals[j] >= 0.0f ? 1.0f : -1.0f;
+            avg += residuals[j] * b[j];
+        }
+        avg /= D;
+        _coeffs1[i] = avg, _coeffs2[i] = avg*2, _coeffs4[i] = avg*4, _coeffs8[i] = avg*8;
+        for (int j = 0; j < D; j++)
+            residuals[j] -= avg*b[j];
+        UINT64 tig = 0;
+        for (int j = 0; j < D; j++)
+            tig = (tig << 1) | (b[j] > 0 ? 1 : 0);
+        _bTIGs[i] = tig;
+    }
+}
+
+void FilterTIG::reconstruct(Mat &w1f){
+    w1f = Mat::zeros(8, 8, CV_32F);
+    float *weight = (float*)w1f.data;
+    for (int i = 0; i < NUM_COMP; i++){
+        UINT64 tig = _bTIGs[i];
+        for (int j = 0; j < D; j++)
+            weight[j] += _coeffs1[i] * (((tig >> (63-j)) & 1) ? 1 : -1);
+    }
+}
+
+// For a W by H gradient magnitude map, find a W-7 by H-7 CV_32F matching score map
+// Please refer to my paper for definition of the variables used in this function
+Mat FilterTIG::matchTemplate(const Mat &mag1u){
+    const int H = mag1u.rows, W = mag1u.cols;
+    const Size sz(W+1, H+1); // Expand original size to avoid dealing with boundary conditions
+    Mat_<INT64> Tig1 = Mat_<INT64>::zeros(sz), Tig2 = Mat_<INT64>::zeros(sz);
+    Mat_<INT64> Tig4 = Mat_<INT64>::zeros(sz), Tig8 = Mat_<INT64>::zeros(sz);
+    Mat_<byte> Row1 = Mat_<byte>::zeros(sz), Row2 = Mat_<byte>::zeros(sz);
+    Mat_<byte> Row4 = Mat_<byte>::zeros(sz), Row8 = Mat_<byte>::zeros(sz);
+    Mat_<float> scores(sz);
+    for(int y = 1; y <= H; y++){
+        const byte* G = mag1u.ptr<byte>(y-1);
+        INT64* T1 = Tig1.ptr<INT64>(y); // Binary TIG of current row
+        INT64* T2 = Tig2.ptr<INT64>(y);
+        INT64* T4 = Tig4.ptr<INT64>(y);
+        INT64* T8 = Tig8.ptr<INT64>(y);
+        INT64* Tu1 = Tig1.ptr<INT64>(y-1); // Binary TIG of upper row
+        INT64* Tu2 = Tig2.ptr<INT64>(y-1);
+        INT64* Tu4 = Tig4.ptr<INT64>(y-1);
+        INT64* Tu8 = Tig8.ptr<INT64>(y-1);
+        byte* R1 = Row1.ptr<byte>(y);
+        byte* R2 = Row2.ptr<byte>(y);
+        byte* R4 = Row4.ptr<byte>(y);
+        byte* R8 = Row8.ptr<byte>(y);
+        float *s = scores.ptr<float>(y);
+        for (int x = 1; x <= W; x++) {
+            byte g = G[x-1];
+            R1[x] = (R1[x-1] << 1) | ((g >> 4) & 1);
+            R2[x] = (R2[x-1] << 1) | ((g >> 5) & 1);
+            R4[x] = (R4[x-1] << 1) | ((g >> 6) & 1);
+            R8[x] = (R8[x-1] << 1) | ((g >> 7) & 1);
+            T1[x] = (Tu1[x] << 8) | R1[x];
+            T2[x] = (Tu2[x] << 8) | R2[x];
+            T4[x] = (Tu4[x] << 8) | R4[x];
+            T8[x] = (Tu8[x] << 8) | R8[x];
+            s[x] = dot(T1[x], T2[x], T4[x], T8[x]);
+        }
+    }
+    Mat matchCost1f;
+    scores(Rect(8, 8, W-7, H-7)).copyTo(matchCost1f);
+    return matchCost1f;
+}

modules/saliency/src/FilterTIG.h

@@ -0,0 +1,45 @@
+#pragma once
+#include "kyheader.h"
+class FilterTIG
+{
+public:
+    void update(CMat &w);
+
+    // For a W by H gradient magnitude map, find a W-7 by H-7 CV_32F matching score map
+    Mat matchTemplate(const Mat &mag1u);
+
+    inline float dot(const INT64 tig1, const INT64 tig2, const INT64 tig4, const INT64 tig8);
+
+public:
+    void reconstruct(Mat &w); // For illustration purpose
+
+private:
+    static const int NUM_COMP = 2; // Number of components
+    static const int D = 64; // Dimension of TIG
+    INT64 _bTIGs[NUM_COMP]; // Binary TIG features
+    float _coeffs1[NUM_COMP]; // Coefficients of binary TIG features
+
+    // For efficiently deals with different bits in CV_8U gradient map
+    float _coeffs2[NUM_COMP], _coeffs4[NUM_COMP], _coeffs8[NUM_COMP];
+};
+
+
+inline float FilterTIG::dot(const INT64 tig1, const INT64 tig2, const INT64 tig4, const INT64 tig8)
+{
+    INT64 bcT1 = __builtin_popcountll(tig1);
+    INT64 bcT2 = __builtin_popcountll(tig2);
+    INT64 bcT4 = __builtin_popcountll(tig4);
+    INT64 bcT8 = __builtin_popcountll(tig8);
+
+    INT64 bc01 = (__builtin_popcountll(_bTIGs[0] & tig1) << 1) - bcT1;
+    INT64 bc02 = ((__builtin_popcountll(_bTIGs[0] & tig2) << 1) - bcT2) << 1;
+    INT64 bc04 = ((__builtin_popcountll(_bTIGs[0] & tig4) << 1) - bcT4) << 2;
+    INT64 bc08 = ((__builtin_popcountll(_bTIGs[0] & tig8) << 1) - bcT8) << 3;
+
+    INT64 bc11 = (__builtin_popcountll(_bTIGs[1] & tig1) << 1) - bcT1;
+    INT64 bc12 = ((__builtin_popcountll(_bTIGs[1] & tig2) << 1) - bcT2) << 1;
+    INT64 bc14 = ((__builtin_popcountll(_bTIGs[1] & tig4) << 1) - bcT4) << 2;
+    INT64 bc18 = ((__builtin_popcountll(_bTIGs[1] & tig8) << 1) - bcT8) << 3;
+
+    return _coeffs1[0] * (bc01 + bc02 + bc04 + bc08) + _coeffs1[1] * (bc11 + bc12 + bc14 + bc18);
+}

modules/saliency/src/ValStructVec.h

@@ -0,0 +1,72 @@
+#pragma once
+
+/************************************************************************/
+/* A value struct vector that supports efficient sorting                */
+/************************************************************************/
+
+template<typename VT, typename ST>
+struct ValStructVec
+{
+    ValStructVec(){clear();}
+    inline int size() const {return sz;}
+    inline void clear() {sz = 0; structVals.clear(); valIdxes.clear();}
+    inline void reserve(int resSz){clear(); structVals.reserve(resSz); valIdxes.reserve(resSz); }
+    inline void pushBack(const VT& val, const ST& structVal) {valIdxes.push_back(make_pair(val, sz)); structVals.push_back(structVal); sz++;}
+
+    inline const VT& operator ()(int i) const {return valIdxes[i].first;} // Should be called after sort
+    inline const ST& operator [](int i) const {return structVals[valIdxes[i].second];} // Should be called after sort
+    inline VT& operator ()(int i) {return valIdxes[i].first;} // Should be called after sort
+    inline ST& operator [](int i) {return structVals[valIdxes[i].second];} // Should be called after sort
+
+    void sort(bool descendOrder = true);
+    const vector<ST> &getSortedStructVal();
+    void append(const ValStructVec<VT, ST> &newVals, int startV = 0);
+
+    vector<ST> structVals; // struct values
+
+private:
+    int sz; // size of the value struct vector
+    vector<pair<VT, int>> valIdxes; // Indexes after sort
+    bool smaller() {return true;};
+    vector<ST> sortedStructVals;
+};
+
+template<typename VT, typename ST>
+void ValStructVec<VT, ST>::append(const ValStructVec<VT, ST> &newVals, int startV)
+{
+    int sz = newVals.size();
+    for (int i = 0; i < sz; i++)
+        pushBack((float)((i+300)*startV)/*newVals(i)*/, newVals[i]);
+}
+
+template<typename VT, typename ST>
+void ValStructVec<VT, ST>::sort(bool descendOrder /* = true */)
+{
+    if (descendOrder)
+        std::sort(valIdxes.begin(), valIdxes.end(), std::greater<pair<VT, int>>());
+    else
+        std::sort(valIdxes.begin(), valIdxes.end(), std::less<pair<VT, int>>());
+}
+
+template<typename VT, typename ST>
+const vector<ST>& ValStructVec<VT, ST>::getSortedStructVal()
+{
+    sortedStructVals.resize(sz);
+    for (int i = 0; i < sz; i++)
+        sortedStructVals[i] = structVals[valIdxes[i].second];
+    return sortedStructVals;
+}
+
+/*
+void valStructVecDemo()
+{
+    ValStructVec<int, string> sVals;
+    sVals.pushBack(3, "String 3");
+    sVals.pushBack(5, "String 5");
+    sVals.pushBack(4, "String 4");
+    sVals.pushBack(1, "String 1");
+    sVals.sort(false);
+    for (int i = 0; i < sVals.size(); i++)
+        printf("%d, %s\n", sVals(i), _S(sVals[i]));
+}
+*/

modules/saliency/src/kyheader.h

@@ -0,0 +1,224 @@
+#ifndef KYHEADER_H
+#define KYHEADER_H
+
+#include <assert.h>
+#include <string>
+#include <vector>
+#include <functional>
+#include <list>
+#include <algorithm>
+#include <iostream>
+#include <cmath>
+#include <time.h>
+#include <fstream>
+#include <random>
+//#include <atlstr.h>
+//#include <atltypes.h>
+#include <omp.h>
+#include <strstream>
+
+
+// TODO: reference additional headers your program requires here
+//#include "LibLinear/linear.h"
+//#include <opencv2/opencv.hpp>
+#include "opencv2/core.hpp"
+
+#define CV_VERSION_ID CVAUX_STR(CV_MAJOR_VERSION) CVAUX_STR(CV_MINOR_VERSION) CVAUX_STR(CV_SUBMINOR_VERSION)
+#ifdef _DEBUG
+#define cvLIB(name) "opencv_" name CV_VERSION_ID "d"
+#else
+#define cvLIB(name) "opencv_" name CV_VERSION_ID
+#endif
+
+#pragma comment( lib, cvLIB("core"))
+#pragma comment( lib, cvLIB("imgproc"))
+#pragma comment( lib, cvLIB("highgui"))
+using namespace cv;
+using namespace std;
+#ifdef WIN32
+/* windows stuff */
+#else
+typedef unsigned long DWORD;
+typedef unsigned short WORD;
+typedef unsigned int UNINT32;
+typedef bool BOOL;
+typedef void *HANDLE;
+typedef unsigned char byte;
+#endif
+
+typedef std::vector<int> vecI;
+typedef const std::string CStr;
+typedef const Mat CMat;
+typedef std::vector<std::string> vecS;
+typedef std::vector<Mat> vecM;
+typedef std::vector<float> vecF;
+typedef std::vector<double> vecD;
+
+enum{CV_FLIP_BOTH = -1, CV_FLIP_VERTICAL = 0, CV_FLIP_HORIZONTAL = 1};
+#define _S(str) ((str).c_str())
+#define CHK_IND(p) ((p).x >= 0 && (p).x < _w && (p).y >= 0 && (p).y < _h)
+#define CV_Assert_(expr, args) \
+{\
+    if(!(expr)) {\
+    String msg = cv::format args; \
+    printf("%s in %s:%d\n", msg.c_str(), __FILE__, __LINE__); \
+    cv::error(cv::Exception(CV_StsAssert, msg, __FUNCTION__, __FILE__, __LINE__) ); }\
+}
+
+using namespace std;
+
+// Return -1 if not in the list
+template<typename T>
+static inline int findFromList(const T &word, const vector<T> &strList) {
+    //TODO delete test code
+    //cout << "\n\n" << "word" <<" "<< word << endl;
+    for(int i=0; i<strList.size(); i++) {
+	//cout <<"test word:"<< word << " " << endl;
+	//cout << "Size w " << word.size() << " Size L "<< strList[i].size() << endl;
+    }
+    
+    auto it = std::find(strList.begin(),strList.end(), word);
+    if (it == strList.end())
+    {
+        return -1;
+    } else
+    {
+        auto index = std::distance(strList.begin(), it);
+	//cout << "index" <<" "<< index << endl;
+        return index;
+    }
+} 
+/*template<typename T>
+static inline int findFromList(const string &word, const vector<T> &strList) {
+    //for(int i=0; i<strList.size(); i++){
+	//cout <<"element: " <<strList[i]<<" "<<word << endl;
+      //if (std::strcmp(word.c_str(),strList[i].c_str())==0) return i;
+  }
+  
+  return -1;
+}
+*/
+
+template<typename T> inline T sqr(T x) { return x * x; } // out of range risk for T = byte, ...
+template<class T, int D> inline T vecSqrDist(const Vec<T, D> &v1, const Vec<T, D> &v2) {T s = 0; for (int i=0; i<D; i++) s += sqr(v1[i] - v2[i]); return s;} // out of range risk for T = byte, ...
+template<class T, int D> inline T    vecDist(const Vec<T, D> &v1, const Vec<T, D> &v2) { return sqrt(vecSqrDist(v1, v2)); } // out of range risk for T = byte, ...
+
+inline Rect Vec4i2Rect(Vec4i &v){return Rect(Point(v[0] - 1, v[1] - 1), Point(v[2], v[3])); }
+
+#ifdef __WIN32
+    #define INT64 long long
+#else
+    #define INT64 long
+    typedef unsigned long UINT64;
+#endif
+
+/////
+#if (_MSC_VER >= 1500)
+# include <intrin.h>
+# define POPCNT(x) __popcnt(x)
+# define POPCNT64(x) __popcnt64(x)
+#endif
+
+#if defined(__GNUC__)
+# define POPCNT(x) __builtin_popcount(x)
+# define POPCNT64(x) __builtin_popcountll(x)
+#endif
+
+inline int popcnt64(register uint64_t u)
+{
+    u = (u & 0x5555555555555555) + ((u >> 1) & 0x5555555555555555);
+    u = (u & 0x3333333333333333) + ((u >> 2) & 0x3333333333333333);
+    u = (u & 0x0f0f0f0f0f0f0f0f) + ((u >> 4) & 0x0f0f0f0f0f0f0f0f);
+    u = (u & 0x00ff00ff00ff00ff) + ((u >> 8) & 0x00ff00ff00ff00ff);
+    u = (u & 0x0000ffff0000ffff) + ((u >>16) & 0x0000ffff0000ffff);
+    u = (u & 0x00000000ffffffff) + ((u >>32) & 0x00000000ffffffff);
+    return u;
+}
+
+inline int popcnt(register uint32_t u)
+{
+    u = (u & 0x55555555) + ((u >> 1) & 0x55555555);
+    u = (u & 0x33333333) + ((u >> 2) & 0x33333333);
+    u = (u & 0x0f0f0f0f) + ((u >> 4) & 0x0f0f0f0f);
+    u = (u & 0x00ff00ff) + ((u >> 8) & 0x00ff00ff);
+    u = (u & 0x0000ffff) + ((u >>16) & 0x0000ffff);
+    return u;
+}
+
+inline int popcnt64_nibble(register uint64_t u)
+{
+    static const uint8_t Table[] = {
+        0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4
+    };
+
+    int c = 0;
+    while (u)
+    {
+        c += Table[u & 0xf];
+        u >>= 4;
+    }
+    return c;
+}
+
+inline int popcnt_nibble(register uint32_t u)
+{
+    static const uint8_t Table[] = {
+        0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4
+    };
+
+    int c = 0;
+    while (u)
+    {
+        c += Table[u & 0xf];
+        u >>= 4;
+    }
+    return c;
+}
+
+inline int popcnt64_byte(register uint64_t u)
+{
+#define B2(k) k, k+1, k+1, k+2
+#define B4(k) B2(k), B2(k+1), B2(k+1), B2(k+2)
+#define B6(k) B4(k), B4(k+1), B4(k+1), B4(k+2)
+    static const uint8_t Table[] = {
+        B6(0), B6(1), B6(1), B6(2)
+    };
+#undef B6
+#undef B4
+#undef B2
+
+    int c = 0;
+    while (u)
+    {
+        c += Table[u & 0xff];
+        u >>= 8;
+    }
+    return c;
+}
+
+inline int popcnt_byte(register uint32_t u)
+{
+#define B2(k) k, k+1, k+1, k+2
+#define B4(k) B2(k), B2(k+1), B2(k+1), B2(k+2)
+#define B6(k) B4(k), B4(k+1), B4(k+1), B4(k+2)
+    static const uint8_t Table[] = {
+        B6(0), B6(1), B6(1), B6(2)
+    };
+#undef B6
+#undef B4
+#undef B2
+
+    int c = 0;
+    while (u)
+    {
+        c += Table[u & 0xff];
+        u >>= 8;
+    }
+    return c;
+}
+
+
+/////
+#include "CmTimer.h"
+#include "CmFile.h"
+#endif // KYHEADER_H

modules/saliency/src/objectnessBING.cpp

@@ -48,33 +48,414 @@ namespace cv
  * BING Objectness
  */
 
-/**
- * Parameters
+const char* ObjectnessBING::_clrName[3] =
+{ "MAXBGR", "HSV", "I" };
 
-ObjectnessBING::Params::Params()
+ObjectnessBING::ObjectnessBING()
 {
+  _base = 2;  // base for window size quantization
+  _W = 8;  // feature window size (W, W)
+  _NSS = 2;  //non-maximal suppress size NSS
+  _logBase( log( _base ) );
+  _minT( cvCeil( log( 10. ) / _logBase ) );
+  _maxT( cvCeil( log( 500. ) / _logBase ) );
+  _numT( _maxT - _minT + 1 );
+  _Clr( MAXBGR );
+
+  setColorSpace( _Clr );
 
+  className = "BING";
 }
 
-void ObjectnessBING::Params::read( const cv::FileNode& fn )
+ObjectnessBING::~ObjectnessBING()
 {
-  //resizedImageSize=Size(fn["resizedImageSize"]);
 
 }
 
-void ObjectnessBING::Params::write( cv::FileStorage& fs ) const
+void ObjectnessBING::setColorSpace(int clr)
 {
-  //fs << "resizedImageSize" << resizedImageSize;
-} */
+    _Clr = clr;
+  //  _modelName = _voc.resDir + string(format("ObjNessB%gW%d%s", _base, _W, _clrName[_Clr]).c_str());
+  //  _trainDirSI = _voc.localDir + string(format("TrainS1B%gW%d%s/", _base, _W, _clrName[_Clr]).c_str());
+  //  _bbResDir = _voc.resDir + string(format("BBoxesB%gW%d%s/", _base, _W, _clrName[_Clr]).c_str());
+}
 
-ObjectnessBING::ObjectnessBING()
+int ObjectnessBING::loadTrainedModel(string modelName) // Return -1, 0, or 1 if partial, none, or all loaded
 {
-  className = "BING";
+    if (modelName.size() == 0)
+        modelName = _modelName;
+    CStr s1 = modelName + ".wS1", s2 = modelName + ".wS2", sI = modelName + ".idx";
+    Mat filters1f, reW1f, idx1i, show3u;
+    if (!matRead(s1, filters1f) || !matRead(sI, idx1i)){
+        printf("Can't load model: %s or %s\n", _S(s1), _S(sI));
+        return 0;
+    }
+
+    //filters1f = aFilter(0.8f, 8);
+    //normalize(filters1f, filters1f, p, 1, NORM_MINMAX);
+
+    normalize(filters1f, show3u, 1, 255, NORM_MINMAX, CV_8U);
+    _tigF.update(filters1f);
+    //_tigF.reconstruct(filters1f);
+
+    _svmSzIdxs = idx1i;
+    CV_Assert(_svmSzIdxs.size() > 1 && filters1f.size() == Size(_W, _W) && filters1f.type() == CV_32F);
+    _svmFilter = filters1f;
+
+    if (!matRead(s2, _svmReW1f) || _svmReW1f.size() != Size(2, _svmSzIdxs.size())){
+        _svmReW1f = Mat();
+        return -1;
+    }
+    return 1;
 }
 
-ObjectnessBING::~ObjectnessBING()
+void ObjectnessBING::predictBBoxSI(CMat &img3u, ValStructVec<float, Vec4i> &valBoxes, vecI &sz, int NUM_WIN_PSZ, bool fast)
+{
+    const int numSz = _svmSzIdxs.size();
+    const int imgW = img3u.cols, imgH = img3u.rows;
+    valBoxes.reserve(10000);
+    sz.clear(); sz.reserve(10000);
+    for (int ir = numSz - 1; ir >= 0; ir--){
+        int r = _svmSzIdxs[ir];
+        int height = cvRound(pow(_base, r/_numT + _minT)), width = cvRound(pow(_base, r%_numT + _minT));
+        if (height > imgH * _base || width > imgW * _base)
+            continue;
+
+        height = min(height, imgH), width = min(width, imgW);
+        Mat im3u, matchCost1f, mag1u;
+        resize(img3u, im3u, Size(cvRound(_W*imgW*1.0/width), cvRound(_W*imgH*1.0/height)));
+        gradientMag(im3u, mag1u);
+
+        //imwrite(_voc.localDir + format("%d.png", r), mag1u);
+        //Mat mag1f;
+        //mag1u.convertTo(mag1f, CV_32F);
+        //matchTemplate(mag1f, _svmFilter, matchCost1f, CV_TM_CCORR);
+
+        matchCost1f = _tigF.matchTemplate(mag1u);
+
+        ValStructVec<float, Point> matchCost;
+        nonMaxSup(matchCost1f, matchCost, _NSS, NUM_WIN_PSZ, fast);
+
+        // Find true locations and match values
+        double ratioX = width/_W, ratioY = height/_W;
+        int iMax = min(matchCost.size(), NUM_WIN_PSZ);
+        for (int i = 0; i < iMax; i++){
+            float mVal = matchCost(i);
+            Point pnt = matchCost[i];
+            Vec4i box(cvRound(pnt.x * ratioX), cvRound(pnt.y*ratioY));
+            box[2] = cvRound(min(box[0] + width, imgW));
+            box[3] = cvRound(min(box[1] + height, imgH));
+            box[0] ++;
+            box[1] ++;
+            valBoxes.pushBack(mVal, box);
+            sz.push_back(ir);
+        }
+    }
+
+}
+
+void ObjectnessBING::predictBBoxSII(ValStructVec<float, Vec4i> &valBoxes, const vecI &sz)
 {
+    int numI = valBoxes.size();
+    for (int i = 0; i < numI; i++){
+        const float* svmIIw = _svmReW1f.ptr<float>(sz[i]);
+        valBoxes(i) = valBoxes(i) * svmIIw[0] + svmIIw[1];
+    }
+    valBoxes.sort();
+}
+
+// Get potential bounding boxes, each of which is represented by a Vec4i for (minX, minY, maxX, maxY).
+// The trained model should be prepared before calling this function: loadTrainedModel() or trainStageI() + trainStageII().
+// Use numDet to control the final number of proposed bounding boxes, and number of per size (scale and aspect ratio)
+void ObjectnessBING::getObjBndBoxes(CMat &img3u, ValStructVec<float, Vec4i> &valBoxes, int numDetPerSize)
+{
+    //CV_Assert_(filtersLoaded() , ("SVM filters should be initialized before getting object proposals\n"));
+    vecI sz;
+    predictBBoxSI(img3u, valBoxes, sz, numDetPerSize, false);
+    predictBBoxSII(valBoxes, sz);
+    return;
+}
+
+void ObjectnessBING::nonMaxSup(CMat &matchCost1f, ValStructVec<float, Point> &matchCost, int NSS, int maxPoint, bool fast)
+{
+    const int _h = matchCost1f.rows, _w = matchCost1f.cols;
+    Mat isMax1u = Mat::ones(_h, _w, CV_8U), costSmooth1f;
+    ValStructVec<float, Point> valPnt;
+    matchCost.reserve(_h * _w);
+    valPnt.reserve(_h * _w);
+    if (fast){
+        blur(matchCost1f, costSmooth1f, Size(3, 3));
+        for (int r = 0; r < _h; r++){
+            const float* d = matchCost1f.ptr<float>(r);
+            const float* ds = costSmooth1f.ptr<float>(r);
+            for (int c = 0; c < _w; c++)
+                if (d[c] >= ds[c])
+                    valPnt.pushBack(d[c], Point(c, r));
+        }
+    }
+    else{
+        for (int r = 0; r < _h; r++){
+            const float* d = matchCost1f.ptr<float>(r);
+            for (int c = 0; c < _w; c++)
+                valPnt.pushBack(d[c], Point(c, r));
+        }
+    }
+
+    valPnt.sort();
+    for (int i = 0; i < valPnt.size(); i++){
+        Point &pnt = valPnt[i];
+        if (isMax1u.at<byte>(pnt)){
+            matchCost.pushBack(valPnt(i), pnt);
+            for (int dy = -NSS; dy <= NSS; dy++) for (int dx = -NSS; dx <= NSS; dx++){
+                Point neighbor = pnt + Point(dx, dy);
+                if (!CHK_IND(neighbor))
+                    continue;
+                isMax1u.at<byte>(neighbor) = false;
+            }
+        }
+        if (matchCost.size() >= maxPoint)
+            return;
+    }
+}
+
+void ObjectnessBING::gradientMag(CMat &imgBGR3u, Mat &mag1u)
+{
+    switch (_Clr){
+    case MAXBGR:
+        gradientRGB(imgBGR3u, mag1u); break;
+    case G:
+        gradientGray(imgBGR3u, mag1u); break;
+    case HSV:
+        gradientHSV(imgBGR3u, mag1u); break;
+    default:
+        printf("Error: not recognized color space\n");
+    }
+}
+
+void ObjectnessBING::gradientRGB(CMat &bgr3u, Mat &mag1u)
+{
+    const int H = bgr3u.rows, W = bgr3u.cols;
+    Mat Ix(H, W, CV_32S), Iy(H, W, CV_32S);
+
+    // Left/right most column Ix
+    for (int y = 0; y < H; y++){
+        Ix.at<int>(y, 0) = bgrMaxDist(bgr3u.at<Vec3b>(y, 1), bgr3u.at<Vec3b>(y, 0))*2;
+        Ix.at<int>(y, W-1) = bgrMaxDist(bgr3u.at<Vec3b>(y, W-1), bgr3u.at<Vec3b>(y, W-2))*2;
+    }
+
+    // Top/bottom most column Iy
+    for (int x = 0; x < W; x++) {
+        Iy.at<int>(0, x) = bgrMaxDist(bgr3u.at<Vec3b>(1, x), bgr3u.at<Vec3b>(0, x))*2;
+        Iy.at<int>(H-1, x) = bgrMaxDist(bgr3u.at<Vec3b>(H-1, x), bgr3u.at<Vec3b>(H-2, x))*2;
+    }
+
+    // Find the gradient for inner regions
+    for (int y = 0; y < H; y++){
+        const Vec3b *dataP = bgr3u.ptr<Vec3b>(y);
+        for (int x = 2; x < W; x++)
+            Ix.at<int>(y, x-1) = bgrMaxDist(dataP[x-2], dataP[x]); //  bgr3u.at<Vec3b>(y, x+1), bgr3u.at<Vec3b>(y, x-1));
+    }
+    for (int y = 1; y < H-1; y++){
+        const Vec3b *tP = bgr3u.ptr<Vec3b>(y-1);
+        const Vec3b *bP = bgr3u.ptr<Vec3b>(y+1);
+        for (int x = 0; x < W; x++)
+            Iy.at<int>(y, x) = bgrMaxDist(tP[x], bP[x]);
+    }
+    gradientXY(Ix, Iy, mag1u);
+}
+
+void ObjectnessBING::gradientGray(CMat &bgr3u, Mat &mag1u)
+{
+    Mat g1u;
+    cvtColor(bgr3u, g1u, COLOR_BGR2GRAY);
+    const int H = g1u.rows, W = g1u.cols;
+    Mat Ix(H, W, CV_32S), Iy(H, W, CV_32S);
+
+    // Left/right most column Ix
+    for (int y = 0; y < H; y++){
+        Ix.at<int>(y, 0) = abs(g1u.at<byte>(y, 1) - g1u.at<byte>(y, 0)) * 2;
+        Ix.at<int>(y, W-1) = abs(g1u.at<byte>(y, W-1) - g1u.at<byte>(y, W-2)) * 2;
+    }
+
+    // Top/bottom most column Iy
+    for (int x = 0; x < W; x++) {
+        Iy.at<int>(0, x) = abs(g1u.at<byte>(1, x) - g1u.at<byte>(0, x)) * 2;
+        Iy.at<int>(H-1, x) = abs(g1u.at<byte>(H-1, x) - g1u.at<byte>(H-2, x)) * 2;
+    }
+
+    // Find the gradient for inner regions
+    for (int y = 0; y < H; y++)
+        for (int x = 1; x < W-1; x++)
+            Ix.at<int>(y, x) = abs(g1u.at<byte>(y, x+1) - g1u.at<byte>(y, x-1));
+    for (int y = 1; y < H-1; y++)
+        for (int x = 0; x < W; x++)
+            Iy.at<int>(y, x) = abs(g1u.at<byte>(y+1, x) - g1u.at<byte>(y-1, x));
 
+    gradientXY(Ix, Iy, mag1u);
+}
+
+
+void ObjectnessBING::gradientHSV(CMat &bgr3u, Mat &mag1u)
+{
+    Mat hsv3u;
+    cvtColor(bgr3u, hsv3u, COLOR_BGR2HSV);
+    const int H = hsv3u.rows, W = hsv3u.cols;
+    Mat Ix(H, W, CV_32S), Iy(H, W, CV_32S);
+
+    // Left/right most column Ix
+    for (int y = 0; y < H; y++){
+        Ix.at<int>(y, 0) = vecDist3b(hsv3u.at<Vec3b>(y, 1), hsv3u.at<Vec3b>(y, 0));
+        Ix.at<int>(y, W-1) = vecDist3b(hsv3u.at<Vec3b>(y, W-1), hsv3u.at<Vec3b>(y, W-2));
+    }
+
+    // Top/bottom most column Iy
+    for (int x = 0; x < W; x++) {
+        Iy.at<int>(0, x) = vecDist3b(hsv3u.at<Vec3b>(1, x), hsv3u.at<Vec3b>(0, x));
+        Iy.at<int>(H-1, x) = vecDist3b(hsv3u.at<Vec3b>(H-1, x), hsv3u.at<Vec3b>(H-2, x));
+    }
+
+    // Find the gradient for inner regions
+    for (int y = 0; y < H; y++)
+        for (int x = 1; x < W-1; x++)
+            Ix.at<int>(y, x) = vecDist3b(hsv3u.at<Vec3b>(y, x+1), hsv3u.at<Vec3b>(y, x-1))/2;
+    for (int y = 1; y < H-1; y++)
+        for (int x = 0; x < W; x++)
+            Iy.at<int>(y, x) = vecDist3b(hsv3u.at<Vec3b>(y+1, x), hsv3u.at<Vec3b>(y-1, x))/2;
+
+    gradientXY(Ix, Iy, mag1u);
+}
+
+void ObjectnessBING::gradientXY(CMat &x1i, CMat &y1i, Mat &mag1u)
+{
+    const int H = x1i.rows, W = x1i.cols;
+    mag1u.create(H, W, CV_8U);
+    for (int r = 0; r < H; r++){
+        const int *x = x1i.ptr<int>(r), *y = y1i.ptr<int>(r);
+        byte* m = mag1u.ptr<byte>(r);
+        for (int c = 0; c < W; c++)
+            m[c] = min(x[c] + y[c], 255);   //((int)sqrt(sqr(x[c]) + sqr(y[c])), 255);
+    }
+}
+
+void ObjectnessBING::getObjBndBoxesForImage(Mat img, ValStructVec<float, Vec4i> &finalBoxes, int numDetPerSize)
+{
+    const int TestNum = 1;
+    vecM imgs3u(TestNum);
+    vector<ValStructVec<float, Vec4i>> boxesTests;
+    boxesTests.resize(TestNum);
+
+
+#pragma omp parallel for
+    for (int i = 0; i < TestNum; i++){
+        imgs3u[i] = img;
+        boxesTests[i].reserve(10000);
+    }
+
+    int scales[3] = {1, 3, 5};
+    for (int clr = MAXBGR; clr <= G; clr++){
+        setColorSpace(clr);
+        loadTrainedModel();
+        CmTimer tm("Predict");
+        //tm.Start();
+
+#pragma omp parallel for
+        for (int i = 0; i < TestNum; i++){
+            ValStructVec<float, Vec4i> boxes;
+            getObjBndBoxes(imgs3u[i], boxes, numDetPerSize);
+            boxesTests[i].append(boxes, scales[clr]);
+
+            //boxes.valIdxes[0].first;
+        }
+
+        //tm.Stop();
+        //printf("Average time for predicting an image (%s) is %gs\n", _clrName[_Clr], tm.TimeInSeconds()/TestNum);
+    }
+
+/*    _boxesTests.resize(TestNum);
+    CmFile::MkDir(_bbResDir);
+#pragma omp parallel for
+    for (int i = 0; i < TestNum; i++){
+
+        CStr fName = _bbResDir + _voc.testSet[i];
+        ValStructVec<float, Vec4i> &boxes = boxesTests[i];
+        FILE *f = fopen(_S(fName + ".txt"), "w");
+        fprintf(f, "%d\n", boxes.size());
+        for (size_t k = 0; k < boxes.size(); k++)
+            //fprintf(f, "%g, %s\n", boxes(k), _S(strVec4i(boxes[k])));
+        fclose(f);
+
+        _boxesTests[i].resize(boxesTests[i].size());
+        for (int j = 0; j < boxesTests[i].size(); j++)
+            _boxesTests[i][j] = boxesTests[i][j];
+    } */
+
+    //evaluatePerImgRecall(_boxesTests, "PerImgAllNS.m", 5000);
+
+#pragma omp parallel for
+    for (int i = 0; i < TestNum; i++){
+        boxesTests[i].sort(false);
+        //for (int j = 0; j < boxesTests[i].size(); j++)
+          //  _boxesTests[i][j] = boxesTests[i][j];
+        finalBoxes=boxesTests[i];
+
+    }
+    //evaluatePerImgRecall(_boxesTests, "PerImgAllS.m", 5000);
+
+}
+
+struct MatchPathSeparator
+{
+    bool operator()( char ch ) const
+    {
+        return ch == '/';
+    }
+};
+
+std::string inline basename( std::string const& pathname )
+{
+    return std::string(
+        std::find_if( pathname.rbegin(), pathname.rend(),
+                      MatchPathSeparator() ).base(),
+        pathname.end() );
+}
+
+std::string inline removeExtension( std::string const& filename )
+{
+    std::string::const_reverse_iterator
+                        pivot
+            = std::find( filename.rbegin(), filename.rend(), '.' );
+    return pivot == filename.rend()
+        ? filename
+        : std::string( filename.begin(), pivot.base() - 1 );
+}
+
+
+// Read matrix from binary file
+bool ObjectnessBING::matRead(const string& filename, Mat& _M){
+
+    FileStorage fs2(filename+".yml.gz", FileStorage::READ);
+    String fileNameString(filename.c_str());
+
+    Mat M;
+    fs2[removeExtension(basename(fileNameString))]>> M;
+
+    /*FILE* f = fopen(_S(filename), "rb");
+    if (f == NULL)
+        return false;
+    char buf[8];
+    int pre = fread(buf,sizeof(char), 5, f);
+    if (strncmp(buf, "CmMat", 5) != 0)  {
+        printf("Invalidate CvMat data file %s\n", _S(filename));
+        return false;
+    }
+    int headData[3]; // Width, height, type
+    fread(headData, sizeof(int), 3, f);
+    Mat M(headData[1], headData[0], headData[2]);
+    fread(M.data, sizeof(char), M.step * M.rows, f);
+    fclose(f); */
+
+    M.copyTo(_M);
+    return true;
 }
 
 void ObjectnessBING::read( const cv::FileNode& fn )
@@ -89,6 +470,13 @@ void ObjectnessBING::write( cv::FileStorage& fs ) const
 
 bool ObjectnessBING::computeSaliencyImpl( const InputArray& src, OutputArray& dst )
 {
+  ValStructVec<float, Vec4i> &finalBoxes;
+  getObjBndBoxesForImage( src.getMat(), finalBoxes, 250 );
+
+  // list of rectangles returned by objectess function
+  //std::vector<Vec4i> boxesList = finalBoxes.structVals;
+  dst = finalBoxes.getSortedStructVal();
+  //dst = finalBoxes.structVals;
 
   return true;
 }