Implementation of SelectiveSearchSegmentation

modules/ximgproc/README.md

@@ -8,3 +8,4 @@ Extended Image Processing
 5. Joint Bilateral Filter
 6. Superpixels
 7. Graph segmentation
+8. Selective search from segmentation

modules/ximgproc/doc/ximgproc.bib

@@ -67,6 +67,18 @@
   publisher={Springer}
 }
 
+
+@article{uijlings2013selective,
+  title={Selective search for object recognition},
+  author={Uijlings, Jasper RR and van de Sande, Koen EA and Gevers, Theo and Smeulders, Arnold WM},
+  journal={International journal of computer vision},
+  volume={104},
+  number={2},
+  pages={154--171},
+  year={2013},
+  publisher={Springer}
+}
+
 @article{Min2014,
   title={Fast global image smoothing based on weighted least squares},
   author={Min, Dongbo and Choi, Sunghwan and Lu, Jiangbo and Ham, Bumsub and Sohn, Kwanghoon and Do, Minh N},

modules/ximgproc/include/opencv2/ximgproc/segmentation.hpp

@@ -67,55 +67,183 @@ namespace cv {
                         @param min_size The minimum size of segments
                      */
                     CV_EXPORTS_W Ptr<GraphSegmentation> createGraphSegmentation(double sigma=0.5, float k=300, int min_size=100);
-            //! @}
 
-            // Represent an edge between two pixels
-            class Edge {
-                public:
-                    int from;
-                    int to;
-                    float weight;
+                    /** @brief Strategie for the selective search segmentation algorithm
+                        The class implements a generic stragery for the algorithm described in @cite uijlings2013selective.
+                     */
+                    class CV_EXPORTS_W SelectiveSearchSegmentationStrategy : public Algorithm {
+                        public:
+                            /** @brief Set a initial image, with a segementation.
+                                @param img The input image. Any number of channel can be provided
+                                @param regions A segementation of the image. The parameter must be the same size of img.
+                                @param sizes The sizes of different regions
+                                @param image_id If not set to -1, try to cache pre-computations. If the same set og (img, regions, size) is used, the image_id need to be the same.
+                            */
+                            CV_WRAP virtual void setImage(InputArray img, InputArray regions, InputArray sizes, int image_id = -1) = 0;
 
-                    bool operator <(const Edge& e) const {
-                        return weight < e.weight;
-                    }
-            };
+                            /** @brief Return the score between two regions (between 0 and 1)
+                                @param r1 The first region
+                                @param r2 The second region
+                            */
+                            CV_WRAP virtual float get(int r1, int r2) = 0;
 
-            // A point in the sets of points
-            class PointSetElement {
-                public:
-                    int p;
-                    int size;
+                            /** @brief Inform the strategy that two regions will be merged
+                                @param r1 The first region
+                                @param r2 The second region
+                            */
+                            CV_WRAP virtual void merge(int r1, int r2) = 0;
+                    };
+
+                    /** @brief Color-based strategy for the selective search segmentation algorithm
+                        The class is implemented from the algorithm described in @cite uijlings2013selective.
+                     */
+                    class CV_EXPORTS_W SelectiveSearchSegmentationStrategyColor : public SelectiveSearchSegmentationStrategy {
+                    };
+
+                    /** @brief Create a new color-based strategy */
+                    CV_EXPORTS_W Ptr<SelectiveSearchSegmentationStrategyColor> createSelectiveSearchSegmentationStrategyColor();
+
+                    /** @brief Size-based strategy for the selective search segmentation algorithm
+                        The class is implemented from the algorithm described in @cite uijlings2013selective.
+                     */
+                    class CV_EXPORTS_W SelectiveSearchSegmentationStrategySize : public SelectiveSearchSegmentationStrategy {
+                    };
 
-                    PointSetElement() { }
+                    /** @brief Create a new size-based strategy */
+                    CV_EXPORTS_W Ptr<SelectiveSearchSegmentationStrategySize> createSelectiveSearchSegmentationStrategySize();
 
-                    PointSetElement(int p_) {
-                        p = p_;
-                        size = 1;
-                    }
-            };
+                    /** @brief Texture-based strategy for the selective search segmentation algorithm
+                        The class is implemented from the algorithm described in @cite uijlings2013selective.
+                     */
+                    class CV_EXPORTS_W SelectiveSearchSegmentationStrategyTexture : public SelectiveSearchSegmentationStrategy {
+                    };
 
-            // An object to manage set of points, who can be fusionned
-            class PointSet {
-                public:
-                    PointSet(int nb_elements_);
-                    ~PointSet();
+                    /** @brief Create a new size-based strategy */
+                    CV_EXPORTS_W Ptr<SelectiveSearchSegmentationStrategyTexture> createSelectiveSearchSegmentationStrategyTexture();
 
-                    int nb_elements;
+                    /** @brief Fill-based strategy for the selective search segmentation algorithm
+                        The class is implemented from the algorithm described in @cite uijlings2013selective.
+                     */
+                    class CV_EXPORTS_W SelectiveSearchSegmentationStrategyFill : public SelectiveSearchSegmentationStrategy {
+                    };
 
-                    // Return the main point of the point's set
-                    int getBasePoint(int p);
+                    /** @brief Create a new fill-based strategy */
+                    CV_EXPORTS_W Ptr<SelectiveSearchSegmentationStrategyFill> createSelectiveSearchSegmentationStrategyFill();
 
-                    // Join two sets of points, based on their main point
-                    void joinPoints(int p_a, int p_b);
+                    /** @brief Regroup multiple strategies for the selective search segmentation algorithm
+                     */
+                    class CV_EXPORTS_W SelectiveSearchSegmentationStrategyMultiple : public SelectiveSearchSegmentationStrategy {
+                        public:
 
-                    // Return the set size of a set (based on the main point)
-                    int size(unsigned int p) { return mapping[p].size; }
+                            /** @brief Add a new sub-strategy
+                                @param g The strategy
+                                @param weight The weight of the strategy
+                            */
+                            CV_WRAP virtual void addStrategy(Ptr<SelectiveSearchSegmentationStrategy> g, float weight) = 0;
+                            /** @brief Remove all sub-strategies
+                            */
+                            CV_WRAP virtual void clearStrategies() = 0;
+                    };
 
-                private:
-                    PointSetElement* mapping;
+                    /** @brief Create a new multiple strategy */
+                    CV_EXPORTS_W Ptr<SelectiveSearchSegmentationStrategyMultiple> createSelectiveSearchSegmentationStrategyMultiple();
+
+                    /** @brief Create a new multiple strategy and set one subtrategy
+                        @param s1 The first strategy
+                    */
+                    CV_EXPORTS_W Ptr<SelectiveSearchSegmentationStrategyMultiple> createSelectiveSearchSegmentationStrategyMultiple(Ptr<SelectiveSearchSegmentationStrategy> s1);
+
+                    /** @brief Create a new multiple strategy and set two subtrategies, with equal weights
+                        @param s1 The first strategy
+                        @param s2 The second strategy
+                    */
+                    CV_EXPORTS_W Ptr<SelectiveSearchSegmentationStrategyMultiple> createSelectiveSearchSegmentationStrategyMultiple(Ptr<SelectiveSearchSegmentationStrategy> s1, Ptr<SelectiveSearchSegmentationStrategy> s2);
+
+
+                    /** @brief Create a new multiple strategy and set three subtrategies, with equal weights
+                        @param s1 The first strategy
+                        @param s2 The second strategy
+                        @param s3 The third strategy
+                    */
+                    CV_EXPORTS_W Ptr<SelectiveSearchSegmentationStrategyMultiple> createSelectiveSearchSegmentationStrategyMultiple(Ptr<SelectiveSearchSegmentationStrategy> s1, Ptr<SelectiveSearchSegmentationStrategy> s2, Ptr<SelectiveSearchSegmentationStrategy> s3);
+
+                    /** @brief Create a new multiple strategy and set four subtrategies, with equal weights
+                        @param s1 The first strategy
+                        @param s2 The second strategy
+                        @param s3 The third strategy
+                        @param s4 The forth strategy
+                    */
+                    CV_EXPORTS_W Ptr<SelectiveSearchSegmentationStrategyMultiple> createSelectiveSearchSegmentationStrategyMultiple(Ptr<SelectiveSearchSegmentationStrategy> s1, Ptr<SelectiveSearchSegmentationStrategy> s2, Ptr<SelectiveSearchSegmentationStrategy> s3, Ptr<SelectiveSearchSegmentationStrategy> s4);
+
+                    /** @brief Selective search segmentation algorithm
+                        The class implements the algorithm described in @cite uijlings2013selective.
+                     */
+                    class CV_EXPORTS_W SelectiveSearchSegmentation : public Algorithm {
+                        public:
 
-            };
+                            /** @brief Set a image used by switch* functions to initialize the class
+                                @param img The image
+                            */
+                            CV_WRAP virtual void setBaseImage(InputArray img) = 0;
+
+                            /** @brief Initialize the class with the 'Single stragegy' parameters describled in @cite uijlings2013selective.
+                                @param k The k parameter for the graph segmentation
+                                @param sigma The sigma parameter for the graph segmentation
+                            */
+                            CV_WRAP virtual void switchToSingleStrategy(int k = 200, float sigma = 0.8f) = 0;
+
+                            /** @brief Initialize the class with the 'Selective search fast' parameters describled in @cite uijlings2013selective.
+                                @param base_k The k parameter for the first graph segmentation
+                                @param inc_k The increment of the k parameter for all graph segmentations
+                                @param sigma The sigma parameter for the graph segmentation
+                            */
+                            CV_WRAP virtual void switchToSelectiveSearchFast(int base_k = 150, int inc_k = 150, float sigma = 0.8f) = 0;
+
+                            /** @brief Initialize the class with the 'Selective search fast' parameters describled in @cite uijlings2013selective.
+                                @param base_k The k parameter for the first graph segmentation
+                                @param inc_k The increment of the k parameter for all graph segmentations
+                                @param sigma The sigma parameter for the graph segmentation
+                            */
+                            CV_WRAP virtual void switchToSelectiveSearchQuality(int base_k = 150, int inc_k = 150, float sigma = 0.8f) = 0;
+
+                            /** @brief Add a new image in the list of images to process.
+                                @param img The image
+                            */
+                            CV_WRAP virtual void addImage(InputArray img) = 0;
+
+                            /** @brief Clear the list of images to process
+                            */
+                            CV_WRAP virtual void clearImages() = 0;
+
+                            /** @brief Add a new graph segmentation in the list of graph segementations to process.
+                                @param g The graph segmentation
+                            */
+                            CV_WRAP virtual void addGraphSegmentation(Ptr<GraphSegmentation> g) = 0;
+
+                            /** @brief Clear the list of graph segmentations to process;
+                            */
+                            CV_WRAP virtual void clearGraphSegmentations() = 0;
+
+                            /** @brief Add a new strategy in the list of strategy to process.
+                                @param s The strategy
+                            */
+                            CV_WRAP virtual void addStrategy(Ptr<SelectiveSearchSegmentationStrategy> s) = 0;
+
+                            /** @brief Clear the list of strategy to process;
+                            */
+                            CV_WRAP virtual void clearStrategies() = 0;
+
+                            /** @brief Based on all images, graph segmentations and stragies, computes all possible rects and return them
+                                @param rects The list of rects. The first ones are more relevents than the lasts ones.
+                            */
+                            CV_WRAP virtual void process(std::vector<Rect>& rects) = 0;
+                    };
+
+                    /** @brief Create a new SelectiveSearchSegmentation class.
+                     */
+                    CV_EXPORTS_W Ptr<SelectiveSearchSegmentation> createSelectiveSearchSegmentation();
+
+            //! @}
 
         }
     }

modules/ximgproc/samples/graphsegmentation_demo.cpp

@@ -32,13 +32,16 @@ the use of this software, even if advised of the possibility of such damage.
 
 #include "opencv2/ximgproc/segmentation.hpp"
 #include "opencv2/highgui.hpp"
-#include <opencv2/core/utility.hpp>
-#include <opencv2/opencv.hpp>
+#include "opencv2/core.hpp"
+#include "opencv2/imgproc.hpp"
 #include <iostream>
 
 using namespace cv;
 using namespace cv::ximgproc::segmentation;
 
+Scalar hsv_to_rgb(Scalar);
+Scalar color_mapping(int);
+
 static void help() {
     std::cout << std::endl <<
     "A program demonstrating the use and capabilities of a particular graph based image" << std::endl <<
@@ -55,9 +58,9 @@ Scalar hsv_to_rgb(Scalar c) {
 
     float * p = in.ptr<float>(0);
 
-    p[0] = c[0] * 360;
-    p[1] = c[1];
-    p[2] = c[2];
+    p[0] = (float)c[0] * 360.0f;
+    p[1] = (float)c[1];
+    p[2] = (float)c[2];
 
     cvtColor(in, out, COLOR_HSV2RGB);
 
@@ -97,7 +100,7 @@ int main(int argc, char** argv) {
         gs->setSigma(atof(argv[3]));
 
     if (argc > 4)
-        gs->setK(atoi(argv[4]));
+        gs->setK((float)atoi(argv[4]));
 
     if (argc > 5)
         gs->setMinSize(atoi(argv[5]));
@@ -137,9 +140,9 @@ int main(int argc, char** argv) {
 
         for (int j = 0; j < output.cols; j++) {
             Scalar color = color_mapping(p[j]);
-            p2[j*3] = color[0];
-            p2[j*3 + 1] = color[1];
-            p2[j*3 + 2] = color[2];
+            p2[j*3] = (uchar)color[0];
+            p2[j*3 + 1] = (uchar)color[1];
+            p2[j*3 + 2] = (uchar)color[2];
         }
     }
 

modules/ximgproc/samples/selectivesearchsegmentation_demo.cpp

@@ -0,0 +1,115 @@
+/*
+By downloading, copying, installing or using the software you agree to this
+license. If you do not agree to this license, do not download, install,
+copy or use the software.
+                          License Agreement
+               For Open Source Computer Vision Library
+                       (3-clause BSD License)
+Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+Third party copyrights are property of their respective owners.
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+  * Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+  * Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+  * Neither the names of the copyright holders nor the names of the contributors
+    may be used to endorse or promote products derived from this software
+    without specific prior written permission.
+This software is provided by the copyright holders and contributors "as is" and
+any express or implied warranties, including, but not limited to, the implied
+warranties of merchantability and fitness for a particular purpose are
+disclaimed. In no event shall copyright holders or contributors be liable for
+any direct, indirect, incidental, special, exemplary, or consequential damages
+(including, but not limited to, procurement of substitute goods or services;
+loss of use, data, or profits; or business interruption) however caused
+and on any theory of liability, whether in contract, strict liability,
+or tort (including negligence or otherwise) arising in any way out of
+the use of this software, even if advised of the possibility of such damage.
+*/
+
+
+#include "opencv2/ximgproc/segmentation.hpp"
+#include "opencv2/highgui.hpp"
+#include "opencv2/core.hpp"
+#include "opencv2/imgproc.hpp"
+#include <iostream>
+#include <ctime>
+
+using namespace cv;
+using namespace cv::ximgproc::segmentation;
+
+static void help() {
+    std::cout << std::endl <<
+    "A program demonstrating the use and capabilities of a particular image segmentation algorithm described" << std::endl <<
+    " in Jasper R. R. Uijlings, Koen E. A. van de Sande, Theo Gevers, Arnold W. M. Smeulders: " << std::endl <<
+    "                       \"Selective Search for Object Recognition\"" << std::endl <<
+    "International Journal of Computer Vision, Volume 104 (2), page 154-171, 2013" << std::endl << std::endl <<
+    "Usage:" << std::endl <<
+    "./selectivesearchsegmentation_demo input_image (single|fast|quality)" << std::endl <<
+    "Use a to display less rects, d to display more rects, q to quit" << std::endl;
+}
+
+
+int main(int argc, char** argv) {
+
+    if (argc < 3) {
+        help();
+        return -1;
+    }
+
+    setUseOptimized(true);
+    setNumThreads(8);
+
+    std::srand((int)std::time(0));
+
+    Mat img = imread(argv[1]);
+
+    Ptr<SelectiveSearchSegmentation> gs = createSelectiveSearchSegmentation();
+    gs->setBaseImage(img);
+
+    if (argv[2][0] == 's') {
+        gs->switchToSingleStrategy();
+    } else if (argv[2][0] == 'f') {
+        gs->switchToSelectiveSearchFast();
+    } else if (argv[2][0] == 'q') {
+        gs->switchToSelectiveSearchQuality();
+    } else {
+        help();
+        return -2;
+    }
+
+    std::vector<Rect> rects;
+    gs->process(rects);
+
+    int nb_rects = 10;
+
+    char c = (char)waitKey();
+
+    while(c != 'q') {
+
+        Mat wimg = img.clone();
+
+        int i = 0;
+
+        for(std::vector<Rect>::iterator it = rects.begin(); it != rects.end(); ++it) {
+            if (i++ < nb_rects) {
+                rectangle(wimg, *it, Scalar(0, 0, 255));
+            }
+        }
+
+        imshow("Output", wimg);
+        c = (char)waitKey();
+
+        if (c == 'd') {
+            nb_rects += 10;
+        }
+
+        if (c == 'a' && nb_rects > 10) {
+            nb_rects -= 10;
+        }
+    }
+
+    return 0;
+}

modules/ximgproc/src/graphsegmentation.cpp

@@ -47,6 +47,56 @@ namespace cv {
     namespace ximgproc {
         namespace segmentation {
 
+            // Helpers
+
+            // Represent an edge between two pixels
+            class Edge {
+                public:
+                    int from;
+                    int to;
+                    float weight;
+
+                    bool operator <(const Edge& e) const {
+                        return weight < e.weight;
+                    }
+            };
+
+            // A point in the sets of points
+            class PointSetElement {
+                public:
+                    int p;
+                    int size;
+
+                    PointSetElement() { }
+
+                    PointSetElement(int p_) {
+                        p = p_;
+                        size = 1;
+                    }
+            };
+
+            // An object to manage set of points, who can be fusionned
+            class PointSet {
+                public:
+                    PointSet(int nb_elements_);
+                    ~PointSet();
+
+                    int nb_elements;
+
+                    // Return the main point of the point's set
+                    int getBasePoint(int p);
+
+                    // Join two sets of points, based on their main point
+                    void joinPoints(int p_a, int p_b);
+
+                    // Return the set size of a set (based on the main point)
+                    int size(unsigned int p) { return mapping[p].size; }
+
+                private:
+                    PointSetElement* mapping;
+
+            };
+
             class GraphSegmentationImpl : public GraphSegmentation {
                 public:
                     GraphSegmentationImpl() {