From b754372ce56deebc80c53712fbf5aa32f696c7ff Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 3 Nov 2015 17:14:00 +0100 Subject: [PATCH 1/3] Rewrote GraphSegmentation in ximgproc --- modules/ximgproc/doc/ximgproc.bib | 11 + modules/ximgproc/include/opencv2/ximgproc.hpp | 3 + .../include/opencv2/ximgproc/segmentation.hpp | 124 +++++++ .../samples/cpp/graphsegmentation_demo.cpp | 151 ++++++++ modules/ximgproc/src/graphsegmentation.cpp | 332 ++++++++++++++++++ modules/ximgproc/src/precomp.hpp | 9 +- 6 files changed, 627 insertions(+), 3 deletions(-) create mode 100644 modules/ximgproc/include/opencv2/ximgproc/segmentation.hpp create mode 100644 modules/ximgproc/samples/cpp/graphsegmentation_demo.cpp create mode 100644 modules/ximgproc/src/graphsegmentation.cpp diff --git a/modules/ximgproc/doc/ximgproc.bib b/modules/ximgproc/doc/ximgproc.bib index 75a59331b..00dc7f57b 100644 --- a/modules/ximgproc/doc/ximgproc.bib +++ b/modules/ximgproc/doc/ximgproc.bib @@ -56,6 +56,17 @@ organization={IEEE} } +@incollection{PFF2004, + title={Efficient graph-based image segmentation}, + author={Felzenszwalb, Pedro F and Huttenlocher, Daniel P}, + journal={International Journal of Computer Vision}, + volume={59}, + number={2}, + pages={167--181}, + year={2004}, + publisher={Springer} +} + @article{Min2014, title={Fast global image smoothing based on weighted least squares}, author={Min, Dongbo and Choi, Sunghwan and Lu, Jiangbo and Ham, Bumsub and Sohn, Kwanghoon and Do, Minh N}, diff --git a/modules/ximgproc/include/opencv2/ximgproc.hpp b/modules/ximgproc/include/opencv2/ximgproc.hpp index cb55da078..41c0d5a50 100644 --- a/modules/ximgproc/include/opencv2/ximgproc.hpp +++ b/modules/ximgproc/include/opencv2/ximgproc.hpp @@ -42,6 +42,7 @@ #include "ximgproc/sparse_match_interpolator.hpp" #include "ximgproc/structured_edge_detection.hpp" #include "ximgproc/seeds.hpp" +#include "ximgproc/segmentation.hpp" #include "ximgproc/fast_hough_transform.hpp" /** @defgroup ximgproc Extended Image Processing @@ -54,6 +55,8 @@ which somehow takes into account pixel affinities in natural images. @defgroup ximgproc_filters Filters @defgroup ximgproc_superpixel Superpixels + + @defgroup ximgproc_segmentation Image segmentation @} */ diff --git a/modules/ximgproc/include/opencv2/ximgproc/segmentation.hpp b/modules/ximgproc/include/opencv2/ximgproc/segmentation.hpp new file mode 100644 index 000000000..a98848d67 --- /dev/null +++ b/modules/ximgproc/include/opencv2/ximgproc/segmentation.hpp @@ -0,0 +1,124 @@ +/* +By downloading, copying, installing or using the software you agree to this +license. If you do not agree to this license, do not download, install, +copy or use the software. + License Agreement + For Open Source Computer Vision Library + (3-clause BSD License) +Copyright (C) 2013, OpenCV Foundation, all rights reserved. +Third party copyrights are property of their respective owners. +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the names of the copyright holders nor the names of the contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. +This software is provided by the copyright holders and contributors "as is" and +any express or implied warranties, including, but not limited to, the implied +warranties of merchantability and fitness for a particular purpose are +disclaimed. In no event shall copyright holders or contributors be liable for +any direct, indirect, incidental, special, exemplary, or consequential damages +(including, but not limited to, procurement of substitute goods or services; +loss of use, data, or profits; or business interruption) however caused +and on any theory of liability, whether in contract, strict liability, +or tort (including negligence or otherwise) arising in any way out of +the use of this software, even if advised of the possibility of such damage. +*/ + +#ifndef __OPENCV_XIMGPROC_SEGMENTATION_HPP__ +#define __OPENCV_XIMGPROC_SEGMENTATION_HPP__ + +#include + +namespace cv { + namespace ximgproc { + namespace segmentation { + //! @addtogroup ximgproc_segmentation + //! @{ + + /** @brief Graph Based Segmentation Algorithm. + The class implements the algorithm described in @cite PFF2004 . + */ + class CV_EXPORTS_W GraphSegmentation : public Algorithm { + public: + /** @brief Segment an image and store output in dst + @param src The input image. Any number of channel (1 (Eg: Gray), 3 (Eg: RGB), 4 (Eg: RGB-D)) can be provided + @param dst The output segmentation. It's a CV_32SC1 Mat with the same number of cols and rows as input image, with an unique, sequential, id for each pixel. + */ + CV_WRAP virtual void processImage(InputArray src, OutputArray dst) = 0; + + CV_WRAP virtual void setSigma(double sigma) = 0; + CV_WRAP virtual double getSigma() = 0; + + CV_WRAP virtual void setK(float k) = 0; + CV_WRAP virtual float getK() = 0; + + CV_WRAP virtual void setMinSize(int min_size) = 0; + CV_WRAP virtual int getMinSize() = 0; + }; + + /** @brief Creates a graph based segmentor + @param simga The sigma parameter, used to smooth image + @param k The k parameter of the algorythm + @param min_size The minimum size of segments + */ + CV_EXPORTS_W Ptr createGraphSegmentation(double sigma=0.5, float k=300, int min_size=100); + //! @} + + // Represent an edge between two pixels + class Edge { + public: + int from; + int to; + float weight; + + bool operator <(const Edge& e) const { + return weight < e.weight; + } + }; + + // A point in the sets of points + class PointSetElement { + public: + int p; + int size; + + PointSetElement() { } + + PointSetElement(int p_) { + p = p_; + size = 1; + } + }; + + // An object to manage set of points, who can be fusionned + class PointSet { + public: + PointSet(int nb_elements_); + ~PointSet(); + + int nb_elements; + + // Return the main point of the point's set + int getBasePoint(int p); + + // Join two sets of points, based on their main point + void joinPoints(int p_a, int p_b); + + // Return the set size of a set (based on the main point) + int size(unsigned int p) { return mapping[p].size; } + + private: + PointSetElement* mapping; + + }; + + } + } +} + +#endif diff --git a/modules/ximgproc/samples/cpp/graphsegmentation_demo.cpp b/modules/ximgproc/samples/cpp/graphsegmentation_demo.cpp new file mode 100644 index 000000000..10e5e6dfa --- /dev/null +++ b/modules/ximgproc/samples/cpp/graphsegmentation_demo.cpp @@ -0,0 +1,151 @@ +/* +By downloading, copying, installing or using the software you agree to this +license. If you do not agree to this license, do not download, install, +copy or use the software. + License Agreement + For Open Source Computer Vision Library + (3-clause BSD License) +Copyright (C) 2013, OpenCV Foundation, all rights reserved. +Third party copyrights are property of their respective owners. +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the names of the copyright holders nor the names of the contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. +This software is provided by the copyright holders and contributors "as is" and +any express or implied warranties, including, but not limited to, the implied +warranties of merchantability and fitness for a particular purpose are +disclaimed. In no event shall copyright holders or contributors be liable for +any direct, indirect, incidental, special, exemplary, or consequential damages +(including, but not limited to, procurement of substitute goods or services; +loss of use, data, or profits; or business interruption) however caused +and on any theory of liability, whether in contract, strict liability, +or tort (including negligence or otherwise) arising in any way out of +the use of this software, even if advised of the possibility of such damage. +*/ + + +#include "opencv2/ximgproc/segmentation.hpp" +#include "opencv2/highgui.hpp" +#include +#include +#include + +using namespace cv; +using namespace cv::ximgproc::segmentation; + +static void help() { + std::cout << std::endl << + "A program demonstrating the use and capabilities of a particular graph based image" << std::endl << + "segmentation algorithm described in P. Felzenszwalb, D. Huttenlocher," << std::endl << + " \"Efficient Graph-Based Image Segmentation\"" << std::endl << + "International Journal of Computer Vision, Vol. 59, No. 2, September 2004" << std::endl << std::endl << + "Usage:" << std::endl << + "./graphsegmentation_demo input_image output_image [simga=0.5] [k=300] [min_size=100]" << std::endl; +} + +Scalar hsv_to_rgb(Scalar c) { + Mat in(1, 1, CV_32FC3); + Mat out(1, 1, CV_32FC3); + + float * p = in.ptr(0); + + p[0] = c[0] * 360; + p[1] = c[1]; + p[2] = c[2]; + + cvtColor(in, out, COLOR_HSV2RGB); + + Scalar t; + + Vec3f p2 = out.at(0, 0); + + t[0] = (int)(p2[0] * 255); + t[1] = (int)(p2[1] * 255); + t[2] = (int)(p2[2] * 255); + + return t; + +} + +Scalar color_mapping(int segment_id) { + + double base = (double)(segment_id) * 0.618033988749895 + 0.24443434; + + return hsv_to_rgb(Scalar(fmod(base, 1.2), 0.95, 0.80)); + +} + +int main(int argc, char** argv) { + + if (argc < 2 || argc > 6) { + help(); + return -1; + } + + setUseOptimized(true); + setNumThreads(8); + + Ptr gs = createGraphSegmentation(); + + if (argc > 3) + gs->setSigma(atof(argv[3])); + + if (argc > 4) + gs->setK(atoi(argv[4])); + + if (argc > 5) + gs->setMinSize(atoi(argv[5])); + + if (!gs) { + std::cerr << "Failed to create GraphSegmentation Algorithm." << std::endl; + return -2; + } + + Mat input, output, output_image; + + input = imread(argv[1]); + + if (!input.data) { + std::cerr << "Failed to load input image" << std::endl; + return -3; + } + + gs->processImage(input, output); + + double min, max; + minMaxLoc(output, &min, &max); + + int nb_segs = (int)max + 1; + + std::cout << nb_segs << " segments" << std::endl; + + output_image = Mat::zeros(output.rows, output.cols, CV_8UC3); + + uint* p; + uchar* p2; + + for (int i = 0; i < output.rows; i++) { + + p = output.ptr(i); + p2 = output_image.ptr(i); + + for (int j = 0; j < output.cols; j++) { + Scalar color = color_mapping(p[j]); + p2[j*3] = color[0]; + p2[j*3 + 1] = color[1]; + p2[j*3 + 2] = color[2]; + } + } + + imwrite(argv[2], output_image); + + std::cout << "Image written to " << argv[2] << std::endl; + + return 0; +} diff --git a/modules/ximgproc/src/graphsegmentation.cpp b/modules/ximgproc/src/graphsegmentation.cpp new file mode 100644 index 000000000..52fc1d47f --- /dev/null +++ b/modules/ximgproc/src/graphsegmentation.cpp @@ -0,0 +1,332 @@ +/* +By downloading, copying, installing or using the software you agree to this +license. If you do not agree to this license, do not download, install, +copy or use the software. + License Agreement + For Open Source Computer Vision Library + (3-clause BSD License) +Copyright (C) 2013, OpenCV Foundation, all rights reserved. +Third party copyrights are property of their respective owners. +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the names of the copyright holders nor the names of the contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. +This software is provided by the copyright holders and contributors "as is" and +any express or implied warranties, including, but not limited to, the implied +warranties of merchantability and fitness for a particular purpose are +disclaimed. In no event shall copyright holders or contributors be liable for +any direct, indirect, incidental, special, exemplary, or consequential damages +(including, but not limited to, procurement of substitute goods or services; +loss of use, data, or profits; or business interruption) however caused +and on any theory of liability, whether in contract, strict liability, +or tort (including negligence or otherwise) arising in any way out of +the use of this software, even if advised of the possibility of such damage. +*/ + +/******************************************************************************\ +* Graph based segmentation * +* This code implements the segmentation method described in: * +* P. Felzenszwalb, D. Huttenlocher: "Graph-Based Image Segmentation" * +* International Journal of Computer Vision, Vol. 59, No. 2, September 2004 * +* * +* Author: Maximilien Cuony / LTS2 / EPFL / 2015 * +*******************************************************************************/ + +#include "precomp.hpp" +#include "opencv2/ximgproc/segmentation.hpp" + +#include + +namespace cv { + namespace ximgproc { + namespace segmentation { + + class GraphSegmentationImpl : public GraphSegmentation { + public: + GraphSegmentationImpl() { + sigma = 0.5; + k = 300; + min_size = 100; + name_ = "GraphSegmentation"; + } + + ~GraphSegmentationImpl() { + }; + + virtual void processImage(InputArray src, OutputArray dst); + + virtual void setSigma(double sigma_) { if (sigma_ <= 0) { sigma_ = 0.001; } sigma = sigma_; } + virtual double getSigma() { return sigma; } + + virtual void setK(float k_) { k = k_; } + virtual float getK() { return k; } + + virtual void setMinSize(int min_size_) { min_size = min_size_; } + virtual int getMinSize() { return min_size; } + + virtual void write(FileStorage& fs) const { + fs << "name" << name_ + << "sigma" << sigma + << "k" << k + << "min_size" << (int)min_size; + } + + virtual void read(const FileNode& fn) { + CV_Assert( (String)fn["name"] == name_ ); + + sigma = (double)fn["sigma"]; + k = (float)fn["k"]; + min_size = (int)(int)fn["min_size"]; + } + + private: + double sigma; + float k; + int min_size; + String name_; + + // Pre-filter the image + void filter(const Mat &img, Mat &img_filtered); + + // Build the graph between each pixels + void buildGraph(Edge **edges, int &nb_edges, const Mat &img_filtered); + + // Segment the graph + void segmentGraph(Edge * edges, const int &nb_edges, const Mat & img_filtered, PointSet **es); + + // Remove areas too small + void filterSmallAreas(Edge *edges, const int &nb_edges, PointSet *es); + + // Map the segemented graph to a Mat with uniques, sequentials ids + void finalMapping(PointSet *es, Mat &output); + }; + + void GraphSegmentationImpl::filter(const Mat &img, Mat &img_filtered) { + + Mat img_converted; + + // Switch to float + img.convertTo(img_converted, CV_32F); + + // Apply gaussian filter + GaussianBlur(img_converted, img_filtered, Size(0, 0), sigma, sigma); + } + + void GraphSegmentationImpl::buildGraph(Edge **edges, int &nb_edges, const Mat &img_filtered) { + + *edges = new Edge[img_filtered.rows * img_filtered.cols * 4]; + + nb_edges = 0; + + int nb_channels = img_filtered.channels(); + + for (int i = 0; i < (int)img_filtered.rows; i++) { + const float* p = img_filtered.ptr(i); + + for (int j = 0; j < (int)img_filtered.cols; j++) { + + //Take the right, left, top and down pixel + for (int delta = -1; delta <= 1; delta += 2) { + for (int delta_j = 0, delta_i = 1; delta_j <= 1; delta_j++ || delta_i--) { + + int i2 = i + delta * delta_i; + int j2 = j + delta * delta_j; + + if (i2 >= 0 && i2 < img_filtered.rows && j2 >= 0 && j2 < img_filtered.cols) { + const float* p2 = img_filtered.ptr(i2); + + float tmp_total = 0; + + for ( int channel = 0; channel < nb_channels; channel++) { + tmp_total += pow(p[j * nb_channels + channel] - p2[j2 * nb_channels + channel], 2); + } + + float diff = 0; + diff = sqrt(tmp_total); + + (*edges)[nb_edges].weight = diff; + (*edges)[nb_edges].from = i * img_filtered.cols + j; + (*edges)[nb_edges].to = i2 * img_filtered.cols + j2; + + nb_edges++; + } + } + } + } + } + } + + void GraphSegmentationImpl::segmentGraph(Edge *edges, const int &nb_edges, const Mat &img_filtered, PointSet **es) { + + int total_points = ( int)(img_filtered.rows * img_filtered.cols); + + // Sort edges + std::sort(edges, edges + nb_edges); + + // Create a set with all point (by default mapped to themselfs) + *es = new PointSet(img_filtered.cols * img_filtered.rows); + + // Thresholds + float* thresholds = new float[total_points]; + + for (int i = 0; i < total_points; i++) + thresholds[i] = k; + + for ( int i = 0; i < nb_edges; i++) { + + int p_a = (*es)->getBasePoint(edges[i].from); + int p_b = (*es)->getBasePoint(edges[i].to); + + if (p_a != p_b) { + if (edges[i].weight <= thresholds[p_a] && edges[i].weight <= thresholds[p_b]) { + (*es)->joinPoints(p_a, p_b); + p_a = (*es)->getBasePoint(p_a); + thresholds[p_a] = edges[i].weight + k / (*es)->size(p_a); + + edges[i].weight = 0; + } + } + } + } + + void GraphSegmentationImpl::filterSmallAreas(Edge *edges, const int &nb_edges, PointSet *es) { + + for ( int i = 0; i < nb_edges; i++) { + + if (edges[i].weight > 0) { + + int p_a = es->getBasePoint(edges[i].from); + int p_b = es->getBasePoint(edges[i].to); + + if (p_a != p_b && (es->size(p_a) < min_size || es->size(p_b) < min_size)) { + es->joinPoints(p_a, p_b); + + } + } + } + + } + + void GraphSegmentationImpl::finalMapping(PointSet *es, Mat &output) { + + int maximum_size = ( int)(output.rows * output.cols); + + int last_id = 0; + int * mapped_id = new int[maximum_size]; + + for ( int i = 0; i < maximum_size; i++) + mapped_id[i] = -1; + + int rows = output.rows; + int cols = output.cols; + + if (output.isContinuous()) { + cols *= rows; + rows = 1; + } + + for (int i = 0; i < rows; i++) { + + int* p = output.ptr(i); + + for (int j = 0; j < cols; j++) { + + int point = es->getBasePoint(i * cols + j); + + if (mapped_id[point] == -1) { + mapped_id[point] = last_id; + last_id++; + } + + p[j] = mapped_id[point]; + } + } + } + + void GraphSegmentationImpl::processImage(InputArray src, OutputArray dst) { + + Mat img = src.getMat(); + + dst.create(img.rows, img.cols, CV_32SC1); + Mat output = dst.getMat(); + output.setTo(0); + + // Filter graph + Mat img_filtered; + filter(img, img_filtered); + + // Build graph + Edge *edges; + int nb_edges; + + buildGraph(&edges, nb_edges, img_filtered); + + // Segment graph + PointSet *es; + + segmentGraph(edges, nb_edges, img_filtered, &es); + + // Remove small areas + filterSmallAreas(edges, nb_edges, es); + + // Map to final output + finalMapping(es, output); + + } + + Ptr createGraphSegmentation(double sigma, float k, int min_size) { + + Ptr graphseg = makePtr(); + + graphseg->setSigma(sigma); + graphseg->setK(k); + graphseg->setMinSize(min_size); + + return graphseg; + } + + PointSet::PointSet(int nb_elements_) { + nb_elements = nb_elements_; + + mapping = new PointSetElement[nb_elements]; + + for ( int i = 0; i < nb_elements; i++) { + mapping[i] = PointSetElement(i); + } + } + + int PointSet::getBasePoint( int p) { + + int base_p = p; + + while (base_p != mapping[base_p].p) { + base_p = mapping[base_p].p; + } + + // Save mapping for faster acces later + mapping[p].p = base_p; + + return base_p; + } + + void PointSet::joinPoints(int p_a, int p_b) { + + // Always target smaller set, to avoid redirection in getBasePoint + if (mapping[p_a].size < mapping[p_b].size) + swap(p_a, p_b); + + mapping[p_b].p = p_a; + mapping[p_a].size += mapping[p_b].size; + + nb_elements--; + } + } + } +} + diff --git a/modules/ximgproc/src/precomp.hpp b/modules/ximgproc/src/precomp.hpp index bfa7ce504..821cd2e0d 100644 --- a/modules/ximgproc/src/precomp.hpp +++ b/modules/ximgproc/src/precomp.hpp @@ -34,8 +34,8 @@ * the use of this software, even if advised of the possibility of such damage. */ -#ifndef _OPENCV_EDGEFILTER_PRECOMP_HPP_ -#define _OPENCV_EDGEFILTER_PRECOMP_HPP_ +#ifndef _OPENCV_XIMGPROC_PRECOMP_HPP_ +#define _OPENCV_XIMGPROC_PRECOMP_HPP_ #include #include @@ -48,4 +48,7 @@ #include -#endif \ No newline at end of file +#include +#include + +#endif From e18fd2deb08a6f70ba49034e879792389ea3f56a Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 3 Nov 2015 17:26:59 +0100 Subject: [PATCH 2/3] Graph segmentation: updated README --- modules/ximgproc/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ximgproc/README.md b/modules/ximgproc/README.md index a31a14611..70c24d3ac 100644 --- a/modules/ximgproc/README.md +++ b/modules/ximgproc/README.md @@ -7,3 +7,4 @@ Extended Image Processing 4. Adaptive Manifold Filter 5. Joint Bilateral Filter 6. Superpixels +7. Graph segmentation From 8f9ac94ba4641e09df39035a5695c4150907d534 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Wed, 4 Nov 2015 10:35:20 +0100 Subject: [PATCH 3/3] Graphsegmentation: fix typo in comment and blank line at EOF --- modules/ximgproc/include/opencv2/ximgproc/segmentation.hpp | 2 +- modules/ximgproc/src/graphsegmentation.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/ximgproc/include/opencv2/ximgproc/segmentation.hpp b/modules/ximgproc/include/opencv2/ximgproc/segmentation.hpp index a98848d67..990fc6413 100644 --- a/modules/ximgproc/include/opencv2/ximgproc/segmentation.hpp +++ b/modules/ximgproc/include/opencv2/ximgproc/segmentation.hpp @@ -62,7 +62,7 @@ namespace cv { }; /** @brief Creates a graph based segmentor - @param simga The sigma parameter, used to smooth image + @param sigma The sigma parameter, used to smooth image @param k The k parameter of the algorythm @param min_size The minimum size of segments */ diff --git a/modules/ximgproc/src/graphsegmentation.cpp b/modules/ximgproc/src/graphsegmentation.cpp index 52fc1d47f..7e0344675 100644 --- a/modules/ximgproc/src/graphsegmentation.cpp +++ b/modules/ximgproc/src/graphsegmentation.cpp @@ -329,4 +329,3 @@ namespace cv { } } } -