From a2f0963d66a7a7cfdb1771e4936cf63b69aa0d05 Mon Sep 17 00:00:00 2001 From: "joao.faro" Date: Wed, 2 Sep 2015 17:14:40 +0100 Subject: [PATCH 01/19] SVMSGD class added --- doc/opencv.bib | 18 ++- modules/ml/include/opencv2/ml.hpp | 121 ++++++++++++++++++ modules/ml/src/svmsgd.cpp | 201 ++++++++++++++++++++++++++++++ 3 files changed, 330 insertions(+), 10 deletions(-) create mode 100644 modules/ml/src/svmsgd.cpp diff --git a/doc/opencv.bib b/doc/opencv.bib index 021c5b4b3f..17eedf7172 100644 --- a/doc/opencv.bib +++ b/doc/opencv.bib @@ -415,16 +415,6 @@ pages = {2548--2555}, organization = {IEEE} } -@ARTICLE{Louhichi07, - author = {Louhichi, H. and Fournel, T. and Lavest, J. M. and Ben Aissia, H.}, - title = {Self-calibration of Scheimpflug cameras: an easy protocol}, - year = {2007}, - pages = {2616–2622}, - journal = {Meas. Sci. Technol.}, - volume = {18}, - number = {8}, - publisher = {IOP Publishing Ltd} -} @ARTICLE{LibSVM, author = {Chang, Chih-Chung and Lin, Chih-Jen}, title = {LIBSVM: a library for support vector machines}, @@ -874,3 +864,11 @@ year={2007}, organization={IEEE} } +@incollection{bottou2010large, + title={Large-scale machine learning with stochastic gradient descent}, + author={Bottou, L{\'e}on}, + booktitle={Proceedings of COMPSTAT'2010}, + pages={177--186}, + year={2010}, + publisher={Springer} +} \ No newline at end of file diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index 7acce7f33c..d5debdbf18 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1513,6 +1513,127 @@ CV_EXPORTS void randMVNormal( InputArray mean, InputArray cov, int nsamples, Out CV_EXPORTS void createConcentricSpheresTestSet( int nsamples, int nfeatures, int nclasses, OutputArray samples, OutputArray responses); +/****************************************************************************************\ +* Stochastic Gradient Descent SVM Classifier * +\****************************************************************************************/ + +/*! +@brief Stochastic Gradient Descent SVM classifier + +SVMSGD provides a fast and easy-to-use implementation of the SVM classifier using the Stochastic Gradient Descent approach, as presented in @cite bottou2010large. +The gradient descent show amazing performance for large-scale problems, reducing the computing time. This allows a fast and reliable online update of the classifier for each new feature which +is fundamental when dealing with variations of data over time (like weather and illumination changes in videosurveillance, for example). + +First, create the SVMSGD object. To enable the online update, a value for updateFrequency should be defined. + +Then the SVM model can be trained using the train features and the correspondent labels. + +After that, the label of a new feature vector can be predicted using the predict function. If the updateFrequency was defined in the constructor, the predict function will update the weights automatically. + +@code +// Initialize object +SVMSGD SvmSgd; + +// Train the Stochastic Gradient Descent SVM +SvmSgd.train(trainFeatures, labels); + +// Predict label for the new feature vector (1xM) +predictedLabel = SvmSgd.predict(newFeatureVector); +@endcode + +*/ +class CV_EXPORTS_W SVMSGD { + + public: + /** @brief SGDSVM constructor. + + @param lambda regularization + @param learnRate learning rate + @param nIterations number of training iterations + + */ + SVMSGD(float lambda = 0.000001, float learnRate = 2, uint nIterations = 100000); + + /** @brief SGDSVM constructor. + + @param updateFrequency online update frequency + @param learnRateDecay learn rate decay over time: learnRate = learnRate * learnDecay + @param lambda regularization + @param learnRate learning rate + @param nIterations number of training iterations + + */ + SVMSGD(uint updateFrequency, float learnRateDecay = 1, float lambda = 0.000001, float learnRate = 2, uint nIterations = 100000); + virtual ~SVMSGD(); + virtual SVMSGD* clone() const; + + /** @brief Train the SGDSVM classifier. + + The function trains the SGDSVM classifier using the train features and the correspondent labels (-1 or 1). + + @param trainFeatures features used for training. Each row is a new sample. + @param labels mat (size Nx1 with N = number of features) with the label of each training feature. + + */ + virtual void train(cv::Mat trainFeatures, cv::Mat labels); + + /** @brief Predict the label of a new feature vector. + + The function predicts and returns the label of a new feature vector, using the previously trained SVM model. + + @param newFeature new feature vector used for prediction + + */ + virtual float predict(cv::Mat newFeature); + + /** @brief Returns the weights of the trained model. + + */ + virtual std::vector getWeights(){ return _weights; }; + + /** @brief Sets the weights of the trained model. + + @param weights weights used to predict the label of a new feature vector. + + */ + virtual void setWeights(std::vector weights){ _weights = weights; }; + + private: + void updateWeights(); + void generateRandomIndex(); + float calcInnerProduct(float *rowDataPointer); + void updateWeights(float innerProduct, float *rowDataPointer, int label); + + // Vector with SVM weights + std::vector _weights; + + // Random index generation + long long int _randomNumber; + unsigned int _randomIndex; + + // Number of features and samples + unsigned int _nFeatures; + unsigned int _nTrainSamples; + + // Parameters for learning + float _lambda; //regularization + float _learnRate; //learning rate + unsigned int _nIterations; //number of training iterations + + // Vars to control the features slider matrix + bool _onlineUpdate; + bool _initPredict; + uint _slidingWindowSize; + uint _predictSlidingWindowSize; + float* _labelSlider; + float _learnRateDecay; + + // Mat with features slider and correspondent counter + unsigned int _sliderCounter; + cv::Mat _featuresSlider; + +}; + //! @} ml } diff --git a/modules/ml/src/svmsgd.cpp b/modules/ml/src/svmsgd.cpp new file mode 100644 index 0000000000..3114e43d9f --- /dev/null +++ b/modules/ml/src/svmsgd.cpp @@ -0,0 +1,201 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000, Intel Corporation, all rights reserved. +// Copyright (C) 2014, Itseez Inc, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "precomp.hpp" + +/****************************************************************************************\ +* Stochastic Gradient Descent SVM Classifier * +\****************************************************************************************/ + +namespace cv { +namespace ml { + +SVMSGD::SVMSGD(float lambda, float learnRate, uint nIterations){ + + // Initialize with random seed + _randomNumber = 1; + + // Initialize constants + _slidingWindowSize = 0; + _nFeatures = 0; + _predictSlidingWindowSize = 1; + + // Initialize sliderCounter at index 0 + _sliderCounter = 0; + + // Parameters for learning + _lambda = lambda; // regularization + _learnRate = learnRate; // learning rate (ideally should be large at beginning and decay each iteration) + _nIterations = nIterations; // number of training iterations + + // True only in the first predict iteration + _initPredict = true; + + // Online update flag + _onlineUpdate = false; +} + +SVMSGD::SVMSGD(uint updateFrequency, float learnRateDecay, float lambda, float learnRate, uint nIterations){ + + // Initialize with random seed + _randomNumber = 1; + + // Initialize constants + _slidingWindowSize = 0; + _nFeatures = 0; + _predictSlidingWindowSize = updateFrequency; + + // Initialize sliderCounter at index 0 + _sliderCounter = 0; + + // Parameters for learning + _lambda = lambda; // regularization + _learnRate = learnRate; // learning rate (ideally should be large at beginning and decay each iteration) + _nIterations = nIterations; // number of training iterations + + // True only in the first predict iteration + _initPredict = true; + + // Online update flag + _onlineUpdate = true; + + // Learn rate decay: _learnRate = _learnRate * _learnDecay + _learnRateDecay = learnRateDecay; +} + +SVMSGD::~SVMSGD(){ + +} + +SVMSGD* SVMSGD::clone() const{ + return new SVMSGD(*this); +} + +void SVMSGD::train(cv::Mat trainFeatures, cv::Mat labels){ + + // Initialize _nFeatures + _slidingWindowSize = trainFeatures.rows; + _nFeatures = trainFeatures.cols; + + float innerProduct; + // Initialize weights vector with zeros + if (_weights.size()==0){ + _weights.reserve(_nFeatures); + for (uint feat = 0; feat < _nFeatures; ++feat){ + _weights.push_back(0.0); + } + } + + // Stochastic gradient descent SVM + for (uint iter = 0; iter < _nIterations; ++iter){ + generateRandomIndex(); + innerProduct = calcInnerProduct(trainFeatures.ptr(_randomIndex)); + int label = (labels.at(_randomIndex,0) > 0) ? 1 : -1; // ensure that labels are -1 or 1 + updateWeights(innerProduct, trainFeatures.ptr(_randomIndex), label ); + } +} + +float SVMSGD::predict(cv::Mat newFeature){ + float innerProduct; + + if (_initPredict){ + _nFeatures = newFeature.cols; + _slidingWindowSize = _predictSlidingWindowSize; + _featuresSlider = cv::Mat::zeros(_slidingWindowSize, _nFeatures, CV_32F); + _initPredict = false; + _labelSlider = new float[_predictSlidingWindowSize](); + _learnRate = _learnRate * _learnRateDecay; + } + + innerProduct = calcInnerProduct(newFeature.ptr(0)); + + // Resultant label (-1 or 1) + int label = (innerProduct>=0) ? 1 : -1; + + if (_onlineUpdate){ + // Update the featuresSlider with newFeature and _labelSlider with label + newFeature.row(0).copyTo(_featuresSlider.row(_sliderCounter)); + _labelSlider[_sliderCounter] = float(label); + + // Update weights with a random index + if (_sliderCounter == _slidingWindowSize-1){ + generateRandomIndex(); + updateWeights(innerProduct, _featuresSlider.ptr(_randomIndex), int(_labelSlider[_randomIndex]) ); + } + + // _sliderCounter++ if < _slidingWindowSize + _sliderCounter = (_sliderCounter == _slidingWindowSize-1) ? 0 : (_sliderCounter+1); + } + + return float(label); +} + +void SVMSGD::generateRandomIndex(){ + // Choose random sample, using Mikolov's fast almost-uniform random number + _randomNumber = _randomNumber * (unsigned long long) 25214903917 + 11; + _randomIndex = uint(_randomNumber % (unsigned long long) _slidingWindowSize); +} + +float SVMSGD::calcInnerProduct(float *rowDataPointer){ + float innerProduct = 0; + for (uint feat = 0; feat < _nFeatures; ++feat){ + innerProduct += _weights[feat] * rowDataPointer[feat]; + } + return innerProduct; +} + +void SVMSGD::updateWeights(float innerProduct, float *rowDataPointer, int label){ + if (label * innerProduct > 1) { + // Not a support vector, only apply weight decay + for (uint feat = 0; feat < _nFeatures; feat++) { + _weights[feat] -= _learnRate * _lambda * _weights[feat]; + } + } else { + // It's a support vector, add it to the weights + for (uint feat = 0; feat < _nFeatures; feat++) { + _weights[feat] -= _learnRate * (_lambda * _weights[feat] - label * rowDataPointer[feat]); + } + } +} + +} +} From 40bf97c6d11a8144f212a07e4ff1eff0eaa9d1a7 Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Wed, 20 Jan 2016 12:59:44 +0300 Subject: [PATCH 02/19] Refactored SVMSGD class --- include/opencv2/opencv.hpp | 1 + modules/ml/include/opencv2/ml.hpp | 120 ------ modules/ml/include/opencv2/ml/svmsgd.hpp | 134 +++++++ modules/ml/src/precomp.hpp | 2 +- modules/ml/src/svmsgd.cpp | 473 ++++++++++++++++++----- modules/ml/test/test_mltests2.cpp | 31 +- modules/ml/test/test_precomp.hpp | 3 + modules/ml/test/test_save_load.cpp | 17 +- modules/ml/test/test_svmsgd.cpp | 182 +++++++++ modules/ts/src/ts_gtest.cpp | 2 +- samples/cpp/train_svmsgd.cpp | 226 +++++++++++ 11 files changed, 965 insertions(+), 226 deletions(-) create mode 100644 modules/ml/include/opencv2/ml/svmsgd.hpp create mode 100644 modules/ml/test/test_svmsgd.cpp create mode 100644 samples/cpp/train_svmsgd.cpp diff --git a/include/opencv2/opencv.hpp b/include/opencv2/opencv.hpp index 49b6a6691f..e411621d0c 100644 --- a/include/opencv2/opencv.hpp +++ b/include/opencv2/opencv.hpp @@ -75,6 +75,7 @@ #endif #ifdef HAVE_OPENCV_ML #include "opencv2/ml.hpp" +#include "opencv2/ml/svmsgd.hpp" #endif #endif diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index d5debdbf18..791f580093 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1513,126 +1513,6 @@ CV_EXPORTS void randMVNormal( InputArray mean, InputArray cov, int nsamples, Out CV_EXPORTS void createConcentricSpheresTestSet( int nsamples, int nfeatures, int nclasses, OutputArray samples, OutputArray responses); -/****************************************************************************************\ -* Stochastic Gradient Descent SVM Classifier * -\****************************************************************************************/ - -/*! -@brief Stochastic Gradient Descent SVM classifier - -SVMSGD provides a fast and easy-to-use implementation of the SVM classifier using the Stochastic Gradient Descent approach, as presented in @cite bottou2010large. -The gradient descent show amazing performance for large-scale problems, reducing the computing time. This allows a fast and reliable online update of the classifier for each new feature which -is fundamental when dealing with variations of data over time (like weather and illumination changes in videosurveillance, for example). - -First, create the SVMSGD object. To enable the online update, a value for updateFrequency should be defined. - -Then the SVM model can be trained using the train features and the correspondent labels. - -After that, the label of a new feature vector can be predicted using the predict function. If the updateFrequency was defined in the constructor, the predict function will update the weights automatically. - -@code -// Initialize object -SVMSGD SvmSgd; - -// Train the Stochastic Gradient Descent SVM -SvmSgd.train(trainFeatures, labels); - -// Predict label for the new feature vector (1xM) -predictedLabel = SvmSgd.predict(newFeatureVector); -@endcode - -*/ -class CV_EXPORTS_W SVMSGD { - - public: - /** @brief SGDSVM constructor. - - @param lambda regularization - @param learnRate learning rate - @param nIterations number of training iterations - - */ - SVMSGD(float lambda = 0.000001, float learnRate = 2, uint nIterations = 100000); - - /** @brief SGDSVM constructor. - - @param updateFrequency online update frequency - @param learnRateDecay learn rate decay over time: learnRate = learnRate * learnDecay - @param lambda regularization - @param learnRate learning rate - @param nIterations number of training iterations - - */ - SVMSGD(uint updateFrequency, float learnRateDecay = 1, float lambda = 0.000001, float learnRate = 2, uint nIterations = 100000); - virtual ~SVMSGD(); - virtual SVMSGD* clone() const; - - /** @brief Train the SGDSVM classifier. - - The function trains the SGDSVM classifier using the train features and the correspondent labels (-1 or 1). - - @param trainFeatures features used for training. Each row is a new sample. - @param labels mat (size Nx1 with N = number of features) with the label of each training feature. - - */ - virtual void train(cv::Mat trainFeatures, cv::Mat labels); - - /** @brief Predict the label of a new feature vector. - - The function predicts and returns the label of a new feature vector, using the previously trained SVM model. - - @param newFeature new feature vector used for prediction - - */ - virtual float predict(cv::Mat newFeature); - - /** @brief Returns the weights of the trained model. - - */ - virtual std::vector getWeights(){ return _weights; }; - - /** @brief Sets the weights of the trained model. - - @param weights weights used to predict the label of a new feature vector. - - */ - virtual void setWeights(std::vector weights){ _weights = weights; }; - - private: - void updateWeights(); - void generateRandomIndex(); - float calcInnerProduct(float *rowDataPointer); - void updateWeights(float innerProduct, float *rowDataPointer, int label); - - // Vector with SVM weights - std::vector _weights; - - // Random index generation - long long int _randomNumber; - unsigned int _randomIndex; - - // Number of features and samples - unsigned int _nFeatures; - unsigned int _nTrainSamples; - - // Parameters for learning - float _lambda; //regularization - float _learnRate; //learning rate - unsigned int _nIterations; //number of training iterations - - // Vars to control the features slider matrix - bool _onlineUpdate; - bool _initPredict; - uint _slidingWindowSize; - uint _predictSlidingWindowSize; - float* _labelSlider; - float _learnRateDecay; - - // Mat with features slider and correspondent counter - unsigned int _sliderCounter; - cv::Mat _featuresSlider; - -}; //! @} ml diff --git a/modules/ml/include/opencv2/ml/svmsgd.hpp b/modules/ml/include/opencv2/ml/svmsgd.hpp new file mode 100644 index 0000000000..f61a905963 --- /dev/null +++ b/modules/ml/include/opencv2/ml/svmsgd.hpp @@ -0,0 +1,134 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000, Intel Corporation, all rights reserved. +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Copyright (C) 2014, Itseez Inc, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#ifndef __OPENCV_ML_SVMSGD_HPP__ +#define __OPENCV_ML_SVMSGD_HPP__ + +#ifdef __cplusplus + +#include "opencv2/ml.hpp" + +namespace cv +{ +namespace ml +{ + + +/****************************************************************************************\ +* Stochastic Gradient Descent SVM Classifier * +\****************************************************************************************/ + +/*! +@brief Stochastic Gradient Descent SVM classifier + +SVMSGD provides a fast and easy-to-use implementation of the SVM classifier using the Stochastic Gradient Descent approach, as presented in @cite bottou2010large. +The gradient descent show amazing performance for large-scale problems, reducing the computing time. This allows a fast and reliable online update of the classifier for each new feature which +is fundamental when dealing with variations of data over time (like weather and illumination changes in videosurveillance, for example). + +First, create the SVMSGD object. To enable the online update, a value for updateFrequency should be defined. + +Then the SVM model can be trained using the train features and the correspondent labels. + +After that, the label of a new feature vector can be predicted using the predict function. If the updateFrequency was defined in the constructor, the predict function will update the weights automatically. + +@code +// Initialize object +SVMSGD SvmSgd; + +// Train the Stochastic Gradient Descent SVM +SvmSgd.train(trainFeatures, labels); + +// Predict label for the new feature vector (1xM) +predictedLabel = SvmSgd.predict(newFeatureVector); +@endcode + +*/ + +class CV_EXPORTS_W SVMSGD : public cv::ml::StatModel +{ +public: + + enum SvmsgdType + { + ILLEGAL_VALUE, + SGD, //Stochastic Gradient Descent + ASGD //Average Stochastic Gradient Descent + }; + + /** + * @return the weights of the trained model. + */ + CV_WRAP virtual Mat getWeights() = 0; + + CV_WRAP virtual float getShift() = 0; + + CV_WRAP static Ptr create(); + + CV_WRAP virtual void setOptimalParameters(int type = ASGD) = 0; + + CV_WRAP virtual int getType() const = 0; + + CV_WRAP virtual void setType(int type) = 0; + + CV_WRAP virtual float getLambda() const = 0; + + CV_WRAP virtual void setLambda(float lambda) = 0; + + CV_WRAP virtual float getGamma0() const = 0; + + CV_WRAP virtual void setGamma0(float gamma0) = 0; + + CV_WRAP virtual float getC() const = 0; + + CV_WRAP virtual void setC(float c) = 0; + + CV_WRAP virtual cv::TermCriteria getTermCriteria() const = 0; + + CV_WRAP virtual void setTermCriteria(const cv::TermCriteria &val) = 0; +}; + +} //ml +} //cv + +#endif // __clpusplus +#endif // __OPENCV_ML_SVMSGD_HPP diff --git a/modules/ml/src/precomp.hpp b/modules/ml/src/precomp.hpp index 84821988b6..9318e4a78c 100644 --- a/modules/ml/src/precomp.hpp +++ b/modules/ml/src/precomp.hpp @@ -45,7 +45,7 @@ #include "opencv2/ml.hpp" #include "opencv2/core/core_c.h" #include "opencv2/core/utility.hpp" - +#include "opencv2/ml/svmsgd.hpp" #include "opencv2/core/private.hpp" #include diff --git a/modules/ml/src/svmsgd.cpp b/modules/ml/src/svmsgd.cpp index 3114e43d9f..91377cfc4f 100644 --- a/modules/ml/src/svmsgd.cpp +++ b/modules/ml/src/svmsgd.cpp @@ -41,161 +41,430 @@ //M*/ #include "precomp.hpp" +#include "limits" /****************************************************************************************\ * Stochastic Gradient Descent SVM Classifier * \****************************************************************************************/ -namespace cv { -namespace ml { +namespace cv +{ +namespace ml +{ -SVMSGD::SVMSGD(float lambda, float learnRate, uint nIterations){ +class SVMSGDImpl : public SVMSGD +{ - // Initialize with random seed - _randomNumber = 1; +public: + SVMSGDImpl(); - // Initialize constants - _slidingWindowSize = 0; - _nFeatures = 0; - _predictSlidingWindowSize = 1; + virtual ~SVMSGDImpl() {} - // Initialize sliderCounter at index 0 - _sliderCounter = 0; + virtual bool train(const Ptr& data, int); - // Parameters for learning - _lambda = lambda; // regularization - _learnRate = learnRate; // learning rate (ideally should be large at beginning and decay each iteration) - _nIterations = nIterations; // number of training iterations + virtual float predict( InputArray samples, OutputArray results=noArray(), int flags = 0 ) const; - // True only in the first predict iteration - _initPredict = true; + virtual bool isClassifier() const { return params.svmsgdType == SGD || params.svmsgdType == ASGD; } - // Online update flag - _onlineUpdate = false; -} + virtual bool isTrained() const; -SVMSGD::SVMSGD(uint updateFrequency, float learnRateDecay, float lambda, float learnRate, uint nIterations){ + virtual void clear(); - // Initialize with random seed - _randomNumber = 1; + virtual void write(FileStorage& fs) const; - // Initialize constants - _slidingWindowSize = 0; - _nFeatures = 0; - _predictSlidingWindowSize = updateFrequency; + virtual void read(const FileNode& fn); - // Initialize sliderCounter at index 0 - _sliderCounter = 0; + virtual Mat getWeights(){ return weights_; } - // Parameters for learning - _lambda = lambda; // regularization - _learnRate = learnRate; // learning rate (ideally should be large at beginning and decay each iteration) - _nIterations = nIterations; // number of training iterations + virtual float getShift(){ return shift_; } - // True only in the first predict iteration - _initPredict = true; + virtual int getVarCount() const { return weights_.cols; } - // Online update flag - _onlineUpdate = true; + virtual String getDefaultName() const {return "opencv_ml_svmsgd";} - // Learn rate decay: _learnRate = _learnRate * _learnDecay - _learnRateDecay = learnRateDecay; -} + virtual void setOptimalParameters(int type = ASGD); -SVMSGD::~SVMSGD(){ + virtual int getType() const; -} + virtual void setType(int type); + + CV_IMPL_PROPERTY(float, Lambda, params.lambda) + CV_IMPL_PROPERTY(float, Gamma0, params.gamma0) + CV_IMPL_PROPERTY(float, C, params.c) + CV_IMPL_PROPERTY_S(cv::TermCriteria, TermCriteria, params.termCrit) -SVMSGD* SVMSGD::clone() const{ - return new SVMSGD(*this); + private: + void updateWeights(InputArray sample, bool is_first_class, float gamma); + float calcShift(InputArray trainSamples, InputArray trainResponses) const; + std::pair areClassesEmpty(Mat responses); + void writeParams( FileStorage& fs ) const; + void readParams( const FileNode& fn ); + static inline bool isFirstClass(float val) { return val > 0; } + + + // Vector with SVM weights + Mat weights_; + float shift_; + + // Random index generation + RNG rng_; + + // Parameters for learning + struct SVMSGDParams + { + float lambda; //regularization + float gamma0; //learning rate + float c; + TermCriteria termCrit; + SvmsgdType svmsgdType; + }; + + SVMSGDParams params; +}; + +Ptr SVMSGD::create() +{ + return makePtr(); } -void SVMSGD::train(cv::Mat trainFeatures, cv::Mat labels){ - // Initialize _nFeatures - _slidingWindowSize = trainFeatures.rows; - _nFeatures = trainFeatures.cols; +bool SVMSGDImpl::train(const Ptr& data, int) +{ + clear(); + + Mat trainSamples = data->getTrainSamples(); + + // Initialize varCount + int trainSamplesCount_ = trainSamples.rows; + int varCount = trainSamples.cols; - float innerProduct; // Initialize weights vector with zeros - if (_weights.size()==0){ - _weights.reserve(_nFeatures); - for (uint feat = 0; feat < _nFeatures; ++feat){ - _weights.push_back(0.0); - } + weights_ = Mat::zeros(1, varCount, CV_32F); + + Mat trainResponses = data->getTrainResponses(); // (trainSamplesCount x 1) matrix + + std::pair are_empty = areClassesEmpty(trainResponses); + + if ( are_empty.first && are_empty.second ) + { + weights_.release(); + return false; + } + if ( are_empty.first || are_empty.second ) + { + shift_ = are_empty.first ? -1 : 1; + return true; + } + + + Mat currentSample; + float gamma = 0; + Mat lastWeights = Mat::zeros(1, varCount, CV_32F); //weights vector for calculating terminal criterion + Mat averageWeights; //average weights vector for ASGD model + double err = DBL_MAX; + if (params.svmsgdType == ASGD) + { + averageWeights = Mat::zeros(1, varCount, CV_32F); } // Stochastic gradient descent SVM - for (uint iter = 0; iter < _nIterations; ++iter){ - generateRandomIndex(); - innerProduct = calcInnerProduct(trainFeatures.ptr(_randomIndex)); - int label = (labels.at(_randomIndex,0) > 0) ? 1 : -1; // ensure that labels are -1 or 1 - updateWeights(innerProduct, trainFeatures.ptr(_randomIndex), label ); + for (int iter = 0; (iter < params.termCrit.maxCount)&&(err > params.termCrit.epsilon); iter++) + { + //generate sample number + int randomNumber = rng_.uniform(0, trainSamplesCount_); + + currentSample = trainSamples.row(randomNumber); + + //update gamma + gamma = params.gamma0 * std::pow((1 + params.lambda * params.gamma0 * (float)iter), (-params.c)); + + bool is_first_class = isFirstClass(trainResponses.at(randomNumber)); + updateWeights( currentSample, is_first_class, gamma ); + + //average weights (only for ASGD model) + if (params.svmsgdType == ASGD) + { + averageWeights = ((float)iter/ (1 + (float)iter)) * averageWeights + weights_ / (1 + (float) iter); + } + + err = norm(weights_ - lastWeights); + weights_.copyTo(lastWeights); + } + + if (params.svmsgdType == ASGD) + { + weights_ = averageWeights; } + + shift_ = calcShift(trainSamples, trainResponses); + + return true; } -float SVMSGD::predict(cv::Mat newFeature){ - float innerProduct; +std::pair SVMSGDImpl::areClassesEmpty(Mat responses) +{ + std::pair are_classes_empty(true, true); + int limit_index = responses.rows; + + for(int index = 0; index < limit_index; index++) + { + if (isFirstClass(responses.at(index,0))) + are_classes_empty.first = false; + else + are_classes_empty.second = false; - if (_initPredict){ - _nFeatures = newFeature.cols; - _slidingWindowSize = _predictSlidingWindowSize; - _featuresSlider = cv::Mat::zeros(_slidingWindowSize, _nFeatures, CV_32F); - _initPredict = false; - _labelSlider = new float[_predictSlidingWindowSize](); - _learnRate = _learnRate * _learnRateDecay; + if (!are_classes_empty.first && ! are_classes_empty.second) + break; } - innerProduct = calcInnerProduct(newFeature.ptr(0)); + return are_classes_empty; +} - // Resultant label (-1 or 1) - int label = (innerProduct>=0) ? 1 : -1; +float SVMSGDImpl::calcShift(InputArray _samples, InputArray _responses) const +{ + float distance_to_classes[2] = { std::numeric_limits::max(), std::numeric_limits::max() }; - if (_onlineUpdate){ - // Update the featuresSlider with newFeature and _labelSlider with label - newFeature.row(0).copyTo(_featuresSlider.row(_sliderCounter)); - _labelSlider[_sliderCounter] = float(label); + Mat trainSamples = _samples.getMat(); + int trainSamplesCount = trainSamples.rows; - // Update weights with a random index - if (_sliderCounter == _slidingWindowSize-1){ - generateRandomIndex(); - updateWeights(innerProduct, _featuresSlider.ptr(_randomIndex), int(_labelSlider[_randomIndex]) ); - } + Mat trainResponses = _responses.getMat(); + + for (int samplesIndex = 0; samplesIndex < trainSamplesCount; samplesIndex++) + { + Mat currentSample = trainSamples.row(samplesIndex); + float scalar_product = currentSample.dot(weights_); - // _sliderCounter++ if < _slidingWindowSize - _sliderCounter = (_sliderCounter == _slidingWindowSize-1) ? 0 : (_sliderCounter+1); + bool is_first_class = isFirstClass(trainResponses.at(samplesIndex)); + int index = is_first_class ? 0:1; + float sign_to_mul = is_first_class ? 1 : -1; + float cur_distance = scalar_product * sign_to_mul ; + + if (cur_distance < distance_to_classes[index]) + { + distance_to_classes[index] = cur_distance; + } } - return float(label); + //todo: areClassesEmpty(); make const; + return -(distance_to_classes[0] - distance_to_classes[1]) / 2.f; } -void SVMSGD::generateRandomIndex(){ - // Choose random sample, using Mikolov's fast almost-uniform random number - _randomNumber = _randomNumber * (unsigned long long) 25214903917 + 11; - _randomIndex = uint(_randomNumber % (unsigned long long) _slidingWindowSize); -} +float SVMSGDImpl::predict( InputArray _samples, OutputArray _results, int ) const +{ + float result = 0; + cv::Mat samples = _samples.getMat(); + int nSamples = samples.rows; + cv::Mat results; -float SVMSGD::calcInnerProduct(float *rowDataPointer){ - float innerProduct = 0; - for (uint feat = 0; feat < _nFeatures; ++feat){ - innerProduct += _weights[feat] * rowDataPointer[feat]; + CV_Assert( samples.cols == weights_.cols && samples.type() == CV_32F ); + + if( _results.needed() ) + { + _results.create( nSamples, 1, samples.type() ); + results = _results.getMat(); + } + else + { + CV_Assert( nSamples == 1 ); + results = Mat(1, 1, CV_32F, &result); } - return innerProduct; + + Mat currentSample; + float criterion = 0; + + for (int sampleIndex = 0; sampleIndex < nSamples; sampleIndex++) + { + currentSample = samples.row(sampleIndex); + criterion = currentSample.dot(weights_) + shift_; + results.at(sampleIndex) = (criterion >= 0) ? 1 : -1; + } + + return result; } -void SVMSGD::updateWeights(float innerProduct, float *rowDataPointer, int label){ - if (label * innerProduct > 1) { +void SVMSGDImpl::updateWeights(InputArray _sample, bool is_first_class, float gamma) +{ + Mat sample = _sample.getMat(); + + int responce = is_first_class ? 1 : -1; // ensure that trainResponses are -1 or 1 + + if ( sample.dot(weights_) * responce > 1) + { // Not a support vector, only apply weight decay - for (uint feat = 0; feat < _nFeatures; feat++) { - _weights[feat] -= _learnRate * _lambda * _weights[feat]; - } - } else { + weights_ *= (1.f - gamma * params.lambda); + } + else + { // It's a support vector, add it to the weights - for (uint feat = 0; feat < _nFeatures; feat++) { - _weights[feat] -= _learnRate * (_lambda * _weights[feat] - label * rowDataPointer[feat]); - } + weights_ -= (gamma * params.lambda) * weights_ - gamma * responce * sample; + //std::cout << "sample " << sample << std::endl; + //std::cout << "weights_ " << weights_ << std::endl; + } +} + +bool SVMSGDImpl::isTrained() const +{ + return !weights_.empty(); +} + +void SVMSGDImpl::write(FileStorage& fs) const +{ + if( !isTrained() ) + CV_Error( CV_StsParseError, "SVMSGD model data is invalid, it hasn't been trained" ); + + writeParams( fs ); + + fs << "shift" << shift_; + fs << "weights" << weights_; +} + +void SVMSGDImpl::writeParams( FileStorage& fs ) const +{ + String SvmsgdTypeStr; + + switch (params.svmsgdType) + { + case SGD: + SvmsgdTypeStr = "SGD"; + break; + case ASGD: + SvmsgdTypeStr = "ASGD"; + break; + case ILLEGAL_VALUE: + SvmsgdTypeStr = format("Uknown_%d", params.svmsgdType); + default: + std::cout << "params.svmsgdType isn't initialized" << std::endl; + } + + + fs << "svmsgdType" << SvmsgdTypeStr; + + fs << "lambda" << params.lambda; + fs << "gamma0" << params.gamma0; + fs << "c" << params.c; + + fs << "term_criteria" << "{:"; + if( params.termCrit.type & TermCriteria::EPS ) + fs << "epsilon" << params.termCrit.epsilon; + if( params.termCrit.type & TermCriteria::COUNT ) + fs << "iterations" << params.termCrit.maxCount; + fs << "}"; +} + + + +void SVMSGDImpl::read(const FileNode& fn) +{ + clear(); + + readParams(fn); + + shift_ = (float) fn["shift"]; + fn["weights"] >> weights_; +} + +void SVMSGDImpl::readParams( const FileNode& fn ) +{ + String svmsgdTypeStr = (String)fn["svmsgdType"]; + SvmsgdType svmsgdType = + svmsgdTypeStr == "SGD" ? SGD : + svmsgdTypeStr == "ASGD" ? ASGD : ILLEGAL_VALUE; + + if( svmsgdType == ILLEGAL_VALUE ) + CV_Error( CV_StsParseError, "Missing or invalid SVMSGD type" ); + + params.svmsgdType = svmsgdType; + + CV_Assert ( fn["lambda"].isReal() ); + params.lambda = (float)fn["lambda"]; + + CV_Assert ( fn["gamma0"].isReal() ); + params.gamma0 = (float)fn["gamma0"]; + + CV_Assert ( fn["c"].isReal() ); + params.c = (float)fn["c"]; + + FileNode tcnode = fn["term_criteria"]; + if( !tcnode.empty() ) + { + params.termCrit.epsilon = (double)tcnode["epsilon"]; + params.termCrit.maxCount = (int)tcnode["iterations"]; + params.termCrit.type = (params.termCrit.epsilon > 0 ? TermCriteria::EPS : 0) + + (params.termCrit.maxCount > 0 ? TermCriteria::COUNT : 0); } + else + params.termCrit = TermCriteria( TermCriteria::EPS + TermCriteria::COUNT, 1000, FLT_EPSILON ); + } +void SVMSGDImpl::clear() +{ + weights_.release(); + shift_ = 0; } + + +SVMSGDImpl::SVMSGDImpl() +{ + clear(); + rng_(0); + + params.svmsgdType = ILLEGAL_VALUE; + + // Parameters for learning + params.lambda = 0; // regularization + params.gamma0 = 0; // learning rate (ideally should be large at beginning and decay each iteration) + params.c = 0; + + TermCriteria _termCrit(TermCriteria::COUNT + TermCriteria::EPS, 0, 0); + params.termCrit = _termCrit; +} + +void SVMSGDImpl::setOptimalParameters(int type) +{ + switch (type) + { + case SGD: + params.svmsgdType = SGD; + params.lambda = 0.00001; + params.gamma0 = 0.05; + params.c = 1; + params.termCrit.maxCount = 50000; + params.termCrit.epsilon = 0.00000001; + break; + + case ASGD: + params.svmsgdType = ASGD; + params.lambda = 0.00001; + params.gamma0 = 0.5; + params.c = 0.75; + params.termCrit.maxCount = 100000; + params.termCrit.epsilon = 0.000001; + break; + + default: + CV_Error( CV_StsParseError, "SVMSGD model data is invalid" ); + } +} + +void SVMSGDImpl::setType(int type) +{ + switch (type) + { + case SGD: + params.svmsgdType = SGD; + break; + case ASGD: + params.svmsgdType = ASGD; + break; + default: + params.svmsgdType = ILLEGAL_VALUE; + } +} + +int SVMSGDImpl::getType() const +{ + return params.svmsgdType; } +} //ml +} //cv diff --git a/modules/ml/test/test_mltests2.cpp b/modules/ml/test/test_mltests2.cpp index 919fae6ce4..6603a35c5b 100644 --- a/modules/ml/test/test_mltests2.cpp +++ b/modules/ml/test/test_mltests2.cpp @@ -193,6 +193,16 @@ int str_to_boost_type( String& str ) // 8. rtrees // 9. ertrees +int str_to_svmsgd_type( String& str ) +{ + if ( !str.compare("SGD") ) + return SVMSGD::SGD; + if ( !str.compare("ASGD") ) + return SVMSGD::ASGD; + CV_Error( CV_StsBadArg, "incorrect boost type string" ); + return -1; +} + // ---------------------------------- MLBaseTest --------------------------------------------------- CV_MLBaseTest::CV_MLBaseTest(const char* _modelName) @@ -248,7 +258,9 @@ void CV_MLBaseTest::run( int ) { string filename = ts->get_data_path(); filename += get_validation_filename(); + validationFS.open( filename, FileStorage::READ ); + read_params( *validationFS ); int code = cvtest::TS::OK; @@ -436,6 +448,21 @@ int CV_MLBaseTest::train( int testCaseIdx ) model = m; } + else if( modelName == CV_SVMSGD ) + { + String svmsgdTypeStr; + modelParamsNode["svmsgdType"] >> svmsgdTypeStr; + Ptr m = SVMSGD::create(); + int type = str_to_svmsgd_type( svmsgdTypeStr ); + m->setType(type); + //m->setType(str_to_svmsgd_type( svmsgdTypeStr )); + m->setLambda(modelParamsNode["lambda"]); + m->setGamma0(modelParamsNode["gamma0"]); + m->setC(modelParamsNode["c"]); + m->setTermCriteria(TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 10000, 0.00001)); + model = m; + } + if( !model.empty() ) is_trained = model->train(data, 0); @@ -457,7 +484,7 @@ float CV_MLBaseTest::get_test_error( int /*testCaseIdx*/, vector *resp ) else if( modelName == CV_ANN ) err = ann_calc_error( model, data, cls_map, type, resp ); else if( modelName == CV_DTREE || modelName == CV_BOOST || modelName == CV_RTREES || - modelName == CV_SVM || modelName == CV_NBAYES || modelName == CV_KNEAREST ) + modelName == CV_SVM || modelName == CV_NBAYES || modelName == CV_KNEAREST || modelName == CV_SVMSGD ) err = model->calcError( data, true, _resp ); if( !_resp.empty() && resp ) _resp.convertTo(*resp, CV_32F); @@ -485,6 +512,8 @@ void CV_MLBaseTest::load( const char* filename ) model = Algorithm::load( filename ); else if( modelName == CV_RTREES ) model = Algorithm::load( filename ); + else if( modelName == CV_SVMSGD ) + model = Algorithm::load( filename ); else CV_Error( CV_StsNotImplemented, "invalid stat model name"); } diff --git a/modules/ml/test/test_precomp.hpp b/modules/ml/test/test_precomp.hpp index 329b9bd6c0..18cee968fb 100644 --- a/modules/ml/test/test_precomp.hpp +++ b/modules/ml/test/test_precomp.hpp @@ -13,6 +13,7 @@ #include #include "opencv2/ts.hpp" #include "opencv2/ml.hpp" +#include "opencv2/ml/svmsgd.hpp" #include "opencv2/core/core_c.h" #define CV_NBAYES "nbayes" @@ -24,6 +25,7 @@ #define CV_BOOST "boost" #define CV_RTREES "rtrees" #define CV_ERTREES "ertrees" +#define CV_SVMSGD "svmsgd" enum { CV_TRAIN_ERROR=0, CV_TEST_ERROR=1 }; @@ -38,6 +40,7 @@ using cv::ml::ANN_MLP; using cv::ml::DTrees; using cv::ml::Boost; using cv::ml::RTrees; +using cv::ml::SVMSGD; class CV_MLBaseTest : public cvtest::BaseTest { diff --git a/modules/ml/test/test_save_load.cpp b/modules/ml/test/test_save_load.cpp index 2d6f144bb9..354c6e0307 100644 --- a/modules/ml/test/test_save_load.cpp +++ b/modules/ml/test/test_save_load.cpp @@ -150,12 +150,20 @@ int CV_SLMLTest::validate_test_results( int testCaseIdx ) TEST(ML_NaiveBayes, save_load) { CV_SLMLTest test( CV_NBAYES ); test.safe_run(); } TEST(ML_KNearest, save_load) { CV_SLMLTest test( CV_KNEAREST ); test.safe_run(); } -TEST(ML_SVM, save_load) { CV_SLMLTest test( CV_SVM ); test.safe_run(); } +TEST(ML_SVM, save_load) +{ + CV_SLMLTest test( CV_SVM ); + test.safe_run(); +} TEST(ML_ANN, save_load) { CV_SLMLTest test( CV_ANN ); test.safe_run(); } TEST(ML_DTree, save_load) { CV_SLMLTest test( CV_DTREE ); test.safe_run(); } TEST(ML_Boost, save_load) { CV_SLMLTest test( CV_BOOST ); test.safe_run(); } TEST(ML_RTrees, save_load) { CV_SLMLTest test( CV_RTREES ); test.safe_run(); } TEST(DISABLED_ML_ERTrees, save_load) { CV_SLMLTest test( CV_ERTREES ); test.safe_run(); } +TEST(MV_SVMSGD, save_load){ + CV_SLMLTest test( CV_SVMSGD ); + test.safe_run(); +} class CV_LegacyTest : public cvtest::BaseTest { @@ -201,6 +209,8 @@ protected: model = Algorithm::load(filename); else if (modelName == CV_RTREES) model = Algorithm::load(filename); + else if (modelName == CV_SVMSGD) + model = Algorithm::load(filename); if (!model) { code = cvtest::TS::FAIL_INVALID_TEST_DATA; @@ -260,6 +270,11 @@ TEST(ML_DTree, legacy_load) { CV_LegacyTest test(CV_DTREE, "_abalone.xml;_mushro TEST(ML_NBayes, legacy_load) { CV_LegacyTest test(CV_NBAYES, "_waveform.xml"); test.safe_run(); } TEST(ML_SVM, legacy_load) { CV_LegacyTest test(CV_SVM, "_poletelecomm.xml;_waveform.xml"); test.safe_run(); } TEST(ML_RTrees, legacy_load) { CV_LegacyTest test(CV_RTREES, "_waveform.xml"); test.safe_run(); } +TEST(ML_SVMSGD, legacy_load) +{ + CV_LegacyTest test(CV_SVMSGD, "_waveform.xml"); + test.safe_run(); +} /*TEST(ML_SVM, throw_exception_when_save_untrained_model) { diff --git a/modules/ml/test/test_svmsgd.cpp b/modules/ml/test/test_svmsgd.cpp new file mode 100644 index 0000000000..9f4aafc08b --- /dev/null +++ b/modules/ml/test/test_svmsgd.cpp @@ -0,0 +1,182 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// Intel License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of Intel Corporation may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "test_precomp.hpp" +#include "opencv2/highgui.hpp" + +using namespace cv; +using namespace cv::ml; +using cv::ml::SVMSGD; +using cv::ml::TrainData; + + + +class CV_SVMSGDTrainTest : public cvtest::BaseTest +{ +public: + CV_SVMSGDTrainTest(Mat _weights, float _shift); +private: + virtual void run( int start_from ); + float decisionFunction(Mat sample, Mat weights, float shift); + + cv::Ptr data; + cv::Mat testSamples; + cv::Mat testResponses; + static const int TEST_VALUE_LIMIT = 50; +}; + +CV_SVMSGDTrainTest::CV_SVMSGDTrainTest(Mat weights, float shift) +{ + int datasize = 100000; + int varCount = weights.cols; + cv::Mat samples = cv::Mat::zeros( datasize, varCount, CV_32FC1 ); + cv::Mat responses = cv::Mat::zeros( datasize, 1, CV_32FC1 ); + cv::RNG rng(0); + + float lowerLimit = -TEST_VALUE_LIMIT; + float upperLimit = TEST_VALUE_LIMIT; + + + rng.fill(samples, RNG::UNIFORM, lowerLimit, upperLimit); + for (int sampleIndex = 0; sampleIndex < datasize; sampleIndex++) + { + responses.at( sampleIndex ) = decisionFunction(samples.row(sampleIndex), weights, shift) > 0 ? 1 : -1; + } + + data = TrainData::create( samples, cv::ml::ROW_SAMPLE, responses ); + + int testSamplesCount = 100000; + + testSamples.create(testSamplesCount, varCount, CV_32FC1); + rng.fill(testSamples, RNG::UNIFORM, lowerLimit, upperLimit); + testResponses.create(testSamplesCount, 1, CV_32FC1); + + for (int i = 0 ; i < testSamplesCount; i++) + { + testResponses.at(i) = decisionFunction(testSamples.row(i), weights, shift) > 0 ? 1 : -1; + } +} + +void CV_SVMSGDTrainTest::run( int /*start_from*/ ) +{ + cv::Ptr svmsgd = SVMSGD::create(); + + svmsgd->setOptimalParameters(SVMSGD::ASGD); + + svmsgd->train( data ); + + Mat responses; + + svmsgd->predict(testSamples, responses); + + int errCount = 0; + int testSamplesCount = testSamples.rows; + + for (int i = 0; i < testSamplesCount; i++) + { + if (responses.at(i) * testResponses.at(i) < 0 ) + errCount++; + } + + float err = (float)errCount / testSamplesCount; + std::cout << "err " << err << std::endl; + + if ( err > 0.01 ) + { + ts->set_failed_test_info( cvtest::TS::FAIL_BAD_ACCURACY ); + } +} + +float CV_SVMSGDTrainTest::decisionFunction(Mat sample, Mat weights, float shift) +{ + return sample.dot(weights) + shift; +} + +TEST(ML_SVMSGD, train0) +{ + int varCount = 2; + + Mat weights; + weights.create(1, varCount, CV_32FC1); + weights.at(0) = 1; + weights.at(1) = 0; + + float shift = 5; + + CV_SVMSGDTrainTest test(weights, shift); + test.safe_run(); +} + +TEST(ML_SVMSGD, train1) +{ + int varCount = 5; + + Mat weights; + weights.create(1, varCount, CV_32FC1); + + float lowerLimit = -1; + float upperLimit = 1; + cv::RNG rng(0); + rng.fill(weights, RNG::UNIFORM, lowerLimit, upperLimit); + + float shift = rng.uniform(-5.f, 5.f); + + CV_SVMSGDTrainTest test(weights, shift); + test.safe_run(); +} + +TEST(ML_SVMSGD, train2) +{ + int varCount = 100; + + Mat weights; + weights.create(1, varCount, CV_32FC1); + + float lowerLimit = -1; + float upperLimit = 1; + cv::RNG rng(0); + rng.fill(weights, RNG::UNIFORM, lowerLimit, upperLimit); + + float shift = rng.uniform(-1000.f, 1000.f); + + CV_SVMSGDTrainTest test(weights, shift); + test.safe_run(); +} diff --git a/modules/ts/src/ts_gtest.cpp b/modules/ts/src/ts_gtest.cpp index 29a3996be8..5604eb7e62 100644 --- a/modules/ts/src/ts_gtest.cpp +++ b/modules/ts/src/ts_gtest.cpp @@ -5659,7 +5659,7 @@ class TestCaseNameIs { // Returns true iff the name of test_case matches name_. bool operator()(const TestCase* test_case) const { - return test_case != NULL && strcmp(test_case->name(), name_.c_str()) == 0; + return test_case != NULL && strcmp(test_case->name(), name_.c_str()) == 0; } private: diff --git a/samples/cpp/train_svmsgd.cpp b/samples/cpp/train_svmsgd.cpp new file mode 100644 index 0000000000..cee821739f --- /dev/null +++ b/samples/cpp/train_svmsgd.cpp @@ -0,0 +1,226 @@ +#include +#include "opencv2/video/tracking.hpp" +#include "opencv2/imgproc/imgproc.hpp" +#include "opencv2/highgui/highgui.hpp" + +using namespace cv; +using namespace cv::ml; + +#define WIDTH 841 +#define HEIGHT 594 + +struct Data +{ + Mat img; + Mat samples; + Mat responses; + RNG rng; + //Point points[2]; + + Data() + { + img = Mat::zeros(HEIGHT, WIDTH, CV_8UC3); + imshow("Train svmsgd", img); + } +}; + +bool doTrain(const Mat samples,const Mat responses, Mat &weights, float &shift); +bool findPointsForLine(const Mat &weights, float shift, Point (&points)[2]); +bool findCrossPoint(const Mat &weights, float shift, const std::pair &segment, Point &crossPoint); +void fillSegments(std::vector > &segments); +void redraw(Data data, const Point points[2]); +void addPointsRetrainAndRedraw(Data &data, int x, int y); + + +bool doTrain( const Mat samples, const Mat responses, Mat &weights, float &shift) +{ + cv::Ptr svmsgd = SVMSGD::create(); + svmsgd->setOptimalParameters(SVMSGD::ASGD); + svmsgd->setTermCriteria(TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 50000, 0.0000001)); + svmsgd->setLambda(0.01); + svmsgd->setGamma0(1); + // svmsgd->setC(5); + + cv::Ptr train_data = TrainData::create( samples, cv::ml::ROW_SAMPLE, responses ); + svmsgd->train( train_data ); + + if (svmsgd->isTrained()) + { + weights = svmsgd->getWeights(); + shift = svmsgd->getShift(); + + std::cout << weights << std::endl; + std::cout << shift << std::endl; + + return true; + } + return false; +} + + +bool findCrossPoint(const Mat &weights, float shift, const std::pair &segment, Point &crossPoint) +{ + int x = 0; + int y = 0; + //с (0,0) всё плохо + if (segment.first.x == segment.second.x && weights.at(1) != 0) + { + x = segment.first.x; + y = -(weights.at(0) * x + shift) / weights.at(1); + if (y >= 0 && y <= HEIGHT) + { + crossPoint.x = x; + crossPoint.y = y; + return true; + } + } + else if (segment.first.y == segment.second.y && weights.at(0) != 0) + { + y = segment.first.y; + x = - (weights.at(1) * y + shift) / weights.at(0); + if (x >= 0 && x <= WIDTH) + { + crossPoint.x = x; + crossPoint.y = y; + return true; + } + } + return false; +} + +bool findPointsForLine(const Mat &weights, float shift, Point (&points)[2]) +{ + if (weights.empty()) + { + return false; + } + + int foundPointsCount = 0; + std::vector > segments; + fillSegments(segments); + + for (int i = 0; i < 4; i++) + { + if (findCrossPoint(weights, shift, segments[i], points[foundPointsCount])) + foundPointsCount++; + if (foundPointsCount > 2) + break; + } + return true; +} + +void fillSegments(std::vector > &segments) +{ + std::pair curSegment; + + curSegment.first = Point(0,0); + curSegment.second = Point(0,HEIGHT); + segments.push_back(curSegment); + + curSegment.first = Point(0,0); + curSegment.second = Point(WIDTH,0); + segments.push_back(curSegment); + + curSegment.first = Point(WIDTH,0); + curSegment.second = Point(WIDTH,HEIGHT); + segments.push_back(curSegment); + + curSegment.first = Point(0,HEIGHT); + curSegment.second = Point(WIDTH,HEIGHT); + segments.push_back(curSegment); +} + +void redraw(Data data, const Point points[2]) +{ + data.img = Mat::zeros(HEIGHT, WIDTH, CV_8UC3); + Point center; + int radius = 3; + Scalar color; + for (int i = 0; i < data.samples.rows; i++) + { + center.x = data.samples.at(i,0); + center.y = data.samples.at(i,1); + color = (data.responses.at(i) > 0) ? Scalar(128,128,0) : Scalar(0,128,128); + circle(data.img, center, radius, color, 5); + } + line(data.img, points[0],points[1],cv::Scalar(1,255,1)); + + imshow("Train svmsgd", data.img); +} + +void addPointsRetrainAndRedraw(Data &data, int x, int y) +{ + + Mat currentSample(1, 2, CV_32F); + //start +/* + Mat _weights; + _weights.create(1, 2, CV_32FC1); + _weights.at(0) = 1; + _weights.at(1) = -1; + + int _x, _y; + + for (int i=0;i<199;i++) + { + _x = data.rng.uniform(0,800); + _y = data.rng.uniform(0,500);*/ + currentSample.at(0,0) = x; + currentSample.at(0,1) = y; + //if (currentSample.dot(_weights) > 0) + //data.responses.push_back(1); + // else data.responses.push_back(-1); + + //finish + data.samples.push_back(currentSample); + + + + Mat weights(1, 2, CV_32F); + float shift = 0; + + if (doTrain(data.samples, data.responses, weights, shift)) + { + Point points[2]; + shift = 0; + + findPointsForLine(weights, shift, points); + + redraw(data, points); + } +} + + +static void onMouse( int event, int x, int y, int, void* pData) +{ + Data &data = *(Data*)pData; + + switch( event ) + { + case CV_EVENT_LBUTTONUP: + data.responses.push_back(1); + addPointsRetrainAndRedraw(data, x, y); + + break; + + case CV_EVENT_RBUTTONDOWN: + data.responses.push_back(-1); + addPointsRetrainAndRedraw(data, x, y); + break; + } + +} + +int main() +{ + + Data data; + + setMouseCallback( "Train svmsgd", onMouse, &data ); + waitKey(); + + + + + return 0; +} From acd74037b32ddec5e1bf28dea5ec161570daa227 Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Wed, 3 Feb 2016 15:31:05 +0300 Subject: [PATCH 03/19] Increasing the dimension of features space in the SVMSGD::train function. --- include/opencv2/opencv.hpp | 1 - modules/ml/include/opencv2/ml.hpp | 115 +++++++++ modules/ml/include/opencv2/ml/svmsgd.hpp | 134 ---------- modules/ml/src/precomp.hpp | 1 - modules/ml/src/svmsgd.cpp | 305 +++++++++++++++-------- modules/ml/test/test_precomp.hpp | 1 - modules/ml/test/test_svmsgd.cpp | 28 ++- samples/cpp/train_svmsgd.cpp | 146 +++++------ 8 files changed, 397 insertions(+), 334 deletions(-) delete mode 100644 modules/ml/include/opencv2/ml/svmsgd.hpp diff --git a/include/opencv2/opencv.hpp b/include/opencv2/opencv.hpp index e411621d0c..49b6a6691f 100644 --- a/include/opencv2/opencv.hpp +++ b/include/opencv2/opencv.hpp @@ -75,7 +75,6 @@ #endif #ifdef HAVE_OPENCV_ML #include "opencv2/ml.hpp" -#include "opencv2/ml/svmsgd.hpp" #endif #endif diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index 791f580093..3a04368e1e 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1496,6 +1496,121 @@ public: CV_WRAP static Ptr create(); }; + +/****************************************************************************************\ +* Stochastic Gradient Descent SVM Classifier * +\****************************************************************************************/ + +/*! +@brief Stochastic Gradient Descent SVM classifier + +SVMSGD provides a fast and easy-to-use implementation of the SVM classifier using the Stochastic Gradient Descent approach, as presented in @cite bottou2010large. +The gradient descent show amazing performance for large-scale problems, reducing the computing time. + + + +First, create the SVMSGD object. Set parametrs of model (type, lambda, gamma0, c) using the functions setType, setLambda, setGamma0 and setC or the function setOptimalParametrs. +Recommended model type is ASGD. + +Then the SVM model can be trained using the train features and the correspondent labels. + +After that, the label of a new feature vector can be predicted using the predict function. + +@code +// Initialize object +SVMSGD SvmSgd; + +// Train the Stochastic Gradient Descent SVM +SvmSgd.train(trainFeatures, labels); + +// Predict label for the new feature vector (1xM) +predictedLabel = SvmSgd.predict(newFeatureVector); +@endcode + +*/ + +class CV_EXPORTS_W SVMSGD : public cv::ml::StatModel +{ +public: + + /** SVMSGD type. + ASGD is often the preferable choice. */ + enum SvmsgdType + { + ILLEGAL_VALUE, + SGD, //!Stochastic Gradient Descent + ASGD //!Average Stochastic Gradient Descent + }; + + /** + * @return the weights of the trained model (decision function f(x) = weights * x + shift). + */ + CV_WRAP virtual Mat getWeights() = 0; + + /** + * @return the shift of the trained model (decision function f(x) = weights * x + shift). + */ + CV_WRAP virtual float getShift() = 0; + + + /** Creates empty model. + Use StatModel::train to train the model. Since %SVMSGD has several parameters, you may want to + find the best parameters for your problem or use setOptimalParameters() to set some default parameters. + */ + CV_WRAP static Ptr create(); + + /** Function sets optimal parameters values for chosen SVM SGD model. + * If chosen type is ASGD, function sets the following values for parameters of model: + * lambda = 0.00001; + * gamma0 = 0.05; + * c = 0.75; + * termCrit.maxCount = 100000; + * termCrit.epsilon = 0.00001; + * + * If SGD: + * lambda = 0.0001; + * gamma0 = 0.05; + * c = 1; + * termCrit.maxCount = 100000; + * termCrit.epsilon = 0.00001; + * @param type is the type of SVMSGD classifier. Legal values are SvmsgdType::SGD and SvmsgdType::ASGD. + * Recommended value is SvmsgdType::ASGD (by default). + */ + CV_WRAP virtual void setOptimalParameters(int type = ASGD) = 0; + + /** %Algorithm type, one of SVMSGD::SvmsgdType. */ + /** @see setAlgorithmType */ + CV_WRAP virtual int getType() const = 0; + /** @copybrief getAlgorithmType @see getAlgorithmType */ + CV_WRAP virtual void setType(int type) = 0; + + /** Parameter _Lambda_ of a %SVMSGD optimization problem. Default value is 0. */ + /** @see setLambda */ + CV_WRAP virtual float getLambda() const = 0; + /** @copybrief getLambda @see getLambda */ + CV_WRAP virtual void setLambda(float lambda) = 0; + + /** Parameter _Gamma0_ of a %SVMSGD optimization problem. Default value is 0. */ + /** @see setGamma0 */ + CV_WRAP virtual float getGamma0() const = 0; + CV_WRAP virtual void setGamma0(float gamma0) = 0; + + /** Parameter _C_ of a %SVMSGD optimization problem. Default value is 0. */ + /** @see setC */ + CV_WRAP virtual float getC() const = 0; + /** @copybrief getC @see getC */ + CV_WRAP virtual void setC(float c) = 0; + + /** @brief Termination criteria of the training algorithm. + You can specify the maximum number of iterations (maxCount) and/or how much the error could + change between the iterations to make the algorithm continue (epsilon).*/ + /** @see setTermCriteria */ + CV_WRAP virtual TermCriteria getTermCriteria() const = 0; + /** @copybrief getTermCriteria @see getTermCriteria */ + CV_WRAP virtual void setTermCriteria(const cv::TermCriteria &val) = 0; +}; + + /****************************************************************************************\ * Auxilary functions declarations * \****************************************************************************************/ diff --git a/modules/ml/include/opencv2/ml/svmsgd.hpp b/modules/ml/include/opencv2/ml/svmsgd.hpp deleted file mode 100644 index f61a905963..0000000000 --- a/modules/ml/include/opencv2/ml/svmsgd.hpp +++ /dev/null @@ -1,134 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// License Agreement -// For Open Source Computer Vision Library -// -// Copyright (C) 2000, Intel Corporation, all rights reserved. -// Copyright (C) 2013, OpenCV Foundation, all rights reserved. -// Copyright (C) 2014, Itseez Inc, all rights reserved. -// Third party copyrights are property of their respective owners. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of the copyright holders may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ - -#ifndef __OPENCV_ML_SVMSGD_HPP__ -#define __OPENCV_ML_SVMSGD_HPP__ - -#ifdef __cplusplus - -#include "opencv2/ml.hpp" - -namespace cv -{ -namespace ml -{ - - -/****************************************************************************************\ -* Stochastic Gradient Descent SVM Classifier * -\****************************************************************************************/ - -/*! -@brief Stochastic Gradient Descent SVM classifier - -SVMSGD provides a fast and easy-to-use implementation of the SVM classifier using the Stochastic Gradient Descent approach, as presented in @cite bottou2010large. -The gradient descent show amazing performance for large-scale problems, reducing the computing time. This allows a fast and reliable online update of the classifier for each new feature which -is fundamental when dealing with variations of data over time (like weather and illumination changes in videosurveillance, for example). - -First, create the SVMSGD object. To enable the online update, a value for updateFrequency should be defined. - -Then the SVM model can be trained using the train features and the correspondent labels. - -After that, the label of a new feature vector can be predicted using the predict function. If the updateFrequency was defined in the constructor, the predict function will update the weights automatically. - -@code -// Initialize object -SVMSGD SvmSgd; - -// Train the Stochastic Gradient Descent SVM -SvmSgd.train(trainFeatures, labels); - -// Predict label for the new feature vector (1xM) -predictedLabel = SvmSgd.predict(newFeatureVector); -@endcode - -*/ - -class CV_EXPORTS_W SVMSGD : public cv::ml::StatModel -{ -public: - - enum SvmsgdType - { - ILLEGAL_VALUE, - SGD, //Stochastic Gradient Descent - ASGD //Average Stochastic Gradient Descent - }; - - /** - * @return the weights of the trained model. - */ - CV_WRAP virtual Mat getWeights() = 0; - - CV_WRAP virtual float getShift() = 0; - - CV_WRAP static Ptr create(); - - CV_WRAP virtual void setOptimalParameters(int type = ASGD) = 0; - - CV_WRAP virtual int getType() const = 0; - - CV_WRAP virtual void setType(int type) = 0; - - CV_WRAP virtual float getLambda() const = 0; - - CV_WRAP virtual void setLambda(float lambda) = 0; - - CV_WRAP virtual float getGamma0() const = 0; - - CV_WRAP virtual void setGamma0(float gamma0) = 0; - - CV_WRAP virtual float getC() const = 0; - - CV_WRAP virtual void setC(float c) = 0; - - CV_WRAP virtual cv::TermCriteria getTermCriteria() const = 0; - - CV_WRAP virtual void setTermCriteria(const cv::TermCriteria &val) = 0; -}; - -} //ml -} //cv - -#endif // __clpusplus -#endif // __OPENCV_ML_SVMSGD_HPP diff --git a/modules/ml/src/precomp.hpp b/modules/ml/src/precomp.hpp index 9318e4a78c..0a5fa06035 100644 --- a/modules/ml/src/precomp.hpp +++ b/modules/ml/src/precomp.hpp @@ -45,7 +45,6 @@ #include "opencv2/ml.hpp" #include "opencv2/core/core_c.h" #include "opencv2/core/utility.hpp" -#include "opencv2/ml/svmsgd.hpp" #include "opencv2/core/private.hpp" #include diff --git a/modules/ml/src/svmsgd.cpp b/modules/ml/src/svmsgd.cpp index 91377cfc4f..5c75023b7d 100644 --- a/modules/ml/src/svmsgd.cpp +++ b/modules/ml/src/svmsgd.cpp @@ -42,6 +42,12 @@ #include "precomp.hpp" #include "limits" +//#include "math.h" + +#include + +using std::cout; +using std::endl; /****************************************************************************************\ * Stochastic Gradient Descent SVM Classifier * @@ -64,7 +70,7 @@ public: virtual float predict( InputArray samples, OutputArray results=noArray(), int flags = 0 ) const; - virtual bool isClassifier() const { return params.svmsgdType == SGD || params.svmsgdType == ASGD; } + virtual bool isClassifier() const; virtual bool isTrained() const; @@ -93,22 +99,29 @@ public: CV_IMPL_PROPERTY(float, C, params.c) CV_IMPL_PROPERTY_S(cv::TermCriteria, TermCriteria, params.termCrit) - private: - void updateWeights(InputArray sample, bool is_first_class, float gamma); - float calcShift(InputArray trainSamples, InputArray trainResponses) const; +private: + void updateWeights(InputArray sample, bool isFirstClass, float gamma, Mat weights); + std::pair areClassesEmpty(Mat responses); + void writeParams( FileStorage& fs ) const; + void readParams( const FileNode& fn ); + static inline bool isFirstClass(float val) { return val > 0; } + static void normalizeSamples(Mat &matrix, Mat &multiplier, Mat &average); + + float calcShift(InputArray _samples, InputArray _responses) const; + + static void makeExtendedTrainSamples(const Mat trainSamples, Mat &extendedTrainSamples, Mat &multiplier); + + // Vector with SVM weights Mat weights_; float shift_; - // Random index generation - RNG rng_; - // Parameters for learning struct SVMSGDParams { @@ -127,97 +140,88 @@ Ptr SVMSGD::create() return makePtr(); } - -bool SVMSGDImpl::train(const Ptr& data, int) +std::pair SVMSGDImpl::areClassesEmpty(Mat responses) { - clear(); - - Mat trainSamples = data->getTrainSamples(); + CV_Assert(responses.cols == 1); + std::pair emptyInClasses(true, true); + int limit_index = responses.rows; - // Initialize varCount - int trainSamplesCount_ = trainSamples.rows; - int varCount = trainSamples.cols; + for(int index = 0; index < limit_index; index++) + { + if (isFirstClass(responses.at(index))) + emptyInClasses.first = false; + else + emptyInClasses.second = false; - // Initialize weights vector with zeros - weights_ = Mat::zeros(1, varCount, CV_32F); + if (!emptyInClasses.first && ! emptyInClasses.second) + break; + } - Mat trainResponses = data->getTrainResponses(); // (trainSamplesCount x 1) matrix + return emptyInClasses; +} - std::pair are_empty = areClassesEmpty(trainResponses); +void SVMSGDImpl::normalizeSamples(Mat &samples, Mat &multiplier, Mat &average) +{ + int featuresCount = samples.cols; + int samplesCount = samples.rows; - if ( are_empty.first && are_empty.second ) + average = Mat(1, featuresCount, samples.type()); + for (int featureIndex = 0; featureIndex < featuresCount; featureIndex++) { - weights_.release(); - return false; + average.at(featureIndex) = mean(samples.col(featureIndex))[0]; } - if ( are_empty.first || are_empty.second ) + + for (int sampleIndex = 0; sampleIndex < samplesCount; sampleIndex++) { - shift_ = are_empty.first ? -1 : 1; - return true; + samples.row(sampleIndex) -= average; } - - Mat currentSample; - float gamma = 0; - Mat lastWeights = Mat::zeros(1, varCount, CV_32F); //weights vector for calculating terminal criterion - Mat averageWeights; //average weights vector for ASGD model - double err = DBL_MAX; - if (params.svmsgdType == ASGD) + Mat featureNorm(1, featuresCount, samples.type()); + for (int featureIndex = 0; featureIndex < featuresCount; featureIndex++) { - averageWeights = Mat::zeros(1, varCount, CV_32F); + featureNorm.at(featureIndex) = norm(samples.col(featureIndex)); } - // Stochastic gradient descent SVM - for (int iter = 0; (iter < params.termCrit.maxCount)&&(err > params.termCrit.epsilon); iter++) + multiplier = sqrt(samplesCount) / featureNorm; + for (int sampleIndex = 0; sampleIndex < samplesCount; sampleIndex++) { - //generate sample number - int randomNumber = rng_.uniform(0, trainSamplesCount_); - - currentSample = trainSamples.row(randomNumber); - - //update gamma - gamma = params.gamma0 * std::pow((1 + params.lambda * params.gamma0 * (float)iter), (-params.c)); + samples.row(sampleIndex) = samples.row(sampleIndex).mul(multiplier); + } +} - bool is_first_class = isFirstClass(trainResponses.at(randomNumber)); - updateWeights( currentSample, is_first_class, gamma ); +void SVMSGDImpl::makeExtendedTrainSamples(const Mat trainSamples, Mat &extendedTrainSamples, Mat &multiplier) +{ + Mat normalisedTrainSamples = trainSamples.clone(); + int samplesCount = normalisedTrainSamples.rows; - //average weights (only for ASGD model) - if (params.svmsgdType == ASGD) - { - averageWeights = ((float)iter/ (1 + (float)iter)) * averageWeights + weights_ / (1 + (float) iter); - } + Mat average; - err = norm(weights_ - lastWeights); - weights_.copyTo(lastWeights); - } + normalizeSamples(normalisedTrainSamples, multiplier, average); - if (params.svmsgdType == ASGD) - { - weights_ = averageWeights; - } - - shift_ = calcShift(trainSamples, trainResponses); + Mat onesCol = Mat::ones(samplesCount, 1, CV_32F); + cv::hconcat(normalisedTrainSamples, onesCol, extendedTrainSamples); - return true; + //cout << "SVMSGDImpl::makeExtendedTrainSamples average: \n" << average << endl; + //cout << "SVMSGDImpl::makeExtendedTrainSamples multiplier: \n" << multiplier << endl; } -std::pair SVMSGDImpl::areClassesEmpty(Mat responses) + +void SVMSGDImpl::updateWeights(InputArray _sample, bool firstClass, float gamma, Mat weights) { - std::pair are_classes_empty(true, true); - int limit_index = responses.rows; + Mat sample = _sample.getMat(); - for(int index = 0; index < limit_index; index++) - { - if (isFirstClass(responses.at(index,0))) - are_classes_empty.first = false; - else - are_classes_empty.second = false; + int response = firstClass ? 1 : -1; // ensure that trainResponses are -1 or 1 - if (!are_classes_empty.first && ! are_classes_empty.second) - break; + if ( sample.dot(weights) * response > 1) + { + // Not a support vector, only apply weight decay + weights *= (1.f - gamma * params.lambda); + } + else + { + // It's a support vector, add it to the weights + weights -= (gamma * params.lambda) * weights - (gamma * response) * sample; } - - return are_classes_empty; } float SVMSGDImpl::calcShift(InputArray _samples, InputArray _responses) const @@ -232,12 +236,12 @@ float SVMSGDImpl::calcShift(InputArray _samples, InputArray _responses) const for (int samplesIndex = 0; samplesIndex < trainSamplesCount; samplesIndex++) { Mat currentSample = trainSamples.row(samplesIndex); - float scalar_product = currentSample.dot(weights_); + float dotProduct = currentSample.dot(weights_); - bool is_first_class = isFirstClass(trainResponses.at(samplesIndex)); - int index = is_first_class ? 0:1; - float sign_to_mul = is_first_class ? 1 : -1; - float cur_distance = scalar_product * sign_to_mul ; + bool firstClass = isFirstClass(trainResponses.at(samplesIndex)); + int index = firstClass ? 0:1; + float signToMul = firstClass ? 1 : -1; + float cur_distance = dotProduct * signToMul; if (cur_distance < distance_to_classes[index]) { @@ -245,10 +249,109 @@ float SVMSGDImpl::calcShift(InputArray _samples, InputArray _responses) const } } - //todo: areClassesEmpty(); make const; return -(distance_to_classes[0] - distance_to_classes[1]) / 2.f; } +bool SVMSGDImpl::train(const Ptr& data, int) +{ + //cout << "SVMSGDImpl::train begin" << endl; + clear(); + CV_Assert( isClassifier() ); //toDo: consider + + Mat trainSamples = data->getTrainSamples(); + + //cout << "SVMSGDImpl::train trainSamples: \n" << trainSamples << endl; + + int featureCount = trainSamples.cols; + Mat trainResponses = data->getTrainResponses(); // (trainSamplesCount x 1) matrix + + //cout << "SVMSGDImpl::train trainresponses: \n" << trainResponses << endl; + + std::pair areEmpty = areClassesEmpty(trainResponses); + + //cout << "SVMSGDImpl::train areEmpty" << areEmpty.first << "," << areEmpty.second << endl; + + if ( areEmpty.first && areEmpty.second ) + { + return false; + } + if ( areEmpty.first || areEmpty.second ) + { + weights_ = Mat::zeros(1, featureCount, CV_32F); + shift_ = areEmpty.first ? -1 : 1; + return true; + } + + Mat extendedTrainSamples; + Mat multiplier; + makeExtendedTrainSamples(trainSamples, extendedTrainSamples, multiplier); + + //cout << "SVMSGDImpl::train extendedTrainSamples: \n" << extendedTrainSamples << endl; + + int extendedTrainSamplesCount = extendedTrainSamples.rows; + int extendedFeatureCount = extendedTrainSamples.cols; + + Mat extendedWeights = Mat::zeros(1, extendedFeatureCount, CV_32F); // Initialize extendedWeights vector with zeros + Mat previousWeights = Mat::zeros(1, extendedFeatureCount, CV_32F); //extendedWeights vector for calculating terminal criterion + Mat averageExtendedWeights; //average extendedWeights vector for ASGD model + if (params.svmsgdType == ASGD) + { + averageExtendedWeights = Mat::zeros(1, extendedFeatureCount, CV_32F); + } + + RNG rng(0); + + int maxCount = (params.termCrit.type & TermCriteria::COUNT) ? params.termCrit.maxCount : INT_MAX; + double epsilon = (params.termCrit.type & TermCriteria::EPS) ? params.termCrit.epsilon : 0; + + double err = DBL_MAX; + // Stochastic gradient descent SVM + for (int iter = 0; (iter < maxCount) && (err > epsilon); iter++) + { + int randomNumber = rng.uniform(0, extendedTrainSamplesCount); //generate sample number + + Mat currentSample = extendedTrainSamples.row(randomNumber); + bool firstClass = isFirstClass(trainResponses.at(randomNumber)); + + float gamma = params.gamma0 * std::pow((1 + params.lambda * params.gamma0 * (float)iter), (-params.c)); //update gamma + + updateWeights( currentSample, firstClass, gamma, extendedWeights ); + + //average weights (only for ASGD model) + if (params.svmsgdType == ASGD) + { + averageExtendedWeights = ((float)iter/ (1 + (float)iter)) * averageExtendedWeights + extendedWeights / (1 + (float) iter); + err = norm(averageExtendedWeights - previousWeights); + averageExtendedWeights.copyTo(previousWeights); + } + else + { + err = norm(extendedWeights - previousWeights); + extendedWeights.copyTo(previousWeights); + } + } + + if (params.svmsgdType == ASGD) + { + extendedWeights = averageExtendedWeights; + } + + //cout << "SVMSGDImpl::train extendedWeights: \n" << extendedWeights << endl; + + Rect roi(0, 0, featureCount, 1); + weights_ = extendedWeights(roi); + weights_ = weights_.mul(1/multiplier); + + //cout << "SVMSGDImpl::train weights: \n" << weights_ << endl; + + shift_ = calcShift(trainSamples, trainResponses); + + //cout << "SVMSGDImpl::train shift = " << shift_ << endl; + + return true; +} + + float SVMSGDImpl::predict( InputArray _samples, OutputArray _results, int ) const { float result = 0; @@ -269,37 +372,21 @@ float SVMSGDImpl::predict( InputArray _samples, OutputArray _results, int ) cons results = Mat(1, 1, CV_32F, &result); } - Mat currentSample; - float criterion = 0; - for (int sampleIndex = 0; sampleIndex < nSamples; sampleIndex++) { - currentSample = samples.row(sampleIndex); - criterion = currentSample.dot(weights_) + shift_; + Mat currentSample = samples.row(sampleIndex); + float criterion = currentSample.dot(weights_) + shift_; results.at(sampleIndex) = (criterion >= 0) ? 1 : -1; } return result; } -void SVMSGDImpl::updateWeights(InputArray _sample, bool is_first_class, float gamma) +bool SVMSGDImpl::isClassifier() const { - Mat sample = _sample.getMat(); - - int responce = is_first_class ? 1 : -1; // ensure that trainResponses are -1 or 1 - - if ( sample.dot(weights_) * responce > 1) - { - // Not a support vector, only apply weight decay - weights_ *= (1.f - gamma * params.lambda); - } - else - { - // It's a support vector, add it to the weights - weights_ -= (gamma * params.lambda) * weights_ - gamma * responce * sample; - //std::cout << "sample " << sample << std::endl; - //std::cout << "weights_ " << weights_ << std::endl; - } + return (params.svmsgdType == SGD || params.svmsgdType == ASGD) + && + (params.lambda > 0) && (params.gamma0 > 0) && (params.c >= 0); } bool SVMSGDImpl::isTrained() const @@ -314,8 +401,8 @@ void SVMSGDImpl::write(FileStorage& fs) const writeParams( fs ); - fs << "shift" << shift_; fs << "weights" << weights_; + fs << "shift" << shift_; } void SVMSGDImpl::writeParams( FileStorage& fs ) const @@ -359,8 +446,8 @@ void SVMSGDImpl::read(const FileNode& fn) readParams(fn); - shift_ = (float) fn["shift"]; fn["weights"] >> weights_; + fn["shift"] >> shift_; } void SVMSGDImpl::readParams( const FileNode& fn ) @@ -393,21 +480,19 @@ void SVMSGDImpl::readParams( const FileNode& fn ) (params.termCrit.maxCount > 0 ? TermCriteria::COUNT : 0); } else - params.termCrit = TermCriteria( TermCriteria::EPS + TermCriteria::COUNT, 1000, FLT_EPSILON ); + params.termCrit = TermCriteria( TermCriteria::EPS + TermCriteria::COUNT, 100000, FLT_EPSILON ); } void SVMSGDImpl::clear() { weights_.release(); - shift_ = 0; } SVMSGDImpl::SVMSGDImpl() { clear(); - rng_(0); params.svmsgdType = ILLEGAL_VALUE; @@ -426,20 +511,20 @@ void SVMSGDImpl::setOptimalParameters(int type) { case SGD: params.svmsgdType = SGD; - params.lambda = 0.00001; + params.lambda = 0.0001; params.gamma0 = 0.05; params.c = 1; - params.termCrit.maxCount = 50000; - params.termCrit.epsilon = 0.00000001; + params.termCrit.maxCount = 100000; + params.termCrit.epsilon = 0.00001; break; case ASGD: params.svmsgdType = ASGD; params.lambda = 0.00001; - params.gamma0 = 0.5; + params.gamma0 = 0.05; params.c = 0.75; params.termCrit.maxCount = 100000; - params.termCrit.epsilon = 0.000001; + params.termCrit.epsilon = 0.00001; break; default: diff --git a/modules/ml/test/test_precomp.hpp b/modules/ml/test/test_precomp.hpp index 18cee968fb..3147a9d96c 100644 --- a/modules/ml/test/test_precomp.hpp +++ b/modules/ml/test/test_precomp.hpp @@ -13,7 +13,6 @@ #include #include "opencv2/ts.hpp" #include "opencv2/ml.hpp" -#include "opencv2/ml/svmsgd.hpp" #include "opencv2/core/core_c.h" #define CV_NBAYES "nbayes" diff --git a/modules/ml/test/test_svmsgd.cpp b/modules/ml/test/test_svmsgd.cpp index 9f4aafc08b..5445271eec 100644 --- a/modules/ml/test/test_svmsgd.cpp +++ b/modules/ml/test/test_svmsgd.cpp @@ -52,7 +52,7 @@ using cv::ml::TrainData; class CV_SVMSGDTrainTest : public cvtest::BaseTest { public: - CV_SVMSGDTrainTest(Mat _weights, float _shift); + CV_SVMSGDTrainTest(Mat _weights, float shift); private: virtual void run( int start_from ); float decisionFunction(Mat sample, Mat weights, float shift); @@ -60,7 +60,7 @@ private: cv::Ptr data; cv::Mat testSamples; cv::Mat testResponses; - static const int TEST_VALUE_LIMIT = 50; + static const int TEST_VALUE_LIMIT = 500; }; CV_SVMSGDTrainTest::CV_SVMSGDTrainTest(Mat weights, float shift) @@ -81,6 +81,11 @@ CV_SVMSGDTrainTest::CV_SVMSGDTrainTest(Mat weights, float shift) responses.at( sampleIndex ) = decisionFunction(samples.row(sampleIndex), weights, shift) > 0 ? 1 : -1; } + + + std::cout << "real weights\n" << weights/norm(weights) << "\n" << std::endl; + std::cout << "real shift \n" << shift/norm(weights) << "\n" << std::endl; + data = TrainData::create( samples, cv::ml::ROW_SAMPLE, responses ); int testSamplesCount = 100000; @@ -100,8 +105,9 @@ void CV_SVMSGDTrainTest::run( int /*start_from*/ ) cv::Ptr svmsgd = SVMSGD::create(); svmsgd->setOptimalParameters(SVMSGD::ASGD); + svmsgd->setTermCriteria(TermCriteria(TermCriteria::EPS, 0, 0.00005)); - svmsgd->train( data ); + svmsgd->train(data); Mat responses; @@ -116,6 +122,12 @@ void CV_SVMSGDTrainTest::run( int /*start_from*/ ) errCount++; } + + float normW = norm(svmsgd->getWeights()); + + std::cout << "found weights\n" << svmsgd->getWeights()/normW << "\n" << std::endl; + std::cout << "found shift \n" << svmsgd->getShift()/normW << "\n" << std::endl; + float err = (float)errCount / testSamplesCount; std::cout << "err " << err << std::endl; @@ -138,8 +150,8 @@ TEST(ML_SVMSGD, train0) weights.create(1, varCount, CV_32FC1); weights.at(0) = 1; weights.at(1) = 0; - - float shift = 5; + cv::RNG rng(1); + float shift = rng.uniform(-varCount, varCount); CV_SVMSGDTrainTest test(weights, shift); test.safe_run(); @@ -157,7 +169,7 @@ TEST(ML_SVMSGD, train1) cv::RNG rng(0); rng.fill(weights, RNG::UNIFORM, lowerLimit, upperLimit); - float shift = rng.uniform(-5.f, 5.f); + float shift = rng.uniform(-varCount, varCount); CV_SVMSGDTrainTest test(weights, shift); test.safe_run(); @@ -175,8 +187,8 @@ TEST(ML_SVMSGD, train2) cv::RNG rng(0); rng.fill(weights, RNG::UNIFORM, lowerLimit, upperLimit); - float shift = rng.uniform(-1000.f, 1000.f); + float shift = rng.uniform(-varCount, varCount); - CV_SVMSGDTrainTest test(weights, shift); + CV_SVMSGDTrainTest test(weights,shift); test.safe_run(); } diff --git a/samples/cpp/train_svmsgd.cpp b/samples/cpp/train_svmsgd.cpp index cee821739f..19c30ffca9 100644 --- a/samples/cpp/train_svmsgd.cpp +++ b/samples/cpp/train_svmsgd.cpp @@ -12,10 +12,8 @@ using namespace cv::ml; struct Data { Mat img; - Mat samples; - Mat responses; - RNG rng; - //Point points[2]; + Mat samples; //Set of train samples. Contains points on image + Mat responses; //Set of responses for train samples Data() { @@ -24,24 +22,36 @@ struct Data } }; -bool doTrain(const Mat samples,const Mat responses, Mat &weights, float &shift); -bool findPointsForLine(const Mat &weights, float shift, Point (&points)[2]); -bool findCrossPoint(const Mat &weights, float shift, const std::pair &segment, Point &crossPoint); -void fillSegments(std::vector > &segments); +//Train with SVMSGD algorithm +//(samples, responses) is a train set +//weights is a required vector for decision function of SVMSGD algorithm +bool doTrain(const Mat samples, const Mat responses, Mat &weights, float &shift); + +//function finds two points for drawing line (wx = 0) +bool findPointsForLine(const Mat &weights, float shift, Point (&points)[2], int width, int height); + +// function finds cross point of line (wx = 0) and segment ( (y = HEIGHT, 0 <= x <= WIDTH) or (x = WIDTH, 0 <= y <= HEIGHT) ) +bool findCrossPointWithBorders(const Mat &weights, float shift, const std::pair &segment, Point &crossPoint); + +//segments' initialization ( (y = HEIGHT, 0 <= x <= WIDTH) and (x = WIDTH, 0 <= y <= HEIGHT) ) +void fillSegments(std::vector > &segments, int width, int height); + +//redraw points' set and line (wx = 0) void redraw(Data data, const Point points[2]); -void addPointsRetrainAndRedraw(Data &data, int x, int y); + +//add point in train set, train SVMSGD algorithm and draw results on image +void addPointRetrainAndRedraw(Data &data, int x, int y); bool doTrain( const Mat samples, const Mat responses, Mat &weights, float &shift) { cv::Ptr svmsgd = SVMSGD::create(); svmsgd->setOptimalParameters(SVMSGD::ASGD); - svmsgd->setTermCriteria(TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 50000, 0.0000001)); - svmsgd->setLambda(0.01); - svmsgd->setGamma0(1); - // svmsgd->setC(5); + svmsgd->setTermCriteria(TermCriteria(TermCriteria::EPS, 0, 0.00000001)); + svmsgd->setLambda(0.00000001); + - cv::Ptr train_data = TrainData::create( samples, cv::ml::ROW_SAMPLE, responses ); + cv::Ptr train_data = TrainData::create(samples, cv::ml::ROW_SAMPLE, responses); svmsgd->train( train_data ); if (svmsgd->isTrained()) @@ -49,36 +59,39 @@ bool doTrain( const Mat samples, const Mat responses, Mat &weights, float &shift weights = svmsgd->getWeights(); shift = svmsgd->getShift(); - std::cout << weights << std::endl; - std::cout << shift << std::endl; - return true; } return false; } -bool findCrossPoint(const Mat &weights, float shift, const std::pair &segment, Point &crossPoint) +bool findCrossPointWithBorders(const Mat &weights, float shift, const std::pair &segment, Point &crossPoint) { int x = 0; int y = 0; - //с (0,0) всё плохо - if (segment.first.x == segment.second.x && weights.at(1) != 0) + int xMin = std::min(segment.first.x, segment.second.x); + int xMax = std::max(segment.first.x, segment.second.x); + int yMin = std::min(segment.first.y, segment.second.y); + int yMax = std::max(segment.first.y, segment.second.y); + + CV_Assert(xMin == xMax || yMin == yMax); + + if (xMin == xMax && weights.at(1) != 0) { - x = segment.first.x; - y = -(weights.at(0) * x + shift) / weights.at(1); - if (y >= 0 && y <= HEIGHT) + x = xMin; + y = std::floor( - (weights.at(0) * x + shift) / weights.at(1)); + if (y >= yMin && y <= yMax) { crossPoint.x = x; crossPoint.y = y; return true; } } - else if (segment.first.y == segment.second.y && weights.at(0) != 0) + else if (yMin == yMax && weights.at(0) != 0) { - y = segment.first.y; - x = - (weights.at(1) * y + shift) / weights.at(0); - if (x >= 0 && x <= WIDTH) + y = yMin; + x = std::floor( - (weights.at(1) * y + shift) / weights.at(0)); + if (x >= xMin && x <= xMax) { crossPoint.x = x; crossPoint.y = y; @@ -88,7 +101,7 @@ bool findCrossPoint(const Mat &weights, float shift, const std::pair > segments; - fillSegments(segments); + fillSegments(segments, width, height); - for (int i = 0; i < 4; i++) + for (uint i = 0; i < segments.size(); i++) { - if (findCrossPoint(weights, shift, segments[i], points[foundPointsCount])) + if (findCrossPointWithBorders(weights, shift, segments[i], points[foundPointsCount])) foundPointsCount++; - if (foundPointsCount > 2) + if (foundPointsCount >= 2) break; } + return true; } -void fillSegments(std::vector > &segments) +void fillSegments(std::vector > &segments, int width, int height) { - std::pair curSegment; + std::pair currentSegment; - curSegment.first = Point(0,0); - curSegment.second = Point(0,HEIGHT); - segments.push_back(curSegment); + currentSegment.first = Point(width, 0); + currentSegment.second = Point(width, height); + segments.push_back(currentSegment); - curSegment.first = Point(0,0); - curSegment.second = Point(WIDTH,0); - segments.push_back(curSegment); + currentSegment.first = Point(0, height); + currentSegment.second = Point(width, height); + segments.push_back(currentSegment); - curSegment.first = Point(WIDTH,0); - curSegment.second = Point(WIDTH,HEIGHT); - segments.push_back(curSegment); + currentSegment.first = Point(0, 0); + currentSegment.second = Point(width, 0); + segments.push_back(currentSegment); - curSegment.first = Point(0,HEIGHT); - curSegment.second = Point(WIDTH,HEIGHT); - segments.push_back(curSegment); + currentSegment.first = Point(0, 0); + currentSegment.second = Point(0, height); + segments.push_back(currentSegment); } void redraw(Data data, const Point points[2]) { - data.img = Mat::zeros(HEIGHT, WIDTH, CV_8UC3); + data.img.setTo(0); Point center; int radius = 3; Scalar color; @@ -143,48 +157,26 @@ void redraw(Data data, const Point points[2]) color = (data.responses.at(i) > 0) ? Scalar(128,128,0) : Scalar(0,128,128); circle(data.img, center, radius, color, 5); } - line(data.img, points[0],points[1],cv::Scalar(1,255,1)); + line(data.img, points[0], points[1],cv::Scalar(1,255,1)); imshow("Train svmsgd", data.img); } -void addPointsRetrainAndRedraw(Data &data, int x, int y) +void addPointRetrainAndRedraw(Data &data, int x, int y) { - Mat currentSample(1, 2, CV_32F); - //start -/* - Mat _weights; - _weights.create(1, 2, CV_32FC1); - _weights.at(0) = 1; - _weights.at(1) = -1; - int _x, _y; - - for (int i=0;i<199;i++) - { - _x = data.rng.uniform(0,800); - _y = data.rng.uniform(0,500);*/ currentSample.at(0,0) = x; currentSample.at(0,1) = y; - //if (currentSample.dot(_weights) > 0) - //data.responses.push_back(1); - // else data.responses.push_back(-1); - - //finish data.samples.push_back(currentSample); - - Mat weights(1, 2, CV_32F); float shift = 0; if (doTrain(data.samples, data.responses, weights, shift)) - { + { Point points[2]; - shift = 0; - - findPointsForLine(weights, shift, points); + findPointsForLine(weights, shift, points, data.img.cols, data.img.rows); redraw(data, points); } @@ -199,13 +191,13 @@ static void onMouse( int event, int x, int y, int, void* pData) { case CV_EVENT_LBUTTONUP: data.responses.push_back(1); - addPointsRetrainAndRedraw(data, x, y); + addPointRetrainAndRedraw(data, x, y); break; case CV_EVENT_RBUTTONDOWN: data.responses.push_back(-1); - addPointsRetrainAndRedraw(data, x, y); + addPointRetrainAndRedraw(data, x, y); break; } @@ -213,14 +205,10 @@ static void onMouse( int event, int x, int y, int, void* pData) int main() { - Data data; setMouseCallback( "Train svmsgd", onMouse, &data ); waitKey(); - - - return 0; } From bfdca05f251398e0638dc168436ded0705df258b Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Tue, 9 Feb 2016 18:42:23 +0300 Subject: [PATCH 04/19] Added margin type, added tests with different scales of features. Also fixed documentation, refactored sample. --- modules/ml/include/opencv2/ml.hpp | 127 +++++++++++++----- modules/ml/src/svmsgd.cpp | 180 +++++++++++++++---------- modules/ml/test/test_mltests2.cpp | 25 +++- modules/ml/test/test_save_load.cpp | 6 +- modules/ml/test/test_svmsgd.cpp | 203 +++++++++++++++++++++-------- samples/cpp/train_svmsgd.cpp | 63 +++++---- 6 files changed, 406 insertions(+), 198 deletions(-) diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index 3a04368e1e..ec7c0730ec 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1504,27 +1504,78 @@ public: /*! @brief Stochastic Gradient Descent SVM classifier -SVMSGD provides a fast and easy-to-use implementation of the SVM classifier using the Stochastic Gradient Descent approach, as presented in @cite bottou2010large. +SVMSGD provides a fast and easy-to-use implementation of the SVM classifier using the Stochastic Gradient Descent approach, +as presented in @cite bottou2010large. The gradient descent show amazing performance for large-scale problems, reducing the computing time. +The classifier has 5 parameters. These are +- model type, +- margin type, +- \f$\lambda\f$ (strength of restrictions on outliers), +- \f$\gamma_0\f$ (initial step size), +- \f$c\f$ (power coefficient for decreasing of step size), +- and termination criteria. +The model type may have one of the following values: \ref SGD and \ref ASGD. -First, create the SVMSGD object. Set parametrs of model (type, lambda, gamma0, c) using the functions setType, setLambda, setGamma0 and setC or the function setOptimalParametrs. -Recommended model type is ASGD. +- \ref SGD is the classic version of SVMSGD classifier: every next step is calculated by the formula + \f[w_{t+1} = w_t - \gamma(t) \frac{dQ_i}{dw} |_{w = w_t}\f] + where + - \f$w_t\f$ is the weights vector for decision function at step \f$t\f$, + - \f$\gamma(t)\f$ is the step size of model parameters at the iteration \f$t\f$, it is decreased on each step by the formula + \f$\gamma(t) = \gamma_0 (1 + \lambda \gamma_0 t) ^ {-c}\f$ + - \f$Q_i\f$ is the target functional from SVM task for sample with number \f$i\f$, this sample is chosen stochastically on each step of the algorithm. -Then the SVM model can be trained using the train features and the correspondent labels. +- \ref ASGD is Average Stochastic Gradient Descent SVM Classifier. ASGD classifier averages weights vector on each step of algorithm by the formula +\f$\widehat{w}_{t+1} = \frac{t}{1+t}\widehat{w}_{t} + \frac{1}{1+t}w_{t+1}\f$ -After that, the label of a new feature vector can be predicted using the predict function. +The recommended model type is ASGD (following @cite bottou2010large). + +The margin type may have one of the following values: \ref SOFT_MARGIN or \ref HARD_MARGIN. + +- You should use \ref HARD_MARGIN type, if you have linearly separable sets. +- You should use \ref SOFT_MARGIN type, if you have non-linearly separable sets or sets with outliers. +- In the general case (if you know nothing about linearly separability of your sets), use SOFT_MARGIN. + +The other parameters may be described as follows: +- \f$\lambda\f$ parameter is responsible for weights decreasing at each step and for the strength of restrictions on outliers + (the less the parameter, the less probability that an outlier will be ignored). + Recommended value for SGD model is 0.0001, for ASGD model is 0.00001. + +- \f$\gamma_0\f$ parameter is the initial value for the step size \f$\gamma(t)\f$. + You will have to find the best \f$\gamma_0\f$ for your problem. + +- \f$c\f$ is the power parameter for \f$\gamma(t)\f$ decreasing by the formula, mentioned above. + Recommended value for SGD model is 1, for ASGD model is 0.75. + +- Termination criteria can be TermCriteria::COUNT, TermCriteria::EPS or TermCriteria::COUNT + TermCriteria::EPS. + You will have to find the best termination criteria for your problem. + +Note that the parameters \f$\lambda\f$, \f$\gamma_0\f$, and \f$c\f$ should be positive. + +To use SVMSGD algorithm do as follows: + +- first, create the SVMSGD object. + +- then set parameters (model type, margin type, \f$\lambda\f$, \f$\gamma_0\f$, \f$c\f$) using the functions + setSvmsgdType(), setMarginType(), setLambda(), setGamma0(), and setC(), or the function setOptimalParameters(). + +- then the SVM model can be trained using the train features and the correspondent labels by the method train(). + +- after that, the label of a new feature vector can be predicted using the method predict(). @code -// Initialize object -SVMSGD SvmSgd; +// Create empty object +cv::Ptr svmsgd = SVMSGD::create(); + +// Set parameters +svmsgd->setOptimalParameters(); // Train the Stochastic Gradient Descent SVM -SvmSgd.train(trainFeatures, labels); +SvmSgd->train(trainData); -// Predict label for the new feature vector (1xM) -predictedLabel = SvmSgd.predict(newFeatureVector); +// Predict labels for the new samples +svmsgd->predict(samples, responses); @endcode */ @@ -1537,9 +1588,17 @@ public: ASGD is often the preferable choice. */ enum SvmsgdType { - ILLEGAL_VALUE, - SGD, //!Stochastic Gradient Descent - ASGD //!Average Stochastic Gradient Descent + ILLEGAL_SVMSGD_TYPE, + SGD, //!< Stochastic Gradient Descent + ASGD //!< Average Stochastic Gradient Descent + }; + + /** Margin type.*/ + enum MarginType + { + ILLEGAL_MARGIN_TYPE, + SOFT_MARGIN, //!< General case, suits to the case of non-linearly separable sets, allows outliers. + HARD_MARGIN //!< More accurate for the case of linearly separable sets. }; /** @@ -1553,49 +1612,59 @@ public: CV_WRAP virtual float getShift() = 0; - /** Creates empty model. + /** @brief Creates empty model. Use StatModel::train to train the model. Since %SVMSGD has several parameters, you may want to find the best parameters for your problem or use setOptimalParameters() to set some default parameters. */ CV_WRAP static Ptr create(); - /** Function sets optimal parameters values for chosen SVM SGD model. + /** @brief Function sets optimal parameters values for chosen SVM SGD model. * If chosen type is ASGD, function sets the following values for parameters of model: - * lambda = 0.00001; - * gamma0 = 0.05; - * c = 0.75; + * \f$\lambda = 0.00001\f$; + * \f$\gamma_0 = 0.05\f$; + * \f$c = 0.75\f$; * termCrit.maxCount = 100000; * termCrit.epsilon = 0.00001; * * If SGD: - * lambda = 0.0001; - * gamma0 = 0.05; - * c = 1; + * \f$\lambda = 0.0001\f$; + * \f$\gamma_0 = 0.05\f$; + * \f$c = 1\f$; * termCrit.maxCount = 100000; * termCrit.epsilon = 0.00001; - * @param type is the type of SVMSGD classifier. Legal values are SvmsgdType::SGD and SvmsgdType::ASGD. + * @param svmsgdType is the type of SVMSGD classifier. Legal values are SvmsgdType::SGD and SvmsgdType::ASGD. * Recommended value is SvmsgdType::ASGD (by default). + * @param marginType is the type of margin constraint. Legal values are MarginType::SOFT_MARGIN and MarginType::HARD_MARGIN. + * Default value is MarginType::SOFT_MARGIN. */ - CV_WRAP virtual void setOptimalParameters(int type = ASGD) = 0; + CV_WRAP virtual void setOptimalParameters(int svmsgdType = ASGD, int marginType = SOFT_MARGIN) = 0; - /** %Algorithm type, one of SVMSGD::SvmsgdType. */ + /** @brief %Algorithm type, one of SVMSGD::SvmsgdType. */ /** @see setAlgorithmType */ - CV_WRAP virtual int getType() const = 0; + CV_WRAP virtual int getSvmsgdType() const = 0; /** @copybrief getAlgorithmType @see getAlgorithmType */ - CV_WRAP virtual void setType(int type) = 0; + CV_WRAP virtual void setSvmsgdType(int svmsgdType) = 0; + + /** @brief %Margin type, one of SVMSGD::MarginType. */ + /** @see setMarginType */ + CV_WRAP virtual int getMarginType() const = 0; + /** @copybrief getMarginType @see getMarginType */ + CV_WRAP virtual void setMarginType(int marginType) = 0; + - /** Parameter _Lambda_ of a %SVMSGD optimization problem. Default value is 0. */ + /** @brief Parameter \f$\lambda\f$ of a %SVMSGD optimization problem. Default value is 0. */ /** @see setLambda */ CV_WRAP virtual float getLambda() const = 0; /** @copybrief getLambda @see getLambda */ CV_WRAP virtual void setLambda(float lambda) = 0; - /** Parameter _Gamma0_ of a %SVMSGD optimization problem. Default value is 0. */ + /** @brief Parameter \f$\gamma_0\f$ of a %SVMSGD optimization problem. Default value is 0. */ /** @see setGamma0 */ CV_WRAP virtual float getGamma0() const = 0; + /** @copybrief getGamma0 @see getGamma0 */ CV_WRAP virtual void setGamma0(float gamma0) = 0; - /** Parameter _C_ of a %SVMSGD optimization problem. Default value is 0. */ + /** @brief Parameter \f$c\f$ of a %SVMSGD optimization problem. Default value is 0. */ /** @see setC */ CV_WRAP virtual float getC() const = 0; /** @copybrief getC @see getC */ diff --git a/modules/ml/src/svmsgd.cpp b/modules/ml/src/svmsgd.cpp index 5c75023b7d..cfb2c760e0 100644 --- a/modules/ml/src/svmsgd.cpp +++ b/modules/ml/src/svmsgd.cpp @@ -42,7 +42,6 @@ #include "precomp.hpp" #include "limits" -//#include "math.h" #include @@ -76,9 +75,9 @@ public: virtual void clear(); - virtual void write(FileStorage& fs) const; + virtual void write(FileStorage &fs) const; - virtual void read(const FileNode& fn); + virtual void read(const FileNode &fn); virtual Mat getWeights(){ return weights_; } @@ -88,11 +87,15 @@ public: virtual String getDefaultName() const {return "opencv_ml_svmsgd";} - virtual void setOptimalParameters(int type = ASGD); + virtual void setOptimalParameters(int svmsgdType = ASGD, int marginType = SOFT_MARGIN); - virtual int getType() const; + virtual int getSvmsgdType() const; - virtual void setType(int type); + virtual void setSvmsgdType(int svmsgdType); + + virtual int getMarginType() const; + + virtual void setMarginType(int marginType); CV_IMPL_PROPERTY(float, Lambda, params.lambda) CV_IMPL_PROPERTY(float, Gamma0, params.gamma0) @@ -100,21 +103,21 @@ public: CV_IMPL_PROPERTY_S(cv::TermCriteria, TermCriteria, params.termCrit) private: - void updateWeights(InputArray sample, bool isFirstClass, float gamma, Mat weights); + void updateWeights(InputArray sample, bool isFirstClass, float gamma, Mat &weights); std::pair areClassesEmpty(Mat responses); - void writeParams( FileStorage& fs ) const; + void writeParams( FileStorage &fs ) const; - void readParams( const FileNode& fn ); + void readParams( const FileNode &fn ); static inline bool isFirstClass(float val) { return val > 0; } - static void normalizeSamples(Mat &matrix, Mat &multiplier, Mat &average); + static void normalizeSamples(Mat &matrix, Mat &average, float &multiplier); float calcShift(InputArray _samples, InputArray _responses) const; - static void makeExtendedTrainSamples(const Mat trainSamples, Mat &extendedTrainSamples, Mat &multiplier); + static void makeExtendedTrainSamples(const Mat &trainSamples, Mat &extendedTrainSamples, Mat &average, float &multiplier); @@ -130,6 +133,7 @@ private: float c; TermCriteria termCrit; SvmsgdType svmsgdType; + MarginType marginType; }; SVMSGDParams params; @@ -160,7 +164,7 @@ std::pair SVMSGDImpl::areClassesEmpty(Mat responses) return emptyInClasses; } -void SVMSGDImpl::normalizeSamples(Mat &samples, Mat &multiplier, Mat &average) +void SVMSGDImpl::normalizeSamples(Mat &samples, Mat &average, float &multiplier) { int featuresCount = samples.cols; int samplesCount = samples.rows; @@ -176,37 +180,25 @@ void SVMSGDImpl::normalizeSamples(Mat &samples, Mat &multiplier, Mat &average) samples.row(sampleIndex) -= average; } - Mat featureNorm(1, featuresCount, samples.type()); - for (int featureIndex = 0; featureIndex < featuresCount; featureIndex++) - { - featureNorm.at(featureIndex) = norm(samples.col(featureIndex)); - } + double normValue = norm(samples); - multiplier = sqrt(samplesCount) / featureNorm; - for (int sampleIndex = 0; sampleIndex < samplesCount; sampleIndex++) - { - samples.row(sampleIndex) = samples.row(sampleIndex).mul(multiplier); - } + multiplier = sqrt(samples.total()) / normValue; + + samples *= multiplier; } -void SVMSGDImpl::makeExtendedTrainSamples(const Mat trainSamples, Mat &extendedTrainSamples, Mat &multiplier) +void SVMSGDImpl::makeExtendedTrainSamples(const Mat &trainSamples, Mat &extendedTrainSamples, Mat &average, float &multiplier) { Mat normalisedTrainSamples = trainSamples.clone(); int samplesCount = normalisedTrainSamples.rows; - Mat average; - - normalizeSamples(normalisedTrainSamples, multiplier, average); + normalizeSamples(normalisedTrainSamples, average, multiplier); Mat onesCol = Mat::ones(samplesCount, 1, CV_32F); cv::hconcat(normalisedTrainSamples, onesCol, extendedTrainSamples); - - //cout << "SVMSGDImpl::makeExtendedTrainSamples average: \n" << average << endl; - //cout << "SVMSGDImpl::makeExtendedTrainSamples multiplier: \n" << multiplier << endl; } - -void SVMSGDImpl::updateWeights(InputArray _sample, bool firstClass, float gamma, Mat weights) +void SVMSGDImpl::updateWeights(InputArray _sample, bool firstClass, float gamma, Mat& weights) { Mat sample = _sample.getMat(); @@ -226,7 +218,7 @@ void SVMSGDImpl::updateWeights(InputArray _sample, bool firstClass, float gamma, float SVMSGDImpl::calcShift(InputArray _samples, InputArray _responses) const { - float distance_to_classes[2] = { std::numeric_limits::max(), std::numeric_limits::max() }; + float distanceToClasses[2] = { std::numeric_limits::max(), std::numeric_limits::max() }; Mat trainSamples = _samples.getMat(); int trainSamplesCount = trainSamples.rows; @@ -241,36 +233,29 @@ float SVMSGDImpl::calcShift(InputArray _samples, InputArray _responses) const bool firstClass = isFirstClass(trainResponses.at(samplesIndex)); int index = firstClass ? 0:1; float signToMul = firstClass ? 1 : -1; - float cur_distance = dotProduct * signToMul; + float curDistance = dotProduct * signToMul; - if (cur_distance < distance_to_classes[index]) + if (curDistance < distanceToClasses[index]) { - distance_to_classes[index] = cur_distance; + distanceToClasses[index] = curDistance; } } - return -(distance_to_classes[0] - distance_to_classes[1]) / 2.f; + return -(distanceToClasses[0] - distanceToClasses[1]) / 2.f; } bool SVMSGDImpl::train(const Ptr& data, int) { - //cout << "SVMSGDImpl::train begin" << endl; clear(); CV_Assert( isClassifier() ); //toDo: consider Mat trainSamples = data->getTrainSamples(); - //cout << "SVMSGDImpl::train trainSamples: \n" << trainSamples << endl; - int featureCount = trainSamples.cols; Mat trainResponses = data->getTrainResponses(); // (trainSamplesCount x 1) matrix - //cout << "SVMSGDImpl::train trainresponses: \n" << trainResponses << endl; - std::pair areEmpty = areClassesEmpty(trainResponses); - //cout << "SVMSGDImpl::train areEmpty" << areEmpty.first << "," << areEmpty.second << endl; - if ( areEmpty.first && areEmpty.second ) { return false; @@ -283,10 +268,9 @@ bool SVMSGDImpl::train(const Ptr& data, int) } Mat extendedTrainSamples; - Mat multiplier; - makeExtendedTrainSamples(trainSamples, extendedTrainSamples, multiplier); - - //cout << "SVMSGDImpl::train extendedTrainSamples: \n" << extendedTrainSamples << endl; + Mat average; + float multiplier = 0; + makeExtendedTrainSamples(trainSamples, extendedTrainSamples, average, multiplier); int extendedTrainSamplesCount = extendedTrainSamples.rows; int extendedFeatureCount = extendedTrainSamples.cols; @@ -301,6 +285,7 @@ bool SVMSGDImpl::train(const Ptr& data, int) RNG rng(0); + CV_Assert (params.termCrit.type & TermCriteria::COUNT || params.termCrit.type & TermCriteria::EPS); int maxCount = (params.termCrit.type & TermCriteria::COUNT) ? params.termCrit.maxCount : INT_MAX; double epsilon = (params.termCrit.type & TermCriteria::EPS) ? params.termCrit.epsilon : 0; @@ -336,17 +321,20 @@ bool SVMSGDImpl::train(const Ptr& data, int) extendedWeights = averageExtendedWeights; } - //cout << "SVMSGDImpl::train extendedWeights: \n" << extendedWeights << endl; - Rect roi(0, 0, featureCount, 1); weights_ = extendedWeights(roi); - weights_ = weights_.mul(1/multiplier); - - //cout << "SVMSGDImpl::train weights: \n" << weights_ << endl; + weights_ *= multiplier; - shift_ = calcShift(trainSamples, trainResponses); + CV_Assert(params.marginType == SOFT_MARGIN || params.marginType == HARD_MARGIN); - //cout << "SVMSGDImpl::train shift = " << shift_ << endl; + if (params.marginType == SOFT_MARGIN) + { + shift_ = extendedWeights.at(featureCount) - weights_.dot(average); + } + else + { + shift_ = calcShift(trainSamples, trainResponses); + } return true; } @@ -385,6 +373,8 @@ float SVMSGDImpl::predict( InputArray _samples, OutputArray _results, int ) cons bool SVMSGDImpl::isClassifier() const { return (params.svmsgdType == SGD || params.svmsgdType == ASGD) + && + (params.marginType == SOFT_MARGIN || params.marginType == HARD_MARGIN) && (params.lambda > 0) && (params.gamma0 > 0) && (params.c >= 0); } @@ -417,15 +407,32 @@ void SVMSGDImpl::writeParams( FileStorage& fs ) const case ASGD: SvmsgdTypeStr = "ASGD"; break; - case ILLEGAL_VALUE: - SvmsgdTypeStr = format("Uknown_%d", params.svmsgdType); + case ILLEGAL_SVMSGD_TYPE: + SvmsgdTypeStr = format("Unknown_%d", params.svmsgdType); default: std::cout << "params.svmsgdType isn't initialized" << std::endl; } - fs << "svmsgdType" << SvmsgdTypeStr; + String marginTypeStr; + + switch (params.marginType) + { + case SOFT_MARGIN: + marginTypeStr = "SOFT_MARGIN"; + break; + case HARD_MARGIN: + marginTypeStr = "HARD_MARGIN"; + break; + case ILLEGAL_MARGIN_TYPE: + marginTypeStr = format("Unknown_%d", params.marginType); + default: + std::cout << "params.marginType isn't initialized" << std::endl; + } + + fs << "marginType" << marginTypeStr; + fs << "lambda" << params.lambda; fs << "gamma0" << params.gamma0; fs << "c" << params.c; @@ -438,8 +445,6 @@ void SVMSGDImpl::writeParams( FileStorage& fs ) const fs << "}"; } - - void SVMSGDImpl::read(const FileNode& fn) { clear(); @@ -455,13 +460,23 @@ void SVMSGDImpl::readParams( const FileNode& fn ) String svmsgdTypeStr = (String)fn["svmsgdType"]; SvmsgdType svmsgdType = svmsgdTypeStr == "SGD" ? SGD : - svmsgdTypeStr == "ASGD" ? ASGD : ILLEGAL_VALUE; + svmsgdTypeStr == "ASGD" ? ASGD : ILLEGAL_SVMSGD_TYPE; - if( svmsgdType == ILLEGAL_VALUE ) + if( svmsgdType == ILLEGAL_SVMSGD_TYPE ) CV_Error( CV_StsParseError, "Missing or invalid SVMSGD type" ); params.svmsgdType = svmsgdType; + String marginTypeStr = (String)fn["marginType"]; + MarginType marginType = + marginTypeStr == "SOFT_MARGIN" ? SOFT_MARGIN : + marginTypeStr == "HARD_MARGIN" ? HARD_MARGIN : ILLEGAL_MARGIN_TYPE; + + if( marginType == ILLEGAL_MARGIN_TYPE ) + CV_Error( CV_StsParseError, "Missing or invalid margin type" ); + + params.marginType = marginType; + CV_Assert ( fn["lambda"].isReal() ); params.lambda = (float)fn["lambda"]; @@ -494,7 +509,8 @@ SVMSGDImpl::SVMSGDImpl() { clear(); - params.svmsgdType = ILLEGAL_VALUE; + params.svmsgdType = ILLEGAL_SVMSGD_TYPE; + params.marginType = ILLEGAL_MARGIN_TYPE; // Parameters for learning params.lambda = 0; // regularization @@ -505,26 +521,28 @@ SVMSGDImpl::SVMSGDImpl() params.termCrit = _termCrit; } -void SVMSGDImpl::setOptimalParameters(int type) +void SVMSGDImpl::setOptimalParameters(int svmsgdType, int marginType) { - switch (type) + switch (svmsgdType) { case SGD: params.svmsgdType = SGD; + params.marginType = (marginType == SOFT_MARGIN) ? SOFT_MARGIN : + (marginType == HARD_MARGIN) ? HARD_MARGIN : ILLEGAL_MARGIN_TYPE; params.lambda = 0.0001; params.gamma0 = 0.05; params.c = 1; - params.termCrit.maxCount = 100000; - params.termCrit.epsilon = 0.00001; + params.termCrit = TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 100000, 0.00001); break; case ASGD: params.svmsgdType = ASGD; + params.marginType = (marginType == SOFT_MARGIN) ? SOFT_MARGIN : + (marginType == HARD_MARGIN) ? HARD_MARGIN : ILLEGAL_MARGIN_TYPE; params.lambda = 0.00001; params.gamma0 = 0.05; params.c = 0.75; - params.termCrit.maxCount = 100000; - params.termCrit.epsilon = 0.00001; + params.termCrit = TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 100000, 0.00001); break; default: @@ -532,7 +550,7 @@ void SVMSGDImpl::setOptimalParameters(int type) } } -void SVMSGDImpl::setType(int type) +void SVMSGDImpl::setSvmsgdType(int type) { switch (type) { @@ -543,13 +561,33 @@ void SVMSGDImpl::setType(int type) params.svmsgdType = ASGD; break; default: - params.svmsgdType = ILLEGAL_VALUE; + params.svmsgdType = ILLEGAL_SVMSGD_TYPE; } } -int SVMSGDImpl::getType() const +int SVMSGDImpl::getSvmsgdType() const { return params.svmsgdType; } + +void SVMSGDImpl::setMarginType(int type) +{ + switch (type) + { + case HARD_MARGIN: + params.marginType = HARD_MARGIN; + break; + case SOFT_MARGIN: + params.marginType = SOFT_MARGIN; + break; + default: + params.marginType = ILLEGAL_MARGIN_TYPE; + } +} + +int SVMSGDImpl::getMarginType() const +{ + return params.marginType; +} } //ml } //cv diff --git a/modules/ml/test/test_mltests2.cpp b/modules/ml/test/test_mltests2.cpp index 6603a35c5b..9cdd368c4c 100644 --- a/modules/ml/test/test_mltests2.cpp +++ b/modules/ml/test/test_mltests2.cpp @@ -199,10 +199,19 @@ int str_to_svmsgd_type( String& str ) return SVMSGD::SGD; if ( !str.compare("ASGD") ) return SVMSGD::ASGD; - CV_Error( CV_StsBadArg, "incorrect boost type string" ); + CV_Error( CV_StsBadArg, "incorrect svmsgd type string" ); return -1; } +int str_to_margin_type( String& str ) +{ + if ( !str.compare("SOFT_MARGIN") ) + return SVMSGD::SOFT_MARGIN; + if ( !str.compare("HARD_MARGIN") ) + return SVMSGD::HARD_MARGIN; + CV_Error( CV_StsBadArg, "incorrect svmsgd margin type string" ); + return -1; +} // ---------------------------------- MLBaseTest --------------------------------------------------- CV_MLBaseTest::CV_MLBaseTest(const char* _modelName) @@ -452,10 +461,16 @@ int CV_MLBaseTest::train( int testCaseIdx ) { String svmsgdTypeStr; modelParamsNode["svmsgdType"] >> svmsgdTypeStr; - Ptr m = SVMSGD::create(); - int type = str_to_svmsgd_type( svmsgdTypeStr ); - m->setType(type); - //m->setType(str_to_svmsgd_type( svmsgdTypeStr )); + + Ptr m = SVMSGD::create(); + int svmsgdType = str_to_svmsgd_type( svmsgdTypeStr ); + m->setSvmsgdType(svmsgdType); + + String marginTypeStr; + modelParamsNode["marginType"] >> marginTypeStr; + int marginType = str_to_margin_type( marginTypeStr ); + m->setMarginType(marginType); + m->setLambda(modelParamsNode["lambda"]); m->setGamma0(modelParamsNode["gamma0"]); m->setC(modelParamsNode["c"]); diff --git a/modules/ml/test/test_save_load.cpp b/modules/ml/test/test_save_load.cpp index 354c6e0307..1bec3cc274 100644 --- a/modules/ml/test/test_save_load.cpp +++ b/modules/ml/test/test_save_load.cpp @@ -270,11 +270,7 @@ TEST(ML_DTree, legacy_load) { CV_LegacyTest test(CV_DTREE, "_abalone.xml;_mushro TEST(ML_NBayes, legacy_load) { CV_LegacyTest test(CV_NBAYES, "_waveform.xml"); test.safe_run(); } TEST(ML_SVM, legacy_load) { CV_LegacyTest test(CV_SVM, "_poletelecomm.xml;_waveform.xml"); test.safe_run(); } TEST(ML_RTrees, legacy_load) { CV_LegacyTest test(CV_RTREES, "_waveform.xml"); test.safe_run(); } -TEST(ML_SVMSGD, legacy_load) -{ - CV_LegacyTest test(CV_SVMSGD, "_waveform.xml"); - test.safe_run(); -} +TEST(ML_SVMSGD, legacy_load) { CV_LegacyTest test(CV_SVMSGD, "_waveform.xml"); test.safe_run(); } /*TEST(ML_SVM, throw_exception_when_save_untrained_model) { diff --git a/modules/ml/test/test_svmsgd.cpp b/modules/ml/test/test_svmsgd.cpp index 5445271eec..7fe298172c 100644 --- a/modules/ml/test/test_svmsgd.cpp +++ b/modules/ml/test/test_svmsgd.cpp @@ -52,45 +52,99 @@ using cv::ml::TrainData; class CV_SVMSGDTrainTest : public cvtest::BaseTest { public: - CV_SVMSGDTrainTest(Mat _weights, float shift); + enum TrainDataType + { + UNIFORM_SAME_SCALE, + UNIFORM_DIFFERENT_SCALES + }; + + CV_SVMSGDTrainTest(Mat _weights, float shift, TrainDataType type, double precision = 0.01); private: virtual void run( int start_from ); - float decisionFunction(Mat sample, Mat weights, float shift); - + static float decisionFunction(const Mat &sample, const Mat &weights, float shift); + void makeTrainData(Mat weights, float shift); + void makeTestData(Mat weights, float shift); + void generateSameScaleData(Mat &samples); + void generateDifferentScalesData(Mat &samples, float shift); + + TrainDataType type; + double precision; cv::Ptr data; cv::Mat testSamples; cv::Mat testResponses; static const int TEST_VALUE_LIMIT = 500; }; -CV_SVMSGDTrainTest::CV_SVMSGDTrainTest(Mat weights, float shift) +void CV_SVMSGDTrainTest::generateSameScaleData(Mat &samples) { - int datasize = 100000; - int varCount = weights.cols; - cv::Mat samples = cv::Mat::zeros( datasize, varCount, CV_32FC1 ); - cv::Mat responses = cv::Mat::zeros( datasize, 1, CV_32FC1 ); + float lowerLimit = -TEST_VALUE_LIMIT; + float upperLimit = TEST_VALUE_LIMIT; cv::RNG rng(0); + rng.fill(samples, RNG::UNIFORM, lowerLimit, upperLimit); +} + +void CV_SVMSGDTrainTest::generateDifferentScalesData(Mat &samples, float shift) +{ + int featureCount = samples.cols; float lowerLimit = -TEST_VALUE_LIMIT; float upperLimit = TEST_VALUE_LIMIT; + cv::RNG rng(10); - rng.fill(samples, RNG::UNIFORM, lowerLimit, upperLimit); - for (int sampleIndex = 0; sampleIndex < datasize; sampleIndex++) + for (int featureIndex = 0; featureIndex < featureCount; featureIndex++) { - responses.at( sampleIndex ) = decisionFunction(samples.row(sampleIndex), weights, shift) > 0 ? 1 : -1; + int crit = rng.uniform(0, 2); + + if (crit > 0) + { + rng.fill(samples.col(featureIndex), RNG::UNIFORM, lowerLimit - shift, upperLimit - shift); + } + else + { + rng.fill(samples.col(featureIndex), RNG::UNIFORM, lowerLimit/10, upperLimit/10); + } } +} +void CV_SVMSGDTrainTest::makeTrainData(Mat weights, float shift) +{ + int datasize = 100000; + int featureCount = weights.cols; + cv::Mat samples = cv::Mat::zeros(datasize, featureCount, CV_32FC1); + cv::Mat responses = cv::Mat::zeros(datasize, 1, CV_32FC1); + switch(type) + { + case UNIFORM_SAME_SCALE: + generateSameScaleData(samples); + break; + case UNIFORM_DIFFERENT_SCALES: + generateDifferentScalesData(samples, shift); + break; + default: + CV_Error(CV_StsBadArg, "Unknown train data type"); + } - std::cout << "real weights\n" << weights/norm(weights) << "\n" << std::endl; - std::cout << "real shift \n" << shift/norm(weights) << "\n" << std::endl; + for (int sampleIndex = 0; sampleIndex < datasize; sampleIndex++) + { + responses.at(sampleIndex) = decisionFunction(samples.row(sampleIndex), weights, shift) > 0 ? 1 : -1; + } - data = TrainData::create( samples, cv::ml::ROW_SAMPLE, responses ); + data = TrainData::create(samples, cv::ml::ROW_SAMPLE, responses); +} +void CV_SVMSGDTrainTest::makeTestData(Mat weights, float shift) +{ int testSamplesCount = 100000; + int featureCount = weights.cols; + + float lowerLimit = -TEST_VALUE_LIMIT; + float upperLimit = TEST_VALUE_LIMIT; + + cv::RNG rng(0); - testSamples.create(testSamplesCount, varCount, CV_32FC1); + testSamples.create(testSamplesCount, featureCount, CV_32FC1); rng.fill(testSamples, RNG::UNIFORM, lowerLimit, upperLimit); testResponses.create(testSamplesCount, 1, CV_32FC1); @@ -100,12 +154,24 @@ CV_SVMSGDTrainTest::CV_SVMSGDTrainTest(Mat weights, float shift) } } +CV_SVMSGDTrainTest::CV_SVMSGDTrainTest(Mat weights, float shift, TrainDataType _type, double _precision) +{ + type = _type; + precision = _precision; + makeTrainData(weights, shift); + makeTestData(weights, shift); +} + +float CV_SVMSGDTrainTest::decisionFunction(const Mat &sample, const Mat &weights, float shift) +{ + return sample.dot(weights) + shift; +} + void CV_SVMSGDTrainTest::run( int /*start_from*/ ) { cv::Ptr svmsgd = SVMSGD::create(); - svmsgd->setOptimalParameters(SVMSGD::ASGD); - svmsgd->setTermCriteria(TermCriteria(TermCriteria::EPS, 0, 0.00005)); + svmsgd->setOptimalParameters(); svmsgd->train(data); @@ -118,77 +184,106 @@ void CV_SVMSGDTrainTest::run( int /*start_from*/ ) for (int i = 0; i < testSamplesCount; i++) { - if (responses.at(i) * testResponses.at(i) < 0 ) + if (responses.at(i) * testResponses.at(i) < 0) errCount++; } - - float normW = norm(svmsgd->getWeights()); - - std::cout << "found weights\n" << svmsgd->getWeights()/normW << "\n" << std::endl; - std::cout << "found shift \n" << svmsgd->getShift()/normW << "\n" << std::endl; - float err = (float)errCount / testSamplesCount; std::cout << "err " << err << std::endl; - if ( err > 0.01 ) + if ( err > precision ) { - ts->set_failed_test_info( cvtest::TS::FAIL_BAD_ACCURACY ); + ts->set_failed_test_info(cvtest::TS::FAIL_BAD_ACCURACY); } } -float CV_SVMSGDTrainTest::decisionFunction(Mat sample, Mat weights, float shift) + +void makeWeightsAndShift(int featureCount, Mat &weights, float &shift) { - return sample.dot(weights) + shift; + weights.create(1, featureCount, CV_32FC1); + cv::RNG rng(0); + double lowerLimit = -1; + double upperLimit = 1; + + rng.fill(weights, RNG::UNIFORM, lowerLimit, upperLimit); + shift = rng.uniform(-featureCount, featureCount); } -TEST(ML_SVMSGD, train0) + +TEST(ML_SVMSGD, trainSameScale2) { - int varCount = 2; + int featureCount = 2; Mat weights; - weights.create(1, varCount, CV_32FC1); - weights.at(0) = 1; - weights.at(1) = 0; - cv::RNG rng(1); - float shift = rng.uniform(-varCount, varCount); - CV_SVMSGDTrainTest test(weights, shift); + float shift = 0; + makeWeightsAndShift(featureCount, weights, shift); + + CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_SAME_SCALE); test.safe_run(); } -TEST(ML_SVMSGD, train1) +TEST(ML_SVMSGD, trainSameScale5) { - int varCount = 5; + int featureCount = 5; Mat weights; - weights.create(1, varCount, CV_32FC1); - float lowerLimit = -1; - float upperLimit = 1; - cv::RNG rng(0); - rng.fill(weights, RNG::UNIFORM, lowerLimit, upperLimit); + float shift = 0; + makeWeightsAndShift(featureCount, weights, shift); + + CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_SAME_SCALE); + test.safe_run(); +} + +TEST(ML_SVMSGD, trainSameScale100) +{ + int featureCount = 100; + + Mat weights; - float shift = rng.uniform(-varCount, varCount); + float shift = 0; + makeWeightsAndShift(featureCount, weights, shift); - CV_SVMSGDTrainTest test(weights, shift); + CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_SAME_SCALE); test.safe_run(); } -TEST(ML_SVMSGD, train2) +TEST(ML_SVMSGD, trainDifferentScales2) { - int varCount = 100; + int featureCount = 2; Mat weights; - weights.create(1, varCount, CV_32FC1); - float lowerLimit = -1; - float upperLimit = 1; - cv::RNG rng(0); - rng.fill(weights, RNG::UNIFORM, lowerLimit, upperLimit); + float shift = 0; + makeWeightsAndShift(featureCount, weights, shift); + + CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_DIFFERENT_SCALES, 0.01); + test.safe_run(); +} + +TEST(ML_SVMSGD, trainDifferentScales5) +{ + int featureCount = 5; + + Mat weights; + + float shift = 0; + makeWeightsAndShift(featureCount, weights, shift); + + CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_DIFFERENT_SCALES, 0.05); + test.safe_run(); +} + +TEST(ML_SVMSGD, trainDifferentScales100) +{ + int featureCount = 100; + + Mat weights; - float shift = rng.uniform(-varCount, varCount); + float shift = 0; + makeWeightsAndShift(featureCount, weights, shift); - CV_SVMSGDTrainTest test(weights,shift); + CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_DIFFERENT_SCALES, 0.10); test.safe_run(); } diff --git a/samples/cpp/train_svmsgd.cpp b/samples/cpp/train_svmsgd.cpp index 19c30ffca9..7d537341f8 100644 --- a/samples/cpp/train_svmsgd.cpp +++ b/samples/cpp/train_svmsgd.cpp @@ -40,16 +40,13 @@ void fillSegments(std::vector > &segments, int width, int void redraw(Data data, const Point points[2]); //add point in train set, train SVMSGD algorithm and draw results on image -void addPointRetrainAndRedraw(Data &data, int x, int y); +void addPointRetrainAndRedraw(Data &data, int x, int y, int response); bool doTrain( const Mat samples, const Mat responses, Mat &weights, float &shift) { cv::Ptr svmsgd = SVMSGD::create(); - svmsgd->setOptimalParameters(SVMSGD::ASGD); - svmsgd->setTermCriteria(TermCriteria(TermCriteria::EPS, 0, 0.00000001)); - svmsgd->setLambda(0.00000001); - + svmsgd->setOptimalParameters(); cv::Ptr train_data = TrainData::create(samples, cv::ml::ROW_SAMPLE, responses); svmsgd->train( train_data ); @@ -64,6 +61,27 @@ bool doTrain( const Mat samples, const Mat responses, Mat &weights, float &shift return false; } +void fillSegments(std::vector > &segments, int width, int height) +{ + std::pair currentSegment; + + currentSegment.first = Point(width, 0); + currentSegment.second = Point(width, height); + segments.push_back(currentSegment); + + currentSegment.first = Point(0, height); + currentSegment.second = Point(width, height); + segments.push_back(currentSegment); + + currentSegment.first = Point(0, 0); + currentSegment.second = Point(width, 0); + segments.push_back(currentSegment); + + currentSegment.first = Point(0, 0); + currentSegment.second = Point(0, height); + segments.push_back(currentSegment); +} + bool findCrossPointWithBorders(const Mat &weights, float shift, const std::pair &segment, Point &crossPoint) { @@ -123,27 +141,6 @@ bool findPointsForLine(const Mat &weights, float shift, Point (&points)[2], int return true; } -void fillSegments(std::vector > &segments, int width, int height) -{ - std::pair currentSegment; - - currentSegment.first = Point(width, 0); - currentSegment.second = Point(width, height); - segments.push_back(currentSegment); - - currentSegment.first = Point(0, height); - currentSegment.second = Point(width, height); - segments.push_back(currentSegment); - - currentSegment.first = Point(0, 0); - currentSegment.second = Point(width, 0); - segments.push_back(currentSegment); - - currentSegment.first = Point(0, 0); - currentSegment.second = Point(0, height); - segments.push_back(currentSegment); -} - void redraw(Data data, const Point points[2]) { data.img.setTo(0); @@ -162,19 +159,20 @@ void redraw(Data data, const Point points[2]) imshow("Train svmsgd", data.img); } -void addPointRetrainAndRedraw(Data &data, int x, int y) +void addPointRetrainAndRedraw(Data &data, int x, int y, int response) { Mat currentSample(1, 2, CV_32F); currentSample.at(0,0) = x; currentSample.at(0,1) = y; data.samples.push_back(currentSample); + data.responses.push_back(response); Mat weights(1, 2, CV_32F); float shift = 0; if (doTrain(data.samples, data.responses, weights, shift)) - { + { Point points[2]; findPointsForLine(weights, shift, points, data.img.cols, data.img.rows); @@ -189,15 +187,12 @@ static void onMouse( int event, int x, int y, int, void* pData) switch( event ) { - case CV_EVENT_LBUTTONUP: - data.responses.push_back(1); - addPointRetrainAndRedraw(data, x, y); - + case CV_EVENT_LBUTTONUP: + addPointRetrainAndRedraw(data, x, y, 1); break; case CV_EVENT_RBUTTONDOWN: - data.responses.push_back(-1); - addPointRetrainAndRedraw(data, x, y); + addPointRetrainAndRedraw(data, x, y, -1); break; } From 41c0a38344361ad455f9df3a7c9c68ac3633b24c Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Wed, 10 Feb 2016 14:52:50 +0300 Subject: [PATCH 05/19] Fixed test samples for tests with different borders Added new test (separating two points) --- modules/ml/src/svmsgd.cpp | 2 +- modules/ml/test/test_save_load.cpp | 5 +- modules/ml/test/test_svmsgd.cpp | 111 +++++++++++++++++++++-------- samples/cpp/train_svmsgd.cpp | 4 +- 4 files changed, 84 insertions(+), 38 deletions(-) diff --git a/modules/ml/src/svmsgd.cpp b/modules/ml/src/svmsgd.cpp index cfb2c760e0..4ad103e383 100644 --- a/modules/ml/src/svmsgd.cpp +++ b/modules/ml/src/svmsgd.cpp @@ -146,7 +146,7 @@ Ptr SVMSGD::create() std::pair SVMSGDImpl::areClassesEmpty(Mat responses) { - CV_Assert(responses.cols == 1); + CV_Assert(responses.cols == 1 || responses.rows == 1); std::pair emptyInClasses(true, true); int limit_index = responses.rows; diff --git a/modules/ml/test/test_save_load.cpp b/modules/ml/test/test_save_load.cpp index 1bec3cc274..f55f9b8a11 100644 --- a/modules/ml/test/test_save_load.cpp +++ b/modules/ml/test/test_save_load.cpp @@ -160,10 +160,7 @@ TEST(ML_DTree, save_load) { CV_SLMLTest test( CV_DTREE ); test.safe_run(); } TEST(ML_Boost, save_load) { CV_SLMLTest test( CV_BOOST ); test.safe_run(); } TEST(ML_RTrees, save_load) { CV_SLMLTest test( CV_RTREES ); test.safe_run(); } TEST(DISABLED_ML_ERTrees, save_load) { CV_SLMLTest test( CV_ERTREES ); test.safe_run(); } -TEST(MV_SVMSGD, save_load){ - CV_SLMLTest test( CV_SVMSGD ); - test.safe_run(); -} +TEST(MV_SVMSGD, save_load){ CV_SLMLTest test( CV_SVMSGD ); test.safe_run(); } class CV_LegacyTest : public cvtest::BaseTest { diff --git a/modules/ml/test/test_svmsgd.cpp b/modules/ml/test/test_svmsgd.cpp index 7fe298172c..666ef18be6 100644 --- a/modules/ml/test/test_svmsgd.cpp +++ b/modules/ml/test/test_svmsgd.cpp @@ -58,39 +58,40 @@ public: UNIFORM_DIFFERENT_SCALES }; - CV_SVMSGDTrainTest(Mat _weights, float shift, TrainDataType type, double precision = 0.01); + CV_SVMSGDTrainTest(const Mat &_weights, float shift, TrainDataType type, double precision = 0.01); private: virtual void run( int start_from ); static float decisionFunction(const Mat &sample, const Mat &weights, float shift); void makeTrainData(Mat weights, float shift); void makeTestData(Mat weights, float shift); - void generateSameScaleData(Mat &samples); - void generateDifferentScalesData(Mat &samples, float shift); + void generateSameBorders(int featureCount); + void generateDifferentBorders(int featureCount); TrainDataType type; double precision; + std::vector > borders; cv::Ptr data; cv::Mat testSamples; cv::Mat testResponses; static const int TEST_VALUE_LIMIT = 500; }; -void CV_SVMSGDTrainTest::generateSameScaleData(Mat &samples) +void CV_SVMSGDTrainTest::generateSameBorders(int featureCount) { float lowerLimit = -TEST_VALUE_LIMIT; float upperLimit = TEST_VALUE_LIMIT; - cv::RNG rng(0); - rng.fill(samples, RNG::UNIFORM, lowerLimit, upperLimit); + + for (int featureIndex = 0; featureIndex < featureCount; featureIndex++) + { + borders.push_back(std::pair(lowerLimit, upperLimit)); + } } -void CV_SVMSGDTrainTest::generateDifferentScalesData(Mat &samples, float shift) +void CV_SVMSGDTrainTest::generateDifferentBorders(int featureCount) { - int featureCount = samples.cols; - float lowerLimit = -TEST_VALUE_LIMIT; float upperLimit = TEST_VALUE_LIMIT; - cv::RNG rng(10); - + cv::RNG rng(0); for (int featureIndex = 0; featureIndex < featureCount; featureIndex++) { @@ -98,11 +99,11 @@ void CV_SVMSGDTrainTest::generateDifferentScalesData(Mat &samples, float shift) if (crit > 0) { - rng.fill(samples.col(featureIndex), RNG::UNIFORM, lowerLimit - shift, upperLimit - shift); + borders.push_back(std::pair(lowerLimit, upperLimit)); } else { - rng.fill(samples.col(featureIndex), RNG::UNIFORM, lowerLimit/10, upperLimit/10); + borders.push_back(std::pair(lowerLimit/1000, upperLimit/1000)); } } } @@ -111,21 +112,16 @@ void CV_SVMSGDTrainTest::makeTrainData(Mat weights, float shift) { int datasize = 100000; int featureCount = weights.cols; + RNG rng(0); + cv::Mat samples = cv::Mat::zeros(datasize, featureCount, CV_32FC1); - cv::Mat responses = cv::Mat::zeros(datasize, 1, CV_32FC1); - switch(type) + for (int featureIndex = 0; featureIndex < featureCount; featureIndex++) { - case UNIFORM_SAME_SCALE: - generateSameScaleData(samples); - break; - case UNIFORM_DIFFERENT_SCALES: - generateDifferentScalesData(samples, shift); - break; - default: - CV_Error(CV_StsBadArg, "Unknown train data type"); + rng.fill(samples.col(featureIndex), RNG::UNIFORM, borders[featureIndex].first, borders[featureIndex].second); } + cv::Mat responses = cv::Mat::zeros(datasize, 1, CV_32FC1); for (int sampleIndex = 0; sampleIndex < datasize; sampleIndex++) { responses.at(sampleIndex) = decisionFunction(samples.row(sampleIndex), weights, shift) > 0 ? 1 : -1; @@ -138,14 +134,14 @@ void CV_SVMSGDTrainTest::makeTestData(Mat weights, float shift) { int testSamplesCount = 100000; int featureCount = weights.cols; - - float lowerLimit = -TEST_VALUE_LIMIT; - float upperLimit = TEST_VALUE_LIMIT; - cv::RNG rng(0); testSamples.create(testSamplesCount, featureCount, CV_32FC1); - rng.fill(testSamples, RNG::UNIFORM, lowerLimit, upperLimit); + for (int featureIndex = 0; featureIndex < featureCount; featureIndex++) + { + rng.fill(testSamples.col(featureIndex), RNG::UNIFORM, borders[featureIndex].first, borders[featureIndex].second); + } + testResponses.create(testSamplesCount, 1, CV_32FC1); for (int i = 0 ; i < testSamplesCount; i++) @@ -154,10 +150,25 @@ void CV_SVMSGDTrainTest::makeTestData(Mat weights, float shift) } } -CV_SVMSGDTrainTest::CV_SVMSGDTrainTest(Mat weights, float shift, TrainDataType _type, double _precision) +CV_SVMSGDTrainTest::CV_SVMSGDTrainTest(const Mat &weights, float shift, TrainDataType _type, double _precision) { type = _type; precision = _precision; + + int featureCount = weights.cols; + + switch(type) + { + case UNIFORM_SAME_SCALE: + generateSameBorders(featureCount); + break; + case UNIFORM_DIFFERENT_SCALES: + generateDifferentBorders(featureCount); + break; + default: + CV_Error(CV_StsBadArg, "Unknown train data type"); + } + makeTrainData(weights, shift); makeTestData(weights, shift); } @@ -271,7 +282,7 @@ TEST(ML_SVMSGD, trainDifferentScales5) float shift = 0; makeWeightsAndShift(featureCount, weights, shift); - CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_DIFFERENT_SCALES, 0.05); + CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_DIFFERENT_SCALES, 0.01); test.safe_run(); } @@ -284,6 +295,44 @@ TEST(ML_SVMSGD, trainDifferentScales100) float shift = 0; makeWeightsAndShift(featureCount, weights, shift); - CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_DIFFERENT_SCALES, 0.10); + CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_DIFFERENT_SCALES, 0.01); test.safe_run(); } + +TEST(ML_SVMSGD, twoPoints) +{ + Mat samples(2, 2, CV_32FC1); + samples.at(0,0) = 0; + samples.at(0,1) = 0; + samples.at(1,0) = 1000; + samples.at(1,1) = 1; + + Mat responses(2, 1, CV_32FC1); + responses.at(0) = -1; + responses.at(1) = 1; + + cv::Ptr trainData = TrainData::create(samples, cv::ml::ROW_SAMPLE, responses); + + Mat realWeights(1, 2, CV_32FC1); + realWeights.at(0) = 1000; + realWeights.at(1) = 1; + + float realShift = -500000.5; + + float normRealWeights = norm(realWeights); + realWeights /= normRealWeights; + realShift /= normRealWeights; + + cv::Ptr svmsgd = SVMSGD::create(); + svmsgd->setOptimalParameters(); + svmsgd->train( trainData ); + + Mat foundWeights = svmsgd->getWeights(); + float foundShift = svmsgd->getShift(); + + float normFoundWeights = norm(foundWeights); + foundWeights /= normFoundWeights; + foundShift /= normFoundWeights; + CV_Assert((norm(foundWeights - realWeights) < 0.001) && (abs((foundShift - realShift) / realShift) < 0.05)); +} + diff --git a/samples/cpp/train_svmsgd.cpp b/samples/cpp/train_svmsgd.cpp index 7d537341f8..cca845ef58 100644 --- a/samples/cpp/train_svmsgd.cpp +++ b/samples/cpp/train_svmsgd.cpp @@ -48,8 +48,8 @@ bool doTrain( const Mat samples, const Mat responses, Mat &weights, float &shift cv::Ptr svmsgd = SVMSGD::create(); svmsgd->setOptimalParameters(); - cv::Ptr train_data = TrainData::create(samples, cv::ml::ROW_SAMPLE, responses); - svmsgd->train( train_data ); + cv::Ptr trainData = TrainData::create(samples, cv::ml::ROW_SAMPLE, responses); + svmsgd->train( trainData ); if (svmsgd->isTrained()) { From 05353a149256ea1a2548cebe3a2b03c43d17bc96 Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Wed, 10 Feb 2016 15:40:09 +0300 Subject: [PATCH 06/19] Removed trailing whitespaces --- modules/ml/src/svmsgd.cpp | 4 ++-- modules/ml/test/test_svmsgd.cpp | 1 - samples/cpp/train_svmsgd.cpp | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/ml/src/svmsgd.cpp b/modules/ml/src/svmsgd.cpp index 4ad103e383..8de076fa23 100644 --- a/modules/ml/src/svmsgd.cpp +++ b/modules/ml/src/svmsgd.cpp @@ -140,7 +140,7 @@ private: }; Ptr SVMSGD::create() -{ +{ return makePtr(); } @@ -265,7 +265,7 @@ bool SVMSGDImpl::train(const Ptr& data, int) weights_ = Mat::zeros(1, featureCount, CV_32F); shift_ = areEmpty.first ? -1 : 1; return true; - } + } Mat extendedTrainSamples; Mat average; diff --git a/modules/ml/test/test_svmsgd.cpp b/modules/ml/test/test_svmsgd.cpp index 666ef18be6..993a0de6b8 100644 --- a/modules/ml/test/test_svmsgd.cpp +++ b/modules/ml/test/test_svmsgd.cpp @@ -335,4 +335,3 @@ TEST(ML_SVMSGD, twoPoints) foundShift /= normFoundWeights; CV_Assert((norm(foundWeights - realWeights) < 0.001) && (abs((foundShift - realShift) / realShift) < 0.05)); } - diff --git a/samples/cpp/train_svmsgd.cpp b/samples/cpp/train_svmsgd.cpp index cca845ef58..616665e87d 100644 --- a/samples/cpp/train_svmsgd.cpp +++ b/samples/cpp/train_svmsgd.cpp @@ -187,7 +187,7 @@ static void onMouse( int event, int x, int y, int, void* pData) switch( event ) { - case CV_EVENT_LBUTTONUP: + case CV_EVENT_LBUTTONUP: addPointRetrainAndRedraw(data, x, y, 1); break; From c52217236732542b7ce9e70036bdb9a7574bfcd8 Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Wed, 10 Feb 2016 16:44:16 +0300 Subject: [PATCH 07/19] Fixed small bug in SVMSGD::clear(). --- modules/ml/src/svmsgd.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ml/src/svmsgd.cpp b/modules/ml/src/svmsgd.cpp index 8de076fa23..64b44814c5 100644 --- a/modules/ml/src/svmsgd.cpp +++ b/modules/ml/src/svmsgd.cpp @@ -502,6 +502,7 @@ void SVMSGDImpl::readParams( const FileNode& fn ) void SVMSGDImpl::clear() { weights_.release(); + shift_ = 0; } From 617dd5db5b4cea0dbf4ae89d83bc5dfb2e324721 Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Wed, 10 Feb 2016 17:37:05 +0300 Subject: [PATCH 08/19] Fixed doc/opencv.bib --- doc/opencv.bib | 12 +++++++++++- modules/ml/include/opencv2/ml.hpp | 1 - modules/ml/src/precomp.hpp | 1 + modules/ml/test/test_mltests2.cpp | 2 -- modules/ml/test/test_save_load.cpp | 6 +----- modules/ts/src/ts_gtest.cpp | 2 +- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/doc/opencv.bib b/doc/opencv.bib index 17eedf7172..58ede18322 100644 --- a/doc/opencv.bib +++ b/doc/opencv.bib @@ -415,6 +415,16 @@ pages = {2548--2555}, organization = {IEEE} } +@ARTICLE{Louhichi07, + author = {Louhichi, H. and Fournel, T. and Lavest, J. M. and Ben Aissia, H.}, + title = {Self-calibration of Scheimpflug cameras: an easy protocol}, + year = {2007}, + pages = {2616–2622}, + journal = {Meas. Sci. Technol.}, + volume = {18}, + number = {8}, + publisher = {IOP Publishing Ltd} +} @ARTICLE{LibSVM, author = {Chang, Chih-Chung and Lin, Chih-Jen}, title = {LIBSVM: a library for support vector machines}, @@ -871,4 +881,4 @@ pages={177--186}, year={2010}, publisher={Springer} -} \ No newline at end of file +} diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index ec7c0730ec..3a8c2ba25b 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1697,7 +1697,6 @@ CV_EXPORTS void randMVNormal( InputArray mean, InputArray cov, int nsamples, Out CV_EXPORTS void createConcentricSpheresTestSet( int nsamples, int nfeatures, int nclasses, OutputArray samples, OutputArray responses); - //! @} ml } diff --git a/modules/ml/src/precomp.hpp b/modules/ml/src/precomp.hpp index 0a5fa06035..84821988b6 100644 --- a/modules/ml/src/precomp.hpp +++ b/modules/ml/src/precomp.hpp @@ -45,6 +45,7 @@ #include "opencv2/ml.hpp" #include "opencv2/core/core_c.h" #include "opencv2/core/utility.hpp" + #include "opencv2/core/private.hpp" #include diff --git a/modules/ml/test/test_mltests2.cpp b/modules/ml/test/test_mltests2.cpp index 9cdd368c4c..01f5568daf 100644 --- a/modules/ml/test/test_mltests2.cpp +++ b/modules/ml/test/test_mltests2.cpp @@ -267,9 +267,7 @@ void CV_MLBaseTest::run( int ) { string filename = ts->get_data_path(); filename += get_validation_filename(); - validationFS.open( filename, FileStorage::READ ); - read_params( *validationFS ); int code = cvtest::TS::OK; diff --git a/modules/ml/test/test_save_load.cpp b/modules/ml/test/test_save_load.cpp index f55f9b8a11..d127471cc9 100644 --- a/modules/ml/test/test_save_load.cpp +++ b/modules/ml/test/test_save_load.cpp @@ -150,11 +150,7 @@ int CV_SLMLTest::validate_test_results( int testCaseIdx ) TEST(ML_NaiveBayes, save_load) { CV_SLMLTest test( CV_NBAYES ); test.safe_run(); } TEST(ML_KNearest, save_load) { CV_SLMLTest test( CV_KNEAREST ); test.safe_run(); } -TEST(ML_SVM, save_load) -{ - CV_SLMLTest test( CV_SVM ); - test.safe_run(); -} +TEST(ML_SVM, save_load) { CV_SLMLTest test( CV_SVM ); test.safe_run(); } TEST(ML_ANN, save_load) { CV_SLMLTest test( CV_ANN ); test.safe_run(); } TEST(ML_DTree, save_load) { CV_SLMLTest test( CV_DTREE ); test.safe_run(); } TEST(ML_Boost, save_load) { CV_SLMLTest test( CV_BOOST ); test.safe_run(); } diff --git a/modules/ts/src/ts_gtest.cpp b/modules/ts/src/ts_gtest.cpp index 5604eb7e62..29a3996be8 100644 --- a/modules/ts/src/ts_gtest.cpp +++ b/modules/ts/src/ts_gtest.cpp @@ -5659,7 +5659,7 @@ class TestCaseNameIs { // Returns true iff the name of test_case matches name_. bool operator()(const TestCase* test_case) const { - return test_case != NULL && strcmp(test_case->name(), name_.c_str()) == 0; + return test_case != NULL && strcmp(test_case->name(), name_.c_str()) == 0; } private: From 0307dd19fdd7f4b0b5fa25c532dd0f34ad9574cd Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Wed, 10 Feb 2016 18:21:45 +0300 Subject: [PATCH 09/19] Minor fix in declaration of SVMSGD::setOptimalParameters --- modules/ml/include/opencv2/ml.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index 3a8c2ba25b..e23c242d5c 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1637,7 +1637,7 @@ public: * @param marginType is the type of margin constraint. Legal values are MarginType::SOFT_MARGIN and MarginType::HARD_MARGIN. * Default value is MarginType::SOFT_MARGIN. */ - CV_WRAP virtual void setOptimalParameters(int svmsgdType = ASGD, int marginType = SOFT_MARGIN) = 0; + CV_WRAP virtual void setOptimalParameters(int svmsgdType = SVMSGD::ASGD, int marginType = SVMSGD::SOFT_MARGIN) = 0; /** @brief %Algorithm type, one of SVMSGD::SvmsgdType. */ /** @see setAlgorithmType */ From 5496dedd6a38db7561f2a3a2d3c13407dd4972ea Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Wed, 10 Feb 2016 19:46:24 +0300 Subject: [PATCH 10/19] Fixed warnings. --- modules/ml/include/opencv2/ml.hpp | 4 ++-- modules/ml/src/svmsgd.cpp | 31 ++++++++++++++++--------------- modules/ml/test/test_svmsgd.cpp | 12 ++++++------ samples/cpp/train_svmsgd.cpp | 12 ++++++------ 4 files changed, 30 insertions(+), 29 deletions(-) diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index e23c242d5c..8b56f0d798 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1640,9 +1640,9 @@ public: CV_WRAP virtual void setOptimalParameters(int svmsgdType = SVMSGD::ASGD, int marginType = SVMSGD::SOFT_MARGIN) = 0; /** @brief %Algorithm type, one of SVMSGD::SvmsgdType. */ - /** @see setAlgorithmType */ + /** @see setSvmsgdType */ CV_WRAP virtual int getSvmsgdType() const = 0; - /** @copybrief getAlgorithmType @see getAlgorithmType */ + /** @copybrief getSvmsgdType @see getSvmsgdType */ CV_WRAP virtual void setSvmsgdType(int svmsgdType) = 0; /** @brief %Margin type, one of SVMSGD::MarginType. */ diff --git a/modules/ml/src/svmsgd.cpp b/modules/ml/src/svmsgd.cpp index 64b44814c5..3ae051a9dc 100644 --- a/modules/ml/src/svmsgd.cpp +++ b/modules/ml/src/svmsgd.cpp @@ -172,7 +172,8 @@ void SVMSGDImpl::normalizeSamples(Mat &samples, Mat &average, float &multiplier) average = Mat(1, featuresCount, samples.type()); for (int featureIndex = 0; featureIndex < featuresCount; featureIndex++) { - average.at(featureIndex) = mean(samples.col(featureIndex))[0]; + Scalar scalAverage = mean(samples.col(featureIndex))[0]; + average.at(featureIndex) = static_cast(scalAverage[0]); } for (int sampleIndex = 0; sampleIndex < samplesCount; sampleIndex++) @@ -182,7 +183,7 @@ void SVMSGDImpl::normalizeSamples(Mat &samples, Mat &average, float &multiplier) double normValue = norm(samples); - multiplier = sqrt(samples.total()) / normValue; + multiplier = static_cast(sqrt(samples.total()) / normValue); samples *= multiplier; } @@ -228,11 +229,11 @@ float SVMSGDImpl::calcShift(InputArray _samples, InputArray _responses) const for (int samplesIndex = 0; samplesIndex < trainSamplesCount; samplesIndex++) { Mat currentSample = trainSamples.row(samplesIndex); - float dotProduct = currentSample.dot(weights_); + float dotProduct = static_cast(currentSample.dot(weights_)); bool firstClass = isFirstClass(trainResponses.at(samplesIndex)); - int index = firstClass ? 0:1; - float signToMul = firstClass ? 1 : -1; + int index = firstClass ? 0 : 1; + float signToMul = firstClass ? 1.f : -1.f; float curDistance = dotProduct * signToMul; if (curDistance < distanceToClasses[index]) @@ -263,7 +264,7 @@ bool SVMSGDImpl::train(const Ptr& data, int) if ( areEmpty.first || areEmpty.second ) { weights_ = Mat::zeros(1, featureCount, CV_32F); - shift_ = areEmpty.first ? -1 : 1; + shift_ = areEmpty.first ? -1.f : 1.f; return true; } @@ -329,7 +330,7 @@ bool SVMSGDImpl::train(const Ptr& data, int) if (params.marginType == SOFT_MARGIN) { - shift_ = extendedWeights.at(featureCount) - weights_.dot(average); + shift_ = extendedWeights.at(featureCount) - static_cast(weights_.dot(average)); } else { @@ -363,8 +364,8 @@ float SVMSGDImpl::predict( InputArray _samples, OutputArray _results, int ) cons for (int sampleIndex = 0; sampleIndex < nSamples; sampleIndex++) { Mat currentSample = samples.row(sampleIndex); - float criterion = currentSample.dot(weights_) + shift_; - results.at(sampleIndex) = (criterion >= 0) ? 1 : -1; + float criterion = static_cast(currentSample.dot(weights_)) + shift_; + results.at(sampleIndex) = (criterion >= 0) ? 1.f : -1.f; } return result; @@ -530,9 +531,9 @@ void SVMSGDImpl::setOptimalParameters(int svmsgdType, int marginType) params.svmsgdType = SGD; params.marginType = (marginType == SOFT_MARGIN) ? SOFT_MARGIN : (marginType == HARD_MARGIN) ? HARD_MARGIN : ILLEGAL_MARGIN_TYPE; - params.lambda = 0.0001; - params.gamma0 = 0.05; - params.c = 1; + params.lambda = 0.0001f; + params.gamma0 = 0.05f; + params.c = 1.f; params.termCrit = TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 100000, 0.00001); break; @@ -540,9 +541,9 @@ void SVMSGDImpl::setOptimalParameters(int svmsgdType, int marginType) params.svmsgdType = ASGD; params.marginType = (marginType == SOFT_MARGIN) ? SOFT_MARGIN : (marginType == HARD_MARGIN) ? HARD_MARGIN : ILLEGAL_MARGIN_TYPE; - params.lambda = 0.00001; - params.gamma0 = 0.05; - params.c = 0.75; + params.lambda = 0.00001f; + params.gamma0 = 0.05f; + params.c = 0.75f; params.termCrit = TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 100000, 0.00001); break; diff --git a/modules/ml/test/test_svmsgd.cpp b/modules/ml/test/test_svmsgd.cpp index 993a0de6b8..592e66b3b0 100644 --- a/modules/ml/test/test_svmsgd.cpp +++ b/modules/ml/test/test_svmsgd.cpp @@ -124,7 +124,7 @@ void CV_SVMSGDTrainTest::makeTrainData(Mat weights, float shift) cv::Mat responses = cv::Mat::zeros(datasize, 1, CV_32FC1); for (int sampleIndex = 0; sampleIndex < datasize; sampleIndex++) { - responses.at(sampleIndex) = decisionFunction(samples.row(sampleIndex), weights, shift) > 0 ? 1 : -1; + responses.at(sampleIndex) = decisionFunction(samples.row(sampleIndex), weights, shift) > 0 ? 1.f : -1.f; } data = TrainData::create(samples, cv::ml::ROW_SAMPLE, responses); @@ -146,7 +146,7 @@ void CV_SVMSGDTrainTest::makeTestData(Mat weights, float shift) for (int i = 0 ; i < testSamplesCount; i++) { - testResponses.at(i) = decisionFunction(testSamples.row(i), weights, shift) > 0 ? 1 : -1; + testResponses.at(i) = decisionFunction(testSamples.row(i), weights, shift) > 0 ? 1.f : -1.f; } } @@ -175,7 +175,7 @@ CV_SVMSGDTrainTest::CV_SVMSGDTrainTest(const Mat &weights, float shift, TrainDat float CV_SVMSGDTrainTest::decisionFunction(const Mat &sample, const Mat &weights, float shift) { - return sample.dot(weights) + shift; + return static_cast(sample.dot(weights)) + shift; } void CV_SVMSGDTrainTest::run( int /*start_from*/ ) @@ -217,7 +217,7 @@ void makeWeightsAndShift(int featureCount, Mat &weights, float &shift) double upperLimit = 1; rng.fill(weights, RNG::UNIFORM, lowerLimit, upperLimit); - shift = rng.uniform(-featureCount, featureCount); + shift = static_cast(rng.uniform(-featureCount, featureCount)); } @@ -319,7 +319,7 @@ TEST(ML_SVMSGD, twoPoints) float realShift = -500000.5; - float normRealWeights = norm(realWeights); + float normRealWeights = static_cast(norm(realWeights)); realWeights /= normRealWeights; realShift /= normRealWeights; @@ -330,7 +330,7 @@ TEST(ML_SVMSGD, twoPoints) Mat foundWeights = svmsgd->getWeights(); float foundShift = svmsgd->getShift(); - float normFoundWeights = norm(foundWeights); + float normFoundWeights = static_cast(norm(foundWeights)); foundWeights /= normFoundWeights; foundShift /= normFoundWeights; CV_Assert((norm(foundWeights - realWeights) < 0.001) && (abs((foundShift - realShift) / realShift) < 0.05)); diff --git a/samples/cpp/train_svmsgd.cpp b/samples/cpp/train_svmsgd.cpp index 616665e87d..aed8228c4b 100644 --- a/samples/cpp/train_svmsgd.cpp +++ b/samples/cpp/train_svmsgd.cpp @@ -97,7 +97,7 @@ bool findCrossPointWithBorders(const Mat &weights, float shift, const std::pair< if (xMin == xMax && weights.at(1) != 0) { x = xMin; - y = std::floor( - (weights.at(0) * x + shift) / weights.at(1)); + y = static_cast(std::floor( - (weights.at(0) * x + shift) / weights.at(1))); if (y >= yMin && y <= yMax) { crossPoint.x = x; @@ -108,7 +108,7 @@ bool findCrossPointWithBorders(const Mat &weights, float shift, const std::pair< else if (yMin == yMax && weights.at(0) != 0) { y = yMin; - x = std::floor( - (weights.at(1) * y + shift) / weights.at(0)); + x = static_cast(std::floor( - (weights.at(1) * y + shift) / weights.at(0))); if (x >= xMin && x <= xMax) { crossPoint.x = x; @@ -149,8 +149,8 @@ void redraw(Data data, const Point points[2]) Scalar color; for (int i = 0; i < data.samples.rows; i++) { - center.x = data.samples.at(i,0); - center.y = data.samples.at(i,1); + center.x = static_cast(data.samples.at(i,0)); + center.y = static_cast(data.samples.at(i,1)); color = (data.responses.at(i) > 0) ? Scalar(128,128,0) : Scalar(0,128,128); circle(data.img, center, radius, color, 5); } @@ -163,8 +163,8 @@ void addPointRetrainAndRedraw(Data &data, int x, int y, int response) { Mat currentSample(1, 2, CV_32F); - currentSample.at(0,0) = x; - currentSample.at(0,1) = y; + currentSample.at(0,0) = (float)x; + currentSample.at(0,1) = (float)y; data.samples.push_back(currentSample); data.responses.push_back(response); From ff549527697af8a66889c5a7d010fe6a768b1076 Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Mon, 15 Feb 2016 14:35:36 +0300 Subject: [PATCH 11/19] Corrected spelling mistakes --- modules/ml/include/opencv2/ml.hpp | 2 +- modules/ml/src/svmsgd.cpp | 23 +++++++++++------------ modules/ml/test/test_svmsgd.cpp | 2 +- samples/cpp/train_svmsgd.cpp | 4 ++-- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index 8b56f0d798..346df8f15c 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1535,7 +1535,7 @@ The margin type may have one of the following values: \ref SOFT_MARGIN or \ref H - You should use \ref HARD_MARGIN type, if you have linearly separable sets. - You should use \ref SOFT_MARGIN type, if you have non-linearly separable sets or sets with outliers. -- In the general case (if you know nothing about linearly separability of your sets), use SOFT_MARGIN. +- In the general case (if you know nothing about linear separability of your sets), use SOFT_MARGIN. The other parameters may be described as follows: - \f$\lambda\f$ parameter is responsible for weights decreasing at each step and for the strength of restrictions on outliers diff --git a/modules/ml/src/svmsgd.cpp b/modules/ml/src/svmsgd.cpp index 3ae051a9dc..77ac2ad67b 100644 --- a/modules/ml/src/svmsgd.cpp +++ b/modules/ml/src/svmsgd.cpp @@ -11,7 +11,7 @@ // For Open Source Computer Vision Library // // Copyright (C) 2000, Intel Corporation, all rights reserved. -// Copyright (C) 2014, Itseez Inc, all rights reserved. +// Copyright (C) 2016, Itseez Inc, all rights reserved. // Third party copyrights are property of their respective owners. // // Redistribution and use in source and binary forms, with or without modification, @@ -103,7 +103,7 @@ public: CV_IMPL_PROPERTY_S(cv::TermCriteria, TermCriteria, params.termCrit) private: - void updateWeights(InputArray sample, bool isFirstClass, float gamma, Mat &weights); + void updateWeights(InputArray sample, bool isPositive, float gamma, Mat &weights); std::pair areClassesEmpty(Mat responses); @@ -111,7 +111,7 @@ private: void readParams( const FileNode &fn ); - static inline bool isFirstClass(float val) { return val > 0; } + static inline bool isPositive(float val) { return val > 0; } static void normalizeSamples(Mat &matrix, Mat &average, float &multiplier); @@ -152,7 +152,7 @@ std::pair SVMSGDImpl::areClassesEmpty(Mat responses) for(int index = 0; index < limit_index; index++) { - if (isFirstClass(responses.at(index))) + if (isPositive(responses.at(index))) emptyInClasses.first = false; else emptyInClasses.second = false; @@ -172,7 +172,7 @@ void SVMSGDImpl::normalizeSamples(Mat &samples, Mat &average, float &multiplier) average = Mat(1, featuresCount, samples.type()); for (int featureIndex = 0; featureIndex < featuresCount; featureIndex++) { - Scalar scalAverage = mean(samples.col(featureIndex))[0]; + Scalar scalAverage = mean(samples.col(featureIndex)); average.at(featureIndex) = static_cast(scalAverage[0]); } @@ -190,13 +190,13 @@ void SVMSGDImpl::normalizeSamples(Mat &samples, Mat &average, float &multiplier) void SVMSGDImpl::makeExtendedTrainSamples(const Mat &trainSamples, Mat &extendedTrainSamples, Mat &average, float &multiplier) { - Mat normalisedTrainSamples = trainSamples.clone(); - int samplesCount = normalisedTrainSamples.rows; + Mat normalizedTrainSamples = trainSamples.clone(); + int samplesCount = normalizedTrainSamples.rows; - normalizeSamples(normalisedTrainSamples, average, multiplier); + normalizeSamples(normalizedTrainSamples, average, multiplier); Mat onesCol = Mat::ones(samplesCount, 1, CV_32F); - cv::hconcat(normalisedTrainSamples, onesCol, extendedTrainSamples); + cv::hconcat(normalizedTrainSamples, onesCol, extendedTrainSamples); } void SVMSGDImpl::updateWeights(InputArray _sample, bool firstClass, float gamma, Mat& weights) @@ -231,7 +231,7 @@ float SVMSGDImpl::calcShift(InputArray _samples, InputArray _responses) const Mat currentSample = trainSamples.row(samplesIndex); float dotProduct = static_cast(currentSample.dot(weights_)); - bool firstClass = isFirstClass(trainResponses.at(samplesIndex)); + bool firstClass = isPositive(trainResponses.at(samplesIndex)); int index = firstClass ? 0 : 1; float signToMul = firstClass ? 1.f : -1.f; float curDistance = dotProduct * signToMul; @@ -297,11 +297,10 @@ bool SVMSGDImpl::train(const Ptr& data, int) int randomNumber = rng.uniform(0, extendedTrainSamplesCount); //generate sample number Mat currentSample = extendedTrainSamples.row(randomNumber); - bool firstClass = isFirstClass(trainResponses.at(randomNumber)); float gamma = params.gamma0 * std::pow((1 + params.lambda * params.gamma0 * (float)iter), (-params.c)); //update gamma - updateWeights( currentSample, firstClass, gamma, extendedWeights ); + updateWeights( currentSample, isPositive(trainResponses.at(randomNumber)), gamma, extendedWeights ); //average weights (only for ASGD model) if (params.svmsgdType == ASGD) diff --git a/modules/ml/test/test_svmsgd.cpp b/modules/ml/test/test_svmsgd.cpp index 592e66b3b0..00ebbf3341 100644 --- a/modules/ml/test/test_svmsgd.cpp +++ b/modules/ml/test/test_svmsgd.cpp @@ -134,7 +134,7 @@ void CV_SVMSGDTrainTest::makeTestData(Mat weights, float shift) { int testSamplesCount = 100000; int featureCount = weights.cols; - cv::RNG rng(0); + cv::RNG rng(42); testSamples.create(testSamplesCount, featureCount, CV_32FC1); for (int featureIndex = 0; featureIndex < featureCount; featureIndex++) diff --git a/samples/cpp/train_svmsgd.cpp b/samples/cpp/train_svmsgd.cpp index aed8228c4b..a68f613b2f 100644 --- a/samples/cpp/train_svmsgd.cpp +++ b/samples/cpp/train_svmsgd.cpp @@ -6,8 +6,6 @@ using namespace cv; using namespace cv::ml; -#define WIDTH 841 -#define HEIGHT 594 struct Data { @@ -17,6 +15,8 @@ struct Data Data() { + const int WIDTH = 841; + const int HEIGHT = 594; img = Mat::zeros(HEIGHT, WIDTH, CV_8UC3); imshow("Train svmsgd", img); } From 02cd8cf039bf475b24f86709e0d42911844ece35 Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Mon, 15 Feb 2016 15:09:59 +0300 Subject: [PATCH 12/19] Deleted illegal type values. --- modules/ml/include/opencv2/ml.hpp | 2 - modules/ml/src/svmsgd.cpp | 92 +++++++------------------------ 2 files changed, 21 insertions(+), 73 deletions(-) diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index 346df8f15c..75fb6d134b 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1588,7 +1588,6 @@ public: ASGD is often the preferable choice. */ enum SvmsgdType { - ILLEGAL_SVMSGD_TYPE, SGD, //!< Stochastic Gradient Descent ASGD //!< Average Stochastic Gradient Descent }; @@ -1596,7 +1595,6 @@ public: /** Margin type.*/ enum MarginType { - ILLEGAL_MARGIN_TYPE, SOFT_MARGIN, //!< General case, suits to the case of non-linearly separable sets, allows outliers. HARD_MARGIN //!< More accurate for the case of linearly separable sets. }; diff --git a/modules/ml/src/svmsgd.cpp b/modules/ml/src/svmsgd.cpp index 77ac2ad67b..0f1efcfea6 100644 --- a/modules/ml/src/svmsgd.cpp +++ b/modules/ml/src/svmsgd.cpp @@ -89,14 +89,8 @@ public: virtual void setOptimalParameters(int svmsgdType = ASGD, int marginType = SOFT_MARGIN); - virtual int getSvmsgdType() const; - - virtual void setSvmsgdType(int svmsgdType); - - virtual int getMarginType() const; - - virtual void setMarginType(int marginType); - + CV_IMPL_PROPERTY(int, SvmsgdType, params.svmsgdType) + CV_IMPL_PROPERTY(int, MarginType, params.marginType) CV_IMPL_PROPERTY(float, Lambda, params.lambda) CV_IMPL_PROPERTY(float, Gamma0, params.gamma0) CV_IMPL_PROPERTY(float, C, params.c) @@ -132,8 +126,8 @@ private: float gamma0; //learning rate float c; TermCriteria termCrit; - SvmsgdType svmsgdType; - MarginType marginType; + int svmsgdType; + int marginType; }; SVMSGDParams params; @@ -148,9 +142,9 @@ std::pair SVMSGDImpl::areClassesEmpty(Mat responses) { CV_Assert(responses.cols == 1 || responses.rows == 1); std::pair emptyInClasses(true, true); - int limit_index = responses.rows; + int limitIndex = responses.rows; - for(int index = 0; index < limit_index; index++) + for(int index = 0; index < limitIndex; index++) { if (isPositive(responses.at(index))) emptyInClasses.first = false; @@ -276,9 +270,9 @@ bool SVMSGDImpl::train(const Ptr& data, int) int extendedTrainSamplesCount = extendedTrainSamples.rows; int extendedFeatureCount = extendedTrainSamples.cols; - Mat extendedWeights = Mat::zeros(1, extendedFeatureCount, CV_32F); // Initialize extendedWeights vector with zeros - Mat previousWeights = Mat::zeros(1, extendedFeatureCount, CV_32F); //extendedWeights vector for calculating terminal criterion - Mat averageExtendedWeights; //average extendedWeights vector for ASGD model + Mat extendedWeights = Mat::zeros(1, extendedFeatureCount, CV_32F); + Mat previousWeights = Mat::zeros(1, extendedFeatureCount, CV_32F); + Mat averageExtendedWeights; if (params.svmsgdType == ASGD) { averageExtendedWeights = Mat::zeros(1, extendedFeatureCount, CV_32F); @@ -407,10 +401,8 @@ void SVMSGDImpl::writeParams( FileStorage& fs ) const case ASGD: SvmsgdTypeStr = "ASGD"; break; - case ILLEGAL_SVMSGD_TYPE: - SvmsgdTypeStr = format("Unknown_%d", params.svmsgdType); default: - std::cout << "params.svmsgdType isn't initialized" << std::endl; + SvmsgdTypeStr = format("Unknown_%d", params.svmsgdType); } fs << "svmsgdType" << SvmsgdTypeStr; @@ -425,10 +417,8 @@ void SVMSGDImpl::writeParams( FileStorage& fs ) const case HARD_MARGIN: marginTypeStr = "HARD_MARGIN"; break; - case ILLEGAL_MARGIN_TYPE: - marginTypeStr = format("Unknown_%d", params.marginType); default: - std::cout << "params.marginType isn't initialized" << std::endl; + marginTypeStr = format("Unknown_%d", params.marginType); } fs << "marginType" << marginTypeStr; @@ -458,21 +448,21 @@ void SVMSGDImpl::read(const FileNode& fn) void SVMSGDImpl::readParams( const FileNode& fn ) { String svmsgdTypeStr = (String)fn["svmsgdType"]; - SvmsgdType svmsgdType = + int svmsgdType = svmsgdTypeStr == "SGD" ? SGD : - svmsgdTypeStr == "ASGD" ? ASGD : ILLEGAL_SVMSGD_TYPE; + svmsgdTypeStr == "ASGD" ? ASGD : -1; - if( svmsgdType == ILLEGAL_SVMSGD_TYPE ) + if( svmsgdType < 0 ) CV_Error( CV_StsParseError, "Missing or invalid SVMSGD type" ); params.svmsgdType = svmsgdType; String marginTypeStr = (String)fn["marginType"]; - MarginType marginType = + int marginType = marginTypeStr == "SOFT_MARGIN" ? SOFT_MARGIN : - marginTypeStr == "HARD_MARGIN" ? HARD_MARGIN : ILLEGAL_MARGIN_TYPE; + marginTypeStr == "HARD_MARGIN" ? HARD_MARGIN : -1; - if( marginType == ILLEGAL_MARGIN_TYPE ) + if( marginType < 0 ) CV_Error( CV_StsParseError, "Missing or invalid margin type" ); params.marginType = marginType; @@ -510,8 +500,8 @@ SVMSGDImpl::SVMSGDImpl() { clear(); - params.svmsgdType = ILLEGAL_SVMSGD_TYPE; - params.marginType = ILLEGAL_MARGIN_TYPE; + params.svmsgdType = -1; + params.marginType = -1; // Parameters for learning params.lambda = 0; // regularization @@ -529,7 +519,7 @@ void SVMSGDImpl::setOptimalParameters(int svmsgdType, int marginType) case SGD: params.svmsgdType = SGD; params.marginType = (marginType == SOFT_MARGIN) ? SOFT_MARGIN : - (marginType == HARD_MARGIN) ? HARD_MARGIN : ILLEGAL_MARGIN_TYPE; + (marginType == HARD_MARGIN) ? HARD_MARGIN : -1; params.lambda = 0.0001f; params.gamma0 = 0.05f; params.c = 1.f; @@ -539,7 +529,7 @@ void SVMSGDImpl::setOptimalParameters(int svmsgdType, int marginType) case ASGD: params.svmsgdType = ASGD; params.marginType = (marginType == SOFT_MARGIN) ? SOFT_MARGIN : - (marginType == HARD_MARGIN) ? HARD_MARGIN : ILLEGAL_MARGIN_TYPE; + (marginType == HARD_MARGIN) ? HARD_MARGIN : -1; params.lambda = 0.00001f; params.gamma0 = 0.05f; params.c = 0.75f; @@ -550,45 +540,5 @@ void SVMSGDImpl::setOptimalParameters(int svmsgdType, int marginType) CV_Error( CV_StsParseError, "SVMSGD model data is invalid" ); } } - -void SVMSGDImpl::setSvmsgdType(int type) -{ - switch (type) - { - case SGD: - params.svmsgdType = SGD; - break; - case ASGD: - params.svmsgdType = ASGD; - break; - default: - params.svmsgdType = ILLEGAL_SVMSGD_TYPE; - } -} - -int SVMSGDImpl::getSvmsgdType() const -{ - return params.svmsgdType; -} - -void SVMSGDImpl::setMarginType(int type) -{ - switch (type) - { - case HARD_MARGIN: - params.marginType = HARD_MARGIN; - break; - case SOFT_MARGIN: - params.marginType = SOFT_MARGIN; - break; - default: - params.marginType = ILLEGAL_MARGIN_TYPE; - } -} - -int SVMSGDImpl::getMarginType() const -{ - return params.marginType; -} } //ml } //cv From f3c4a6aab8c1d60e72e696fa524c01ba644d42fe Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Wed, 24 Feb 2016 13:22:07 +0300 Subject: [PATCH 13/19] Rename parameters lambda, gamma0 and c. --- modules/ml/include/opencv2/ml.hpp | 60 +++++++++++------------ modules/ml/src/svmsgd.cpp | 81 +++++++++++++++---------------- modules/ml/test/test_mltests2.cpp | 6 +-- modules/ml/test/test_svmsgd.cpp | 1 - samples/cpp/train_svmsgd.cpp | 4 +- 5 files changed, 75 insertions(+), 77 deletions(-) diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index 75fb6d134b..7710abdae1 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1511,9 +1511,9 @@ The gradient descent show amazing performance for large-scale problems, reducing The classifier has 5 parameters. These are - model type, - margin type, -- \f$\lambda\f$ (strength of restrictions on outliers), -- \f$\gamma_0\f$ (initial step size), -- \f$c\f$ (power coefficient for decreasing of step size), +- marginRegularization (\f$\lambda\f$), +- initialStepSize (\f$\gamma_0\f$), +- stepDecreasingPower (\f$c\f$), - and termination criteria. The model type may have one of the following values: \ref SGD and \ref ASGD. @@ -1538,27 +1538,27 @@ The margin type may have one of the following values: \ref SOFT_MARGIN or \ref H - In the general case (if you know nothing about linear separability of your sets), use SOFT_MARGIN. The other parameters may be described as follows: -- \f$\lambda\f$ parameter is responsible for weights decreasing at each step and for the strength of restrictions on outliers +- marginRegularization parameter is responsible for weights decreasing at each step and for the strength of restrictions on outliers (the less the parameter, the less probability that an outlier will be ignored). Recommended value for SGD model is 0.0001, for ASGD model is 0.00001. -- \f$\gamma_0\f$ parameter is the initial value for the step size \f$\gamma(t)\f$. +- initialStepSize parameter is the initial value for the step size \f$\gamma(t)\f$. You will have to find the best \f$\gamma_0\f$ for your problem. -- \f$c\f$ is the power parameter for \f$\gamma(t)\f$ decreasing by the formula, mentioned above. +- stepDecreasingPower is the power parameter for \f$\gamma(t)\f$ decreasing by the formula, mentioned above. Recommended value for SGD model is 1, for ASGD model is 0.75. - Termination criteria can be TermCriteria::COUNT, TermCriteria::EPS or TermCriteria::COUNT + TermCriteria::EPS. You will have to find the best termination criteria for your problem. -Note that the parameters \f$\lambda\f$, \f$\gamma_0\f$, and \f$c\f$ should be positive. +Note that the parameters marginRegularization, initialStepSize, and stepDecreasingPower should be positive. To use SVMSGD algorithm do as follows: - first, create the SVMSGD object. -- then set parameters (model type, margin type, \f$\lambda\f$, \f$\gamma_0\f$, \f$c\f$) using the functions - setSvmsgdType(), setMarginType(), setLambda(), setGamma0(), and setC(), or the function setOptimalParameters(). +- then set parameters (model type, margin type, marginRegularization, initialStepSize, stepDecreasingPower) using the functions + setSvmsgdType(), setMarginType(), setMarginRegularization(), setInitialStepSize(), and setStepDecreasingPower(), or the function setOptimalParameters(). - then the SVM model can be trained using the train features and the correspondent labels by the method train(). @@ -1618,16 +1618,16 @@ public: /** @brief Function sets optimal parameters values for chosen SVM SGD model. * If chosen type is ASGD, function sets the following values for parameters of model: - * \f$\lambda = 0.00001\f$; - * \f$\gamma_0 = 0.05\f$; - * \f$c = 0.75\f$; + * marginRegularization = 0.00001; + * initialStepSize = 0.05; + * stepDecreasingPower = 0.75; * termCrit.maxCount = 100000; * termCrit.epsilon = 0.00001; * * If SGD: - * \f$\lambda = 0.0001\f$; - * \f$\gamma_0 = 0.05\f$; - * \f$c = 1\f$; + * marginRegularization = 0.0001; + * initialStepSize = 0.05; + * stepDecreasingPower = 1; * termCrit.maxCount = 100000; * termCrit.epsilon = 0.00001; * @param svmsgdType is the type of SVMSGD classifier. Legal values are SvmsgdType::SGD and SvmsgdType::ASGD. @@ -1650,23 +1650,23 @@ public: CV_WRAP virtual void setMarginType(int marginType) = 0; - /** @brief Parameter \f$\lambda\f$ of a %SVMSGD optimization problem. Default value is 0. */ - /** @see setLambda */ - CV_WRAP virtual float getLambda() const = 0; - /** @copybrief getLambda @see getLambda */ - CV_WRAP virtual void setLambda(float lambda) = 0; + /** @brief Parameter marginRegularization of a %SVMSGD optimization problem. Default value is 0. */ + /** @see setMarginRegularization */ + CV_WRAP virtual float getMarginRegularization() const = 0; + /** @copybrief getMarginRegularization @see getMarginRegularization */ + CV_WRAP virtual void setMarginRegularization(float marginRegularization) = 0; /** @brief Parameter \f$\gamma_0\f$ of a %SVMSGD optimization problem. Default value is 0. */ - /** @see setGamma0 */ - CV_WRAP virtual float getGamma0() const = 0; - /** @copybrief getGamma0 @see getGamma0 */ - CV_WRAP virtual void setGamma0(float gamma0) = 0; - - /** @brief Parameter \f$c\f$ of a %SVMSGD optimization problem. Default value is 0. */ - /** @see setC */ - CV_WRAP virtual float getC() const = 0; - /** @copybrief getC @see getC */ - CV_WRAP virtual void setC(float c) = 0; + /** @see setInitialStepSize */ + CV_WRAP virtual float getInitialStepSize() const = 0; + /** @copybrief getInitialStepSize @see getInitialStepSize */ + CV_WRAP virtual void setInitialStepSize(float InitialStepSize) = 0; + + /** @brief Parameter stepDecreasingPower of a %SVMSGD optimization problem. Default value is 0. */ + /** @see setStepDecreasingPower */ + CV_WRAP virtual float getStepDecreasingPower() const = 0; + /** @copybrief getStepDecreasingPower @see getStepDecreasingPower */ + CV_WRAP virtual void setStepDecreasingPower(float stepDecreasingPower) = 0; /** @brief Termination criteria of the training algorithm. You can specify the maximum number of iterations (maxCount) and/or how much the error could diff --git a/modules/ml/src/svmsgd.cpp b/modules/ml/src/svmsgd.cpp index 0f1efcfea6..17feeff17c 100644 --- a/modules/ml/src/svmsgd.cpp +++ b/modules/ml/src/svmsgd.cpp @@ -91,13 +91,13 @@ public: CV_IMPL_PROPERTY(int, SvmsgdType, params.svmsgdType) CV_IMPL_PROPERTY(int, MarginType, params.marginType) - CV_IMPL_PROPERTY(float, Lambda, params.lambda) - CV_IMPL_PROPERTY(float, Gamma0, params.gamma0) - CV_IMPL_PROPERTY(float, C, params.c) + CV_IMPL_PROPERTY(float, MarginRegularization, params.marginRegularization) + CV_IMPL_PROPERTY(float, InitialStepSize, params.initialStepSize) + CV_IMPL_PROPERTY(float, StepDecreasingPower, params.stepDecreasingPower) CV_IMPL_PROPERTY_S(cv::TermCriteria, TermCriteria, params.termCrit) private: - void updateWeights(InputArray sample, bool isPositive, float gamma, Mat &weights); + void updateWeights(InputArray sample, bool isPositive, float stepSize, Mat &weights); std::pair areClassesEmpty(Mat responses); @@ -122,9 +122,9 @@ private: // Parameters for learning struct SVMSGDParams { - float lambda; //regularization - float gamma0; //learning rate - float c; + float marginRegularization; + float initialStepSize; + float stepDecreasingPower; TermCriteria termCrit; int svmsgdType; int marginType; @@ -166,8 +166,7 @@ void SVMSGDImpl::normalizeSamples(Mat &samples, Mat &average, float &multiplier) average = Mat(1, featuresCount, samples.type()); for (int featureIndex = 0; featureIndex < featuresCount; featureIndex++) { - Scalar scalAverage = mean(samples.col(featureIndex)); - average.at(featureIndex) = static_cast(scalAverage[0]); + average.at(featureIndex) = static_cast(mean(samples.col(featureIndex))[0]); } for (int sampleIndex = 0; sampleIndex < samplesCount; sampleIndex++) @@ -193,7 +192,7 @@ void SVMSGDImpl::makeExtendedTrainSamples(const Mat &trainSamples, Mat &extended cv::hconcat(normalizedTrainSamples, onesCol, extendedTrainSamples); } -void SVMSGDImpl::updateWeights(InputArray _sample, bool firstClass, float gamma, Mat& weights) +void SVMSGDImpl::updateWeights(InputArray _sample, bool firstClass, float stepSize, Mat& weights) { Mat sample = _sample.getMat(); @@ -202,18 +201,18 @@ void SVMSGDImpl::updateWeights(InputArray _sample, bool firstClass, float gamma, if ( sample.dot(weights) * response > 1) { // Not a support vector, only apply weight decay - weights *= (1.f - gamma * params.lambda); + weights *= (1.f - stepSize * params.marginRegularization); } else { // It's a support vector, add it to the weights - weights -= (gamma * params.lambda) * weights - (gamma * response) * sample; + weights -= (stepSize * params.marginRegularization) * weights - (stepSize * response) * sample; } } float SVMSGDImpl::calcShift(InputArray _samples, InputArray _responses) const { - float distanceToClasses[2] = { std::numeric_limits::max(), std::numeric_limits::max() }; + float margin[2] = { std::numeric_limits::max(), std::numeric_limits::max() }; Mat trainSamples = _samples.getMat(); int trainSamplesCount = trainSamples.rows; @@ -225,18 +224,18 @@ float SVMSGDImpl::calcShift(InputArray _samples, InputArray _responses) const Mat currentSample = trainSamples.row(samplesIndex); float dotProduct = static_cast(currentSample.dot(weights_)); - bool firstClass = isPositive(trainResponses.at(samplesIndex)); - int index = firstClass ? 0 : 1; - float signToMul = firstClass ? 1.f : -1.f; - float curDistance = dotProduct * signToMul; + bool positive = isPositive(trainResponses.at(samplesIndex)); + int index = positive ? 0 : 1; + float signToMul = positive ? 1.f : -1.f; + float curMargin = dotProduct * signToMul; - if (curDistance < distanceToClasses[index]) + if (curMargin < margin[index]) { - distanceToClasses[index] = curDistance; + margin[index] = curMargin; } } - return -(distanceToClasses[0] - distanceToClasses[1]) / 2.f; + return -(margin[0] - margin[1]) / 2.f; } bool SVMSGDImpl::train(const Ptr& data, int) @@ -292,9 +291,9 @@ bool SVMSGDImpl::train(const Ptr& data, int) Mat currentSample = extendedTrainSamples.row(randomNumber); - float gamma = params.gamma0 * std::pow((1 + params.lambda * params.gamma0 * (float)iter), (-params.c)); //update gamma + float stepSize = params.initialStepSize * std::pow((1 + params.marginRegularization * params.initialStepSize * (float)iter), (-params.stepDecreasingPower)); //update stepSize - updateWeights( currentSample, isPositive(trainResponses.at(randomNumber)), gamma, extendedWeights ); + updateWeights( currentSample, isPositive(trainResponses.at(randomNumber)), stepSize, extendedWeights ); //average weights (only for ASGD model) if (params.svmsgdType == ASGD) @@ -370,7 +369,7 @@ bool SVMSGDImpl::isClassifier() const && (params.marginType == SOFT_MARGIN || params.marginType == HARD_MARGIN) && - (params.lambda > 0) && (params.gamma0 > 0) && (params.c >= 0); + (params.marginRegularization > 0) && (params.initialStepSize > 0) && (params.stepDecreasingPower >= 0); } bool SVMSGDImpl::isTrained() const @@ -423,9 +422,9 @@ void SVMSGDImpl::writeParams( FileStorage& fs ) const fs << "marginType" << marginTypeStr; - fs << "lambda" << params.lambda; - fs << "gamma0" << params.gamma0; - fs << "c" << params.c; + fs << "marginRegularization" << params.marginRegularization; + fs << "initialStepSize" << params.initialStepSize; + fs << "stepDecreasingPower" << params.stepDecreasingPower; fs << "term_criteria" << "{:"; if( params.termCrit.type & TermCriteria::EPS ) @@ -467,14 +466,14 @@ void SVMSGDImpl::readParams( const FileNode& fn ) params.marginType = marginType; - CV_Assert ( fn["lambda"].isReal() ); - params.lambda = (float)fn["lambda"]; + CV_Assert ( fn["marginRegularization"].isReal() ); + params.marginRegularization = (float)fn["marginRegularization"]; - CV_Assert ( fn["gamma0"].isReal() ); - params.gamma0 = (float)fn["gamma0"]; + CV_Assert ( fn["initialStepSize"].isReal() ); + params.initialStepSize = (float)fn["initialStepSize"]; - CV_Assert ( fn["c"].isReal() ); - params.c = (float)fn["c"]; + CV_Assert ( fn["stepDecreasingPower"].isReal() ); + params.stepDecreasingPower = (float)fn["stepDecreasingPower"]; FileNode tcnode = fn["term_criteria"]; if( !tcnode.empty() ) @@ -504,9 +503,9 @@ SVMSGDImpl::SVMSGDImpl() params.marginType = -1; // Parameters for learning - params.lambda = 0; // regularization - params.gamma0 = 0; // learning rate (ideally should be large at beginning and decay each iteration) - params.c = 0; + params.marginRegularization = 0; // regularization + params.initialStepSize = 0; // learning rate (ideally should be large at beginning and decay each iteration) + params.stepDecreasingPower = 0; TermCriteria _termCrit(TermCriteria::COUNT + TermCriteria::EPS, 0, 0); params.termCrit = _termCrit; @@ -520,9 +519,9 @@ void SVMSGDImpl::setOptimalParameters(int svmsgdType, int marginType) params.svmsgdType = SGD; params.marginType = (marginType == SOFT_MARGIN) ? SOFT_MARGIN : (marginType == HARD_MARGIN) ? HARD_MARGIN : -1; - params.lambda = 0.0001f; - params.gamma0 = 0.05f; - params.c = 1.f; + params.marginRegularization = 0.0001f; + params.initialStepSize = 0.05f; + params.stepDecreasingPower = 1.f; params.termCrit = TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 100000, 0.00001); break; @@ -530,9 +529,9 @@ void SVMSGDImpl::setOptimalParameters(int svmsgdType, int marginType) params.svmsgdType = ASGD; params.marginType = (marginType == SOFT_MARGIN) ? SOFT_MARGIN : (marginType == HARD_MARGIN) ? HARD_MARGIN : -1; - params.lambda = 0.00001f; - params.gamma0 = 0.05f; - params.c = 0.75f; + params.marginRegularization = 0.00001f; + params.initialStepSize = 0.05f; + params.stepDecreasingPower = 0.75f; params.termCrit = TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 100000, 0.00001); break; diff --git a/modules/ml/test/test_mltests2.cpp b/modules/ml/test/test_mltests2.cpp index 01f5568daf..15ae20096c 100644 --- a/modules/ml/test/test_mltests2.cpp +++ b/modules/ml/test/test_mltests2.cpp @@ -469,9 +469,9 @@ int CV_MLBaseTest::train( int testCaseIdx ) int marginType = str_to_margin_type( marginTypeStr ); m->setMarginType(marginType); - m->setLambda(modelParamsNode["lambda"]); - m->setGamma0(modelParamsNode["gamma0"]); - m->setC(modelParamsNode["c"]); + m->setMarginRegularization(modelParamsNode["marginRegularization"]); + m->setInitialStepSize(modelParamsNode["initialStepSize"]); + m->setStepDecreasingPower(modelParamsNode["stepDecreasingPower"]); m->setTermCriteria(TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 10000, 0.00001)); model = m; } diff --git a/modules/ml/test/test_svmsgd.cpp b/modules/ml/test/test_svmsgd.cpp index 00ebbf3341..31fd71729a 100644 --- a/modules/ml/test/test_svmsgd.cpp +++ b/modules/ml/test/test_svmsgd.cpp @@ -200,7 +200,6 @@ void CV_SVMSGDTrainTest::run( int /*start_from*/ ) } float err = (float)errCount / testSamplesCount; - std::cout << "err " << err << std::endl; if ( err > precision ) { diff --git a/samples/cpp/train_svmsgd.cpp b/samples/cpp/train_svmsgd.cpp index a68f613b2f..efcff01cb8 100644 --- a/samples/cpp/train_svmsgd.cpp +++ b/samples/cpp/train_svmsgd.cpp @@ -28,7 +28,7 @@ struct Data bool doTrain(const Mat samples, const Mat responses, Mat &weights, float &shift); //function finds two points for drawing line (wx = 0) -bool findPointsForLine(const Mat &weights, float shift, Point (&points)[2], int width, int height); +bool findPointsForLine(const Mat &weights, float shift, Point points[], int width, int height); // function finds cross point of line (wx = 0) and segment ( (y = HEIGHT, 0 <= x <= WIDTH) or (x = WIDTH, 0 <= y <= HEIGHT) ) bool findCrossPointWithBorders(const Mat &weights, float shift, const std::pair &segment, Point &crossPoint); @@ -119,7 +119,7 @@ bool findCrossPointWithBorders(const Mat &weights, float shift, const std::pair< return false; } -bool findPointsForLine(const Mat &weights, float shift, Point (&points)[2], int width, int height) +bool findPointsForLine(const Mat &weights, float shift, Point points[2], int width, int height) { if (weights.empty()) { From 9d9a5bbbfda3e5109f2550454fc18cb82c35a195 Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Wed, 24 Feb 2016 13:33:43 +0300 Subject: [PATCH 14/19] Fixed documentation. --- modules/ml/include/opencv2/ml.hpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index 7710abdae1..f9d7418d5e 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1511,9 +1511,9 @@ The gradient descent show amazing performance for large-scale problems, reducing The classifier has 5 parameters. These are - model type, - margin type, -- marginRegularization (\f$\lambda\f$), -- initialStepSize (\f$\gamma_0\f$), -- stepDecreasingPower (\f$c\f$), +- margin regularization (\f$\lambda\f$), +- initial step size (\f$\gamma_0\f$), +- step decreasing power (\f$c\f$), - and termination criteria. The model type may have one of the following values: \ref SGD and \ref ASGD. @@ -1538,26 +1538,26 @@ The margin type may have one of the following values: \ref SOFT_MARGIN or \ref H - In the general case (if you know nothing about linear separability of your sets), use SOFT_MARGIN. The other parameters may be described as follows: -- marginRegularization parameter is responsible for weights decreasing at each step and for the strength of restrictions on outliers +- Margin regularization parameter is responsible for weights decreasing at each step and for the strength of restrictions on outliers (the less the parameter, the less probability that an outlier will be ignored). Recommended value for SGD model is 0.0001, for ASGD model is 0.00001. -- initialStepSize parameter is the initial value for the step size \f$\gamma(t)\f$. +- Initial step size parameter is the initial value for the step size \f$\gamma(t)\f$. You will have to find the best \f$\gamma_0\f$ for your problem. -- stepDecreasingPower is the power parameter for \f$\gamma(t)\f$ decreasing by the formula, mentioned above. +- Step decreasing power is the power parameter for \f$\gamma(t)\f$ decreasing by the formula, mentioned above. Recommended value for SGD model is 1, for ASGD model is 0.75. - Termination criteria can be TermCriteria::COUNT, TermCriteria::EPS or TermCriteria::COUNT + TermCriteria::EPS. You will have to find the best termination criteria for your problem. -Note that the parameters marginRegularization, initialStepSize, and stepDecreasingPower should be positive. +Note that the parameters margin regularization, initial step size, and step decreasing power should be positive. To use SVMSGD algorithm do as follows: - first, create the SVMSGD object. -- then set parameters (model type, margin type, marginRegularization, initialStepSize, stepDecreasingPower) using the functions +- then set parameters (model type, margin type, margin regularization, initial step size, step decreasing power) using the functions setSvmsgdType(), setMarginType(), setMarginRegularization(), setInitialStepSize(), and setStepDecreasingPower(), or the function setOptimalParameters(). - then the SVM model can be trained using the train features and the correspondent labels by the method train(). @@ -1656,7 +1656,7 @@ public: /** @copybrief getMarginRegularization @see getMarginRegularization */ CV_WRAP virtual void setMarginRegularization(float marginRegularization) = 0; - /** @brief Parameter \f$\gamma_0\f$ of a %SVMSGD optimization problem. Default value is 0. */ + /** @brief Parameter initialStepSize of a %SVMSGD optimization problem. Default value is 0. */ /** @see setInitialStepSize */ CV_WRAP virtual float getInitialStepSize() const = 0; /** @copybrief getInitialStepSize @see getInitialStepSize */ From 068677ad50a72b911445ed7a2c7472b70906a882 Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Wed, 24 Feb 2016 14:41:51 +0300 Subject: [PATCH 15/19] Fixed documentation. --- modules/ml/include/opencv2/ml.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index f9d7418d5e..6343b344bc 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1506,7 +1506,6 @@ public: SVMSGD provides a fast and easy-to-use implementation of the SVM classifier using the Stochastic Gradient Descent approach, as presented in @cite bottou2010large. -The gradient descent show amazing performance for large-scale problems, reducing the computing time. The classifier has 5 parameters. These are - model type, From 74c87a26a561999ecdaf23102e8ed47519a8b313 Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Thu, 25 Feb 2016 15:31:07 +0300 Subject: [PATCH 16/19] Delete function areClassesEmpty(). --- modules/ml/include/opencv2/ml.hpp | 7 ++--- modules/ml/src/svmsgd.cpp | 48 +++++++------------------------ modules/ml/test/test_svmsgd.cpp | 2 -- samples/cpp/train_svmsgd.cpp | 1 - 4 files changed, 12 insertions(+), 46 deletions(-) diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index 6343b344bc..e596ead0ed 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1507,7 +1507,7 @@ public: SVMSGD provides a fast and easy-to-use implementation of the SVM classifier using the Stochastic Gradient Descent approach, as presented in @cite bottou2010large. -The classifier has 5 parameters. These are +The classifier has following parameters: - model type, - margin type, - margin regularization (\f$\lambda\f$), @@ -1567,11 +1567,8 @@ To use SVMSGD algorithm do as follows: // Create empty object cv::Ptr svmsgd = SVMSGD::create(); -// Set parameters -svmsgd->setOptimalParameters(); - // Train the Stochastic Gradient Descent SVM -SvmSgd->train(trainData); +svmsgd->train(trainData); // Predict labels for the new samples svmsgd->predict(samples, responses); diff --git a/modules/ml/src/svmsgd.cpp b/modules/ml/src/svmsgd.cpp index 17feeff17c..0a40481687 100644 --- a/modules/ml/src/svmsgd.cpp +++ b/modules/ml/src/svmsgd.cpp @@ -99,8 +99,6 @@ public: private: void updateWeights(InputArray sample, bool isPositive, float stepSize, Mat &weights); - std::pair areClassesEmpty(Mat responses); - void writeParams( FileStorage &fs ) const; void readParams( const FileNode &fn ); @@ -138,26 +136,6 @@ Ptr SVMSGD::create() return makePtr(); } -std::pair SVMSGDImpl::areClassesEmpty(Mat responses) -{ - CV_Assert(responses.cols == 1 || responses.rows == 1); - std::pair emptyInClasses(true, true); - int limitIndex = responses.rows; - - for(int index = 0; index < limitIndex; index++) - { - if (isPositive(responses.at(index))) - emptyInClasses.first = false; - else - emptyInClasses.second = false; - - if (!emptyInClasses.first && ! emptyInClasses.second) - break; - } - - return emptyInClasses; -} - void SVMSGDImpl::normalizeSamples(Mat &samples, Mat &average, float &multiplier) { int featuresCount = samples.cols; @@ -248,16 +226,20 @@ bool SVMSGDImpl::train(const Ptr& data, int) int featureCount = trainSamples.cols; Mat trainResponses = data->getTrainResponses(); // (trainSamplesCount x 1) matrix - std::pair areEmpty = areClassesEmpty(trainResponses); + CV_Assert(trainResponses.rows == trainSamples.rows); - if ( areEmpty.first && areEmpty.second ) + if (trainResponses.empty()) { return false; } - if ( areEmpty.first || areEmpty.second ) + + int positiveCount = countNonZero(trainResponses >= 0); + int negativeCount = countNonZero(trainResponses < 0); + + if ( positiveCount <= 0 || negativeCount <= 0 ) { weights_ = Mat::zeros(1, featureCount, CV_32F); - shift_ = areEmpty.first ? -1.f : 1.f; + shift_ = (positiveCount > 0) ? 1.f : -1.f; return true; } @@ -340,7 +322,7 @@ float SVMSGDImpl::predict( InputArray _samples, OutputArray _results, int ) cons int nSamples = samples.rows; cv::Mat results; - CV_Assert( samples.cols == weights_.cols && samples.type() == CV_32F ); + CV_Assert( samples.cols == weights_.cols && samples.type() == CV_32FC1); if( _results.needed() ) { @@ -498,17 +480,7 @@ void SVMSGDImpl::clear() SVMSGDImpl::SVMSGDImpl() { clear(); - - params.svmsgdType = -1; - params.marginType = -1; - - // Parameters for learning - params.marginRegularization = 0; // regularization - params.initialStepSize = 0; // learning rate (ideally should be large at beginning and decay each iteration) - params.stepDecreasingPower = 0; - - TermCriteria _termCrit(TermCriteria::COUNT + TermCriteria::EPS, 0, 0); - params.termCrit = _termCrit; + setOptimalParameters(); } void SVMSGDImpl::setOptimalParameters(int svmsgdType, int marginType) diff --git a/modules/ml/test/test_svmsgd.cpp b/modules/ml/test/test_svmsgd.cpp index 31fd71729a..9a206e47ed 100644 --- a/modules/ml/test/test_svmsgd.cpp +++ b/modules/ml/test/test_svmsgd.cpp @@ -182,8 +182,6 @@ void CV_SVMSGDTrainTest::run( int /*start_from*/ ) { cv::Ptr svmsgd = SVMSGD::create(); - svmsgd->setOptimalParameters(); - svmsgd->train(data); Mat responses; diff --git a/samples/cpp/train_svmsgd.cpp b/samples/cpp/train_svmsgd.cpp index efcff01cb8..580346a473 100644 --- a/samples/cpp/train_svmsgd.cpp +++ b/samples/cpp/train_svmsgd.cpp @@ -46,7 +46,6 @@ void addPointRetrainAndRedraw(Data &data, int x, int y, int response); bool doTrain( const Mat samples, const Mat responses, Mat &weights, float &shift) { cv::Ptr svmsgd = SVMSGD::create(); - svmsgd->setOptimalParameters(); cv::Ptr trainData = TrainData::create(samples, cv::ml::ROW_SAMPLE, responses); svmsgd->train( trainData ); From d484893839c568ceebd09fdc9e7ba15fb002dcd5 Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Thu, 25 Feb 2016 16:57:03 +0300 Subject: [PATCH 17/19] Deleted functions makeTrainData() and makeTestData() in test_svmsgd.cpp. Added function makeData() in test_svmsgd.cpp. --- modules/ml/include/opencv2/ml.hpp | 8 ++--- modules/ml/src/svmsgd.cpp | 12 ++++--- modules/ml/test/test_svmsgd.cpp | 57 +++++++++++-------------------- 3 files changed, 31 insertions(+), 46 deletions(-) diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index e596ead0ed..bcd22c46ba 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1626,10 +1626,10 @@ public: * stepDecreasingPower = 1; * termCrit.maxCount = 100000; * termCrit.epsilon = 0.00001; - * @param svmsgdType is the type of SVMSGD classifier. Legal values are SvmsgdType::SGD and SvmsgdType::ASGD. - * Recommended value is SvmsgdType::ASGD (by default). - * @param marginType is the type of margin constraint. Legal values are MarginType::SOFT_MARGIN and MarginType::HARD_MARGIN. - * Default value is MarginType::SOFT_MARGIN. + * @param svmsgdType is the type of SVMSGD classifier. Legal values are SVMSGD::SvmsgdType::SGD and SVMSGD::SvmsgdType::ASGD. + * Recommended value is SVMSGD::SvmsgdType::ASGD (by default). + * @param marginType is the type of margin constraint. Legal values are SVMSGD::MarginType::SOFT_MARGIN and SVMSGD::MarginType::HARD_MARGIN. + * Default value is SVMSGD::MarginType::SOFT_MARGIN. */ CV_WRAP virtual void setOptimalParameters(int svmsgdType = SVMSGD::ASGD, int marginType = SVMSGD::SOFT_MARGIN) = 0; diff --git a/modules/ml/src/svmsgd.cpp b/modules/ml/src/svmsgd.cpp index 0a40481687..9a03ed9ac6 100644 --- a/modules/ml/src/svmsgd.cpp +++ b/modules/ml/src/svmsgd.cpp @@ -142,6 +142,7 @@ void SVMSGDImpl::normalizeSamples(Mat &samples, Mat &average, float &multiplier) int samplesCount = samples.rows; average = Mat(1, featuresCount, samples.type()); + CV_Assert(average.type() == CV_32FC1); for (int featureIndex = 0; featureIndex < featuresCount; featureIndex++) { average.at(featureIndex) = static_cast(mean(samples.col(featureIndex))[0]); @@ -170,11 +171,11 @@ void SVMSGDImpl::makeExtendedTrainSamples(const Mat &trainSamples, Mat &extended cv::hconcat(normalizedTrainSamples, onesCol, extendedTrainSamples); } -void SVMSGDImpl::updateWeights(InputArray _sample, bool firstClass, float stepSize, Mat& weights) +void SVMSGDImpl::updateWeights(InputArray _sample, bool positive, float stepSize, Mat& weights) { Mat sample = _sample.getMat(); - int response = firstClass ? 1 : -1; // ensure that trainResponses are -1 or 1 + int response = positive ? 1 : -1; // ensure that trainResponses are -1 or 1 if ( sample.dot(weights) * response > 1) { @@ -197,6 +198,7 @@ float SVMSGDImpl::calcShift(InputArray _samples, InputArray _responses) const Mat trainResponses = _responses.getMat(); + CV_Assert(trainResponses.type() == CV_32FC1); for (int samplesIndex = 0; samplesIndex < trainSamplesCount; samplesIndex++) { Mat currentSample = trainSamples.row(samplesIndex); @@ -261,7 +263,7 @@ bool SVMSGDImpl::train(const Ptr& data, int) RNG rng(0); - CV_Assert (params.termCrit.type & TermCriteria::COUNT || params.termCrit.type & TermCriteria::EPS); + CV_Assert ((params.termCrit.type & TermCriteria::COUNT || params.termCrit.type & TermCriteria::EPS) && (trainResponses.type() == CV_32FC1)); int maxCount = (params.termCrit.type & TermCriteria::COUNT) ? params.termCrit.maxCount : INT_MAX; double epsilon = (params.termCrit.type & TermCriteria::EPS) ? params.termCrit.epsilon : 0; @@ -300,7 +302,7 @@ bool SVMSGDImpl::train(const Ptr& data, int) weights_ = extendedWeights(roi); weights_ *= multiplier; - CV_Assert(params.marginType == SOFT_MARGIN || params.marginType == HARD_MARGIN); + CV_Assert((params.marginType == SOFT_MARGIN || params.marginType == HARD_MARGIN) && (extendedWeights.type() == CV_32FC1)); if (params.marginType == SOFT_MARGIN) { @@ -332,7 +334,7 @@ float SVMSGDImpl::predict( InputArray _samples, OutputArray _results, int ) cons else { CV_Assert( nSamples == 1 ); - results = Mat(1, 1, CV_32F, &result); + results = Mat(1, 1, CV_32FC1, &result); } for (int sampleIndex = 0; sampleIndex < nSamples; sampleIndex++) diff --git a/modules/ml/test/test_svmsgd.cpp b/modules/ml/test/test_svmsgd.cpp index 9a206e47ed..b6aed3c7e5 100644 --- a/modules/ml/test/test_svmsgd.cpp +++ b/modules/ml/test/test_svmsgd.cpp @@ -62,8 +62,7 @@ public: private: virtual void run( int start_from ); static float decisionFunction(const Mat &sample, const Mat &weights, float shift); - void makeTrainData(Mat weights, float shift); - void makeTestData(Mat weights, float shift); + void makeData(int samplesCount, Mat weights, float shift, RNG rng, Mat &samples, Mat & responses); void generateSameBorders(int featureCount); void generateDifferentBorders(int featureCount); @@ -108,46 +107,28 @@ void CV_SVMSGDTrainTest::generateDifferentBorders(int featureCount) } } -void CV_SVMSGDTrainTest::makeTrainData(Mat weights, float shift) +float CV_SVMSGDTrainTest::decisionFunction(const Mat &sample, const Mat &weights, float shift) { - int datasize = 100000; - int featureCount = weights.cols; - RNG rng(0); - - cv::Mat samples = cv::Mat::zeros(datasize, featureCount, CV_32FC1); - - for (int featureIndex = 0; featureIndex < featureCount; featureIndex++) - { - rng.fill(samples.col(featureIndex), RNG::UNIFORM, borders[featureIndex].first, borders[featureIndex].second); - } - - cv::Mat responses = cv::Mat::zeros(datasize, 1, CV_32FC1); - for (int sampleIndex = 0; sampleIndex < datasize; sampleIndex++) - { - responses.at(sampleIndex) = decisionFunction(samples.row(sampleIndex), weights, shift) > 0 ? 1.f : -1.f; - } - - data = TrainData::create(samples, cv::ml::ROW_SAMPLE, responses); + return static_cast(sample.dot(weights)) + shift; } -void CV_SVMSGDTrainTest::makeTestData(Mat weights, float shift) +void CV_SVMSGDTrainTest::makeData(int samplesCount, Mat weights, float shift, RNG rng, Mat &samples, Mat & responses) { - int testSamplesCount = 100000; int featureCount = weights.cols; - cv::RNG rng(42); - testSamples.create(testSamplesCount, featureCount, CV_32FC1); + samples.create(samplesCount, featureCount, CV_32FC1); for (int featureIndex = 0; featureIndex < featureCount; featureIndex++) { - rng.fill(testSamples.col(featureIndex), RNG::UNIFORM, borders[featureIndex].first, borders[featureIndex].second); + rng.fill(samples.col(featureIndex), RNG::UNIFORM, borders[featureIndex].first, borders[featureIndex].second); } - testResponses.create(testSamplesCount, 1, CV_32FC1); + responses.create(samplesCount, 1, CV_32FC1); - for (int i = 0 ; i < testSamplesCount; i++) + for (int i = 0 ; i < samplesCount; i++) { - testResponses.at(i) = decisionFunction(testSamples.row(i), weights, shift) > 0 ? 1.f : -1.f; + responses.at(i) = decisionFunction(samples.row(i), weights, shift) > 0 ? 1.f : -1.f; } + } CV_SVMSGDTrainTest::CV_SVMSGDTrainTest(const Mat &weights, float shift, TrainDataType _type, double _precision) @@ -169,13 +150,16 @@ CV_SVMSGDTrainTest::CV_SVMSGDTrainTest(const Mat &weights, float shift, TrainDat CV_Error(CV_StsBadArg, "Unknown train data type"); } - makeTrainData(weights, shift); - makeTestData(weights, shift); -} + RNG rng(0); -float CV_SVMSGDTrainTest::decisionFunction(const Mat &sample, const Mat &weights, float shift) -{ - return static_cast(sample.dot(weights)) + shift; + Mat trainSamples; + Mat trainResponses; + int trainSamplesCount = 10000; + makeData(trainSamplesCount, weights, shift, rng, trainSamples, trainResponses); + data = TrainData::create(trainSamples, cv::ml::ROW_SAMPLE, trainResponses); + + int testSamplesCount = 100000; + makeData(testSamplesCount, weights, shift, rng, testSamples, testResponses); } void CV_SVMSGDTrainTest::run( int /*start_from*/ ) @@ -205,7 +189,6 @@ void CV_SVMSGDTrainTest::run( int /*start_from*/ ) } } - void makeWeightsAndShift(int featureCount, Mat &weights, float &shift) { weights.create(1, featureCount, CV_32FC1); @@ -253,7 +236,7 @@ TEST(ML_SVMSGD, trainSameScale100) float shift = 0; makeWeightsAndShift(featureCount, weights, shift); - CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_SAME_SCALE); + CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_SAME_SCALE, 0.02); test.safe_run(); } From 53711ec29d2715969a0fdbf54ff1f1a6825dda10 Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Thu, 25 Feb 2016 19:12:54 +0300 Subject: [PATCH 18/19] Deleted default value for parameters in docs. Added some asserts. --- modules/ml/include/opencv2/ml.hpp | 33 +++++-------------- modules/ml/src/svmsgd.cpp | 53 ++++++++++++++----------------- modules/ml/test/test_svmsgd.cpp | 5 +-- samples/cpp/train_svmsgd.cpp | 6 ++-- 4 files changed, 38 insertions(+), 59 deletions(-) diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index bcd22c46ba..666710bad1 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1542,7 +1542,7 @@ The other parameters may be described as follows: Recommended value for SGD model is 0.0001, for ASGD model is 0.00001. - Initial step size parameter is the initial value for the step size \f$\gamma(t)\f$. - You will have to find the best \f$\gamma_0\f$ for your problem. + You will have to find the best initial step for your problem. - Step decreasing power is the power parameter for \f$\gamma(t)\f$ decreasing by the formula, mentioned above. Recommended value for SGD model is 1, for ASGD model is 0.75. @@ -1605,31 +1605,15 @@ public: */ CV_WRAP virtual float getShift() = 0; - /** @brief Creates empty model. - Use StatModel::train to train the model. Since %SVMSGD has several parameters, you may want to - find the best parameters for your problem or use setOptimalParameters() to set some default parameters. + * Use StatModel::train to train the model. Since %SVMSGD has several parameters, you may want to + * find the best parameters for your problem or use setOptimalParameters() to set some default parameters. */ CV_WRAP static Ptr create(); /** @brief Function sets optimal parameters values for chosen SVM SGD model. - * If chosen type is ASGD, function sets the following values for parameters of model: - * marginRegularization = 0.00001; - * initialStepSize = 0.05; - * stepDecreasingPower = 0.75; - * termCrit.maxCount = 100000; - * termCrit.epsilon = 0.00001; - * - * If SGD: - * marginRegularization = 0.0001; - * initialStepSize = 0.05; - * stepDecreasingPower = 1; - * termCrit.maxCount = 100000; - * termCrit.epsilon = 0.00001; - * @param svmsgdType is the type of SVMSGD classifier. Legal values are SVMSGD::SvmsgdType::SGD and SVMSGD::SvmsgdType::ASGD. - * Recommended value is SVMSGD::SvmsgdType::ASGD (by default). - * @param marginType is the type of margin constraint. Legal values are SVMSGD::MarginType::SOFT_MARGIN and SVMSGD::MarginType::HARD_MARGIN. - * Default value is SVMSGD::MarginType::SOFT_MARGIN. + * @param svmsgdType is the type of SVMSGD classifier. + * @param marginType is the type of margin constraint. */ CV_WRAP virtual void setOptimalParameters(int svmsgdType = SVMSGD::ASGD, int marginType = SVMSGD::SOFT_MARGIN) = 0; @@ -1645,20 +1629,19 @@ public: /** @copybrief getMarginType @see getMarginType */ CV_WRAP virtual void setMarginType(int marginType) = 0; - - /** @brief Parameter marginRegularization of a %SVMSGD optimization problem. Default value is 0. */ + /** @brief Parameter marginRegularization of a %SVMSGD optimization problem. */ /** @see setMarginRegularization */ CV_WRAP virtual float getMarginRegularization() const = 0; /** @copybrief getMarginRegularization @see getMarginRegularization */ CV_WRAP virtual void setMarginRegularization(float marginRegularization) = 0; - /** @brief Parameter initialStepSize of a %SVMSGD optimization problem. Default value is 0. */ + /** @brief Parameter initialStepSize of a %SVMSGD optimization problem. */ /** @see setInitialStepSize */ CV_WRAP virtual float getInitialStepSize() const = 0; /** @copybrief getInitialStepSize @see getInitialStepSize */ CV_WRAP virtual void setInitialStepSize(float InitialStepSize) = 0; - /** @brief Parameter stepDecreasingPower of a %SVMSGD optimization problem. Default value is 0. */ + /** @brief Parameter stepDecreasingPower of a %SVMSGD optimization problem. */ /** @see setStepDecreasingPower */ CV_WRAP virtual float getStepDecreasingPower() const = 0; /** @copybrief getStepDecreasingPower @see getStepDecreasingPower */ diff --git a/modules/ml/src/svmsgd.cpp b/modules/ml/src/svmsgd.cpp index 9a03ed9ac6..ed452e35ef 100644 --- a/modules/ml/src/svmsgd.cpp +++ b/modules/ml/src/svmsgd.cpp @@ -97,7 +97,7 @@ public: CV_IMPL_PROPERTY_S(cv::TermCriteria, TermCriteria, params.termCrit) private: - void updateWeights(InputArray sample, bool isPositive, float stepSize, Mat &weights); + void updateWeights(InputArray sample, bool positive, float stepSize, Mat &weights); void writeParams( FileStorage &fs ) const; @@ -111,8 +111,6 @@ private: static void makeExtendedTrainSamples(const Mat &trainSamples, Mat &extendedTrainSamples, Mat &average, float &multiplier); - - // Vector with SVM weights Mat weights_; float shift_; @@ -263,11 +261,12 @@ bool SVMSGDImpl::train(const Ptr& data, int) RNG rng(0); - CV_Assert ((params.termCrit.type & TermCriteria::COUNT || params.termCrit.type & TermCriteria::EPS) && (trainResponses.type() == CV_32FC1)); + CV_Assert (params.termCrit.type & TermCriteria::COUNT || params.termCrit.type & TermCriteria::EPS); int maxCount = (params.termCrit.type & TermCriteria::COUNT) ? params.termCrit.maxCount : INT_MAX; double epsilon = (params.termCrit.type & TermCriteria::EPS) ? params.termCrit.epsilon : 0; double err = DBL_MAX; + CV_Assert (trainResponses.type() == CV_32FC1); // Stochastic gradient descent SVM for (int iter = 0; (iter < maxCount) && (err > epsilon); iter++) { @@ -288,8 +287,8 @@ bool SVMSGDImpl::train(const Ptr& data, int) } else { - err = norm(extendedWeights - previousWeights); - extendedWeights.copyTo(previousWeights); + err = norm(extendedWeights - previousWeights); + extendedWeights.copyTo(previousWeights); } } @@ -316,7 +315,6 @@ bool SVMSGDImpl::train(const Ptr& data, int) return true; } - float SVMSGDImpl::predict( InputArray _samples, OutputArray _results, int ) const { float result = 0; @@ -417,17 +415,6 @@ void SVMSGDImpl::writeParams( FileStorage& fs ) const fs << "iterations" << params.termCrit.maxCount; fs << "}"; } - -void SVMSGDImpl::read(const FileNode& fn) -{ - clear(); - - readParams(fn); - - fn["weights"] >> weights_; - fn["shift"] >> shift_; -} - void SVMSGDImpl::readParams( const FileNode& fn ) { String svmsgdTypeStr = (String)fn["svmsgdType"]; @@ -443,7 +430,7 @@ void SVMSGDImpl::readParams( const FileNode& fn ) String marginTypeStr = (String)fn["marginType"]; int marginType = marginTypeStr == "SOFT_MARGIN" ? SOFT_MARGIN : - marginTypeStr == "HARD_MARGIN" ? HARD_MARGIN : -1; + marginTypeStr == "HARD_MARGIN" ? HARD_MARGIN : -1; if( marginType < 0 ) CV_Error( CV_StsParseError, "Missing or invalid margin type" ); @@ -460,16 +447,22 @@ void SVMSGDImpl::readParams( const FileNode& fn ) params.stepDecreasingPower = (float)fn["stepDecreasingPower"]; FileNode tcnode = fn["term_criteria"]; - if( !tcnode.empty() ) - { - params.termCrit.epsilon = (double)tcnode["epsilon"]; - params.termCrit.maxCount = (int)tcnode["iterations"]; - params.termCrit.type = (params.termCrit.epsilon > 0 ? TermCriteria::EPS : 0) + - (params.termCrit.maxCount > 0 ? TermCriteria::COUNT : 0); - } - else - params.termCrit = TermCriteria( TermCriteria::EPS + TermCriteria::COUNT, 100000, FLT_EPSILON ); + CV_Assert(!tcnode.empty()); + params.termCrit.epsilon = (double)tcnode["epsilon"]; + params.termCrit.maxCount = (int)tcnode["iterations"]; + params.termCrit.type = (params.termCrit.epsilon > 0 ? TermCriteria::EPS : 0) + + (params.termCrit.maxCount > 0 ? TermCriteria::COUNT : 0); + CV_Assert ((params.termCrit.type & TermCriteria::COUNT || params.termCrit.type & TermCriteria::EPS)); +} +void SVMSGDImpl::read(const FileNode& fn) +{ + clear(); + + readParams(fn); + + fn["weights"] >> weights_; + fn["shift"] >> shift_; } void SVMSGDImpl::clear() @@ -492,7 +485,7 @@ void SVMSGDImpl::setOptimalParameters(int svmsgdType, int marginType) case SGD: params.svmsgdType = SGD; params.marginType = (marginType == SOFT_MARGIN) ? SOFT_MARGIN : - (marginType == HARD_MARGIN) ? HARD_MARGIN : -1; + (marginType == HARD_MARGIN) ? HARD_MARGIN : -1; params.marginRegularization = 0.0001f; params.initialStepSize = 0.05f; params.stepDecreasingPower = 1.f; @@ -502,7 +495,7 @@ void SVMSGDImpl::setOptimalParameters(int svmsgdType, int marginType) case ASGD: params.svmsgdType = ASGD; params.marginType = (marginType == SOFT_MARGIN) ? SOFT_MARGIN : - (marginType == HARD_MARGIN) ? HARD_MARGIN : -1; + (marginType == HARD_MARGIN) ? HARD_MARGIN : -1; params.marginRegularization = 0.00001f; params.initialStepSize = 0.05f; params.stepDecreasingPower = 0.75f; diff --git a/modules/ml/test/test_svmsgd.cpp b/modules/ml/test/test_svmsgd.cpp index b6aed3c7e5..8d9103e0fb 100644 --- a/modules/ml/test/test_svmsgd.cpp +++ b/modules/ml/test/test_svmsgd.cpp @@ -62,7 +62,7 @@ public: private: virtual void run( int start_from ); static float decisionFunction(const Mat &sample, const Mat &weights, float shift); - void makeData(int samplesCount, Mat weights, float shift, RNG rng, Mat &samples, Mat & responses); + void makeData(int samplesCount, const Mat &weights, float shift, RNG &rng, Mat &samples, Mat & responses); void generateSameBorders(int featureCount); void generateDifferentBorders(int featureCount); @@ -112,7 +112,7 @@ float CV_SVMSGDTrainTest::decisionFunction(const Mat &sample, const Mat &weights return static_cast(sample.dot(weights)) + shift; } -void CV_SVMSGDTrainTest::makeData(int samplesCount, Mat weights, float shift, RNG rng, Mat &samples, Mat & responses) +void CV_SVMSGDTrainTest::makeData(int samplesCount, const Mat &weights, float shift, RNG &rng, Mat &samples, Mat & responses) { int featureCount = weights.cols; @@ -175,6 +175,7 @@ void CV_SVMSGDTrainTest::run( int /*start_from*/ ) int errCount = 0; int testSamplesCount = testSamples.rows; + CV_Assert((responses.type() == CV_32FC1) && (testResponses.type() == CV_32FC1)); for (int i = 0; i < testSamplesCount; i++) { if (responses.at(i) * testResponses.at(i) < 0) diff --git a/samples/cpp/train_svmsgd.cpp b/samples/cpp/train_svmsgd.cpp index 580346a473..e2942d01ea 100644 --- a/samples/cpp/train_svmsgd.cpp +++ b/samples/cpp/train_svmsgd.cpp @@ -91,6 +91,7 @@ bool findCrossPointWithBorders(const Mat &weights, float shift, const std::pair< int yMin = std::min(segment.first.y, segment.second.y); int yMax = std::max(segment.first.y, segment.second.y); + CV_Assert(weights.type() == CV_32FC1); CV_Assert(xMin == xMax || yMin == yMax); if (xMin == xMax && weights.at(1) != 0) @@ -146,6 +147,7 @@ void redraw(Data data, const Point points[2]) Point center; int radius = 3; Scalar color; + CV_Assert((data.samples.type() == CV_32FC1) && (data.responses.type() == CV_32FC1)); for (int i = 0; i < data.samples.rows; i++) { center.x = static_cast(data.samples.at(i,0)); @@ -160,14 +162,14 @@ void redraw(Data data, const Point points[2]) void addPointRetrainAndRedraw(Data &data, int x, int y, int response) { - Mat currentSample(1, 2, CV_32F); + Mat currentSample(1, 2, CV_32FC1); currentSample.at(0,0) = (float)x; currentSample.at(0,1) = (float)y; data.samples.push_back(currentSample); data.responses.push_back(response); - Mat weights(1, 2, CV_32F); + Mat weights(1, 2, CV_32FC1); float shift = 0; if (doTrain(data.samples, data.responses, weights, shift)) From 3f0a51bda153567b24b13132236e3d3497a6b0b4 Mon Sep 17 00:00:00 2001 From: Marina Noskova Date: Fri, 26 Feb 2016 14:40:23 +0300 Subject: [PATCH 19/19] Fixed documentation. --- modules/ml/include/opencv2/ml.hpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index 666710bad1..29e4685d15 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1554,10 +1554,8 @@ Note that the parameters margin regularization, initial step size, and step decr To use SVMSGD algorithm do as follows: -- first, create the SVMSGD object. - -- then set parameters (model type, margin type, margin regularization, initial step size, step decreasing power) using the functions - setSvmsgdType(), setMarginType(), setMarginRegularization(), setInitialStepSize(), and setStepDecreasingPower(), or the function setOptimalParameters(). +- first, create the SVMSGD object. The algoorithm will set optimal parameters by default, but you can set your own parameters via functions setSvmsgdType(), + setMarginType(), setMarginRegularization(), setInitialStepSize(), and setStepDecreasingPower(). - then the SVM model can be trained using the train features and the correspondent labels by the method train().