SVMSGD class added

doc/opencv.bib

@@ -415,16 +415,6 @@
   pages = {2548--2555},
   organization = {IEEE}
 }
-@ARTICLE{Louhichi07,
-  author = {Louhichi, H. and Fournel, T. and Lavest, J. M. and Ben Aissia, H.},
-  title = {Self-calibration of Scheimpflug cameras: an easy protocol},
-  year = {2007},
-  pages = {2616–2622},
-  journal = {Meas. Sci. Technol.},
-  volume = {18},
-  number = {8},
-  publisher = {IOP Publishing Ltd}
-}
 @ARTICLE{LibSVM,
   author = {Chang, Chih-Chung and Lin, Chih-Jen},
   title = {LIBSVM: a library for support vector machines},
@@ -874,3 +864,11 @@
   year={2007},
   organization={IEEE}
 }
+@incollection{bottou2010large,
+  title={Large-scale machine learning with stochastic gradient descent},
+  author={Bottou, L{\'e}on},
+  booktitle={Proceedings of COMPSTAT'2010},
+  pages={177--186},
+  year={2010},
+  publisher={Springer}
+}

modules/ml/include/opencv2/ml.hpp

@@ -1513,6 +1513,127 @@ CV_EXPORTS void randMVNormal( InputArray mean, InputArray cov, int nsamples, Out
 CV_EXPORTS void createConcentricSpheresTestSet( int nsamples, int nfeatures, int nclasses,
                                                 OutputArray samples, OutputArray responses);
 
+/****************************************************************************************\
+*                        Stochastic Gradient Descent SVM Classifier                      *
+\****************************************************************************************/
+
+/*!
+@brief Stochastic Gradient Descent SVM classifier
+
+SVMSGD provides a fast and easy-to-use implementation of the SVM classifier using the Stochastic Gradient Descent approach, as presented in @cite bottou2010large.
+The gradient descent show amazing performance for large-scale problems, reducing the computing time. This allows a fast and reliable online update of the classifier for each new feature which
+is fundamental when dealing with variations of data over time (like weather and illumination changes in videosurveillance, for example).
+
+First, create the SVMSGD object. To enable the online update, a value for updateFrequency should be defined.
+
+Then the SVM model can be trained using the train features and the correspondent labels.
+
+After that, the label of a new feature vector can be predicted using the predict function. If the updateFrequency was defined in the constructor, the predict function will update the weights automatically.
+
+@code
+// Initialize object
+SVMSGD SvmSgd;
+
+// Train the Stochastic Gradient Descent SVM
+SvmSgd.train(trainFeatures, labels);
+
+// Predict label for the new feature vector (1xM)
+predictedLabel = SvmSgd.predict(newFeatureVector);
+@endcode
+
+*/
+class CV_EXPORTS_W SVMSGD {
+
+    public:
+        /** @brief SGDSVM constructor.
+
+        @param lambda regularization
+        @param learnRate learning rate
+        @param nIterations number of training iterations
+
+        */
+        SVMSGD(float lambda = 0.000001, float learnRate = 2, uint nIterations = 100000);
+
+        /** @brief SGDSVM constructor.
+
+        @param updateFrequency online update frequency
+        @param learnRateDecay learn rate decay over time: learnRate = learnRate * learnDecay
+        @param lambda regularization
+        @param learnRate learning rate
+        @param nIterations number of training iterations
+
+        */
+        SVMSGD(uint updateFrequency, float learnRateDecay = 1, float lambda = 0.000001, float learnRate = 2, uint nIterations = 100000);
+        virtual ~SVMSGD();
+        virtual SVMSGD* clone() const;
+
+        /** @brief Train the SGDSVM classifier.
+
+        The function trains the SGDSVM classifier using the train features and the correspondent labels (-1 or 1).
+
+        @param trainFeatures features used for training. Each row is a new sample.
+        @param labels mat (size Nx1 with N = number of features) with the label of each training feature.
+
+        */
+        virtual void train(cv::Mat trainFeatures, cv::Mat labels);
+
+        /** @brief Predict the label of a new feature vector.
+
+        The function predicts and returns the label of a new feature vector, using the previously trained SVM model.
+
+        @param newFeature new feature vector used for prediction
+
+        */
+        virtual float predict(cv::Mat newFeature);
+
+        /** @brief Returns the weights of the trained model.
+
+        */
+        virtual std::vector<float> getWeights(){ return _weights; };
+
+        /** @brief Sets the weights of the trained model.
+
+        @param weights weights used to predict the label of a new feature vector.
+
+        */
+        virtual void setWeights(std::vector<float> weights){ _weights = weights; };
+
+    private:
+        void updateWeights();
+        void generateRandomIndex();
+        float calcInnerProduct(float *rowDataPointer);
+        void updateWeights(float innerProduct, float *rowDataPointer, int label);
+
+        // Vector with SVM weights
+        std::vector<float> _weights;
+
+        // Random index generation
+        long long int _randomNumber;
+        unsigned int _randomIndex;
+
+        // Number of features and samples
+        unsigned int _nFeatures;
+        unsigned int _nTrainSamples;
+
+        // Parameters for learning
+        float _lambda;  //regularization
+        float _learnRate;  //learning rate
+        unsigned int _nIterations; //number of training iterations
+
+        // Vars to control the features slider matrix
+        bool _onlineUpdate;
+        bool _initPredict;
+        uint _slidingWindowSize;
+        uint _predictSlidingWindowSize;
+        float* _labelSlider;
+        float _learnRateDecay;
+
+        // Mat with features slider and correspondent counter
+        unsigned int _sliderCounter;
+        cv::Mat _featuresSlider;
+
+};
+
 //! @} ml
 
 }

modules/ml/src/svmsgd.cpp

@@ -0,0 +1,201 @@
+/*M///////////////////////////////////////////////////////////////////////////////////////
+//
+//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+//
+//  By downloading, copying, installing or using the software you agree to this license.
+//  If you do not agree to this license, do not download, install,
+//  copy or use the software.
+//
+//
+//                           License Agreement
+//                For Open Source Computer Vision Library
+//
+// Copyright (C) 2000, Intel Corporation, all rights reserved.
+// Copyright (C) 2014, Itseez Inc, all rights reserved.
+// Third party copyrights are property of their respective owners.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+//   * Redistribution's of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//
+//   * Redistribution's in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//
+//   * The name of the copyright holders may not be used to endorse or promote products
+//     derived from this software without specific prior written permission.
+//
+// This software is provided by the copyright holders and contributors "as is" and
+// any express or implied warranties, including, but not limited to, the implied
+// warranties of merchantability and fitness for a particular purpose are disclaimed.
+// In no event shall the Intel Corporation or contributors be liable for any direct,
+// indirect, incidental, special, exemplary, or consequential damages
+// (including, but not limited to, procurement of substitute goods or services;
+// loss of use, data, or profits; or business interruption) however caused
+// and on any theory of liability, whether in contract, strict liability,
+// or tort (including negligence or otherwise) arising in any way out of
+// the use of this software, even if advised of the possibility of such damage.
+//
+//M*/
+
+#include "precomp.hpp"
+
+/****************************************************************************************\
+*                        Stochastic Gradient Descent SVM Classifier                      *
+\****************************************************************************************/
+
+namespace cv {
+namespace ml {
+
+SVMSGD::SVMSGD(float lambda, float learnRate, uint nIterations){
+
+    // Initialize with random seed
+    _randomNumber = 1;
+
+    // Initialize constants
+    _slidingWindowSize = 0;
+    _nFeatures = 0;
+    _predictSlidingWindowSize = 1;
+
+    // Initialize sliderCounter at index 0
+    _sliderCounter = 0;
+
+    // Parameters for learning
+    _lambda = lambda;  // regularization
+    _learnRate = learnRate;  // learning rate (ideally should be large at beginning and decay each iteration)
+    _nIterations = nIterations;  // number of training iterations
+
+    // True only in the first predict iteration
+    _initPredict = true;
+
+    // Online update flag
+    _onlineUpdate = false;
+}
+
+SVMSGD::SVMSGD(uint updateFrequency, float learnRateDecay, float lambda, float learnRate, uint nIterations){
+
+    // Initialize with random seed
+    _randomNumber = 1;
+
+    // Initialize constants
+    _slidingWindowSize = 0;
+    _nFeatures = 0;
+    _predictSlidingWindowSize = updateFrequency;
+
+    // Initialize sliderCounter at index 0
+    _sliderCounter = 0;
+
+    // Parameters for learning
+    _lambda = lambda;  // regularization
+    _learnRate = learnRate;  // learning rate (ideally should be large at beginning and decay each iteration)
+    _nIterations = nIterations;  // number of training iterations
+
+    // True only in the first predict iteration
+    _initPredict = true;
+
+    // Online update flag
+    _onlineUpdate = true;
+
+    // Learn rate decay: _learnRate = _learnRate * _learnDecay
+    _learnRateDecay = learnRateDecay;
+}
+
+SVMSGD::~SVMSGD(){
+
+}
+
+SVMSGD* SVMSGD::clone() const{
+    return new SVMSGD(*this);
+}
+
+void SVMSGD::train(cv::Mat trainFeatures, cv::Mat labels){
+
+    // Initialize _nFeatures
+    _slidingWindowSize = trainFeatures.rows;
+    _nFeatures = trainFeatures.cols;
+
+    float innerProduct;
+    // Initialize weights vector with zeros
+    if (_weights.size()==0){
+        _weights.reserve(_nFeatures);
+        for (uint feat = 0; feat < _nFeatures; ++feat){
+            _weights.push_back(0.0);
+        }
+    }
+
+    // Stochastic gradient descent SVM
+    for (uint iter = 0; iter < _nIterations; ++iter){
+        generateRandomIndex();
+        innerProduct = calcInnerProduct(trainFeatures.ptr<float>(_randomIndex));
+        int label = (labels.at<int>(_randomIndex,0) > 0) ? 1 : -1; // ensure that labels are -1 or 1
+        updateWeights(innerProduct, trainFeatures.ptr<float>(_randomIndex), label );
+    }
+}
+
+float SVMSGD::predict(cv::Mat newFeature){
+    float innerProduct;
+
+    if (_initPredict){
+        _nFeatures = newFeature.cols;
+        _slidingWindowSize = _predictSlidingWindowSize;
+        _featuresSlider = cv::Mat::zeros(_slidingWindowSize, _nFeatures, CV_32F);
+        _initPredict = false;
+        _labelSlider = new float[_predictSlidingWindowSize]();
+        _learnRate = _learnRate * _learnRateDecay;
+    }
+
+    innerProduct = calcInnerProduct(newFeature.ptr<float>(0));
+
+    // Resultant label (-1 or 1)
+    int label = (innerProduct>=0) ? 1 : -1;
+
+    if (_onlineUpdate){
+        // Update the featuresSlider with newFeature and _labelSlider with label
+        newFeature.row(0).copyTo(_featuresSlider.row(_sliderCounter));
+        _labelSlider[_sliderCounter] = float(label);
+
+        // Update weights with a random index
+        if (_sliderCounter == _slidingWindowSize-1){
+            generateRandomIndex();
+            updateWeights(innerProduct, _featuresSlider.ptr<float>(_randomIndex), int(_labelSlider[_randomIndex]) );
+        }
+
+        // _sliderCounter++ if < _slidingWindowSize
+        _sliderCounter = (_sliderCounter == _slidingWindowSize-1) ? 0 : (_sliderCounter+1);
+    }
+
+    return float(label);
+}
+
+void SVMSGD::generateRandomIndex(){
+    // Choose random sample, using Mikolov's fast almost-uniform random number
+    _randomNumber = _randomNumber * (unsigned long long) 25214903917 + 11;
+    _randomIndex = uint(_randomNumber % (unsigned long long) _slidingWindowSize);
+}
+
+float SVMSGD::calcInnerProduct(float *rowDataPointer){
+    float innerProduct = 0;
+    for (uint feat = 0; feat < _nFeatures; ++feat){
+        innerProduct += _weights[feat] * rowDataPointer[feat];
+    }
+    return innerProduct;
+}
+
+void SVMSGD::updateWeights(float innerProduct, float *rowDataPointer, int label){
+    if (label * innerProduct > 1) {
+        // Not a support vector, only apply weight decay
+        for (uint feat = 0; feat < _nFeatures; feat++) {
+            _weights[feat] -= _learnRate * _lambda * _weights[feat];
+        }
+    } else {
+        // It's a support vector, add it to the weights
+        for (uint feat = 0; feat < _nFeatures; feat++) {
+            _weights[feat] -= _learnRate * (_lambda * _weights[feat] - label * rowDataPointer[feat]);
+        }
+    }
+}
+
+}
+}