From 51e96d89d797e90ef4b90e0a89acb3e7aa05561d Mon Sep 17 00:00:00 2001 From: berak Date: Mon, 26 Mar 2018 13:39:14 +0200 Subject: [PATCH] face: refactor landmarks interfaces --- modules/face/include/opencv2/face.hpp | 1 + .../include/opencv2/face/face_alignment.hpp | 17 +- .../face/include/opencv2/face/facemark.hpp | 376 +---------------- .../face/include/opencv2/face/facemarkAAM.hpp | 12 +- .../face/include/opencv2/face/facemarkLBF.hpp | 8 +- .../include/opencv2/face/facemark_train.hpp | 392 ++++++++++++++++++ modules/face/misc/java/gen_dict.json | 16 + modules/face/samples/Facemark.java | 48 +++ modules/face/samples/facemark_demo_aam.cpp | 2 +- modules/face/samples/facemark_demo_lbf.cpp | 2 +- modules/face/samples/facemark_lbf_fitting.cpp | 2 +- .../sample_train_landmark_detector2.cpp | 4 +- modules/face/src/face_alignment.cpp | 7 +- modules/face/src/face_alignmentimpl.hpp | 9 +- modules/face/src/facemark.cpp | 2 +- modules/face/src/facemarkAAM.cpp | 30 +- modules/face/src/facemarkLBF.cpp | 13 +- modules/face/src/getlandmarks.cpp | 4 +- modules/face/test/test_facemark_aam.cpp | 6 +- modules/face/test/test_facemark_lbf.cpp | 6 +- .../face_landmark_detection.markdown | 90 ++-- 21 files changed, 590 insertions(+), 457 deletions(-) create mode 100644 modules/face/include/opencv2/face/facemark_train.hpp create mode 100644 modules/face/misc/java/gen_dict.json create mode 100644 modules/face/samples/Facemark.java diff --git a/modules/face/include/opencv2/face.hpp b/modules/face/include/opencv2/face.hpp index 87cf4b34e..890f5b6a0 100644 --- a/modules/face/include/opencv2/face.hpp +++ b/modules/face/include/opencv2/face.hpp @@ -375,6 +375,7 @@ protected: #include "opencv2/face/facerec.hpp" #include "opencv2/face/facemark.hpp" +#include "opencv2/face/facemark_train.hpp" #include "opencv2/face/facemarkLBF.hpp" #include "opencv2/face/facemarkAAM.hpp" #include "opencv2/face/face_alignment.hpp" diff --git a/modules/face/include/opencv2/face/face_alignment.hpp b/modules/face/include/opencv2/face/face_alignment.hpp index 0cad017bc..612bce12c 100644 --- a/modules/face/include/opencv2/face/face_alignment.hpp +++ b/modules/face/include/opencv2/face/face_alignment.hpp @@ -4,11 +4,11 @@ #ifndef __OPENCV_FACE_ALIGNMENT_HPP__ #define __OPENCV_FACE_ALIGNMENT_HPP__ -#include "facemark.hpp" +#include "opencv2/face/facemark_train.hpp" namespace cv{ namespace face{ -class CV_EXPORTS_W FacemarkKazemi : public Algorithm +class CV_EXPORTS_W FacemarkKazemi : public Facemark { public: struct CV_EXPORTS Params @@ -39,8 +39,6 @@ public: static Ptr create(const FacemarkKazemi::Params ¶meters = FacemarkKazemi::Params()); virtual ~FacemarkKazemi(); - /// @brief training the facemark model, input are the file names of image list and landmark annotation - virtual void training(String imageList, String groundTruth)=0; /** @brief This function is used to train the model using gradient boosting to get a cascade of regressors *which can then be used to predict shape. *@param images A vector of type cv::Mat which stores the images which are used in training samples. @@ -51,16 +49,7 @@ public: *@returns A boolean value. The function returns true if the model is trained properly or false if it is not trained. */ virtual bool training(std::vector& images, std::vector< std::vector >& landmarks,std::string configfile,Size scale,std::string modelFilename = "face_landmarks.dat")=0; - /** @brief This function is used to load the trained model.. - *@param filename A variable of type cv::String which stores the name of the file in which trained model is stored. - */ - virtual void loadModel(String filename)=0; - /** @brief This functions retrieves a centered and scaled face shape, according to the bounding rectangle. - *@param image A variable of type cv::InputArray which stores the image whose landmarks have to be found - *@param faces A variable of type cv::InputArray which stores the bounding boxes of faces found in a given image. - *@param landmarks A variable of type cv::InputOutputArray which stores the landmarks of all the faces found in the image - */ - virtual bool fit( InputArray image, InputArray faces, InputOutputArray landmarks )=0;//!< from many ROIs + /// set the custom face detector virtual bool setFaceDetector(bool(*f)(InputArray , OutputArray, void*), void* userData)=0; /// get faces using the custom detector diff --git a/modules/face/include/opencv2/face/facemark.hpp b/modules/face/include/opencv2/face/facemark.hpp index 285f2a598..70849bc4b 100644 --- a/modules/face/include/opencv2/face/facemark.hpp +++ b/modules/face/include/opencv2/face/facemark.hpp @@ -18,323 +18,37 @@ Mentor: Delia Passalacqua - The Facemark API */ -#include "opencv2/face.hpp" -#include "opencv2/objdetect.hpp" +#include "opencv2/core.hpp" #include -#include namespace cv { namespace face { -//! @addtogroup face -//! @{ - -typedef bool(*FN_FaceDetector)(InputArray, OutputArray, void* userData); - -struct CParams{ - String cascade; //!< the face detector - double scaleFactor; //!< Parameter specifying how much the image size is reduced at each image scale. - int minNeighbors; //!< Parameter specifying how many neighbors each candidate rectangle should have to retain it. - Size minSize; //!< Minimum possible object size. - Size maxSize; //!< Maximum possible object size. - - CV_EXPORTS CParams( - String cascade_model, - double sf = 1.1, - int minN = 3, - Size minSz = Size(30, 30), - Size maxSz = Size() - ); - - CascadeClassifier face_cascade; -}; - -/** @brief Default face detector -This function is mainly utilized by the implementation of a Facemark Algorithm. -End users are advised to use function Facemark::getFaces which can be manually defined -and circumvented to the algorithm by Facemark::setFaceDetector. - -@param image The input image to be processed. -@param faces Output of the function which represent region of interest of the detected faces. -Each face is stored in cv::Rect container. -@param params detector parameters - -Example of usage -@code -std::vector faces; -CParams params("haarcascade_frontalface_alt.xml"); -cv::face::getFaces(frame, faces, ¶ms); -for(int j=0;j images_train; -std::vector landmarks_train; -loadDatasetList(imageFiles,ptsFiles,images_train,landmarks_train); -@endcode -*/ -CV_EXPORTS_W bool loadDatasetList(String imageList, - String annotationList, - std::vector & images, - std::vector & annotations); - -/** @brief A utility to load facial landmark dataset from a single file. - -@param filename The filename of a file that contains the dataset information. -Each line contains the filename of an image followed by -pairs of x and y values of facial landmarks points separated by a space. -Example -@code -/home/user/ibug/image_003_1.jpg 336.820955 240.864510 334.238298 260.922709 335.266918 ... -/home/user/ibug/image_005_1.jpg 376.158428 230.845712 376.736984 254.924635 383.265403 ... -@endcode -@param images A vector where each element represent the filename of image in the dataset. -Images are not loaded by default to save the memory. -@param facePoints The loaded landmark points for all training data. -@param delim Delimiter between each element, the default value is a whitespace. -@param offset An offset value to adjust the loaded points. - -Example of usage -@code -cv::String imageFiles = "../data/images_train.txt"; -cv::String ptsFiles = "../data/points_train.txt"; -std::vector images; -std::vector > facePoints; -loadTrainingData(imageFiles, ptsFiles, images, facePoints, 0.0f); -@endcode -*/ -CV_EXPORTS_W bool loadTrainingData( String filename , std::vector & images, - OutputArray facePoints, - char delim = ' ', float offset = 0.0f); - -/** @brief A utility to load facial landmark information from the dataset. - -@param imageList A file contains the list of image filenames in the training dataset. -@param groundTruth A file contains the list of filenames -where the landmarks points information are stored. -The content in each file should follow the standard format (see face::loadFacePoints). -@param images A vector where each element represent the filename of image in the dataset. -Images are not loaded by default to save the memory. -@param facePoints The loaded landmark points for all training data. -@param offset An offset value to adjust the loaded points. - -Example of usage -@code -cv::String imageFiles = "../data/images_train.txt"; -cv::String ptsFiles = "../data/points_train.txt"; -std::vector images; -std::vector > facePoints; -loadTrainingData(imageFiles, ptsFiles, images, facePoints, 0.0f); -@endcode - -example of content in the images_train.txt -@code -/home/user/ibug/image_003_1.jpg -/home/user/ibug/image_004_1.jpg -/home/user/ibug/image_005_1.jpg -/home/user/ibug/image_006.jpg -@endcode - -example of content in the points_train.txt -@code -/home/user/ibug/image_003_1.pts -/home/user/ibug/image_004_1.pts -/home/user/ibug/image_005_1.pts -/home/user/ibug/image_006.pts -@endcode -*/ -CV_EXPORTS_W bool loadTrainingData( String imageList, String groundTruth, - std::vector & images, - OutputArray facePoints, - float offset = 0.0f); - -/** @brief This function extracts the data for training from .txt files which contains the corresponding image name and landmarks. -*The first file in each file should give the path of the image whose -*landmarks are being described in the file. Then in the subsequent -*lines there should be coordinates of the landmarks in the image -*i.e each line should be of the form x,y -*where x represents the x coordinate of the landmark and y represents -*the y coordinate of the landmark. -* -*For reference you can see the files as provided in the -*HELEN dataset -* -* @param filename A vector of type cv::String containing name of the .txt files. -* @param trainlandmarks A vector of type cv::Point2f that would store shape or landmarks of all images. -* @param trainimages A vector of type cv::String which stores the name of images whose landmarks are tracked -* @returns A boolean value. It returns true when it reads the data successfully and false otherwise -*/ -CV_EXPORTS_W bool loadTrainingData(std::vector filename,std::vector< std::vector > - &trainlandmarks,std::vector & trainimages); - -/** @brief A utility to load facial landmark information from a given file. - -@param filename The filename of file contains the facial landmarks data. -@param points The loaded facial landmark points. -@param offset An offset value to adjust the loaded points. - -Example of usage -@code -std::vector points; -face::loadFacePoints("filename.txt", points, 0.0f); -@endcode - -The annotation file should follow the default format which is -@code -version: 1 -n_points: 68 -{ -212.716603 499.771793 -230.232816 566.290071 -... -} -@endcode -where n_points is the number of points considered -and each point is represented as its position in x and y. -*/ -CV_EXPORTS_W bool loadFacePoints( String filename, OutputArray points, - float offset = 0.0f); - -/** @brief Utility to draw the detected facial landmark points - -@param image The input image to be processed. -@param points Contains the data of points which will be drawn. -@param color The color of points in BGR format represented by cv::Scalar. - -Example of usage -@code -std::vector faces; -std::vector > landmarks; -facemark->getFaces(img, faces); -facemark->fit(img, faces, landmarks); -for(int j=0;j facemark = FacemarkLBF::create(); +Ptr facemark = createFacemarkLBF(); @endcode -The typical pipeline for facemark detection is listed as follows: -- (Non-mandatory) Set a user defined face detection using Facemark::setFaceDetector. - The facemark algorithms are desgined to fit the facial points into a face. - Therefore, the face information should be provided to the facemark algorithm. - Some algorithms might provides a default face recognition function. - However, the users might prefer to use their own face detector to obtains the best possible detection result. -- (Non-mandatory) Training the model for a specific algorithm using Facemark::training. - In this case, the model should be automatically saved by the algorithm. - If the user already have a trained model, then this part can be omitted. +The typical pipeline for facemark detection is as follows: - Load the trained model using Facemark::loadModel. -- Perform the fitting via the Facemark::fit. +- Perform the fitting on an image via Facemark::fit. */ class CV_EXPORTS_W Facemark : public virtual Algorithm { public: - virtual void read( const FileNode& fn ) CV_OVERRIDE = 0; - virtual void write( FileStorage& fs ) const CV_OVERRIDE = 0; - - /** @brief Add one training sample to the trainer. - - @param image Input image. - @param landmarks The ground-truth of facial landmarks points corresponds to the image. - - Example of usage - @code - String imageFiles = "../data/images_train.txt"; - String ptsFiles = "../data/points_train.txt"; - std::vector images_train; - std::vector landmarks_train; - - // load the list of dataset: image paths and landmark file paths - loadDatasetList(imageFiles,ptsFiles,images_train,landmarks_train); - - Mat image; - std::vector facial_points; - for(size_t i=0;iaddTrainingSample(image, facial_points); - } - @endcode - - The contents in the training files should follows the standard format. - Here are examples for the contents in these files. - example of content in the images_train.txt - @code - /home/user/ibug/image_003_1.jpg - /home/user/ibug/image_004_1.jpg - /home/user/ibug/image_005_1.jpg - /home/user/ibug/image_006.jpg - @endcode - - example of content in the points_train.txt - @code - /home/user/ibug/image_003_1.pts - /home/user/ibug/image_004_1.pts - /home/user/ibug/image_005_1.pts - /home/user/ibug/image_006.pts - @endcode - - */ - virtual bool addTrainingSample(InputArray image, InputArray landmarks)=0; - - /** @brief Trains a Facemark algorithm using the given dataset. - Before the training process, training samples should be added to the trainer - using face::addTrainingSample function. - - @param parameters Optional extra parameters (algorithm dependent). - - Example of usage - @code - FacemarkLBF::Params params; - params.model_filename = "ibug68.model"; // filename to save the trained model - Ptr facemark = FacemarkLBF::create(params); - - // add training samples (see Facemark::addTrainingSample) - - facemark->training(); - @endcode - */ - - virtual void training(void* parameters=0)=0; - /** @brief A function to load the trained model before the fitting process. - @param model A string represent the filename of a trained model. Example of usage @@ -342,16 +56,14 @@ public: facemark->loadModel("../data/lbf.model"); @endcode */ - virtual void loadModel(String model)=0; + CV_WRAP virtual void loadModel( String model ) = 0; // virtual void saveModel(String fs)=0; - /** @brief Trains a Facemark algorithm using the given dataset. - + /** @brief Detect facial landmarks from an image. @param image Input image. @param faces Output of the function which represent region of interest of the detected faces. Each face is stored in cv::Rect container. @param landmarks The detected landmark points for each faces. - @param config Algorithm specific for running time parameters. Example of usage @code @@ -360,74 +72,24 @@ public: std::vector > landmarks; facemark->fit(image, faces, landmarks); @endcode - - TODO remove "config" from here - */ - virtual bool fit( InputArray image, - InputArray faces, - InputOutputArray landmarks, - void * config = 0)=0; - - /** @brief Set a user defined face detector for the Facemark algorithm. - @param detector The user defined face detector function - @param userData Detector parameters - - Example of usage - @code - MyDetectorParameters detectorParameters(...); - facemark->setFaceDetector(myDetector, &detectorParameters); - @endcode - - Example of a user defined face detector - @code - bool myDetector( InputArray image, OutputArray faces, void* userData) - { - MyDetectorParameters* params = (MyDetectorParameters*)userData; - // -------- do something -------- - } - @endcode - - TODO Lifetime of detector parameters is uncontrolled. Rework interface design to "Ptr". */ - virtual bool setFaceDetector(FN_FaceDetector detector, void* userData = 0)=0; + CV_WRAP virtual bool fit( InputArray image, + InputArray faces, + OutputArrayOfArrays landmarks ) = 0; +}; /* Facemark*/ - /** @brief Detect faces from a given image using default or user defined face detector. - Some Algorithm might not provide a default face detector. - @param image Input image. - @param faces Output of the function which represent region of interest of the detected faces. Each face is stored in cv::Rect container. +//! construct an AAM facemark detector +CV_EXPORTS_W Ptr createFacemarkAAM(); - Example of usage - @code - std::vector faces; - facemark->getFaces(img, faces); - for(int j=0;j createFacemarkLBF(); - /** @brief Get data from an algorithm +//! construct a Kazemi facemark detector +CV_EXPORTS_W Ptr createFacemarkKazemi(); - @param items The obtained data, algorithm dependent. - Example of usage - @code - Ptr facemark = FacemarkAAM::create(); - facemark->loadModel("AAM.yml"); - - FacemarkAAM::Data data; - facemark->getData(&data); - std::vector s0 = data.s0; - - cout<scales; //!< defines the scales considered to build the model @@ -147,7 +145,11 @@ public: }; - //!< initializer + //! overload with additional Config structures + virtual bool fitConfig( InputArray image, InputArray roi, OutputArrayOfArrays _landmarks, const std::vector &runtime_params ) = 0; + + + //! initializer static Ptr create(const FacemarkAAM::Params ¶meters = FacemarkAAM::Params() ); virtual ~FacemarkAAM() {} diff --git a/modules/face/include/opencv2/face/facemarkLBF.hpp b/modules/face/include/opencv2/face/facemarkLBF.hpp index 90ba3fb86..7f4cd325c 100644 --- a/modules/face/include/opencv2/face/facemarkLBF.hpp +++ b/modules/face/include/opencv2/face/facemarkLBF.hpp @@ -37,7 +37,7 @@ Mentor: Delia Passalacqua #ifndef __OPENCV_FACEMARK_LBF_HPP__ #define __OPENCV_FACEMARK_LBF_HPP__ -#include "opencv2/face/facemark.hpp" +#include "opencv2/face/facemark_train.hpp" namespace cv { namespace face { @@ -45,7 +45,7 @@ namespace face { //! @addtogroup face //! @{ -class CV_EXPORTS_W FacemarkLBF : public Facemark +class CV_EXPORTS_W FacemarkLBF : public FacemarkTrain { public: struct CV_EXPORTS Params @@ -99,8 +99,8 @@ public: ~BBox(); BBox(double x, double y, double w, double h); - cv::Mat project(const cv::Mat &shape) const; - cv::Mat reproject(const cv::Mat &shape) const; + Mat project(const Mat &shape) const; + Mat reproject(const Mat &shape) const; double x, y; double x_center, y_center; diff --git a/modules/face/include/opencv2/face/facemark_train.hpp b/modules/face/include/opencv2/face/facemark_train.hpp new file mode 100644 index 000000000..33ecb9464 --- /dev/null +++ b/modules/face/include/opencv2/face/facemark_train.hpp @@ -0,0 +1,392 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +/* +This file was part of GSoC Project: Facemark API for OpenCV +Final report: https://gist.github.com/kurnianggoro/74de9121e122ad0bd825176751d47ecc +Student: Laksono Kurnianggoro +Mentor: Delia Passalacqua +*/ + +#ifndef __OPENCV_FACELANDMARKTRAIN_HPP__ +#define __OPENCV_FACELANDMARKTRAIN_HPP__ + +/** +@defgroup face Face Analysis +- @ref tutorial_table_of_content_facemark +- The Facemark API +*/ + +#include "opencv2/face/facemark.hpp" +#include "opencv2/objdetect.hpp" +#include +#include + + +namespace cv { +namespace face { + +//! @addtogroup face +//! @{ + +typedef bool(*FN_FaceDetector)(InputArray, OutputArray, void* userData); + +struct CParams{ + String cascade; //!< the face detector + double scaleFactor; //!< Parameter specifying how much the image size is reduced at each image scale. + int minNeighbors; //!< Parameter specifying how many neighbors each candidate rectangle should have to retain it. + Size minSize; //!< Minimum possible object size. + Size maxSize; //!< Maximum possible object size. + + CV_EXPORTS CParams( + String cascade_model, + double sf = 1.1, + int minN = 3, + Size minSz = Size(30, 30), + Size maxSz = Size() + ); + + CascadeClassifier face_cascade; +}; + +/** @brief Default face detector +This function is mainly utilized by the implementation of a Facemark Algorithm. +End users are advised to use function Facemark::getFaces which can be manually defined +and circumvented to the algorithm by Facemark::setFaceDetector. + +@param image The input image to be processed. +@param faces Output of the function which represent region of interest of the detected faces. +Each face is stored in cv::Rect container. +@param params detector parameters + +Example of usage +@code +std::vector faces; +CParams params("haarcascade_frontalface_alt.xml"); +cv::face::getFaces(frame, faces, ¶ms); +for(int j=0;j images_train; +std::vector landmarks_train; +loadDatasetList(imageFiles,ptsFiles,images_train,landmarks_train); +@endcode +*/ +CV_EXPORTS_W bool loadDatasetList(String imageList, + String annotationList, + std::vector & images, + std::vector & annotations); + +/** @brief A utility to load facial landmark dataset from a single file. + +@param filename The filename of a file that contains the dataset information. +Each line contains the filename of an image followed by +pairs of x and y values of facial landmarks points separated by a space. +Example +@code +/home/user/ibug/image_003_1.jpg 336.820955 240.864510 334.238298 260.922709 335.266918 ... +/home/user/ibug/image_005_1.jpg 376.158428 230.845712 376.736984 254.924635 383.265403 ... +@endcode +@param images A vector where each element represent the filename of image in the dataset. +Images are not loaded by default to save the memory. +@param facePoints The loaded landmark points for all training data. +@param delim Delimiter between each element, the default value is a whitespace. +@param offset An offset value to adjust the loaded points. + +Example of usage +@code +cv::String imageFiles = "../data/images_train.txt"; +cv::String ptsFiles = "../data/points_train.txt"; +std::vector images; +std::vector > facePoints; +loadTrainingData(imageFiles, ptsFiles, images, facePoints, 0.0f); +@endcode +*/ +CV_EXPORTS_W bool loadTrainingData( String filename , std::vector & images, + OutputArray facePoints, + char delim = ' ', float offset = 0.0f); + +/** @brief A utility to load facial landmark information from the dataset. + +@param imageList A file contains the list of image filenames in the training dataset. +@param groundTruth A file contains the list of filenames +where the landmarks points information are stored. +The content in each file should follow the standard format (see face::loadFacePoints). +@param images A vector where each element represent the filename of image in the dataset. +Images are not loaded by default to save the memory. +@param facePoints The loaded landmark points for all training data. +@param offset An offset value to adjust the loaded points. + +Example of usage +@code +cv::String imageFiles = "../data/images_train.txt"; +cv::String ptsFiles = "../data/points_train.txt"; +std::vector images; +std::vector > facePoints; +loadTrainingData(imageFiles, ptsFiles, images, facePoints, 0.0f); +@endcode + +example of content in the images_train.txt +@code +/home/user/ibug/image_003_1.jpg +/home/user/ibug/image_004_1.jpg +/home/user/ibug/image_005_1.jpg +/home/user/ibug/image_006.jpg +@endcode + +example of content in the points_train.txt +@code +/home/user/ibug/image_003_1.pts +/home/user/ibug/image_004_1.pts +/home/user/ibug/image_005_1.pts +/home/user/ibug/image_006.pts +@endcode +*/ +CV_EXPORTS_W bool loadTrainingData( String imageList, String groundTruth, + std::vector & images, + OutputArray facePoints, + float offset = 0.0f); + +/** @brief This function extracts the data for training from .txt files which contains the corresponding image name and landmarks. +*The first file in each file should give the path of the image whose +*landmarks are being described in the file. Then in the subsequent +*lines there should be coordinates of the landmarks in the image +*i.e each line should be of the form x,y +*where x represents the x coordinate of the landmark and y represents +*the y coordinate of the landmark. +* +*For reference you can see the files as provided in the +*HELEN dataset +* +* @param filename A vector of type cv::String containing name of the .txt files. +* @param trainlandmarks A vector of type cv::Point2f that would store shape or landmarks of all images. +* @param trainimages A vector of type cv::String which stores the name of images whose landmarks are tracked +* @returns A boolean value. It returns true when it reads the data successfully and false otherwise +*/ +CV_EXPORTS_W bool loadTrainingData(std::vector filename,std::vector< std::vector > + &trainlandmarks,std::vector & trainimages); + +/** @brief A utility to load facial landmark information from a given file. + +@param filename The filename of file contains the facial landmarks data. +@param points The loaded facial landmark points. +@param offset An offset value to adjust the loaded points. + +Example of usage +@code +std::vector points; +face::loadFacePoints("filename.txt", points, 0.0f); +@endcode + +The annotation file should follow the default format which is +@code +version: 1 +n_points: 68 +{ +212.716603 499.771793 +230.232816 566.290071 +... +} +@endcode +where n_points is the number of points considered +and each point is represented as its position in x and y. +*/ +CV_EXPORTS_W bool loadFacePoints( String filename, OutputArray points, + float offset = 0.0f); + +/** @brief Utility to draw the detected facial landmark points + +@param image The input image to be processed. +@param points Contains the data of points which will be drawn. +@param color The color of points in BGR format represented by cv::Scalar. + +Example of usage +@code +std::vector faces; +std::vector > landmarks; +facemark->getFaces(img, faces); +facemark->fit(img, faces, landmarks); +for(int j=0;j facemark = FacemarkLBF::create(); +@endcode + + +The typical pipeline for facemark detection is listed as follows: +- (Non-mandatory) Set a user defined face detection using FacemarkTrain::setFaceDetector. + The facemark algorithms are desgined to fit the facial points into a face. + Therefore, the face information should be provided to the facemark algorithm. + Some algorithms might provides a default face recognition function. + However, the users might prefer to use their own face detector to obtains the best possible detection result. +- (Non-mandatory) Training the model for a specific algorithm using FacemarkTrain::training. + In this case, the model should be automatically saved by the algorithm. + If the user already have a trained model, then this part can be omitted. +- Load the trained model using Facemark::loadModel. +- Perform the fitting via the Facemark::fit. +*/ +class CV_EXPORTS_W FacemarkTrain : public Facemark +{ +public: + /** @brief Add one training sample to the trainer. + + @param image Input image. + @param landmarks The ground-truth of facial landmarks points corresponds to the image. + + Example of usage + @code + String imageFiles = "../data/images_train.txt"; + String ptsFiles = "../data/points_train.txt"; + std::vector images_train; + std::vector landmarks_train; + + // load the list of dataset: image paths and landmark file paths + loadDatasetList(imageFiles,ptsFiles,images_train,landmarks_train); + + Mat image; + std::vector facial_points; + for(size_t i=0;iaddTrainingSample(image, facial_points); + } + @endcode + + The contents in the training files should follows the standard format. + Here are examples for the contents in these files. + example of content in the images_train.txt + @code + /home/user/ibug/image_003_1.jpg + /home/user/ibug/image_004_1.jpg + /home/user/ibug/image_005_1.jpg + /home/user/ibug/image_006.jpg + @endcode + + example of content in the points_train.txt + @code + /home/user/ibug/image_003_1.pts + /home/user/ibug/image_004_1.pts + /home/user/ibug/image_005_1.pts + /home/user/ibug/image_006.pts + @endcode + + */ + virtual bool addTrainingSample(InputArray image, InputArray landmarks)=0; + + /** @brief Trains a Facemark algorithm using the given dataset. + Before the training process, training samples should be added to the trainer + using face::addTrainingSample function. + + @param parameters Optional extra parameters (algorithm dependent). + + Example of usage + @code + FacemarkLBF::Params params; + params.model_filename = "ibug68.model"; // filename to save the trained model + Ptr facemark = FacemarkLBF::create(params); + + // add training samples (see Facemark::addTrainingSample) + + facemark->training(); + @endcode + */ + + virtual void training(void* parameters=0)=0; + + /** @brief Set a user defined face detector for the Facemark algorithm. + @param detector The user defined face detector function + @param userData Detector parameters + + Example of usage + @code + MyDetectorParameters detectorParameters(...); + facemark->setFaceDetector(myDetector, &detectorParameters); + @endcode + + Example of a user defined face detector + @code + bool myDetector( InputArray image, OutputArray faces, void* userData) + { + MyDetectorParameters* params = (MyDetectorParameters*)userData; + // -------- do something -------- + } + @endcode + + TODO Lifetime of detector parameters is uncontrolled. Rework interface design to "Ptr". + */ + virtual bool setFaceDetector(FN_FaceDetector detector, void* userData = 0)=0; + + /** @brief Detect faces from a given image using default or user defined face detector. + Some Algorithm might not provide a default face detector. + + @param image Input image. + @param faces Output of the function which represent region of interest of the detected faces. Each face is stored in cv::Rect container. + + Example of usage + @code + std::vector faces; + facemark->getFaces(img, faces); + for(int j=0;jExample of usage + @code + Ptr facemark = FacemarkAAM::create(); + facemark->loadModel("AAM.yml"); + + FacemarkAAM::Data data; + facemark->getData(&data); + std::vector s0 = data.s0; + + cout< landmarks = new ArrayList(); + fm.fit(img, faces, landmarks); + + // draw them + for (int i=0; i > landmarks; double newtime = (double)getTickCount(); - facemark->fit(image, faces_eyes, landmarks, (void*)&conf); + facemark->fitConfig(image, faces_eyes, landmarks, conf); double fittime = ((getTickCount() - newtime)/getTickFrequency()); for(unsigned j=0;j facemark = FacemarkLBF::create(params); + Ptr facemark = FacemarkLBF::create(params); CascadeClassifier face_cascade; face_cascade.load(params.cascade_face.c_str()); diff --git a/modules/face/samples/facemark_lbf_fitting.cpp b/modules/face/samples/facemark_lbf_fitting.cpp index 961ed900d..60f0b133a 100644 --- a/modules/face/samples/facemark_lbf_fitting.cpp +++ b/modules/face/samples/facemark_lbf_fitting.cpp @@ -74,7 +74,7 @@ int main(int argc, char** argv ){ params.model_filename = model_path; params.cascade_face = cascade_path; - Ptr facemark = FacemarkLBF::create(params); + Ptr facemark = FacemarkLBF::create(params); facemark->setFaceDetector((FN_FaceDetector)myDetector, &face_cascade); facemark->loadModel(params.model_filename.c_str()); diff --git a/modules/face/samples/sample_train_landmark_detector2.cpp b/modules/face/samples/sample_train_landmark_detector2.cpp index c337ccfb2..b268c367a 100644 --- a/modules/face/samples/sample_train_landmark_detector2.cpp +++ b/modules/face/samples/sample_train_landmark_detector2.cpp @@ -1,5 +1,5 @@ /*---------------------------------------------- - * the user should provides the list of training images_train + * the user should provide the list of training images_train, * accompanied by their corresponding landmarks location in separated files. * example of contents for images.txt: * ../trainset/image_0001.png @@ -131,4 +131,4 @@ int main(int argc,char** argv){ facemark->training(Trainimages,Trainlandmarks,configfile_name,scale,modelfile_name); cout<<"Training complete"< > & trainlandmarks, Ptr FacemarkKazemi::create(const FacemarkKazemi::Params ¶meters){ return Ptr(new FacemarkKazemiImpl(parameters)); } + +Ptr createFacemarkKazemi() { + FacemarkKazemi::Params parameters; + return Ptr(new FacemarkKazemiImpl(parameters)); +} }//cv -}//face \ No newline at end of file +}//face diff --git a/modules/face/src/face_alignmentimpl.hpp b/modules/face/src/face_alignmentimpl.hpp index c8ae3ac42..4c50cc337 100644 --- a/modules/face/src/face_alignmentimpl.hpp +++ b/modules/face/src/face_alignmentimpl.hpp @@ -73,11 +73,14 @@ public: void loadModel(String fs) CV_OVERRIDE; bool setFaceDetector(FN_FaceDetector f, void* userdata) CV_OVERRIDE; bool getFaces(InputArray image, OutputArray faces) CV_OVERRIDE; - bool fit(InputArray image, InputArray faces, InputOutputArray landmarks) CV_OVERRIDE; - void training(String imageList, String groundTruth) CV_OVERRIDE; + bool fit(InputArray image, InputArray faces, OutputArrayOfArrays landmarks ) CV_OVERRIDE; + void training(String imageList, String groundTruth); bool training(vector& images, vector< vector >& landmarks,string filename,Size scale,string modelFilename) CV_OVERRIDE; // Destructor for the class. - virtual ~FacemarkKazemiImpl(); + virtual ~FacemarkKazemiImpl() CV_OVERRIDE; + + virtual void read( const FileNode& ) CV_OVERRIDE {} + virtual void write( FileStorage& ) const CV_OVERRIDE {} protected: FacemarkKazemi::Params params; diff --git a/modules/face/src/facemark.cpp b/modules/face/src/facemark.cpp index c614af56f..652c14a1c 100644 --- a/modules/face/src/facemark.cpp +++ b/modules/face/src/facemark.cpp @@ -10,7 +10,7 @@ Mentor: Delia Passalacqua */ #include "precomp.hpp" -#include "opencv2/face.hpp" +#include "opencv2/face/facemark_train.hpp" /*dataset parser*/ #include diff --git a/modules/face/src/facemarkAAM.cpp b/modules/face/src/facemarkAAM.cpp index 9d4a9008e..00a77b406 100644 --- a/modules/face/src/facemarkAAM.cpp +++ b/modules/face/src/facemarkAAM.cpp @@ -103,9 +103,12 @@ public: bool getData(void * items) CV_OVERRIDE; + bool fitConfig( InputArray image, InputArray roi, OutputArrayOfArrays _landmarks, const std::vector &runtime_params ); + protected: - bool fit( InputArray image, InputArray faces, InputOutputArray landmarks, void * runtime_params) CV_OVERRIDE;//!< from many ROIs + bool fit( InputArray image, InputArray faces, OutputArrayOfArrays landmarks ); + //bool fit( InputArray image, InputArray faces, InputOutputArray landmarks, void * runtime_params);//!< from many ROIs bool fitImpl( const Mat image, std::vector& landmarks,const Mat R,const Point2f T,const float scale, const int sclIdx=0 ); bool addTrainingSample(InputArray image, InputArray landmarks) CV_OVERRIDE; @@ -153,6 +156,14 @@ Ptr FacemarkAAM::create(const FacemarkAAM::Params ¶meters){ return Ptr(new FacemarkAAMImpl(parameters)); } +/* +* Constructor +*/ +Ptr createFacemarkAAM(){ + FacemarkAAM::Params parameters; + return Ptr(new FacemarkAAMImpl(parameters)); +} + FacemarkAAMImpl::FacemarkAAMImpl( const FacemarkAAM::Params ¶meters ) : params( parameters ), faceDetector(NULL), faceDetectorData(NULL) @@ -312,7 +323,13 @@ void FacemarkAAMImpl::training(void* parameters){ if(params.verbose) printf("Training is completed\n"); } -bool FacemarkAAMImpl::fit( InputArray image, InputArray roi, InputOutputArray _landmarks, void * runtime_params) +bool FacemarkAAMImpl::fit( InputArray image, InputArray roi, OutputArrayOfArrays _landmarks ) +{ + std::vector config; // empty + return fitConfig(image, roi, _landmarks, config); +} + +bool FacemarkAAMImpl::fitConfig( InputArray image, InputArray roi, OutputArrayOfArrays _landmarks, const std::vector &configs ) { std::vector & faces = *(std::vector *)roi.getObj(); if(faces.size()<1) return false; @@ -322,14 +339,13 @@ bool FacemarkAAMImpl::fit( InputArray image, InputArray roi, InputOutputArray _l landmarks.resize(faces.size()); Mat img = image.getMat(); - if(runtime_params!=0){ + if (! configs.empty()){ - std::vector conf = *(std::vector*)runtime_params; - if (conf.size()!=faces.size()) { + if (configs.size()!=faces.size()) { CV_Error(Error::StsBadArg, "Number of faces and extra_parameters are different!"); } - for(size_t i=0; i & landmarks );//!< from a face bool addTrainingSample(InputArray image, InputArray landmarks) CV_OVERRIDE; @@ -248,6 +248,13 @@ private: Ptr FacemarkLBF::create(const FacemarkLBF::Params ¶meters){ return Ptr(new FacemarkLBFImpl(parameters)); } +/* +* Constructor +*/ +Ptr createFacemarkLBF(){ + const FacemarkLBF::Params parameters; + return Ptr(new FacemarkLBFImpl(parameters)); +} FacemarkLBFImpl::FacemarkLBFImpl( const FacemarkLBF::Params ¶meters ) : faceDetector(NULL), faceDetectorData(NULL) @@ -363,10 +370,8 @@ void FacemarkLBFImpl::training(void* parameters){ isModelTrained = true; } -bool FacemarkLBFImpl::fit( InputArray image, InputArray roi, InputOutputArray _landmarks, void * runtime_params ) +bool FacemarkLBFImpl::fit( InputArray image, InputArray roi, OutputArrayOfArrays _landmarks ) { - CV_UNUSED(runtime_params); - // FIXIT std::vector & faces = *(std::vector *)roi.getObj(); if (faces.empty()) return false; diff --git a/modules/face/src/getlandmarks.cpp b/modules/face/src/getlandmarks.cpp index cf6c79e39..a7da5271f 100644 --- a/modules/face/src/getlandmarks.cpp +++ b/modules/face/src/getlandmarks.cpp @@ -163,7 +163,7 @@ void FacemarkKazemiImpl :: loadModel(String filename){ f.close(); isModelLoaded = true; } -bool FacemarkKazemiImpl::fit(InputArray img, InputArray roi, InputOutputArray landmarks){ +bool FacemarkKazemiImpl::fit(InputArray img, InputArray roi, OutputArrayOfArrays landmarks){ if(!isModelLoaded){ String error_message = "No model loaded. Aborting...."; CV_Error(Error::StsBadArg, error_message); @@ -241,4 +241,4 @@ bool FacemarkKazemiImpl::fit(InputArray img, InputArray roi, InputOutputArray la return true; } }//cv -}//face \ No newline at end of file +}//face diff --git a/modules/face/test/test_facemark_aam.cpp b/modules/face/test/test_facemark_aam.cpp index 79cb399d7..5f4c520c6 100644 --- a/modules/face/test/test_facemark_aam.cpp +++ b/modules/face/test/test_facemark_aam.cpp @@ -64,7 +64,7 @@ static bool customDetector( InputArray image, OutputArray ROIs, CascadeClassifie TEST(CV_Face_FacemarkAAM, can_create_default) { FacemarkAAM::Params params; - Ptr facemark; + Ptr facemark; EXPECT_NO_THROW(facemark = FacemarkAAM::create(params)); EXPECT_FALSE(facemark.empty()); } @@ -75,7 +75,7 @@ TEST(CV_Face_FacemarkAAM, can_set_custom_detector) { CascadeClassifier face_detector; EXPECT_TRUE(face_detector.load(cascade_filename)); - Ptr facemark = FacemarkAAM::create(); + Ptr facemark = FacemarkAAM::create(); EXPECT_TRUE(facemark->setFaceDetector((cv::face::FN_FaceDetector)customDetector, &face_detector)); } @@ -104,7 +104,7 @@ TEST(CV_Face_FacemarkAAM, test_workflow) { params.m = 1; params.verbose = false; params.save_model = false; - Ptr facemark = FacemarkAAM::create(params); + Ptr facemark = FacemarkAAM::create(params); Mat image; std::vector landmarks; diff --git a/modules/face/test/test_facemark_lbf.cpp b/modules/face/test/test_facemark_lbf.cpp index fde5a52ea..144c87338 100644 --- a/modules/face/test/test_facemark_lbf.cpp +++ b/modules/face/test/test_facemark_lbf.cpp @@ -70,7 +70,7 @@ TEST(CV_Face_FacemarkLBF, can_create_default) { FacemarkLBF::Params params; params.n_landmarks = 68; - Ptr facemark; + Ptr facemark; EXPECT_NO_THROW(facemark = FacemarkLBF::create(params)); EXPECT_FALSE(facemark.empty()); } @@ -81,7 +81,7 @@ TEST(CV_Face_FacemarkLBF, can_set_custom_detector) { EXPECT_TRUE(cascade_detector.load(cascade_filename)); - Ptr facemark = FacemarkLBF::create(); + Ptr facemark = FacemarkLBF::create(); EXPECT_TRUE(facemark->setFaceDetector(myCustomDetector)); } @@ -107,7 +107,7 @@ TEST(CV_Face_FacemarkLBF, test_workflow) { params.verbose = false; params.save_model = false; - Ptr facemark = FacemarkLBF::create(params); + Ptr facemark = FacemarkLBF::create(params); Mat image; std::vector landmarks; diff --git a/modules/face/tutorials/face_landmark/face_landmark_detection.markdown b/modules/face/tutorials/face_landmark/face_landmark_detection.markdown index b87d8569d..bc566fa14 100644 --- a/modules/face/tutorials/face_landmark/face_landmark_detection.markdown +++ b/modules/face/tutorials/face_landmark/face_landmark_detection.markdown @@ -26,72 +26,66 @@ This tutorial will explain the sample code for face landmark detection. Jumping ``` c++ CascadeClassifier face_cascade; -bool myDetector( InputArray image, OutputArray ROIs ); +face_cascade.load(cascade_name); -bool myDetector( InputArray image, OutputArray ROIs ){ - Mat gray; - std::vector faces; - if(image.channels()>1){ - cvtColor(image.getMat(),gray,CV_BGR2GRAY); - } - else{ - gray = image.getMat().clone(); - } - equalizeHist( gray, gray ); - face_cascade.detectMultiScale( gray, faces, 1.1, 3,0, Size(30, 30) ); - Mat(faces).copyTo(ROIs); - return true; -} +Mat img = imread(image); +Ptr facemark = createFacemarkKazemi()); +facemark->loadModel(filename); +cout<<"Loaded model"< facemark = FacemarkKazemi::create(params); -facemark->setFaceDetector(myDetector); - -``` -The above code creates a pointer of the face landmark detection class. The face detector created above has to be passed -as function pointer to the facemark pointer created for detecting faces while landmark detection. The above code also loads the image -in which landmarks have to be detected. ``` c++ -facemark->loadModel(filename); -cout<<"Loaded model"< faces; resize(img,img,Size(460,460),0,0,INTER_LINEAR_EXACT); -facemark->getFaces(img,faces); -vector< vector > shapes; -``` c++ -The above code loads a trained model for face landmark detection and creates a vector to store the detected faces. It then resizes the image to a smaller size as processing speed is faster with small images. It then creates a vector of vector to store shapes for each face detected. +Mat gray; +std::vector faces; +if(img.channels()>1){ + cvtColor(img.getMat(),gray,CV_BGR2GRAY); +} +else{ + gray = img.getMat().clone(); +} +equalizeHist( gray, gray ); + +face_cascade.detectMultiScale( gray, faces, 1.1, 3,0, Size(30, 30) ); +``` + +After doing some preprocessing, we first have to detect possible face regions (which will be stored in a `vector`. +Also, the image is resized to a smaller size as processing speed is faster with small images. + ``` c++ -if(facemark->fit(img,faces,shapes)) -{ -for( size_t i = 0; i < faces.size(); i++ ) +vector< vector > shapes; + +if (facemark->fit(img,faces,shapes)) { - cv::rectangle(img,faces[i],Scalar( 255, 0, 0 )); -} -for(unsigned long i=0;i