diff --git a/modules/face/include/opencv2/face.hpp b/modules/face/include/opencv2/face.hpp
index 52fe97569..54fa4c086 100644
--- a/modules/face/include/opencv2/face.hpp
+++ b/modules/face/include/opencv2/face.hpp
@@ -40,7 +40,7 @@ the use of this software, even if advised of the possibility of such damage.
#define __OPENCV_FACE_HPP__
/**
-@defgroup face Face Recognition
+@defgroup face Face Analysis
- @ref face_changelog
- @ref tutorial_face_main
@@ -374,5 +374,7 @@ protected:
}}
#include "opencv2/face/facerec.hpp"
-
+#include "opencv2/face/facemark.hpp"
+#include "opencv2/face/facemarkLBF.hpp"
+#include "opencv2/face/facemarkAAM.hpp"
#endif
diff --git a/modules/face/include/opencv2/face/facemark.hpp b/modules/face/include/opencv2/face/facemark.hpp
new file mode 100644
index 000000000..f2eacc81f
--- /dev/null
+++ b/modules/face/include/opencv2/face/facemark.hpp
@@ -0,0 +1,439 @@
+/*
+By downloading, copying, installing or using the software you agree to this
+license. If you do not agree to this license, do not download, install,
+copy or use the software.
+ License Agreement
+ For Open Source Computer Vision Library
+ (3-clause BSD License)
+Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+Third party copyrights are property of their respective owners.
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ * Neither the names of the copyright holders nor the names of the contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+This software is provided by the copyright holders and contributors "as is" and
+any express or implied warranties, including, but not limited to, the implied
+warranties of merchantability and fitness for a particular purpose are
+disclaimed. In no event shall copyright holders or contributors be liable for
+any direct, indirect, incidental, special, exemplary, or consequential damages
+(including, but not limited to, procurement of substitute goods or services;
+loss of use, data, or profits; or business interruption) however caused
+and on any theory of liability, whether in contract, strict liability,
+or tort (including negligence or otherwise) arising in any way out of
+the use of this software, even if advised of the possibility of such damage.
+
+This file was part of GSoC Project: Facemark API for OpenCV
+Final report: https://gist.github.com/kurnianggoro/74de9121e122ad0bd825176751d47ecc
+Student: Laksono Kurnianggoro
+Mentor: Delia Passalacqua
+*/
+
+#ifndef __OPENCV_FACELANDMARK_HPP__
+#define __OPENCV_FACELANDMARK_HPP__
+
+/**
+@defgroup face Face Analysis
+- @ref tutorial_table_of_content_facemark
+- The Facemark API
+*/
+
+#include "opencv2/face.hpp"
+#include "opencv2/objdetect.hpp"
+#include "opencv2/objdetect/objdetect_c.h"
+#include "opencv2/imgproc/types_c.h"
+
+namespace cv {
+namespace face {
+
+//! @addtogroup face
+//! @{
+ struct CV_EXPORTS_W CParams{
+ String cascade; //!< the face detector
+ double scaleFactor; //!< Parameter specifying how much the image size is reduced at each image scale.
+ int minNeighbors; //!< Parameter specifying how many neighbors each candidate rectangle should have to retain it.
+ Size minSize; //!< Minimum possible object size.
+ Size maxSize; //!< Maximum possible object size.
+
+ CParams(
+ String cascade_model,
+ double sf = 1.1,
+ int minN = 3,
+ Size minSz = Size(30, 30),
+ Size maxSz = Size()
+ );
+ };
+ /** @brief Default face detector
+ This function is mainly utilized by the implementation of a Facemark Algorithm.
+ End users are advised to use function Facemark::getFaces which can be manually defined
+ and circumvented to the algorithm by Facemark::setFaceDetector.
+
+ @param image The input image to be processed.
+ @param faces Output of the function which represent region of interest of the detected faces.
+ Each face is stored in cv::Rect container.
+ @param extra_params extra parameters
+
+ Example of usage
+ @code
+ std::vector faces;
+ CParams params("haarcascade_frontalface_alt.xml");
+ cv::face::getFaces(frame, faces, ¶ms);
+ for(int j=0;j images_train;
+ std::vector landmarks_train;
+ loadDatasetList(imageFiles,ptsFiles,images_train,landmarks_train);
+ @endcode
+
+ */
+ CV_EXPORTS_W bool loadDatasetList(String imageList,
+ String annotationList,
+ std::vector & images,
+ std::vector & annotations);
+
+ /** @brief A utility to load facial landmark dataset from a single file.
+
+ @param filename The filename of a file that contains the dataset information.
+ Each line contains the filename of an image followed by
+ pairs of x and y values of facial landmarks points separated by a space.
+ Example
+ @code
+ /home/user/ibug/image_003_1.jpg 336.820955 240.864510 334.238298 260.922709 335.266918 ...
+ /home/user/ibug/image_005_1.jpg 376.158428 230.845712 376.736984 254.924635 383.265403 ...
+ @endcode
+ @param images A vector where each element represent the filename of image in the dataset.
+ Images are not loaded by default to save the memory.
+ @param facePoints The loaded landmark points for all training data.
+ @param delim Delimiter between each element, the default value is a whitespace.
+ @param offset An offset value to adjust the loaded points.
+
+ Example of usage
+ @code
+ cv::String imageFiles = "../data/images_train.txt";
+ cv::String ptsFiles = "../data/points_train.txt";
+ std::vector images;
+ std::vector > facePoints;
+ loadTrainingData(imageFiles, ptsFiles, images, facePoints, 0.0);
+ @endcode
+ */
+
+ CV_EXPORTS_W bool loadTrainingData( String filename , std::vector & images,
+ OutputArray facePoints,
+ char delim = ' ', float offset = 0.0);
+
+ /** @brief A utility to load facial landmark information from the dataset.
+
+ @param imageList A file contains the list of image filenames in the training dataset.
+ @param groundTruth A file contains the list of filenames
+ where the landmarks points information are stored.
+ The content in each file should follow the standard format (see face::loadFacePoints).
+ @param images A vector where each element represent the filename of image in the dataset.
+ Images are not loaded by default to save the memory.
+ @param facePoints The loaded landmark points for all training data.
+ @param offset An offset value to adjust the loaded points.
+
+ Example of usage
+ @code
+ cv::String imageFiles = "../data/images_train.txt";
+ cv::String ptsFiles = "../data/points_train.txt";
+ std::vector images;
+ std::vector > facePoints;
+ loadTrainingData(imageFiles, ptsFiles, images, facePoints, 0.0);
+ @endcode
+
+ example of content in the images_train.txt
+ @code
+ /home/user/ibug/image_003_1.jpg
+ /home/user/ibug/image_004_1.jpg
+ /home/user/ibug/image_005_1.jpg
+ /home/user/ibug/image_006.jpg
+ @endcode
+
+ example of content in the points_train.txt
+ @code
+ /home/user/ibug/image_003_1.pts
+ /home/user/ibug/image_004_1.pts
+ /home/user/ibug/image_005_1.pts
+ /home/user/ibug/image_006.pts
+ @endcode
+ */
+
+ CV_EXPORTS_W bool loadTrainingData( String imageList, String groundTruth,
+ std::vector & images,
+ OutputArray facePoints,
+ float offset = 0.0);
+
+ /** @brief A utility to load facial landmark information from a given file.
+
+ @param filename The filename of file contains the facial landmarks data.
+ @param points The loaded facial landmark points.
+ @param offset An offset value to adjust the loaded points.
+
+ Example of usage
+ @code
+ std::vector points;
+ face::loadFacePoints("filename.txt", points, 0.0);
+ @endcode
+
+ The annotation file should follow the default format which is
+ @code
+ version: 1
+ n_points: 68
+ {
+ 212.716603 499.771793
+ 230.232816 566.290071
+ ...
+ }
+ @endcode
+ where n_points is the number of points considered
+ and each point is represented as its position in x and y.
+ */
+
+ CV_EXPORTS_W bool loadFacePoints( String filename, OutputArray points,
+ float offset = 0.0);
+
+ /** @brief Utility to draw the detected facial landmark points
+
+ @param image The input image to be processed.
+ @param points Contains the data of points which will be drawn.
+ @param color The color of points in BGR format represented by cv::Scalar.
+
+ Example of usage
+ @code
+ std::vector faces;
+ std::vector > landmarks;
+ facemark->getFaces(img, faces);
+ facemark->fit(img, faces, landmarks);
+ for(int j=0;j facemark = FacemarkLBF::create();
+ @endcode
+
+ The typical pipeline for facemark detection is listed as follows:
+ - (Non-mandatory) Set a user defined face detection using Facemark::setFaceDetector.
+ The facemark algorithms are desgined to fit the facial points into a face.
+ Therefore, the face information should be provided to the facemark algorithm.
+ Some algorithms might provides a default face recognition function.
+ However, the users might prefer to use their own face detector to obtains the best possible detection result.
+ - (Non-mandatory) Training the model for a specific algorithm using Facemark::training.
+ In this case, the model should be automatically saved by the algorithm.
+ If the user already have a trained model, then this part can be omitted.
+ - Load the trained model using Facemark::loadModel.
+ - Perform the fitting via the Facemark::fit.
+ */
+ class CV_EXPORTS_W Facemark : public virtual Algorithm
+ {
+ public:
+
+ virtual void read( const FileNode& fn )=0;
+ virtual void write( FileStorage& fs ) const=0;
+
+ /** @brief Add one training sample to the trainer.
+
+ @param image Input image.
+ @param landmarks The ground-truth of facial landmarks points corresponds to the image.
+
+ Example of usage
+ @code
+ String imageFiles = "../data/images_train.txt";
+ String ptsFiles = "../data/points_train.txt";
+ std::vector images_train;
+ std::vector landmarks_train;
+
+ // load the list of dataset: image paths and landmark file paths
+ loadDatasetList(imageFiles,ptsFiles,images_train,landmarks_train);
+
+ Mat image;
+ std::vector facial_points;
+ for(size_t i=0;iaddTrainingSample(image, facial_points);
+ }
+ @endcode
+
+ The contents in the training files should follows the standard format.
+ Here are examples for the contents in these files.
+ example of content in the images_train.txt
+ @code
+ /home/user/ibug/image_003_1.jpg
+ /home/user/ibug/image_004_1.jpg
+ /home/user/ibug/image_005_1.jpg
+ /home/user/ibug/image_006.jpg
+ @endcode
+
+ example of content in the points_train.txt
+ @code
+ /home/user/ibug/image_003_1.pts
+ /home/user/ibug/image_004_1.pts
+ /home/user/ibug/image_005_1.pts
+ /home/user/ibug/image_006.pts
+ @endcode
+
+ */
+ virtual bool addTrainingSample(InputArray image, InputArray landmarks)=0;
+
+ /** @brief Trains a Facemark algorithm using the given dataset.
+ Before the training process, training samples should be added to the trainer
+ using face::addTrainingSample function.
+
+ @param parameters Optional extra parameters (algorithm dependent).
+
+ Example of usage
+ @code
+ FacemarkLBF::Params params;
+ params.model_filename = "ibug68.model"; // filename to save the trained model
+ Ptr facemark = FacemarkLBF::create(params);
+
+ // add training samples (see Facemark::addTrainingSample)
+
+ facemark->training();
+ @endcode
+ */
+
+ virtual void training(void* parameters=0)=0;
+
+ /** @brief A function to load the trained model before the fitting process.
+
+ @param model A string represent the filename of a trained model.
+
+ Example of usage
+ @code
+ facemark->loadModel("../data/lbf.model");
+ @endcode
+ */
+ virtual void loadModel(String model)=0;
+ // virtual void saveModel(String fs)=0;
+
+ /** @brief Trains a Facemark algorithm using the given dataset.
+
+ @param image Input image.
+ @param faces Output of the function which represent region of interest of the detected faces.
+ Each face is stored in cv::Rect container.
+ @param landmarks The detected landmark points for each faces.
+ @param config Algorithm specific for running time parameters.
+
+ Example of usage
+ @code
+ Mat image = imread("image.jpg");
+ std::vector faces;
+ std::vector > landmarks;
+ facemark->fit(image, faces, landmarks);
+ @endcode
+ */
+ virtual bool fit( InputArray image,\
+ InputArray faces,\
+ InputOutputArray landmarks,\
+ void * config = 0)=0;
+
+ /** @brief Set a user defined face detector for the Facemark algorithm.
+ @param f The user defined face detector function
+ Example of usage
+ @code
+ facemark->setFaceDetector(myDetector);
+ @endcode
+
+ Example of a user defined face detector
+ @code
+ bool myDetector( InputArray image, OutputArray ROIs ){
+ std::vector & faces = *(std::vector*) ROIs.getObj();
+ faces.clear();
+
+ Mat img = image.getMat();
+
+ // -------- do something --------
+ }
+ @endcode
+ */
+ virtual bool setFaceDetector(bool(*f)(InputArray , OutputArray, void * ))=0;
+
+ /** @brief Detect faces from a given image using default or user defined face detector.
+ Some Algorithm might not provide a default face detector.
+
+ @param image Input image.
+ @param faces Output of the function which represent region of interest of the detected faces.
+ Each face is stored in cv::Rect container.
+ @param extra_params Optional extra-parameters for the face detector function.
+
+ Example of usage
+ @code
+ std::vector faces;
+ facemark->getFaces(img, faces);
+ for(int j=0;jExample of usage
+ @code
+ Ptr facemark = FacemarkAAM::create();
+ facemark->loadModel("AAM.yml");
+
+ FacemarkAAM::Data data;
+ facemark->getData(&data);
+ std::vector s0 = data.s0;
+
+ cout<scales;
+ };
+
+ /**
+ * \brief Optional parameter for fitting process.
+ */
+ struct CV_EXPORTS Config
+ {
+ Config( Mat rot = Mat::eye(2,2,CV_32F),
+ Point2f trans = Point2f(0.0,0.0),
+ float scaling = 1.0,
+ int scale_id=0
+ );
+
+ Mat R;
+ Point2f t;
+ float scale;
+ int model_scale_idx;
+
+ };
+
+ /**
+ * \brief Data container for the facemark::getData function
+ */
+ struct CV_EXPORTS Data
+ {
+ std::vector s0;
+ };
+
+ /**
+ * \brief The model of AAM Algorithm
+ */
+ struct CV_EXPORTS Model
+ {
+ int npts; //!< unused delete
+ int max_n; //!< unused delete
+ std::vectorscales;
+ //!< defines the scales considered to build the model
+
+ /*warping*/
+ std::vector triangles;
+ //!< each element contains 3 values, represent index of facemarks that construct one triangle (obtained using delaunay triangulation)
+
+ struct Texture{
+ int max_m; //!< unused delete
+ Rect resolution;
+ //!< resolution of the current scale
+ Mat A;
+ //!< gray values from all face region in the dataset, projected in PCA space
+ Mat A0;
+ //!< average of gray values from all face region in the dataset
+ Mat AA;
+ //!< gray values from all erorded face region in the dataset, projected in PCA space
+ Mat AA0;
+ //!< average of gray values from all erorded face region in the dataset
+
+ std::vector > textureIdx;
+ //!< index for warping of each delaunay triangle region constructed by 3 facemarks
+ std::vector base_shape;
+ //!< basic shape, normalized to be fit in an image with current detection resolution
+ std::vector ind1;
+ //!< index of pixels for mapping process to obtains the grays values of face region
+ std::vector ind2;
+ //!< index of pixels for mapping process to obtains the grays values of eroded face region
+ };
+ std::vector textures;
+ //!< a container to holds the texture data for each scale of fitting
+
+ /*shape*/
+ std::vector s0;
+ //!< the basic shape obtained from training dataset
+ Mat S,Q;
+ //!< the encoded shapes from training data
+
+ };
+
+ //!< initializer
+ static Ptr create(const FacemarkAAM::Params ¶meters = FacemarkAAM::Params() );
+ virtual ~FacemarkAAM() {}
+
+ }; /* AAM */
+
+//! @}
+
+} /* namespace face */
+} /* namespace cv */
+#endif
diff --git a/modules/face/include/opencv2/face/facemarkLBF.hpp b/modules/face/include/opencv2/face/facemarkLBF.hpp
new file mode 100644
index 000000000..682d586c6
--- /dev/null
+++ b/modules/face/include/opencv2/face/facemarkLBF.hpp
@@ -0,0 +1,120 @@
+/*
+By downloading, copying, installing or using the software you agree to this
+license. If you do not agree to this license, do not download, install,
+copy or use the software.
+ License Agreement
+ For Open Source Computer Vision Library
+ (3-clause BSD License)
+Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+Third party copyrights are property of their respective owners.
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ * Neither the names of the copyright holders nor the names of the contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+This software is provided by the copyright holders and contributors "as is" and
+any express or implied warranties, including, but not limited to, the implied
+warranties of merchantability and fitness for a particular purpose are
+disclaimed. In no event shall copyright holders or contributors be liable for
+any direct, indirect, incidental, special, exemplary, or consequential damages
+(including, but not limited to, procurement of substitute goods or services;
+loss of use, data, or profits; or business interruption) however caused
+and on any theory of liability, whether in contract, strict liability,
+or tort (including negligence or otherwise) arising in any way out of
+the use of this software, even if advised of the possibility of such damage.
+
+This file was part of GSoC Project: Facemark API for OpenCV
+Final report: https://gist.github.com/kurnianggoro/74de9121e122ad0bd825176751d47ecc
+Student: Laksono Kurnianggoro
+Mentor: Delia Passalacqua
+*/
+
+#ifndef __OPENCV_FACEMARK_LBF_HPP__
+#define __OPENCV_FACEMARK_LBF_HPP__
+
+#include "opencv2/face/facemark.hpp"
+
+namespace cv {
+namespace face {
+
+//! @addtogroup face
+//! @{
+
+ class CV_EXPORTS_W FacemarkLBF : public Facemark
+ {
+ public:
+ struct CV_EXPORTS Params
+ {
+ /**
+ * \brief Constructor
+ */
+ Params();
+
+ double shape_offset;
+ //!< offset for the loaded face landmark points
+ String cascade_face;
+ //!< filename of the face detector model
+ bool verbose;
+ //!< show the training print-out
+
+ int n_landmarks;
+ //!< number of landmark points
+ int initShape_n;
+ //!< multiplier for augment the training data
+
+ int stages_n;
+ //!< number of refinement stages
+ int tree_n;
+ //!< number of tree in the model for each landmark point refinement
+ int tree_depth;
+ //!< the depth of decision tree, defines the size of feature
+ double bagging_overlap;
+ //!< overlap ratio for training the LBF feature
+
+ std::string model_filename;
+ //!< filename where the trained model will be saved
+ bool save_model; //!< flag to save the trained model or not
+ unsigned int seed; //!< seed for shuffling the training data
+
+ std::vector feats_m;
+ std::vector radius_m;
+ std::vector pupils[2];
+ //!< index of facemark points on pupils of left and right eye
+
+ Rect detectROI;
+
+ void read(const FileNode& /*fn*/);
+ void write(FileStorage& /*fs*/) const;
+
+ };
+
+ class BBox {
+ public:
+ BBox();
+ ~BBox();
+ BBox(double x, double y, double w, double h);
+
+ cv::Mat project(const cv::Mat &shape) const;
+ cv::Mat reproject(const cv::Mat &shape) const;
+
+ double x, y;
+ double x_center, y_center;
+ double x_scale, y_scale;
+ double width, height;
+ };
+
+ static Ptr create(const FacemarkLBF::Params ¶meters = FacemarkLBF::Params() );
+ virtual ~FacemarkLBF(){};
+ }; /* LBF */
+
+//! @}
+
+} /* namespace face */
+}/* namespace cv */
+
+#endif
diff --git a/modules/face/samples/facemark_demo_aam.cpp b/modules/face/samples/facemark_demo_aam.cpp
new file mode 100644
index 000000000..f1702d11b
--- /dev/null
+++ b/modules/face/samples/facemark_demo_aam.cpp
@@ -0,0 +1,318 @@
+/*
+By downloading, copying, installing or using the software you agree to this
+license. If you do not agree to this license, do not download, install,
+copy or use the software.
+ License Agreement
+ For Open Source Computer Vision Library
+ (3-clause BSD License)
+Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+Third party copyrights are property of their respective owners.
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ * Neither the names of the copyright holders nor the names of the contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+This software is provided by the copyright holders and contributors "as is" and
+any express or implied warranties, including, but not limited to, the implied
+warranties of merchantability and fitness for a particular purpose are
+disclaimed. In no event shall copyright holders or contributors be liable for
+any direct, indirect, incidental, special, exemplary, or consequential damages
+(including, but not limited to, procurement of substitute goods or services;
+loss of use, data, or profits; or business interruption) however caused
+and on any theory of liability, whether in contract, strict liability,
+or tort (including negligence or otherwise) arising in any way out of
+the use of this software, even if advised of the possibility of such damage.
+
+This file was part of GSoC Project: Facemark API for OpenCV
+Final report: https://gist.github.com/kurnianggoro/74de9121e122ad0bd825176751d47ecc
+Student: Laksono Kurnianggoro
+Mentor: Delia Passalacqua
+*/
+
+/*----------------------------------------------
+ * Usage:
+ * facemark_demo_aam [test_files]
+ *
+ * Example:
+ * facemark_demo_aam ../face_cascade.xml ../eyes_cascade.xml ../images_train.txt ../points_train.txt ../test.txt
+ *
+ * Notes:
+ * the user should provides the list of training images_train
+ * accompanied by their corresponding landmarks location in separated files.
+ * example of contents for images_train.txt:
+ * ../trainset/image_0001.png
+ * ../trainset/image_0002.png
+ * example of contents for points_train.txt:
+ * ../trainset/image_0001.pts
+ * ../trainset/image_0002.pts
+ * where the image_xxxx.pts contains the position of each face landmark.
+ * example of the contents:
+ * version: 1
+ * n_points: 68
+ * {
+ * 115.167660 220.807529
+ * 116.164839 245.721357
+ * 120.208690 270.389841
+ * ...
+ * }
+ * example of the dataset is available at https://ibug.doc.ic.ac.uk/download/annotations/lfpw.zip
+ *--------------------------------------------------*/
+
+ #include
+ #include
+ #include
+ #include "opencv2/core.hpp"
+ #include "opencv2/highgui.hpp"
+ #include "opencv2/imgproc.hpp"
+ #include "opencv2/face.hpp"
+
+ #include
+ #include
+ #include
+
+ using namespace std;
+ using namespace cv;
+ using namespace cv::face;
+
+ bool myDetector( InputArray image, OutputArray ROIs, CascadeClassifier face_cascade);
+ bool getInitialFitting(Mat image, Rect face, std::vector s0,
+ CascadeClassifier eyes_cascade, Mat & R, Point2f & Trans, float & scale);
+ bool parseArguments(int argc, char** argv, CommandLineParser & , String & cascade,
+ String & model, String & images, String & annotations, String & testImages
+ );
+
+ int main(int argc, char** argv )
+ {
+ CommandLineParser parser(argc, argv,"");
+ String cascade_path,eyes_cascade_path,images_path, annotations_path, test_images_path;
+ if(!parseArguments(argc, argv, parser,cascade_path,eyes_cascade_path,images_path, annotations_path, test_images_path))
+ return -1;
+
+ //! [instance_creation]
+ /*create the facemark instance*/
+ FacemarkAAM::Params params;
+ params.scales.push_back(2.0);
+ params.scales.push_back(4.0);
+ params.model_filename = "AAM.yaml";
+ Ptr facemark = FacemarkAAM::create(params);
+ //! [instance_creation]
+
+ //! [load_dataset]
+ /*Loads the dataset*/
+ std::vector images_train;
+ std::vector landmarks_train;
+ loadDatasetList(images_path,annotations_path,images_train,landmarks_train);
+ //! [load_dataset]
+
+ //! [add_samples]
+ Mat image;
+ std::vector facial_points;
+ for(size_t i=0;iaddTrainingSample(image, facial_points);
+ }
+ //! [add_samples]
+
+ //! [training]
+ /* trained model will be saved to AAM.yml */
+ facemark->training();
+ //! [training]
+
+ //! [load_test_images]
+ /*test using some images*/
+ String testFiles(images_path), testPts(annotations_path);
+ if(!test_images_path.empty()){
+ testFiles = test_images_path;
+ testPts = test_images_path; //unused
+ }
+ std::vector images;
+ std::vector facePoints;
+ loadDatasetList(testFiles, testPts, images, facePoints);
+ //! [load_test_images]
+
+ //! [trainsformation_variables]
+ float scale ;
+ Point2f T;
+ Mat R;
+ //! [trainsformation_variables]
+
+ //! [base_shape]
+ FacemarkAAM::Data data;
+ facemark->getData(&data);
+ std::vector s0 = data.s0;
+ //! [base_shape]
+
+ //! [fitting]
+ /*fitting process*/
+ std::vector faces;
+ //! [load_cascade_models]
+ CascadeClassifier face_cascade(cascade_path);
+ CascadeClassifier eyes_cascade(eyes_cascade_path);
+ //! [load_cascade_models]
+ for(int i=0;i<(int)images.size();i++){
+ printf("image #%i ", i);
+ //! [detect_face]
+ image = imread(images[i]);
+ myDetector(image, faces, face_cascade);
+ //! [detect_face]
+ if(faces.size()>0){
+ //! [get_initialization]
+ std::vector conf;
+ std::vector faces_eyes;
+ for(unsigned j=0;j0){
+ printf(" - face with eyes found %i ", (int)conf.size());
+ std::vector > landmarks;
+ double newtime = (double)getTickCount();
+ facemark->fit(image, faces_eyes, landmarks, (void*)&conf);
+ double fittime = ((getTickCount() - newtime)/getTickFrequency());
+ for(unsigned j=0;j & faces = *(std::vector*) ROIs.getObj();
+ faces.clear();
+
+ if(image.channels()>1){
+ cvtColor(image.getMat(),gray,CV_BGR2GRAY);
+ }else{
+ gray = image.getMat().clone();
+ }
+ equalizeHist( gray, gray );
+
+ face_cascade.detectMultiScale( gray, faces, 1.2, 2, CV_HAAR_SCALE_IMAGE, Size(30, 30) );
+ return true;
+ }
+
+ bool getInitialFitting(Mat image, Rect face, std::vector s0 ,CascadeClassifier eyes_cascade, Mat & R, Point2f & Trans, float & scale){
+ std::vector mybase;
+ std::vector T;
+ std::vector base = Mat(Mat(s0)+Scalar(image.cols/2,image.rows/2)).reshape(2);
+
+ std::vector base_shape,base_shape2 ;
+ Point2f e1 = Point2f((float)((base[39].x+base[36].x)/2.0),(float)((base[39].y+base[36].y)/2.0)); //eye1
+ Point2f e2 = Point2f((float)((base[45].x+base[42].x)/2.0),(float)((base[45].y+base[42].y)/2.0)); //eye2
+
+ if(face.width==0 || face.height==0) return false;
+
+ std::vector eye;
+ bool found=false;
+
+ Mat faceROI = image( face);
+ std::vector eyes;
+
+ //-- In each face, detect eyes
+ eyes_cascade.detectMultiScale( faceROI, eyes, 1.1, 2, 0 |CV_HAAR_SCALE_IMAGE, Size(20, 20) );
+ if(eyes.size()==2){
+ found = true;
+ int j=0;
+ Point2f c1( (float)(face.x + eyes[j].x + eyes[j].width*0.5), (float)(face.y + eyes[j].y + eyes[j].height*0.5));
+
+ j=1;
+ Point2f c2( (float)(face.x + eyes[j].x + eyes[j].width*0.5), (float)(face.y + eyes[j].y + eyes[j].height*0.5));
+
+ Point2f pivot;
+ double a0,a1;
+ if(c1.x("face-cascade"));
+ model = String(parser.get("eyes-cascade"));
+ images = String(parser.get("images"));
+ annotations = String(parser.get("annotations"));
+ test_images = String(parser.get("test-images"));
+
+ if(cascade.empty() || model.empty() || images.empty() || annotations.empty()){
+ std::cerr << "one or more required arguments are not found" << '\n';
+ cout<<"face-cascade : "< [test_files]
+ *
+ * Example:
+ * facemark_demo_lbf ../face_cascade.xml ../LBF.model ../images_train.txt ../points_train.txt ../test.txt
+ *
+ * Notes:
+ * the user should provides the list of training images_train
+ * accompanied by their corresponding landmarks location in separated files.
+ * example of contents for images_train.txt:
+ * ../trainset/image_0001.png
+ * ../trainset/image_0002.png
+ * example of contents for points_train.txt:
+ * ../trainset/image_0001.pts
+ * ../trainset/image_0002.pts
+ * where the image_xxxx.pts contains the position of each face landmark.
+ * example of the contents:
+ * version: 1
+ * n_points: 68
+ * {
+ * 115.167660 220.807529
+ * 116.164839 245.721357
+ * 120.208690 270.389841
+ * ...
+ * }
+ * example of the dataset is available at https://ibug.doc.ic.ac.uk/download/annotations/ibug.zip
+ *--------------------------------------------------*/
+
+ #include
+ #include
+ #include
+ #include
+ #include "opencv2/core.hpp"
+ #include "opencv2/highgui.hpp"
+ #include "opencv2/imgproc.hpp"
+ #include "opencv2/face.hpp"
+
+ using namespace std;
+ using namespace cv;
+ using namespace cv::face;
+
+ CascadeClassifier face_cascade;
+ bool myDetector( InputArray image, OutputArray roi, void * config=0 );
+ bool parseArguments(int argc, char** argv, CommandLineParser & , String & cascade,
+ String & model, String & images, String & annotations, String & testImages
+ );
+
+ int main(int argc, char** argv)
+ {
+ CommandLineParser parser(argc, argv,"");
+ String cascade_path,model_path,images_path, annotations_path, test_images_path;
+ if(!parseArguments(argc, argv, parser,cascade_path,model_path,images_path, annotations_path, test_images_path))
+ return -1;
+
+ /*create the facemark instance*/
+ FacemarkLBF::Params params;
+ params.model_filename = model_path;
+ params.cascade_face = cascade_path;
+ Ptr facemark = FacemarkLBF::create(params);
+
+ face_cascade.load(params.cascade_face.c_str());
+ facemark->setFaceDetector(myDetector);
+
+ /*Loads the dataset*/
+ std::vector images_train;
+ std::vector landmarks_train;
+ loadDatasetList(images_path,annotations_path,images_train,landmarks_train);
+
+ Mat image;
+ std::vector facial_points;
+ for(size_t i=0;iaddTrainingSample(image, facial_points);
+ }
+
+ /*train the Algorithm*/
+ facemark->training();
+
+ /*test using some images*/
+ String testFiles(images_path), testPts(annotations_path);
+ if(!test_images_path.empty()){
+ testFiles = test_images_path;
+ testPts = test_images_path; //unused
+ }
+ std::vector images;
+ std::vector facePoints;
+ loadDatasetList(testFiles, testPts, images, facePoints);
+
+ std::vector rects;
+ CascadeClassifier cc(params.cascade_face.c_str());
+ for(size_t i=0;i > landmarks;
+ cout<getFaces(img, rects);
+ facemark->fit(img, rects, landmarks);
+
+ for(size_t j=0;j0){
+ cout< & faces = *(std::vector*) roi.getObj();
+ faces.clear();
+
+ if(config!=0){
+ //do nothing
+ }
+
+ if(image.channels()>1){
+ cvtColor(image,gray,CV_BGR2GRAY);
+ }else{
+ gray = image.getMat().clone();
+ }
+ equalizeHist( gray, gray );
+
+ face_cascade.detectMultiScale( gray, faces, 1.4, 2, CV_HAAR_SCALE_IMAGE, Size(30, 30) );
+
+ return true;
+ }
+
+ bool parseArguments(int argc, char** argv, CommandLineParser & parser,
+ String & cascade,
+ String & model,
+ String & images,
+ String & annotations,
+ String & test_images
+ ){
+ const String keys =
+ "{ @c cascade | | (required) path to the face cascade xml file fo the face detector }"
+ "{ @i images | | (required) path of a text file contains the list of paths to all training images}"
+ "{ @a annotations | | (required) Path of a text file contains the list of paths to all annotations files}"
+ "{ @m model | | (required) path to save the trained model }"
+ "{ t test-images | | Path of a text file contains the list of paths to the test images}"
+ "{ help h usage ? | | facemark_demo_lbf -cascade -images -annotations -model [-t] \n"
+ " example: facemark_demo_lbf ../face_cascade.xml ../images_train.txt ../points_train.txt ../lbf.model}"
+ ;
+ parser = CommandLineParser(argc, argv,keys);
+ parser.about("hello");
+
+ if (parser.has("help")){
+ parser.printMessage();
+ return false;
+ }
+
+ cascade = String(parser.get("cascade"));
+ model = String(parser.get("model"));
+ images = String(parser.get("images"));
+ annotations = String(parser.get("annotations"));
+ test_images = String(parser.get("t"));
+
+ cout<<"cascade : "<
+ *
+ * example:
+ * facemark_lbf_fitting ../face_cascade.xml ../LBF.model ../video.mp4
+ *
+ * note: do not forget to provide the LBF_MODEL and DETECTOR_MODEL
+ * the model are available at opencv_contrib/modules/face/data/
+ *--------------------------------------------------*/
+
+#include
+#include
+ #include
+#include "opencv2/core.hpp"
+#include "opencv2/highgui.hpp"
+#include "opencv2/imgproc.hpp"
+#include "opencv2/face.hpp"
+
+using namespace std;
+using namespace cv;
+using namespace cv::face;
+
+CascadeClassifier face_cascade;
+bool myDetector( InputArray image, OutputArray ROIs, void * config = 0);
+bool parseArguments(int argc, char** argv, CommandLineParser & parser,
+ String & cascade, String & model,String & video);
+
+int main(int argc, char** argv ){
+ CommandLineParser parser(argc, argv,"");
+ String cascade_path,model_path,images_path, video_path;
+ if(!parseArguments(argc, argv, parser,cascade_path,model_path,video_path))
+ return -1;
+
+ face_cascade.load(cascade_path);
+
+ FacemarkLBF::Params params;
+ params.model_filename = model_path;
+ params.cascade_face = cascade_path;
+
+ Ptr facemark = FacemarkLBF::create(params);
+ facemark->setFaceDetector(myDetector);
+ facemark->loadModel(params.model_filename.c_str());
+
+ VideoCapture capture(video_path);
+ Mat frame;
+
+ if( !capture.isOpened() ){
+ printf("Error when reading vide\n");
+ return 0;
+ }
+
+ Mat img;
+ String text;
+ char buff[255];
+ double fittime;
+ int nfaces;
+ std::vector rects,rects_scaled;
+ std::vector > landmarks;
+ CascadeClassifier cc(params.cascade_face.c_str());
+ namedWindow( "w", 1);
+ for( ; ; )
+ {
+ capture >> frame;
+ if(frame.empty())
+ break;
+
+ double __time__ = (double)getTickCount();
+
+ float scale = (float)(400.0/frame.cols);
+ resize(frame, img, Size((int)(frame.cols*scale), (int)(frame.rows*scale)));
+
+ facemark->getFaces(img, rects);
+ rects_scaled.clear();
+
+ for(int j=0;j<(int)rects.size();j++){
+ rects_scaled.push_back(Rect(
+ (int)(rects[j].x/scale),
+ (int)(rects[j].y/scale),
+ (int)(rects[j].width/scale),
+ (int)(rects[j].height/scale)));
+ }
+ rects = rects_scaled;
+ fittime=0;
+ nfaces = (int)rects.size();
+ if(rects.size()>0){
+ double newtime = (double)getTickCount();
+
+ facemark->fit(frame, rects, landmarks);
+
+
+ fittime = ((getTickCount() - newtime)/getTickFrequency());
+ for(int j=0;j<(int)rects.size();j++){
+ landmarks[j] = Mat(Mat(landmarks[j]));
+ drawFacemarks(frame, landmarks[j], Scalar(0,0,255));
+ }
+ }
+
+
+ double fps = (getTickFrequency()/(getTickCount() - __time__));
+ sprintf(buff, "faces: %i %03.2f fps, fit:%03.0f ms",nfaces,fps,fittime*1000);
+ text = buff;
+ putText(frame, text, Point(20,40), FONT_HERSHEY_PLAIN , 2.0,Scalar::all(255), 2, 8);
+
+ imshow("w", frame);
+ waitKey(1); // waits to display frame
+ }
+ waitKey(0); // key press to close window
+}
+
+bool myDetector( InputArray image, OutputArray ROIs, void * config ){
+ Mat gray;
+ std::vector & faces = *(std::vector*) ROIs.getObj();
+ faces.clear();
+
+ if(config!=0){
+ //do nothing
+ }
+
+ if(image.channels()>1){
+ cvtColor(image.getMat(),gray,CV_BGR2GRAY);
+ }else{
+ gray = image.getMat().clone();
+ }
+ equalizeHist( gray, gray );
+
+ face_cascade.detectMultiScale( gray, faces, 1.4, 2, CV_HAAR_SCALE_IMAGE, Size(30, 30) );
+ return true;
+}
+
+bool parseArguments(int argc, char** argv, CommandLineParser & parser,
+ String & cascade,
+ String & model,
+ String & video
+){
+ const String keys =
+ "{ @c cascade | | (required) path to the cascade model file for the face detector }"
+ "{ @m model | | (required) path to the trained model }"
+ "{ @v video | | (required) path input video}"
+ "{ help h usage ? | | facemark_lbf_fitting -cascade -model -video [-t]\n"
+ " example: facemark_lbf_fitting ../face_cascade.xml ../LBF.model ../video.mp4}"
+ ;
+ parser = CommandLineParser(argc, argv,keys);
+ parser.about("hello");
+
+ if (parser.has("help")){
+ parser.printMessage();
+ return false;
+ }
+
+ cascade = String(parser.get("cascade"));
+ model = String(parser.get("model"));
+ video = String(parser.get("video"));
+
+
+ if(cascade.empty() || model.empty() || video.empty() ){
+ std::cerr << "one or more required arguments are not found" << '\n';
+ cout<<"cascade : "<
+#include
+#include
+#include /* atoi */
+
+namespace cv {
+namespace face {
+ CParams::CParams(String s, double sf, int minN, Size minSz, Size maxSz){
+ cascade = s;
+ scaleFactor = sf;
+ minNeighbors = minN;
+ minSize = minSz;
+ maxSize = maxSz;
+ }
+
+ bool getFaces(InputArray image, OutputArray faces, void * parameters){
+ Mat gray;
+ std::vector roi;
+
+ if(parameters!=0){
+ CParams * params = (CParams *)parameters;
+ cvtColor( image.getMat(), gray, CV_BGR2GRAY );
+ equalizeHist( gray, gray );
+
+ CascadeClassifier face_cascade;
+ if( !face_cascade.load( params->cascade ) ){ printf("--(!)Error loading face_cascade\n"); return false; };
+ face_cascade.detectMultiScale( gray, roi, params->scaleFactor, params->minNeighbors, 0|CV_HAAR_SCALE_IMAGE, params->minSize, params->maxSize);
+
+ Mat(roi).copyTo(faces);
+ return true;
+ }else{
+ return false;
+ }
+
+ }
+
+ bool loadDatasetList(String imageList, String groundTruth, std::vector & images, std::vector & landmarks){
+ std::string line;
+
+ /*clear the output containers*/
+ images.clear();
+ landmarks.clear();
+
+ /*open the files*/
+ std::ifstream infile;
+ infile.open(imageList.c_str(), std::ios::in);
+ std::ifstream ss_gt;
+ ss_gt.open(groundTruth.c_str(), std::ios::in);
+ if ((!infile) || !(ss_gt)) {
+ printf("No valid input file was given, please check the given filename.\n");
+ return false;
+ }
+
+ /*load the images path*/
+ while (getline (infile, line)){
+ images.push_back(line);
+ }
+
+ /*load the points*/
+ while (getline (ss_gt, line)){
+ landmarks.push_back(line);
+ }
+
+ return true;
+ }
+
+ bool loadTrainingData(String filename, std::vector & images, OutputArray _facePoints, char delim, float offset){
+ std::string line;
+ std::string item;
+ std::vector pts;
+ std::vector raw;
+
+ std::vector > & facePoints =
+ *(std::vector >*) _facePoints.getObj();
+
+ std::ifstream infile;
+ infile.open(filename.c_str(), std::ios::in);
+ if (!infile) {
+ std::string error_message = "No valid input file was given, please check the given filename.";
+ CV_Error(CV_StsBadArg, error_message);
+ }
+
+ /*clear the output containers*/
+ images.clear();
+ facePoints.clear();
+
+ /*the main loading process*/
+ while (getline (infile, line)){
+ std::istringstream ss(line); // string stream for the current line
+
+ /*pop the image path*/
+ getline (ss, item, delim);
+ images.push_back(item);
+
+ /*load all numbers*/
+ raw.clear();
+ while (getline (ss, item, delim)){
+ raw.push_back((float)atof(item.c_str()));
+ }
+
+ /*convert to opencv points*/
+ pts.clear();
+ for(unsigned i = 0;i< raw.size();i+=2){
+ pts.push_back(Point2f(raw[i]+offset,raw[i+1]+offset));
+ }
+ facePoints.push_back(pts);
+ } // main loading process
+
+ return true;
+ }
+
+ bool loadTrainingData(String imageList, String groundTruth, std::vector & images, OutputArray _facePoints, float offset){
+ std::string line;
+ std::vector facePts;
+
+ std::vector > & facePoints =
+ *(std::vector >*) _facePoints.getObj();
+
+ /*clear the output containers*/
+ images.clear();
+ facePoints.clear();
+
+ /*load the images path*/
+ std::ifstream infile;
+ infile.open(imageList.c_str(), std::ios::in);
+ if (!infile) {
+ std::string error_message = "No valid input file was given, please check the given filename.";
+ CV_Error(CV_StsBadArg, error_message);
+ }
+
+ while (getline (infile, line)){
+ images.push_back(line);
+ }
+
+ /*load the points*/
+ std::ifstream ss_gt(groundTruth.c_str());
+ while (getline (ss_gt, line)){
+ facePts.clear();
+ loadFacePoints(line, facePts, offset);
+ facePoints.push_back(facePts);
+ }
+
+ return true;
+ }
+
+ bool loadFacePoints(String filename, OutputArray points, float offset){
+ std::vector & pts = *(std::vector *)points.getObj();
+
+ std::string line, item;
+ std::ifstream infile(filename.c_str());
+
+ /*pop the version*/
+ std::getline(infile, line);
+ CV_Assert(line.compare(0,7,"version")==0);
+
+ /*pop the number of points*/
+ std::getline(infile, line);
+ CV_Assert(line.compare(0,8,"n_points")==0);
+
+ /*get the number of points*/
+ std::string item_npts;
+ int npts;
+
+ std::istringstream linestream(line);
+ linestream>>item_npts>>npts;
+
+ /*pop out '{' character*/
+ std::getline(infile, line);
+
+ /*main process*/
+ int cnt = 0;
+ std::string x, y;
+ pts.clear();
+ while (std::getline(infile, line) && cnt>x>>y;
+ pts.push_back(Point2f((float)atof(x.c_str())+offset,(float)atof(y.c_str())+offset));
+
+ }
+
+ return true;
+ }
+
+ void drawFacemarks(InputOutputArray image, InputArray points, Scalar color){
+ Mat img = image.getMat();
+ std::vector pts = *(std::vector*)points.getObj();
+ for(size_t i=0;i> model_filename;
+
+ if (!fn["m"].empty()) fn["m"] >> m;
+ if (!fn["n"].empty()) fn["n"] >> m;
+ if (!fn["n_iter"].empty()) fn["n_iter"] >> m;
+ if (!fn["verbose"].empty()) fn["verbose"] >> m;
+ if (!fn["max_m"].empty()) fn["max_m"] >> m;
+ if (!fn["max_n"].empty()) fn["max_n"] >> m;
+ if (!fn["texture_max_m"].empty()) fn["texture_max_m"] >> m;
+ if (!fn["scales"].empty()) fn["scales"] >> m;
+ }
+
+ void FacemarkAAM::Params::write( cv::FileStorage& fs ) const{
+ fs << "model_filename" << model_filename;
+ fs << "m" << m;
+ fs << "n" << n;
+ fs << "n_iter" << n_iter;
+ fs << "verbose" << verbose;
+ fs << "max_m" << verbose;
+ fs << "max_n" << verbose;
+ fs << "texture_max_m" << verbose;
+ fs << "scales" << verbose;
+ }
+
+ class FacemarkAAMImpl : public FacemarkAAM {
+ public:
+ FacemarkAAMImpl( const FacemarkAAM::Params ¶meters = FacemarkAAM::Params() );
+ void read( const FileNode& /*fn*/ );
+ void write( FileStorage& /*fs*/ ) const;
+
+ void saveModel(String fs);
+ void loadModel(String fs);
+
+ bool setFaceDetector(bool(*f)(InputArray , OutputArray, void * ));
+ bool getFaces( InputArray image ,OutputArray faces, void * extra_params);
+
+ bool getData(void * items);
+
+ protected:
+
+ bool fit( InputArray image, InputArray faces, InputOutputArray landmarks, void * runtime_params);//!< from many ROIs
+ bool fitImpl( const Mat image, std::vector& landmarks,const Mat R,const Point2f T,const float scale, const int sclIdx=0 );
+
+ bool addTrainingSample(InputArray image, InputArray landmarks);
+ void training(void* parameters);
+
+ Mat procrustes(std::vector , std::vector , Mat & , Scalar & , float & );
+ void calcMeanShape(std::vector > ,std::vector & );
+ void procrustesAnalysis(std::vector > , std::vector > & , std::vector & );
+
+ inline Mat linearize(Mat );
+ inline Mat linearize(std::vector );
+ void getProjection(const Mat , Mat &, int );
+ void calcSimilarityEig(std::vector ,Mat , Mat & , Mat & );
+ Mat orthonormal(Mat );
+ void delaunay(std::vector , std::vector & );
+ Mat createMask(std::vector , Rect );
+ Mat createTextureBase(std::vector , std::vector , Rect , std::vector > & );
+ Mat warpImage(const Mat ,const std::vector ,const std::vector ,
+ const std::vector , const Rect , const std::vector > );
+ template
+ Mat getFeature(const Mat , std::vector map);
+ void createMaskMapping(const Mat mask, const Mat mask2, std::vector & , std::vector &, std::vector &);
+
+ void warpUpdate(std::vector & shape, Mat delta, std::vector s0, Mat S, Mat Q, std::vector triangles,std::vector > Tp);
+ Mat computeWarpParts(std::vector curr_shape,std::vector s0, Mat ds0, std::vector triangles,std::vector > Tp);
+ void image_jacobian(const Mat gx, const Mat gy, const Mat Jx, const Mat Jy, Mat & G);
+ void gradient(const Mat M, Mat & gx, Mat & gy);
+ void createWarpJacobian(Mat S, Mat Q, std::vector , Model::Texture & T, Mat & Wx_dp, Mat & Wy_dp, std::vector > & Tp);
+
+ std::vector images;
+ std::vector > facePoints;
+ FacemarkAAM::Params params;
+ FacemarkAAM::Model AAM;
+ bool(*faceDetector)(InputArray , OutputArray, void *);
+ bool isSetDetector;
+
+ private:
+ bool isModelTrained;
+ };
+
+ /*
+ * Constructor
+ */
+ Ptr FacemarkAAM::create(const FacemarkAAM::Params ¶meters){
+ return Ptr(new FacemarkAAMImpl(parameters));
+ }
+
+ FacemarkAAMImpl::FacemarkAAMImpl( const FacemarkAAM::Params ¶meters ) :
+ params( parameters )
+ {
+ isSetDetector =false;
+ isModelTrained = false;
+ }
+
+ void FacemarkAAMImpl::read( const cv::FileNode& fn ){
+ params.read( fn );
+ }
+
+ void FacemarkAAMImpl::write( cv::FileStorage& fs ) const {
+ params.write( fs );
+ }
+
+ bool FacemarkAAMImpl::setFaceDetector(bool(*f)(InputArray , OutputArray, void *)){
+ faceDetector = f;
+ isSetDetector = true;
+ return true;
+ }
+
+
+ bool FacemarkAAMImpl::getFaces( InputArray image , OutputArray roi, void * extra_params){
+
+ if(!isSetDetector){
+ return false;
+ }
+
+ if(extra_params!=0){
+ //do nothing
+ }
+
+ std::vector faces;
+ faces.clear();
+
+ faceDetector(image.getMat(), faces, extra_params);
+ Mat(faces).copyTo(roi);
+ return true;
+ }
+
+ bool FacemarkAAMImpl::getData(void * items){
+ if(items==0){
+ return true;
+ }else{
+ Data * data = (Data*)items;
+ data->s0 = AAM.s0;
+ return true;
+ }
+ }
+
+ bool FacemarkAAMImpl::addTrainingSample(InputArray image, InputArray landmarks){
+ std::vector & _landmarks = *(std::vector*)landmarks.getObj();
+
+ images.push_back(image.getMat());
+ facePoints.push_back(_landmarks);
+
+ return true;
+ }
+
+ void FacemarkAAMImpl::training(void* parameters){
+ if(parameters!=0){/*do nothing*/}
+ if (images.size()<1) {
+ std::string error_message =
+ "Training data is not provided. Consider to add using addTrainingSample() function!";
+ CV_Error(CV_StsBadArg, error_message);
+ }
+
+ if(strcmp(params.model_filename.c_str(),"")==0 && params.save_model){
+ std::string error_message = "The model_filename parameter should be set!";
+ CV_Error(CV_StsBadArg, error_message);
+ }
+
+ std::vector > normalized;
+ Mat erode_kernel = getStructuringElement(MORPH_RECT, Size(3,3), Point(1,1));
+ Mat image;
+
+ int param_max_m = params.max_m;//550;
+ int param_max_n = params.max_n;//136;
+
+ AAM.scales = params.scales;
+ AAM.textures.resize(AAM.scales.size());
+
+ /*-------------- A. Load the training data---------*/
+ procrustesAnalysis(facePoints, normalized,AAM.s0);
+
+ /*-------------- B. Create the shape model---------*/
+ Mat s0_lin = linearize(AAM.s0).t() ;
+ // linearize all shapes data, all x and then all y for each shape
+ Mat M;
+ for(unsigned i=0;i s0_scaled = s0_scaled_m.reshape(2); //convert to points
+
+ /*get the min and max of x and y coordinate*/
+ double min_x, max_x, min_y, max_y;
+ s0_scaled_m = s0_scaled_m.reshape(1);
+ Mat s0_scaled_x = s0_scaled_m.col(0);
+ Mat s0_scaled_y = s0_scaled_m.col(1);
+ minMaxIdx(s0_scaled_x, &min_x, &max_x);
+ minMaxIdx(s0_scaled_y, &min_y, &max_y);
+
+ std::vector base_shape = Mat(Mat(s0_scaled)-Scalar(min_x-2.0,min_y-2.0)).reshape(2);
+ AAM.textures[scale].base_shape = base_shape;
+ AAM.textures[scale].resolution = Rect(0,0,(int)ceil(max_x-min_x+3),(int)ceil(max_y-min_y+3));
+
+ Mat base_texture = createTextureBase(base_shape, AAM.triangles, AAM.textures[scale].resolution, AAM.textures[scale].textureIdx);
+
+ Mat mask1 = base_texture>0;
+ Mat mask2;
+ erode(mask1, mask1, erode_kernel);
+ erode(mask1, mask2, erode_kernel);
+
+ Mat warped;
+ std::vector fe_map;
+ createMaskMapping(mask1,mask2, AAM.textures[scale].ind1, AAM.textures[scale].ind2,fe_map);//ok
+
+ /* ------------ Part D. Get textures -------------*/
+ Mat texture_feats, feat;
+ if(params.verbose) printf("(1/4) Feature extraction ...\n");
+ for(size_t i=0; i(warped, AAM.textures[scale].ind1);
+ texture_feats.push_back(feat.t());
+ }
+ Mat T= texture_feats.t();
+
+ /* -------------- E. Create the texture model -----------------*/
+ reduce(T,AAM.textures[scale].A0,1, CV_REDUCE_AVG);
+
+ if(params.verbose) printf("(2/4) Compute the feature average ...\n");
+ Mat A0_mtx = repeat(AAM.textures[scale].A0,1,T.cols);
+ Mat textures_normalized = T - A0_mtx;
+
+ if(params.verbose) printf("(3/4) Projecting the features ...\n");
+ getProjection(textures_normalized, AAM.textures[scale].A ,param_max_m);
+ AAM.textures[scale].AA0 = getFeature(AAM.textures[scale].A0, fe_map);
+
+ if(params.verbose) printf("(4/4) Extraction of the eroded face features ...\n");
+ Mat U_data, ud;
+ for(int i =0;i(c,fe_map);
+ U_data.push_back(ud.t());
+ }
+ Mat U = U_data.t();
+ AAM.textures[scale].AA = orthonormal(U);
+ } // scale
+
+ images.clear();
+ if(params.save_model){
+ if(params.verbose) printf("Saving the model\n");
+ saveModel(params.model_filename);
+ }
+ isModelTrained = true;
+ if(params.verbose) printf("Training is completed\n");
+ }
+
+ bool FacemarkAAMImpl::fit( InputArray image, InputArray roi, InputOutputArray _landmarks, void * runtime_params)
+ {
+ std::vector & faces = *(std::vector *)roi.getObj();
+ if(faces.size()<1) return false;
+
+ std::vector > & landmarks =
+ *(std::vector >*) _landmarks.getObj();
+ landmarks.resize(faces.size());
+
+ Mat img = image.getMat();
+ if(runtime_params!=0){
+
+ std::vector conf = *(std::vector*)runtime_params;
+ if (conf.size()!=faces.size()) {
+ std::string error_message =
+ "Number of faces and extra_parameters are different!";
+ CV_Error(CV_StsBadArg, error_message);
+ }
+ for(size_t i=0; i& landmarks, const Mat R, const Point2f T, const float scale, int _scl){
+ if (landmarks.size()>0)
+ landmarks.clear();
+
+ CV_Assert(isModelTrained);
+
+ int param_n = params.n, param_m = params.m;
+ int scl = _scl<(int)AAM.scales.size()?_scl:(int)AAM.scales.size();
+
+ /*variables*/
+ std::vector s0 = Mat(Mat(AAM.s0)/AAM.scales[scl]).reshape(2);
+
+ /*pre-computation*/
+ Mat S = Mat(AAM.S, Range::all(), Range(0,param_n>AAM.S.cols?AAM.S.cols:param_n)).clone(); // chop the shape data
+ std::vector > Tp;
+ Mat Wx_dp, Wy_dp;
+ createWarpJacobian(S, AAM.Q, AAM.triangles, AAM.textures[scl],Wx_dp, Wy_dp, Tp);
+
+ std::vector s0_init = Mat(Mat(R*scale*AAM.scales[scl]*Mat(Mat(s0).reshape(1)).t()).t()).reshape(2);
+ std::vector curr_shape = Mat(Mat(s0_init)+Scalar(T.x,T.y));
+ curr_shape = Mat(1.0/scale*Mat(curr_shape)).reshape(2);
+
+ Mat imgray;
+ Mat img;
+ if(image.channels()>1){
+ cvtColor(image,imgray,CV_BGR2GRAY);
+ }else{
+ imgray = image;
+ }
+
+ resize(imgray,img,Size(int(image.cols/scale),int(image.rows/scale)));// matlab use bicubic interpolation, the result is float numbers
+
+ /*chop the textures model*/
+ int maxCol = param_m;
+ if(AAM.textures[scl].A.cols(warped, AAM.textures[scl].ind1);
+ II = getFeature(warped, AAM.textures[scl].ind2);
+
+ if(t==0){
+ c = A.t()*(I-AAM.textures[scl].A0); //little bit different to matlab, probably due to datatype
+ }else{
+ c = c+dc;
+ }
+
+ Irec_feat = (AAM.textures[scl].A0+A*c);
+ Irec = Mat::zeros(AAM.textures[scl].resolution.width, AAM.textures[scl].resolution.height, CV_32FC1);
+
+ for(int j=0;j<(int)AAM.textures[scl].ind1.size();j++){
+ Irec.at(AAM.textures[scl].ind1[j]) = Irec_feat.at(j);
+ }
+ Mat irec = Irec.t();
+
+ gradient(irec, gx, gy);
+
+ Mat Jc;
+ image_jacobian(Mat(gx.t()).reshape(0,1).t(),Mat(gy.t()).reshape(0,1).t(),Wx_dp, Wy_dp,Jc);
+
+ Mat J;
+ std::vector Irec_vec;
+ for(size_t j=0;j(AAM.textures[scl].ind2[j]));
+ }
+
+ /*compute Jfsic and Hfsic*/
+ Mat Jfsic = J - AA*(AA.t()*J);
+ Mat Hfsic = Jfsic.t()*Jfsic;
+ Mat iHfsic;
+ invert(Hfsic, iHfsic);
+
+ /*compute dp dq and dc*/
+ Mat dqp = iHfsic*Jfsic.t()*(II-AAM.textures[scl].AA0);
+ dc = AA.t()*(II-Mat(Irec_vec)-J*dqp);
+ warpUpdate(curr_shape, dqp, s0,S, AAM.Q, AAM.triangles,Tp);
+ }
+ landmarks = Mat(scale*Mat(curr_shape)).reshape(2);
+ return true;
+ }
+
+ void FacemarkAAMImpl::saveModel(String s){
+ FileStorage fs(s.c_str(),FileStorage::WRITE_BASE64);
+ fs << "AAM_tri" << AAM.triangles;
+ fs << "scales" << AAM.scales;
+ fs << "s0" << AAM.s0;
+ fs << "S" << AAM.S;
+ fs << "Q" << AAM.Q;
+
+ String x;
+ for(int i=0;i< (int)AAM.scales.size();i++){
+ x = cv::format("scale%i_max_m",i);
+ fs << x << AAM.textures[i].max_m;
+
+ x = cv::format("scale%i_resolution",i);
+ fs << x << AAM.textures[i].resolution;
+
+ x = cv::format("scale%i_textureIdx",i);
+ fs << x << AAM.textures[i].textureIdx;
+
+ x = cv::format("scale%i_base_shape",i);
+ fs << x << AAM.textures[i].base_shape;
+
+ x = cv::format("scale%i_A",i);
+ fs << x << AAM.textures[i].A;
+
+ x = cv::format("scale%i_A0",i);
+ fs << x << AAM.textures[i].A0;
+
+ x = cv::format("scale%i_AA",i);
+ fs << x << AAM.textures[i].AA;
+
+ x = cv::format("scale%i_AA0",i);
+ fs << x << AAM.textures[i].AA0;
+
+ x = cv::format("scale%i_ind1",i);
+ fs << x << AAM.textures[i].ind1;
+
+ x = cv::format("scale%i_ind2",i);
+ fs << x << AAM.textures[i].ind2;
+
+ }
+ fs.release();
+ if(params.verbose) printf("The model is successfully saved! \n");
+ }
+
+ void FacemarkAAMImpl::loadModel(String s){
+ FileStorage fs(s.c_str(),FileStorage::READ);
+ String x;
+ fs["AAM_tri"] >> AAM.triangles;
+ fs["scales"] >> AAM.scales;
+ fs["s0"] >> AAM.s0;
+ fs["S"] >> AAM.S;
+ fs["Q"] >> AAM.Q;
+
+
+ AAM.textures.resize(AAM.scales.size());
+ for(int i=0;i< (int)AAM.scales.size();i++){
+ x = cv::format("scale%i_max_m",i);
+ fs[x] >> AAM.textures[i].max_m;
+
+ x = cv::format("scale%i_resolution",i);
+ fs[x] >> AAM.textures[i].resolution;
+
+ x = cv::format("scale%i_textureIdx",i);
+ fs[x] >> AAM.textures[i].textureIdx;
+
+ x = cv::format("scale%i_base_shape",i);
+ fs[x] >> AAM.textures[i].base_shape;
+
+ x = cv::format("scale%i_A",i);
+ fs[x] >> AAM.textures[i].A;
+
+ x = cv::format("scale%i_A0",i);
+ fs[x] >> AAM.textures[i].A0;
+
+ x = cv::format("scale%i_AA",i);
+ fs[x] >> AAM.textures[i].AA;
+
+ x = cv::format("scale%i_AA0",i);
+ fs[x] >> AAM.textures[i].AA0;
+
+ x = cv::format("scale%i_ind1",i);
+ fs[x] >> AAM.textures[i].ind1;
+
+ x = cv::format("scale%i_ind2",i);
+ fs[x] >> AAM.textures[i].ind2;
+ }
+
+ fs.release();
+ isModelTrained = true;
+ if(params.verbose) printf("the model has been loaded\n");
+ }
+
+ Mat FacemarkAAMImpl::procrustes(std::vector P, std::vector Q, Mat & rot, Scalar & trans, float & scale){
+
+ // calculate average
+ Scalar mx = mean(P);
+ Scalar my = mean(Q);
+
+ // zero centered data
+ Mat X0 = Mat(P) - mx;
+ Mat Y0 = Mat(Q) - my;
+
+ // calculate magnitude
+ Mat Xs, Ys;
+ multiply(X0,X0,Xs);
+ multiply(Y0,Y0,Ys);
+
+ // calculate the sum
+ Mat sumXs, sumYs;
+ reduce(Xs,sumXs, 0, CV_REDUCE_SUM);
+ reduce(Ys,sumYs, 0, CV_REDUCE_SUM);
+
+ //calculate the normrnd
+ double normX = sqrt(Mat(sumXs.reshape(1)).at(0)+Mat(sumXs.reshape(1)).at(1));
+ double normY = sqrt(Mat(sumYs.reshape(1)).at(0)+Mat(sumYs.reshape(1)).at(1));
+
+ //normalization
+ X0 = X0/normX;
+ Y0 = Y0/normY;
+
+ //reshape, convert to 2D Matrix
+ Mat Xn=X0.reshape(1);
+ Mat Yn=Y0.reshape(1);
+
+ //calculate the covariance matrix
+ Mat M = Xn.t()*Yn;
+
+ // decompose
+ Mat U,S,Vt;
+ SVD::compute(M, S, U, Vt);
+
+ // extract the transformations
+ scale = (S.at(0)+S.at(1))*(float)normX/(float)normY;
+ rot = Vt.t()*U.t();
+
+ Mat muX(mx),mX; muX.pop_back();muX.pop_back();
+ Mat muY(my),mY; muY.pop_back();muY.pop_back();
+ muX.convertTo(mX,CV_32FC1);
+ muY.convertTo(mY,CV_32FC1);
+
+ Mat t = mX.t()-scale*mY.t()*rot;
+ trans[0] = t.at(0);
+ trans[1] = t.at(1);
+
+ // calculate the recovered form
+ Mat Qmat = Mat(Q).reshape(1);
+
+ return Mat(scale*Qmat*rot+trans).clone();
+ }
+
+ void FacemarkAAMImpl::procrustesAnalysis(std::vector > shapes, std::vector > & normalized, std::vector & new_mean){
+
+ std::vector mean_every_shape;
+ mean_every_shape.resize(shapes.size());
+
+ Point2f temp;
+
+ // calculate the mean of every shape
+ for(size_t i=0; i< shapes.size();i++){
+ mean_every_shape[i] = mean(shapes[i]);
+ }
+
+ //normalize every shapes
+ Mat tShape;
+ normalized.clear();
+ for(size_t i=0; i< shapes.size();i++){
+ normalized.push_back((Mat)(Mat(shapes[i]) - mean_every_shape[i]));
+ }
+
+ // calculate the mean shape
+ std::vector mean_shape;
+ calcMeanShape(normalized, mean_shape);
+
+ // update the mean shape and normalized shapes iteratively
+ int maxIter = 100;
+ Mat R;
+ Scalar t;
+ float s;
+ Mat aligned;
+ for(int i=0;i > shapes,std::vector & mean){
+ mean.resize(shapes[0].size());
+ Point2f tmp;
+ for(unsigned i=0;in)k=n;
+ if(k>M.rows)k=M.rows;
+ if(k>M.cols)k=M.cols;
+
+ // cut the column of eigen vector
+ U.colRange(0,k).copyTo(P);
+ }else{
+ // SVD::compute(M.t()*M, S, U, Vt);
+ eigen(M.t()*M, S, Ut);U=Ut.t();
+
+ // threshold(S,S1,0.00001,1,THRESH_BINARY);
+ k= S.rows; //countNonZero(S1);
+ if(k>n)k=n;
+ if(k>M.rows)k=M.rows;
+ if(k>M.cols)k=M.cols;
+
+ // cut the eigen values to k-amount
+ Mat D = Mat::zeros(k,k,CV_32FC1);
+ Mat diag = D.diag();
+ Mat s; pow(S,-0.5,s);
+ s(Range(0,k), Range::all()).copyTo(diag);
+
+ // cut the eigen vector to k-column,
+ P = Mat(M*U.colRange(0,k)*D).clone();
+
+ }
+ }
+
+ Mat FacemarkAAMImpl::orthonormal(Mat Mo){
+ Mat M;
+ Mo.convertTo(M,CV_32FC1);
+
+ // TODO: float precission is only 1e-7, but MATLAB version use thresh=2.2204e-16
+ float thresh = (float)2.2204e-6;
+
+ Mat O = Mat::zeros(M.rows, M.cols, CV_32FC1);
+
+ int k = 0; //storing index
+
+ Mat w,nv;
+ float n;
+ for(int i=0;ithresh){
+ Mat ok=O.col(k);
+ // nv=v/n;
+ normalize(v,nv);
+ nv.copyTo(ok);
+ k+=1;
+ }
+
+ }
+
+ return O.colRange(0,k).clone();
+ }
+
+ void FacemarkAAMImpl::calcSimilarityEig(std::vector s0,Mat S, Mat & Q_orth, Mat & S_orth){
+ int npts = (int)s0.size();
+
+ Mat Q = Mat::zeros(2*npts,4,CV_32FC1);
+ Mat c0 = Q.col(0);
+ Mat c1 = Q.col(1);
+ Mat c2 = Q.col(2);
+ Mat c3 = Q.col(3);
+
+ /*c0 = s0(:)*/
+ Mat w = linearize(s0);
+ // w.convertTo(w, CV_64FC1);
+ w.copyTo(c0);
+
+ /*c1 = [-s0(npts:2*npts); s0(0:npts-1)]*/
+ Mat s0_mat = Mat(s0).reshape(1);
+ // s0_mat.convertTo(s0_mat, CV_64FC1);
+ Mat swapper = Mat::zeros(2,npts,CV_32FC1);
+ Mat s00 = s0_mat.col(0);
+ Mat s01 = s0_mat.col(1);
+ Mat sw0 = swapper.row(0);
+ Mat sw1 = swapper.row(1);
+ Mat(s00.t()).copyTo(sw1);
+ s01 = -s01;
+ Mat(s01.t()).copyTo(sw0);
+
+ Mat(swapper.reshape(1,2*npts)).copyTo(c1);
+
+ /*c2 - [ones(npts); zeros(npts)]*/
+ Mat ones = Mat::ones(1,npts,CV_32FC1);
+ Mat c2_mat = Mat::zeros(2,npts,CV_32FC1);
+ Mat c20 = c2_mat.row(0);
+ ones.copyTo(c20);
+ Mat(c2_mat.reshape(1,2*npts)).copyTo(c2);
+
+ /*c3 - [zeros(npts); ones(npts)]*/
+ Mat c3_mat = Mat::zeros(2,npts,CV_32FC1);
+ Mat c31 = c3_mat.row(1);
+ ones.copyTo(c31);
+ Mat(c3_mat.reshape(1,2*npts)).copyTo(c3);
+
+ Mat Qo = orthonormal(Q);
+
+ Mat all = Qo.t();
+ all.push_back(S.t());
+
+ Mat allOrth = orthonormal(all.t());
+ Q_orth = allOrth.colRange(0,4).clone();
+ S_orth = allOrth.colRange(4,allOrth.cols).clone();
+
+ }
+
+ inline Mat FacemarkAAMImpl::linearize(Mat s){ // all x values and then all y values
+ return Mat(s.reshape(1).t()).reshape(1,2*s.rows);
+ }
+ inline Mat FacemarkAAMImpl::linearize(std::vector s){ // all x values and then all y values
+ return linearize(Mat(s));
+ }
+
+ void FacemarkAAMImpl::delaunay(std::vector s, std::vector & triangles){
+
+ triangles.clear();
+
+ std::vector idx;
+ std::vector tp;
+
+ double min_x, max_x, min_y, max_y;
+ Mat S = Mat(s).reshape(1);
+ Mat s_x = S.col(0);
+ Mat s_y = S.col(1);
+ minMaxIdx(s_x, &min_x, &max_x);
+ minMaxIdx(s_y, &min_y, &max_y);
+
+ // TODO: set the rectangle as configurable parameter
+ Subdiv2D subdiv(Rect(-500,-500,1000,1000));
+ subdiv.insert(s);
+
+ int a,b;
+ subdiv.locate(s.back(),a,b);
+ idx.resize(b+1);
+
+ Point2f p;
+ for(unsigned i=0;i=min_x && t[0]<=max_x && t[1]>=min_y && t[1]<=max_y
+ && t[2]>=min_x && t[2]<=max_x && t[3]>=min_y && t[3]<=max_y
+ && t[4]>=min_x && t[4]<=max_x && t[5]>=min_y && t[5]<=max_y
+ ){
+ subdiv.locate(Point2f(t[0],t[1]),a,v1);
+ subdiv.locate(Point2f(t[2],t[3]),a,v2);
+ subdiv.locate(Point2f(t[4],t[5]),a,v3);
+ triangles.push_back(Vec3i(idx[v1],idx[v2],idx[v3]));
+ } //if
+ } // for
+ }
+
+ Mat FacemarkAAMImpl::createMask(std::vector base_shape, Rect res){
+ Mat mask = Mat::zeros(res.height, res.width, CV_8U);
+ std::vector hull;
+ std::vector shape;
+ Mat(base_shape).convertTo(shape, CV_32S);
+ convexHull(shape,hull);
+ fillConvexPoly(mask, &hull[0], (int)hull.size(), 255, 8 ,0);
+ return mask.clone();
+ }
+
+ Mat FacemarkAAMImpl::createTextureBase(std::vector shape, std::vector triangles, Rect res, std::vector > & textureIdx){
+ // max supported amount of triangles only 255
+ Mat mask = Mat::zeros(res.height, res.width, CV_8U);
+
+ std::vector p(3);
+ textureIdx.clear();
+ for(size_t i=0;i polygon;
+ approxPolyDP(p,polygon, 1.0, true);
+ fillConvexPoly(mask, &polygon[0], (int)polygon.size(), (double)i+1,8,0 );
+
+ std::vector list;
+ for(int y=0;y(y,x)==(uchar)(i+1)){
+ list.push_back(Point(x,y));
+ }
+ }
+ }
+ textureIdx.push_back(list);
+
+ }
+
+ return mask.clone();
+ }
+
+ Mat FacemarkAAMImpl::warpImage(
+ const Mat img, const std::vector target_shape,
+ const std::vector curr_shape, const std::vector triangles,
+ const Rect res, const std::vector > textureIdx)
+ {
+ // TODO: this part can be optimized, collect tranformation pair form all triangles first, then do one time remapping
+ Mat warped = Mat::zeros(res.height, res.width, CV_8U);
+ Mat warped2 = Mat::zeros(res.height, res.width, CV_8U);
+ Mat image,part, warped_part;
+
+ if(img.channels()>1){
+ cvtColor(img,image,CV_BGR2GRAY);
+ }else{
+ image = img;
+ }
+
+ Mat A,R,t;
+ A = Mat::zeros(2,3,CV_64F);
+ std::vector target(3),source(3);
+ std::vector polygon;
+ for(size_t i=0;i(0) = ((target[2].y-target[0].y)*(source[1].x-source[0].x)-
+ (target[1].y-target[0].y)*(source[2].x-source[0].x))/denominator;
+ A.at(1) = ((target[1].x-target[0].x)*(source[2].x-source[0].x)-
+ (target[2].x-target[0].x)*(source[1].x-source[0].x))/denominator;
+ A.at