Merge pull request #1592 from berak:landmarks

modules/face/include/opencv2/face.hpp

@@ -375,6 +375,7 @@ protected:
 
 #include "opencv2/face/facerec.hpp"
 #include "opencv2/face/facemark.hpp"
+#include "opencv2/face/facemark_train.hpp"
 #include "opencv2/face/facemarkLBF.hpp"
 #include "opencv2/face/facemarkAAM.hpp"
 #include "opencv2/face/face_alignment.hpp"

modules/face/include/opencv2/face/face_alignment.hpp

@@ -4,11 +4,11 @@
 #ifndef __OPENCV_FACE_ALIGNMENT_HPP__
 #define __OPENCV_FACE_ALIGNMENT_HPP__
 
-#include "facemark.hpp"
+#include "opencv2/face/facemark_train.hpp"
 
 namespace cv{
 namespace face{
-class CV_EXPORTS_W FacemarkKazemi : public Algorithm
+class CV_EXPORTS_W FacemarkKazemi : public Facemark
 {
 public:
     struct CV_EXPORTS Params
@@ -39,8 +39,6 @@ public:
     static Ptr<FacemarkKazemi> create(const FacemarkKazemi::Params &parameters = FacemarkKazemi::Params());
     virtual ~FacemarkKazemi();
 
-    /// @brief training the facemark model, input are the file names of image list and landmark annotation
-    virtual void training(String imageList, String groundTruth)=0;
     /** @brief This function is used to train the model using gradient boosting to get a cascade of regressors
     *which can then be used to predict shape.
     *@param images A vector of type cv::Mat which stores the images which are used in training samples.
@@ -51,16 +49,7 @@ public:
     *@returns A boolean value. The function returns true if the model is trained properly or false if it is not trained.
     */
     virtual bool training(std::vector<Mat>& images, std::vector< std::vector<Point2f> >& landmarks,std::string configfile,Size scale,std::string modelFilename = "face_landmarks.dat")=0;
-    /** @brief This function is used to load the trained model..
-    *@param filename A variable of type cv::String which stores the name of the file in which trained model is stored.
-    */
-    virtual void loadModel(String filename)=0;
-    /** @brief This functions retrieves a centered and scaled face shape, according to the bounding rectangle.
-    *@param image A variable of type cv::InputArray which stores the image whose landmarks have to be found
-    *@param faces A variable of type cv::InputArray which stores the bounding boxes of faces found in a given image.
-    *@param landmarks A variable of type cv::InputOutputArray which stores the landmarks of all the faces found in the image
-    */
-    virtual bool fit( InputArray image, InputArray faces, InputOutputArray landmarks )=0;//!< from many ROIs
+
     /// set the custom face detector
     virtual bool setFaceDetector(bool(*f)(InputArray , OutputArray, void*), void* userData)=0;
     /// get faces using the custom detector

modules/face/include/opencv2/face/facemarkAAM.hpp

@@ -37,14 +37,14 @@ Mentor: Delia Passalacqua
 #ifndef __OPENCV_FACEMARK_AAM_HPP__
 #define __OPENCV_FACEMARK_AAM_HPP__
 
-#include "opencv2/face/facemark.hpp"
+#include "opencv2/face/facemark_train.hpp"
 namespace cv {
 namespace face {
 
 //! @addtogroup face
 //! @{
 
-class CV_EXPORTS_W FacemarkAAM : public Facemark
+class CV_EXPORTS_W FacemarkAAM : public FacemarkTrain
 {
 public:
     struct CV_EXPORTS Params
@@ -105,8 +105,6 @@ public:
     */
     struct CV_EXPORTS Model
     {
-        int npts; //!<  unused delete
-        int max_n; //!<  unused delete
         std::vector<float>scales;
         //!<  defines the scales considered to build the model
 
@@ -147,7 +145,11 @@ public:
 
     };
 
-    //!<  initializer
+    //! overload with additional Config structures
+    virtual bool fitConfig( InputArray image, InputArray roi, OutputArrayOfArrays _landmarks, const std::vector<Config> &runtime_params ) = 0;
+
+
+    //!  initializer
     static Ptr<FacemarkAAM> create(const FacemarkAAM::Params &parameters = FacemarkAAM::Params() );
     virtual ~FacemarkAAM() {}
 

modules/face/include/opencv2/face/facemarkLBF.hpp

@@ -37,7 +37,7 @@ Mentor: Delia Passalacqua
 #ifndef __OPENCV_FACEMARK_LBF_HPP__
 #define __OPENCV_FACEMARK_LBF_HPP__
 
-#include "opencv2/face/facemark.hpp"
+#include "opencv2/face/facemark_train.hpp"
 
 namespace cv {
 namespace face {
@@ -45,7 +45,7 @@ namespace face {
 //! @addtogroup face
 //! @{
 
-class CV_EXPORTS_W FacemarkLBF : public Facemark
+class CV_EXPORTS_W FacemarkLBF : public FacemarkTrain
 {
 public:
     struct CV_EXPORTS Params
@@ -99,8 +99,8 @@ public:
         ~BBox();
         BBox(double x, double y, double w, double h);
 
-        cv::Mat project(const cv::Mat &shape) const;
-        cv::Mat reproject(const cv::Mat &shape) const;
+        Mat project(const Mat &shape) const;
+        Mat reproject(const Mat &shape) const;
 
         double x, y;
         double x_center, y_center;

modules/face/misc/java/gen_dict.json

+{
+    "const_ignore_list": [
+    ],
+    "const_private_list" : [
+    ],
+    "missing_consts" : {
+    },
+    "ManualFuncs" : {
+    },
+    "func_arg_fix" : {
+        "fit" : {
+            "landmarks" : {"ctype" : "vector_vector_Point2f"},
+            "faces" : {"ctype" : "vector_Rect"}
+        }
+    }
+}

modules/face/samples/Facemark.java

+import org.opencv.core.*;
+import org.opencv.face.*;
+import org.opencv.imgcodecs.*;
+import org.opencv.imgproc.*;
+import org.opencv.objdetect.*;
+import java.util.*;
+
+
+public class Facemark {
+    static {
+        System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
+    }
+
+    public static void main(String[] args) {
+        if (args.length < 3) {
+            System.out.println("use: java Facemark [image file] [cascade file] [model file]");
+            return;
+        }
+        // read the image
+        Mat img = Imgcodecs.imread(args[0]);
+
+        // setup face detection
+        CascadeClassifier cascade = new CascadeClassifier(args[1]);
+        MatOfRect faces = new MatOfRect();
+        // detect faces
+        cascade.detectMultiScale(img, faces);
+
+        // setup landmarks detector
+        Facemark fm = Face.createFacemarkKazemi();
+        fm.loadModel(args[2]);
+
+        // fit landmarks for each found face
+        ArrayList<MatOfPoint2f> landmarks = new ArrayList<MatOfPoint2f>();
+        fm.fit(img, faces, landmarks);
+
+        // draw them
+        for (int i=0; i<landmarks.size(); i++) {
+            MatOfPoint2f lm = landmarks.get(i);
+            for (int j=0; j<lm.rows(); j++) {
+                double [] dp = lm.get(j,0);
+                Point p = new Point(dp[0], dp[1]);
+                Imgproc.circle(img,p,2,new Scalar(222),1);
+            }
+        }
+        // save result
+        Imgcodecs.imwrite("landmarks.jpg",img);
+    }
+}

modules/face/samples/facemark_demo_aam.cpp

@@ -149,7 +149,7 @@ int main(int argc, char** argv )
                 printf(" - face with eyes found %i ", (int)conf.size());
                 std::vector<std::vector<Point2f> > landmarks;
                 double newtime = (double)getTickCount();
-                facemark->fit(image, faces_eyes, landmarks, (void*)&conf);
+                facemark->fitConfig(image, faces_eyes, landmarks, conf);
                 double fittime = ((getTickCount() - newtime)/getTickFrequency());
                 for(unsigned j=0;j<landmarks.size();j++){
                     drawFacemarks(image, landmarks[j],Scalar(0,255,0));

modules/face/samples/facemark_demo_lbf.cpp

@@ -63,7 +63,7 @@ int main(int argc, char** argv)
     FacemarkLBF::Params params;
     params.model_filename = model_path;
     params.cascade_face = cascade_path;
-    Ptr<Facemark> facemark = FacemarkLBF::create(params);
+    Ptr<FacemarkLBF> facemark = FacemarkLBF::create(params);
 
     CascadeClassifier face_cascade;
     face_cascade.load(params.cascade_face.c_str());

modules/face/samples/facemark_lbf_fitting.cpp

@@ -74,7 +74,7 @@ int main(int argc, char** argv ){
     params.model_filename = model_path;
     params.cascade_face = cascade_path;
 
-    Ptr<Facemark> facemark = FacemarkLBF::create(params);
+    Ptr<FacemarkLBF> facemark = FacemarkLBF::create(params);
     facemark->setFaceDetector((FN_FaceDetector)myDetector, &face_cascade);
     facemark->loadModel(params.model_filename.c_str());
 

modules/face/samples/sample_train_landmark_detector2.cpp

 /*----------------------------------------------
- * the user should provides the list of training images_train
+ * the user should provide the list of training images_train,
  * accompanied by their corresponding landmarks location in separated files.
  * example of contents for images.txt:
  * ../trainset/image_0001.png
@@ -131,4 +131,4 @@ int main(int argc,char** argv){
     facemark->training(Trainimages,Trainlandmarks,configfile_name,scale,modelfile_name);
     cout<<"Training complete"<<endl;
     return 0;
-}
\ No newline at end of file
+}

modules/face/src/face_alignment.cpp

@@ -186,5 +186,10 @@ bool FacemarkKazemiImpl::scaleData( vector< vector<Point2f> > & trainlandmarks,
 Ptr<FacemarkKazemi> FacemarkKazemi::create(const FacemarkKazemi::Params &parameters){
     return Ptr<FacemarkKazemiImpl>(new FacemarkKazemiImpl(parameters));
 }
+
+Ptr<Facemark> createFacemarkKazemi() {
+    FacemarkKazemi::Params parameters;
+    return Ptr<FacemarkKazemiImpl>(new FacemarkKazemiImpl(parameters));
+}
 }//cv
-}//face
\ No newline at end of file
+}//face

modules/face/src/face_alignmentimpl.hpp

@@ -73,11 +73,14 @@ public:
     void loadModel(String fs) CV_OVERRIDE;
     bool setFaceDetector(FN_FaceDetector f, void* userdata) CV_OVERRIDE;
     bool getFaces(InputArray image, OutputArray faces) CV_OVERRIDE;
-    bool fit(InputArray image, InputArray faces, InputOutputArray landmarks) CV_OVERRIDE;
-    void training(String imageList, String groundTruth) CV_OVERRIDE;
+    bool fit(InputArray image, InputArray faces, OutputArrayOfArrays landmarks ) CV_OVERRIDE;
+    void training(String imageList, String groundTruth);
     bool training(vector<Mat>& images, vector< vector<Point2f> >& landmarks,string filename,Size scale,string modelFilename) CV_OVERRIDE;
     // Destructor for the class.
-    virtual ~FacemarkKazemiImpl();
+    virtual ~FacemarkKazemiImpl() CV_OVERRIDE;
+
+    virtual void read( const FileNode& ) CV_OVERRIDE {}
+    virtual void write( FileStorage& ) const CV_OVERRIDE {}
 
 protected:
     FacemarkKazemi::Params params;

modules/face/src/facemark.cpp

@@ -10,7 +10,7 @@ Mentor: Delia Passalacqua
 */
 
 #include "precomp.hpp"
-#include "opencv2/face.hpp"
+#include "opencv2/face/facemark_train.hpp"
 
 /*dataset parser*/
 #include <fstream>

modules/face/src/facemarkAAM.cpp

@@ -103,9 +103,12 @@ public:
 
     bool getData(void * items) CV_OVERRIDE;
 
+    bool fitConfig( InputArray image, InputArray roi, OutputArrayOfArrays _landmarks, const std::vector<Config> &runtime_params );
+
 protected:
 
-    bool fit( InputArray image, InputArray faces, InputOutputArray landmarks, void * runtime_params) CV_OVERRIDE;//!< from many ROIs
+    bool fit( InputArray image, InputArray faces, OutputArrayOfArrays landmarks );
+    //bool fit( InputArray image, InputArray faces, InputOutputArray landmarks, void * runtime_params);//!< from many ROIs
     bool fitImpl( const Mat image, std::vector<Point2f>& landmarks,const  Mat R,const  Point2f T,const  float scale, const int sclIdx=0 );
 
     bool addTrainingSample(InputArray image, InputArray landmarks) CV_OVERRIDE;
@@ -153,6 +156,14 @@ Ptr<FacemarkAAM> FacemarkAAM::create(const FacemarkAAM::Params &parameters){
     return Ptr<FacemarkAAMImpl>(new FacemarkAAMImpl(parameters));
 }
 
+/*
+* Constructor
+*/
+Ptr<Facemark> createFacemarkAAM(){
+    FacemarkAAM::Params parameters;
+    return Ptr<FacemarkAAMImpl>(new FacemarkAAMImpl(parameters));
+}
+
 FacemarkAAMImpl::FacemarkAAMImpl( const FacemarkAAM::Params &parameters ) :
     params( parameters ),
     faceDetector(NULL), faceDetectorData(NULL)
@@ -312,7 +323,13 @@ void FacemarkAAMImpl::training(void* parameters){
     if(params.verbose) printf("Training is completed\n");
 }
 
-bool FacemarkAAMImpl::fit( InputArray image, InputArray roi, InputOutputArray _landmarks, void * runtime_params)
+bool FacemarkAAMImpl::fit( InputArray image, InputArray roi, OutputArrayOfArrays _landmarks )
+{
+    std::vector<Config> config; // empty
+    return fitConfig(image, roi, _landmarks, config);
+}
+
+bool FacemarkAAMImpl::fitConfig( InputArray image, InputArray roi, OutputArrayOfArrays _landmarks, const std::vector<Config> &configs )
 {
     std::vector<Rect> & faces = *(std::vector<Rect> *)roi.getObj();
     if(faces.size()<1) return false;
@@ -322,14 +339,13 @@ bool FacemarkAAMImpl::fit( InputArray image, InputArray roi, InputOutputArray _l
     landmarks.resize(faces.size());
 
     Mat img = image.getMat();
-    if(runtime_params!=0){
+    if (! configs.empty()){
 
-        std::vector<Config> conf = *(std::vector<Config>*)runtime_params;
-        if (conf.size()!=faces.size()) {
+        if (configs.size()!=faces.size()) {
             CV_Error(Error::StsBadArg, "Number of faces and extra_parameters are different!");
         }
-        for(size_t i=0; i<conf.size();i++){
-            fitImpl(img, landmarks[i], conf[i].R,conf[i].t, conf[i].scale, conf[i].model_scale_idx);
+        for(size_t i=0; i<configs.size();i++){
+            fitImpl(img, landmarks[i], configs[i].R,configs[i].t, configs[i].scale, configs[i].model_scale_idx);
         }
     }else{
         Mat R =  Mat::eye(2, 2, CV_32F);

modules/face/src/facemarkLBF.cpp

@@ -115,7 +115,7 @@ public:
 
 protected:
 
-    bool fit( InputArray image, InputArray faces, InputOutputArray landmarks, void * runtime_params ) CV_OVERRIDE;//!< from many ROIs
+    bool fit( InputArray image, InputArray faces, OutputArrayOfArrays landmarks );//!< from many ROIs
     bool fitImpl( const Mat image, std::vector<Point2f> & landmarks );//!< from a face
 
     bool addTrainingSample(InputArray image, InputArray landmarks) CV_OVERRIDE;
@@ -248,6 +248,13 @@ private:
 Ptr<FacemarkLBF> FacemarkLBF::create(const FacemarkLBF::Params &parameters){
     return Ptr<FacemarkLBFImpl>(new FacemarkLBFImpl(parameters));
 }
+/*
+* Constructor
+*/
+Ptr<Facemark> createFacemarkLBF(){
+    const FacemarkLBF::Params parameters;
+    return Ptr<FacemarkLBFImpl>(new FacemarkLBFImpl(parameters));
+}
 
 FacemarkLBFImpl::FacemarkLBFImpl( const FacemarkLBF::Params &parameters ) :
     faceDetector(NULL), faceDetectorData(NULL)
@@ -363,10 +370,8 @@ void FacemarkLBFImpl::training(void* parameters){
     isModelTrained = true;
 }
 
-bool FacemarkLBFImpl::fit( InputArray image, InputArray roi, InputOutputArray  _landmarks, void * runtime_params )
+bool FacemarkLBFImpl::fit( InputArray image, InputArray roi, OutputArrayOfArrays  _landmarks )
 {
-    CV_UNUSED(runtime_params);
-
     // FIXIT
     std::vector<Rect> & faces = *(std::vector<Rect> *)roi.getObj();
     if (faces.empty()) return false;

modules/face/src/getlandmarks.cpp

@@ -163,7 +163,7 @@ void FacemarkKazemiImpl :: loadModel(String filename){
     f.close();
     isModelLoaded = true;
 }
-bool FacemarkKazemiImpl::fit(InputArray img, InputArray roi, InputOutputArray landmarks){
+bool FacemarkKazemiImpl::fit(InputArray img, InputArray roi, OutputArrayOfArrays landmarks){
     if(!isModelLoaded){
         String error_message = "No model loaded. Aborting....";
         CV_Error(Error::StsBadArg, error_message);
@@ -241,4 +241,4 @@ bool FacemarkKazemiImpl::fit(InputArray img, InputArray roi, InputOutputArray la
     return true;
 }
 }//cv
-}//face
\ No newline at end of file
+}//face

modules/face/test/test_facemark_aam.cpp

@@ -64,7 +64,7 @@ static bool customDetector( InputArray image, OutputArray ROIs, CascadeClassifie
 TEST(CV_Face_FacemarkAAM, can_create_default) {
     FacemarkAAM::Params params;
 
-    Ptr<Facemark> facemark;
+    Ptr<FacemarkAAM> facemark;
     EXPECT_NO_THROW(facemark = FacemarkAAM::create(params));
     EXPECT_FALSE(facemark.empty());
 }
@@ -75,7 +75,7 @@ TEST(CV_Face_FacemarkAAM, can_set_custom_detector) {
     CascadeClassifier face_detector;
     EXPECT_TRUE(face_detector.load(cascade_filename));
 
-    Ptr<Facemark> facemark = FacemarkAAM::create();
+    Ptr<FacemarkAAM> facemark = FacemarkAAM::create();
     EXPECT_TRUE(facemark->setFaceDetector((cv::face::FN_FaceDetector)customDetector, &face_detector));
 }
 
@@ -104,7 +104,7 @@ TEST(CV_Face_FacemarkAAM, test_workflow) {
     params.m = 1;
     params.verbose = false;
     params.save_model = false;
-    Ptr<Facemark> facemark = FacemarkAAM::create(params);
+    Ptr<FacemarkAAM> facemark = FacemarkAAM::create(params);
 
     Mat image;
     std::vector<Point2f> landmarks;

modules/face/test/test_facemark_lbf.cpp

@@ -70,7 +70,7 @@ TEST(CV_Face_FacemarkLBF, can_create_default) {
     FacemarkLBF::Params params;
     params.n_landmarks = 68;
 
-    Ptr<Facemark> facemark;
+    Ptr<FacemarkLBF> facemark;
     EXPECT_NO_THROW(facemark = FacemarkLBF::create(params));
     EXPECT_FALSE(facemark.empty());
 }
@@ -81,7 +81,7 @@ TEST(CV_Face_FacemarkLBF, can_set_custom_detector) {
 
     EXPECT_TRUE(cascade_detector.load(cascade_filename));
 
-    Ptr<Facemark> facemark = FacemarkLBF::create();
+    Ptr<FacemarkLBF> facemark = FacemarkLBF::create();
     EXPECT_TRUE(facemark->setFaceDetector(myCustomDetector));
 }
 
@@ -107,7 +107,7 @@ TEST(CV_Face_FacemarkLBF, test_workflow) {
     params.verbose = false;
     params.save_model = false;
 
-    Ptr<Facemark> facemark = FacemarkLBF::create(params);
+    Ptr<FacemarkLBF> facemark = FacemarkLBF::create(params);
 
     Mat image;
     std::vector<Point2f> landmarks;

modules/face/tutorials/face_landmark/face_landmark_detection.markdown

@@ -26,72 +26,66 @@ This tutorial will explain the sample code for face landmark detection. Jumping
 
 ``` c++
 CascadeClassifier face_cascade;
-bool myDetector( InputArray image, OutputArray ROIs );
+face_cascade.load(cascade_name);
 
-bool myDetector( InputArray image, OutputArray ROIs ){
-    Mat gray;
-    std::vector<Rect> faces;
-    if(image.channels()>1){
-        cvtColor(image.getMat(),gray,CV_BGR2GRAY);
-    }
-    else{
-        gray = image.getMat().clone();
-    }
-    equalizeHist( gray, gray );
-    face_cascade.detectMultiScale( gray, faces, 1.1, 3,0, Size(30, 30) );
-    Mat(faces).copyTo(ROIs);
-    return true;
-}
+Mat img = imread(image);
+Ptr<Facemark> facemark = createFacemarkKazemi());
+facemark->loadModel(filename);
+cout<<"Loaded model"<<endl;
 ```
 
-The facemark API provides the functionality to the user to use their own face detector to be used in training.The above code creartes a sample face detector. The above function would be passed to a function pointer in the facemark API.
+The above code creates a CascadeClassifier to detect face regions, and an instance of the face landmark detection class.
+We need to load a pretrained model for face landmark detection, and a cascade file for the face detection.
+It also loads the image in which landmarks have to be detected.
 
-``` c++
-Mat img = imread(image);
-face_cascade.load(cascade_name);
-FacemarkKazemi::Params params;
-params.configfile = configfile_name;
-Ptr<Facemark> facemark = FacemarkKazemi::create(params);
-facemark->setFaceDetector(myDetector);
-
-```
-The above code creates a pointer of the face landmark detection class. The face detector created above has to be passed
-as function pointer to the facemark pointer created for detecting faces while landmark detection. The above code also loads the image
-in which landmarks have to be detected.
 
 ``` c++
-facemark->loadModel(filename);
-cout<<"Loaded model"<<endl;
 vector<Rect> faces;
 resize(img,img,Size(460,460),0,0,INTER_LINEAR_EXACT);
-facemark->getFaces(img,faces);
-vector< vector<Point2f> > shapes;
 
-``` c++
-The above code loads a trained model for face landmark detection and creates a vector to store the detected faces. It then resizes the image to a smaller size as processing speed is faster with small images. It then creates a vector of vector to store shapes for each face detected.
+Mat gray;
+std::vector<Rect> faces;
+if(img.channels()>1){
+    cvtColor(img.getMat(),gray,CV_BGR2GRAY);
+}
+else{
+    gray = img.getMat().clone();
+}
+equalizeHist( gray, gray );
+
+face_cascade.detectMultiScale( gray, faces, 1.1, 3,0, Size(30, 30) );
+```
+
+After doing some preprocessing, we first have to detect possible face regions (which will be stored in a `vector<Rect>`.
+Also, the image is resized to a smaller size as processing speed is faster with small images.
+
 
 ``` c++
-if(facemark->fit(img,faces,shapes))
-{
-for( size_t i = 0; i < faces.size(); i++ )
+vector< vector<Point2f> > shapes;
+
+if (facemark->fit(img,faces,shapes))
 {
-    cv::rectangle(img,faces[i],Scalar( 255, 0, 0 ));
-}
-for(unsigned long i=0;i<faces.size();i++){
-    for(unsigned long k=0;k<shapes[i].size();k++)
-        cv::circle(img,shapes[i][k],5,cv::Scalar(0,0,255),FILLED);
-}
-namedWindow("Detected_shape");
-imshow("Detected_shape",img);
-waitKey(0);
+    for ( size_t i = 0; i < faces.size(); i++ )
+    {
+        cv::rectangle(img,faces[i],Scalar( 255, 0, 0 ));
+    }
+    for (unsigned long i=0;i<faces.size();i++){
+        for(unsigned long k=0;k<shapes[i].size();k++)
+            cv::circle(img,shapes[i][k],5,cv::Scalar(0,0,255),FILLED);
+    }
+    namedWindow("Detected_shape");
+    imshow("Detected_shape",img);
+    waitKey(0);
 }
 ```
 
-The above code then calls the function fit to get shapes of all detected faces in the image
+
+It then creates a vector of vector to store shapes for each face detected.
+The above code calls the function fit to get shapes of all detected faces in the image
 and then draws the rectangles bounding the faces and marks the desired landmarks.
 
 ### Detection Results
 
 ![](ab.jpg)
 
-![](ab-1.jpg)
\ No newline at end of file
+![](ab-1.jpg)