face_alignmentimpl.hpp

// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.

#ifndef __OPENCV_FACE_ALIGNMENTIMPL_HPP__
#define __OPENCV_FACE_ALIGNMENTIMPL_HPP__
#include "opencv2/face.hpp"
#include <string>
#include <sstream>
#include <vector>
#include <fstream>
#include <queue>
#include <algorithm>
#include <ctime>

using namespace std;
namespace cv{
namespace face{
/**@brief structure determining split in regression tree
*/
struct splitr{
        //!index1 Index of the first coordinates among the test coordinates for deciding split.
        uint64_t index1;
        //! index2 index of the second coordinate among the test coordinates for deciding split.
        uint64_t index2;
        //! thresh threshold for deciding the split.
        float thresh;
};
/** @brief represents a node of the regression tree*/
struct node_info{
    //First pixel coordinate of split
    long index1;
    //Second pixel coordinate .split
    long index2;
    long depth;
    long node_no;
};
/** @brief regression tree structure. Each leaf node is a vector storing residual shape.
* The tree is represented as vector of leaves.
*/
struct tree_node{
    splitr split;
    std::vector<Point2f> leaf;
};
struct regtree{
    std::vector<tree_node> nodes;
};
/** @brief Represents a training sample
*It contains current shape, difference between actual shape
*and current shape. It also stores the image whose shape is being
*detected.
*/
struct training_sample{
    //! shapeResiduals vector which stores the residual shape remaining to be corrected.
    std::vector<Point2f> shapeResiduals;
    //! current_shape vector containing current estimate of the shape
    std::vector<Point2f> current_shape;
    //! actual_shape vector containing the actual shape of the face or the ground truth.
    std::vector<Point2f> actual_shape;
    //! image A mat object which stores the image.
    Mat image ;
    //! pixel_intensities vector containing pixel intensities of the coordinates chosen for testing
    std::vector<int> pixel_intensities;
    //! pixel_coordinates vector containing pixel coordinates used for testing
    std::vector<Point2f> pixel_coordinates;
    //! bound Rectangle enclosing the face found  in the image for training
    Rect bound;
};
class FacemarkKazemiImpl : public FacemarkKazemi{

public:
    FacemarkKazemiImpl(const FacemarkKazemi::Params& parameters);
    void loadModel(String fs) CV_OVERRIDE;
    bool setFaceDetector(FN_FaceDetector f, void* userdata) CV_OVERRIDE;
    bool getFaces(InputArray image, OutputArray faces) CV_OVERRIDE;
    bool fit(InputArray image, InputArray faces, OutputArrayOfArrays landmarks ) CV_OVERRIDE;
    void training(String imageList, String groundTruth);
    bool training(vector<Mat>& images, vector< vector<Point2f> >& landmarks,string filename,Size scale,string modelFilename) CV_OVERRIDE;
    // Destructor for the class.
    virtual ~FacemarkKazemiImpl() CV_OVERRIDE;

    virtual void read( const FileNode& ) CV_OVERRIDE {}
    virtual void write( FileStorage& ) const CV_OVERRIDE {}

protected:
    FacemarkKazemi::Params params;
    float minmeanx;
    float maxmeanx;
    float minmeany;
    float maxmeany;
    bool isModelLoaded;
    /* meanshape This is a vector which stores the mean shape of all the images used in training*/
    std::vector<Point2f> meanshape;
    std::vector< std::vector<regtree> > loaded_forests;
    std::vector< std::vector<Point2f> > loaded_pixel_coordinates;
    FN_FaceDetector faceDetector;
    void* faceDetectorData;
    bool findNearestLandmarks(std::vector< std::vector<int> >& nearest);
    /*Extract left node of the current node in the regression tree*/
    unsigned long left(unsigned long index);
    // Extract the right node of the current node in the regression tree
    unsigned long right(unsigned long index);
    // This function randomly  generates test splits to get the best split.
    splitr getTestSplits(std::vector<Point2f> pixel_coordinates,int seed);
    // This function writes a split node to the XML file storing the trained model
    void writeSplit(std::ofstream& os, const splitr& split);
    // This function writes a leaf node to the binary file storing the trained model
    void writeLeaf(std::ofstream& os, const std::vector<Point2f> &leaf);
    // This function writes a tree to the binary file containing the model
    void writeTree(std::ofstream &f,regtree tree);
    // This function saves the pixel coordinates to a binary file
    void writePixels(std::ofstream& f,int index);
    // This function saves model to the binary file
    bool saveModel(String filename);
    // This funcrion reads pixel coordinates from the model file
    void readPixels(std::ifstream& is,uint64_t index);
    //This function reads the split node of the tree from binary file
    void readSplit(std::ifstream& is, splitr &vec);
    //This function reads a leaf node of the tree.
    void readLeaf(std::ifstream& is, std::vector<Point2f> &leaf);
    /* This function generates pixel intensities of the randomly generated test coordinates used to decide the split.
    */
    bool getPixelIntensities(Mat img,std::vector<Point2f> pixel_coordinates_,std::vector<int>& pixel_intensities_,Rect face);
    //This function initialises the training parameters.
    bool setTrainingParameters(String filename);
    //This function finds a warp matrix that warp the pixels from the normalised space to the actual space
    bool convertToActual(Rect r,Mat &warp);
    //This function finds a warp matrix that warps the pixels from the actual space to normaluised space
    bool convertToUnit(Rect r,Mat &warp);
    /** @brief This function calculates mean shape while training.
    * This function is only called when new training data is supplied by the train function.
    *@param trainlandmarks A vector of type cv::Point2f which stores the landmarks of corresponding images.
    *@param trainimages A vector of type cv::Mat which stores the images which serve as training data.
    *@param faces A vector of type cv::Rect which stores the bounding recatngle of each training image
    *@returns A boolean value. It returns true if mean shape is found successfully else returns false.
    */
    bool calcMeanShape(std::vector< std::vector<Point2f> > & trainlandmarks,std::vector<Mat>& trainimages,std::vector<Rect>& faces);
    /** @brief This functions scales the annotations to a common size which is considered same for all images.
    * @param trainlandmarks A vector of type cv::Point2f stores the landmarks of the corresponding training images.
    * @param trainimages A vector of type cv::Mat which stores the images which are to be scaled.
    * @param s A variable of type cv::Size stores the common size to which all the images are scaled.
    * @returns A boolean value. It returns true when data is scaled properly else returns false.
    */
    bool scaleData(std::vector< std::vector<Point2f> >& trainlandmarks,
                                    std::vector<Mat>& trainimages , Size s=Size(460,460) );
    // This function gets the landmarks in the meanshape nearest to the pixel coordinates.
    unsigned long getNearestLandmark (Point2f pixels );
    // This function gets the relative position of the test pixel coordinates relative to the current shape.
    bool getRelativePixels(std::vector<Point2f> sample,std::vector<Point2f>& pixel_coordinates , std::vector<int> nearest_landmark = std::vector<int>());
    // This function partitions samples according to the split
    unsigned long divideSamples (splitr split,std::vector<training_sample>& samples,unsigned long start,unsigned long end);
    // This function fits a regression tree according to the shape residuals calculated to give weak learners for GBT algorithm.
    bool buildRegtree(regtree &tree,std::vector<training_sample>& samples,std::vector<Point2f> pixel_coordinates);
    // This function greedily decides the best split among the test splits generated.
    bool getBestSplit(std::vector<Point2f> pixel_coordinates, std::vector<training_sample>& samples,unsigned long start ,
                                        unsigned long end,splitr& split,std::vector< std::vector<Point2f> >& sum,long node_no);
    // This function randomly generates test coordinates for each level of cascade.
    void getTestCoordinates ();
    // This function implements gradient boosting by fitting regression trees
    std::vector<regtree> gradientBoosting(std::vector<training_sample>& samples,std::vector<Point2f> pixel_coordinates);
    // This function creates training sample by randomly assigning a current shape from set of shapes available.
    void createLeafNode(regtree& tree,long node_no,std::vector<Point2f> assign);
    // This function creates a split node in the regression tree.
    void createSplitNode(regtree& tree, splitr split,long node_no);
    // This function prepares the training samples
    bool createTrainingSamples(std::vector<training_sample> &samples,std::vector<Mat> images,std::vector< std::vector<Point2f> > landmarks,
    std::vector<Rect> rectangle);
    //This function generates a split
    bool generateSplit(std::queue<node_info>& curr,std::vector<Point2f> pixel_coordinates, std::vector<training_sample>& samples,
                                        splitr &split , std::vector< std::vector<Point2f> >& sum);
    bool setMeanExtreme();
    //friend class getRelShape;
    friend class getRelPixels;
};
}//face
}//cv
#endif