Merge pull request #239 from sbokov/disparityFiltering

Interface and demo for disparity filtering, basic WLS implementation

modules/ximgproc/CMakeLists.txt

 set(the_description "Extended image processing module. It includes edge-aware filters and etc.")
 set(OPENCV_MODULE_IS_PART_OF_WORLD OFF)
-ocv_define_module(ximgproc opencv_imgproc opencv_core opencv_highgui WRAP python)
+ocv_define_module(ximgproc opencv_imgproc opencv_core opencv_highgui opencv_calib3d WRAP python)
 
 target_link_libraries(opencv_ximgproc)

modules/ximgproc/include/opencv2/ximgproc/disparity_filter.hpp

+/*
+ *  By downloading, copying, installing or using the software you agree to this license.
+ *  If you do not agree to this license, do not download, install,
+ *  copy or use the software.
+ *
+ *
+ *  License Agreement
+ *  For Open Source Computer Vision Library
+ *  (3 - clause BSD License)
+ *
+ *  Redistribution and use in source and binary forms, with or without modification,
+ *  are permitted provided that the following conditions are met :
+ *
+ *  *Redistributions of source code must retain the above copyright notice,
+ *  this list of conditions and the following disclaimer.
+ *
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *  this list of conditions and the following disclaimer in the documentation
+ *  and / or other materials provided with the distribution.
+ *
+ *  * Neither the names of the copyright holders nor the names of the contributors
+ *  may be used to endorse or promote products derived from this software
+ *  without specific prior written permission.
+ *
+ *  This software is provided by the copyright holders and contributors "as is" and
+ *  any express or implied warranties, including, but not limited to, the implied
+ *  warranties of merchantability and fitness for a particular purpose are disclaimed.
+ *  In no event shall copyright holders or contributors be liable for any direct,
+ *  indirect, incidental, special, exemplary, or consequential damages
+ *  (including, but not limited to, procurement of substitute goods or services;
+ *  loss of use, data, or profits; or business interruption) however caused
+ *  and on any theory of liability, whether in contract, strict liability,
+ *  or tort(including negligence or otherwise) arising in any way out of
+ *  the use of this software, even if advised of the possibility of such damage.
+ */
+
+#ifndef __OPENCV_DISPARITYFILTER_HPP__
+#define __OPENCV_DISPARITYFILTER_HPP__
+#ifdef __cplusplus
+
+#include <opencv2/core.hpp>
+
+namespace cv {
+namespace ximgproc {
+
+class CV_EXPORTS_W DisparityFilter : public Algorithm
+{
+public:
+    CV_WRAP virtual void filter(InputArray disparity_map, InputArray left_view, OutputArray filtered_disparity_map,Rect ROI) = 0;
+};
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+class CV_EXPORTS_W DisparityDTFilter : public DisparityFilter
+{
+public:
+    CV_WRAP virtual double getSigmaSpatial() = 0;
+    CV_WRAP virtual void setSigmaSpatial(double _sigmaSpatial) = 0;
+    CV_WRAP virtual double getSigmaColor() = 0;
+    CV_WRAP virtual void setSigmaColor(double _sigmaColor) = 0;
+};
+
+CV_EXPORTS_W
+Ptr<DisparityDTFilter> createDisparityDTFilter();
+
+class CV_EXPORTS_W DisparityGuidedFilter : public DisparityFilter
+{
+public:
+    CV_WRAP virtual double getEps() = 0;
+    CV_WRAP virtual void setEps(double _eps) = 0;
+    CV_WRAP virtual int getRadius() = 0;
+    CV_WRAP virtual void setRadius(int _radius) = 0;
+};
+
+CV_EXPORTS_W
+Ptr<DisparityGuidedFilter> createDisparityGuidedFilter();
+
+class CV_EXPORTS_W DisparityWLSFilter : public DisparityFilter
+{
+    CV_WRAP virtual double getLambda() = 0;
+    CV_WRAP virtual void setLambda(double _lambda) = 0;
+    CV_WRAP virtual double getSigmaColor() = 0;
+    CV_WRAP virtual void setSigmaColor(double _sigma_color) = 0;
+};
+
+CV_EXPORTS_W
+Ptr<DisparityWLSFilter> createDisparityWLSFilter();
+
+}
+}
+#endif
+#endif

modules/ximgproc/include/opencv2/ximgproc/edge_filter.hpp

@@ -318,6 +318,17 @@ void jointBilateralFilter(InputArray joint, InputArray src, OutputArray dst, int
 
 //! @}
 
+class CV_EXPORTS_W WeightedLeastSquaresFilter : public Algorithm
+{
+public:
+    CV_WRAP virtual void filter(InputArray src, OutputArray dst) = 0;
+    virtual ~WeightedLeastSquaresFilter();
+};
+
+CV_EXPORTS_W Ptr<WeightedLeastSquaresFilter> createWeightedLeastSquaresFilter(InputArray guide, double lambda, double sigma_color, int num_iter=3);
+
+CV_EXPORTS_W void weightedLeastSquaresFilter(InputArray guide, InputArray src, OutputArray dst, double lambda, double sigma_color, int num_iter=3);
+
 }
 }
 #endif

modules/ximgproc/samples/disparity_filtering.cpp

+#include "opencv2/calib3d.hpp"
+#include "opencv2/imgproc.hpp"
+#include "opencv2/imgcodecs.hpp"
+#include "opencv2/highgui.hpp"
+#include "opencv2/core/utility.hpp"
+#include "opencv2/ximgproc/disparity_filter.hpp"
+#include <stdio.h>
+#include <string>
+#include <vector>
+#include <map>
+
+#if defined(_WIN32)
+#include <direct.h>
+#else 
+#include <sys/stat.h>
+#endif
+
+using namespace cv;
+using namespace cv::ximgproc;
+using namespace std;
+
+#define UNKNOWN_DISPARITY 16320
+ 
+static void print_help()
+{
+    printf("\nDemo for disparity filtering, evaluating speed and performance of different filters\n");
+    printf("\nUsage: disparity_filtering.exe <path_to_dataset_folder> <path_to_results_folder>\n");
+}
+
+struct dataset_entry
+{
+    string name;
+    string dataset_folder;
+    string left_file,right_file,GT_file;
+    dataset_entry(string _dataset_folder): dataset_folder(_dataset_folder){}
+    void readEntry(Mat& dst_left,Mat& dst_right,Mat& dst_GT)
+    {
+        dst_left  = imread(dataset_folder+"/"+left_file,  IMREAD_COLOR);
+        dst_right = imread(dataset_folder+"/"+right_file, IMREAD_COLOR);
+        Mat raw_disp = imread(dataset_folder+"/"+GT_file, IMREAD_COLOR);
+        dst_GT = Mat(raw_disp.rows,raw_disp.cols,CV_16S);
+        for(int i=0;i<raw_disp.rows;i++)
+            for(int j=0;j<raw_disp.cols;j++)
+            {
+                Vec3b bgrPixel = raw_disp.at<Vec3b>(i, j);
+                dst_GT.at<short>(i,j) = 64*bgrPixel.val[2]+bgrPixel.val[1]/4; //16-multiplied disparity
+            }
+    }
+};
+
+struct config
+{
+    Ptr<StereoMatcher> matcher_instance;
+    Ptr<DisparityFilter> filter_instance;
+    config(Ptr<StereoMatcher> _matcher_instance,Ptr<DisparityFilter> _filter_instance)
+    {
+        matcher_instance = _matcher_instance;
+        filter_instance = _filter_instance;
+    }
+    config() {}
+};
+
+void operator>>(const FileNode& node,dataset_entry& entry);
+double computeMSE(Mat& GT, Mat& src, Rect ROI);
+double computeBadPixelPercent(Mat& GT, Mat& src, Rect ROI, int thresh=24/*1.5 pixels*/);
+void getDisparityVis(Mat& disparity_map,Mat& dst);
+Rect computeROI(Size2i src_sz, Ptr<StereoMatcher> matcher_instance);
+void setConfigsForTesting(map<string,config>& cfgs);
+void CreateDir(string path);
+
+int main(int argc, char** argv)
+{
+    if(argc < 3)
+    {
+        print_help();
+        return 0;
+    }
+    string dataset_folder(argv[1]);
+    string res_folder(argv[2]);
+
+    map<string,config> configs_for_testing;
+    setConfigsForTesting(configs_for_testing);
+    CreateDir(res_folder);
+
+    for (map<string,config>::iterator cfg = configs_for_testing.begin(); cfg != configs_for_testing.end(); cfg++)
+    {
+        string vis_folder = res_folder+"/vis_"+cfg->first;
+        CreateDir(vis_folder);
+
+        string cfg_file_name = res_folder+"/"+cfg->first+"_res.csv";
+        FILE* cur_cfg_res_file = fopen(cfg_file_name.c_str(),"w");
+        fprintf(cur_cfg_res_file,"Name,MSE,MSE after postfiltering,Percent bad,Percent bad after postfiltering,Matcher Execution Time(s),Filter Execution Time(s)\n");
+
+        printf("Processing configuration: %s\n",cfg->first.c_str());
+
+        FileStorage fs(dataset_folder + "/_dataset.xml", FileStorage::READ);
+        FileNode n = fs["data_set"];
+        double MSE_pre,percent_pre,MSE_post,percent_post,matching_time,filtering_time;
+        double average_MSE_pre=0,average_percent_pre=0,average_MSE_post=0,
+            average_percent_post=0,average_matching_time=0,average_filtering_time=0;
+        int cnt = 0;
+        for (FileNodeIterator it = n.begin(); it != n.end(); it++)
+        {
+            dataset_entry entry(dataset_folder);
+            (*it)>>entry;
+            printf("%s ",entry.name.c_str());
+            Mat left,right,GT;
+            entry.readEntry(left,right,GT);
+            Mat raw_disp;
+            Mat left_gray; cvtColor(left, left_gray, COLOR_BGR2GRAY );
+            Mat right_gray; cvtColor(right, right_gray, COLOR_BGR2GRAY );
+            matching_time = (double)getTickCount();
+            cfg->second.matcher_instance->compute(left_gray,right_gray,raw_disp);
+            matching_time = ((double)getTickCount() - matching_time)/getTickFrequency();
+
+            Rect ROI = computeROI(left.size(),cfg->second.matcher_instance);
+            Mat filtered_disp;
+            filtering_time = (double)getTickCount();
+            cfg->second.filter_instance->filter(raw_disp,left,filtered_disp,ROI);
+            filtering_time = ((double)getTickCount() - filtering_time)/getTickFrequency();
+
+
+            MSE_pre = computeMSE(GT,raw_disp,ROI);
+            percent_pre = computeBadPixelPercent(GT,raw_disp,ROI);
+            MSE_post = computeMSE(GT,filtered_disp,ROI);
+            percent_post = computeBadPixelPercent(GT,filtered_disp,ROI);
+
+            fprintf(cur_cfg_res_file,"%s,%.1f,%.1f,%.1f,%.1f,%.3f,%.3f\n",entry.name.c_str(),MSE_pre,MSE_post,
+                percent_pre,percent_post,matching_time,filtering_time);
+
+            average_matching_time+=matching_time; average_filtering_time+=filtering_time;
+            average_MSE_pre+=MSE_pre; average_percent_pre+=percent_pre;
+            average_MSE_post+=MSE_post; average_percent_post+=percent_post;
+            cnt++;
+
+            // dump visualizations:
+            imwrite(vis_folder + "/" + entry.name + "_left.png",left);
+            Mat GT_vis,raw_disp_vis,filtered_disp_vis;
+            getDisparityVis(GT,GT_vis);
+            getDisparityVis(raw_disp,raw_disp_vis);
+            getDisparityVis(filtered_disp,filtered_disp_vis);
+            imwrite(vis_folder + "/" + entry.name + "_disparity_GT.png",GT_vis);
+            imwrite(vis_folder + "/" + entry.name + "_disparity_raw.png",raw_disp_vis);
+            imwrite(vis_folder + "/" + entry.name + "_disparity_filtered.png",filtered_disp_vis);
+
+            printf("- Done\n");
+
+        }
+        fprintf(cur_cfg_res_file,"%s,%.1f,%.1f,%.1f,%.1f,%.3f,%.3f\n","average",average_MSE_pre/cnt,
+            average_MSE_post/cnt,average_percent_pre/cnt,average_percent_post/cnt,
+            average_matching_time/cnt,average_filtering_time/cnt);
+        fclose(cur_cfg_res_file);
+    }
+    return 0;
+}
+
+void operator>>(const FileNode& node,dataset_entry& entry) 
+{
+    node["name"] >> entry.name;
+    node["left_file"] >> entry.left_file;
+    node["right_file"] >> entry.right_file;
+    node["GT_file"] >> entry.GT_file;
+}
+
+double computeMSE(Mat& GT, Mat& src, Rect ROI)
+{
+    double res = 0;
+    Mat GT_ROI(GT,ROI);
+    Mat src_ROI(src,ROI);
+    int cnt=0;
+    for(int i=0;i<src_ROI.rows;i++)
+        for(int j=0;j<src_ROI.cols;j++)
+        {
+            if(GT_ROI.at<short>(i,j)!=UNKNOWN_DISPARITY)
+            {
+                res += (GT_ROI.at<short>(i,j) - src_ROI.at<short>(i,j))*(GT_ROI.at<short>(i,j) - src_ROI.at<short>(i,j));
+                cnt++;
+            }
+        }
+    res /= cnt*256;
+    return res;
+}
+
+double computeBadPixelPercent(Mat& GT, Mat& src, Rect ROI, int thresh)
+{
+    int bad_pixel_num = 0;
+    Mat GT_ROI(GT,ROI);
+    Mat src_ROI(src,ROI);
+    int cnt=0;
+    for(int i=0;i<src_ROI.rows;i++)
+        for(int j=0;j<src_ROI.cols;j++)
+        {
+            if(GT_ROI.at<short>(i,j)!=UNKNOWN_DISPARITY)
+            {
+                if( abs(GT_ROI.at<short>(i,j) - src_ROI.at<short>(i,j))>=thresh )
+                    bad_pixel_num++;
+                cnt++;
+            }
+        }
+    return (100.0*bad_pixel_num)/cnt;
+}
+
+void getDisparityVis(Mat& disparity_map,Mat& dst)
+{
+    dst = Mat(disparity_map.rows,disparity_map.cols,CV_8UC3);
+    for(int i=0;i<dst.rows;i++)
+        for(int j=0;j<dst.cols;j++)
+        {
+            if(disparity_map.at<short>(i,j)==UNKNOWN_DISPARITY)
+                dst.at<Vec3b>(i,j) = Vec3b(0,0,0);
+            else
+                dst.at<Vec3b>(i,j) = Vec3b(saturate_cast<unsigned char>(disparity_map.at<short>(i,j)/8),
+                                           saturate_cast<unsigned char>(disparity_map.at<short>(i,j)/8),
+                                           saturate_cast<unsigned char>(disparity_map.at<short>(i,j)/8));
+        }
+}
+
+Rect computeROI(Size2i src_sz, Ptr<StereoMatcher> matcher_instance)
+{
+    int min_disparity = matcher_instance->getMinDisparity();
+    int num_disparities = matcher_instance->getNumDisparities();
+    int block_size = matcher_instance->getBlockSize();
+
+    int bs2 = block_size/2;
+    int minD = min_disparity, maxD = min_disparity + num_disparities - 1;
+
+    int xmin = maxD + bs2;
+    int xmax = src_sz.width - minD - bs2;
+    int ymin = bs2;
+    int ymax = src_sz.height - bs2;
+
+    Rect r(xmin, ymin, xmax - xmin, ymax - ymin);
+    return r;
+}
+
+void setConfigsForTesting(map<string,config>& cfgs)
+{
+    Ptr<StereoBM> stereobm_matcher = StereoBM::create(128,21); 
+    stereobm_matcher->setTextureThreshold(0); 
+    stereobm_matcher->setUniquenessRatio(0);
+
+    Ptr<DisparityFilter> wls_filter = createDisparityWLSFilter();
+    Ptr<DisparityFilter> dt_filter = createDisparityDTFilter();
+    Ptr<DisparityFilter> guided_filter = createDisparityGuidedFilter();
+
+    cfgs["stereobm_wls"] = config(stereobm_matcher,wls_filter);
+    cfgs["stereobm_dtf"] = config(stereobm_matcher,dt_filter);
+    cfgs["stereobm_gf"]  = config(stereobm_matcher,guided_filter);
+}
+
+void CreateDir(string path)
+{
+#if defined(_WIN32)
+    _mkdir(path.c_str());
+#else 
+    mkdir(path.c_str(), 0777);
+#endif
+}

modules/ximgproc/src/disparity_filters.cpp

+/*
+ *  By downloading, copying, installing or using the software you agree to this license.
+ *  If you do not agree to this license, do not download, install,
+ *  copy or use the software.
+ *  
+ *  
+ *  License Agreement
+ *  For Open Source Computer Vision Library
+ *  (3 - clause BSD License)
+ *  
+ *  Redistribution and use in source and binary forms, with or without modification,
+ *  are permitted provided that the following conditions are met :
+ *  
+ *  *Redistributions of source code must retain the above copyright notice,
+ *  this list of conditions and the following disclaimer.
+ *  
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *  this list of conditions and the following disclaimer in the documentation
+ *  and / or other materials provided with the distribution.
+ *  
+ *  * Neither the names of the copyright holders nor the names of the contributors
+ *  may be used to endorse or promote products derived from this software
+ *  without specific prior written permission.
+ *  
+ *  This software is provided by the copyright holders and contributors "as is" and
+ *  any express or implied warranties, including, but not limited to, the implied
+ *  warranties of merchantability and fitness for a particular purpose are disclaimed.
+ *  In no event shall copyright holders or contributors be liable for any direct,
+ *  indirect, incidental, special, exemplary, or consequential damages
+ *  (including, but not limited to, procurement of substitute goods or services;
+ *  loss of use, data, or profits; or business interruption) however caused
+ *  and on any theory of liability, whether in contract, strict liability,
+ *  or tort(including negligence or otherwise) arising in any way out of
+ *  the use of this software, even if advised of the possibility of such damage.
+ */
+
+#include "precomp.hpp"
+#include "opencv2/ximgproc/disparity_filter.hpp"
+#include <math.h>
+
+namespace cv {
+namespace ximgproc {
+
+void setROI(InputArray disparity_map, InputArray left_view, OutputArray filtered_disparity_map,
+            Mat& disp,Mat& src,Mat& dst,Rect ROI);
+
+class DisparityDTFilterImpl : public DisparityDTFilter
+{
+protected:
+    double sigmaSpatial,sigmaColor;
+    void init(double _sigmaSpatial, double _sigmaColor)
+    {
+        sigmaColor = _sigmaColor;
+        sigmaSpatial = _sigmaSpatial;
+    }
+public:
+    double getSigmaSpatial() {return sigmaSpatial;}
+    void setSigmaSpatial(double _sigmaSpatial) {sigmaSpatial = _sigmaSpatial;}
+    double getSigmaColor() {return sigmaColor;}
+    void setSigmaColor(double _sigmaColor) {sigmaColor = _sigmaColor;}
+
+    static Ptr<DisparityDTFilterImpl> create()
+    {
+        DisparityDTFilterImpl *dtf = new DisparityDTFilterImpl();
+        dtf->init(25.0,60.0); 
+        return Ptr<DisparityDTFilterImpl>(dtf);
+    }
+
+    void filter(InputArray disparity_map, InputArray left_view, OutputArray filtered_disparity_map,Rect ROI)
+    {
+        Mat disp,src,dst;
+        setROI(disparity_map,left_view,filtered_disparity_map,disp,src,dst,ROI);
+
+        Mat disp_32F,dst_32F;
+        disp.convertTo(disp_32F,CV_32F);
+        dtFilter(src,disp_32F,dst_32F,sigmaSpatial,sigmaColor);
+        dst_32F.convertTo(dst,CV_16S);
+    }
+};
+
+CV_EXPORTS_W
+Ptr<DisparityDTFilter> createDisparityDTFilter()
+{
+    return Ptr<DisparityDTFilter>(DisparityDTFilterImpl::create());
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////
+    
+class DisparityGuidedFilterImpl : public DisparityGuidedFilter
+{
+protected:
+    int radius;
+    double eps;
+    void init(int _radius, double _eps)
+    {
+        radius = _radius;
+        eps = _eps;
+    }
+public:
+    double getEps() {return eps;}
+    void setEps(double _eps) {eps = _eps;}
+    int getRadius() {return radius;}
+    void setRadius(int _radius) {radius = _radius;}
+
+    static Ptr<DisparityGuidedFilterImpl> create()
+    {
+        DisparityGuidedFilterImpl *gf = new DisparityGuidedFilterImpl();
+        gf->init(20,100.0);
+        return Ptr<DisparityGuidedFilterImpl>(gf);
+    }
+
+    void filter(InputArray disparity_map, InputArray left_view, OutputArray filtered_disparity_map,Rect ROI)
+    {
+        Mat disp,src,dst;
+        setROI(disparity_map,left_view,filtered_disparity_map,disp,src,dst,ROI);
+
+        Mat disp_32F,dst_32F;
+        disp.convertTo(disp_32F,CV_32F);
+        guidedFilter(src,disp_32F,dst_32F,radius,eps);
+        dst_32F.convertTo(dst,CV_16S);
+    }
+};
+
+CV_EXPORTS_W
+Ptr<DisparityGuidedFilter> createDisparityGuidedFilter()
+{
+    return Ptr<DisparityGuidedFilter>(DisparityGuidedFilterImpl::create());
+}
+    
+//////////////////////////////////////////////////////////////////////////////////////////////////////
+
+class DisparityWLSFilterImpl : public DisparityWLSFilter
+{
+protected:
+    double lambda,sigma_color;
+    void init(double _lambda, double _sigma_color)
+    {
+        lambda = _lambda;
+        sigma_color = _sigma_color;
+    }
+public:
+    double getLambda() {return lambda;}
+    void setLambda(double _lambda) {lambda = _lambda;}
+    double getSigmaColor() {return sigma_color;}
+    void setSigmaColor(double _sigma_color) {sigma_color = _sigma_color;}
+
+    static Ptr<DisparityWLSFilterImpl> create()
+    {
+        DisparityWLSFilterImpl *wls = new DisparityWLSFilterImpl();
+        wls->init(500.0,2.0);
+        return Ptr<DisparityWLSFilterImpl>(wls);
+    }
+
+    void filter(InputArray disparity_map, InputArray left_view, OutputArray filtered_disparity_map,Rect ROI)
+    {
+        Mat disp,src,dst;
+        setROI(disparity_map,left_view,filtered_disparity_map,disp,src,dst,ROI);
+        weightedLeastSquaresFilter(src,disp,dst,lambda,sigma_color);
+    }
+};
+
+CV_EXPORTS_W
+Ptr<DisparityWLSFilter> createDisparityWLSFilter()
+{
+    return Ptr<DisparityWLSFilter>(DisparityWLSFilterImpl::create());
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void setROI(InputArray disparity_map, InputArray left_view, OutputArray filtered_disparity_map,
+    Mat& disp,Mat& src,Mat& dst,Rect ROI)
+{
+    Mat disp_full_size = disparity_map.getMat();
+    Mat src_full_size = left_view.getMat();
+    CV_Assert( (disp_full_size.depth() == CV_16S) && (disp_full_size.channels() == 1) );
+    CV_Assert( (src_full_size.depth()  == CV_8U)  && (src_full_size.channels()  <= 4) );
+    disp = Mat(disp_full_size,ROI);
+    src  = Mat(src_full_size ,ROI);
+    filtered_disparity_map.create(disp_full_size.size(), disp_full_size.type());
+    Mat& dst_full_size = filtered_disparity_map.getMatRef();
+    dst = Mat(dst_full_size,ROI);
+}
+
+}
+}