Merge pull request #303 from sbokov:improvingStereoSGBM

modules/ximgproc/include/opencv2/ximgproc/disparity_filter.hpp

@@ -39,6 +39,7 @@
 #ifdef __cplusplus
 
 #include <opencv2/core.hpp>
+#include <opencv2/calib3d.hpp>
 
 namespace cv {
 namespace ximgproc {
@@ -63,15 +64,15 @@ public:
 
     @param filtered_disparity_map output disparity map.
 
-    @param ROI region of the disparity map to filter.
-
     @param disparity_map_right optional argument, some implementations might also use the disparity map
     of the right view to compute confidence maps, for instance.
 
+    @param ROI region of the disparity map to filter. Optional, usually it should be set automatically.
+
     @param right_view optional argument, some implementations might also use the right view of the original
     stereo-pair.
      */
-    CV_WRAP virtual void filter(InputArray disparity_map_left, InputArray left_view, OutputArray filtered_disparity_map, Rect ROI, InputArray disparity_map_right = Mat(), InputArray right_view = Mat()) = 0;
+    CV_WRAP virtual void filter(InputArray disparity_map_left, InputArray left_view, OutputArray filtered_disparity_map, InputArray disparity_map_right = Mat(), Rect ROI = Rect(), InputArray right_view = Mat()) = 0;
 };
 
 /** @brief Disparity map filter based on Weighted Least Squares filter (in form of Fast Global Smoother that
@@ -106,8 +107,7 @@ public:
     /** @see getLRCthresh */
     CV_WRAP virtual void setLRCthresh(int _LRC_thresh) = 0;
     /** @brief DepthDiscontinuityRadius is a parameter used in confidence computation. It defines the size of
-    low-confidence regions around depth discontinuities. For typical window sizes used in stereo matching the
-    optimal value is around 5.
+    low-confidence regions around depth discontinuities.
      */
     CV_WRAP virtual int getDepthDiscontinuityRadius() = 0;
     /** @see getDepthDiscontinuityRadius */
@@ -117,16 +117,36 @@ public:
     correct disparity values with a high degree of confidence).
      */
     CV_WRAP virtual Mat getConfidenceMap() = 0;
-
+    /** @brief Get the ROI used in the last filter call
+     */
+    CV_WRAP virtual Rect getROI() = 0;
 };
 
-/** @brief Factory method, create instance of DisparityWLSFilter and execute the initialization routines.
+/** @brief Convenience factory method that creates an instance of DisparityWLSFilter and sets up all the relevant
+filter parameters automatically based on the matcher instance. Currently supports only StereoBM and StereoSGBM.
+
+@param matcher_left stereo matcher instance that will be used with the filter
+*/
+CV_EXPORTS_W
+Ptr<DisparityWLSFilter> createDisparityWLSFilter(Ptr<StereoMatcher> matcher_left);
+
+/** @brief Convenience method to set up the matcher for computing the right-view disparity map
+that is required in case of filtering with confidence.
+
+@param matcher_left main stereo matcher instance that will be used with the filter
+*/
+CV_EXPORTS_W
+Ptr<StereoMatcher> createRightMatcher(Ptr<StereoMatcher> matcher_left);
+
+/** @brief More generic factory method, create instance of DisparityWLSFilter and execute basic
+initialization routines. When using this method you will need to set-up the ROI, matchers and
+other parameters by yourself.
 
 @param use_confidence filtering with confidence requires two disparity maps (for the left and right views) and is
 approximately two times slower. However, quality is typically significantly better.
 */
 CV_EXPORTS_W
-Ptr<DisparityWLSFilter> createDisparityWLSFilter(bool use_confidence);
+Ptr<DisparityWLSFilter> createDisparityWLSFilterGeneric(bool use_confidence);
 
 //////////////////////////////////////////////////////////////////////////
 //////////////////////////////////////////////////////////////////////////

modules/ximgproc/perf/perf_disparity_wls_filter.cpp

@@ -85,8 +85,8 @@ PERF_TEST_P( DisparityWLSFilterPerfTest, perf, Combine(GuideTypes::all(), SrcTyp
     cv::setNumThreads(cv::getNumberOfCPUs());
     TEST_CYCLE_N(10)
     {
-        Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilter(use_conf);
-        wls_filter->filter(disp_left,guide,dst,ROI,disp_right);
+        Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilterGeneric(use_conf);
+        wls_filter->filter(disp_left,guide,dst,disp_right,ROI);
     }
 
     SANITY_CHECK(dst);

modules/ximgproc/samples/disparity_filtering.cpp

@@ -23,13 +23,13 @@ const String keys =
     "{algorithm      |bm                | stereo matching method (bm or sgbm)                               }"
     "{filter         |wls_conf          | used post-filtering (wls_conf or wls_no_conf)                     }"
     "{no-display     |                  | don't display results                                             }"
-    "{no-downscale   |                  | prevent stereo matching on downscaled views                       }"
+    "{no-downscale   |                  | force stereo matching on full-sized views to improve quality      }"
     "{dst_conf_path  |None              | optional path to save the confidence map used in filtering        }"
     "{vis_mult       |1.0               | coefficient used to scale disparity map visualizations            }"
     "{max_disparity  |160               | parameter of stereo matching                                      }"
-    "{window_size    |19                | parameter of stereo matching                                      }"
+    "{window_size    |-1                | parameter of stereo matching                                      }"
     "{wls_lambda     |8000.0            | parameter of post-filtering                                       }"
-    "{wls_sigma      |1.0               | parameter of post-filtering                                       }"
+    "{wls_sigma      |1.5               | parameter of post-filtering                                       }"
     ;
 
 int main(int argc, char** argv)
@@ -54,17 +54,30 @@ int main(int argc, char** argv)
     bool no_display = parser.has("no-display");
     bool no_downscale = parser.has("no-downscale");
     int max_disp = parser.get<int>("max_disparity");
-    int wsize = parser.get<int>("window_size");
     double lambda = parser.get<double>("wls_lambda");
     double sigma  = parser.get<double>("wls_sigma");
     double vis_mult = parser.get<double>("vis_mult");
 
+    int wsize;
+    if(parser.get<int>("window_size")>=0) //user provided window_size value
+        wsize = parser.get<int>("window_size");
+    else
+    {
+        if(algo=="sgbm")
+            wsize = 3; //default window size for SGBM
+        else if(!no_downscale && algo=="bm" && filter=="wls_conf")
+            wsize = 7; //default window size for BM on downscaled views (downscaling is performed only for wls_conf)
+        else
+            wsize = 15; //default window size for BM on full-sized views
+    }
+
     if (!parser.check())
     {
         parser.printErrors();
         return -1;
     }
 
+    //! [load_views]
     Mat left  = imread(left_im ,IMREAD_COLOR);
     if ( left.empty() )
     {
@@ -78,6 +91,7 @@ int main(int argc, char** argv)
         cout<<"Cannot read image file: "<<right_im;
         return -1;
     }
+    //! [load_views]
 
     bool noGT;
     Mat GT_disp;
@@ -99,6 +113,7 @@ int main(int argc, char** argv)
     Mat conf_map = Mat(left.rows,left.cols,CV_8U);
     conf_map = Scalar(255);
     Rect ROI;
+    Ptr<DisparityWLSFilter> wls_filter;
     double matching_time, filtering_time;
     if(max_disp<=0 || max_disp%16!=0)
     {
@@ -110,17 +125,19 @@ int main(int argc, char** argv)
         cout<<"Incorrect window_size value: it should be positive and odd";
         return -1;
     }
-    if(filter=="wls_conf")
+    if(filter=="wls_conf") // filtering with confidence (significantly better quality than wls_no_conf)
     {
         if(!no_downscale)
         {
-            wsize = wsize/2;
-            if(wsize%2==0) wsize++;
+            // downscale the views to speed-up the matching stage, as we will need to compute both left
+            // and right disparity maps for confidence map computation
+            //! [downscale]
             max_disp/=2;
             if(max_disp%16!=0)
                 max_disp += 16-(max_disp%16);
             resize(left ,left_for_matcher ,Size(),0.5,0.5);
             resize(right,right_for_matcher,Size(),0.5,0.5);
+            //! [downscale]
         }
         else
         {
@@ -128,38 +145,31 @@ int main(int argc, char** argv)
             right_for_matcher = right.clone();
         }
 
-
         if(algo=="bm")
         {
-            Ptr<StereoBM> left_matcher  = StereoBM::create(max_disp,wsize);
-            left_matcher->setMinDisparity(0);
-            Ptr<StereoBM> right_matcher = StereoBM::create(max_disp,wsize);
-            right_matcher->setMinDisparity(-max_disp+1);
-            left_matcher->setTextureThreshold(0);
-            left_matcher->setUniquenessRatio(0);
-            right_matcher->setTextureThreshold(0);
-            right_matcher->setUniquenessRatio(0);
-            cvtColor(left_for_matcher,  left_for_matcher, COLOR_BGR2GRAY);
+            //! [matching]
+            Ptr<StereoBM> left_matcher = StereoBM::create(max_disp,wsize);
+            wls_filter = createDisparityWLSFilter(left_matcher);
+            Ptr<StereoMatcher> right_matcher = createRightMatcher(left_matcher);
+
+            cvtColor(left_for_matcher,  left_for_matcher,  COLOR_BGR2GRAY);
             cvtColor(right_for_matcher, right_for_matcher, COLOR_BGR2GRAY);
-            ROI = computeROI(left_for_matcher.size(),left_matcher);
 
             matching_time = (double)getTickCount();
             left_matcher-> compute(left_for_matcher, right_for_matcher,left_disp);
             right_matcher->compute(right_for_matcher,left_for_matcher, right_disp);
             matching_time = ((double)getTickCount() - matching_time)/getTickFrequency();
+            //! [matching]
         }
         else if(algo=="sgbm")
         {
             Ptr<StereoSGBM> left_matcher  = StereoSGBM::create(0,max_disp,wsize);
-            left_matcher->setMinDisparity(0);
-            Ptr<StereoSGBM> right_matcher = StereoSGBM::create(-max_disp+1,max_disp,wsize);
-            left_matcher->setUniquenessRatio(0);
-            left_matcher->setDisp12MaxDiff(1000000);
-            left_matcher->setSpeckleWindowSize(0);
-            right_matcher->setUniquenessRatio(0);
-            right_matcher->setDisp12MaxDiff(1000000);
-            right_matcher->setSpeckleWindowSize(0);
-            ROI = computeROI(left_for_matcher.size(),left_matcher);
+            left_matcher->setP1(24*wsize*wsize);
+            left_matcher->setP2(96*wsize*wsize);
+            left_matcher->setPreFilterCap(63);
+            left_matcher->setMode(StereoSGBM::MODE_SGBM_3WAY);
+            wls_filter = createDisparityWLSFilter(left_matcher);
+            Ptr<StereoMatcher> right_matcher = createRightMatcher(left_matcher);
 
             matching_time = (double)getTickCount();
             left_matcher-> compute(left_for_matcher, right_for_matcher,left_disp);
@@ -172,14 +182,17 @@ int main(int argc, char** argv)
             return -1;
         }
 
-        Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilter(true);
+        //! [filtering]
         wls_filter->setLambda(lambda);
         wls_filter->setSigmaColor(sigma);
         filtering_time = (double)getTickCount();
-        wls_filter->filter(left_disp,left,filtered_disp,ROI,right_disp);
+        wls_filter->filter(left_disp,left,filtered_disp,right_disp);
         filtering_time = ((double)getTickCount() - filtering_time)/getTickFrequency();
+        //! [filtering]
         conf_map = wls_filter->getConfidenceMap();
 
+        // Get the ROI that was used in the last filter call:
+        ROI = wls_filter->getROI();
         if(!no_downscale)
         {
             // upscale raw disparity and ROI back for a proper comparison:
@@ -190,6 +203,9 @@ int main(int argc, char** argv)
     }
     else if(filter=="wls_no_conf")
     {
+        /* There is no convenience function for the case of filtering with no confidence, so we
+        will need to set the ROI and matcher parameters manually */
+
         left_for_matcher  = left.clone();
         right_for_matcher = right.clone();
 
@@ -201,6 +217,8 @@ int main(int argc, char** argv)
             cvtColor(left_for_matcher,  left_for_matcher, COLOR_BGR2GRAY);
             cvtColor(right_for_matcher, right_for_matcher, COLOR_BGR2GRAY);
             ROI = computeROI(left_for_matcher.size(),matcher);
+            wls_filter = createDisparityWLSFilterGeneric(false);
+            wls_filter->setDepthDiscontinuityRadius((int)ceil(0.33*wsize));
 
             matching_time = (double)getTickCount();
             matcher->compute(left_for_matcher,right_for_matcher,left_disp);
@@ -212,7 +230,12 @@ int main(int argc, char** argv)
             matcher->setUniquenessRatio(0);
             matcher->setDisp12MaxDiff(1000000);
             matcher->setSpeckleWindowSize(0);
+            matcher->setP1(24*wsize*wsize);
+            matcher->setP2(96*wsize*wsize);
+            matcher->setMode(StereoSGBM::MODE_SGBM_3WAY);
             ROI = computeROI(left_for_matcher.size(),matcher);
+            wls_filter = createDisparityWLSFilterGeneric(false);
+            wls_filter->setDepthDiscontinuityRadius((int)ceil(0.5*wsize));
 
             matching_time = (double)getTickCount();
             matcher->compute(left_for_matcher,right_for_matcher,left_disp);
@@ -224,11 +247,10 @@ int main(int argc, char** argv)
             return -1;
         }
 
-        Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilter(false);
         wls_filter->setLambda(lambda);
         wls_filter->setSigmaColor(sigma);
         filtering_time = (double)getTickCount();
-        wls_filter->filter(left_disp,left,filtered_disp,ROI);
+        wls_filter->filter(left_disp,left,filtered_disp,Mat(),ROI);
         filtering_time = ((double)getTickCount() - filtering_time)/getTickFrequency();
     }
     else
@@ -292,6 +314,7 @@ int main(int argc, char** argv)
             imshow("ground-truth disparity", GT_disp_vis);
         }
 
+        //! [visualization]
         Mat raw_disp_vis;
         getDisparityVis(left_disp,raw_disp_vis,vis_mult);
         namedWindow("raw disparity", WINDOW_AUTOSIZE);
@@ -301,6 +324,7 @@ int main(int argc, char** argv)
         namedWindow("filtered disparity", WINDOW_AUTOSIZE);
         imshow("filtered disparity", filtered_disp_vis);
         waitKey();
+        //! [visualization]
     }
 
     return 0;

modules/ximgproc/test/test_disparity_wls_filter.cpp

@@ -83,10 +83,10 @@ TEST(DisparityWLSFilterTest, ReferenceAccuracy)
     cv::setNumThreads(cv::getNumberOfCPUs());
     Mat res;
 
-    Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilter(true);
+    Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilterGeneric(true);
     wls_filter->setLambda(8000.0);
     wls_filter->setSigmaColor(0.5);
-    wls_filter->filter(left_disp,left,res,ROI,right_disp);
+    wls_filter->filter(left_disp,left,res,right_disp,ROI);
 
     double MSE = computeMSE(GT,res,ROI);
     double BadPercent = computeBadPixelPercent(GT,res,ROI);
@@ -134,17 +134,17 @@ TEST_P(DisparityWLSFilterTest, MultiThreadReproducibility)
         double lambda = rng.uniform(100.0, 10000.0);
         double sigma  = rng.uniform(1.0, 100.0);
 
-        Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilter(use_conf);
+        Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilterGeneric(use_conf);
         wls_filter->setLambda(lambda);
         wls_filter->setSigmaColor(sigma);
 
         cv::setNumThreads(cv::getNumberOfCPUs());
         Mat resMultiThread;
-        wls_filter->filter(left_disp,left,resMultiThread,ROI,right_disp);
+        wls_filter->filter(left_disp,left,resMultiThread,right_disp,ROI);
 
         cv::setNumThreads(1);
         Mat resSingleThread;
-        wls_filter->filter(left_disp,left,resSingleThread,ROI,right_disp);
+        wls_filter->filter(left_disp,left,resSingleThread,right_disp,ROI);
 
         EXPECT_LE(cv::norm(resSingleThread, resMultiThread, NORM_INF), MAX_DIF);
         EXPECT_LE(cv::norm(resSingleThread, resMultiThread, NORM_L1), MAX_MEAN_DIF*left.total());

modules/ximgproc/tutorials/disparity_filtering.markdown

+Disparity map post-filtering {#tutorial_ximgproc_disparity_filtering}
+============================
+
+Introduction
+------------
+
+Stereo matching algorithms, especially highly-optimized ones that are intended for real-time processing
+on CPU, tend to make quite a few errors on challenging sequences. These errors are usually concentrated
+in uniform texture-less areas, half-occlusions and regions near depth discontinuities. One way of dealing
+with stereo-matching errors is to use various techniques of detecting potentially inaccurate disparity
+values and invalidate them, therefore making the disparity map semi-sparse. Several such techniques are
+already implemented in the StereoBM and StereoSGBM algorithms. Another way would be to use some kind of
+filtering procedure to align the disparity map edges with those of the source image and to propagate
+the disparity values from high- to low-confidence regions like half-occlusions. Recent advances in
+edge-aware filtering have enabled performing such post-filtering under the constraints of real-time
+processing on CPU.
+
+In this tutorial you will learn how to use the disparity map post-filtering to improve the results
+of StereoBM and StereoSGBM algorithms.
+
+Source Stereoscopic Image
+-------------------------
+
+![Left view](images/ambush_5_left.jpg)
+![Right view](images/ambush_5_right.jpg)
+
+Source Code
+-----------
+
+We will be using snippets from the example application, that can be downloaded [here ](https://github.com/Itseez/opencv_contrib/blob/master/modules/ximgproc/samples/disparity_filtering.cpp).
+
+Explanation
+-----------
+
+The provided example has several options that yield different trade-offs between the speed and
+the quality of the resulting disparity map. Both the speed and the quality are measured if the user
+has provided the ground-truth disparity map. In this tutorial we will take a detailed look at the
+default pipeline, that was designed to provide the best possible quality under the constraints of
+real-time processing on CPU.
+
+-#  **Load left and right views**
+    @snippet ximgproc/samples/disparity_filtering.cpp load_views
+    We start by loading the source stereopair. For this tutorial we will take a somewhat challenging
+    example from the MPI-Sintel dataset with a lot of texture-less regions.
+
+-#  **Prepare the views for matching**
+    @snippet ximgproc/samples/disparity_filtering.cpp downscale
+    We perform downscaling of the views to speed-up the matching stage at the cost of minor
+    quality degradation. To get the best possible quality downscaling should be avoided.
+
+-#  **Perform matching and create the filter instance**
+    @snippet ximgproc/samples/disparity_filtering.cpp matching
+    We are using StereoBM for faster processing. If speed is not critical, though,
+    StereoSGBM would provide better quality. The filter instance is created by providing
+    the StereoMatcher instance that we intend to use. Another matcher instance is
+    returned by the createRightMatcher function. These two matcher instances are then
+    used to compute disparity maps both for the left and right views, that are required
+    by the filter.
+
+-#  **Perform filtering**
+    @snippet ximgproc/samples/disparity_filtering.cpp filtering
+    Disparity maps computed by the respective matcher instances, as well as the source left view
+    are passed to the filter. Note that we are using the original non-downscaled view to guide the
+    filtering process. The disparity map is automatically upscaled in an edge-aware fashion to match
+    the original view resolution. The result is stored in filtered_disp.
+
+-#  **Visualize the disparity maps**
+    @snippet ximgproc/samples/disparity_filtering.cpp visualization
+    We use a convenience function getDisparityVis to visualize the disparity maps. The second parameter
+    defines the contrast (all disparity values are scaled by this value in the visualization).
+
+Results
+-------
+
+![Result of the StereoBM](images/ambush_5_bm.png)
+![Result of the demonstrated pipeline (StereoBM on downscaled views with post-filtering)](images/ambush_5_bm_with_filter.png)