Merge pull request #15338 from rayonnant14:my_detect_and_decode_3.4

QR-Code detector : multiple detection

* change in qr-codes detection

* change in qr-codes detection

* change in test

* change in test

* add multiple detection

* multiple detection

* multiple detect

* add parallel implementation

* add functional for performance tests

* change in test

* add perftest

* returned implementation for 1 qr-code, added support for vector<Mat> and vector<vector<Point2f>> in MultipleDetectAndDecode

* deleted all lambda expressions

* changing in triangle sort

* fixed warnings

* fixed errors

* add java and python tests

* change in java tests

* change in java and python tests

* change in perf test

* change in qrcode.cpp

* add spaces

* change in qrcode.cpp

* change in qrcode.cpp

* change in qrcode.cpp

* change in java tests

* change in java tests

* solved problems

* solved problems

* change in java and python tests

* change in python tests

* change in python tests

* change in python tests

* change in methods name

* deleted sample qrcode_multi, change in qrcode.cpp

* change in perf tests

* change in objdetect.hpp

* deleted code duplication in sample qrcode.cpp

* returned spaces

* added spaces

* deleted draw function

* change in qrcode.cpp

* change in qrcode.cpp

* deleted all draw functions

* objdetect(QR): extractVerticalLines

* objdetect(QR): whitespaces

* objdetect(QR): simplify operations, avoid duplicated code

* change in interface, additional checks in java and python tests, added new key in sample for saving original image from camera

* fix warnings and errors in python test

* fix

* write in file with space key

* solved error with empty mat check in python test

* correct path to test image

* deleted spaces

* solved error with check empty mat in python tests

* added check of empty vector of points

* samples: rework qrcode.cpp

* objdetect(QR): fix API, input parameters must be first

* objdetect(QR): test/fix points layout

modules/java/generator/gen_java.py

@@ -914,7 +914,10 @@ class JavaWrapperGenerator(object):
                     c_epilogue.append("Mat* _retval_ = new Mat();")
                     c_epilogue.append(fi.ctype+"_to_Mat(_ret_val_vector_, *_retval_);")
                 else:
-                    c_epilogue.append("return " + fi.ctype + "_to_List(env, _ret_val_vector_);")
+                    if ret:
+                        c_epilogue.append("jobject _retval_ = " + fi.ctype + "_to_List(env, _ret_val_vector_);")
+                    else:
+                        c_epilogue.append("return " + fi.ctype + "_to_List(env, _ret_val_vector_);")
             if fi.classname:
                 if not fi.ctype: # c-tor
                     retval = fi.fullClass(isCPP=True) + "* _retval_ = "

modules/objdetect/include/opencv2/objdetect.hpp

@@ -694,8 +694,8 @@ public:
     CV_WRAP bool detect(InputArray img, OutputArray points) const;
 
     /** @brief Decodes QR code in image once it's found by the detect() method.
-     Returns UTF8-encoded output string or empty string if the code cannot be decoded.
 
+     Returns UTF8-encoded output string or empty string if the code cannot be decoded.
      @param img grayscale or color (BGR) image containing QR code.
      @param points Quadrangle vertices found by detect() method (or some other algorithm).
      @param straight_qrcode The optional output image containing rectified and binarized QR code
@@ -705,11 +705,44 @@ public:
     /** @brief Both detects and decodes QR code
 
      @param img grayscale or color (BGR) image containing QR code.
-     @param points opiotnal output array of vertices of the found QR code quadrangle. Will be empty if not found.
+     @param points optional output array of vertices of the found QR code quadrangle. Will be empty if not found.
      @param straight_qrcode The optional output image containing rectified and binarized QR code
      */
     CV_WRAP cv::String detectAndDecode(InputArray img, OutputArray points=noArray(),
                                         OutputArray straight_qrcode = noArray());
+    /** @brief Detects QR codes in image and returns the vector of the quadrangles containing the codes.
+     @param img grayscale or color (BGR) image containing (or not) QR codes.
+     @param points Output vector of vector of vertices of the minimum-area quadrangle containing the codes.
+     */
+    CV_WRAP
+    bool detectMulti(InputArray img, OutputArray points) const;
+
+    /** @brief Decodes QR codes in image once it's found by the detect() method.
+     @param img grayscale or color (BGR) image containing QR codes.
+     @param decoded_info UTF8-encoded output vector of string or empty vector of string if the codes cannot be decoded.
+     @param points vector of Quadrangle vertices found by detect() method (or some other algorithm).
+     @param straight_qrcode The optional output vector of images containing rectified and binarized QR codes
+     */
+    CV_WRAP
+    bool decodeMulti(
+            InputArray img, InputArray points,
+            CV_OUT std::vector<cv::String>& decoded_info,
+            OutputArrayOfArrays straight_qrcode = noArray()
+    ) const;
+
+    /** @brief Both detects and decodes QR codes
+    @param img grayscale or color (BGR) image containing QR codes.
+    @param decoded_info UTF8-encoded output vector of string or empty vector of string if the codes cannot be decoded.
+    @param points optional output vector of vertices of the found QR code quadrangles. Will be empty if not found.
+    @param straight_qrcode The optional output vector of images containing rectified and binarized QR codes
+    */
+    CV_WRAP
+    bool detectAndDecodeMulti(
+            InputArray img, CV_OUT std::vector<cv::String>& decoded_info,
+            OutputArray points = noArray(),
+            OutputArrayOfArrays straight_qrcode = noArray()
+    ) const;
+
 protected:
     struct Impl;
     Ptr<Impl> p;
@@ -731,6 +764,29 @@ CV_EXPORTS bool detectQRCode(InputArray in, std::vector<Point> &points, double e
     */
 CV_EXPORTS bool decodeQRCode(InputArray in, InputArray points, std::string &decoded_info, OutputArray straight_qrcode = noArray());
 
+/** @brief Detect QR codes in image and return vector of minimum area of quadrangle that describes QR codes.
+    @param in  Matrix of the type CV_8UC1 containing an image where QR codes are detected.
+    @param points Output vector of vertices of quadrangles of minimal area that describes QR codes.
+    @param eps_x Epsilon neighborhood, which allows you to determine the horizontal pattern of the scheme 1:1:3:1:1 according to QR code standard.
+    @param eps_y Epsilon neighborhood, which allows you to determine the vertical pattern of the scheme 1:1:3:1:1 according to QR code standard.
+    */
+CV_EXPORTS
+bool detectQRCodeMulti(
+        InputArray in, std::vector<Point> &points,
+        double eps_x = 0.2, double eps_y = 0.1);
+
+/** @brief Decode QR codes in image and return text that is encrypted in QR code.
+    @param in  Matrix of the type CV_8UC1 containing an image where QR code are detected.
+    @param points Input vector of vertices of quadrangles of minimal area that describes QR codes.
+    @param decoded_info vector of String information that is encrypted in QR codes.
+    @param straight_qrcode vector of Matrixes of the type CV_8UC1 containing an binary straight QR codes.
+    */
+CV_EXPORTS
+bool decodeQRCodeMulti(
+        InputArray in, InputArray points,
+        CV_OUT std::vector<std::string> &decoded_info,
+        OutputArrayOfArrays straight_qrcode = noArray());
+
 //! @} objdetect
 }
 

modules/objdetect/misc/java/test/QRCodeDetectorTest.java

 package org.opencv.test.objdetect;
 
+import java.util.List;
 import org.opencv.core.Mat;
 import org.opencv.objdetect.QRCodeDetector;
 import org.opencv.imgcodecs.Imgcodecs;
 import org.opencv.test.OpenCVTestCase;
+import java.util.ArrayList;
 
 public class QRCodeDetectorTest extends OpenCVTestCase {
 
@@ -21,9 +23,27 @@ public class QRCodeDetectorTest extends OpenCVTestCase {
 
     public void testDetectAndDecode() {
         Mat img = Imgcodecs.imread(testDataPath + "/cv/qrcode/link_ocv.jpg");
+        assertFalse(img.empty());
         QRCodeDetector detector = new QRCodeDetector();
+        assertNotNull(detector);
         String output = detector.detectAndDecode(img);
         assertEquals(output, "https://opencv.org/");
     }
 
+    public void testDetectAndDecodeMulti() {
+        Mat img = Imgcodecs.imread(testDataPath + "/cv/qrcode/multiple/6_qrcodes.png");
+        assertFalse(img.empty());
+        QRCodeDetector detector = new QRCodeDetector();
+        assertNotNull(detector);
+        List < String > output = new ArrayList< String >();
+        boolean result = detector.detectAndDecodeMulti(img, output);
+        assertTrue(result);
+        assertEquals(output.size(), 6);
+        assertEquals(output.get(0), "SKIP");
+        assertEquals(output.get(1), "EXTRA");
+        assertEquals(output.get(2), "TWO STEPS FORWARD");
+        assertEquals(output.get(3), "STEP BACK");
+        assertEquals(output.get(4), "QUESTION");
+        assertEquals(output.get(5), "STEP FORWARD");
+    }
 }

modules/objdetect/misc/python/test/test_qrcode_detect.py

@@ -11,8 +11,42 @@ import cv2 as cv
 from tests_common import NewOpenCVTests
 
 class qrcode_detector_test(NewOpenCVTests):
+
+    def test_detect(self):
+        img = cv.imread(os.path.join(self.extraTestDataPath, 'cv/qrcode/link_ocv.jpg'))
+        self.assertFalse(img is None)
+        detector = cv.QRCodeDetector()
+        retval, points = detector.detect(img)
+        self.assertTrue(retval)
+        self.assertEqual(points.shape, (1, 4, 2))
+
     def test_detect_and_decode(self):
         img = cv.imread(os.path.join(self.extraTestDataPath, 'cv/qrcode/link_ocv.jpg'))
+        self.assertFalse(img is None)
         detector = cv.QRCodeDetector()
         retval, points, straight_qrcode = detector.detectAndDecode(img)
-        self.assertEqual(retval, "https://opencv.org/");
+        self.assertEqual(retval, "https://opencv.org/")
+        self.assertEqual(points.shape, (1, 4, 2))
+
+    def test_detect_multi(self):
+        img = cv.imread(os.path.join(self.extraTestDataPath, 'cv/qrcode/multiple/6_qrcodes.png'))
+        self.assertFalse(img is None)
+        detector = cv.QRCodeDetector()
+        retval, points = detector.detectMulti(img)
+        self.assertTrue(retval)
+        self.assertEqual(points.shape, (6, 4, 2))
+
+    def test_detect_and_decode_multi(self):
+        img = cv.imread(os.path.join(self.extraTestDataPath, 'cv/qrcode/multiple/6_qrcodes.png'))
+        self.assertFalse(img is None)
+        detector = cv.QRCodeDetector()
+        retval, decoded_data, points, straight_qrcode = detector.detectAndDecodeMulti(img)
+        self.assertTrue(retval)
+        self.assertEqual(len(decoded_data), 6)
+        self.assertEqual(decoded_data[0], "TWO STEPS FORWARD")
+        self.assertEqual(decoded_data[1], "EXTRA")
+        self.assertEqual(decoded_data[2], "SKIP")
+        self.assertEqual(decoded_data[3], "STEP FORWARD")
+        self.assertEqual(decoded_data[4], "STEP BACK")
+        self.assertEqual(decoded_data[5], "QUESTION")
+        self.assertEqual(points.shape, (6, 4, 2))

modules/objdetect/perf/perf_qrcode_pipeline.cpp

@@ -53,6 +53,56 @@ PERF_TEST_P_(Perf_Objdetect_QRCode, decode)
 }
 #endif
 
+typedef ::perf::TestBaseWithParam< std::string > Perf_Objdetect_QRCode_Multi;
+
+PERF_TEST_P_(Perf_Objdetect_QRCode_Multi, detectMulti)
+{
+    const std::string name_current_image = GetParam();
+    const std::string root = "cv/qrcode/multiple/";
+
+    std::string image_path = findDataFile(root + name_current_image);
+    Mat src = imread(image_path);
+    ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path;
+    std::vector<Point2f> corners;
+    QRCodeDetector qrcode;
+    TEST_CYCLE() ASSERT_TRUE(qrcode.detectMulti(src, corners));
+    SANITY_CHECK(corners);
+}
+
+#ifdef HAVE_QUIRC
+PERF_TEST_P_(Perf_Objdetect_QRCode_Multi, decodeMulti)
+{
+    const std::string name_current_image = GetParam();
+    const std::string root = "cv/qrcode/multiple/";
+
+    std::string image_path = findDataFile(root + name_current_image);
+    Mat src = imread(image_path);
+    ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path;
+    QRCodeDetector qrcode;
+    std::vector<Point2f> corners;
+    ASSERT_TRUE(qrcode.detectMulti(src, corners));
+    std::vector<Mat> straight_barcode;
+    std::vector< cv::String > decoded_info;
+    TEST_CYCLE()
+    {
+        ASSERT_TRUE(qrcode.decodeMulti(src, corners, decoded_info, straight_barcode));
+        for(size_t i = 0; i < decoded_info.size(); i++)
+        {
+            ASSERT_FALSE(decoded_info[i].empty());
+        }
+    }
+    std::vector < std::vector< uint8_t > > decoded_info_uint8_t;
+    for(size_t i = 0; i < decoded_info.size(); i++)
+    {
+        std::vector< uint8_t > tmp(decoded_info[i].begin(), decoded_info[i].end());
+        decoded_info_uint8_t.push_back(tmp);
+    }
+    SANITY_CHECK(decoded_info_uint8_t);
+    SANITY_CHECK(straight_barcode);
+
+}
+#endif
+
 INSTANTIATE_TEST_CASE_P(/*nothing*/, Perf_Objdetect_QRCode,
     ::testing::Values(
         "version_1_down.jpg", "version_1_left.jpg", "version_1_right.jpg", "version_1_up.jpg", "version_1_top.jpg",
@@ -61,6 +111,13 @@ INSTANTIATE_TEST_CASE_P(/*nothing*/, Perf_Objdetect_QRCode,
     )
 );
 
+INSTANTIATE_TEST_CASE_P(/*nothing*/, Perf_Objdetect_QRCode_Multi,
+    ::testing::Values(
+      "2_qrcodes.png", "3_close_qrcodes.png", "3_qrcodes.png", "4_qrcodes.png",
+       "5_qrcodes.png", "6_qrcodes.png", "7_qrcodes.png", "8_close_qrcodes.png"
+    )
+);
+
 typedef ::perf::TestBaseWithParam< tuple< std::string, Size > > Perf_Objdetect_Not_QRCode;
 
 PERF_TEST_P_(Perf_Objdetect_Not_QRCode, detect)

modules/objdetect/test/test_qrcode.cpp

@@ -21,7 +21,11 @@ std::string qrcode_images_close[] = {
 std::string qrcode_images_monitor[] = {
   "monitor_1.png", "monitor_2.png", "monitor_3.png", "monitor_4.png", "monitor_5.png"
 };
-// #define UPDATE_QRCODE_TEST_DATA
+std::string qrcode_images_multiple[] = {
+  "2_qrcodes.png", "3_close_qrcodes.png", "3_qrcodes.png", "4_qrcodes.png",
+   "5_qrcodes.png", "6_qrcodes.png", "7_qrcodes.png", "8_close_qrcodes.png"
+};
+//#define UPDATE_QRCODE_TEST_DATA
 #ifdef  UPDATE_QRCODE_TEST_DATA
 
 TEST(Objdetect_QRCode, generate_test_data)
@@ -134,6 +138,66 @@ TEST(Objdetect_QRCode_Monitor, generate_test_data)
     file_config.release();
 }
 
+
+TEST(Objdetect_QRCode_Multi, generate_test_data)
+{
+    const std::string root = "qrcode/multiple/";
+    const std::string dataset_config = findDataFile(root + "dataset_config.json");
+    FileStorage file_config(dataset_config, FileStorage::WRITE);
+
+    file_config << "multiple_images" << "[:";
+    size_t multiple_count = sizeof(qrcode_images_multiple) / sizeof(qrcode_images_multiple[0]);
+    for (size_t i = 0; i < multiple_count; i++)
+    {
+        file_config << "{:" << "image_name" << qrcode_images_multiple[i];
+        std::string image_path = findDataFile(root + qrcode_images_multiple[i]);
+        Mat src = imread(image_path);
+
+        ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path;
+        std::vector<Point> corners;
+        EXPECT_TRUE(detectQRCodeMulti(src, corners));
+#ifdef HAVE_QUIRC
+        std::vector<cv::String> decoded_info;
+        std::vector<Mat> straight_barcode;
+        EXPECT_TRUE(decodeQRCodeMulti(src, corners, decoded_info, straight_barcode));
+#endif
+        file_config << "x" << "[:";
+        for(size_t j = 0; j < corners.size(); j += 4)
+        {
+            file_config << "[:";
+            for (size_t k = 0; k < 4; k++)
+            {
+                file_config << corners[j + k].x;
+            }
+            file_config << "]";
+        }
+        file_config << "]";
+        file_config << "y" << "[:";
+        for(size_t j = 0; j < corners.size(); j += 4)
+        {
+            file_config << "[:";
+            for (size_t k = 0; k < 4; k++)
+            {
+                file_config << corners[j + k].y;
+            }
+            file_config << "]";
+        }
+        file_config << "]";
+        file_config << "info";
+        file_config << "[:";
+
+        for(size_t j = 0; j < decoded_info.size(); j++)
+        {
+            file_config << decoded_info[j];
+        }
+        file_config << "]";
+        file_config << "}";
+    }
+
+    file_config << "]";
+    file_config.release();
+}
+
 #else
 
 typedef testing::TestWithParam< std::string > Objdetect_QRCode;
@@ -326,9 +390,96 @@ TEST_P(Objdetect_QRCode_Monitor, regression)
     }
 }
 
+typedef testing::TestWithParam < std::string > Objdetect_QRCode_Multi;
+TEST_P(Objdetect_QRCode_Multi, regression)
+{
+    const std::string name_current_image = GetParam();
+    const std::string root = "qrcode/multiple/";
+    const int pixels_error = 3;
+
+    std::string image_path = findDataFile(root + name_current_image);
+    Mat src = imread(image_path);
+    ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path;
+    QRCodeDetector qrcode;
+    std::vector<Point> corners;
+#ifdef HAVE_QUIRC
+    std::vector<cv::String> decoded_info;
+    std::vector<Mat> straight_barcode;
+    EXPECT_TRUE(qrcode.detectAndDecodeMulti(src, decoded_info, corners, straight_barcode));
+    ASSERT_FALSE(corners.empty());
+    ASSERT_FALSE(decoded_info.empty());
+#else
+    ASSERT_TRUE(qrcode.detectMulti(src, corners));
+#endif
+
+    const std::string dataset_config = findDataFile(root + "dataset_config.json");
+    FileStorage file_config(dataset_config, FileStorage::READ);
+    ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config;
+    {
+        FileNode images_list = file_config["multiple_images"];
+        size_t images_count = static_cast<size_t>(images_list.size());
+        ASSERT_GT(images_count, 0u) << "Can't find validation data entries in 'test_images': " << dataset_config;
+        for (size_t index = 0; index < images_count; index++)
+        {
+            FileNode config = images_list[(int)index];
+            std::string name_test_image = config["image_name"];
+            if (name_test_image == name_current_image)
+            {
+                for(int j = 0; j < int(corners.size()); j += 4)
+                {
+                    bool ok = false;
+                    for (int k = 0; k < int(corners.size() / 4); k++)
+                    {
+                        int count_eq_points = 0;
+                        for (int i = 0; i < 4; i++)
+                        {
+                            int x = config["x"][k][i];
+                            int y = config["y"][k][i];
+                            if(((abs(corners[j + i].x - x)) <= pixels_error) && ((abs(corners[j + i].y - y)) <= pixels_error))
+                              count_eq_points++;
+                        }
+                        if (count_eq_points == 4)
+                        {
+                            ok = true;
+                            break;
+                        }
+                    }
+                    EXPECT_TRUE(ok);
+                }
+
+#ifdef HAVE_QUIRC
+                  size_t count_eq_info = 0;
+                  for(int i = 0; i < int(decoded_info.size()); i++)
+                  {
+                      for(int j = 0; j < int(decoded_info.size()); j++)
+                      {
+                          std::string original_info = config["info"][j];
+                          if(original_info == decoded_info[i])
+                          {
+                             count_eq_info++;
+                             break;
+                          }
+                      }
+                  }
+                  EXPECT_EQ(decoded_info.size(), count_eq_info);
+#endif
+
+                  return; // done
+            }
+        }
+        std::cerr
+            << "Not found results for '" << name_current_image
+            << "' image in config file:" << dataset_config << std::endl
+            << "Re-run tests with enabled UPDATE_QRCODE_TEST_DATA macro to update test data."
+            << std::endl;
+    }
+}
+
+
 INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode, testing::ValuesIn(qrcode_images_name));
 INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Close, testing::ValuesIn(qrcode_images_close));
 INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Monitor, testing::ValuesIn(qrcode_images_monitor));
+INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Multi, testing::ValuesIn(qrcode_images_multiple));
 
 TEST(Objdetect_QRCode_basic, not_found_qrcode)
 {