Merge pull request #13858 from rgarnov:fluid_nv12_support

modules/gapi/include/opencv2/gapi/fluid/gfluidbuffer.hpp

@@ -53,6 +53,10 @@ public:
 
         inline const uint8_t* linePtr(int index) const
         {
+            // "out_of_window" check:
+            // user must not request the lines which are outside of specified kernel window
+            GAPI_DbgAssert(index >= -m_border_size
+                        && index <  -m_border_size + static_cast<int>(m_linePtrs.size()));
             return m_linePtrs[index + m_border_size];
         }
     };

modules/gapi/include/opencv2/gapi/fluid/gfluidkernel.hpp

@@ -49,7 +49,8 @@ public:
     enum class Kind
     {
         Filter,
-        Resize
+        Resize,
+        NV12toRGB
     };
 
     // This function is a generic "doWork" callback

modules/gapi/src/backends/fluid/gfluidbackend.hpp

@@ -4,10 +4,12 @@
 //
 // Copyright (C) 2018 Intel Corporation
 
-
 #ifndef OPENCV_GAPI_FLUID_BACKEND_HPP
 #define OPENCV_GAPI_FLUID_BACKEND_HPP
 
+// FIXME? Actually gfluidbackend.hpp is not included anywhere
+// and can be placed in gfluidbackend.cpp
+
 #include "opencv2/gapi/garg.hpp"
 #include "opencv2/gapi/gproto.hpp"
 #include "opencv2/gapi/fluid/gfluidkernel.hpp"
@@ -25,7 +27,7 @@ struct FluidUnit
     GFluidKernel k;
     gapi::fluid::BorderOpt border;
     int border_size;
-    int line_consumption;
+    std::vector<int> line_consumption;
     double ratio;
 };
 
@@ -90,8 +92,8 @@ public:
 private:
     // FIXME!!!
     // move to another class
-    virtual int firstWindow() const = 0;
-    virtual std::pair<int,int> linesReadAndnextWindow() const = 0;
+    virtual int firstWindow(std::size_t inPort) const = 0;
+    virtual std::pair<int,int> linesReadAndnextWindow(std::size_t inPort) const = 0;
 };
 
 class GFluidExecutable final: public GIslandExecutable

modules/gapi/src/backends/fluid/gfluidbuffer.cpp

@@ -402,20 +402,6 @@ fluid::ViewPrivWithoutOwnBorder::ViewPrivWithoutOwnBorder(const Buffer *parent,
     m_border_size = borderSize;
 }
 
-const uint8_t* fluid::ViewPrivWithoutOwnBorder::InLineB(int index) const
-{
-    GAPI_DbgAssert(m_p);
-
-    const auto &p_priv = m_p->priv();
-
-    GAPI_DbgAssert(index >= -m_border_size
-                && index <  -m_border_size + m_lines_next_iter);
-
-    const int log_idx = m_read_caret + index;
-
-    return p_priv.storage().inLineB(log_idx, m_p->meta().size.height);
-}
-
 void fluid::ViewPrivWithoutOwnBorder::allocate(int lineConsumption, BorderOpt)
 {
     initCache(lineConsumption);
@@ -475,17 +461,6 @@ std::size_t fluid::ViewPrivWithOwnBorder::size() const
     return m_own_storage.size();
 }
 
-const uint8_t* fluid::ViewPrivWithOwnBorder::InLineB(int index) const
-{
-    GAPI_DbgAssert(m_p);
-    GAPI_DbgAssert(index >= -m_border_size
-                && index <  -m_border_size + m_lines_next_iter);
-
-    const int log_idx = m_read_caret + index;
-
-    return m_own_storage.inLineB(log_idx, m_p->meta().size.height);
-}
-
 bool fluid::View::ready() const
 {
     return m_priv->ready();
@@ -544,6 +519,9 @@ void fluid::Buffer::Priv::allocate(BorderOpt border,
     // Init physical buffer
 
     // FIXME? combine line_consumption with skew?
+    // FIXME? This formula serves general case to avoid possible deadlock,
+    // in some cases this value can be smaller:
+    // 2 lines produced, 2 consumed, data_height can be 2, not 3
     auto data_height = std::max(line_consumption, skew) + m_writer_lpi - 1;
 
     m_storage = createStorage(data_height,
@@ -641,13 +619,6 @@ int fluid::Buffer::Priv::linesReady() const
     }
 }
 
-uint8_t* fluid::Buffer::Priv::OutLineB(int index)
-{
-    GAPI_DbgAssert(index >= 0 && index < m_writer_lpi);
-
-    return m_storage->ptr(m_write_caret + index);
-}
-
 int fluid::Buffer::Priv::lpi() const
 {
     // FIXME:

modules/gapi/src/backends/fluid/gfluidbuffer_priv.hpp

@@ -197,9 +197,6 @@ public:
 
     // Does the view have enough unread lines for next iteration
     bool ready() const;
-
-    // API used (indirectly) by user code
-    virtual const uint8_t* InLineB(int index) const = 0;
 };
 
 class ViewPrivWithoutOwnBorder final : public View::Priv
@@ -212,9 +209,6 @@ public:
     virtual void prepareToRead() override;
 
     inline virtual std::size_t size() const override { return 0; }
-
-    // API used (indirectly) by user code
-    virtual const uint8_t* InLineB(int index) const override;
 };
 
 class ViewPrivWithOwnBorder final : public View::Priv
@@ -228,9 +222,6 @@ public:
     virtual void allocate(int lineConsumption, BorderOpt border) override;
     virtual void prepareToRead() override;
     virtual std::size_t size() const override;
-
-    // API used (indirectly) by user code
-    virtual const uint8_t* InLineB(int index) const override;
 };
 
 void debugBufferPriv(const Buffer& buffer, std::ostream &os);
@@ -290,7 +281,6 @@ public:
     inline int writer_lpi()     const { return m_writer_lpi; }
 
     // API used (indirectly) by user code
-    uint8_t* OutLineB(int index = 0);
     int lpi() const;
 
     inline int readStart()   const { return m_readStart; }

modules/gapi/test/gapi_fluid_resize_test.cpp

@@ -717,4 +717,77 @@ INSTANTIATE_TEST_CASE_P(ResizeTestCPU, BlursAfterResizeTest,
                                        std::make_tuple(cv::Size{64,64},
                                                        cv::Size{49,49}, cv::Rect{0,39,49,10}))));
 
+struct NV12PlusResizeTest : public TestWithParam <std::tuple<cv::Size, cv::Size, cv::Rect>> {};
+TEST_P(NV12PlusResizeTest, Test)
+{
+    cv::Size y_sz, out_sz;
+    cv::Rect roi;
+    std::tie(y_sz, out_sz, roi) = GetParam();
+    int interp = cv::INTER_LINEAR;
+
+    cv::Size uv_sz(y_sz.width / 2, y_sz.height / 2);
+    cv::Size in_sz(y_sz.width, y_sz.height*3/2);
+
+    cv::Mat in_mat = cv::Mat(in_sz, CV_8UC1);
+
+    cv::Scalar mean   = cv::Scalar(127.0f);
+    cv::Scalar stddev = cv::Scalar(40.f);
+    cv::randn(in_mat, mean, stddev);
+
+    cv::Mat y_mat  = cv::Mat(y_sz, CV_8UC1, in_mat.data);
+    cv::Mat uv_mat = cv::Mat(uv_sz, CV_8UC2, in_mat.data + in_mat.step1() * y_sz.height);
+    cv::Mat out_mat, out_mat_ocv;
+
+    cv::GMat y, uv;
+    auto rgb = cv::gapi::NV12toRGB(y, uv);
+    auto out = cv::gapi::resize(rgb, out_sz, 0, 0, interp);
+    cv::GComputation c(cv::GIn(y, uv), cv::GOut(out));
+
+    auto pkg = cv::gapi::combine(fluidTestPackage, cv::gapi::core::fluid::kernels(), cv::unite_policy::KEEP);
+
+    c.apply(cv::gin(y_mat, uv_mat), cv::gout(out_mat)
+           ,cv::compile_args(pkg, cv::GFluidOutputRois{{to_own(roi)}}));
+
+    cv::Mat rgb_mat;
+    cv::cvtColor(in_mat, rgb_mat, cv::COLOR_YUV2RGB_NV12);
+    cv::resize(rgb_mat, out_mat_ocv, out_sz, 0, 0, interp);
+
+    EXPECT_EQ(0, cv::countNonZero(out_mat(roi) != out_mat_ocv(roi)));
+}
+
+INSTANTIATE_TEST_CASE_P(Fluid, NV12PlusResizeTest,
+                        Values(std::make_tuple(cv::Size{8, 8},
+                                               cv::Size{4, 4}, cv::Rect{0, 0, 4, 4})
+                              ,std::make_tuple(cv::Size{8, 8},
+                                               cv::Size{4, 4}, cv::Rect{0, 0, 4, 1})
+                              ,std::make_tuple(cv::Size{8, 8},
+                                               cv::Size{4, 4}, cv::Rect{0, 1, 4, 2})
+                              ,std::make_tuple(cv::Size{8, 8},
+                                               cv::Size{4, 4}, cv::Rect{0, 2, 4, 2})
+                              ,std::make_tuple(cv::Size{64, 64},
+                                               cv::Size{49, 49}, cv::Rect{0,  0, 49, 49})
+                              ,std::make_tuple(cv::Size{64, 64},
+                                               cv::Size{49, 49}, cv::Rect{0,  0, 49, 12})
+                              ,std::make_tuple(cv::Size{64, 64},
+                                               cv::Size{49, 49}, cv::Rect{0, 11, 49, 15})
+                              ,std::make_tuple(cv::Size{64, 64},
+                                               cv::Size{49, 49}, cv::Rect{0, 39, 49, 10})
+                              ,std::make_tuple(cv::Size{1920, 1080},
+                                               cv::Size{ 320,  256}, cv::Rect{0, 0, 320, 64})
+                              ,std::make_tuple(cv::Size{1920, 1080},
+                                               cv::Size{ 320,  256}, cv::Rect{0, 64, 320, 64})
+                              ,std::make_tuple(cv::Size{1920, 1080},
+                                               cv::Size{ 320,  256}, cv::Rect{0, 128, 320, 64})
+                              ,std::make_tuple(cv::Size{1920, 1080},
+                                               cv::Size{ 320,  256}, cv::Rect{0, 192, 320, 64})
+                              ,std::make_tuple(cv::Size{256, 400},
+                                               cv::Size{ 32,  64}, cv::Rect{0,  0, 32, 16})
+                              ,std::make_tuple(cv::Size{256, 400},
+                                               cv::Size{ 32,  64}, cv::Rect{0, 16, 32, 16})
+                              ,std::make_tuple(cv::Size{256, 400},
+                                               cv::Size{ 32,  64}, cv::Rect{0, 32, 32, 16})
+                              ,std::make_tuple(cv::Size{256, 400},
+                                               cv::Size{ 32,  64}, cv::Rect{0, 48, 32, 16})
+                               ));
+
 } // namespace opencv_test

modules/gapi/test/gapi_fluid_test.cpp

@@ -710,4 +710,46 @@ TEST(FluidTwoIslands, SanityTest)
     EXPECT_EQ(0, countNonZero(in_mat2 != out_mat2));
 }
 
+struct NV12RoiTest : public TestWithParam <std::pair<cv::Size, cv::Rect>> {};
+TEST_P(NV12RoiTest, Test)
+{
+    cv::Size y_sz;
+    cv::Rect roi;
+    std::tie(y_sz, roi) = GetParam();
+
+    cv::Size uv_sz(y_sz.width / 2, y_sz.height / 2);
+    cv::Size in_sz(y_sz.width, y_sz.height*3/2);
+
+    cv::Mat in_mat = cv::Mat(in_sz, CV_8UC1);
+
+    cv::Scalar mean   = cv::Scalar(127.0f);
+    cv::Scalar stddev = cv::Scalar(40.f);
+    cv::randn(in_mat, mean, stddev);
+
+    cv::Mat y_mat  = cv::Mat(y_sz, CV_8UC1, in_mat.data);
+    cv::Mat uv_mat = cv::Mat(uv_sz, CV_8UC2, in_mat.data + in_mat.step1() * y_sz.height);
+    cv::Mat out_mat, out_mat_ocv;
+
+    cv::GMat y, uv;
+    auto rgb = cv::gapi::NV12toRGB(y, uv);
+    cv::GComputation c(cv::GIn(y, uv), cv::GOut(rgb));
+
+    c.apply(cv::gin(y_mat, uv_mat), cv::gout(out_mat), cv::compile_args(fluidTestPackage, cv::GFluidOutputRois{{to_own(roi)}}));
+
+    cv::cvtColor(in_mat, out_mat_ocv, cv::COLOR_YUV2RGB_NV12);
+
+    EXPECT_EQ(0, cv::countNonZero(out_mat(roi) != out_mat_ocv(roi)));
+}
+
+INSTANTIATE_TEST_CASE_P(Fluid, NV12RoiTest,
+                        Values(std::make_pair(cv::Size{8, 8}, cv::Rect{0, 0, 8, 2})
+                              ,std::make_pair(cv::Size{8, 8}, cv::Rect{0, 2, 8, 2})
+                              ,std::make_pair(cv::Size{8, 8}, cv::Rect{0, 4, 8, 2})
+                              ,std::make_pair(cv::Size{8, 8}, cv::Rect{0, 6, 8, 2})
+                              ,std::make_pair(cv::Size{1920, 1080}, cv::Rect{0,   0, 1920, 270})
+                              ,std::make_pair(cv::Size{1920, 1080}, cv::Rect{0, 270, 1920, 270})
+                              ,std::make_pair(cv::Size{1920, 1080}, cv::Rect{0, 540, 1920, 270})
+                              ,std::make_pair(cv::Size{1920, 1080}, cv::Rect{0, 710, 1920, 270})
+                              ));
+
 } // namespace opencv_test

modules/gapi/test/gapi_fluid_test_kernels.cpp

@@ -416,6 +416,76 @@ GAPI_FLUID_KERNEL(FSum2MatsAndScalar, TSum2MatsAndScalar, false)
     }
 };
 
+static const int ITUR_BT_601_CY = 1220542;
+static const int ITUR_BT_601_CUB = 2116026;
+static const int ITUR_BT_601_CUG = -409993;
+static const int ITUR_BT_601_CVG = -852492;
+static const int ITUR_BT_601_CVR = 1673527;
+static const int ITUR_BT_601_SHIFT = 20;
+
+static inline void uvToRGBuv(const uchar u, const uchar v, int& ruv, int& guv, int& buv)
+{
+    int uu, vv;
+    uu = int(u) - 128;
+    vv = int(v) - 128;
+
+    ruv = (1 << (ITUR_BT_601_SHIFT - 1)) + ITUR_BT_601_CVR * vv;
+    guv = (1 << (ITUR_BT_601_SHIFT - 1)) + ITUR_BT_601_CVG * vv + ITUR_BT_601_CUG * uu;
+    buv = (1 << (ITUR_BT_601_SHIFT - 1)) + ITUR_BT_601_CUB * uu;
+}
+
+static inline void yRGBuvToRGB(const uchar vy, const int ruv, const int guv, const int buv,
+                                uchar& r, uchar& g, uchar& b)
+{
+    int y = std::max(0, vy - 16) * ITUR_BT_601_CY;
+    r = saturate_cast<uchar>((y + ruv) >> ITUR_BT_601_SHIFT);
+    g = saturate_cast<uchar>((y + guv) >> ITUR_BT_601_SHIFT);
+    b = saturate_cast<uchar>((y + buv) >> ITUR_BT_601_SHIFT);
+}
+
+GAPI_FLUID_KERNEL(FNV12toRGB, cv::gapi::imgproc::GNV12toRGB, false)
+{
+    static const int Window = 1;
+    static const int LPI    = 2;
+    static const auto Kind = GFluidKernel::Kind::NV12toRGB;
+
+    static void run(const cv::gapi::fluid::View   &in1,
+                    const cv::gapi::fluid::View   &in2,
+                          cv::gapi::fluid::Buffer &out)
+    {
+        const auto w = out.length();
+
+        GAPI_Assert(w % 2 == 0);
+        GAPI_Assert(out.lpi() == 2);
+
+        const uchar* uv_row = in2.InLineB(0);
+        const uchar*   y_rows[] = {in1. InLineB(0), in1. InLineB(1)};
+              uchar* out_rows[] = {out.OutLineB(0), out.OutLineB(1)};
+
+        for (int i = 0; i < w/2; i++)
+        {
+            uchar u = uv_row[2*i];
+            uchar v = uv_row[2*i + 1];
+            int ruv, guv, buv;
+            uvToRGBuv(u, v, ruv, guv, buv);
+
+            for (int y = 0; y < 2; y++)
+            {
+                for (int x = 0; x < 2; x++)
+                {
+                    uchar vy = y_rows[y][2*i + x];
+                    uchar r, g, b;
+                    yRGBuvToRGB(vy, ruv, guv, buv, r, g, b);
+
+                    out_rows[y][3*(2*i + x)]     = r;
+                    out_rows[y][3*(2*i + x) + 1] = g;
+                    out_rows[y][3*(2*i + x) + 2] = b;
+                }
+            }
+        }
+    }
+};
+
 cv::gapi::GKernelPackage fluidTestPackage = cv::gapi::kernels
         <FAddSimple
         ,FAddCSimple
@@ -428,6 +498,7 @@ cv::gapi::GKernelPackage fluidTestPackage = cv::gapi::kernels
         ,FBlur5x5_2lpi
         ,FIdentity
         ,FId7x7
+        ,FNV12toRGB
         ,FPlusRow0
         ,FSum2MatsAndScalar
         ,FTestSplit3