d3d11-nv12 interop

fixed issues with ocl nv12 cvt kernel

finisged ocl nv12-to-rgba kernel, update dx-interop samples. (ocl rgba-to-nv12 kernel will be added later)

an attempt to fix build issue

fix for non opencl build issue

fix typo

fix compilation warnings

fix compile issue for Mac (OpenCL)

add convertion from rgba to nv12 (still need to debug kernel)

remove empty line at the EOF

fixed compilation warning

modules/core/src/directx.cpp

@@ -44,6 +44,7 @@
 #include "opencv2/core.hpp"
 #include "opencv2/core/ocl.hpp"
 #include "opencv2/core/directx.hpp"
+#include "opencl_kernels_core.hpp"
 
 #ifdef HAVE_DIRECTX
 #include <vector>
@@ -167,6 +168,7 @@ int getTypeFromDXGI_FORMAT(const int iDXGI_FORMAT)
     //case DXGI_FORMAT_BC7_TYPELESS:
     //case DXGI_FORMAT_BC7_UNORM:
     //case DXGI_FORMAT_BC7_UNORM_SRGB:
+    case DXGI_FORMAT_NV12: return CV_8UC4;
     default: break;
     }
     return errorType;
@@ -701,6 +703,59 @@ static void __OpenCLinitializeD3D11()
 }
 #endif // defined(HAVE_DIRECTX) && defined(HAVE_OPENCL)
 
+} // namespace directx
+
+
+namespace ocl {
+
+#if defined(HAVE_DIRECTX) && defined(HAVE_OPENCL)
+static
+bool ocl_convert_nv12_to_rgba(
+    cl_mem clImageY,
+    cl_mem clImageUV,
+    cl_mem clBuffer,
+    int step,
+    int cols,
+    int rows)
+{
+    ocl::Kernel k;
+    k.create("YUV2RGBA_NV12_8u", cv::ocl::core::cvtclr_dx_oclsrc, "");
+    if (k.empty())
+        return false;
+
+    k.args(clImageY, clImageUV, clBuffer, step, cols, rows);
+
+    size_t globalsize[] = { cols, rows };
+    return k.run(2, globalsize, 0, false);
+}
+
+
+static
+bool ocl_convert_rgba_to_nv12(
+    cl_mem clBuffer,
+    int step,
+    int cols,
+    int rows,
+    cl_mem clImageY,
+    cl_mem clImageUV)
+{
+    ocl::Kernel k;
+    k.create("RGBA2YUV_NV12_8u", cv::ocl::core::cvtclr_dx_oclsrc, "");
+    if (k.empty())
+        return false;
+
+    k.args(clBuffer, step, cols, rows, clImageY, clImageUV);
+
+    size_t globalsize[] = { cols, rows };
+    return k.run(2, globalsize, 0, false);
+}
+#endif // HAVE_DIRECTX && HAVE_OPENCL
+
+} // namespace ocl
+
+
+namespace directx {
+
 void convertToD3D11Texture2D(InputArray src, ID3D11Texture2D* pD3D11Texture2D)
 {
     (void)src; (void)pD3D11Texture2D;
@@ -719,33 +774,63 @@ void convertToD3D11Texture2D(InputArray src, ID3D11Texture2D* pD3D11Texture2D)
     Size srcSize = src.size();
     CV_Assert(srcSize.width == (int)desc.Width && srcSize.height == (int)desc.Height);
 
-    using namespace cv::ocl;
-    Context& ctx = Context::getDefault();
-    cl_context context = (cl_context)ctx.ptr();
-
     UMat u = src.getUMat();
 
     // TODO Add support for roi
     CV_Assert(u.offset == 0);
     CV_Assert(u.isContinuous());
 
+    cl_mem clBuffer = (cl_mem)u.handle(ACCESS_READ);
+
+    using namespace cv::ocl;
+    Context& ctx = Context::getDefault();
+    cl_context context = (cl_context)ctx.ptr();
+
     cl_int status = 0;
-    cl_mem clImage = clCreateFromD3D11Texture2DKHR(context, CL_MEM_WRITE_ONLY, pD3D11Texture2D, 0, &status);
+    cl_mem clImage = 0;
+    cl_mem clImageUV = 0;
+
+    clImage = clCreateFromD3D11Texture2DKHR(context, CL_MEM_WRITE_ONLY, pD3D11Texture2D, 0, &status);
     if (status != CL_SUCCESS)
         CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DKHR failed");
 
-    cl_mem clBuffer = (cl_mem)u.handle(ACCESS_READ);
+    if(DXGI_FORMAT_NV12 == desc.Format)
+    {
+        clImageUV = clCreateFromD3D11Texture2DKHR(context, CL_MEM_WRITE_ONLY, pD3D11Texture2D, 1, &status);
+        if (status != CL_SUCCESS)
+            CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DKHR failed");
+    }
 
     cl_command_queue q = (cl_command_queue)Queue::getDefault().ptr();
+
     status = clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL);
     if (status != CL_SUCCESS)
         CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsKHR failed");
-    size_t offset = 0; // TODO
-    size_t dst_origin[3] = {0, 0, 0};
-    size_t region[3] = {u.cols, u.rows, 1};
-    status = clEnqueueCopyBufferToImage(q, clBuffer, clImage, offset, dst_origin, region, 0, NULL, NULL);
-    if (status != CL_SUCCESS)
-        CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyBufferToImage failed");
+
+    if(DXGI_FORMAT_NV12 == desc.Format)
+    {
+        status = clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL);
+        if (status != CL_SUCCESS)
+            CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsKHR failed");
+
+        if(!ocl::ocl_convert_rgba_to_nv12(clBuffer, (int)u.step[0], u.cols, u.rows, clImage, clImageUV))
+            CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: ocl_convert_rgba_to_nv12 failed");
+
+        status = clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL);
+        if (status != CL_SUCCESS)
+            CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsKHR failed");
+    }
+    else
+    {
+        size_t offset = 0; // TODO
+        size_t origin[3] = { 0, 0, 0 };
+        size_t region[3] = { u.cols, u.rows, 1 };
+
+        status = clEnqueueCopyBufferToImage(q, clBuffer, clImage, offset, origin, region, 0, NULL, NULL);
+        if (status != CL_SUCCESS)
+            CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyBufferToImage failed");
+    }
+
     status = clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL);
     if (status != CL_SUCCESS)
         CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsKHR failed");
@@ -757,11 +842,20 @@ void convertToD3D11Texture2D(InputArray src, ID3D11Texture2D* pD3D11Texture2D)
     status = clReleaseMemObject(clImage); // TODO RAII
     if (status != CL_SUCCESS)
         CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clReleaseMem failed");
+
+    if(DXGI_FORMAT_NV12 == desc.Format)
+    {
+        status = clReleaseMemObject(clImageUV);
+        if (status != CL_SUCCESS)
+            CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clReleaseMem failed");
+    }
 #else
     // TODO memcpy
     NO_OPENCL_SUPPORT_ERROR;
 #endif
 }
+
+
 void convertFromD3D11Texture2D(ID3D11Texture2D* pD3D11Texture2D, OutputArray dst)
 {
     (void)pD3D11Texture2D; (void)dst;
@@ -776,10 +870,6 @@ void convertFromD3D11Texture2D(ID3D11Texture2D* pD3D11Texture2D, OutputArray dst
     int textureType = getTypeFromDXGI_FORMAT(desc.Format);
     CV_Assert(textureType >= 0);
 
-    using namespace cv::ocl;
-    Context& ctx = Context::getDefault();
-    cl_context context = (cl_context)ctx.ptr();
-
     // TODO Need to specify ACCESS_WRITE here somehow to prevent useless data copying!
     dst.create(Size(desc.Width, desc.Height), textureType);
     UMat u = dst.getUMat();
@@ -788,23 +878,57 @@ void convertFromD3D11Texture2D(ID3D11Texture2D* pD3D11Texture2D, OutputArray dst
     CV_Assert(u.offset == 0);
     CV_Assert(u.isContinuous());
 
+    cl_mem clBuffer = (cl_mem)u.handle(ACCESS_READ);
+
+    using namespace cv::ocl;
+    Context& ctx = Context::getDefault();
+    cl_context context = (cl_context)ctx.ptr();
+
     cl_int status = 0;
-    cl_mem clImage = clCreateFromD3D11Texture2DKHR(context, CL_MEM_READ_ONLY, pD3D11Texture2D, 0, &status);
+    cl_mem clImage = 0;
+    cl_mem clImageUV = 0;
+
+    clImage = clCreateFromD3D11Texture2DKHR(context, CL_MEM_READ_ONLY, pD3D11Texture2D, 0, &status);
     if (status != CL_SUCCESS)
         CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DKHR failed");
 
-    cl_mem clBuffer = (cl_mem)u.handle(ACCESS_READ);
+    if(DXGI_FORMAT_NV12 == desc.Format)
+    {
+        clImageUV = clCreateFromD3D11Texture2DKHR(context, CL_MEM_READ_ONLY, pD3D11Texture2D, 1, &status);
+        if (status != CL_SUCCESS)
+            CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DKHR failed");
+    }
 
     cl_command_queue q = (cl_command_queue)Queue::getDefault().ptr();
+
     status = clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL);
     if (status != CL_SUCCESS)
         CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsKHR failed");
-    size_t offset = 0; // TODO
-    size_t src_origin[3] = {0, 0, 0};
-    size_t region[3] = {u.cols, u.rows, 1};
-    status = clEnqueueCopyImageToBuffer(q, clImage, clBuffer, src_origin, region, offset, 0, NULL, NULL);
-    if (status != CL_SUCCESS)
-        CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyImageToBuffer failed");
+
+    if(DXGI_FORMAT_NV12 == desc.Format)
+    {
+        status = clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL);
+        if (status != CL_SUCCESS)
+            CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsKHR failed");
+
+        if(!ocl::ocl_convert_nv12_to_rgba(clImage, clImageUV, clBuffer, (int)u.step[0], u.cols, u.rows))
+            CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: ocl_convert_nv12_to_rgba failed");
+
+        status = clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL);
+        if (status != CL_SUCCESS)
+            CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsKHR failed");
+    }
+    else
+    {
+        size_t offset = 0; // TODO
+        size_t origin[3] = { 0, 0, 0 };
+        size_t region[3] = { u.cols, u.rows, 1 };
+
+        status = clEnqueueCopyImageToBuffer(q, clImage, clBuffer, origin, region, offset, 0, NULL, NULL);
+        if (status != CL_SUCCESS)
+            CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyImageToBuffer failed");
+    }
+
     status = clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL);
     if (status != CL_SUCCESS)
         CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsKHR failed");
@@ -816,6 +940,13 @@ void convertFromD3D11Texture2D(ID3D11Texture2D* pD3D11Texture2D, OutputArray dst
     status = clReleaseMemObject(clImage); // TODO RAII
     if (status != CL_SUCCESS)
         CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clReleaseMem failed");
+
+    if(DXGI_FORMAT_NV12 == desc.Format)
+    {
+        status = clReleaseMemObject(clImageUV);
+        if (status != CL_SUCCESS)
+            CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clReleaseMem failed");
+    }
 #else
     // TODO memcpy
     NO_OPENCL_SUPPORT_ERROR;

modules/core/src/opencl/cvtclr_dx.cl

+/*M///////////////////////////////////////////////////////////////////////////////////////
+//
+//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+//
+//  By downloading, copying, installing or using the software you agree to this license.
+//  If you do not agree to this license, do not download, install,
+//  copy or use the software.
+//
+//
+//                           License Agreement
+//                For Open Source Computer Vision Library
+//
+// Copyright (C) 2010-2012, Institute Of Software Chinese Academy Of Science, all rights reserved.
+// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
+// Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+// Third party copyrights are property of their respective owners.
+//
+// @Authors
+//    Jia Haipeng, jiahaipeng95@gmail.com
+//
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+//   * Redistribution's of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//
+//   * Redistribution's in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//
+//   * The name of the copyright holders may not be used to endorse or promote products
+//     derived from this software without specific prior written permission.
+//
+// This software is provided by the copyright holders and contributors as is and
+// any express or implied warranties, including, but not limited to, the implied
+// warranties of merchantability and fitness for a particular purpose are disclaimed.
+// In no event shall the copyright holders or contributors be liable for any direct,
+// indirect, incidental, special, exemplary, or consequential damages
+// (including, but not limited to, procurement of substitute goods or services;
+// loss of use, data, or profits; or business interruption) however caused
+// and on any theory of liability, whether in contract, strict liability,
+// or tort (including negligence or otherwise) arising in any way out of
+// the use of this software, even if advised of the possibility of such damage.
+//
+//M*/
+
+#ifdef DOUBLE_SUPPORT
+#ifdef cl_amd_fp64
+#pragma OPENCL EXTENSION cl_amd_fp64:enable
+#elif defined cl_khr_fp64
+#pragma OPENCL EXTENSION cl_khr_fp64:enable
+#endif
+#endif
+
+#ifdef INTEL_DEVICE
+#pragma OPENCL FP_CONTRACT ON
+#pragma OPENCL FP_FAST_FMAF ON
+#pragma OPENCL FP_FAST_FMA ON
+#endif
+
+static
+__constant
+float c_YUV2RGBCoeffs_420[5] =
+{
+     1.163999557f,
+     2.017999649f,
+    -0.390999794f,
+    -0.812999725f,
+     1.5959997177f
+};
+
+
+__kernel
+void YUV2RGBA_NV12_8u(
+    read_only image2d_t imgY,
+    read_only image2d_t imgUV,
+    __global unsigned char* pRGBA,
+   int rgbaStep,
+   int cols,
+   int rows)
+{
+    int x = get_global_id(0);
+    int y = get_global_id(1);
+
+    if (x < cols)
+    {
+        if (y < rows)
+        {
+            __global uchar* pDstRow1 = pRGBA + mad24(y, rgbaStep, mad24(x, 4, 0));
+            __global uchar* pDstRow2 = pDstRow1 + rgbaStep;
+
+            float4 Y1 = read_imagef(imgY, (int2)(x+0, y+0)) * 255.0f;
+            float4 Y2 = read_imagef(imgY, (int2)(x+1, y+0)) * 255.0f;
+            float4 Y3 = read_imagef(imgY, (int2)(x+0, y+1)) * 255.0f;
+            float4 Y4 = read_imagef(imgY, (int2)(x+1, y+1)) * 255.0f;
+
+            float4 UV = read_imagef(imgUV, (int2)(x/2, y/2)) * 255.0f - 128.0f;
+
+            __constant float* coeffs = c_YUV2RGBCoeffs_420;
+            float ruv = fma(coeffs[4], UV.y, 0.5f);
+            float guv = fma(coeffs[3], UV.y, fma(coeffs[2], UV.x, 0.5f));
+            float buv = fma(coeffs[1], UV.x, 0.5f);
+
+            Y1 = max(0.f, Y1 - 16.f) * coeffs[0];
+            pDstRow1[0+0] = convert_uchar_sat(Y1.x + ruv);
+            pDstRow1[1+0] = convert_uchar_sat(Y1.x + guv);
+            pDstRow1[2+0] = convert_uchar_sat(Y1.x + buv);
+            pDstRow1[3+0] = 255;
+
+            Y2 = max(0.f, Y2 - 16.f) * coeffs[0];
+            pDstRow1[0+4] = convert_uchar_sat(Y2.x + ruv);
+            pDstRow1[1+4] = convert_uchar_sat(Y2.x + guv);
+            pDstRow1[2+4] = convert_uchar_sat(Y2.x + buv);
+            pDstRow1[3+4] = 255;
+
+            Y3 = max(0.f, Y3 - 16.f) * coeffs[0];
+            pDstRow2[0+0] = convert_uchar_sat(Y3.x + ruv);
+            pDstRow2[1+0] = convert_uchar_sat(Y3.x + guv);
+            pDstRow2[2+0] = convert_uchar_sat(Y3.x + buv);
+            pDstRow2[3+0] = 255;
+
+            Y4 = max(0.f, Y4 - 16.f) * coeffs[0];
+            pDstRow2[0+4] = convert_uchar_sat(Y4.x + ruv);
+            pDstRow2[1+4] = convert_uchar_sat(Y4.x + guv);
+            pDstRow2[2+4] = convert_uchar_sat(Y4.x + buv);
+            pDstRow2[3+4] = 255;
+        }
+    }
+}
+
+
+static
+__constant float c_RGB2YUVCoeffs_420[8] =
+{
+     0.256999969f,  0.50399971f,   0.09799957f,   -0.1479988098f,
+    -0.2909994125f, 0.438999176f, -0.3679990768f, -0.0709991455f
+};
+
+#define scn 4
+__kernel
+void RGBA2YUV_NV12_8u(
+    __global unsigned char* pRGBA,
+    int rgbaStep,
+    int cols,
+    int rows,
+    write_only image2d_t imgY,
+    write_only image2d_t imgUV)
+{
+    int x = get_global_id(0);
+    int y = get_global_id(1);
+
+    if (x < cols)
+    {
+        if (y < rows)
+        {
+            __global const uchar* pSrcRow1 = pRGBA + mad24(y, rgbaStep, mad24(x, scn, 0));
+            __global const uchar* pSrcRow2 = pSrcRow1 + rgbaStep;
+
+            float4 src_pix1 = convert_float4(vload4(0, pSrcRow1 + 0));
+            float4 src_pix2 = convert_float4(vload4(0, pSrcRow1 + scn));
+            float4 src_pix3 = convert_float4(vload4(0, pSrcRow2 + 0));
+            float4 src_pix4 = convert_float4(vload4(0, pSrcRow2 + scn));
+
+            __constant float* coeffs = c_RGB2YUVCoeffs_420;
+
+            uchar Y1 = convert_uchar_sat(fma(coeffs[0], src_pix1.x, fma(coeffs[1], src_pix1.y, fma(coeffs[2], src_pix1.z, 16.5f))));
+            uchar Y2 = convert_uchar_sat(fma(coeffs[0], src_pix2.x, fma(coeffs[1], src_pix2.y, fma(coeffs[2], src_pix2.z, 16.5f))));
+            uchar Y3 = convert_uchar_sat(fma(coeffs[0], src_pix3.x, fma(coeffs[1], src_pix3.y, fma(coeffs[2], src_pix3.z, 16.5f))));
+            uchar Y4 = convert_uchar_sat(fma(coeffs[0], src_pix4.x, fma(coeffs[1], src_pix4.y, fma(coeffs[2], src_pix4.z, 16.5f))));
+
+            write_imageui(imgY, (int2)(x+0, y+0), Y1);
+            write_imageui(imgY, (int2)(x+1, y+0), Y2);
+            write_imageui(imgY, (int2)(x+0, y+1), Y3);
+            write_imageui(imgY, (int2)(x+1, y+1), Y4);
+
+            float uf = fma(coeffs[3], src_pix1.x, fma(coeffs[4], src_pix1.y, fma(coeffs[5], src_pix1.z, 128.5f)));
+            float vf = fma(coeffs[5], src_pix1.x, fma(coeffs[6], src_pix1.y, fma(coeffs[7], src_pix1.z, 128.5f)));
+
+            uchar U = convert_uchar_sat(uf);
+            uchar V = convert_uchar_sat(vf);
+
+            write_imageui(imgUV, (int2)((x/2)+0, (y/2)), U);
+            write_imageui(imgUV, (int2)((x/2)+1, (y/2)), V);
+        }
+    }
+}
\ No newline at end of file

samples/directx/d3d10_interop.cpp

@@ -135,7 +135,7 @@ public:
         if (!m_cap.read(m_frame_bgr))
             return -1;
 
-        cv::cvtColor(m_frame_bgr, m_frame_rgba, CV_RGB2BGRA);
+        cv::cvtColor(m_frame_bgr, m_frame_rgba, CV_BGR2RGBA);
 
         UINT subResource = ::D3D10CalcSubresource(0, 0, 1);
 
@@ -166,6 +166,9 @@ public:
             if (m_shutdown)
                 return 0;
 
+            // capture user input once
+            MODE mode = (m_mode == MODE_GPU_NV12) ? MODE_GPU_RGBA : m_mode;
+
             HRESULT r;
             ID3D10Texture2D* pSurface;
 
@@ -177,7 +180,7 @@ public:
 
             m_timer.start();
 
-            switch (m_mode)
+            switch (mode)
             {
                 case MODE_CPU:
                 {
@@ -214,7 +217,7 @@ public:
                     break;
                 }
 
-                case MODE_GPU:
+                case MODE_GPU_RGBA:
                 {
                     // process video frame on GPU
                     cv::UMat u;
@@ -227,7 +230,7 @@ public:
                         cv::blur(u, u, cv::Size(15, 15), cv::Point(-7, -7));
                     }
 
-                    cv::String strMode = cv::format("mode: %s", m_modeStr[MODE_GPU].c_str());
+                    cv::String strMode = cv::format("mode: %s", m_modeStr[MODE_GPU_RGBA].c_str());
                     cv::String strProcessing = m_demo_processing ? "blur frame" : "copy frame";
                     cv::String strTime = cv::format("time: %4.1f msec", m_timer.time(Timer::UNITS::MSEC));
                     cv::String strDevName = cv::format("OpenCL device: %s", m_oclDevName.c_str());

samples/directx/d3d9_interop.cpp

@@ -108,7 +108,7 @@ public:
         if (!m_cap.read(m_frame_bgr))
             return -1;
 
-        cv::cvtColor(m_frame_bgr, m_frame_rgba, CV_RGB2RGBA);
+        cv::cvtColor(m_frame_bgr, m_frame_rgba, CV_BGR2BGRA);
 
         D3DLOCKED_RECT memDesc = { 0, NULL };
         RECT rc = { 0, 0, m_width, m_height };
@@ -143,6 +143,9 @@ public:
             if (m_shutdown)
                 return 0;
 
+            // capture user input once
+            MODE mode = (m_mode == MODE_GPU_NV12) ? MODE_GPU_RGBA : m_mode;
+
             HRESULT r;
             LPDIRECT3DSURFACE9 pSurface;
 
@@ -154,7 +157,7 @@ public:
 
             m_timer.start();
 
-            switch (m_mode)
+            switch (mode)
             {
                 case MODE_CPU:
                 {
@@ -185,7 +188,7 @@ public:
                     break;
                 }
 
-                case MODE_GPU:
+                case MODE_GPU_RGBA:
                 {
                     // process video frame on GPU
                     cv::UMat u;
@@ -207,7 +210,7 @@ public:
 
             m_timer.stop();
 
-            print_info(pSurface, m_mode, m_timer.time(Timer::UNITS::MSEC), m_oclDevName);
+            print_info(pSurface, mode, m_timer.time(Timer::UNITS::MSEC), m_oclDevName);
 
             // traditional DX render pipeline:
             //   BitBlt surface to backBuffer and flip backBuffer to frontBuffer

samples/directx/d3d9ex_interop.cpp

@@ -108,7 +108,7 @@ public:
         if (!m_cap.read(m_frame_bgr))
             return -1;
 
-        cv::cvtColor(m_frame_bgr, m_frame_rgba, CV_RGB2RGBA);
+        cv::cvtColor(m_frame_bgr, m_frame_rgba, CV_BGR2BGRA);
 
         D3DLOCKED_RECT memDesc = { 0, NULL };
         RECT rc = { 0, 0, m_width, m_height };
@@ -143,6 +143,9 @@ public:
             if (m_shutdown)
                 return 0;
 
+            // capture user input once
+            MODE mode = m_mode == MODE_GPU_NV12 ? MODE_GPU_RGBA : m_mode;
+
             HRESULT r;
             LPDIRECT3DSURFACE9 pSurface;
 
@@ -154,7 +157,7 @@ public:
 
             m_timer.start();
 
-            switch (m_mode)
+            switch (mode)
             {
                 case MODE_CPU:
                 {
@@ -185,7 +188,7 @@ public:
                     break;
                 }
 
-                case MODE_GPU:
+                case MODE_GPU_RGBA:
                 {
                     // process video frame on GPU
                     cv::UMat u;

samples/directx/d3dsample.hpp

@@ -67,7 +67,8 @@ public:
     enum MODE
     {
         MODE_CPU,
-        MODE_GPU
+        MODE_GPU_RGBA,
+        MODE_GPU_NV12
     };
 
     D3DSample(int width, int height, std::string& window_name, cv::VideoCapture& cap) :
@@ -76,7 +77,8 @@ public:
         m_shutdown          = false;
         m_mode              = MODE_CPU;
         m_modeStr[0]        = cv::String("Processing on CPU");
-        m_modeStr[1]        = cv::String("Processing on GPU");
+        m_modeStr[1]        = cv::String("Processing on GPU RGBA");
+        m_modeStr[2]        = cv::String("Processing on GPU NV12");
         m_demo_processing   = false;
         m_cap               = cap;
     }
@@ -104,7 +106,12 @@ protected:
             }
             if (wParam == '2')
             {
-                m_mode = MODE_GPU;
+                m_mode = MODE_GPU_RGBA;
+                return 0;
+            }
+            if (wParam == '3')
+            {
+                m_mode = MODE_GPU_NV12;
                 return 0;
             }
             else if (wParam == VK_SPACE)
@@ -136,7 +143,7 @@ protected:
     bool               m_shutdown;
     bool               m_demo_processing;
     MODE               m_mode;
-    cv::String         m_modeStr[2];
+    cv::String         m_modeStr[3];
     cv::VideoCapture   m_cap;
     cv::Mat            m_frame_bgr;
     cv::Mat            m_frame_rgba;
@@ -151,7 +158,8 @@ static void help()
         "Hot keys: \n"
         "  SPACE - turn processing on/off\n"
         "    1   - process DX surface through OpenCV on CPU\n"
-        "    2   - process DX surface through OpenCV on GPU (via OpenCL)\n"
+        "    2   - process DX RGBA surface through OpenCV on GPU (via OpenCL)\n"
+        "    3   - process DX NV12 surface through OpenCV on GPU (via OpenCL)\n"
         "  ESC   - exit\n\n");
 }