diff --git a/modules/core/src/directx.cpp b/modules/core/src/directx.cpp index 4d9fd6c9e9..9b87302f16 100644 --- a/modules/core/src/directx.cpp +++ b/modules/core/src/directx.cpp @@ -44,6 +44,7 @@ #include "opencv2/core.hpp" #include "opencv2/core/ocl.hpp" #include "opencv2/core/directx.hpp" +#include "opencl_kernels_core.hpp" #ifdef HAVE_DIRECTX #include @@ -167,6 +168,7 @@ int getTypeFromDXGI_FORMAT(const int iDXGI_FORMAT) //case DXGI_FORMAT_BC7_TYPELESS: //case DXGI_FORMAT_BC7_UNORM: //case DXGI_FORMAT_BC7_UNORM_SRGB: + case DXGI_FORMAT_NV12: return CV_8UC4; default: break; } return errorType; @@ -701,6 +703,59 @@ static void __OpenCLinitializeD3D11() } #endif // defined(HAVE_DIRECTX) && defined(HAVE_OPENCL) +} // namespace directx + + +namespace ocl { + +#if defined(HAVE_DIRECTX) && defined(HAVE_OPENCL) +static +bool ocl_convert_nv12_to_rgba( + cl_mem clImageY, + cl_mem clImageUV, + cl_mem clBuffer, + int step, + int cols, + int rows) +{ + ocl::Kernel k; + k.create("YUV2RGBA_NV12_8u", cv::ocl::core::cvtclr_dx_oclsrc, ""); + if (k.empty()) + return false; + + k.args(clImageY, clImageUV, clBuffer, step, cols, rows); + + size_t globalsize[] = { cols, rows }; + return k.run(2, globalsize, 0, false); +} + + +static +bool ocl_convert_rgba_to_nv12( + cl_mem clBuffer, + int step, + int cols, + int rows, + cl_mem clImageY, + cl_mem clImageUV) +{ + ocl::Kernel k; + k.create("RGBA2YUV_NV12_8u", cv::ocl::core::cvtclr_dx_oclsrc, ""); + if (k.empty()) + return false; + + k.args(clBuffer, step, cols, rows, clImageY, clImageUV); + + size_t globalsize[] = { cols, rows }; + return k.run(2, globalsize, 0, false); +} +#endif // HAVE_DIRECTX && HAVE_OPENCL + +} // namespace ocl + + +namespace directx { + void convertToD3D11Texture2D(InputArray src, ID3D11Texture2D* pD3D11Texture2D) { (void)src; (void)pD3D11Texture2D; @@ -719,33 +774,63 @@ void convertToD3D11Texture2D(InputArray src, ID3D11Texture2D* pD3D11Texture2D) Size srcSize = src.size(); CV_Assert(srcSize.width == (int)desc.Width && srcSize.height == (int)desc.Height); - using namespace cv::ocl; - Context& ctx = Context::getDefault(); - cl_context context = (cl_context)ctx.ptr(); - UMat u = src.getUMat(); // TODO Add support for roi CV_Assert(u.offset == 0); CV_Assert(u.isContinuous()); + cl_mem clBuffer = (cl_mem)u.handle(ACCESS_READ); + + using namespace cv::ocl; + Context& ctx = Context::getDefault(); + cl_context context = (cl_context)ctx.ptr(); + cl_int status = 0; - cl_mem clImage = clCreateFromD3D11Texture2DKHR(context, CL_MEM_WRITE_ONLY, pD3D11Texture2D, 0, &status); + cl_mem clImage = 0; + cl_mem clImageUV = 0; + + clImage = clCreateFromD3D11Texture2DKHR(context, CL_MEM_WRITE_ONLY, pD3D11Texture2D, 0, &status); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DKHR failed"); - cl_mem clBuffer = (cl_mem)u.handle(ACCESS_READ); + if(DXGI_FORMAT_NV12 == desc.Format) + { + clImageUV = clCreateFromD3D11Texture2DKHR(context, CL_MEM_WRITE_ONLY, pD3D11Texture2D, 1, &status); + if (status != CL_SUCCESS) + CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DKHR failed"); + } cl_command_queue q = (cl_command_queue)Queue::getDefault().ptr(); + status = clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsKHR failed"); - size_t offset = 0; // TODO - size_t dst_origin[3] = {0, 0, 0}; - size_t region[3] = {u.cols, u.rows, 1}; - status = clEnqueueCopyBufferToImage(q, clBuffer, clImage, offset, dst_origin, region, 0, NULL, NULL); - if (status != CL_SUCCESS) - CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyBufferToImage failed"); + + if(DXGI_FORMAT_NV12 == desc.Format) + { + status = clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL); + if (status != CL_SUCCESS) + CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsKHR failed"); + + if(!ocl::ocl_convert_rgba_to_nv12(clBuffer, (int)u.step[0], u.cols, u.rows, clImage, clImageUV)) + CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: ocl_convert_rgba_to_nv12 failed"); + + status = clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL); + if (status != CL_SUCCESS) + CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsKHR failed"); + } + else + { + size_t offset = 0; // TODO + size_t origin[3] = { 0, 0, 0 }; + size_t region[3] = { u.cols, u.rows, 1 }; + + status = clEnqueueCopyBufferToImage(q, clBuffer, clImage, offset, origin, region, 0, NULL, NULL); + if (status != CL_SUCCESS) + CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyBufferToImage failed"); + } + status = clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsKHR failed"); @@ -757,11 +842,20 @@ void convertToD3D11Texture2D(InputArray src, ID3D11Texture2D* pD3D11Texture2D) status = clReleaseMemObject(clImage); // TODO RAII if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clReleaseMem failed"); + + if(DXGI_FORMAT_NV12 == desc.Format) + { + status = clReleaseMemObject(clImageUV); + if (status != CL_SUCCESS) + CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clReleaseMem failed"); + } #else // TODO memcpy NO_OPENCL_SUPPORT_ERROR; #endif } + + void convertFromD3D11Texture2D(ID3D11Texture2D* pD3D11Texture2D, OutputArray dst) { (void)pD3D11Texture2D; (void)dst; @@ -776,10 +870,6 @@ void convertFromD3D11Texture2D(ID3D11Texture2D* pD3D11Texture2D, OutputArray dst int textureType = getTypeFromDXGI_FORMAT(desc.Format); CV_Assert(textureType >= 0); - using namespace cv::ocl; - Context& ctx = Context::getDefault(); - cl_context context = (cl_context)ctx.ptr(); - // TODO Need to specify ACCESS_WRITE here somehow to prevent useless data copying! dst.create(Size(desc.Width, desc.Height), textureType); UMat u = dst.getUMat(); @@ -788,23 +878,57 @@ void convertFromD3D11Texture2D(ID3D11Texture2D* pD3D11Texture2D, OutputArray dst CV_Assert(u.offset == 0); CV_Assert(u.isContinuous()); + cl_mem clBuffer = (cl_mem)u.handle(ACCESS_READ); + + using namespace cv::ocl; + Context& ctx = Context::getDefault(); + cl_context context = (cl_context)ctx.ptr(); + cl_int status = 0; - cl_mem clImage = clCreateFromD3D11Texture2DKHR(context, CL_MEM_READ_ONLY, pD3D11Texture2D, 0, &status); + cl_mem clImage = 0; + cl_mem clImageUV = 0; + + clImage = clCreateFromD3D11Texture2DKHR(context, CL_MEM_READ_ONLY, pD3D11Texture2D, 0, &status); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DKHR failed"); - cl_mem clBuffer = (cl_mem)u.handle(ACCESS_READ); + if(DXGI_FORMAT_NV12 == desc.Format) + { + clImageUV = clCreateFromD3D11Texture2DKHR(context, CL_MEM_READ_ONLY, pD3D11Texture2D, 1, &status); + if (status != CL_SUCCESS) + CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DKHR failed"); + } cl_command_queue q = (cl_command_queue)Queue::getDefault().ptr(); + status = clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsKHR failed"); - size_t offset = 0; // TODO - size_t src_origin[3] = {0, 0, 0}; - size_t region[3] = {u.cols, u.rows, 1}; - status = clEnqueueCopyImageToBuffer(q, clImage, clBuffer, src_origin, region, offset, 0, NULL, NULL); - if (status != CL_SUCCESS) - CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyImageToBuffer failed"); + + if(DXGI_FORMAT_NV12 == desc.Format) + { + status = clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL); + if (status != CL_SUCCESS) + CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsKHR failed"); + + if(!ocl::ocl_convert_nv12_to_rgba(clImage, clImageUV, clBuffer, (int)u.step[0], u.cols, u.rows)) + CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: ocl_convert_nv12_to_rgba failed"); + + status = clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL); + if (status != CL_SUCCESS) + CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsKHR failed"); + } + else + { + size_t offset = 0; // TODO + size_t origin[3] = { 0, 0, 0 }; + size_t region[3] = { u.cols, u.rows, 1 }; + + status = clEnqueueCopyImageToBuffer(q, clImage, clBuffer, origin, region, offset, 0, NULL, NULL); + if (status != CL_SUCCESS) + CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyImageToBuffer failed"); + } + status = clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsKHR failed"); @@ -816,6 +940,13 @@ void convertFromD3D11Texture2D(ID3D11Texture2D* pD3D11Texture2D, OutputArray dst status = clReleaseMemObject(clImage); // TODO RAII if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clReleaseMem failed"); + + if(DXGI_FORMAT_NV12 == desc.Format) + { + status = clReleaseMemObject(clImageUV); + if (status != CL_SUCCESS) + CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clReleaseMem failed"); + } #else // TODO memcpy NO_OPENCL_SUPPORT_ERROR; diff --git a/modules/core/src/opencl/cvtclr_dx.cl b/modules/core/src/opencl/cvtclr_dx.cl new file mode 100644 index 0000000000..555baaa48e --- /dev/null +++ b/modules/core/src/opencl/cvtclr_dx.cl @@ -0,0 +1,187 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2010-2012, Institute Of Software Chinese Academy Of Science, all rights reserved. +// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved. +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// @Authors +// Jia Haipeng, jiahaipeng95@gmail.com +// +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors as is and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the copyright holders or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#ifdef DOUBLE_SUPPORT +#ifdef cl_amd_fp64 +#pragma OPENCL EXTENSION cl_amd_fp64:enable +#elif defined cl_khr_fp64 +#pragma OPENCL EXTENSION cl_khr_fp64:enable +#endif +#endif + +#ifdef INTEL_DEVICE +#pragma OPENCL FP_CONTRACT ON +#pragma OPENCL FP_FAST_FMAF ON +#pragma OPENCL FP_FAST_FMA ON +#endif + +static +__constant +float c_YUV2RGBCoeffs_420[5] = +{ + 1.163999557f, + 2.017999649f, + -0.390999794f, + -0.812999725f, + 1.5959997177f +}; + + +__kernel +void YUV2RGBA_NV12_8u( + read_only image2d_t imgY, + read_only image2d_t imgUV, + __global unsigned char* pRGBA, + int rgbaStep, + int cols, + int rows) +{ + int x = get_global_id(0); + int y = get_global_id(1); + + if (x < cols) + { + if (y < rows) + { + __global uchar* pDstRow1 = pRGBA + mad24(y, rgbaStep, mad24(x, 4, 0)); + __global uchar* pDstRow2 = pDstRow1 + rgbaStep; + + float4 Y1 = read_imagef(imgY, (int2)(x+0, y+0)) * 255.0f; + float4 Y2 = read_imagef(imgY, (int2)(x+1, y+0)) * 255.0f; + float4 Y3 = read_imagef(imgY, (int2)(x+0, y+1)) * 255.0f; + float4 Y4 = read_imagef(imgY, (int2)(x+1, y+1)) * 255.0f; + + float4 UV = read_imagef(imgUV, (int2)(x/2, y/2)) * 255.0f - 128.0f; + + __constant float* coeffs = c_YUV2RGBCoeffs_420; + float ruv = fma(coeffs[4], UV.y, 0.5f); + float guv = fma(coeffs[3], UV.y, fma(coeffs[2], UV.x, 0.5f)); + float buv = fma(coeffs[1], UV.x, 0.5f); + + Y1 = max(0.f, Y1 - 16.f) * coeffs[0]; + pDstRow1[0+0] = convert_uchar_sat(Y1.x + ruv); + pDstRow1[1+0] = convert_uchar_sat(Y1.x + guv); + pDstRow1[2+0] = convert_uchar_sat(Y1.x + buv); + pDstRow1[3+0] = 255; + + Y2 = max(0.f, Y2 - 16.f) * coeffs[0]; + pDstRow1[0+4] = convert_uchar_sat(Y2.x + ruv); + pDstRow1[1+4] = convert_uchar_sat(Y2.x + guv); + pDstRow1[2+4] = convert_uchar_sat(Y2.x + buv); + pDstRow1[3+4] = 255; + + Y3 = max(0.f, Y3 - 16.f) * coeffs[0]; + pDstRow2[0+0] = convert_uchar_sat(Y3.x + ruv); + pDstRow2[1+0] = convert_uchar_sat(Y3.x + guv); + pDstRow2[2+0] = convert_uchar_sat(Y3.x + buv); + pDstRow2[3+0] = 255; + + Y4 = max(0.f, Y4 - 16.f) * coeffs[0]; + pDstRow2[0+4] = convert_uchar_sat(Y4.x + ruv); + pDstRow2[1+4] = convert_uchar_sat(Y4.x + guv); + pDstRow2[2+4] = convert_uchar_sat(Y4.x + buv); + pDstRow2[3+4] = 255; + } + } +} + + +static +__constant float c_RGB2YUVCoeffs_420[8] = +{ + 0.256999969f, 0.50399971f, 0.09799957f, -0.1479988098f, + -0.2909994125f, 0.438999176f, -0.3679990768f, -0.0709991455f +}; + +#define scn 4 +__kernel +void RGBA2YUV_NV12_8u( + __global unsigned char* pRGBA, + int rgbaStep, + int cols, + int rows, + write_only image2d_t imgY, + write_only image2d_t imgUV) +{ + int x = get_global_id(0); + int y = get_global_id(1); + + if (x < cols) + { + if (y < rows) + { + __global const uchar* pSrcRow1 = pRGBA + mad24(y, rgbaStep, mad24(x, scn, 0)); + __global const uchar* pSrcRow2 = pSrcRow1 + rgbaStep; + + float4 src_pix1 = convert_float4(vload4(0, pSrcRow1 + 0)); + float4 src_pix2 = convert_float4(vload4(0, pSrcRow1 + scn)); + float4 src_pix3 = convert_float4(vload4(0, pSrcRow2 + 0)); + float4 src_pix4 = convert_float4(vload4(0, pSrcRow2 + scn)); + + __constant float* coeffs = c_RGB2YUVCoeffs_420; + + uchar Y1 = convert_uchar_sat(fma(coeffs[0], src_pix1.x, fma(coeffs[1], src_pix1.y, fma(coeffs[2], src_pix1.z, 16.5f)))); + uchar Y2 = convert_uchar_sat(fma(coeffs[0], src_pix2.x, fma(coeffs[1], src_pix2.y, fma(coeffs[2], src_pix2.z, 16.5f)))); + uchar Y3 = convert_uchar_sat(fma(coeffs[0], src_pix3.x, fma(coeffs[1], src_pix3.y, fma(coeffs[2], src_pix3.z, 16.5f)))); + uchar Y4 = convert_uchar_sat(fma(coeffs[0], src_pix4.x, fma(coeffs[1], src_pix4.y, fma(coeffs[2], src_pix4.z, 16.5f)))); + + write_imageui(imgY, (int2)(x+0, y+0), Y1); + write_imageui(imgY, (int2)(x+1, y+0), Y2); + write_imageui(imgY, (int2)(x+0, y+1), Y3); + write_imageui(imgY, (int2)(x+1, y+1), Y4); + + float uf = fma(coeffs[3], src_pix1.x, fma(coeffs[4], src_pix1.y, fma(coeffs[5], src_pix1.z, 128.5f))); + float vf = fma(coeffs[5], src_pix1.x, fma(coeffs[6], src_pix1.y, fma(coeffs[7], src_pix1.z, 128.5f))); + + uchar U = convert_uchar_sat(uf); + uchar V = convert_uchar_sat(vf); + + write_imageui(imgUV, (int2)((x/2)+0, (y/2)), U); + write_imageui(imgUV, (int2)((x/2)+1, (y/2)), V); + } + } +} \ No newline at end of file diff --git a/samples/directx/d3d10_interop.cpp b/samples/directx/d3d10_interop.cpp index 2869e6b86e..a6ca1335c6 100644 --- a/samples/directx/d3d10_interop.cpp +++ b/samples/directx/d3d10_interop.cpp @@ -135,7 +135,7 @@ public: if (!m_cap.read(m_frame_bgr)) return -1; - cv::cvtColor(m_frame_bgr, m_frame_rgba, CV_RGB2BGRA); + cv::cvtColor(m_frame_bgr, m_frame_rgba, CV_BGR2RGBA); UINT subResource = ::D3D10CalcSubresource(0, 0, 1); @@ -166,6 +166,9 @@ public: if (m_shutdown) return 0; + // capture user input once + MODE mode = (m_mode == MODE_GPU_NV12) ? MODE_GPU_RGBA : m_mode; + HRESULT r; ID3D10Texture2D* pSurface; @@ -177,7 +180,7 @@ public: m_timer.start(); - switch (m_mode) + switch (mode) { case MODE_CPU: { @@ -214,7 +217,7 @@ public: break; } - case MODE_GPU: + case MODE_GPU_RGBA: { // process video frame on GPU cv::UMat u; @@ -227,7 +230,7 @@ public: cv::blur(u, u, cv::Size(15, 15), cv::Point(-7, -7)); } - cv::String strMode = cv::format("mode: %s", m_modeStr[MODE_GPU].c_str()); + cv::String strMode = cv::format("mode: %s", m_modeStr[MODE_GPU_RGBA].c_str()); cv::String strProcessing = m_demo_processing ? "blur frame" : "copy frame"; cv::String strTime = cv::format("time: %4.1f msec", m_timer.time(Timer::UNITS::MSEC)); cv::String strDevName = cv::format("OpenCL device: %s", m_oclDevName.c_str()); diff --git a/samples/directx/d3d11_interop.cpp b/samples/directx/d3d11_interop.cpp index 3ac2b063f0..ff62dc2db6 100644 --- a/samples/directx/d3d11_interop.cpp +++ b/samples/directx/d3d11_interop.cpp @@ -57,23 +57,31 @@ public: scd.Flags = DXGI_SWAP_CHAIN_FLAG_ALLOW_MODE_SWITCH; // allow full-screen switching r = ::D3D11CreateDeviceAndSwapChain( - NULL, - D3D_DRIVER_TYPE_HARDWARE, - NULL, - 0, - NULL, - 0, - D3D11_SDK_VERSION, - &scd, - &m_pD3D11SwapChain, - &m_pD3D11Dev, - NULL, - &m_pD3D11Ctx); + NULL, + D3D_DRIVER_TYPE_HARDWARE, + NULL, + 0, + NULL, + 0, + D3D11_SDK_VERSION, + &scd, + &m_pD3D11SwapChain, + &m_pD3D11Dev, + NULL, + &m_pD3D11Ctx); if (FAILED(r)) { throw std::runtime_error("D3D11CreateDeviceAndSwapChain() failed!"); } + m_nv12_available = true; + UINT fmt = 0; + r = m_pD3D11Dev->CheckFormatSupport(DXGI_FORMAT_NV12, &fmt); + if (FAILED(r)) + { + m_nv12_available = false; + } + r = m_pD3D11SwapChain->GetBuffer(0, __uuidof(ID3D11Texture2D), (LPVOID*)&m_pBackBuffer); if (FAILED(r)) { @@ -98,24 +106,70 @@ public: m_pD3D11Ctx->RSSetViewports(1, &viewport); - D3D11_TEXTURE2D_DESC desc; - - desc.Width = m_width; - desc.Height = m_height; - desc.MipLevels = 1; - desc.ArraySize = 1; - desc.Format = DXGI_FORMAT_R8G8B8A8_UNORM; - desc.SampleDesc.Count = 1; - desc.SampleDesc.Quality = 0; - desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; - desc.Usage = D3D11_USAGE_DYNAMIC; - desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; - desc.MiscFlags = 0; - - r = m_pD3D11Dev->CreateTexture2D(&desc, NULL, &m_pSurface); + m_pSurfaceRGBA = 0; + m_pSurfaceNV12 = 0; + + D3D11_TEXTURE2D_DESC desc_rgba; + + desc_rgba.Width = m_width; + desc_rgba.Height = m_height; + desc_rgba.MipLevels = 1; + desc_rgba.ArraySize = 1; + desc_rgba.Format = DXGI_FORMAT_R8G8B8A8_UNORM; + desc_rgba.SampleDesc.Count = 1; + desc_rgba.SampleDesc.Quality = 0; + desc_rgba.BindFlags = D3D11_BIND_SHADER_RESOURCE; + desc_rgba.Usage = D3D11_USAGE_DYNAMIC; + desc_rgba.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; + desc_rgba.MiscFlags = 0; + + r = m_pD3D11Dev->CreateTexture2D(&desc_rgba, 0, &m_pSurfaceRGBA); if (FAILED(r)) { - throw std::runtime_error("Can't create texture with input image"); + throw std::runtime_error("Can't create DX texture"); + } + + if(m_nv12_available) + { + D3D11_TEXTURE2D_DESC desc_nv12; + + desc_nv12.Width = m_width; + desc_nv12.Height = m_height; + desc_nv12.MipLevels = 1; + desc_nv12.ArraySize = 1; + desc_nv12.Format = DXGI_FORMAT_NV12; + desc_nv12.SampleDesc.Count = 1; + desc_nv12.SampleDesc.Quality = 0; + desc_nv12.BindFlags = D3D11_BIND_SHADER_RESOURCE; + desc_nv12.Usage = D3D11_USAGE_DEFAULT; + desc_nv12.CPUAccessFlags = 0; + desc_nv12.MiscFlags = D3D11_RESOURCE_MISC_SHARED; + + r = m_pD3D11Dev->CreateTexture2D(&desc_nv12, 0, &m_pSurfaceNV12); + if (FAILED(r)) + { + throw std::runtime_error("Can't create DX NV12 texture"); + } + + D3D11_TEXTURE2D_DESC desc_nv12_cpu_copy; + + desc_nv12_cpu_copy.Width = m_width; + desc_nv12_cpu_copy.Height = m_height; + desc_nv12_cpu_copy.MipLevels = 1; + desc_nv12_cpu_copy.ArraySize = 1; + desc_nv12_cpu_copy.Format = DXGI_FORMAT_NV12; + desc_nv12_cpu_copy.SampleDesc.Count = 1; + desc_nv12_cpu_copy.SampleDesc.Quality = 0; + desc_nv12_cpu_copy.BindFlags = 0; + desc_nv12_cpu_copy.Usage = D3D11_USAGE_STAGING; + desc_nv12_cpu_copy.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE | D3D11_CPU_ACCESS_READ; + desc_nv12_cpu_copy.MiscFlags = 0; + + r = m_pD3D11Dev->CreateTexture2D(&desc_nv12_cpu_copy, 0, &m_pSurfaceNV12_cpu_copy); + if (FAILED(r)) + { + throw std::runtime_error("Can't create DX NV12 texture"); + } } // initialize OpenCL context of OpenCV lib from DirectX @@ -133,31 +187,42 @@ public: // get media data on DX surface for further processing - int get_surface(ID3D11Texture2D** ppSurface) + int get_surface(ID3D11Texture2D** ppSurface, bool use_nv12) { HRESULT r; if (!m_cap.read(m_frame_bgr)) - throw std::runtime_error("Can't get frame"); + return -1; - cv::cvtColor(m_frame_bgr, m_frame_rgba, CV_RGB2BGRA); + if (use_nv12) + { + cv::cvtColor(m_frame_bgr, m_frame_i420, CV_BGR2YUV_I420); - UINT subResource = ::D3D11CalcSubresource(0, 0, 1); + convert_I420_to_NV12(m_frame_i420, m_frame_nv12, m_width, m_height); - D3D11_MAPPED_SUBRESOURCE mappedTex; - r = m_pD3D11Ctx->Map(m_pSurface, subResource, D3D11_MAP_WRITE_DISCARD, 0, &mappedTex); - if (FAILED(r)) - { - throw std::runtime_error("surface mapping failed!"); + m_pD3D11Ctx->UpdateSubresource(m_pSurfaceNV12, 0, 0, m_frame_nv12.data, (UINT)m_frame_nv12.step[0], (UINT)m_frame_nv12.total()); } + else + { + cv::cvtColor(m_frame_bgr, m_frame_rgba, CV_BGR2RGBA); + + // process video frame on CPU + UINT subResource = ::D3D11CalcSubresource(0, 0, 1); + + D3D11_MAPPED_SUBRESOURCE mappedTex; + r = m_pD3D11Ctx->Map(m_pSurfaceRGBA, subResource, D3D11_MAP_WRITE_DISCARD, 0, &mappedTex); + if (FAILED(r)) + { + throw std::runtime_error("surface mapping failed!"); + } - cv::Mat m(m_height, m_width, CV_8UC4, mappedTex.pData, (int)mappedTex.RowPitch); - // copy video frame data to surface - m_frame_rgba.copyTo(m); + cv::Mat m(m_height, m_width, CV_8UC4, mappedTex.pData, mappedTex.RowPitch); + m_frame_rgba.copyTo(m); - m_pD3D11Ctx->Unmap(m_pSurface, subResource); + m_pD3D11Ctx->Unmap(m_pSurfaceRGBA, subResource); + } - *ppSurface = m_pSurface; + *ppSurface = use_nv12 ? m_pSurfaceNV12 : m_pSurfaceRGBA; return 0; } // get_surface() @@ -171,10 +236,13 @@ public: if (m_shutdown) return 0; + // capture user input once + MODE mode = (m_mode == MODE_GPU_NV12 && !m_nv12_available) ? MODE_GPU_RGBA : m_mode; + HRESULT r; ID3D11Texture2D* pSurface = 0; - r = get_surface(&pSurface); + r = get_surface(&pSurface, mode == MODE_GPU_NV12); if (FAILED(r)) { throw std::runtime_error("get_surface() failed!"); @@ -182,71 +250,113 @@ public: m_timer.start(); - switch (m_mode) + switch (mode) { - case MODE_CPU: + case MODE_CPU: + { + // process video frame on CPU + UINT subResource = ::D3D11CalcSubresource(0, 0, 1); + + D3D11_MAPPED_SUBRESOURCE mappedTex; + r = m_pD3D11Ctx->Map(pSurface, subResource, D3D11_MAP_WRITE_DISCARD, 0, &mappedTex); + if (FAILED(r)) { - // process video frame on CPU - UINT subResource = ::D3D11CalcSubresource(0, 0, 1); + throw std::runtime_error("surface mapping failed!"); + } - D3D11_MAPPED_SUBRESOURCE mappedTex; - r = m_pD3D11Ctx->Map(pSurface, subResource, D3D11_MAP_WRITE_DISCARD, 0, &mappedTex); - if (FAILED(r)) - { - throw std::runtime_error("surface mapping failed!"); - } + cv::Mat m(m_height, m_width, CV_8UC4, mappedTex.pData, (int)mappedTex.RowPitch); - cv::Mat m(m_height, m_width, CV_8UC4, mappedTex.pData, (int)mappedTex.RowPitch); + if (m_demo_processing) + { + // blur data from D3D11 surface with OpenCV on CPU + cv::blur(m, m, cv::Size(15, 15), cv::Point(-7, -7)); + } - if (m_demo_processing) - { - // blur data from D3D11 surface with OpenCV on CPU - cv::blur(m, m, cv::Size(15, 15), cv::Point(-7, -7)); - } + cv::String strMode = cv::format("mode: %s", m_modeStr[MODE_CPU].c_str()); + cv::String strProcessing = m_demo_processing ? "blur frame" : "copy frame"; + cv::String strTime = cv::format("time: %4.1f msec", m_timer.time(Timer::UNITS::MSEC)); + cv::String strDevName = cv::format("OpenCL device: %s", m_oclDevName.c_str()); - cv::String strMode = cv::format("mode: %s", m_modeStr[MODE_CPU].c_str()); - cv::String strProcessing = m_demo_processing ? "blur frame" : "copy frame"; - cv::String strTime = cv::format("time: %4.1f msec", m_timer.time(Timer::UNITS::MSEC)); - cv::String strDevName = cv::format("OpenCL device: %s", m_oclDevName.c_str()); + cv::putText(m, strMode, cv::Point(0, 16), 1, 0.8, cv::Scalar(0, 0, 0)); + cv::putText(m, strProcessing, cv::Point(0, 32), 1, 0.8, cv::Scalar(0, 0, 0)); + cv::putText(m, strTime, cv::Point(0, 48), 1, 0.8, cv::Scalar(0, 0, 0)); + cv::putText(m, strDevName, cv::Point(0, 64), 1, 0.8, cv::Scalar(0, 0, 0)); - cv::putText(m, strMode, cv::Point(0, 16), 1, 0.8, cv::Scalar(0, 0, 0)); - cv::putText(m, strProcessing, cv::Point(0, 32), 1, 0.8, cv::Scalar(0, 0, 0)); - cv::putText(m, strTime, cv::Point(0, 48), 1, 0.8, cv::Scalar(0, 0, 0)); - cv::putText(m, strDevName, cv::Point(0, 64), 1, 0.8, cv::Scalar(0, 0, 0)); + m_pD3D11Ctx->Unmap(pSurface, subResource); - m_pD3D11Ctx->Unmap(pSurface, subResource); + break; + } - break; - } + case MODE_GPU_RGBA: + case MODE_GPU_NV12: + { + // process video frame on GPU + cv::UMat u; + + cv::directx::convertFromD3D11Texture2D(pSurface, u); - case MODE_GPU: + if (m_demo_processing) { - // process video frame on GPU - cv::UMat u; + // blur data from D3D11 surface with OpenCV on GPU with OpenCL + cv::blur(u, u, cv::Size(15, 15), cv::Point(-7, -7)); + } + + cv::String strMode = cv::format("mode: %s", m_modeStr[mode].c_str()); + cv::String strProcessing = m_demo_processing ? "blur frame" : "copy frame"; + cv::String strTime = cv::format("time: %4.1f msec", m_timer.time(Timer::UNITS::MSEC)); + cv::String strDevName = cv::format("OpenCL device: %s", m_oclDevName.c_str()); + + cv::putText(u, strMode, cv::Point(0, 16), 1, 0.8, cv::Scalar(0, 0, 0)); + cv::putText(u, strProcessing, cv::Point(0, 32), 1, 0.8, cv::Scalar(0, 0, 0)); + cv::putText(u, strTime, cv::Point(0, 48), 1, 0.8, cv::Scalar(0, 0, 0)); + cv::putText(u, strDevName, cv::Point(0, 64), 1, 0.8, cv::Scalar(0, 0, 0)); - cv::directx::convertFromD3D11Texture2D(pSurface, u); + cv::directx::convertToD3D11Texture2D(u, pSurface); - if (m_demo_processing) + if (mode == MODE_GPU_NV12) + { + // just for rendering, we need to convert NV12 to RGBA. + m_pD3D11Ctx->CopyResource(m_pSurfaceNV12_cpu_copy, m_pSurfaceNV12); + + // process video frame on CPU { - // blur data from D3D11 surface with OpenCV on GPU with OpenCL - cv::blur(u, u, cv::Size(15, 15), cv::Point(-7, -7)); + UINT subResource = ::D3D11CalcSubresource(0, 0, 1); + + D3D11_MAPPED_SUBRESOURCE mappedTex; + r = m_pD3D11Ctx->Map(m_pSurfaceNV12_cpu_copy, subResource, D3D11_MAP_READ, 0, &mappedTex); + if (FAILED(r)) + { + throw std::runtime_error("surface mapping failed!"); + } + + cv::Mat frame_nv12(m_height + (m_height / 2), m_width, CV_8UC1, mappedTex.pData, mappedTex.RowPitch); + cv::cvtColor(frame_nv12, m_frame_rgba, CV_YUV2RGBA_NV12); + + m_pD3D11Ctx->Unmap(m_pSurfaceNV12_cpu_copy, subResource); } - cv::String strMode = cv::format("mode: %s", m_modeStr[MODE_GPU].c_str()); - cv::String strProcessing = m_demo_processing ? "blur frame" : "copy frame"; - cv::String strTime = cv::format("time: %4.1f msec", m_timer.time(Timer::UNITS::MSEC)); - cv::String strDevName = cv::format("OpenCL device: %s", m_oclDevName.c_str()); + { + UINT subResource = ::D3D11CalcSubresource(0, 0, 1); - cv::putText(u, strMode, cv::Point(0, 16), 1, 0.8, cv::Scalar(0, 0, 0)); - cv::putText(u, strProcessing, cv::Point(0, 32), 1, 0.8, cv::Scalar(0, 0, 0)); - cv::putText(u, strTime, cv::Point(0, 48), 1, 0.8, cv::Scalar(0, 0, 0)); - cv::putText(u, strDevName, cv::Point(0, 64), 1, 0.8, cv::Scalar(0, 0, 0)); + D3D11_MAPPED_SUBRESOURCE mappedTex; + r = m_pD3D11Ctx->Map(m_pSurfaceRGBA, subResource, D3D11_MAP_WRITE_DISCARD, 0, &mappedTex); + if (FAILED(r)) + { + throw std::runtime_error("surface mapping failed!"); + } - cv::directx::convertToD3D11Texture2D(u, pSurface); + cv::Mat m(m_height, m_width, CV_8UC4, mappedTex.pData, mappedTex.RowPitch); + m_frame_rgba.copyTo(m); - break; + m_pD3D11Ctx->Unmap(m_pSurfaceRGBA, subResource); + } + + pSurface = m_pSurfaceRGBA; } + break; + } + } // switch m_timer.stop(); @@ -267,12 +377,14 @@ public: catch (cv::Exception& e) { std::cerr << "Exception: " << e.what() << std::endl; + cleanup(); return 10; } catch (const std::exception& e) { std::cerr << "Exception: " << e.what() << std::endl; + cleanup(); return 11; } @@ -282,7 +394,9 @@ public: int cleanup(void) { - SAFE_RELEASE(m_pSurface); + SAFE_RELEASE(m_pSurfaceRGBA); + SAFE_RELEASE(m_pSurfaceNV12); + SAFE_RELEASE(m_pSurfaceNV12_cpu_copy); SAFE_RELEASE(m_pBackBuffer); SAFE_RELEASE(m_pD3D11SwapChain); SAFE_RELEASE(m_pRenderTarget); @@ -292,16 +406,74 @@ public: return 0; } // cleanup() +protected: + void convert_I420_to_NV12(cv::Mat& i420, cv::Mat& nv12, int width, int height) + { + nv12.create(i420.rows, i420.cols, CV_8UC1); + + unsigned char* pSrcY = i420.data; + unsigned char* pDstY = nv12.data; + size_t srcStep = i420.step[0]; + size_t dstStep = nv12.step[0]; + + { + unsigned char* src; + unsigned char* dst; + + // copy Y plane + for (int i = 0; i < height; i++) + { + src = pSrcY + i*srcStep; + dst = pDstY + i*dstStep; + + for (int j = 0; j < width; j++) + { + dst[j] = src[j]; + } + } + } + + { + // copy U/V planes to UV plane + unsigned char* pSrcU; + unsigned char* pSrcV; + unsigned char* pDstUV; + + size_t uv_offset = height * dstStep; + + for (int i = 0; i < height / 2; i++) + { + pSrcU = pSrcY + height*width + i*(width / 2); + pSrcV = pSrcY + height*width + (height / 2) * (width / 2) + i*(width / 2); + + pDstUV = pDstY + uv_offset + i*dstStep; + + for (int j = 0; j < width / 2; j++) + { + pDstUV[j*2 + 0] = pSrcU[j]; + pDstUV[j*2 + 1] = pSrcV[j]; + } + } + } + + return; + } + private: ID3D11Device* m_pD3D11Dev; IDXGISwapChain* m_pD3D11SwapChain; ID3D11DeviceContext* m_pD3D11Ctx; ID3D11Texture2D* m_pBackBuffer; - ID3D11Texture2D* m_pSurface; + ID3D11Texture2D* m_pSurfaceRGBA; + ID3D11Texture2D* m_pSurfaceNV12; + ID3D11Texture2D* m_pSurfaceNV12_cpu_copy; ID3D11RenderTargetView* m_pRenderTarget; cv::ocl::Context m_oclCtx; cv::String m_oclPlatformName; cv::String m_oclDevName; + bool m_nv12_available; + cv::Mat m_frame_i420; + cv::Mat m_frame_nv12; }; diff --git a/samples/directx/d3d9_interop.cpp b/samples/directx/d3d9_interop.cpp index afe12b3de1..03c8a6bc84 100644 --- a/samples/directx/d3d9_interop.cpp +++ b/samples/directx/d3d9_interop.cpp @@ -108,7 +108,7 @@ public: if (!m_cap.read(m_frame_bgr)) return -1; - cv::cvtColor(m_frame_bgr, m_frame_rgba, CV_RGB2RGBA); + cv::cvtColor(m_frame_bgr, m_frame_rgba, CV_BGR2BGRA); D3DLOCKED_RECT memDesc = { 0, NULL }; RECT rc = { 0, 0, m_width, m_height }; @@ -143,6 +143,9 @@ public: if (m_shutdown) return 0; + // capture user input once + MODE mode = (m_mode == MODE_GPU_NV12) ? MODE_GPU_RGBA : m_mode; + HRESULT r; LPDIRECT3DSURFACE9 pSurface; @@ -154,7 +157,7 @@ public: m_timer.start(); - switch (m_mode) + switch (mode) { case MODE_CPU: { @@ -185,7 +188,7 @@ public: break; } - case MODE_GPU: + case MODE_GPU_RGBA: { // process video frame on GPU cv::UMat u; @@ -207,7 +210,7 @@ public: m_timer.stop(); - print_info(pSurface, m_mode, m_timer.time(Timer::UNITS::MSEC), m_oclDevName); + print_info(pSurface, mode, m_timer.time(Timer::UNITS::MSEC), m_oclDevName); // traditional DX render pipeline: // BitBlt surface to backBuffer and flip backBuffer to frontBuffer diff --git a/samples/directx/d3d9ex_interop.cpp b/samples/directx/d3d9ex_interop.cpp index 187177061c..6d526ba052 100644 --- a/samples/directx/d3d9ex_interop.cpp +++ b/samples/directx/d3d9ex_interop.cpp @@ -108,7 +108,7 @@ public: if (!m_cap.read(m_frame_bgr)) return -1; - cv::cvtColor(m_frame_bgr, m_frame_rgba, CV_RGB2RGBA); + cv::cvtColor(m_frame_bgr, m_frame_rgba, CV_BGR2BGRA); D3DLOCKED_RECT memDesc = { 0, NULL }; RECT rc = { 0, 0, m_width, m_height }; @@ -143,6 +143,9 @@ public: if (m_shutdown) return 0; + // capture user input once + MODE mode = m_mode == MODE_GPU_NV12 ? MODE_GPU_RGBA : m_mode; + HRESULT r; LPDIRECT3DSURFACE9 pSurface; @@ -154,7 +157,7 @@ public: m_timer.start(); - switch (m_mode) + switch (mode) { case MODE_CPU: { @@ -185,7 +188,7 @@ public: break; } - case MODE_GPU: + case MODE_GPU_RGBA: { // process video frame on GPU cv::UMat u; diff --git a/samples/directx/d3dsample.hpp b/samples/directx/d3dsample.hpp index 4b7545fc69..8ab7afb2b5 100644 --- a/samples/directx/d3dsample.hpp +++ b/samples/directx/d3dsample.hpp @@ -67,7 +67,8 @@ public: enum MODE { MODE_CPU, - MODE_GPU + MODE_GPU_RGBA, + MODE_GPU_NV12 }; D3DSample(int width, int height, std::string& window_name, cv::VideoCapture& cap) : @@ -76,7 +77,8 @@ public: m_shutdown = false; m_mode = MODE_CPU; m_modeStr[0] = cv::String("Processing on CPU"); - m_modeStr[1] = cv::String("Processing on GPU"); + m_modeStr[1] = cv::String("Processing on GPU RGBA"); + m_modeStr[2] = cv::String("Processing on GPU NV12"); m_demo_processing = false; m_cap = cap; } @@ -104,7 +106,12 @@ protected: } if (wParam == '2') { - m_mode = MODE_GPU; + m_mode = MODE_GPU_RGBA; + return 0; + } + if (wParam == '3') + { + m_mode = MODE_GPU_NV12; return 0; } else if (wParam == VK_SPACE) @@ -136,7 +143,7 @@ protected: bool m_shutdown; bool m_demo_processing; MODE m_mode; - cv::String m_modeStr[2]; + cv::String m_modeStr[3]; cv::VideoCapture m_cap; cv::Mat m_frame_bgr; cv::Mat m_frame_rgba; @@ -151,7 +158,8 @@ static void help() "Hot keys: \n" " SPACE - turn processing on/off\n" " 1 - process DX surface through OpenCV on CPU\n" - " 2 - process DX surface through OpenCV on GPU (via OpenCL)\n" + " 2 - process DX RGBA surface through OpenCV on GPU (via OpenCL)\n" + " 3 - process DX NV12 surface through OpenCV on GPU (via OpenCL)\n" " ESC - exit\n\n"); }