From fa745553bfcf05b0e7abf093b8cc6ed124471109 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Mon, 19 Feb 2024 10:23:05 +0100 Subject: [PATCH] Merge pull request #24459 from savuor:tri_rasterize Triangle rasterization function #24459 #24065 reopened since the previous one was automatically closed after rebase Connected PR with ground truth data: [#1113@extra](https://github.com/opencv/opencv_extra/pull/1113) ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/3d/include/opencv2/3d.hpp | 130 +++ modules/3d/misc/python/test/test_raster.py | 113 +++ modules/3d/perf/perf_rendering.cpp | 344 +++++++ modules/3d/src/rendering.cpp | 402 ++++++++ modules/3d/test/test_rendering.cpp | 948 +++++++++++++++++++ samples/opengl/CMakeLists.txt | 1 + samples/opengl/opengl_testdata_generator.cpp | 460 +++++++++ 7 files changed, 2398 insertions(+) create mode 100644 modules/3d/misc/python/test/test_raster.py create mode 100644 modules/3d/perf/perf_rendering.cpp create mode 100644 modules/3d/src/rendering.cpp create mode 100644 modules/3d/test/test_rendering.cpp create mode 100644 samples/opengl/opengl_testdata_generator.cpp diff --git a/modules/3d/include/opencv2/3d.hpp b/modules/3d/include/opencv2/3d.hpp index f622b0955c..ee09dd6482 100644 --- a/modules/3d/include/opencv2/3d.hpp +++ b/modules/3d/include/opencv2/3d.hpp @@ -2848,6 +2848,136 @@ CV_EXPORTS_W void loadMesh(const String &filename, OutputArray vertices, OutputA CV_EXPORTS_W void saveMesh(const String &filename, InputArray vertices, InputArrayOfArrays indices, InputArray normals = noArray(), InputArray colors = noArray()); + +//! Triangle fill settings +enum TriangleShadingType +{ + RASTERIZE_SHADING_WHITE = 0, //!< a white color is used for the whole triangle + RASTERIZE_SHADING_FLAT = 1, //!< a color of 1st vertex of each triangle is used + RASTERIZE_SHADING_SHADED = 2 //!< a color is interpolated between 3 vertices with perspective correction +}; + +//! Face culling settings: what faces are drawn after face culling +enum TriangleCullingMode +{ + RASTERIZE_CULLING_NONE = 0, //!< all faces are drawn, no culling is actually performed + RASTERIZE_CULLING_CW = 1, //!< triangles which vertices are given in clockwork order are drawn + RASTERIZE_CULLING_CCW = 2 //!< triangles which vertices are given in counterclockwork order are drawn +}; + +//! GL compatibility settings +enum TriangleGlCompatibleMode +{ + RASTERIZE_COMPAT_DISABLED = 0, //!< Color and depth have their natural values and converted to internal formats if needed + RASTERIZE_COMPAT_INVDEPTH = 1 //!< Color is natural, Depth is transformed from [-zNear; -zFar] to [0; 1] + //!< by the following formula: \f$ \frac{z_{far} \left(z + z_{near}\right)}{z \left(z_{far} - z_{near}\right)} \f$ \n + //!< In this mode the input/output depthBuf is considered to be in this format, + //!< therefore it's faster since there're no conversions performed +}; + +/** + * @brief Structure to keep settings for rasterization + */ +struct CV_EXPORTS_W_SIMPLE TriangleRasterizeSettings +{ + TriangleRasterizeSettings(); + + CV_WRAP TriangleRasterizeSettings& setShadingType(TriangleShadingType st) { shadingType = st; return *this; } + CV_WRAP TriangleRasterizeSettings& setCullingMode(TriangleCullingMode cm) { cullingMode = cm; return *this; } + CV_WRAP TriangleRasterizeSettings& setGlCompatibleMode(TriangleGlCompatibleMode gm) { glCompatibleMode = gm; return *this; } + + TriangleShadingType shadingType; + TriangleCullingMode cullingMode; + TriangleGlCompatibleMode glCompatibleMode; +}; + + +/** @brief Renders a set of triangles on a depth and color image + +Triangles can be drawn white (1.0, 1.0, 1.0), flat-shaded or with a color interpolation between vertices. +In flat-shaded mode the 1st vertex color of each triangle is used to fill the whole triangle. + +The world2cam is an inverted camera pose matrix in fact. It transforms vertices from world to +camera coordinate system. + +The camera coordinate system emulates the OpenGL's coordinate system having coordinate origin in a screen center, +X axis pointing right, Y axis pointing up and Z axis pointing towards the viewer +except that image is vertically flipped after the render. +This means that all visible objects are placed in z-negative area, or exactly in -zNear > z > -zFar since +zNear and zFar are positive. +For example, at fovY = PI/2 the point (0, 1, -1) will be projected to (width/2, 0) screen point, +(1, 0, -1) to (width/2 + height/2, height/2). Increasing fovY makes projection smaller and vice versa. + +The function does not create or clear output images before the rendering. This means that it can be used +for drawing over an existing image or for rendering a model into a 3D scene using pre-filled Z-buffer. + +Empty scene results in a depth buffer filled by the maximum value since every pixel is infinitely far from the camera. +Therefore, before rendering anything from scratch the depthBuf should be filled by zFar values (or by ones in INVDEPTH mode). + +There are special versions of this function named triangleRasterizeDepth and triangleRasterizeColor +for cases if a user needs a color image or a depth image alone; they may run slightly faster. + +@param vertices vertices coordinates array. Should contain values of CV_32FC3 type or a compatible one (e.g. cv::Vec3f, etc.) +@param indices triangle vertices index array, 3 per triangle. Each index indicates a vertex in a vertices array. +Should contain CV_32SC3 values or compatible +@param colors per-vertex colors of CV_32FC3 type or compatible. Can be empty or the same size as vertices array. +If the values are out of [0; 1] range, the result correctness is not guaranteed +@param colorBuf an array representing the final rendered image. Should containt CV_32FC3 values and be the same size as depthBuf. +Not cleared before rendering, i.e. the content is reused as there is some pre-rendered scene. +@param depthBuf an array of floats containing resulting Z buffer. Should contain float values and be the same size as colorBuf. +Not cleared before rendering, i.e. the content is reused as there is some pre-rendered scene. +Empty scene corresponds to all values set to zFar (or to 1.0 in INVDEPTH mode) +@param world2cam a 4x3 or 4x4 float or double matrix containing inverted (sic!) camera pose +@param fovY field of view in vertical direction, given in radians +@param zNear minimum Z value to render, everything closer is clipped +@param zFar maximum Z value to render, everything farther is clipped +@param settings see TriangleRasterizeSettings. By default the smooth shading is on, +with CW culling and with disabled GL compatibility +*/ +CV_EXPORTS_W void triangleRasterize(InputArray vertices, InputArray indices, InputArray colors, + InputOutputArray colorBuf, InputOutputArray depthBuf, + InputArray world2cam, double fovY, double zNear, double zFar, + const TriangleRasterizeSettings& settings = TriangleRasterizeSettings()); + +/** @brief Overloaded version of triangleRasterize() with depth-only rendering + +@param vertices vertices coordinates array. Should contain values of CV_32FC3 type or a compatible one (e.g. cv::Vec3f, etc.) +@param indices triangle vertices index array, 3 per triangle. Each index indicates a vertex in a vertices array. +Should contain CV_32SC3 values or compatible +@param depthBuf an array of floats containing resulting Z buffer. Should contain float values and be the same size as colorBuf. +Not cleared before rendering, i.e. the content is reused as there is some pre-rendered scene. +Empty scene corresponds to all values set to zFar (or to 1.0 in INVDEPTH mode) +@param world2cam a 4x3 or 4x4 float or double matrix containing inverted (sic!) camera pose +@param fovY field of view in vertical direction, given in radians +@param zNear minimum Z value to render, everything closer is clipped +@param zFar maximum Z value to render, everything farther is clipped +@param settings see TriangleRasterizeSettings. By default the smooth shading is on, +with CW culling and with disabled GL compatibility +*/ +CV_EXPORTS_W void triangleRasterizeDepth(InputArray vertices, InputArray indices, InputOutputArray depthBuf, + InputArray world2cam, double fovY, double zNear, double zFar, + const TriangleRasterizeSettings& settings = TriangleRasterizeSettings()); + +/** @brief Overloaded version of triangleRasterize() with color-only rendering + +@param vertices vertices coordinates array. Should contain values of CV_32FC3 type or a compatible one (e.g. cv::Vec3f, etc.) +@param indices triangle vertices index array, 3 per triangle. Each index indicates a vertex in a vertices array. +Should contain CV_32SC3 values or compatible +@param colors per-vertex colors of CV_32FC3 type or compatible. Can be empty or the same size as vertices array. +If the values are out of [0; 1] range, the result correctness is not guaranteed +@param colorBuf an array representing the final rendered image. Should containt CV_32FC3 values and be the same size as depthBuf. +Not cleared before rendering, i.e. the content is reused as there is some pre-rendered scene. +@param world2cam a 4x3 or 4x4 float or double matrix containing inverted (sic!) camera pose +@param fovY field of view in vertical direction, given in radians +@param zNear minimum Z value to render, everything closer is clipped +@param zFar maximum Z value to render, everything farther is clipped +@param settings see TriangleRasterizeSettings. By default the smooth shading is on, +with CW culling and with disabled GL compatibility +*/ +CV_EXPORTS_W void triangleRasterizeColor(InputArray vertices, InputArray indices, InputArray colors, InputOutputArray colorBuf, + InputArray world2cam, double fovY, double zNear, double zFar, + const TriangleRasterizeSettings& settings = TriangleRasterizeSettings()); + //! @} _3d } //end namespace cv diff --git a/modules/3d/misc/python/test/test_raster.py b/modules/3d/misc/python/test/test_raster.py new file mode 100644 index 0000000000..7dce59b4d0 --- /dev/null +++ b/modules/3d/misc/python/test/test_raster.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python + +# Python 2/3 compatibility +from __future__ import print_function + +import os, numpy +import math +import unittest +import cv2 as cv + +from tests_common import NewOpenCVTests + +def lookAtMatrixCal(position, lookat, upVector): + tmp = position - lookat + norm = numpy.linalg.norm(tmp) + w = tmp / norm + tmp = numpy.cross(upVector, w) + norm = numpy.linalg.norm(tmp) + u = tmp / norm + v = numpy.cross(w, u) + res = numpy.array([ + [u[0], u[1], u[2], 0], + [v[0], v[1], v[2], 0], + [w[0], w[1], w[2], 0], + [0, 0, 0, 1.0] + ], dtype=numpy.float32) + translate = numpy.array([ + [1.0, 0, 0, -position[0]], + [0, 1.0, 0, -position[1]], + [0, 0, 1.0, -position[2]], + [0, 0, 0, 1.0] + ], dtype=numpy.float32) + res = numpy.matmul(res, translate) + return res + +class raster_test(NewOpenCVTests): + + def prepareData(self): + self.vertices = numpy.array([ + [ 2.0, 0.0, -2.0], + [ 0.0, -6.0, -2.0], + [-2.0, 0.0, -2.0], + [ 3.5, -1.0, -5.0], + [ 2.5, -2.5, -5.0], + [-1.0, 1.0, -5.0], + [-6.5, -1.0, -3.0], + [-2.5, -2.0, -3.0], + [ 1.0, 1.0, -5.0], + ], dtype=numpy.float32) + + self.indices = numpy.array([ [0, 1, 2], [3, 4, 5], [6, 7, 8]], dtype=int) + + col1 = [ 185.0, 238.0, 217.0 ] + col2 = [ 238.0, 217.0, 185.0 ] + col3 = [ 238.0, 10.0, 150.0 ] + + self.colors = numpy.array([ + col1, col2, col3, + col2, col3, col1, + col3, col1, col2, + ], dtype=numpy.float32) + + self.colors = self.colors / 255.0 + + self.zNear = 0.1 + self.zFar = 50.0 + self.fovy = 45.0 * math.pi / 180.0 + + position = numpy.array([0.0, 0.0, 5.0], dtype=numpy.float32) + lookat = numpy.array([0.0, 0.0, 0.0], dtype=numpy.float32) + upVector = numpy.array([0.0, 1.0, 0.0], dtype=numpy.float32) + self.cameraPose = lookAtMatrixCal(position, lookat, upVector) + + self.depth_buf = numpy.ones((240, 320), dtype=numpy.float32) * self.zFar + self.color_buf = numpy.zeros((240, 320, 3), dtype=numpy.float32) + + self.settings = cv.TriangleRasterizeSettings().setShadingType(cv.RASTERIZE_SHADING_SHADED) + self.settings = self.settings.setCullingMode(cv.RASTERIZE_CULLING_NONE) + + def compareResults(self, needRgb, needDepth): + if needDepth: + depth = self.get_sample('rendering/depth_image_Clipping_320x240_CullNone.png', cv.IMREAD_ANYDEPTH).astype(numpy.float32) + depthFactor = 1000.0 + diff = depth/depthFactor - self.depth_buf + norm = numpy.linalg.norm(diff) + self.assertLessEqual(norm, 356.0) + + if needRgb: + rgb = self.get_sample('rendering/example_image_Clipping_320x240_CullNone_Shaded.png', cv.IMREAD_ANYCOLOR) + diff = rgb/255.0 - self.color_buf + norm = numpy.linalg.norm(diff) + self.assertLessEqual(norm, 11.62) + + def test_rasterizeBoth(self): + self.prepareData() + self.color_buf, self.depth_buf = cv.triangleRasterize(self.vertices, self.indices, self.colors, self.color_buf, self.depth_buf, + self.cameraPose, self.fovy, self.zNear, self.zFar, self.settings) + self.compareResults(needRgb=True, needDepth=True) + + def test_rasterizeDepth(self): + self.prepareData() + self.depth_buf = cv.triangleRasterizeDepth(self.vertices, self.indices, self.depth_buf, + self.cameraPose, self.fovy, self.zNear, self.zFar, self.settings) + self.compareResults(needRgb=False, needDepth=True) + + def test_rasterizeColor(self): + self.prepareData() + self.color_buf = cv.triangleRasterizeColor(self.vertices, self.indices, self.colors, self.color_buf, + self.cameraPose, self.fovy, self.zNear, self.zFar, self.settings) + self.compareResults(needRgb=True, needDepth=False) + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/modules/3d/perf/perf_rendering.cpp b/modules/3d/perf/perf_rendering.cpp new file mode 100644 index 0000000000..48e6bb5739 --- /dev/null +++ b/modules/3d/perf/perf_rendering.cpp @@ -0,0 +1,344 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include "perf_precomp.hpp" + +namespace opencv_test +{ + +// that was easier than using CV_ENUM() macro +namespace +{ +using namespace cv; +struct ShadingTypeEnum +{ + static const std::array vals; + static const std::array svals; + + ShadingTypeEnum(TriangleShadingType v = RASTERIZE_SHADING_WHITE) : val(v) {} + operator TriangleShadingType() const { return val; } + void PrintTo(std::ostream *os) const + { + int v = int(val); + if (v >= 0 && v < (int)vals.size()) + { + *os << svals[v]; + } + else + { + *os << "UNKNOWN"; + } + } + static ::testing::internal::ParamGenerator all() + { + return ::testing::Values(ShadingTypeEnum(vals[0]), + ShadingTypeEnum(vals[1]), + ShadingTypeEnum(vals[2])); + } + +private: + TriangleShadingType val; +}; + +const std::array ShadingTypeEnum::vals +{ + RASTERIZE_SHADING_WHITE, + RASTERIZE_SHADING_FLAT, + RASTERIZE_SHADING_SHADED +}; +const std::array ShadingTypeEnum::svals +{ + std::string("White"), + std::string("Flat"), + std::string("Shaded") +}; + +static inline void PrintTo(const ShadingTypeEnum &t, std::ostream *os) { t.PrintTo(os); } + + +using namespace cv; +struct GlCompatibleModeEnum +{ + static const std::array vals; + static const std::array svals; + + GlCompatibleModeEnum(TriangleGlCompatibleMode v = RASTERIZE_COMPAT_DISABLED) : val(v) {} + operator TriangleGlCompatibleMode() const { return val; } + void PrintTo(std::ostream *os) const + { + int v = int(val); + if (v >= 0 && v < (int)vals.size()) + { + *os << svals[v]; + } + else + { + *os << "UNKNOWN"; + } + } + static ::testing::internal::ParamGenerator all() + { + return ::testing::Values(GlCompatibleModeEnum(vals[0]), + GlCompatibleModeEnum(vals[1])); + } + +private: + TriangleGlCompatibleMode val; +}; + +const std::array GlCompatibleModeEnum::vals +{ + RASTERIZE_COMPAT_DISABLED, + RASTERIZE_COMPAT_INVDEPTH, +}; +const std::array GlCompatibleModeEnum::svals +{ + std::string("Disabled"), + std::string("InvertedDepth"), +}; + +static inline void PrintTo(const GlCompatibleModeEnum &t, std::ostream *os) { t.PrintTo(os); } +} + +enum class Outputs +{ + DepthOnly = 0, + ColorOnly = 1, + DepthColor = 2, +}; + +// that was easier than using CV_ENUM() macro +namespace +{ +using namespace cv; +struct OutputsEnum +{ + static const std::array vals; + static const std::array svals; + + OutputsEnum(Outputs v = Outputs::DepthColor) : val(v) {} + operator Outputs() const { return val; } + void PrintTo(std::ostream *os) const + { + int v = int(val); + if (v >= 0 && v < (int)vals.size()) + { + *os << svals[v]; + } + else + { + *os << "UNKNOWN"; + } + } + static ::testing::internal::ParamGenerator all() + { + return ::testing::Values(OutputsEnum(vals[0]), + OutputsEnum(vals[1]), + OutputsEnum(vals[2])); + } + +private: + Outputs val; +}; + +const std::array OutputsEnum::vals +{ + Outputs::DepthOnly, + Outputs::ColorOnly, + Outputs::DepthColor +}; +const std::array OutputsEnum::svals +{ + std::string("DepthOnly"), + std::string("ColorOnly"), + std::string("DepthColor") +}; + +static inline void PrintTo(const OutputsEnum &t, std::ostream *os) { t.PrintTo(os); } +} + +static Matx44d lookAtMatrixCal(const Vec3d& position, const Vec3d& lookat, const Vec3d& upVector) +{ + Vec3d w = cv::normalize(position - lookat); + Vec3d u = cv::normalize(upVector.cross(w)); + + Vec3d v = w.cross(u); + + Matx44d res(u[0], u[1], u[2], 0, + v[0], v[1], v[2], 0, + w[0], w[1], w[2], 0, + 0, 0, 0, 1.0); + + Matx44d translate(1.0, 0, 0, -position[0], + 0, 1.0, 0, -position[1], + 0, 0, 1.0, -position[2], + 0, 0, 0, 1.0); + res = res * translate; + + return res; +} + + +static void generateNormals(const std::vector& points, const std::vector>& indices, + std::vector& normals) +{ + std::vector> preNormals(points.size(), std::vector()); + + for (const auto& tri : indices) + { + Vec3f p0 = points[tri[0]]; + Vec3f p1 = points[tri[1]]; + Vec3f p2 = points[tri[2]]; + + Vec3f cross = cv::normalize((p1 - p0).cross(p2 - p0)); + for (int i = 0; i < 3; i++) + { + preNormals[tri[i]].push_back(cross); + } + } + + normals.reserve(points.size()); + for (const auto& pn : preNormals) + { + Vec3f sum { }; + for (const auto& n : pn) + { + sum += n; + } + normals.push_back(cv::normalize(sum)); + } +} + +// load model once and keep it in static memory +static void getModelOnce(const std::string& objectPath, std::vector& vertices, + std::vector& indices, std::vector& colors) +{ + static bool load = false; + static std::vector vert, col; + static std::vector ind; + + if (!load) + { + std::vector> indvec; + // using per-vertex normals as colors + loadMesh(objectPath, vert, indvec); + generateNormals(vert, indvec, col); + + for (const auto &vec : indvec) + { + ind.push_back({vec[0], vec[1], vec[2]}); + } + + for (auto &color : col) + { + color = Vec3f(abs(color[0]), abs(color[1]), abs(color[2])); + } + + load = true; + } + + vertices = vert; + colors = col; + indices = ind; +} + +template +std::string printEnum(T v) +{ + std::ostringstream ss; + v.PrintTo(&ss); + return ss.str(); +} + +// resolution, shading type, outputs needed +typedef perf::TestBaseWithParam, ShadingTypeEnum, OutputsEnum, GlCompatibleModeEnum>> RenderingTest; + +PERF_TEST_P(RenderingTest, rasterizeTriangles, ::testing::Combine( + ::testing::Values(std::make_tuple(1920, 1080), std::make_tuple(1024, 768), std::make_tuple(640, 480)), + ShadingTypeEnum::all(), + OutputsEnum::all(), + GlCompatibleModeEnum::all() + )) +{ + auto t = GetParam(); + auto wh = std::get<0>(t); + int width = std::get<0>(wh); + int height = std::get<1>(wh); + auto shadingType = std::get<1>(t); + auto outputs = std::get<2>(t); + auto glCompatibleMode = std::get<3>(t); + + string objectPath = findDataFile("viz/dragon.ply"); + + Vec3f position = Vec3d( 1.9, 0.4, 1.3); + Vec3f lookat = Vec3d( 0.0, 0.0, 0.0); + Vec3f upVector = Vec3d( 0.0, 1.0, 0.0); + + double fovy = 45.0; + + std::vector vertices; + std::vector indices; + std::vector colors; + + getModelOnce(objectPath, vertices, indices, colors); + if (shadingType != RASTERIZE_SHADING_WHITE) + { + // let vertices be in BGR format to avoid later color conversions + // cvtColor does not work with 1d Mats + std::vector xyz; + cv::split(colors, xyz); + cv::merge(std::vector{xyz[2], xyz[1], xyz[0]}, colors); + } + + double zNear = 0.1, zFar = 50.0; + + Matx44d cameraPose = lookAtMatrixCal(position, lookat, upVector); + double fovYradians = fovy * (CV_PI / 180.0); + TriangleRasterizeSettings settings; + settings.setCullingMode(RASTERIZE_CULLING_CW) + .setShadingType(shadingType) + .setGlCompatibleMode(glCompatibleMode); + + Mat depth_buf, color_buf; + while (next()) + { + // Prefilled to measure pure rendering time w/o allocation and clear + float zMax = (glCompatibleMode == RASTERIZE_COMPAT_INVDEPTH) ? 1.f : (float)zFar; + depth_buf = Mat(height, width, CV_32F, zMax); + color_buf = Mat(height, width, CV_32FC3, Scalar::all(0)); + + startTimer(); + if (outputs == Outputs::ColorOnly) + { + cv::triangleRasterizeColor(vertices, indices, colors, color_buf, cameraPose, + fovYradians, zNear, zFar, settings); + } + else if (outputs == Outputs::DepthOnly) + { + cv::triangleRasterizeDepth(vertices, indices, depth_buf, cameraPose, + fovYradians, zNear, zFar, settings); + } + else // Outputs::DepthColor + { + cv::triangleRasterize(vertices, indices, colors, color_buf, depth_buf, + cameraPose, fovYradians, zNear, zFar, settings); + } + stopTimer(); + } + + if (debugLevel > 0) + { + depth_buf.convertTo(depth_buf, CV_16U, 1000.0); + + std::string shadingName = printEnum(shadingType); + std::string suffix = cv::format("%dx%d_%s", width, height, shadingName.c_str()); + + imwrite("perf_color_image_" + suffix + ".png", color_buf * 255.f); + imwrite("perf_depth_image_" + suffix + ".png", depth_buf); + } + + SANITY_CHECK_NOTHING(); +} + +} // namespace diff --git a/modules/3d/src/rendering.cpp b/modules/3d/src/rendering.cpp new file mode 100644 index 0000000000..18c81ff20a --- /dev/null +++ b/modules/3d/src/rendering.cpp @@ -0,0 +1,402 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include "precomp.hpp" + +namespace cv { + +TriangleRasterizeSettings::TriangleRasterizeSettings() +{ + shadingType = RASTERIZE_SHADING_SHADED; + cullingMode = RASTERIZE_CULLING_CW; + glCompatibleMode = RASTERIZE_COMPAT_DISABLED; +} + +static void drawTriangle(Vec4f verts[3], Vec3f colors[3], Mat& depthBuf, Mat& colorBuf, + TriangleRasterizeSettings settings) +{ + // this will be useful during refactoring + // if there's gonna be more supported data types + CV_DbgAssert(depthBuf.empty() || depthBuf.type() == CV_32FC1); + CV_DbgAssert(colorBuf.empty() || colorBuf.type() == CV_32FC3); + + // any of buffers can be empty + int width = std::max(colorBuf.cols, depthBuf.cols); + int height = std::max(colorBuf.rows, depthBuf.rows); + + Point minPt(width, height), maxPt(0, 0); + for (int i = 0; i < 3; i++) + { + // round down to cover the whole pixel + int x = (int)(verts[i][0]), y = (int)(verts[i][1]); + minPt.x = std::min( x, minPt.x); + minPt.y = std::min( y, minPt.y); + maxPt.x = std::max(x + 1, maxPt.x); + maxPt.y = std::max(y + 1, maxPt.y); + } + + minPt.x = std::max(minPt.x, 0); maxPt.x = std::min(maxPt.x, width); + minPt.y = std::max(minPt.y, 0); maxPt.y = std::min(maxPt.y, height); + + Point2f a(verts[0][0], verts[0][1]), b(verts[1][0], verts[1][1]), c(verts[2][0], verts[2][1]); + Point2f bc = b - c, ac = a - c; + float d = ac.x*bc.y - ac.y*bc.x; + + // culling and degenerated triangle removal + if ((settings.cullingMode == RASTERIZE_CULLING_CW && d <= 0) || + (settings.cullingMode == RASTERIZE_CULLING_CCW && d >= 0) || + (abs(d) < 1e-6)) + { + return; + } + + float invd = 1.f / d; + Vec3f zinv { verts[0][2], verts[1][2], verts[2][2] }; + Vec3f w { verts[0][3], verts[1][3], verts[2][3] }; + + for (int y = minPt.y; y < maxPt.y; y++) + { + for (int x = minPt.x; x < maxPt.x; x++) + { + Point2f p(x + 0.5f, y + 0.5f), pc = p - c; + // barycentric coordinates + Vec3f f; + f[0] = ( pc.x * bc.y - pc.y * bc.x) * invd; + f[1] = ( pc.y * ac.x - pc.x * ac.y) * invd; + f[2] = 1.f - f[0] - f[1]; + // if inside the triangle + if ((f[0] >= 0) && (f[1] >= 0) && (f[2] >= 0)) + { + bool update = false; + if (!depthBuf.empty()) + { + float zCurrent = depthBuf.at(height - 1 - y, x); + float zNew = f[0] * zinv[0] + f[1] * zinv[1] + f[2] * zinv[2]; + if (zNew < zCurrent) + { + update = true; + depthBuf.at(height - 1 - y, x) = zNew; + } + } + else // RASTERIZE_SHADING_WHITE + { + update = true; + } + + if (!colorBuf.empty() && update) + { + Vec3f color; + if (settings.shadingType == RASTERIZE_SHADING_WHITE) + { + color = { 1.f, 1.f, 1.f }; + } + else if (settings.shadingType == RASTERIZE_SHADING_FLAT) + { + color = colors[0]; + } + else // TriangleShadingType::Shaded + { + float zInter = 1.0f / (f[0] * w[0] + f[1] * w[1] + f[2] * w[2]); + color = { 0, 0, 0 }; + for (int j = 0; j < 3; j++) + { + color += (f[j] * w[j]) * colors[j]; + } + color *= zInter; + } + colorBuf.at(height - 1 - y, x) = color; + } + } + } + } +} + + +// values outside of [zNear, zFar] have to be restored +// [0, 1] -> [zNear, zFar] +static void linearizeDepth(const Mat& inbuf, const Mat& validMask, Mat outbuf, double zFar, double zNear) +{ + CV_Assert(inbuf.type() == CV_32FC1); + CV_Assert(validMask.type() == CV_8UC1); + CV_Assert(outbuf.type() == CV_32FC1); + CV_Assert(outbuf.size() == inbuf.size()); + + float scaleNear = (float)(1.0 / zNear); + float scaleFar = (float)(1.0 / zFar); + for (int y = 0; y < inbuf.rows; y++) + { + const float* inp = inbuf.ptr(y); + const uchar * validPtr = validMask.ptr(y); + float * outp = outbuf.ptr(y); + for (int x = 0; x < inbuf.cols; x++) + { + if (validPtr[x]) + { + float d = inp[x]; + // precision-optimized version of this: + //float z = - zFar * zNear / (d * (zFar - zNear) - zFar); + float z = 1.f / ((1.f - d) * scaleNear + d * scaleFar ); + outp[x] = z; + } + } + } +} + +// [zNear, zFar] -> [0, 1] +static void invertDepth(const Mat& inbuf, Mat& outbuf, Mat& validMask, double zNear, double zFar) +{ + CV_Assert(inbuf.type() == CV_32FC1); + outbuf.create(inbuf.size(), CV_32FC1); + validMask.create(inbuf.size(), CV_8UC1); + + float fNear = (float)zNear, fFar = (float)zFar; + float zadd = (float)(zFar / (zFar - zNear)); + float zmul = (float)(-zNear * zFar / (zFar - zNear)); + for (int y = 0; y < inbuf.rows; y++) + { + const float * inp = inbuf.ptr(y); + float * outp = outbuf.ptr(y); + uchar * validPtr = validMask.ptr(y); + for (int x = 0; x < inbuf.cols; x++) + { + float z = inp[x]; + uchar m = (z >= fNear) && (z <= fFar); + z = std::max(std::min(z, fFar), fNear); + // precision-optimized version of this: + // outp[x] = (z - zNear) / z * zFar / (zFar - zNear); + outp[x] = zadd + zmul / z; + validPtr[x] = m; + } + } +} + + +static void triangleRasterizeInternal(InputArray _vertices, InputArray _indices, InputArray _colors, + Mat& colorBuf, Mat& depthBuf, + InputArray world2cam, double fovyRadians, double zNear, double zFar, + const TriangleRasterizeSettings& settings) +{ + CV_Assert(world2cam.type() == CV_32FC1 || world2cam.type() == CV_64FC1); + CV_Assert((world2cam.size() == Size {4, 3}) || (world2cam.size() == Size {4, 4})); + + CV_Assert((fovyRadians > 0) && (fovyRadians < CV_PI)); + CV_Assert(zNear > 0); + CV_Assert(zFar > zNear); + + Mat cpMat; + world2cam.getMat().convertTo(cpMat, CV_64FC1); + Matx44d camPoseMat = Matx44d::eye(); + for (int i = 0; i < 3; i++) + { + for (int j = 0; j < 4; j++) + { + camPoseMat(i, j) = cpMat.at(i, j); + } + } + + if(_indices.empty()) + { + return; + } + + CV_CheckFalse(_vertices.empty(), "No vertices provided along with indices array"); + + Mat vertices, colors, triangles; + int nVerts = 0, nColors = 0, nTriangles = 0; + + int vertexType = _vertices.type(); + CV_Assert(vertexType == CV_32FC1 || vertexType == CV_32FC3); + vertices = _vertices.getMat(); + // transform 3xN matrix to Nx3, except 3x3 + if ((_vertices.channels() == 1) && (_vertices.rows() == 3) && (_vertices.cols() != 3)) + { + vertices = vertices.t(); + } + // This transposition is performed on 1xN matrix so it's almost free in terms of performance + vertices = vertices.reshape(3, 1).t(); + nVerts = (int)vertices.total(); + + int indexType = _indices.type(); + CV_Assert(indexType == CV_32SC1 || indexType == CV_32SC3); + triangles = _indices.getMat(); + // transform 3xN matrix to Nx3, except 3x3 + if ((_indices.channels() == 1) && (_indices.rows() == 3) && (_indices.cols() != 3)) + { + triangles = triangles.t(); + } + // This transposition is performed on 1xN matrix so it's almost free in terms of performance + triangles = triangles.reshape(3, 1).t(); + nTriangles = (int)triangles.total(); + + if (!_colors.empty()) + { + int colorType = _colors.type(); + CV_Assert(colorType == CV_32FC1 || colorType == CV_32FC3); + colors = _colors.getMat(); + // transform 3xN matrix to Nx3, except 3x3 + if ((_colors.channels() == 1) && (_colors.rows() == 3) && (_colors.cols() != 3)) + { + colors = colors.t(); + } + colors = colors.reshape(3, 1).t(); + nColors = (int)colors.total(); + + CV_Assert(nColors == nVerts); + } + + // any of buffers can be empty + Size imgSize {std::max(colorBuf.cols, depthBuf.cols), std::max(colorBuf.rows, depthBuf.rows)}; + + // world-to-camera coord system + Matx44d lookAtMatrix = camPoseMat; + + double ys = 1.0 / std::tan(fovyRadians / 2); + double xs = ys / (double)imgSize.width * (double)imgSize.height; + double zz = (zNear + zFar) / (zNear - zFar); + double zw = 2.0 * zFar * zNear / (zNear - zFar); + + // camera to NDC: [-1, 1]^3 + Matx44d perspectMatrix (xs, 0, 0, 0, + 0, ys, 0, 0, + 0, 0, zz, zw, + 0, 0, -1, 0); + + Matx44f mvpMatrix = perspectMatrix * lookAtMatrix; + + // vertex transform stage + + Mat screenVertices(vertices.size(), CV_32FC4); + for (int i = 0; i < nVerts; i++) + { + Vec3f vglobal = vertices.at(i); + + Vec4f vndc = mvpMatrix * Vec4f(vglobal[0], vglobal[1], vglobal[2], 1.f); + + float invw = 1.f / vndc[3]; + Vec4f vdiv = {vndc[0] * invw, vndc[1] * invw, vndc[2] * invw, invw}; + + // [-1, 1]^3 => [0, width] x [0, height] x [0, 1] + Vec4f vscreen = { + (vdiv[0] + 1.f) * 0.5f * (float)imgSize.width, + (vdiv[1] + 1.f) * 0.5f * (float)imgSize.height, + (vdiv[2] + 1.f) * 0.5f, + vdiv[3] + }; + + screenVertices.at(i) = vscreen; + } + + // draw stage + + for (int t = 0; t < nTriangles; t++) + { + Vec3i tri = triangles.at(t); + + Vec3f col[3]; + Vec4f ver[3]; + for (int i = 0; i < 3; i++) + { + int idx = tri[i]; + CV_DbgAssert(idx >= 0 && idx < nVerts); + + col[i] = colors.empty() ? Vec3f::all(0) : colors.at(idx); + ver[i] = screenVertices.at(idx); + } + + drawTriangle(ver, col, depthBuf, colorBuf, settings); + } +} + + +void triangleRasterizeDepth(InputArray _vertices, InputArray _indices, InputOutputArray _depthBuf, + InputArray world2cam, double fovY, double zNear, double zFar, + const TriangleRasterizeSettings& settings) +{ + CV_Assert(!_depthBuf.empty()); + CV_Assert(_depthBuf.type() == CV_32FC1); + + Mat emptyColorBuf; + // out-of-range values from user-provided depthBuf should not be altered, let's mark them + Mat_ validMask; + Mat depthBuf; + if (settings.glCompatibleMode == RASTERIZE_COMPAT_INVDEPTH) + { + depthBuf = _depthBuf.getMat(); + } + else // RASTERIZE_COMPAT_DISABLED + { + invertDepth(_depthBuf.getMat(), depthBuf, validMask, zNear, zFar); + } + + triangleRasterizeInternal(_vertices, _indices, noArray(), emptyColorBuf, depthBuf, world2cam, fovY, zNear, zFar, settings); + + if (settings.glCompatibleMode == RASTERIZE_COMPAT_DISABLED) + { + linearizeDepth(depthBuf, validMask, _depthBuf.getMat(), zFar, zNear); + } +} + +void triangleRasterizeColor(InputArray _vertices, InputArray _indices, InputArray _colors, InputOutputArray _colorBuf, + InputArray world2cam, double fovY, double zNear, double zFar, + const TriangleRasterizeSettings& settings) +{ + CV_Assert(!_colorBuf.empty()); + CV_Assert(_colorBuf.type() == CV_32FC3); + Mat colorBuf = _colorBuf.getMat(); + + Mat depthBuf; + if (_colors.empty()) + { + // full white shading does not require depth test + CV_Assert(settings.shadingType == RASTERIZE_SHADING_WHITE); + } + else + { + // internal depth buffer is not exposed outside + depthBuf.create(_colorBuf.size(), CV_32FC1); + depthBuf.setTo(1.0); + } + + triangleRasterizeInternal(_vertices, _indices, _colors, colorBuf, depthBuf, world2cam, fovY, zNear, zFar, settings); +} + +void triangleRasterize(InputArray _vertices, InputArray _indices, InputArray _colors, + InputOutputArray _colorBuffer, InputOutputArray _depthBuffer, + InputArray world2cam, double fovyRadians, double zNear, double zFar, + const TriangleRasterizeSettings& settings) +{ + if (_colors.empty()) + { + CV_Assert(settings.shadingType == RASTERIZE_SHADING_WHITE); + } + + CV_Assert(!_colorBuffer.empty()); + CV_Assert(_colorBuffer.type() == CV_32FC3); + CV_Assert(!_depthBuffer.empty()); + CV_Assert(_depthBuffer.type() == CV_32FC1); + + CV_Assert(_depthBuffer.size() == _colorBuffer.size()); + + Mat colorBuf = _colorBuffer.getMat(); + + // out-of-range values from user-provided depthBuf should not be altered, let's mark them + Mat_ validMask; + Mat depthBuf; + if (settings.glCompatibleMode == RASTERIZE_COMPAT_INVDEPTH) + { + depthBuf = _depthBuffer.getMat(); + } + else // RASTERIZE_COMPAT_DISABLED + { + invertDepth(_depthBuffer.getMat(), depthBuf, validMask, zNear, zFar); + } + + triangleRasterizeInternal(_vertices, _indices, _colors, colorBuf, depthBuf, world2cam, fovyRadians, zNear, zFar, settings); + + if (settings.glCompatibleMode == RASTERIZE_COMPAT_DISABLED) + { + linearizeDepth(depthBuf, validMask, _depthBuffer.getMat(), zFar, zNear); + } +} +} // namespace cv diff --git a/modules/3d/test/test_rendering.cpp b/modules/3d/test/test_rendering.cpp new file mode 100644 index 0000000000..99b66c0e8e --- /dev/null +++ b/modules/3d/test/test_rendering.cpp @@ -0,0 +1,948 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include "test_precomp.hpp" + +namespace opencv_test { namespace { +using namespace cv; + +// that was easier than using CV_ENUM() macro +namespace +{ +using namespace cv; +struct CullingModeEnum +{ + static const std::array vals; + static const std::array svals; + + CullingModeEnum(TriangleCullingMode v = RASTERIZE_CULLING_NONE) : val(v) {} + operator TriangleCullingMode() const { return val; } + void PrintTo(std::ostream *os) const + { + int v = int(val); + if (v >= 0 && v < (int)vals.size()) + { + *os << svals[v]; + } + else + { + *os << "UNKNOWN"; + } + } + static ::testing::internal::ParamGenerator all() + { + return ::testing::Values(CullingModeEnum(vals[0]), + CullingModeEnum(vals[1]), + CullingModeEnum(vals[2])); + } + +private: + TriangleCullingMode val; +}; + +const std::array CullingModeEnum::vals +{ + RASTERIZE_CULLING_NONE, + RASTERIZE_CULLING_CW, + RASTERIZE_CULLING_CCW +}; +const std::array CullingModeEnum::svals +{ + std::string("None"), + std::string("CW"), + std::string("CCW") +}; + +static inline void PrintTo(const CullingModeEnum &t, std::ostream *os) { t.PrintTo(os); } +} + +// that was easier than using CV_ENUM() macro +namespace +{ +using namespace cv; +struct ShadingTypeEnum +{ + static const std::array vals; + static const std::array svals; + + ShadingTypeEnum(TriangleShadingType v = RASTERIZE_SHADING_WHITE) : val(v) {} + operator TriangleShadingType() const { return val; } + void PrintTo(std::ostream *os) const + { + int v = int(val); + if (v >= 0 && v < (int)vals.size()) + { + *os << svals[v]; + } + else + { + *os << "UNKNOWN"; + } + } + static ::testing::internal::ParamGenerator all() + { + return ::testing::Values(ShadingTypeEnum(vals[0]), + ShadingTypeEnum(vals[1]), + ShadingTypeEnum(vals[2])); + } + +private: + TriangleShadingType val; +}; + +const std::array ShadingTypeEnum::vals +{ + RASTERIZE_SHADING_WHITE, + RASTERIZE_SHADING_FLAT, + RASTERIZE_SHADING_SHADED +}; +const std::array ShadingTypeEnum::svals +{ + std::string("White"), + std::string("Flat"), + std::string("Shaded") +}; + +static inline void PrintTo(const ShadingTypeEnum &t, std::ostream *os) { t.PrintTo(os); } +} + + +enum class ModelType +{ + Empty = 0, + File = 1, + Clipping = 2, + Color = 3, + Centered = 4 +}; + +// that was easier than using CV_ENUM() macro +namespace +{ +using namespace cv; +struct ModelTypeEnum +{ + static const std::array vals; + static const std::array svals; + + ModelTypeEnum(ModelType v = ModelType::Empty) : val(v) {} + operator ModelType() const { return val; } + void PrintTo(std::ostream *os) const + { + int v = int(val); + if (v >= 0 && v < (int)vals.size()) + { + *os << svals[v]; + } + else + { + *os << "UNKNOWN"; + } + } + static ::testing::internal::ParamGenerator all() + { + return ::testing::Values(ModelTypeEnum(vals[0]), + ModelTypeEnum(vals[1]), + ModelTypeEnum(vals[2]), + ModelTypeEnum(vals[3]), + ModelTypeEnum(vals[4])); + } + +private: + ModelType val; +}; + +const std::array ModelTypeEnum::vals +{ + ModelType::Empty, + ModelType::File, + ModelType::Clipping, + ModelType::Color, + ModelType::Centered +}; +const std::array ModelTypeEnum::svals +{ + std::string("Empty"), + std::string("File"), + std::string("Clipping"), + std::string("Color"), + std::string("Centered") +}; + +static inline void PrintTo(const ModelTypeEnum &t, std::ostream *os) { t.PrintTo(os); } +} + +template +std::string printEnum(T v) +{ + std::ostringstream ss; + v.PrintTo(&ss); + return ss.str(); +} + + +static Matx44d lookAtMatrixCal(const Vec3d& position, const Vec3d& lookat, const Vec3d& upVector) +{ + Vec3d w = cv::normalize(position - lookat); + Vec3d u = cv::normalize(upVector.cross(w)); + + Vec3d v = w.cross(u); + + Matx44d res(u[0], u[1], u[2], 0, + v[0], v[1], v[2], 0, + w[0], w[1], w[2], 0, + 0, 0, 0, 1.0); + + Matx44d translate(1.0, 0, 0, -position[0], + 0, 1.0, 0, -position[1], + 0, 0, 1.0, -position[2], + 0, 0, 0, 1.0); + res = res * translate; + + return res; +} + + +static void generateNormals(const std::vector& points, const std::vector>& indices, + std::vector& normals) +{ + std::vector> preNormals(points.size(), std::vector()); + + for (const auto& tri : indices) + { + Vec3f p0 = points[tri[0]]; + Vec3f p1 = points[tri[1]]; + Vec3f p2 = points[tri[2]]; + + Vec3f cross = cv::normalize((p1 - p0).cross(p2 - p0)); + for (int i = 0; i < 3; i++) + { + preNormals[tri[i]].push_back(cross); + } + } + + normals.reserve(points.size()); + for (const auto& pn : preNormals) + { + Vec3f sum { }; + for (const auto& n : pn) + { + sum += n; + } + normals.push_back(cv::normalize(sum)); + } +} + +// load model once and keep it in static memory +static void getModelOnce(const std::string& objectPath, std::vector& vertices, + std::vector& indices, std::vector& colors) +{ + static bool load = false; + static std::vector vert, col; + static std::vector ind; + + if (!load) + { + std::vector> indvec; + // using per-vertex normals as colors + loadMesh(objectPath, vert, indvec); + generateNormals(vert, indvec, col); + + for (const auto &vec : indvec) + { + ind.push_back({vec[0], vec[1], vec[2]}); + } + + for (auto &color : col) + { + color = Vec3f(abs(color[0]), abs(color[1]), abs(color[2])); + } + + load = true; + } + + vertices = vert; + colors = col; + indices = ind; +} + +class ModelData +{ +public: + ModelData(ModelType type = ModelType::Empty) + { + switch (type) + { + case ModelType::Empty: + { + position = Vec3d(0.0, 0.0, 0.0); + lookat = Vec3d(0.0, 0.0, 0.0); + upVector = Vec3d(0.0, 1.0, 0.0); + + fovy = 45.0; + + vertices = std::vector(4, {2.0f, 0, -2.0f}); + colors = std::vector(4, {0, 0, 1.0f}); + indices = { }; + } + break; + case ModelType::File: + { + string objectPath = findDataFile("viz/dragon.ply"); + + position = Vec3d( 1.9, 0.4, 1.3); + lookat = Vec3d( 0.0, 0.0, 0.0); + upVector = Vec3d( 0.0, 1.0, 0.0); + + fovy = 45.0; + + getModelOnce(objectPath, vertices, indices, colors); + } + break; + case ModelType::Clipping: + { + position = Vec3d(0.0, 0.0, 5.0); + lookat = Vec3d(0.0, 0.0, 0.0); + upVector = Vec3d(0.0, 1.0, 0.0); + + fovy = 45.0; + + vertices = + { + { 2.0, 0.0, -2.0}, { 0.0, -6.0, -2.0}, {-2.0, 0.0, -2.0}, + { 3.5, -1.0, -5.0}, { 2.5, -2.5, -5.0}, {-1.0, 1.0, -5.0}, + {-6.5, -1.0, -3.0}, {-2.5, -2.0, -3.0}, { 1.0, 1.0, -5.0}, + }; + + indices = { {0, 1, 2}, {3, 4, 5}, {6, 7, 8} }; + + Vec3f col1(217.0, 238.0, 185.0); + Vec3f col2(185.0, 217.0, 238.0); + Vec3f col3(150.0, 10.0, 238.0); + + col1 *= (1.f / 255.f); + col2 *= (1.f / 255.f); + col3 *= (1.f / 255.f); + + colors = + { + col1, col2, col3, + col2, col3, col1, + col3, col1, col2, + }; + } + break; + case ModelType::Centered: + { + position = Vec3d(0.0, 0.0, 5.0); + lookat = Vec3d(0.0, 0.0, 0.0); + upVector = Vec3d(0.0, 1.0, 0.0); + + fovy = 45.0; + + vertices = + { + { 2.0, 0.0, -2.0}, { 0.0, -2.0, -2.0}, {-2.0, 0.0, -2.0}, + { 3.5, -1.0, -5.0}, { 2.5, -1.5, -5.0}, {-1.0, 0.5, -5.0}, + }; + + indices = { {0, 1, 2}, {3, 4, 5} }; + + Vec3f col1(217.0, 238.0, 185.0); + Vec3f col2(185.0, 217.0, 238.0); + + col1 *= (1.f / 255.f); + col2 *= (1.f / 255.f); + + colors = + { + col1, col2, col1, + col2, col1, col2, + }; + } + break; + case ModelType::Color: + { + position = Vec3d(0.0, 0.0, 5.0); + lookat = Vec3d(0.0, 0.0, 0.0); + upVector = Vec3d(0.0, 1.0, 0.0); + + fovy = 60.0; + + vertices = + { + { 2.0, 0.0, -2.0}, + { 0.0, 2.0, -3.0}, + {-2.0, 0.0, -2.0}, + { 0.0, -2.0, 1.0}, + }; + + indices = { {0, 1, 2}, {0, 2, 3} }; + + colors = + { + { 0.0f, 0.0f, 1.0f}, + { 0.0f, 1.0f, 0.0f}, + { 1.0f, 0.0f, 0.0f}, + { 0.0f, 1.0f, 0.0f}, + }; + } + break; + + default: + CV_Error(Error::StsBadArg, "Unknown model type"); + break; + } + } + + Vec3d position; + Vec3d lookat; + Vec3d upVector; + + double fovy; + + std::vector vertices; + std::vector indices; + std::vector colors; +}; + + +void compareDepth(const cv::Mat& gt, const cv::Mat& mat, cv::Mat& diff, double zFar, double scale, + double maskThreshold, double normInfThreshold, double normL2Threshold) +{ + ASSERT_EQ(CV_16UC1, gt.type()); + ASSERT_EQ(CV_16UC1, mat.type()); + ASSERT_EQ(gt.size(), mat.size()); + + Mat gtMask = gt < zFar*scale; + Mat matMask = mat < zFar*scale; + + Mat diffMask = gtMask != matMask; + int nzDepthDiff = cv::countNonZero(diffMask); + EXPECT_LE(nzDepthDiff, maskThreshold); + + Mat jointMask = gtMask & matMask; + int nzJointMask = cv::countNonZero(jointMask); + double normInfDepth = cv::norm(gt, mat, cv::NORM_INF, jointMask); + EXPECT_LE(normInfDepth, normInfThreshold); + double normL2Depth = nzJointMask ? (cv::norm(gt, mat, cv::NORM_L2, jointMask) / nzJointMask) : 0; + EXPECT_LE(normL2Depth, normL2Threshold); + + // add --test_debug to output differences + if (debugLevel > 0) + { + std::cout << "nzDepthDiff: " << nzDepthDiff << " vs " << maskThreshold << std::endl; + std::cout << "normInfDepth: " << normInfDepth << " vs " << normInfThreshold << std::endl; + std::cout << "normL2Depth: " << normL2Depth << " vs " << normL2Threshold << std::endl; + } + + diff = (gt - mat) + (1 << 15); +} + + +void compareRGB(const cv::Mat& gt, const cv::Mat& mat, cv::Mat& diff, double normInfThreshold, double normL2Threshold) +{ + ASSERT_EQ(CV_32FC3, gt.type()); + ASSERT_EQ(CV_32FC3, mat.type()); + ASSERT_EQ(gt.size(), mat.size()); + + double normInfRgb = cv::norm(gt, mat, cv::NORM_INF); + EXPECT_LE(normInfRgb, normInfThreshold); + double normL2Rgb = cv::norm(gt, mat, cv::NORM_L2) / gt.total(); + EXPECT_LE(normL2Rgb, normL2Threshold); + // add --test_debug to output differences + if (debugLevel > 0) + { + std::cout << "normInfRgb: " << normInfRgb << " vs " << normInfThreshold << std::endl; + std::cout << "normL2Rgb: " << normL2Rgb << " vs " << normL2Threshold << std::endl; + } + diff = (gt - mat) * 0.5 + 0.5; +} + + +struct RenderTestThresholds +{ + RenderTestThresholds( + double _rgbInfThreshold, + double _rgbL2Threshold, + double _depthMaskThreshold, + double _depthInfThreshold, + double _depthL2Threshold) : + rgbInfThreshold(_rgbInfThreshold), + rgbL2Threshold(_rgbL2Threshold), + depthMaskThreshold(_depthMaskThreshold), + depthInfThreshold(_depthInfThreshold), + depthL2Threshold(_depthL2Threshold) + { } + + double rgbInfThreshold; + double rgbL2Threshold; + double depthMaskThreshold; + double depthInfThreshold; + double depthL2Threshold; +}; + +// resolution, shading type, culling mode, model type, float type, index type +typedef std::tuple, ShadingTypeEnum, CullingModeEnum, ModelTypeEnum, MatDepth, MatDepth> RenderTestParamType; + +class RenderingTest : public ::testing::TestWithParam +{ +protected: + void SetUp() override + { + params = GetParam(); + auto wh = std::get<0>(params); + width = std::get<0>(wh); + height = std::get<1>(wh); + shadingType = std::get<1>(params); + cullingMode = std::get<2>(params); + modelType = std::get<3>(params); + modelData = ModelData(modelType); + ftype = std::get<4>(params); + itype = std::get<5>(params); + + zNear = 0.1, zFar = 50.0; + depthScale = 1000.0; + + depth_buf = Mat(height, width, ftype, zFar); + color_buf = Mat(height, width, CV_MAKETYPE(ftype, 3), Scalar::all(0)); + + cameraPose = lookAtMatrixCal(modelData.position, modelData.lookat, modelData.upVector); + fovYradians = modelData.fovy * (CV_PI / 180.0); + + verts = Mat(modelData.vertices); + verts.convertTo(verts, ftype); + + if (shadingType != RASTERIZE_SHADING_WHITE) + { + colors = Mat(modelData.colors); + colors.convertTo(colors, ftype); + // let vertices be in BGR format to avoid later color conversions + // cvtColor does not work with 1d Mats + std::vector xyz; + cv::split(colors, xyz); + cv::merge(std::vector{xyz[2], xyz[1], xyz[0]}, colors); + } + + indices = Mat(modelData.indices); + if (itype != CV_32S) + { + indices.convertTo(indices, itype); + } + + settings = TriangleRasterizeSettings().setCullingMode(cullingMode).setShadingType(shadingType); + + triangleRasterize(verts, indices, colors, color_buf, depth_buf, + cameraPose, fovYradians, zNear, zFar, settings); + } + +public: + RenderTestParamType params; + int width, height; + double zNear, zFar, depthScale; + + Mat depth_buf, color_buf; + + Mat verts, colors, indices; + Matx44d cameraPose; + double fovYradians; + TriangleRasterizeSettings settings; + + ModelData modelData; + ModelTypeEnum modelType; + ShadingTypeEnum shadingType; + CullingModeEnum cullingMode; + int ftype, itype; +}; + + +// depth-only or RGB-only rendering should produce the same result as usual rendering +TEST_P(RenderingTest, noArrays) +{ + Mat depthOnly(height, width, ftype, zFar); + Mat colorOnly(height, width, CV_MAKETYPE(ftype, 3), Scalar::all(0)); + + triangleRasterizeDepth(verts, indices, depthOnly, cameraPose, fovYradians, zNear, zFar, settings); + triangleRasterizeColor(verts, indices, colors, colorOnly, cameraPose, fovYradians, zNear, zFar, settings); + + Mat rgbDiff, depthDiff; + compareRGB(color_buf, colorOnly, rgbDiff, 0, 0); + depth_buf.convertTo(depth_buf, CV_16U, depthScale); + depthOnly.convertTo(depthOnly, CV_16U, depthScale); + compareDepth(depth_buf, depthOnly, depthDiff, zFar, depthScale, 0, 0, 0); + + // add --test_debug to output resulting images + if (debugLevel > 0) + { + std::string modelName = printEnum(modelType); + std::string shadingName = printEnum(shadingType); + std::string cullingName = printEnum(cullingMode); + std::string suffix = cv::format("%s_%dx%d_Cull%s", modelName.c_str(), width, height, cullingName.c_str()); + + std::string outColorPath = "noarray_color_image_" + suffix + "_" + shadingName + ".png"; + std::string outDepthPath = "noarray_depth_image_" + suffix + "_" + shadingName + ".png"; + + imwrite(outColorPath, color_buf * 255.f); + imwrite(outDepthPath, depth_buf); + imwrite("diff_" + outColorPath, rgbDiff * 255.f); + imwrite("diff_" + outDepthPath, depthDiff); + } +} + + +// passing the same parameters in float should give the same result +TEST_P(RenderingTest, floatParams) +{ + Mat depth_buf2(height, width, ftype, zFar); + Mat color_buf2(height, width, CV_MAKETYPE(ftype, 3), Scalar::all(0)); + + // cameraPose can also be float, checking it + triangleRasterize(verts, indices, colors, color_buf2, depth_buf2, + Matx44f(cameraPose), (float)fovYradians, (float)zNear, (float)zFar, settings); + + RenderTestThresholds thr(0, 0, 0, 0, 0); + switch (modelType) + { + case ModelType::Empty: break; + case ModelType::Color: break; + case ModelType::Clipping: + if (width == 320 && height == 240 && shadingType == RASTERIZE_SHADING_FLAT && cullingMode == RASTERIZE_CULLING_CW) + { + thr.depthInfThreshold = 1; + thr.depthL2Threshold = 0.00127; + } + else if (width == 320 && height == 240 && shadingType == RASTERIZE_SHADING_SHADED && cullingMode == RASTERIZE_CULLING_NONE) + { + thr.rgbInfThreshold = 3e-7; + thr.rgbL2Threshold = 1.86e-10; + thr.depthInfThreshold = 1; + thr.depthL2Threshold = 0.000406; + } + else if (width == 256 && height == 256 && shadingType == RASTERIZE_SHADING_SHADED && cullingMode == RASTERIZE_CULLING_CW) + { + thr.rgbInfThreshold = 2.39e-07; + thr.rgbL2Threshold = 1.86e-10; + thr.depthInfThreshold = 1; + thr.depthL2Threshold = 0.0016; + } + else if (width == 256 && height == 256 && shadingType == RASTERIZE_SHADING_FLAT && cullingMode == RASTERIZE_CULLING_CCW) + { + thr.rgbInfThreshold = 0.934; + thr.rgbL2Threshold = 0.000102; + thr.depthMaskThreshold = 21; + } + else if (width == 640 && height == 480 && shadingType == RASTERIZE_SHADING_WHITE && cullingMode == RASTERIZE_CULLING_NONE) + { + thr.rgbL2Threshold = 1; + thr.depthInfThreshold = 1; + thr.depthL2Threshold = 0.000248; + } + else if (width == 700 && height == 700 && shadingType == RASTERIZE_SHADING_FLAT && cullingMode == RASTERIZE_CULLING_CCW) + { + thr.rgbInfThreshold = 0.934; + thr.rgbL2Threshold = 3.18e-5; + thr.depthMaskThreshold = 114; + } + break; + case ModelType::File: + thr.depthInfThreshold = 1; + if (width == 320 && height == 240 && shadingType == RASTERIZE_SHADING_SHADED && cullingMode == RASTERIZE_CULLING_CCW) + { + thr.rgbInfThreshold = 0.000157; + thr.rgbL2Threshold = 6e-09; + thr.depthL2Threshold = 0.000413; + } + else if (width == 700 && height == 700 && shadingType == RASTERIZE_SHADING_SHADED && cullingMode == RASTERIZE_CULLING_CW) + { + thr.rgbInfThreshold = 0.000303; + thr.rgbL2Threshold = 1.9e-09; + thr.depthL2Threshold = 0.00012; + } + else if (width == 700 && height == 700 && shadingType == RASTERIZE_SHADING_WHITE && cullingMode == RASTERIZE_CULLING_NONE) + { + thr.depthL2Threshold = 0.00012; + } + break; + case ModelType::Centered: + if (shadingType == RASTERIZE_SHADING_SHADED && cullingMode != RASTERIZE_CULLING_CW) + { + thr.rgbInfThreshold = 3.58e-07; + thr.rgbL2Threshold = 1.51e-10; + } + break; + } + + Mat rgbDiff, depthDiff; + compareRGB(color_buf, color_buf2, rgbDiff, thr.rgbInfThreshold, thr.rgbL2Threshold); + depth_buf.convertTo(depth_buf, CV_16U, depthScale); + depth_buf2.convertTo(depth_buf2, CV_16U, depthScale); + compareDepth(depth_buf, depth_buf2, depthDiff, zFar, depthScale, thr.depthMaskThreshold, thr.depthInfThreshold, thr.depthL2Threshold); + + // add --test_debug to output resulting images + if (debugLevel > 0) + { + std::string modelName = printEnum(modelType); + std::string shadingName = printEnum(shadingType); + std::string cullingName = printEnum(cullingMode); + std::string suffix = cv::format("%s_%dx%d_Cull%s", modelName.c_str(), width, height, cullingName.c_str()); + + std::string outColorPath = "float_color_image_" + suffix + "_" + shadingName + ".png"; + std::string outDepthPath = "float_depth_image_" + suffix + "_" + shadingName + ".png"; + + imwrite(outColorPath, color_buf * 255.f); + imwrite(outDepthPath, depth_buf); + imwrite("diff_" + outColorPath, rgbDiff * 255.f); + imwrite("diff_" + outDepthPath, depthDiff); + } +} + + +// some culling options produce the same pictures, let's join them +TriangleCullingMode findSameCulling(ModelType modelType, TriangleShadingType shadingType, TriangleCullingMode cullingMode, bool forRgb) +{ + TriangleCullingMode sameCullingMode = cullingMode; + + if ((modelType == ModelType::Centered && cullingMode == RASTERIZE_CULLING_CCW) || + (modelType == ModelType::Color && cullingMode == RASTERIZE_CULLING_CW) || + (modelType == ModelType::File && shadingType == RASTERIZE_SHADING_WHITE && forRgb) || + (modelType == ModelType::File && cullingMode == RASTERIZE_CULLING_CW)) + { + sameCullingMode = RASTERIZE_CULLING_NONE; + } + + return sameCullingMode; +} + +// compare rendering results to the ones produced by samples/opengl/opengl_testdata_generator app +TEST_P(RenderingTest, accuracy) +{ + depth_buf.convertTo(depth_buf, CV_16U, depthScale); + + if (modelType == ModelType::Empty || + (modelType == ModelType::Centered && cullingMode == RASTERIZE_CULLING_CW) || + (modelType == ModelType::Color && cullingMode == RASTERIZE_CULLING_CCW)) + { + // empty image case + EXPECT_EQ(0, cv::norm(color_buf, NORM_INF)); + + Mat depthDiff; + absdiff(depth_buf, Scalar(zFar * depthScale), depthDiff); + EXPECT_EQ(0, cv::norm(depthDiff, cv::NORM_INF)); + } + else + { + RenderTestThresholds thr(0, 0, 0, 0, 0); + switch (modelType) + { + case ModelType::Centered: + if (shadingType == RASTERIZE_SHADING_SHADED) + { + thr.rgbInfThreshold = 0.00218; + thr.rgbL2Threshold = 2.85e-06; + } + break; + case ModelType::Clipping: + if (width == 320 && height == 240 && shadingType == RASTERIZE_SHADING_FLAT && cullingMode == RASTERIZE_CULLING_CW) + { + thr.depthInfThreshold = 1; + thr.depthL2Threshold = 0.0016; + } + else if (width == 320 && height == 240 && shadingType == RASTERIZE_SHADING_SHADED && cullingMode == RASTERIZE_CULLING_NONE) + { + thr.rgbInfThreshold = 0.934; + thr.rgbL2Threshold = 8.03E-05; + thr.depthMaskThreshold = 23; + thr.depthInfThreshold = 1; + thr.depthL2Threshold = 0.000544; + } + else if (width == 256 && height == 256 && shadingType == RASTERIZE_SHADING_SHADED && cullingMode == RASTERIZE_CULLING_CW) + { + thr.rgbInfThreshold = 0.0022; + thr.rgbL2Threshold = 2.54E-06; + thr.depthInfThreshold = 1; + thr.depthL2Threshold = 0.00175; + } + else if (width == 256 && height == 256 && shadingType == RASTERIZE_SHADING_FLAT && cullingMode == RASTERIZE_CULLING_CCW) + { + thr.rgbInfThreshold = 0.934; + thr.rgbL2Threshold = 0.000102; + thr.depthMaskThreshold = 21; + } + else if (width == 640 && height == 480 && shadingType == RASTERIZE_SHADING_WHITE && cullingMode == RASTERIZE_CULLING_NONE) + { + thr.rgbInfThreshold = 1; + thr.rgbL2Threshold = 3.95E-05; + thr.depthMaskThreshold = 49; + thr.depthInfThreshold = 1; + thr.depthL2Threshold = 0.000269; + } + else if (width == 700 && height == 700 && shadingType == RASTERIZE_SHADING_FLAT && cullingMode == RASTERIZE_CULLING_CCW) + { + thr.rgbInfThreshold = 0.934; + thr.rgbL2Threshold = 3.27e-5; + thr.depthMaskThreshold = 121; + } + break; + case ModelType::Color: + thr.depthInfThreshold = 1; + if (width == 320 && height == 240) + { + thr.depthL2Threshold = 0.000989; + } + else if (width == 256 && height == 256) + { + thr.depthL2Threshold = 0.000785; + } + if (shadingType == RASTERIZE_SHADING_SHADED) + { + thr.rgbInfThreshold = 0.0022; + thr.rgbL2Threshold = 3.13e-06; + } + break; + case ModelType::File: + if (width == 320 && height == 240 && shadingType == RASTERIZE_SHADING_SHADED && cullingMode == RASTERIZE_CULLING_CCW) + { + thr.rgbInfThreshold = 0.93; + thr.rgbL2Threshold = 2.45E-05; + thr.depthMaskThreshold = 2; + thr.depthInfThreshold = 99; + thr.depthL2Threshold = 0.00544; + } + else if (width == 700 && height == 700 && shadingType == RASTERIZE_SHADING_SHADED && cullingMode == RASTERIZE_CULLING_CW) + { + thr.rgbInfThreshold = 0.973; + thr.rgbL2Threshold = 4.46E-06; + thr.depthMaskThreshold = 3; + thr.depthInfThreshold = 258; + thr.depthL2Threshold = 0.00142; + } + else if (width == 700 && height == 700 && shadingType == RASTERIZE_SHADING_WHITE && cullingMode == RASTERIZE_CULLING_NONE) + { + thr.rgbInfThreshold = 1; + thr.rgbL2Threshold = 6.13E-06; + thr.depthMaskThreshold = 3; + thr.depthInfThreshold = 258; + thr.depthL2Threshold = 0.00142; + } + break; + default: + break; + } + + CullingModeEnum cullingModeRgb = findSameCulling(modelType, shadingType, cullingMode, true); + CullingModeEnum cullingModeDepth = findSameCulling(modelType, shadingType, cullingMode, false); + + std::string modelName = printEnum(modelType); + std::string shadingName = printEnum(shadingType); + std::string cullingName = printEnum(cullingMode); + std::string cullingRgbName = printEnum(cullingModeRgb); + std::string cullingDepthName = printEnum(cullingModeDepth); + + std::string path = findDataDirectory("rendering"); + std::string suffix = cv::format("%s_%dx%d_Cull%s", modelName.c_str(), width, height, cullingName.c_str()); + std::string suffixRgb = cv::format("%s_%dx%d_Cull%s", modelName.c_str(), width, height, cullingRgbName.c_str()); + std::string suffixDepth = cv::format("%s_%dx%d_Cull%s", modelName.c_str(), width, height, cullingDepthName.c_str()); + std::string gtPathColor = path + "/example_image_" + suffixRgb + "_" + shadingName + ".png"; + std::string gtPathDepth = path + "/depth_image_" + suffixDepth + ".png"; + + Mat rgbDiff, depthDiff; + Mat groundTruthColor = imread(gtPathColor); + groundTruthColor.convertTo(groundTruthColor, CV_32F, (1.f / 255.f)); + compareRGB(groundTruthColor, color_buf, rgbDiff, thr.rgbInfThreshold, thr.rgbL2Threshold); + + Mat groundTruthDepth = imread(gtPathDepth, cv::IMREAD_GRAYSCALE | cv::IMREAD_ANYDEPTH); + compareDepth(groundTruthDepth, depth_buf, depthDiff, zFar, depthScale, thr.depthMaskThreshold, thr.depthInfThreshold, thr.depthL2Threshold); + + // add --test_debug to output resulting images + if (debugLevel > 0) + { + std::string outColorPath = "color_image_" + suffix + "_" + shadingName + ".png"; + std::string outDepthPath = "depth_image_" + suffix + "_" + shadingName + ".png"; + imwrite(outColorPath, color_buf * 255.f); + imwrite(outDepthPath, depth_buf); + imwrite("diff_" + outColorPath, rgbDiff * 255.f); + imwrite("diff_" + outDepthPath, depthDiff); + } + } +} + + +// drawing model as a whole or as two halves should give the same result +TEST_P(RenderingTest, keepDrawnData) +{ + if (modelType != ModelType::Empty) + { + Mat depth_buf2(height, width, ftype, zFar); + Mat color_buf2(height, width, CV_MAKETYPE(ftype, 3), Scalar::all(0)); + + Mat idx1, idx2; + int nTriangles = (int)indices.total(); + idx1 = indices.reshape(3, 1)(Range::all(), Range(0, nTriangles / 2)); + idx2 = indices.reshape(3, 1)(Range::all(), Range(nTriangles / 2, nTriangles)); + + triangleRasterize(verts, idx1, colors, color_buf2, depth_buf2, cameraPose, fovYradians, zNear, zFar, settings); + triangleRasterize(verts, idx2, colors, color_buf2, depth_buf2, cameraPose, fovYradians, zNear, zFar, settings); + + Mat rgbDiff, depthDiff; + compareRGB(color_buf, color_buf2, rgbDiff, 0, 0); + depth_buf.convertTo(depth_buf, CV_16U, depthScale); + depth_buf2.convertTo(depth_buf2, CV_16U, depthScale); + compareDepth(depth_buf, depth_buf2, depthDiff, zFar, depthScale, 0, 0, 0); + } +} + + +TEST_P(RenderingTest, glCompatibleDepth) +{ + Mat depth_buf2(height, width, ftype, 1.0); + + triangleRasterizeDepth(verts, indices, depth_buf2, cameraPose, fovYradians, zNear, zFar, + settings.setGlCompatibleMode(RASTERIZE_COMPAT_INVDEPTH)); + + Mat convertedDepth(height, width, ftype); + // map from [0, 1] to [zNear, zFar] + double scaleNear = (1.0 / zNear); + double scaleFar = (1.0 / zFar); + for (int y = 0; y < height; y++) + { + for (int x = 0; x < width; x++) + { + double z = (double)depth_buf2.at(y, x); + convertedDepth.at(y, x) = (float)(1.0 / ((1.0 - z) * scaleNear + z * scaleFar )); + } + } + + double normL2Diff = cv::norm(depth_buf, convertedDepth, cv::NORM_L2) / (height * width); + const double normL2Threshold = 1.e-9; + EXPECT_LE(normL2Diff, normL2Threshold); + // add --test_debug to output differences + if (debugLevel > 0) + { + std::cout << "normL2Diff: " << normL2Diff << " vs " << normL2Threshold << std::endl; + } +} + + +INSTANTIATE_TEST_CASE_P(Rendering, RenderingTest, ::testing::Values( + RenderTestParamType { std::make_tuple(320, 240), RASTERIZE_SHADING_SHADED, RASTERIZE_CULLING_NONE, ModelType::Centered, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(256, 256), RASTERIZE_SHADING_SHADED, RASTERIZE_CULLING_NONE, ModelType::Centered, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(256, 256), RASTERIZE_SHADING_WHITE, RASTERIZE_CULLING_NONE, ModelType::Centered, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(640, 480), RASTERIZE_SHADING_FLAT, RASTERIZE_CULLING_NONE, ModelType::Centered, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(320, 240), RASTERIZE_SHADING_FLAT, RASTERIZE_CULLING_CW, ModelType::Color, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(320, 240), RASTERIZE_SHADING_SHADED, RASTERIZE_CULLING_NONE, ModelType::Color, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(256, 256), RASTERIZE_SHADING_SHADED, RASTERIZE_CULLING_NONE, ModelType::Color, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(256, 256), RASTERIZE_SHADING_WHITE, RASTERIZE_CULLING_NONE, ModelType::Color, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(320, 240), RASTERIZE_SHADING_FLAT, RASTERIZE_CULLING_CW, ModelType::Clipping, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(320, 240), RASTERIZE_SHADING_SHADED, RASTERIZE_CULLING_NONE, ModelType::Clipping, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(256, 256), RASTERIZE_SHADING_FLAT, RASTERIZE_CULLING_CCW, ModelType::Clipping, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(256, 256), RASTERIZE_SHADING_SHADED, RASTERIZE_CULLING_CW, ModelType::Clipping, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(640, 480), RASTERIZE_SHADING_WHITE, RASTERIZE_CULLING_NONE, ModelType::Clipping, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(700, 700), RASTERIZE_SHADING_FLAT, RASTERIZE_CULLING_CCW, ModelType::Clipping, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(320, 240), RASTERIZE_SHADING_SHADED, RASTERIZE_CULLING_CCW, ModelType::File, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(700, 700), RASTERIZE_SHADING_SHADED, RASTERIZE_CULLING_CW, ModelType::File, CV_32F, CV_32S }, + RenderTestParamType { std::make_tuple(700, 700), RASTERIZE_SHADING_WHITE, RASTERIZE_CULLING_NONE, ModelType::File, CV_32F, CV_32S } +)); + +} // namespace :: +} // namespace opencv_test diff --git a/samples/opengl/CMakeLists.txt b/samples/opengl/CMakeLists.txt index 158151c300..419e83a66c 100644 --- a/samples/opengl/CMakeLists.txt +++ b/samples/opengl/CMakeLists.txt @@ -10,6 +10,7 @@ SET(OPENCV_OPENGL_SAMPLES_REQUIRED_DEPS opencv_core opencv_imgproc opencv_imgcodecs + opencv_3d opencv_videoio opencv_highgui) ocv_check_dependencies(${OPENCV_OPENGL_SAMPLES_REQUIRED_DEPS}) diff --git a/samples/opengl/opengl_testdata_generator.cpp b/samples/opengl/opengl_testdata_generator.cpp new file mode 100644 index 0000000000..0f6bf6e42b --- /dev/null +++ b/samples/opengl/opengl_testdata_generator.cpp @@ -0,0 +1,460 @@ +#include + +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN 1 +#define NOMINMAX 1 +#include +#endif + +#if defined(__APPLE__) +#include +#include +#else +#include +#include +#endif + +#include "opencv2/core.hpp" +#include "opencv2/core/opengl.hpp" +#include "opencv2/3d.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/highgui.hpp" + +using namespace std; +using namespace cv; +using namespace cv::cuda; + +// model data should be identical to the code from tests +enum class ModelType +{ + Empty = 0, + File = 1, + Clipping = 2, + Color = 3, + Centered = 4 +}; + +static void generateNormals(const std::vector& points, const std::vector>& indices, + std::vector& normals) +{ + std::vector> preNormals(points.size(), std::vector()); + + for (const auto& tri : indices) + { + Vec3f p0 = points[tri[0]]; + Vec3f p1 = points[tri[1]]; + Vec3f p2 = points[tri[2]]; + + Vec3f cross = cv::normalize((p1 - p0).cross(p2 - p0)); + for (int i = 0; i < 3; i++) + { + preNormals[tri[i]].push_back(cross); + } + } + + normals.reserve(points.size()); + for (const auto& pn : preNormals) + { + Vec3f sum { }; + for (const auto& n : pn) + { + sum += n; + } + normals.push_back(cv::normalize(sum)); + } +} + +class ModelData +{ +public: + ModelData(ModelType type = ModelType::Empty, std::string objPath = { }) + { + switch (type) + { + case ModelType::Empty: + { + position = Vec3d(0.0, 0.0, 0.0); + lookat = Vec3d(0.0, 0.0, 0.0); + upVector = Vec3d(0.0, 1.0, 0.0); + + fovy = 45.0; + + vertices = std::vector(4, {2.0f, 0, -2.0f}); + colors = std::vector(4, {0, 0, 1.0f}); + indices = { }; + } + break; + case ModelType::File: + { + position = Vec3d( 1.9, 0.4, 1.3); + lookat = Vec3d( 0.0, 0.0, 0.0); + upVector = Vec3d( 0.0, 1.0, 0.0); + + fovy = 45.0; + + objectPath = objPath; + std::vector> indvec; + loadMesh(objectPath, vertices, indvec); + // using per-vertex normals as colors + generateNormals(vertices, indvec, colors); + if (vertices.size() != colors.size()) + { + std::runtime_error("Model should contain normals for each vertex"); + } + for (const auto &vec : indvec) + { + indices.push_back({vec[0], vec[1], vec[2]}); + } + + for (auto &color : colors) + { + color = Vec3f(abs(color[0]), abs(color[1]), abs(color[2])); + } + } + break; + case ModelType::Clipping: + { + position = Vec3d(0.0, 0.0, 5.0); + lookat = Vec3d(0.0, 0.0, 0.0); + upVector = Vec3d(0.0, 1.0, 0.0); + + fovy = 45.0; + + vertices = + { + { 2.0, 0.0, -2.0}, { 0.0, -6.0, -2.0}, {-2.0, 0.0, -2.0}, + { 3.5, -1.0, -5.0}, { 2.5, -2.5, -5.0}, {-1.0, 1.0, -5.0}, + {-6.5, -1.0, -3.0}, {-2.5, -2.0, -3.0}, { 1.0, 1.0, -5.0}, + }; + + indices = { {0, 1, 2}, {3, 4, 5}, {6, 7, 8} }; + + Vec3f col1(217.0, 238.0, 185.0); + Vec3f col2(185.0, 217.0, 238.0); + Vec3f col3(150.0, 10.0, 238.0); + + col1 *= (1.f / 255.f); + col2 *= (1.f / 255.f); + col3 *= (1.f / 255.f); + + colors = + { + col1, col2, col3, + col2, col3, col1, + col3, col1, col2, + }; + } + break; + case ModelType::Centered: + { + position = Vec3d(0.0, 0.0, 5.0); + lookat = Vec3d(0.0, 0.0, 0.0); + upVector = Vec3d(0.0, 1.0, 0.0); + + fovy = 45.0; + + vertices = + { + { 2.0, 0.0, -2.0}, { 0.0, -2.0, -2.0}, {-2.0, 0.0, -2.0}, + { 3.5, -1.0, -5.0}, { 2.5, -1.5, -5.0}, {-1.0, 0.5, -5.0}, + }; + + indices = { {0, 1, 2}, {3, 4, 5} }; + + Vec3f col1(217.0, 238.0, 185.0); + Vec3f col2(185.0, 217.0, 238.0); + + col1 *= (1.f / 255.f); + col2 *= (1.f / 255.f); + + colors = + { + col1, col2, col1, + col2, col1, col2, + }; + } + break; + case ModelType::Color: + { + position = Vec3d(0.0, 0.0, 5.0); + lookat = Vec3d(0.0, 0.0, 0.0); + upVector = Vec3d(0.0, 1.0, 0.0); + + fovy = 60.0; + + vertices = + { + { 2.0, 0.0, -2.0}, + { 0.0, 2.0, -3.0}, + {-2.0, 0.0, -2.0}, + { 0.0, -2.0, 1.0}, + }; + + indices = { {0, 1, 2}, {0, 2, 3} }; + + colors = + { + { 0.0f, 0.0f, 1.0f}, + { 0.0f, 1.0f, 0.0f}, + { 1.0f, 0.0f, 0.0f}, + { 0.0f, 1.0f, 0.0f}, + }; + } + break; + + default: + CV_Error(Error::StsBadArg, "Unknown model type"); + break; + } + } + + Vec3d position; + Vec3d lookat; + Vec3d upVector; + + double fovy; + + std::vector vertices; + std::vector indices; + std::vector colors; + + string objectPath; +}; + + +struct DrawData +{ + ogl::Arrays arr; + ogl::Buffer indices; +}; + +void draw(void* userdata); + +void draw(void* userdata) +{ + DrawData* data = static_cast(userdata); + + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + ogl::render(data->arr, data->indices, ogl::TRIANGLES); +} + +static void generateImage(cv::Size imgSz, TriangleShadingType shadingType, TriangleCullingMode cullingMode, + ModelType modelType, std::string modelPath, cv::Mat& colorImage, cv::Mat& depthImage) +{ + namedWindow("OpenGL", WINDOW_OPENGL); + resizeWindow("OpenGL", imgSz.width, imgSz.height); + + ModelData modelData(modelType, modelPath); + + DrawData data; + + std::vector vertices; + std::vector colors4f; + std::vector idxLinear; + + if (shadingType == RASTERIZE_SHADING_FLAT) + { + // rearrange vertices and colors for flat shading + int ctr = 0; + for (const auto& idx : modelData.indices) + { + for (int i = 0; i < 3; i++) + { + vertices.push_back(modelData.vertices[idx[i]]); + idxLinear.push_back(ctr++); + } + + Vec3f ci = modelData.colors[idx[0]]; + for (int i = 0; i < 3; i++) + { + colors4f.emplace_back(ci[0], ci[1], ci[2], 1.f); + } + } + } + else + { + vertices = modelData.vertices; + for (const auto& c : modelData.colors) + { + Vec3f ci = (shadingType == RASTERIZE_SHADING_SHADED) ? c: cv::Vec3f::all(1.f); + colors4f.emplace_back(ci[0], ci[1], ci[2], 1.0); + } + + for (const auto& idx : modelData.indices) + { + for (int i = 0; i < 3; i++) + { + idxLinear.push_back(idx[i]); + } + } + } + + data.arr.setVertexArray(vertices); + data.arr.setColorArray(colors4f); + data.indices.copyFrom(idxLinear); + + double zNear = 0.1, zFar = 50; + glMatrixMode(GL_PROJECTION); + glLoadIdentity(); + gluPerspective(modelData.fovy, (double)imgSz.width / imgSz.height, zNear, zFar); + + glMatrixMode(GL_MODELVIEW); + glLoadIdentity(); + + //gluLookAt(0, 0, 5, 0, 0, 0, 0, 1, 0); + gluLookAt(modelData.position[0], modelData.position[1], modelData.position[2], + modelData.lookat [0], modelData.lookat [1], modelData.lookat [2], + modelData.upVector[0], modelData.upVector[1], modelData.upVector[2]); + + if (cullingMode == RASTERIZE_CULLING_NONE) + { + glDisable(GL_CULL_FACE); + } + else + { + glEnable(GL_CULL_FACE); + glCullFace(GL_FRONT); + if (cullingMode == RASTERIZE_CULLING_CW) + { + glFrontFace(GL_CW); + } + else + { + glFrontFace(GL_CCW); + } + } + + glEnable(GL_DEPTH_TEST); + + cv::setOpenGlDrawCallback("OpenGL", draw, &data); + + const int framesToSkip = 10; + for (int f = 0; f < framesToSkip; f++) + { + updateWindow("OpenGL"); + + colorImage = cv::Mat(imgSz.height, imgSz.width, CV_8UC3); + glReadPixels(0, 0, imgSz.width, imgSz.height, GL_RGB, GL_UNSIGNED_BYTE, colorImage.data); + cv::cvtColor(colorImage, colorImage, cv::COLOR_RGB2BGR); + cv::flip(colorImage, colorImage, 0); + + depthImage = cv::Mat(imgSz.height, imgSz.width, CV_32F); + glReadPixels(0, 0, imgSz.width, imgSz.height, GL_DEPTH_COMPONENT, GL_FLOAT, depthImage.data); + // map from [0, 1] to [zNear, zFar] + for (auto it = depthImage.begin(); it != depthImage.end(); ++it) + { + *it = (float)(zNear * zFar / (double(*it) * (zNear - zFar) + zFar)); + } + cv::flip(depthImage, depthImage, 0); + depthImage.convertTo(depthImage, CV_16U, 1000.0); + + char key = (char)waitKey(40); + if (key == 27) + break; + } + + cv::setOpenGlDrawCallback("OpenGL", 0, 0); + cv::destroyAllWindows(); +} + + +int main(int argc, char* argv[]) +{ + cv::CommandLineParser parser(argc, argv, + "{ help h usage ? | | show this message }" + "{ outPath | | output path for generated images }" + "{ modelPath | | path to 3d model to render }" + ); + parser.about("This app is used to generate test data for triangleRasterize() function"); + + if (parser.has("help")) + { + parser.printMessage(); + return 0; + } + + std::string modelPath = parser.get("modelPath"); + if (modelPath.empty()) + { + std::cout << "No model path given" << std::endl; + return -1; + } + + std::string outPath = parser.get("outPath"); + if (outPath.empty()) + { + std::cout << "No output path given" << std::endl; + return -1; + } + + std::array resolutions = { cv::Size {700, 700}, cv::Size {640, 480}, cv::Size(256, 256), cv::Size(320, 240) }; + for (const auto& res : resolutions) + { + for (const auto shadingType : { + RASTERIZE_SHADING_WHITE, + RASTERIZE_SHADING_FLAT, + RASTERIZE_SHADING_SHADED + }) + { + std::string shadingName; + switch (shadingType) + { + case RASTERIZE_SHADING_WHITE: shadingName = "White"; break; + case RASTERIZE_SHADING_FLAT: shadingName = "Flat"; break; + case RASTERIZE_SHADING_SHADED: shadingName = "Shaded"; break; + default: + break; + } + + for (const auto cullingMode : { + RASTERIZE_CULLING_NONE, + RASTERIZE_CULLING_CW, + RASTERIZE_CULLING_CCW + }) + { + std::string cullingName; + switch (cullingMode) + { + case RASTERIZE_CULLING_NONE: cullingName = "None"; break; + case RASTERIZE_CULLING_CW: cullingName = "CW"; break; + case RASTERIZE_CULLING_CCW: cullingName = "CCW"; break; + default: break; + } + + for (const auto modelType : { + ModelType::File, + ModelType::Clipping, + ModelType::Color, + ModelType::Centered, + }) + { + std::string modelName; + switch (modelType) + { + case ModelType::File: modelName = "File"; break; + case ModelType::Clipping: modelName = "Clipping"; break; + case ModelType::Color: modelName = "Color"; break; + case ModelType::Centered: modelName = "Centered"; break; + default: + break; + } + + std::string suffix = cv::format("%s_%dx%d_Cull%s", modelName.c_str(), res.width, res.height, cullingName.c_str()); + + std::cout << suffix + "_" + shadingName << "..." << std::endl; + + cv::Mat colorImage, depthImage; + generateImage(res, shadingType, cullingMode, modelType, modelPath, colorImage, depthImage); + + std::string gtPathColor = outPath + "/example_image_" + suffix + "_" + shadingName + ".png"; + std::string gtPathDepth = outPath + "/depth_image_" + suffix + ".png"; + + cv::imwrite(gtPathColor, colorImage); + cv::imwrite(gtPathDepth, depthImage); + } + } + } + } + + return 0; +}