From 4d695cd2f4c8311698ef7961186b866f84371d29 Mon Sep 17 00:00:00 2001 From: Zhangjie Chen Date: Tue, 25 Jul 2023 21:05:56 +0800 Subject: [PATCH] Merge pull request #23805 from starga2er777:5.x GSoC: Modified PLY reader to support color attribute read/write #23805 * Modified PLY reader to support color attribute read/write * Fix bugs & Modified OBJ reader for color IO * Replace with correct test file * Fix I/O of property [w] in OBJ files ### Pull Request Readiness Checklist **Merged with https://github.com/opencv/opencv_extra/pull/1075** [Issue for GSoC 2023](https://github.com/opencv/opencv/issues/23624) The ply loader in 3D module doesn't support color attribute reading. I modified that to support color attribute reading & writing for the color attribute compression as described in proposal. See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/3d/include/opencv2/3d.hpp | 6 ++- modules/3d/src/pointcloud/io_base.cpp | 8 +-- modules/3d/src/pointcloud/io_base.hpp | 8 +-- modules/3d/src/pointcloud/io_obj.cpp | 44 +++++++++++++-- modules/3d/src/pointcloud/io_obj.hpp | 4 +- modules/3d/src/pointcloud/io_ply.cpp | 54 +++++++++++++------ modules/3d/src/pointcloud/io_ply.hpp | 6 +-- .../3d/src/pointcloud/load_point_cloud.cpp | 24 ++++++--- modules/3d/test/test_pointcloud.cpp | 33 +++++++++--- 9 files changed, 139 insertions(+), 48 deletions(-) diff --git a/modules/3d/include/opencv2/3d.hpp b/modules/3d/include/opencv2/3d.hpp index 0063fa34fb..f27ee5614a 100644 --- a/modules/3d/include/opencv2/3d.hpp +++ b/modules/3d/include/opencv2/3d.hpp @@ -2636,8 +2636,9 @@ protected: * @param filename Name of the file. * @param vertices (vector of Point3f) Point coordinates of a point cloud * @param normals (vector of Point3f) Point normals of a point cloud +* @param rgb (vector of Point3_) Point RGB color of a point cloud */ -CV_EXPORTS_W void loadPointCloud(const String &filename, OutputArray vertices, OutputArray normals = noArray()); +CV_EXPORTS_W void loadPointCloud(const String &filename, OutputArray vertices, OutputArray normals = noArray(), OutputArray rgb = noArray()); /** @brief Saves a point cloud to a specified file. * @@ -2647,8 +2648,9 @@ CV_EXPORTS_W void loadPointCloud(const String &filename, OutputArray vertices, O * @param filename Name of the file. * @param vertices (vector of Point3f) Point coordinates of a point cloud * @param normals (vector of Point3f) Point normals of a point cloud +* @param rgb (vector of Point3_) Point RGB color of a point cloud */ -CV_EXPORTS_W void savePointCloud(const String &filename, InputArray vertices, InputArray normals = noArray()); +CV_EXPORTS_W void savePointCloud(const String &filename, InputArray vertices, InputArray normals = noArray(), InputArray rgb = noArray()); /** @brief Loads a mesh from a file. * diff --git a/modules/3d/src/pointcloud/io_base.cpp b/modules/3d/src/pointcloud/io_base.cpp index e87700555c..99b94b496c 100644 --- a/modules/3d/src/pointcloud/io_base.cpp +++ b/modules/3d/src/pointcloud/io_base.cpp @@ -12,10 +12,10 @@ void BasePointCloudDecoder::setSource(const std::string &filename) noexcept m_filename = filename; } -void BasePointCloudDecoder::readData(std::vector &points, std::vector &normals) +void BasePointCloudDecoder::readData(std::vector &points, std::vector &normals, std::vector> &rgb) { std::vector> indices; - readData(points, normals, indices); + readData(points, normals, rgb, indices); } void BasePointCloudEncoder::setDestination(const std::string &filename) noexcept @@ -23,10 +23,10 @@ void BasePointCloudEncoder::setDestination(const std::string &filename) noexcept m_filename = filename; } -void BasePointCloudEncoder::writeData(const std::vector &points, const std::vector &normals) +void BasePointCloudEncoder::writeData(const std::vector &points, const std::vector &normals, const std::vector> &rgb) { std::vector> indices; - writeData(points, normals, indices); + writeData(points, normals, rgb, indices); } } /* namespace cv */ diff --git a/modules/3d/src/pointcloud/io_base.hpp b/modules/3d/src/pointcloud/io_base.hpp index b5fe4bae89..446e80a34a 100644 --- a/modules/3d/src/pointcloud/io_base.hpp +++ b/modules/3d/src/pointcloud/io_base.hpp @@ -25,8 +25,8 @@ public: virtual ~BasePointCloudDecoder() = default; virtual void setSource(const String &filename) noexcept; - virtual void readData(std::vector &points, std::vector &normals); - virtual void readData(std::vector &points, std::vector &normals, std::vector> &indices) = 0; + virtual void readData(std::vector &points, std::vector &normals, std::vector> &rgb); + virtual void readData(std::vector &points, std::vector &normals, std::vector> &rgb, std::vector> &indices) = 0; protected: String m_filename; @@ -39,8 +39,8 @@ public: virtual ~BasePointCloudEncoder() = default; virtual void setDestination(const String &filename) noexcept; - virtual void writeData(const std::vector &points, const std::vector &normals); - virtual void writeData(const std::vector &points, const std::vector &normals, const std::vector> &indices) = 0; + virtual void writeData(const std::vector &points, const std::vector &normals, const std::vector> &rgb); + virtual void writeData(const std::vector &points, const std::vector &normals, const std::vector> &rgb, const std::vector> &indices) = 0; protected: String m_filename; diff --git a/modules/3d/src/pointcloud/io_obj.cpp b/modules/3d/src/pointcloud/io_obj.cpp index 2c7afa4e4c..51b1c47a4f 100644 --- a/modules/3d/src/pointcloud/io_obj.cpp +++ b/modules/3d/src/pointcloud/io_obj.cpp @@ -12,10 +12,11 @@ namespace cv { std::unordered_set ObjDecoder::m_unsupportedKeys; -void ObjDecoder::readData(std::vector &points, std::vector &normals, std::vector> &indices) +void ObjDecoder::readData(std::vector &points, std::vector &normals, std::vector> &rgb, std::vector> &indices) { points.clear(); normals.clear(); + rgb.clear(); indices.clear(); std::ifstream file(m_filename, std::ios::binary); @@ -39,9 +40,34 @@ void ObjDecoder::readData(std::vector &points, std::vector &no continue; else if (key == "v") { + // (x, y, z, [w], [r, g, b]) + auto splitArr = split(s, ' '); + if (splitArr.size() <= 3) + { + CV_LOG_ERROR(NULL, "Vertex should have at least 3 coordinate values."); + return; + } Point3f vertex; ss >> vertex.x >> vertex.y >> vertex.z; points.push_back(vertex); + if (splitArr.size() == 5 || splitArr.size() == 8) + { + float w; + ss >> w; + CV_UNUSED(w); + } + if (splitArr.size() >= 7) + { + Point3f color; + if (ss.rdbuf()->in_avail() != 0) { + Point3_ uc_color; + ss >> color.x >> color.y >> color.z; + uc_color.x = static_cast(std::round(255.f * color.x)); + uc_color.y = static_cast(std::round(255.f * color.y)); + uc_color.z = static_cast(std::round(255.f * color.z)); + rgb.push_back(uc_color); + } + } } else if (key == "vn") { @@ -75,7 +101,7 @@ void ObjDecoder::readData(std::vector &points, std::vector &no file.close(); } -void ObjEncoder::writeData(const std::vector &points, const std::vector &normals, const std::vector> &indices) +void ObjEncoder::writeData(const std::vector &points, const std::vector &normals, const std::vector> &rgb, const std::vector> &indices) { std::ofstream file(m_filename, std::ios::binary); if (!file) { @@ -83,12 +109,22 @@ void ObjEncoder::writeData(const std::vector &points, const std::vector return; } + if (!rgb.empty() && rgb.size() != points.size()) { + CV_LOG_ERROR(NULL, "Vertices and Colors have different size."); + return; + } + file << "# OBJ file writer" << std::endl; file << "o Point_Cloud" << std::endl; - for (const auto& point : points) + for (size_t i = 0; i < points.size(); ++i) { - file << "v " << point.x << " " << point.y << " " << point.z << std::endl; + file << "v " << points[i].x << " " << points[i].y << " " << points[i].z; + if (!rgb.empty()) { + file << " " << static_cast(rgb[i].x) / 255.f << " " << + static_cast(rgb[i].y) / 255.f << " " << static_cast(rgb[i].z) / 255.f; + } + file << std::endl; } for (const auto& normal : normals) diff --git a/modules/3d/src/pointcloud/io_obj.hpp b/modules/3d/src/pointcloud/io_obj.hpp index 924827fa2c..e85782eee3 100644 --- a/modules/3d/src/pointcloud/io_obj.hpp +++ b/modules/3d/src/pointcloud/io_obj.hpp @@ -13,7 +13,7 @@ namespace cv { class ObjDecoder CV_FINAL : public BasePointCloudDecoder { public: - void readData(std::vector &points, std::vector &normals, std::vector> &indices) CV_OVERRIDE; + void readData(std::vector &points, std::vector &normals, std::vector> &rgb, std::vector> &indices) CV_OVERRIDE; protected: static std::unordered_set m_unsupportedKeys; @@ -22,7 +22,7 @@ protected: class ObjEncoder CV_FINAL : public BasePointCloudEncoder { public: - void writeData(const std::vector &points, const std::vector &normals, const std::vector> &indices) CV_OVERRIDE; + void writeData(const std::vector &points, const std::vector &normals, const std::vector> &rgb, const std::vector> &indices) CV_OVERRIDE; }; diff --git a/modules/3d/src/pointcloud/io_ply.cpp b/modules/3d/src/pointcloud/io_ply.cpp index d33b027777..0e8447edf5 100644 --- a/modules/3d/src/pointcloud/io_ply.cpp +++ b/modules/3d/src/pointcloud/io_ply.cpp @@ -11,16 +11,17 @@ namespace cv { -void PlyDecoder::readData(std::vector &points, std::vector &normals, std::vector> &indices) +void PlyDecoder::readData(std::vector &points, std::vector &normals, std::vector> &rgb, std::vector> &indices) { points.clear(); normals.clear(); + rgb.clear(); CV_UNUSED(indices); std::ifstream file(m_filename, std::ios::binary); if (parseHeader(file)) { - parseBody(file, points, normals); + parseBody(file, points, normals, rgb); } } @@ -81,19 +82,30 @@ bool PlyDecoder::parseHeader(std::ifstream &file) if (onVertexRead) { auto splitArrElem = split(s, ' '); - if (splitArrElem[2] == "x" || splitArrElem[2] == "red" || splitArrElem[2] == "nx") + if (splitArrElem[2] == "x" || splitArrElem[2] == "y" || splitArrElem[2] == "z") { if (splitArrElem[1] != "float") { - CV_LOG_ERROR(NULL, "Provided PLY file format '" << splitArrElem[1] << "' is not supported"); + CV_LOG_ERROR(NULL, "Provided property '" << splitArrElem[2] << "' with format '" << splitArrElem[1] + << "' is not supported"); return false; } } - if (splitArrElem[2] == "red") + if (splitArrElem[2] == "red" || splitArrElem[2] == "green" || splitArrElem[2] == "blue") { + if (splitArrElem[1] != "uchar") { + CV_LOG_ERROR(NULL, "Provided property '" << splitArrElem[2] << "' with format '" << splitArrElem[1] + << "' is not supported"); + return false; + } m_hasColour = true; } if (splitArrElem[2] == "nx") { + if (splitArrElem[1] != "float") { + CV_LOG_ERROR(NULL, "Provided property '" << splitArrElem[2] << "' with format '" << splitArrElem[1] + << "' is not supported"); + return false; + } m_hasNormal = true; } } @@ -130,7 +142,7 @@ T readNext(std::ifstream &file, DataFormat format) return val; } -void PlyDecoder::parseBody(std::ifstream &file, std::vector &points, std::vector &normals) +void PlyDecoder::parseBody(std::ifstream &file, std::vector &points, std::vector &normals, std::vector> &rgb) { points.reserve(m_vertexCount); if (m_hasNormal) @@ -146,9 +158,11 @@ void PlyDecoder::parseBody(std::ifstream &file, std::vector &points, st points.push_back(vertex); if (m_hasColour) { - readNext(file, m_inputDataFormat); - readNext(file, m_inputDataFormat); - readNext(file, m_inputDataFormat); + Point3_ colour; + colour.x = readNext(file, m_inputDataFormat) & 0xff; + colour.y = readNext(file, m_inputDataFormat) & 0xff; + colour.z = readNext(file, m_inputDataFormat) & 0xff; + rgb.push_back(colour); } if (m_hasNormal) { @@ -161,7 +175,7 @@ void PlyDecoder::parseBody(std::ifstream &file, std::vector &points, st } } -void PlyEncoder::writeData(const std::vector &points, const std::vector &normals, const std::vector> &indices) +void PlyEncoder::writeData(const std::vector &points, const std::vector &normals, const std::vector> &rgb, const std::vector> &indices) { CV_UNUSED(indices); std::ofstream file(m_filename, std::ios::binary); @@ -169,6 +183,7 @@ void PlyEncoder::writeData(const std::vector &points, const std::vector CV_LOG_ERROR(NULL, "Impossible to open the file: " << m_filename); return; } + bool hasNormals = !normals.empty(), hasColor = !rgb.empty(); file << "ply" << std::endl; file << "format ascii 1.0" << std::endl; @@ -179,23 +194,30 @@ void PlyEncoder::writeData(const std::vector &points, const std::vector file << "property float y" << std::endl; file << "property float z" << std::endl; - file << "end_header" << std::endl; + if(hasColor) { + file << "property uchar red" << std::endl; + file << "property uchar green" << std::endl; + file << "property uchar blue" << std::endl; + } - if (normals.size() != 0) + if (hasNormals) { file << "property float nx" << std::endl; file << "property float ny" << std::endl; file << "property float nz" << std::endl; } - bool isNormals = (normals.size() != 0); + file << "end_header" << std::endl; + for (size_t i = 0; i < points.size(); i++) { file << points[i].x << " " << points[i].y << " " << points[i].z; - if (isNormals) - { - file << normals[i].x << " " << normals[i].y << " " << normals[i].z; + if (hasColor) { + file << " " << static_cast(rgb[i].x) << " " << static_cast(rgb[i].y) << " " << static_cast(rgb[i].z); + } + if (hasNormals) { + file << " " << normals[i].x << " " << normals[i].y << " " << normals[i].z; } file << std::endl; } diff --git a/modules/3d/src/pointcloud/io_ply.hpp b/modules/3d/src/pointcloud/io_ply.hpp index 5fd4c39814..d8fc44e353 100644 --- a/modules/3d/src/pointcloud/io_ply.hpp +++ b/modules/3d/src/pointcloud/io_ply.hpp @@ -21,11 +21,11 @@ enum class DataFormat class PlyDecoder CV_FINAL : public BasePointCloudDecoder { public: - void readData(std::vector &points, std::vector &normals, std::vector> &indices) CV_OVERRIDE; + void readData(std::vector &points, std::vector &normals, std::vector> &rgb, std::vector> &indices) CV_OVERRIDE; protected: bool parseHeader(std::ifstream &file); - void parseBody(std::ifstream &file, std::vector &points, std::vector &normals); + void parseBody(std::ifstream &file, std::vector &points, std::vector &normals, std::vector> &rgb); DataFormat m_inputDataFormat; size_t m_vertexCount{0}; @@ -36,7 +36,7 @@ protected: class PlyEncoder CV_FINAL : public BasePointCloudEncoder { public: - void writeData(const std::vector &points, const std::vector &normals, const std::vector> &indices) CV_OVERRIDE; + void writeData(const std::vector &points, const std::vector &normals, const std::vector> &rgb, const std::vector> &indices) CV_OVERRIDE; }; diff --git a/modules/3d/src/pointcloud/load_point_cloud.cpp b/modules/3d/src/pointcloud/load_point_cloud.cpp index 8c9dd3d50f..f2a052c43b 100644 --- a/modules/3d/src/pointcloud/load_point_cloud.cpp +++ b/modules/3d/src/pointcloud/load_point_cloud.cpp @@ -49,7 +49,7 @@ static PointCloudEncoder findEncoder(const String &filename) #endif -void loadPointCloud(const String &filename, OutputArray vertices, OutputArray normals) +void loadPointCloud(const String &filename, OutputArray vertices, OutputArray normals, OutputArray rgb) { #if OPENCV_HAVE_FILESYSTEM_SUPPORT auto decoder = findDecoder(filename); @@ -63,14 +63,19 @@ void loadPointCloud(const String &filename, OutputArray vertices, OutputArray no std::vector vec_vertices; std::vector vec_normals; + std::vector> vec_rgb; - decoder->readData(vec_vertices, vec_normals); + decoder->readData(vec_vertices, vec_normals, vec_rgb); if (!vec_vertices.empty()) Mat(static_cast(vec_vertices.size()), 1, CV_32FC3, &vec_vertices[0]).copyTo(vertices); if (!vec_normals.empty() && normals.needed()) Mat(static_cast(vec_normals.size()), 1, CV_32FC3, &vec_normals[0]).copyTo(normals); + + if (!vec_rgb.empty() && rgb.needed()) + Mat(static_cast(vec_rgb.size()), 1, CV_8UC3, &vec_rgb[0]).copyTo(rgb); + #else // OPENCV_HAVE_FILESYSTEM_SUPPORT CV_UNUSED(filename); CV_UNUSED(vertices); @@ -79,7 +84,7 @@ void loadPointCloud(const String &filename, OutputArray vertices, OutputArray no #endif } -void savePointCloud(const String &filename, InputArray vertices, InputArray normals) +void savePointCloud(const String &filename, InputArray vertices, InputArray normals, InputArray rgb) { #if OPENCV_HAVE_FILESYSTEM_SUPPORT if (vertices.empty()) { @@ -98,11 +103,16 @@ void savePointCloud(const String &filename, InputArray vertices, InputArray norm std::vector vec_vertices(vertices.getMat()); std::vector vec_normals; + std::vector> vec_rgb; + if (!normals.empty()){ vec_normals = normals.getMat(); } - encoder->writeData(vec_vertices, vec_normals); + if (!rgb.empty()){ + vec_rgb = rgb.getMat(); + } + encoder->writeData(vec_vertices, vec_normals, vec_rgb); #else // OPENCV_HAVE_FILESYSTEM_SUPPORT CV_UNUSED(filename); @@ -126,9 +136,10 @@ void loadMesh(const String &filename, OutputArray vertices, OutputArray normals, std::vector vec_vertices; std::vector vec_normals; + std::vector> vec_rgb; std::vector> vec_indices; - decoder->readData(vec_vertices, vec_normals, vec_indices); + decoder->readData(vec_vertices, vec_normals, vec_rgb, vec_indices); if (!vec_vertices.empty()) { Mat(1, static_cast(vec_vertices.size()), CV_32FC3, vec_vertices.data()).copyTo(vertices); @@ -173,6 +184,7 @@ void saveMesh(const String &filename, InputArray vertices, InputArray normals, I std::vector vec_vertices(vertices.getMat()); std::vector vec_normals; + std::vector> vec_rgb; if (!normals.empty()){ vec_normals = normals.getMat(); } @@ -185,7 +197,7 @@ void saveMesh(const String &filename, InputArray vertices, InputArray normals, I mat_indices[i].copyTo(vec_indices[i]); } - encoder->writeData(vec_vertices, vec_normals, vec_indices); + encoder->writeData(vec_vertices, vec_normals, vec_rgb, vec_indices); #else // OPENCV_HAVE_FILESYSTEM_SUPPORT CV_UNUSED(filename); diff --git a/modules/3d/test/test_pointcloud.cpp b/modules/3d/test/test_pointcloud.cpp index aa72765051..13453bab1f 100644 --- a/modules/3d/test/test_pointcloud.cpp +++ b/modules/3d/test/test_pointcloud.cpp @@ -22,6 +22,7 @@ TEST(PointCloud, LoadObj) {0.399941f, 1.86743f, 2.55837f}, {0.399941f, -0.13257f, -1.16339f}, {0.399941f, 1.86743f, -1.16339f}}; + std::vector normals_gold = { {-1.0000f, 0.0000f, 0.0000f}, {0.0000f, 0.0000f, -1.0000f}, @@ -30,13 +31,25 @@ TEST(PointCloud, LoadObj) {0.0000f, -1.0000f, 0.0000f}, {0.0000f, 1.0000f, 0.0000f}}; + std::vector> rgb_gold = { + {19, 144, 149}, + {219, 28, 216}, + {218, 157, 101}, + {11, 161, 78}, + {248, 183, 214}, + {63, 196, 6}, + {165, 190, 153}, + {89, 203, 11}}; + std::vector points; std::vector normals; + std::vector> rgb; auto folder = cvtest::TS::ptr()->get_data_path(); - cv::loadPointCloud(folder + "pointcloudio/orig.obj", points, normals); + cv::loadPointCloud(folder + "pointcloudio/orig.obj", points, normals, rgb); EXPECT_EQ(points_gold, points); + EXPECT_EQ(rgb_gold, rgb); EXPECT_EQ(normals_gold, normals); } @@ -66,20 +79,23 @@ TEST(PointCloud, SaveObj) { std::vector points_gold; std::vector normals_gold; + std::vector> rgb_gold; auto folder = cvtest::TS::ptr()->get_data_path(); auto new_path = tempfile("new.obj"); - cv::loadPointCloud(folder + "pointcloudio/orig.obj", points_gold, normals_gold); - cv::savePointCloud(new_path, points_gold, normals_gold); + cv::loadPointCloud(folder + "pointcloudio/orig.obj", points_gold, normals_gold, rgb_gold); + cv::savePointCloud(new_path, points_gold, normals_gold, rgb_gold); std::vector points; std::vector normals; + std::vector> rgb; - cv::loadPointCloud(new_path, points, normals); + cv::loadPointCloud(new_path, points, normals, rgb); EXPECT_EQ(normals, normals_gold); EXPECT_EQ(points, points_gold); + EXPECT_EQ(rgb, rgb_gold); std::remove(new_path.c_str()); } @@ -87,20 +103,23 @@ TEST(PointCloud, LoadSavePly) { std::vector points; std::vector normals; + std::vector> rgb; auto folder = cvtest::TS::ptr()->get_data_path(); std::string new_path = tempfile("new.ply"); - cv::loadPointCloud(folder + "pointcloudio/orig.ply", points, normals); - cv::savePointCloud(new_path, points, normals); + cv::loadPointCloud(folder + "pointcloudio/orig.ply", points, normals, rgb); + cv::savePointCloud(new_path, points, normals, rgb); std::vector points_gold; std::vector normals_gold; + std::vector> rgb_gold; - cv::loadPointCloud(new_path, points_gold, normals_gold); + cv::loadPointCloud(new_path, points_gold, normals_gold, rgb_gold); EXPECT_EQ(normals_gold, normals); EXPECT_EQ(points_gold, points); + EXPECT_EQ(rgb_gold, rgb); std::remove(new_path.c_str()); }