From 37c2af63f099909252eeced1258e9a30a317306f Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Mon, 11 Nov 2024 14:13:33 +0300 Subject: [PATCH] Merge pull request #26434 from dkurt:dk/int64_file_storage_4.x int64 data type support for FileStorage. 1d and empty Mat with exact dimensions #26434 ### Pull Request Readiness Checklist Port of https://github.com/opencv/opencv/pull/26399 to 4.x branch See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- .../core/include/opencv2/core/persistence.hpp | 7 ++ modules/core/src/persistence.cpp | 116 ++++++++++++++++-- modules/core/src/persistence.hpp | 2 + modules/core/src/persistence_impl.hpp | 2 + modules/core/src/persistence_json.cpp | 10 +- modules/core/src/persistence_xml.cpp | 8 +- modules/core/src/persistence_yml.cpp | 8 +- modules/core/test/test_io.cpp | 29 +++++ .../predefined_types.py | 1 + 9 files changed, 166 insertions(+), 17 deletions(-) diff --git a/modules/core/include/opencv2/core/persistence.hpp b/modules/core/include/opencv2/core/persistence.hpp index 33f24d62e5..c6ddf6f0ca 100644 --- a/modules/core/include/opencv2/core/persistence.hpp +++ b/modules/core/include/opencv2/core/persistence.hpp @@ -365,6 +365,8 @@ public: */ CV_WRAP void write(const String& name, int val); /// @overload + CV_WRAP void write(const String& name, int64_t val); + /// @overload CV_WRAP void write(const String& name, double val); /// @overload CV_WRAP void write(const String& name, const String& val); @@ -530,6 +532,8 @@ public: CV_WRAP size_t rawSize() const; //! returns the node content as an integer. If the node stores floating-point number, it is rounded. operator int() const; + //! returns the node content as a signed 64bit integer. If the node stores floating-point number, it is rounded. + operator int64_t() const; //! returns the node content as float operator float() const; //! returns the node content as double @@ -654,6 +658,7 @@ protected: /////////////////// XML & YAML I/O implementation ////////////////// CV_EXPORTS void write( FileStorage& fs, const String& name, int value ); +CV_EXPORTS void write( FileStorage& fs, const String& name, int64_t value ); CV_EXPORTS void write( FileStorage& fs, const String& name, float value ); CV_EXPORTS void write( FileStorage& fs, const String& name, double value ); CV_EXPORTS void write( FileStorage& fs, const String& name, const String& value ); @@ -665,11 +670,13 @@ CV_EXPORTS void write( FileStorage& fs, const String& name, const std::vector +static inline void writeInt(uchar* p, T ival) { // On little endian CPUs, both branches produce the same result. On big endian, only the else branch does. #if CV_LITTLE_ENDIAN_MEM_ACCESS memcpy(p, &ival, sizeof(ival)); #else - p[0] = (uchar)ival; - p[1] = (uchar)(ival >> 8); - p[2] = (uchar)(ival >> 16); - p[3] = (uchar)(ival >> 24); + for (size_t i = 0, j = 0; i < sizeof(ival); ++i, j += 8) + p[i] = (uchar)(ival >> j); #endif } @@ -1056,6 +1091,11 @@ void FileStorage::Impl::write(const String &key, int value) { getEmitter().write(key.c_str(), value); } +void FileStorage::Impl::write(const String &key, int64_t value) { + CV_Assert(write_mode); + getEmitter().write(key.c_str(), value); +} + void FileStorage::Impl::write(const String &key, double value) { CV_Assert(write_mode); getEmitter().write(key.c_str(), value); @@ -1408,7 +1448,7 @@ void FileStorage::Impl::convertToCollection(int type, FileNode &node) { bool named = node.isNamed(); uchar *ptr = node.ptr() + 1 + (named ? 4 : 0); - int ival = 0; + int64_t ival = 0; double fval = 0; std::string sval; bool add_first_scalar = false; @@ -1421,7 +1461,7 @@ void FileStorage::Impl::convertToCollection(int type, FileNode &node) { // otherwise we don't know where to get the element names from CV_Assert(type == FileNode::SEQ); if (node_type == FileNode::INT) { - ival = readInt(ptr); + ival = readLong(ptr); add_first_scalar = true; } else if (node_type == FileNode::REAL) { fval = readReal(ptr); @@ -1783,7 +1823,7 @@ char *FileStorage::Impl::parseBase64(char *ptr, int indent, FileNode &collection int fmt_pairs[CV_FS_MAX_FMT_PAIRS * 2]; int fmt_pair_count = fs::decodeFormat(dt, fmt_pairs, CV_FS_MAX_FMT_PAIRS); - int ival = 0; + int64_t ival = 0; double fval = 0; for (;;) { @@ -2023,6 +2063,11 @@ void writeScalar( FileStorage& fs, int value ) fs.p->write(String(), value); } +void writeScalar( FileStorage& fs, int64_t value ) +{ + fs.p->write(String(), value); +} + void writeScalar( FileStorage& fs, float value ) { fs.p->write(String(), (double)value); @@ -2043,6 +2088,11 @@ void write( FileStorage& fs, const String& name, int value ) fs.p->write(name, value); } +void write( FileStorage& fs, const String& name, int64_t value ) +{ + fs.p->write(name, value); +} + void write( FileStorage& fs, const String& name, float value ) { fs.p->write(name, (double)value); @@ -2059,6 +2109,7 @@ void write( FileStorage& fs, const String& name, const String& value ) } void FileStorage::write(const String& name, int val) { p->write(name, val); } +void FileStorage::write(const String& name, int64_t val) { p->write(name, val); } void FileStorage::write(const String& name, double val) { p->write(name, val); } void FileStorage::write(const String& name, const String& val) { p->write(name, val); } void FileStorage::write(const String& name, const Mat& val) { cv::write(*this, name, val); } @@ -2279,6 +2330,27 @@ FileNode::operator int() const return 0x7fffffff; } +FileNode::operator int64_t() const +{ + const uchar* p = ptr(); + if(!p) + return 0; + int tag = *p; + int type = (tag & TYPE_MASK); + p += (tag & NAMED) ? 5 : 1; + + if( type == INT ) + { + return readLong(p); + } + else if( type == REAL ) + { + return cvRound(readReal(p)); + } + else + return 0x7fffffff; +} + FileNode::operator float() const { const uchar* p = ptr(); @@ -2403,7 +2475,13 @@ void FileNode::setValue( int type, const void* value, int len ) sz += 4; if( type == INT ) - sz += 4; + { + int64_t ival = *(const int64_t*)value; + if (ival > INT_MAX || ival < INT_MIN) + sz += 8; + else + sz += 4; + } else if( type == REAL ) sz += 8; else if( type == STRING ) @@ -2423,8 +2501,11 @@ void FileNode::setValue( int type, const void* value, int len ) if( type == INT ) { - int ival = *(const int*)value; - writeInt(p, ival); + int64_t ival = *(const int64_t*)value; + if (sz > 8) + writeInt(p, ival); + else + writeInt(p, static_cast(ival)); } else if( type == REAL ) { @@ -2580,7 +2661,7 @@ FileNodeIterator& FileNodeIterator::readRaw( const String& fmt, void* _data0, si FileNode node = *(*this); if( node.isInt() ) { - int ival = (int)node; + int64_t ival = static_cast(elem_size == 8 ? (int64_t)node : (int)node); switch( elem_type ) { case CV_8U: @@ -2600,7 +2681,7 @@ FileNodeIterator& FileNodeIterator::readRaw( const String& fmt, void* _data0, si data += sizeof(short); break; case CV_32S: - *(int*)data = ival; + *(int*)data = (int)ival; data += sizeof(int); break; case CV_32F: @@ -2702,6 +2783,15 @@ void read(const FileNode& node, int& val, int default_val) } } +void read(const FileNode& node, int64_t& val, int64_t default_val) +{ + val = default_val; + if( !node.empty() ) + { + val = (int64_t)node; + } +} + void read(const FileNode& node, double& val, double default_val) { val = default_val; diff --git a/modules/core/src/persistence.hpp b/modules/core/src/persistence.hpp index 4b579303fa..2a1f73cba8 100644 --- a/modules/core/src/persistence.hpp +++ b/modules/core/src/persistence.hpp @@ -86,6 +86,7 @@ namespace fs { int strcasecmp(const char* str1, const char* str2); char* itoa( int _val, char* buffer, int /*radix*/ ); +char* itoa( int64_t _val, char* buffer, int /*radix*/, bool _signed ); char* floatToString( char* buf, size_t bufSize, float value, bool halfprecision, bool explicitZero ); char* doubleToString( char* buf, size_t bufSize, double value, bool explicitZero ); @@ -193,6 +194,7 @@ public: int struct_flags, const char* type_name=0 ) = 0; virtual void endWriteStruct(const FStructData& current_struct) = 0; virtual void write(const char* key, int value) = 0; + virtual void write(const char* key, int64_t value) = 0; virtual void write(const char* key, double value) = 0; virtual void write(const char* key, const char* value, bool quote) = 0; virtual void writeScalar(const char* key, const char* value) = 0; diff --git a/modules/core/src/persistence_impl.hpp b/modules/core/src/persistence_impl.hpp index 1c261ce772..7f902079a5 100644 --- a/modules/core/src/persistence_impl.hpp +++ b/modules/core/src/persistence_impl.hpp @@ -69,6 +69,8 @@ public: void write( const String& key, int value ); + void write( const String& key, int64_t value ); + void write( const String& key, double value ); void write( const String& key, const String& value ); diff --git a/modules/core/src/persistence_json.cpp b/modules/core/src/persistence_json.cpp index e201215401..c5326c674a 100644 --- a/modules/core/src/persistence_json.cpp +++ b/modules/core/src/persistence_json.cpp @@ -81,6 +81,12 @@ public: writeScalar( key, fs::itoa( value, buf, 10 )); } + void write(const char* key, int64_t value) + { + char buf[128]; + writeScalar( key, fs::itoa( value, buf, 10, true )); + } + void write( const char* key, double value ) { char buf[128]; @@ -596,7 +602,7 @@ public: } else { - int ival = (int)strtol( beg, &ptr, 0 ); + int64_t ival = strtoll( beg, &ptr, 0 ); CV_PERSISTENCE_CHECK_END_OF_BUFFER_BUG_CPP(); node.setValue(FileNode::INT, &ival); @@ -623,7 +629,7 @@ public: else if( (len == 4 && memcmp( beg, "true", 4 ) == 0) || (len == 5 && memcmp( beg, "false", 5 ) == 0) ) { - int ival = *beg == 't' ? 1 : 0; + int64_t ival = *beg == 't' ? 1 : 0; node.setValue(FileNode::INT, &ival); } else diff --git a/modules/core/src/persistence_xml.cpp b/modules/core/src/persistence_xml.cpp index ed699758fc..29c05fdeca 100644 --- a/modules/core/src/persistence_xml.cpp +++ b/modules/core/src/persistence_xml.cpp @@ -145,6 +145,12 @@ public: writeScalar( key, ptr); } + void write(const char* key, int64_t value) + { + char buf[128], *ptr = fs::itoa( value, buf, 10, true ); + writeScalar( key, ptr); + } + void write( const char* key, double value ) { char buf[128]; @@ -556,7 +562,7 @@ public: } else { - int ival = (int)strtol( ptr, &endptr, 0 ); + int64_t ival = strtoll( ptr, &endptr, 0 ); elem->setValue(FileNode::INT, &ival); } diff --git a/modules/core/src/persistence_yml.cpp b/modules/core/src/persistence_yml.cpp index e4f53c353e..009ab7d8b7 100644 --- a/modules/core/src/persistence_yml.cpp +++ b/modules/core/src/persistence_yml.cpp @@ -107,6 +107,12 @@ public: writeScalar( key, fs::itoa( value, buf, 10 )); } + void write(const char* key, int64_t value) + { + char buf[128]; + writeScalar( key, fs::itoa( value, buf, 10, true )); + } + void write( const char* key, double value ) { char buf[128]; @@ -567,7 +573,7 @@ public: else { force_int: - int ival = (int)strtol( ptr, &endptr, 0 ); + int64_t ival = strtoll( ptr, &endptr, 0 ); node.setValue(FileNode::INT, &ival); } diff --git a/modules/core/test/test_io.cpp b/modules/core/test/test_io.cpp index d766abb319..88b72cd3f1 100644 --- a/modules/core/test/test_io.cpp +++ b/modules/core/test/test_io.cpp @@ -2003,4 +2003,33 @@ TEST(Core_InputOutput, FileStorage_invalid_attribute_value_regression_25946) ASSERT_EQ(0, std::remove(fileName.c_str())); } +template +T fsWriteRead(const T& expectedValue, const char* ext) +{ + std::string fname = cv::tempfile(ext); + FileStorage fs_w(fname, FileStorage::WRITE); + fs_w << "value" << expectedValue; + fs_w.release(); + + FileStorage fs_r(fname, FileStorage::READ); + + T value; + fs_r["value"] >> value; + return value; +} + +typedef testing::TestWithParam FileStorage_exact_type; +TEST_P(FileStorage_exact_type, long_int) +{ + for (const int64_t expected : std::vector{INT64_MAX, INT64_MIN, -1, 1, 0}) + { + int64_t value = fsWriteRead(expected, GetParam()); + EXPECT_EQ(value, expected); + } +} + +INSTANTIATE_TEST_CASE_P(Core_InputOutput, + FileStorage_exact_type, Values(".yml", ".xml", ".json") +); + }} // namespace diff --git a/modules/python/src2/typing_stubs_generation/predefined_types.py b/modules/python/src2/typing_stubs_generation/predefined_types.py index 5aed93c8b0..dd764f6112 100644 --- a/modules/python/src2/typing_stubs_generation/predefined_types.py +++ b/modules/python/src2/typing_stubs_generation/predefined_types.py @@ -27,6 +27,7 @@ _PREDEFINED_TYPES = ( PrimitiveTypeNode.int_("int32_t"), PrimitiveTypeNode.int_("uint32_t"), PrimitiveTypeNode.int_("size_t"), + PrimitiveTypeNode.int_("int64_t"), PrimitiveTypeNode.float_("float"), PrimitiveTypeNode.float_("double"), PrimitiveTypeNode.bool_("bool"),