core(persistence): struct storage layout without alignment gaps

modules/core/src/persistence.hpp

@@ -55,7 +55,7 @@ size_t base64_decode_buffer_size(size_t cnt, char  const * src, bool is_end_with
 size_t base64_decode_buffer_size(size_t cnt, uchar const * src, bool is_end_with_zero = true);
 std::string make_base64_header(const char * dt);
 bool read_base64_header(std::vector<char> const & header, std::string & dt);
-void make_seq(void * binary_data, int elem_cnt, const char * dt, CvSeq & seq);
+void make_seq(::CvFileStorage* fs, const uchar* binary_data, size_t elem_cnt, const char * dt, CvSeq & seq);
 void cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt);
 
 class Base64ContextEmitter;

modules/core/src/persistence_base64.cpp

@@ -5,6 +5,8 @@
 
 #include "precomp.hpp"
 #include "persistence.hpp"
+#include <opencv2/core/utils/logger.hpp>
+#include <opencv2/core/utils/configuration.private.hpp>
 
 namespace base64 {
 
@@ -555,7 +557,7 @@ public:
         CV_Assert(len > 0);
 
         /* calc step and to_binary_funcs */
-        make_to_binary_funcs(dt);
+        step_packed = make_to_binary_funcs(dt);
 
         end = beg;
         cur = beg;
@@ -570,10 +572,10 @@ public:
 
         for (size_t i = 0U, n = to_binary_funcs.size(); i < n; i++) {
             elem_to_binary_t & pack = to_binary_funcs[i];
-            pack.func(cur + pack.offset, dst + pack.offset);
+            pack.func(cur + pack.offset, dst + pack.offset_packed);
         }
         cur += step;
-        dst += step;
+        dst += step_packed;
 
         return *this;
     }
@@ -588,14 +590,16 @@ private:
     struct elem_to_binary_t
     {
         size_t      offset;
+        size_t      offset_packed;
         to_binary_t func;
     };
 
 private:
-    void make_to_binary_funcs(const std::string &dt)
+    size_t make_to_binary_funcs(const std::string &dt)
     {
         size_t cnt = 0;
         size_t offset = 0;
+        size_t offset_packed = 0;
         char type = '\0';
 
         std::istringstream iss(dt);
@@ -646,11 +650,15 @@ private:
                 pack.offset = offset;
                 offset += size;
 
+                pack.offset_packed = offset_packed;
+                offset_packed += size;
+
                 to_binary_funcs.push_back(pack);
             }
         }
 
         CV_Assert(iss.eof());
+        return offset_packed;
     }
 
 private:
@@ -659,27 +667,26 @@ private:
     const uchar * end;
 
     size_t step;
+    size_t step_packed;
     std::vector<elem_to_binary_t> to_binary_funcs;
 };
 
 class BinaryToCvSeqConvertor
 {
 public:
-    BinaryToCvSeqConvertor(const void* src, int len, const char* dt)
-        : cur(reinterpret_cast<const uchar *>(src))
-        , beg(reinterpret_cast<const uchar *>(src))
-        , end(reinterpret_cast<const uchar *>(src))
+    BinaryToCvSeqConvertor(CvFileStorage* fs, const uchar* src, size_t total_byte_size, const char* dt)
+        : cur(src)
+        , end(src + total_byte_size)
     {
         CV_Assert(src);
         CV_Assert(dt);
-        CV_Assert(len >= 0);
+        CV_Assert(total_byte_size > 0);
 
-        /* calc binary_to_funcs */
-        make_funcs(dt);
+        step = make_funcs(dt);  // calc binary_to_funcs
         functor_iter = binary_to_funcs.begin();
 
-        step = ::icvCalcStructSize(dt, 0);
-        end = beg + step * static_cast<size_t>(len);
+        if (total_byte_size % step != 0)
+            CV_PARSE_ERROR("Total byte size not match elememt size");
     }
 
     inline BinaryToCvSeqConvertor & operator >> (CvFileNode & dst)
@@ -699,7 +706,7 @@ public:
             double d;
         } buffer; /* for GCC -Wstrict-aliasing */
         std::memset(buffer.mem, 0, sizeof(buffer));
-        functor_iter->func(cur + functor_iter->offset, buffer.mem);
+        functor_iter->func(cur + functor_iter->offset_packed, buffer.mem);
 
         /* set node::data */
         switch (functor_iter->cv_type)
@@ -746,16 +753,17 @@ private:
     struct binary_to_filenode_t
     {
         size_t      cv_type;
-        size_t      offset;
+        size_t      offset_packed;
         binary_to_t func;
     };
 
 private:
-    void make_funcs(const char* dt)
+    size_t make_funcs(const char* dt)
     {
         size_t cnt = 0;
         char type = '\0';
         size_t offset = 0;
+        size_t offset_packed = 0;
 
         std::istringstream iss(dt);
         while (!iss.eof()) {
@@ -803,9 +811,28 @@ private:
                 }; // need a better way for outputting error.
 
                 offset = static_cast<size_t>(cvAlign(static_cast<int>(offset), static_cast<int>(size)));
-                pack.offset = offset;
+                if (offset != offset_packed)
+                {
+                    static bool skip_message = cv::utils::getConfigurationParameterBool("OPENCV_PERSISTENCE_SKIP_PACKED_STRUCT_WARNING",
+#ifdef _DEBUG
+                            false
+#else
+                            true
+#endif
+                    );
+                    if (!skip_message)
+                    {
+                        CV_LOG_WARNING(NULL, "Binary converter: struct storage layout has been changed in OpenCV 3.4.7. Alignment gaps has been removed from the storage containers. "
+                                "Details: https://github.com/opencv/opencv/pull/15050"
+                        );
+                        skip_message = true;
+                    }
+                }
                 offset += size;
 
+                pack.offset_packed = offset_packed;
+                offset_packed += size;
+
                 /* set type */
                 switch (type)
                 {
@@ -827,12 +854,13 @@ private:
 
         CV_Assert(iss.eof());
         CV_Assert(binary_to_funcs.size());
+
+        return offset_packed;
     }
 
 private:
 
     const uchar * cur;
-    const uchar * beg;
     const uchar * end;
 
     size_t step;
@@ -889,11 +917,13 @@ void Base64Writer::check_dt(const char* dt)
 }
 
 
-void make_seq(void * binary, int elem_cnt, const char * dt, ::CvSeq & seq)
+void make_seq(CvFileStorage* fs, const uchar* binary, size_t total_byte_size, const char * dt, ::CvSeq & seq)
 {
+    if (total_byte_size == 0)
+        return;
     ::CvFileNode node;
     node.info = 0;
-    BinaryToCvSeqConvertor convertor(binary, elem_cnt, dt);
+    BinaryToCvSeqConvertor convertor(fs, binary, total_byte_size, dt);
     while (convertor) {
         convertor >> node;
         cvSeqPush(&seq, &node);

modules/core/src/persistence_json.cpp

@@ -259,15 +259,9 @@ static char* icvJSONParseValue( CvFileStorage* fs, char* ptr, CvFileNode* node )
                         parser.flush();
                     }
 
-                    /* save as CvSeq */
-                    int elem_size = ::icvCalcStructSize(dt.c_str(), 0);
-                    if (total_byte_size % elem_size != 0)
-                        CV_PARSE_ERROR("Byte size not match elememt size");
-                    int elem_cnt = total_byte_size / elem_size;
-
                     /* after icvFSCreateCollection, node->tag == struct_flags */
                     icvFSCreateCollection(fs, CV_NODE_FLOW | CV_NODE_SEQ, node);
-                    base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
+                    base64::make_seq(fs, binary_buffer.data(), total_byte_size, dt.c_str(), *node->data.seq);
                 }
                 else
                 {

modules/core/src/persistence_xml.cpp

@@ -167,17 +167,11 @@ static char* icvXMLParseBase64(CvFileStorage* fs, char* ptr, CvFileNode * node)
         parser.flush();
     }
 
-    /* save as CvSeq */
-    int elem_size = ::icvCalcStructSize(dt.c_str(), 0);
-    if (total_byte_size % elem_size != 0)
-        CV_PARSE_ERROR("data size not matches elememt size");
-    int elem_cnt = total_byte_size / elem_size;
-
     node->tag = CV_NODE_NONE;
     int struct_flags = CV_NODE_SEQ;
     /* after icvFSCreateCollection, node->tag == struct_flags */
     icvFSCreateCollection(fs, struct_flags, node);
-    base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
+    base64::make_seq(fs, binary_buffer.data(), total_byte_size, dt.c_str(), *node->data.seq);
 
     if (fs->dummy_eof) {
         /* end of file */

modules/core/src/persistence_yml.cpp

@@ -130,17 +130,11 @@ static char* icvYMLParseBase64(CvFileStorage* fs, char* ptr, int indent, CvFileN
         parser.flush();
     }
 
-    /* save as CvSeq */
-    int elem_size = ::icvCalcStructSize(dt.c_str(), 0);
-    if (total_byte_size % elem_size != 0)
-        CV_PARSE_ERROR("Byte size not match elememt size");
-    int elem_cnt = total_byte_size / elem_size;
-
     node->tag = CV_NODE_NONE;
     int struct_flags = CV_NODE_FLOW | CV_NODE_SEQ;
     /* after icvFSCreateCollection, node->tag == struct_flags */
     icvFSCreateCollection(fs, struct_flags, node);
-    base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
+    base64::make_seq(fs, binary_buffer.data(), total_byte_size, dt.c_str(), *node->data.seq);
 
     if (fs->dummy_eof) {
         /* end of file */

modules/core/test/test_io.cpp

@@ -659,38 +659,29 @@ struct data_t
     }
 };
 
-TEST(Core_InputOutput, filestorage_base64_basic)
+static void test_filestorage_basic(int write_flags, const char* suffix_name, bool testReadWrite, bool useMemory = false)
 {
     const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info();
-    std::string basename = (test_info == 0)
-        ? "filestorage_base64_valid_call"
-        : (std::string(test_info->test_case_name()) + "--" + test_info->name());
-
-    char const * filenames[] = {
-        "core_io_base64_basic_test.yml",
-        "core_io_base64_basic_test.xml",
-        "core_io_base64_basic_test.json",
-        0
-    };
+    CV_Assert(test_info);
+    std::string name = (std::string(test_info->test_case_name()) + "--" + test_info->name() + suffix_name);
+    if (!testReadWrite)
+        name = string(cvtest::TS::ptr()->get_data_path()) + "io/" + name;
 
-    for (char const ** ptr = filenames; *ptr; ptr++)
     {
-        char const * suffix_name = *ptr;
-        std::string name = basename + '_' + suffix_name;
-
+        const size_t rawdata_N = 40;
         std::vector<data_t> rawdata;
 
         cv::Mat _em_out, _em_in;
         cv::Mat _2d_out, _2d_in;
         cv::Mat _nd_out, _nd_in;
-        cv::Mat _rd_out(64, 64, CV_64FC1), _rd_in;
+        cv::Mat _rd_out(8, 16, CV_64FC1), _rd_in;
 
         bool no_type_id = true;
 
         {   /* init */
 
             /* a normal mat */
-            _2d_out = cv::Mat(100, 100, CV_8UC3, cvScalar(1U, 2U, 127U));
+            _2d_out = cv::Mat(10, 20, CV_8UC3, cvScalar(1U, 2U, 127U));
             for (int i = 0; i < _2d_out.rows; ++i)
                 for (int j = 0; j < _2d_out.cols; ++j)
                     _2d_out.at<cv::Vec3b>(i, j)[1] = (i + j) % 256;
@@ -709,7 +700,7 @@ TEST(Core_InputOutput, filestorage_base64_basic)
             cv::randu(_rd_out, cv::Scalar(0.0), cv::Scalar(1.0));
 
             /* raw data */
-            for (int i = 0; i < 1000; i++) {
+            for (int i = 0; i < (int)rawdata_N; i++) {
                 data_t tmp;
                 tmp.u1 = 1;
                 tmp.u2 = 2;
@@ -722,24 +713,41 @@ TEST(Core_InputOutput, filestorage_base64_basic)
                 rawdata.push_back(tmp);
             }
         }
-
-        {   /* write */
-            cv::FileStorage fs(name, cv::FileStorage::WRITE_BASE64);
+#ifdef GENERATE_TEST_DATA
+#else
+        if (testReadWrite || useMemory)
+#endif
+        {
+            cv::FileStorage fs(name, write_flags + (useMemory ? cv::FileStorage::MEMORY : 0));
             fs << "normal_2d_mat" << _2d_out;
             fs << "normal_nd_mat" << _nd_out;
             fs << "empty_2d_mat"  << _em_out;
             fs << "random_mat"    << _rd_out;
 
             cvStartWriteStruct( *fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW, "binary" );
-            for (int i = 0; i < 10; i++)
-                cvWriteRawDataBase64(*fs, rawdata.data() + i * 100, 100, data_t::signature());
+            for (int i = 0; i < (int)rawdata_N/10; i++)
+                cvWriteRawData(*fs, rawdata.data() + i * 10, 10, data_t::signature());
             cvEndWriteStruct( *fs );
 
-            fs.release();
+            size_t sz = 0;
+            if (useMemory)
+            {
+                name = fs.releaseAndGetString();
+                sz = name.size();
+            }
+            else
+            {
+                fs.release();
+                std::ifstream f(name.c_str(), std::ios::in|std::ios::binary);
+                f.seekg(0, std::fstream::end);
+                sz = (size_t)f.tellg();
+                f.close();
+            }
+            std::cout << "Storage size: " << sz << std::endl;
+            EXPECT_LE(sz, (size_t)6000);
         }
-
         {   /* read */
-            cv::FileStorage fs(name, cv::FileStorage::READ);
+            cv::FileStorage fs(name, cv::FileStorage::READ + (useMemory ? cv::FileStorage::MEMORY : 0));
 
             /* mat */
             fs["empty_2d_mat"]  >> _em_in;
@@ -754,14 +762,14 @@ TEST(Core_InputOutput, filestorage_base64_basic)
                 no_type_id = false;
 
             /* raw data */
-            std::vector<data_t>(1000).swap(rawdata);
+            std::vector<data_t>(rawdata_N).swap(rawdata);
             cvReadRawData(*fs, fs["rawdata"].node, rawdata.data(), data_t::signature());
 
             fs.release();
         }
 
         int errors = 0;
-        for (int i = 0; i < 1000; i++)
+        for (int i = 0; i < (int)rawdata_N; i++)
         {
             EXPECT_EQ((int)rawdata[i].u1, 1);
             EXPECT_EQ((int)rawdata[i].u2, 2);
@@ -815,18 +823,54 @@ TEST(Core_InputOutput, filestorage_base64_basic)
         EXPECT_EQ(_nd_in.cols   , _nd_out.cols);
         EXPECT_EQ(_nd_in.dims   , _nd_out.dims);
         EXPECT_EQ(_nd_in.depth(), _nd_out.depth());
-        EXPECT_EQ(cv::countNonZero(cv::mean(_nd_in != _nd_out)), 0);
+        EXPECT_EQ(0, cv::norm(_nd_in, _nd_out, NORM_INF));
 
         EXPECT_EQ(_rd_in.rows   , _rd_out.rows);
         EXPECT_EQ(_rd_in.cols   , _rd_out.cols);
         EXPECT_EQ(_rd_in.dims   , _rd_out.dims);
         EXPECT_EQ(_rd_in.depth(), _rd_out.depth());
-        EXPECT_EQ(cv::countNonZero(cv::mean(_rd_in != _rd_out)), 0);
-
-        remove(name.c_str());
+        EXPECT_EQ(0, cv::norm(_rd_in, _rd_out, NORM_INF));
     }
 }
 
+TEST(Core_InputOutput, filestorage_base64_basic_read_XML)
+{
+    test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", false);
+}
+TEST(Core_InputOutput, filestorage_base64_basic_read_YAML)
+{
+    test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", false);
+}
+TEST(Core_InputOutput, filestorage_base64_basic_read_JSON)
+{
+    test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", false);
+}
+TEST(Core_InputOutput, filestorage_base64_basic_rw_XML)
+{
+    test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true);
+}
+TEST(Core_InputOutput, filestorage_base64_basic_rw_YAML)
+{
+    test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true);
+}
+TEST(Core_InputOutput, filestorage_base64_basic_rw_JSON)
+{
+    test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true);
+}
+TEST(Core_InputOutput, filestorage_base64_basic_memory_XML)
+{
+    test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true, true);
+}
+TEST(Core_InputOutput, filestorage_base64_basic_memory_YAML)
+{
+    test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true, true);
+}
+TEST(Core_InputOutput, filestorage_base64_basic_memory_JSON)
+{
+    test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true, true);
+}
+
+
 TEST(Core_InputOutput, filestorage_base64_valid_call)
 {
     const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info();
@@ -856,10 +900,12 @@ TEST(Core_InputOutput, filestorage_base64_valid_call)
     std::vector<int> rawdata(10, static_cast<int>(0x00010203));
     cv::String str_out = "test_string";
 
-    for (char const ** ptr = filenames; *ptr; ptr++)
+    for (int n = 0; n < 6; n++)
     {
-        char const * suffix_name = *ptr;
+        char const* suffix_name = filenames[n];
+        SCOPED_TRACE(suffix_name);
         std::string name = basename + '_' + suffix_name;
+        std::string file_name = basename + '_' + real_name[n];
 
         EXPECT_NO_THROW(
         {
@@ -877,7 +923,7 @@ TEST(Core_InputOutput, filestorage_base64_valid_call)
         });
 
         {
-            cv::FileStorage fs(name, cv::FileStorage::READ);
+            cv::FileStorage fs(file_name, cv::FileStorage::READ);
             std::vector<int> data_in(rawdata.size());
             fs["manydata"][0].readRaw("i", (uchar *)data_in.data(), data_in.size());
             EXPECT_TRUE(fs["manydata"][0].isSeq());
@@ -905,7 +951,7 @@ TEST(Core_InputOutput, filestorage_base64_valid_call)
         });
 
         {
-            cv::FileStorage fs(name, cv::FileStorage::READ);
+            cv::FileStorage fs(file_name, cv::FileStorage::READ);
             cv::String str_in;
             fs["manydata"][0] >> str_in;
             EXPECT_TRUE(fs["manydata"][0].isString());
@@ -917,7 +963,7 @@ TEST(Core_InputOutput, filestorage_base64_valid_call)
             fs.release();
         }
 
-        remove((basename + '_' + real_name[ptr - filenames]).c_str());
+        EXPECT_EQ(0, remove(file_name.c_str()));
     }
 }