Merge pull request #20143 from rogday:base64_encoding

modules/core/src/persistence.hpp

@@ -163,6 +163,24 @@ public:
     CV_NORETURN
     virtual void parseError(const char* funcname, const std::string& msg,
                             const char* filename, int lineno) = 0;
+
+private:
+    enum Base64State{
+        Uncertain,
+        NotUse,
+        InUse,
+    };
+
+    friend class cv::FileStorage::Impl;
+    friend class cv::FileStorage;
+    friend class JSONEmitter;
+    friend class XMLEmitter;
+    friend class YAMLEmitter;
+
+    virtual void check_if_write_struct_is_delayed(bool change_type_to_base64 = false) = 0;
+    virtual void switch_to_Base64_state(Base64State state) = 0;
+    virtual Base64State get_state_of_writing_base64() = 0;
+    virtual int get_space() = 0;
 };
 
 class FileStorageEmitter

modules/core/src/persistence_base64_encoding.cpp

@@ -0,0 +1,370 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html
+
+#include "precomp.hpp"
+#include "persistence_impl.hpp"
+#include "persistence_base64_encoding.hpp"
+
+namespace cv
+{
+
+class base64::Base64ContextEmitter
+{
+public:
+    explicit Base64ContextEmitter(cv::FileStorage::Impl& fs, bool needs_indent_)
+            : file_storage(fs)
+            , needs_indent(needs_indent_)
+            , binary_buffer(BUFFER_LEN)
+            , base64_buffer(base64_encode_buffer_size(BUFFER_LEN))
+            , src_beg(0)
+            , src_cur(0)
+            , src_end(0)
+    {
+        src_beg = binary_buffer.data();
+        src_end = src_beg + BUFFER_LEN;
+        src_cur = src_beg;
+
+        CV_Assert(fs.write_mode);
+
+        if (needs_indent)
+        {
+            file_storage.flush();
+        }
+    }
+
+    ~Base64ContextEmitter()
+    {
+        /* cleaning */
+        if (src_cur != src_beg)
+            flush();    /* encode the rest binary data to base64 buffer */
+    }
+
+    Base64ContextEmitter & write(const uchar * beg, const uchar * end)
+    {
+        if (beg >= end)
+            return *this;
+
+        while (beg < end) {
+            /* collect binary data and copy to binary buffer */
+            size_t len = std::min(end - beg, src_end - src_cur);
+            std::memcpy(src_cur, beg, len);
+            beg     += len;
+            src_cur += len;
+
+            if (src_cur >= src_end) {
+                /* binary buffer is full. */
+                /* encode it to base64 and send result to fs */
+                flush();
+            }
+        }
+
+        return *this;
+    }
+
+    /*
+     * a convertor must provide :
+     * - `operator >> (uchar * & dst)` for writing current binary data to `dst` and moving to next data.
+     * - `operator bool` for checking if current loaction is valid and not the end.
+     */
+    template<typename _to_binary_convertor_t> inline
+    Base64ContextEmitter & write(_to_binary_convertor_t & convertor)
+    {
+        static const size_t BUFFER_MAX_LEN = 1024U;
+
+        std::vector<uchar> buffer(BUFFER_MAX_LEN);
+        uchar * beg = buffer.data();
+        uchar * end = beg;
+
+        while (convertor) {
+            convertor >> end;
+            write(beg, end);
+            end = beg;
+        }
+
+        return *this;
+    }
+
+    bool flush()
+    {
+        /* control line width, so on. */
+        size_t len = base64_encode(src_beg, base64_buffer.data(), 0U, src_cur - src_beg);
+        if (len == 0U)
+            return false;
+
+        src_cur = src_beg;
+
+        if ( !needs_indent)
+        {
+            file_storage.puts((const char*)base64_buffer.data());
+        }
+        else
+        {
+            const char newline[] = "\n";
+            char space[80];
+            int ident = file_storage.write_stack.back().indent;
+            memset(space, ' ', static_cast<int>(ident));
+            space[ident] = '\0';
+
+            file_storage.puts(space);
+            file_storage.puts((const char*)base64_buffer.data());
+            file_storage.puts(newline);
+            file_storage.flush();
+        }
+
+        return true;
+    }
+
+private:
+    /* because of Base64, we must keep its length a multiple of 3 */
+    static const size_t BUFFER_LEN = 48U;
+    // static_assert(BUFFER_LEN % 3 == 0, "BUFFER_LEN is invalid");
+
+private:
+    cv::FileStorage::Impl& file_storage;
+    bool needs_indent;
+
+    std::vector<uchar> binary_buffer;
+    std::vector<uchar> base64_buffer;
+    uchar * src_beg;
+    uchar * src_cur;
+    uchar * src_end;
+};
+
+std::string base64::make_base64_header(const char *dt) {
+    std::ostringstream oss;
+    oss << dt   << ' ';
+    std::string buffer(oss.str());
+    CV_Assert(buffer.size() < ::base64::HEADER_SIZE);
+
+    buffer.reserve(::base64::HEADER_SIZE);
+    while (buffer.size() < ::base64::HEADER_SIZE)
+        buffer += ' ';
+
+    return buffer;
+}
+
+size_t base64::base64_encode(const uint8_t *src, uint8_t *dst, size_t off, size_t cnt) {
+    if (!src || !dst || !cnt)
+        return 0;
+
+    /* initialize beginning and end */
+    uint8_t       * dst_beg = dst;
+    uint8_t       * dst_cur = dst_beg;
+
+    uint8_t const * src_beg = src + off;
+    uint8_t const * src_cur = src_beg;
+    uint8_t const * src_end = src_cur + cnt / 3U * 3U;
+
+    /* integer multiples part */
+    while (src_cur < src_end) {
+        uint8_t _2 = *src_cur++;
+        uint8_t _1 = *src_cur++;
+        uint8_t _0 = *src_cur++;
+        *dst_cur++ = base64_mapping[ _2          >> 2U];
+        *dst_cur++ = base64_mapping[(_1 & 0xF0U) >> 4U | (_2 & 0x03U) << 4U];
+        *dst_cur++ = base64_mapping[(_0 & 0xC0U) >> 6U | (_1 & 0x0FU) << 2U];
+        *dst_cur++ = base64_mapping[ _0 & 0x3FU];
+    }
+
+    /* remainder part */
+    size_t rst = src_beg + cnt - src_cur;
+    if (rst == 1U) {
+        uint8_t _2 = *src_cur++;
+        *dst_cur++ = base64_mapping[ _2          >> 2U];
+        *dst_cur++ = base64_mapping[(_2 & 0x03U) << 4U];
+    } else if (rst == 2U) {
+        uint8_t _2 = *src_cur++;
+        uint8_t _1 = *src_cur++;
+        *dst_cur++ = base64_mapping[ _2          >> 2U];
+        *dst_cur++ = base64_mapping[(_2 & 0x03U) << 4U | (_1 & 0xF0U) >> 4U];
+        *dst_cur++ = base64_mapping[(_1 & 0x0FU) << 2U];
+    }
+
+    /* padding */
+    switch (rst)
+    {
+        case 1U: *dst_cur++ = base64_padding;
+            /* fallthrough */
+        case 2U: *dst_cur++ = base64_padding;
+            /* fallthrough */
+        default: *dst_cur   = 0;
+            break;
+    }
+
+    return static_cast<size_t>(dst_cur - dst_beg);
+}
+
+int base64::icvCalcStructSize(const char *dt, int initial_size) {
+    int size = cv::fs::calcElemSize( dt, initial_size );
+    size_t elem_max_size = 0;
+    for ( const char * type = dt; *type != '\0'; type++ ) {
+        switch ( *type )
+        {
+            case 'u': { elem_max_size = std::max( elem_max_size, sizeof(uchar ) ); break; }
+            case 'c': { elem_max_size = std::max( elem_max_size, sizeof(schar ) ); break; }
+            case 'w': { elem_max_size = std::max( elem_max_size, sizeof(ushort) ); break; }
+            case 's': { elem_max_size = std::max( elem_max_size, sizeof(short ) ); break; }
+            case 'i': { elem_max_size = std::max( elem_max_size, sizeof(int   ) ); break; }
+            case 'f': { elem_max_size = std::max( elem_max_size, sizeof(float ) ); break; }
+            case 'd': { elem_max_size = std::max( elem_max_size, sizeof(double) ); break; }
+            default: break;
+        }
+    }
+    size = cvAlign( size, static_cast<int>(elem_max_size) );
+    return size;
+}
+
+size_t base64::base64_encode_buffer_size(size_t cnt, bool is_end_with_zero) {
+    size_t additional = static_cast<size_t>(is_end_with_zero == true);
+    return (cnt + 2U) / 3U * 4U + additional;
+}
+
+base64::Base64Writer::Base64Writer(cv::FileStorage::Impl& fs, bool can_indent)
+        : emitter(new Base64ContextEmitter(fs, can_indent))
+        , data_type_string()
+{
+    CV_Assert(fs.write_mode);
+}
+
+void base64::Base64Writer::write(const void* _data, size_t len, const char* dt)
+{
+    check_dt(dt);
+    RawDataToBinaryConvertor convertor(_data, static_cast<int>(len), data_type_string);
+    emitter->write(convertor);
+}
+
+template<typename _to_binary_convertor_t> inline
+void base64::Base64Writer::write(_to_binary_convertor_t & convertor, const char* dt)
+{
+    check_dt(dt);
+    emitter->write(convertor);
+}
+
+base64::Base64Writer::~Base64Writer()
+{
+    delete emitter;
+}
+
+void base64::Base64Writer::check_dt(const char* dt)
+{
+    if ( dt == 0 )
+        CV_Error( cv::Error::StsBadArg, "Invalid \'dt\'." );
+    else if (data_type_string.empty()) {
+        data_type_string = dt;
+
+        /* output header */
+        std::string buffer = make_base64_header(dt);
+        const uchar * beg = reinterpret_cast<const uchar *>(buffer.data());
+        const uchar * end = beg + buffer.size();
+
+        emitter->write(beg, end);
+    } else if ( data_type_string != dt )
+        CV_Error( cv::Error::StsBadArg, "\'dt\' does not match." );
+}
+
+base64::RawDataToBinaryConvertor::RawDataToBinaryConvertor(const void* src, int len, const std::string & dt)
+        : beg(reinterpret_cast<const uchar *>(src))
+        , cur(0)
+        , end(0)
+{
+    CV_Assert(src);
+    CV_Assert(!dt.empty());
+    CV_Assert(len > 0);
+
+    /* calc step and to_binary_funcs */
+    step_packed = make_to_binary_funcs(dt);
+
+    end = beg;
+    cur = beg;
+
+    step = icvCalcStructSize(dt.c_str(), 0);
+    end = beg + static_cast<size_t>(len);
+}
+
+inline  base64::RawDataToBinaryConvertor&  base64::RawDataToBinaryConvertor::operator >>(uchar * & dst)
+{
+    CV_DbgAssert(*this);
+
+    for (size_t i = 0U, n = to_binary_funcs.size(); i < n; i++) {
+        elem_to_binary_t & pack = to_binary_funcs[i];
+        pack.func(cur + pack.offset, dst + pack.offset_packed);
+    }
+    cur += step;
+    dst += step_packed;
+
+    return *this;
+}
+
+inline  base64::RawDataToBinaryConvertor::operator bool() const
+{
+    return cur < end;
+}
+
+size_t base64::RawDataToBinaryConvertor::make_to_binary_funcs(const std::string &dt)
+{
+    size_t cnt = 0;
+    size_t offset = 0;
+    size_t offset_packed = 0;
+    char type = '\0';
+
+    std::istringstream iss(dt);
+    while (!iss.eof()) {
+        if (!(iss >> cnt)) {
+            iss.clear();
+            cnt = 1;
+        }
+        CV_Assert(cnt > 0U);
+        if (!(iss >> type))
+            break;
+
+        while (cnt-- > 0)
+        {
+            elem_to_binary_t pack;
+
+            size_t size = 0;
+            switch (type)
+            {
+                case 'u':
+                case 'c':
+                    size = sizeof(uchar);
+                    pack.func = to_binary<uchar>;
+                    break;
+                case 'w':
+                case 's':
+                    size = sizeof(ushort);
+                    pack.func = to_binary<ushort>;
+                    break;
+                case 'i':
+                    size = sizeof(uint);
+                    pack.func = to_binary<uint>;
+                    break;
+                case 'f':
+                    size = sizeof(float);
+                    pack.func = to_binary<float>;
+                    break;
+                case 'd':
+                    size = sizeof(double);
+                    pack.func = to_binary<double>;
+                    break;
+                case 'r':
+                default:
+                    CV_Error(cv::Error::StsError, "type is not supported");
+            };
+
+            offset = static_cast<size_t>(cvAlign(static_cast<int>(offset), static_cast<int>(size)));
+            pack.offset = offset;
+            offset += size;
+
+            pack.offset_packed = offset_packed;
+            offset_packed += size;
+
+            to_binary_funcs.push_back(pack);
+        }
+    }
+
+    CV_Assert(iss.eof());
+    return offset_packed;
+}
+
+}

modules/core/src/persistence_base64_encoding.hpp

@@ -0,0 +1,127 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html
+
+#ifndef OPENCV_CORE_BASE64_ENCODING_HPP
+#define OPENCV_CORE_BASE64_ENCODING_HPP
+
+namespace cv
+{
+
+namespace base64
+{
+/* A decorator for CvFileStorage
+* - no copyable
+* - not safe for now
+* - move constructor may be needed if C++11
+*/
+uint8_t const base64_mapping[] =
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "abcdefghijklmnopqrstuvwxyz"
+        "0123456789+/";
+
+uint8_t const base64_padding = '=';
+
+std::string make_base64_header(const char * dt);
+
+size_t base64_encode(uint8_t const * src, uint8_t * dst, size_t off, size_t cnt);
+
+
+int icvCalcStructSize( const char* dt, int initial_size );
+
+class Base64ContextEmitter;
+class Impl;
+
+class Base64Writer
+{
+public:
+    Base64Writer(cv::FileStorage::Impl& fs, bool can_indent);
+    ~Base64Writer();
+    void write(const void* _data, size_t len, const char* dt);
+    template<typename _to_binary_convertor_t> void write(_to_binary_convertor_t & convertor, const char* dt);
+
+private:
+    void check_dt(const char* dt);
+
+private:
+    // disable copy and assignment
+    Base64Writer(const Base64Writer &);
+    Base64Writer & operator=(const Base64Writer &);
+
+private:
+
+    Base64ContextEmitter * emitter;
+    std::string data_type_string;
+};
+
+size_t base64_encode_buffer_size(size_t cnt, bool is_end_with_zero = true);
+
+template<typename _uint_t> inline size_t
+to_binary(_uint_t val, uchar * cur)
+{
+    size_t delta = CHAR_BIT;
+    size_t cnt = sizeof(_uint_t);
+    while (cnt --> static_cast<size_t>(0U)) {
+        *cur++ = static_cast<uchar>(val);
+        val >>= delta;
+    }
+    return sizeof(_uint_t);
+}
+
+template<> inline size_t to_binary(double val, uchar * cur)
+{
+    Cv64suf bit64;
+    bit64.f = val;
+    return to_binary(bit64.u, cur);
+}
+
+template<> inline size_t to_binary(float val, uchar * cur)
+{
+    Cv32suf bit32;
+    bit32.f = val;
+    return to_binary(bit32.u, cur);
+}
+
+template<typename _primitive_t> inline size_t
+to_binary(uchar const * val, uchar * cur)
+{
+    return to_binary<_primitive_t>(*reinterpret_cast<_primitive_t const *>(val), cur);
+}
+
+
+
+class RawDataToBinaryConvertor
+{
+public:
+    // NOTE: len is already multiplied by element size here
+    RawDataToBinaryConvertor(const void* src, int len, const std::string & dt);
+
+    inline RawDataToBinaryConvertor & operator >>(uchar * & dst);
+    inline operator bool() const;
+
+private:
+    typedef size_t(*to_binary_t)(const uchar *, uchar *);
+    struct elem_to_binary_t
+    {
+        size_t      offset;
+        size_t      offset_packed;
+        to_binary_t func;
+    };
+
+private:
+    size_t make_to_binary_funcs(const std::string &dt);
+
+private:
+    const uchar * beg;
+    const uchar * cur;
+    const uchar * end;
+
+    size_t step;
+    size_t step_packed;
+    std::vector<elem_to_binary_t> to_binary_funcs;
+};
+
+}
+
+}
+#endif

modules/core/src/persistence_impl.hpp

@@ -0,0 +1,231 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html
+
+#ifndef OPENCV_CORE_PERSISTENCE_IMPL_HPP
+#define OPENCV_CORE_PERSISTENCE_IMPL_HPP
+
+#include "persistence.hpp"
+#include "persistence_base64_encoding.hpp"
+#include <unordered_map>
+#include <iterator>
+
+
+namespace cv
+{
+
+enum Base64State{
+    Uncertain,
+    NotUse,
+    InUse,
+};
+
+class cv::FileStorage::Impl : public FileStorage_API
+{
+public:
+    void init();
+
+    Impl(FileStorage* _fs);
+
+    virtual ~Impl();
+
+    void release(String* out=0);
+
+    void analyze_file_name( const std::string& file_name, std::vector<std::string>& params );
+
+    bool open( const char* filename_or_buf, int _flags, const char* encoding );
+
+    void puts( const char* str );
+
+    char* getsFromFile( char* buf, int count );
+
+    char* gets( size_t maxCount );
+
+    char* gets();
+
+    bool eof();
+
+    void setEof();
+
+    void closeFile();
+
+    void rewind();
+
+    char* resizeWriteBuffer( char* ptr, int len );
+
+    char* flush();
+
+    void endWriteStruct();
+
+    void startWriteStruct_helper( const char* key, int struct_flags,
+                                  const char* type_name );
+
+    void startWriteStruct( const char* key, int struct_flags,
+                           const char* type_name );
+
+    void writeComment( const char* comment, bool eol_comment );
+
+    void startNextStream();
+
+    void write( const String& key, int value );
+
+    void write( const String& key, double value );
+
+    void write( const String& key, const String& value );
+
+    void writeRawData( const std::string& dt, const void* _data, size_t len );
+
+    void workaround();
+
+    void switch_to_Base64_state( FileStorage_API::Base64State new_state);
+
+    void make_write_struct_delayed( const char* key, int struct_flags, const char* type_name );
+
+    void check_if_write_struct_is_delayed( bool change_type_to_base64 );
+
+    void writeRawDataBase64(const void* _data, size_t len, const char* dt );
+
+    String releaseAndGetString();
+
+    FileNode getFirstTopLevelNode() const;
+
+    FileNode root(int streamIdx=0) const;
+
+    FileNode operator[](const String& nodename) const;
+
+    FileNode operator[](const char* /*nodename*/) const;
+
+    int getFormat() const;
+
+    char* bufferPtr() const;
+    char* bufferStart() const;
+    char* bufferEnd() const;
+    void setBufferPtr(char* ptr);
+    int wrapMargin() const;
+
+    FStructData& getCurrentStruct();
+
+    void setNonEmpty();
+
+    void processSpecialDouble( char* buf, double* value, char** endptr );
+
+    double strtod( char* ptr, char** endptr );
+
+    void convertToCollection(int type, FileNode& node);
+
+    // a) allocates new FileNode (for that just set blockIdx to the last block and ofs to freeSpaceOfs) or
+    // b) reallocates just created new node (blockIdx and ofs must be taken from FileNode).
+    //    If there is no enough space in the current block (it should be the last block added so far),
+    //    the last block is shrunk so that it ends immediately before the reallocated node. Then,
+    //    a new block of sufficient size is allocated and the FileNode is placed in the beginning of it.
+    // The case (a) can be used to allocate the very first node by setting blockIdx == ofs == 0.
+    // In the case (b) the existing tag and the name are copied automatically.
+    uchar* reserveNodeSpace(FileNode& node, size_t sz);
+
+    unsigned getStringOfs( const std::string& key ) const;
+
+    FileNode addNode( FileNode& collection, const std::string& key,
+                      int elem_type, const void* value, int len );
+
+    void finalizeCollection( FileNode& collection );
+
+    void normalizeNodeOfs(size_t& blockIdx, size_t& ofs) const;
+
+    Base64State get_state_of_writing_base64();
+
+    int get_space();
+
+    class Base64Decoder
+    {
+    public:
+        Base64Decoder();
+        void init(Ptr<FileStorageParser>& _parser, char* _ptr, int _indent);
+
+        bool readMore(int needed);
+
+        uchar getUInt8();
+
+        ushort getUInt16();
+
+        int getInt32();
+
+        double getFloat64();
+
+        bool endOfStream() const;
+        char* getPtr() const;
+    protected:
+
+        Ptr<FileStorageParser> parser;
+        char* ptr;
+        int indent;
+        std::vector<char> encoded;
+        std::vector<uchar> decoded;
+        size_t ofs;
+        size_t totalchars;
+        bool eos;
+    };
+
+    char* parseBase64(char* ptr, int indent, FileNode& collection);
+
+    void parseError( const char* func_name, const std::string& err_msg, const char* source_file, int source_line );
+
+    const uchar* getNodePtr(size_t blockIdx, size_t ofs) const;
+
+    std::string getName( size_t nameofs ) const;
+
+    FileStorage* getFS();
+
+    FileStorage* fs_ext;
+
+    std::string filename;
+    int flags;
+    bool empty_stream;
+
+    FILE* file;
+    gzFile gzfile;
+
+    bool is_opened;
+    bool dummy_eof;
+    bool write_mode;
+    bool mem_mode;
+    int fmt;
+
+    State state; //!< current state of the FileStorage (used only for writing)
+    bool is_using_base64;
+    bool is_write_struct_delayed;
+    char* delayed_struct_key;
+    int   delayed_struct_flags;
+    char* delayed_type_name;
+    FileStorage_API::Base64State state_of_writing_base64;
+
+    int space, wrap_margin;
+    std::deque<FStructData> write_stack;
+    std::vector<char> buffer;
+    size_t bufofs;
+
+    std::deque<char> outbuf;
+
+    Ptr<FileStorageEmitter> emitter;
+    Ptr<FileStorageParser> parser;
+    Base64Decoder base64decoder;
+    base64::Base64Writer* base64_writer;
+
+    std::vector<FileNode> roots;
+    std::vector<Ptr<std::vector<uchar> > > fs_data;
+    std::vector<uchar*> fs_data_ptrs;
+    std::vector<size_t> fs_data_blksz;
+    size_t freeSpaceOfs;
+    typedef std::unordered_map<std::string, unsigned> str_hash_t;
+    str_hash_t str_hash;
+    std::vector<char> str_hash_data;
+
+    std::vector<char> strbufv;
+    char* strbuf;
+    size_t strbufsize;
+    size_t strbufpos;
+    int lineno;
+};
+
+}
+
+#endif

modules/core/src/persistence_json.cpp

@@ -23,7 +23,7 @@ public:
 
         struct_flags = (struct_flags & (FileNode::TYPE_MASK|FileNode::FLOW)) | FileNode::EMPTY;
         if( !FileNode::isCollection(struct_flags))
-            CV_Error( CV_StsBadArg,
+            CV_Error( cv::Error::StsBadArg,
                      "Some collection type - FileNode::SEQ or FileNode::MAP, must be specified" );
 
         if( type_name && *type_name == '\0' )
@@ -53,29 +53,26 @@ public:
     void endWriteStruct(const FStructData& current_struct)
     {
         int struct_flags = current_struct.flags;
-        CV_Assert( FileNode::isCollection(struct_flags) );
 
-        if( !FileNode::isFlow(struct_flags) )
-        {
-#if 0
-            if ( fs->bufferPtr() <= fs->bufferStart() + fs->space )
-            {
-                /* some bad code for base64_writer... */
-                ptr = fs->bufferPtr();
-                *ptr++ = '\n';
-                *ptr++ = '\0';
-                fs->puts( fs->bufferStart() );
-                fs->setBufferPtr(fs->bufferStart());
+        if (FileNode::isCollection(struct_flags)) {
+            if (!FileNode::isFlow(struct_flags)) {
+                if (fs->bufferPtr() <= fs->bufferStart() + fs->get_space()) {
+                    /* some bad code for base64_writer... */
+                    char *ptr = fs->bufferPtr();
+                    *ptr++ = '\n';
+                    *ptr++ = '\0';
+                    fs->puts(fs->bufferStart());
+                    fs->setBufferPtr(fs->bufferStart());
+                }
+                fs->flush();
             }
-#endif
-            fs->flush();
-        }
 
-        char* ptr = fs->bufferPtr();
-        if( ptr > fs->bufferStart() + current_struct.indent && !FileNode::isEmptyCollection(struct_flags) )
-            *ptr++ = ' ';
-        *ptr++ = FileNode::isMap(struct_flags) ? '}' : ']';
-        fs->setBufferPtr(ptr);
+            char *ptr = fs->bufferPtr();
+            if (ptr > fs->bufferStart() + current_struct.indent && !FileNode::isEmptyCollection(struct_flags))
+                *ptr++ = ' ';
+            *ptr++ = FileNode::isMap(struct_flags) ? '}' : ']';
+            fs->setBufferPtr(ptr);
+        }
     }
 
     void write(const char* key, int value)
@@ -97,11 +94,11 @@ public:
         int i, len;
 
         if( !str )
-            CV_Error( CV_StsNullPtr, "Null string pointer" );
+            CV_Error( cv::Error::StsNullPtr, "Null string pointer" );
 
         len = (int)strlen(str);
         if( len > CV_FS_MAX_LEN )
-            CV_Error( CV_StsBadArg, "The written string is too long" );
+            CV_Error( cv::Error::StsBadArg, "The written string is too long" );
 
         if( quote || len == 0 || str[0] != str[len-1] || (str[0] != '\"' && str[0] != '\'') )
         {
@@ -136,6 +133,20 @@ public:
 
     void writeScalar(const char* key, const char* data)
     {
+        /* check write_struct */
+
+        fs->check_if_write_struct_is_delayed(false);
+        if ( fs->get_state_of_writing_base64() == FileStorage_API::Uncertain )
+        {
+            fs->switch_to_Base64_state( FileStorage_API::NotUse );
+        }
+        else if ( fs->get_state_of_writing_base64() == FileStorage_API::InUse )
+        {
+            CV_Error( cv::Error::StsError, "At present, output Base64 data only." );
+        }
+
+        /* check parameters */
+
         size_t key_len = 0u;
         if( key && *key == '\0' )
             key = 0;
@@ -143,9 +154,9 @@ public:
         {
             key_len = strlen(key);
             if ( key_len == 0u )
-                CV_Error( CV_StsBadArg, "The key is an empty" );
+                CV_Error( cv::Error::StsBadArg, "The key is an empty" );
             else if ( static_cast<int>(key_len) > CV_FS_MAX_LEN )
-                CV_Error( CV_StsBadArg, "The key is too long" );
+                CV_Error( cv::Error::StsBadArg, "The key is too long" );
         }
 
         size_t data_len = 0u;
@@ -157,7 +168,7 @@ public:
         if( FileNode::isCollection(struct_flags) )
         {
             if ( (FileNode::isMap(struct_flags) ^ (key != 0)) )
-                CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, "
+                CV_Error( cv::Error::StsBadArg, "An attempt to add element without a key to a map, "
                          "or add element with key to sequence" );
         } else {
             fs->setNonEmpty();
@@ -199,7 +210,7 @@ public:
         if( key )
         {
             if( !cv_isalpha(key[0]) && key[0] != '_' )
-                CV_Error( CV_StsBadArg, "Key must start with a letter or _" );
+                CV_Error( cv::Error::StsBadArg, "Key must start with a letter or _" );
 
             ptr = fs->resizeWriteBuffer( ptr, static_cast<int>(key_len) );
             *ptr++ = '\"';
@@ -210,7 +221,7 @@ public:
 
                 ptr[i] = c;
                 if( !cv_isalnum(c) && c != '-' && c != '_' && c != ' ' )
-                    CV_Error( CV_StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" );
+                    CV_Error( cv::Error::StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" );
             }
 
             ptr += key_len;
@@ -233,7 +244,7 @@ public:
     void writeComment(const char* comment, bool eol_comment)
     {
         if( !comment )
-            CV_Error( CV_StsNullPtr, "Null comment" );
+            CV_Error( cv::Error::StsNullPtr, "Null comment" );
 
         int len = static_cast<int>(strlen(comment));
         char* ptr = fs->bufferPtr();

modules/core/src/persistence_xml.cpp

@@ -45,7 +45,7 @@ public:
             if( FileNode::isCollection(struct_flags) )
             {
                 if( FileNode::isMap(struct_flags) ^ (key != 0) )
-                    CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, "
+                    CV_Error( cv::Error::StsBadArg, "An attempt to add element without a key to a map, "
                              "or add element with key to sequence" );
             }
             else
@@ -61,26 +61,26 @@ public:
         if( !key )
             key = "_";
         else if( key[0] == '_' && key[1] == '\0' )
-            CV_Error( CV_StsBadArg, "A single _ is a reserved tag name" );
+            CV_Error( cv::Error::StsBadArg, "A single _ is a reserved tag name" );
 
         len = (int)strlen( key );
         *ptr++ = '<';
         if( tag_type == CV_XML_CLOSING_TAG )
         {
             if( !attrlist.empty() )
-                CV_Error( CV_StsBadArg, "Closing tag should not include any attributes" );
+                CV_Error( cv::Error::StsBadArg, "Closing tag should not include any attributes" );
             *ptr++ = '/';
         }
 
         if( !cv_isalpha(key[0]) && key[0] != '_' )
-            CV_Error( CV_StsBadArg, "Key should start with a letter or _" );
+            CV_Error( cv::Error::StsBadArg, "Key should start with a letter or _" );
 
         ptr = fs->resizeWriteBuffer( ptr, len );
         for( i = 0; i < len; i++ )
         {
             char c = key[i];
             if( !cv_isalnum(c) && c != '_' && c != '-' )
-                CV_Error( CV_StsBadArg, "Key name may only contain alphanumeric characters [a-zA-Z0-9], '-' and '_'" );
+                CV_Error( cv::Error::StsBadArg, "Key name may only contain alphanumeric characters [a-zA-Z0-9], '-' and '_'" );
             ptr[i] = c;
         }
         ptr += len;
@@ -158,11 +158,11 @@ public:
         int i, len;
 
         if( !str )
-            CV_Error( CV_StsNullPtr, "Null string pointer" );
+            CV_Error( cv::Error::StsNullPtr, "Null string pointer" );
 
         len = (int)strlen(str);
         if( len > CV_FS_MAX_LEN )
-            CV_Error( CV_StsBadArg, "The written string is too long" );
+            CV_Error( cv::Error::StsBadArg, "The written string is too long" );
 
         if( quote || len == 0 || str[0] != '\"' || str[0] != str[len-1] )
         {
@@ -233,6 +233,16 @@ public:
 
     void writeScalar(const char* key, const char* data)
     {
+        fs->check_if_write_struct_is_delayed(false);
+        if ( fs->get_state_of_writing_base64() == FileStorage_API::Uncertain )
+        {
+            fs->switch_to_Base64_state( FileStorage_API::NotUse );
+        }
+        else if ( fs->get_state_of_writing_base64() == FileStorage_API::InUse )
+        {
+            CV_Error( cv::Error::StsError, "At present, output Base64 data only." );
+        }
+
         int len = (int)strlen(data);
         if( key && *key == '\0' )
             key = 0;
@@ -255,7 +265,7 @@ public:
             int new_offset = (int)(ptr - fs->bufferStart()) + len;
 
             if( key )
-                CV_Error( CV_StsBadArg, "elements with keys can not be written to sequence" );
+                CV_Error( cv::Error::StsBadArg, "elements with keys can not be written to sequence" );
 
             current_struct.flags = FileNode::SEQ;
 
@@ -281,10 +291,10 @@ public:
         char* ptr;
 
         if( !comment )
-            CV_Error( CV_StsNullPtr, "Null comment" );
+            CV_Error( cv::Error::StsNullPtr, "Null comment" );
 
         if( strstr(comment, "--") != 0 )
-            CV_Error( CV_StsBadArg, "Double hyphen \'--\' is not allowed in the comments" );
+            CV_Error( cv::Error::StsBadArg, "Double hyphen \'--\' is not allowed in the comments" );
 
         len = (int)strlen(comment);
         eol = strchr(comment, '\n');

modules/core/src/persistence_yml.cpp

@@ -33,7 +33,7 @@ public:
 
         struct_flags = (struct_flags & (FileNode::TYPE_MASK|FileNode::FLOW)) | FileNode::EMPTY;
         if( !FileNode::isCollection(struct_flags))
-            CV_Error( CV_StsBadArg,
+            CV_Error( cv::Error::StsBadArg,
                      "Some collection type - FileNode::SEQ or FileNode::MAP, must be specified" );
 
         if (type_name && memcmp(type_name, "binary", 6) == 0)
@@ -120,11 +120,11 @@ public:
         int i, len;
 
         if( !str )
-            CV_Error( CV_StsNullPtr, "Null string pointer" );
+            CV_Error( cv::Error::StsNullPtr, "Null string pointer" );
 
         len = (int)strlen(str);
         if( len > CV_FS_MAX_LEN )
-            CV_Error( CV_StsBadArg, "The written string is too long" );
+            CV_Error( cv::Error::StsBadArg, "The written string is too long" );
 
         if( quote || len == 0 || str[0] != str[len-1] || (str[0] != '\"' && str[0] != '\'') )
         {
@@ -174,6 +174,16 @@ public:
 
     void writeScalar(const char* key, const char* data)
     {
+        fs->check_if_write_struct_is_delayed(false);
+        if ( fs->get_state_of_writing_base64() == FileStorage_API::Uncertain )
+        {
+            fs->switch_to_Base64_state( FileStorage_API::NotUse );
+        }
+        else if ( fs->get_state_of_writing_base64() == FileStorage_API::InUse )
+        {
+            CV_Error( cv::Error::StsError, "At present, output Base64 data only." );
+        }
+
         int i, keylen = 0;
         int datalen = 0;
         char* ptr;
@@ -188,7 +198,7 @@ public:
         if( FileNode::isCollection(struct_flags) )
         {
             if( (FileNode::isMap(struct_flags) ^ (key != 0)) )
-                CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, "
+                CV_Error( cv::Error::StsBadArg, "An attempt to add element without a key to a map, "
                          "or add element with key to sequence" );
         }
         else
@@ -201,10 +211,10 @@ public:
         {
             keylen = (int)strlen(key);
             if( keylen == 0 )
-                CV_Error( CV_StsBadArg, "The key is an empty" );
+                CV_Error( cv::Error::StsBadArg, "The key is an empty" );
 
             if( keylen > CV_FS_MAX_LEN )
-                CV_Error( CV_StsBadArg, "The key is too long" );
+                CV_Error( cv::Error::StsBadArg, "The key is too long" );
         }
 
         if( data )
@@ -238,7 +248,7 @@ public:
         if( key )
         {
             if( !cv_isalpha(key[0]) && key[0] != '_' )
-                CV_Error( CV_StsBadArg, "Key must start with a letter or _" );
+                CV_Error( cv::Error::StsBadArg, "Key must start with a letter or _" );
 
             ptr = fs->resizeWriteBuffer( ptr, keylen );
 
@@ -248,7 +258,7 @@ public:
 
                 ptr[i] = c;
                 if( !cv_isalnum(c) && c != '-' && c != '_' && c != ' ' )
-                    CV_Error( CV_StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" );
+                    CV_Error( cv::Error::StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" );
             }
 
             ptr += keylen;
@@ -271,7 +281,7 @@ public:
     void writeComment(const char* comment, bool eol_comment)
     {
         if( !comment )
-            CV_Error( CV_StsNullPtr, "Null comment" );
+            CV_Error( cv::Error::StsNullPtr, "Null comment" );
 
         int len = (int)strlen(comment);
         const char* eol = strchr(comment, '\n');

modules/core/test/test_io.cpp

@@ -586,6 +586,7 @@ static void test_filestorage_basic(int write_flags, const char* suffix_name, boo
     const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info();
     CV_Assert(test_info);
     std::string name = (std::string(test_info->test_case_name()) + "--" + test_info->name() + suffix_name);
+    std::string name_34 = string(cvtest::TS::ptr()->get_data_path()) + "io/3_4/" + name;
     if (!testReadWrite)
         name = string(cvtest::TS::ptr()->get_data_path()) + "io/" + name;
 
@@ -661,7 +662,23 @@ static void test_filestorage_basic(int write_flags, const char* suffix_name, boo
                 std::ifstream f(name.c_str(), std::ios::in|std::ios::binary);
                 f.seekg(0, std::fstream::end);
                 sz = (size_t)f.tellg();
+
+                f.seekg(0, std::ios::beg);
+                std::vector<char> test_data(sz);
+                f.read(&test_data[0], sz);
                 f.close();
+
+                std::ifstream reference(name_34.c_str(), std::ios::in|std::ios::binary);
+                ASSERT_TRUE(reference.is_open());
+                reference.seekg(0, std::fstream::end);
+                size_t ref_sz = (size_t)reference.tellg();
+
+                reference.seekg(0, std::ios::beg);
+                std::vector<char> reference_data(ref_sz);
+                reference.read(&reference_data[0], ref_sz);
+                reference.close();
+
+                EXPECT_EQ(reference_data, test_data);
             }
             std::cout << "Storage size: " << sz << std::endl;
             EXPECT_LE(sz, (size_t)6000);
@@ -757,27 +774,27 @@ TEST(Core_InputOutput, filestorage_base64_basic_read_JSON)
 {
     test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", false);
 }
-TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_rw_XML)
+TEST(Core_InputOutput, filestorage_base64_basic_rw_XML)
 {
     test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true);
 }
-TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_rw_YAML)
+TEST(Core_InputOutput, filestorage_base64_basic_rw_YAML)
 {
     test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true);
 }
-TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_rw_JSON)
+TEST(Core_InputOutput, filestorage_base64_basic_rw_JSON)
 {
     test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true);
 }
-TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_memory_XML)
+TEST(Core_InputOutput, filestorage_base64_basic_memory_XML)
 {
     test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true, true);
 }
-TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_memory_YAML)
+TEST(Core_InputOutput, filestorage_base64_basic_memory_YAML)
 {
     test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true, true);
 }
-TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_memory_JSON)
+TEST(Core_InputOutput, filestorage_base64_basic_memory_JSON)
 {
     test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true, true);
 }

modules/python/test/test_filestorage_io.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 """Algorithm serialization test."""
 from __future__ import print_function
+import base64
+import json
 import tempfile
 import os
 import cv2 as cv
@@ -109,5 +111,96 @@ class filestorage_io_test(NewOpenCVTests):
     def test_json(self):
         self.run_fs_test(".json")
 
+    def test_base64(self):
+        fd, fname = tempfile.mkstemp(prefix="opencv_python_sample_filestorage_base64", suffix=".json")
+        os.close(fd)
+        np.random.seed(42)
+        self.write_base64_json(fname)
+        os.remove(fname)
+
+    @staticmethod
+    def get_normal_2d_mat():
+        rows = 10
+        cols = 20
+        cn = 3
+
+        image = np.zeros((rows, cols, cn), np.uint8)
+        image[:] = (1, 2, 127)
+
+        for i in range(rows):
+            for j in range(cols):
+                image[i, j, 1] = (i + j) % 256
+
+        return image
+
+    @staticmethod
+    def get_normal_nd_mat():
+        shape = (2, 2, 1, 2)
+        cn = 4
+
+        image = np.zeros(shape + (cn,), np.float64)
+        image[:] = (0.888, 0.111, 0.666, 0.444)
+
+        return image
+
+    @staticmethod
+    def get_empty_2d_mat():
+        shape = (0, 0)
+        cn = 1
+
+        image = np.zeros(shape + (cn,), np.uint8)
+
+        return image
+
+    @staticmethod
+    def get_random_mat():
+        rows = 8
+        cols = 16
+        cn = 1
+
+        image = np.random.rand(rows, cols, cn)
+
+        return image
+
+    @staticmethod
+    def decode(data):
+        # strip $base64$
+        encoded = data[8:]
+
+        if len(encoded) == 0:
+            return b''
+
+        # strip info about datatype and padding
+        return base64.b64decode(encoded)[24:]
+
+    def write_base64_json(self, fname):
+        fs = cv.FileStorage(fname, cv.FileStorage_WRITE_BASE64)
+
+        mats = {'normal_2d_mat': self.get_normal_2d_mat(),
+                'normal_nd_mat': self.get_normal_nd_mat(),
+                'empty_2d_mat': self.get_empty_2d_mat(),
+                'random_mat': self.get_random_mat()}
+
+        for name, mat in mats.items():
+            fs.write(name, mat)
+
+        fs.release()
+
+        data = {}
+        with open(fname) as file:
+            data = json.load(file)
+
+        for name, mat in mats.items():
+            buffer = b''
+
+            if mat.size != 0:
+                if hasattr(mat, 'tobytes'):
+                    buffer = mat.tobytes()
+                else:
+                    buffer = mat.tostring()
+
+            self.assertEqual(buffer, self.decode(data[name]['data']))
+
+
 if __name__ == '__main__':
     NewOpenCVTests.bootstrap()