Chiebot-Mirror
/
opencv
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Modify Base64 functions and add test and documentation

Browse Source 
Major changes:

- modify the Base64 functions to compatible with `cvWriteRawData` and so
on.
- add a Base64 flag for FileStorage and outputs raw data in Base64
automatically.
- complete all  testing and documentation.
pull/6949/head
MYLS 9 years ago
parent a9b4cdd36f
commit 617df09143
8 changed files with 908 additions and 375 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 58
      modules/core/include/opencv2/core/core_c.h
  2. 9
      modules/core/include/opencv2/core/persistence.hpp
  3. 2
      modules/core/include/opencv2/core/types_c.h
  4. 86
      modules/core/perf/perf_io_base64.cpp
  5. 705
      modules/core/src/persistence.cpp
  6. 147
      modules/core/test/test_io.cpp
  7. 270
      modules/core/test/test_io_base64.cpp
  8. 6
      modules/ml/test/test_save_load.cpp

58
modules/core/include/opencv2/core/core_c.h
Unescape View File

@ -1976,8 +1976,19 @@ CVAPI(void) cvSetIPLAllocators( Cv_iplCreateImageHeader create_header,
The function opens file storage for reading or writing data. In the latter case, a new file is
created or an existing file is rewritten. The type of the read or written file is determined by the
filename extension: .xml for XML and .yml or .yaml for YAML. The function returns a pointer to the
CvFileStorage structure. If the file cannot be opened then the function returns NULL.
filename extension: .xml for XML and .yml or .yaml for YAML.
At the same time, it also supports adding parameters like "example.xml?base64".
@code
CvFileStorage* fs = cvOpenFileStorage( "example.yml?base64", 0, CV_STORAGE_WRITE );
@endcode
it's exactly the same as
@code
CvFileStorage* fs = cvOpenFileStorage( "example.yml", 0, CV_STORAGE_WRITE_BASE64 );
@endcode
The function returns a pointer to the CvFileStorage structure.
If the file cannot be opened then the function returns NULL.
@param filename Name of the file associated with the storage
@param memstorage Memory storage used for temporary data and for
: storing dynamic structures, such as CvSeq or CvGraph . If it is NULL, a temporary memory
@ -1985,6 +1996,7 @@ CvFileStorage structure. If the file cannot be opened then the function returns
@param flags Can be one of the following:
> - **CV_STORAGE_READ** the storage is open for reading
> - **CV_STORAGE_WRITE** the storage is open for writing
(use **CV_STORAGE_WRITE | CV_STORAGE_WRITE_BASE64** to write rawdata in Base64)
@param encoding
*/
CVAPI(CvFileStorage*) cvOpenFileStorage( const char* filename, CvMemStorage* memstorage,
@ -2162,7 +2174,7 @@ the file with multiple streams looks like this:
@endcode
The YAML file will look like this:
@code{.yaml}
%YAML:1.0
%YAML 1.0
# stream #1 data
...
---
@ -2187,6 +2199,46 @@ to a sequence rather than a map.
CVAPI(void) cvWriteRawData( CvFileStorage* fs, const void* src,
int len, const char* dt );
/** @brief Writes multiple numbers in Base64.
If either CV_STORAGE_WRITE_BASE64 or cv::FileStorage::WRITE_BASE64 is used,
this function will be the same as cvWriteRawData. If neither, the main
difference is that it outputs a sequence in Base64 encoding rather than
in plain text.
This function can only be used to write a sequence with a type "binary".
Consider the following two examples where their output is the same:
@code
std::vector<int> rawdata(10, 0x00010203);
// without the flag CV_STORAGE_WRITE_BASE64.
CvFileStorage* fs = cvOpenFileStorage( "example.yml", 0, CV_STORAGE_WRITE );
// both CV_NODE_SEQ and "binary" are necessary.
cvStartWriteStruct(fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW, "binary");
cvWriteRawDataBase64(fs, rawdata.data(), rawdata.size(), "i");
cvEndWriteStruct(fs);
cvReleaseFileStorage( &fs );
@endcode
and
@code
std::vector<int> rawdata(10, 0x00010203);
// with the flag CV_STORAGE_WRITE_BASE64.
CvFileStorage* fs = cvOpenFileStorage( "example.yml", 0, CV_STORAGE_WRITE_BASE64);
// parameter, typename "binary" could be omitted.
cvStartWriteStruct(fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW);
cvWriteRawData(fs, rawdata.data(), rawdata.size(), "i");
cvEndWriteStruct(fs);
cvReleaseFileStorage( &fs );
@endcode
@param fs File storage
@param src Pointer to the written array
@param len Number of the array elements to write
@param dt Specification of each array element, see @ref format_spec "format specification"
*/
CVAPI(void) cvWriteRawDataBase64( CvFileStorage* fs, const void* _data,
int len, const char* dt );
/** @brief Returns a unique pointer for a given name.
The function returns a unique pointer for each particular file node name. This pointer can be then

9
modules/core/include/opencv2/core/persistence.hpp
Unescape View File

@ -311,7 +311,10 @@ public:
FORMAT_MASK = (7<<3), //!< mask for format flags
FORMAT_AUTO = 0, //!< flag, auto format
FORMAT_XML = (1<<3), //!< flag, XML format
FORMAT_YAML = (2<<3) //!< flag, YAML format
FORMAT_YAML = (2<<3), //!< flag, YAML format
BASE64 = 64, //!< flag, write rawdata in Base64 by default. (consider using WRITE_BASE64)
WRITE_BASE64 = BASE64 | WRITE, //!< flag, enable both WRITE and BASE64
};
enum
{
@ -354,7 +357,9 @@ public:
Extension of the file (.xml or .yml/.yaml) determines its format (XML or YAML respectively).
Also you can append .gz to work with compressed files, for example myHugeMatrix.xml.gz. If both
FileStorage::WRITE and FileStorage::MEMORY flags are specified, source is used just to specify
the output file format (e.g. mydata.xml, .yml etc.).
the output file format (e.g. mydata.xml, .yml etc.). A file name can also contain parameters.
You can use this format, "*?base64" (e.g. "file.xml?base64"), as an alternative to
FileStorage::BASE64 flag. Note: it is case sensitive.
@param flags Mode of operation. One of FileStorage::Mode
@param encoding Encoding of the file. Note that UTF-16 XML encoding is not supported currently and
you should use 8-bit encoding instead of it.

2
modules/core/include/opencv2/core/types_c.h
Unescape View File

@ -1669,6 +1669,8 @@ typedef struct CvFileStorage CvFileStorage;
#define CV_STORAGE_FORMAT_AUTO 0
#define CV_STORAGE_FORMAT_XML 8
#define CV_STORAGE_FORMAT_YAML 16
#define CV_STORAGE_BASE64 64
#define CV_STORAGE_WRITE_BASE64 (CV_STORAGE_BASE64 | CV_STORAGE_WRITE)
/** @brief List of attributes. :

86
modules/core/perf/perf_io_base64.cpp
Unescape View File

@ -0,0 +1,86 @@
#include "perf_precomp.hpp"
using namespace std;
using namespace cv;
using namespace perf;
using std::tr1::make_tuple;
using std::tr1::get;
typedef std::tr1::tuple<cv::Size, MatType, String> Size_MatType_Str_t;
typedef TestBaseWithParam<Size_MatType_Str_t> Size_Mat_StrType;
#define MAT_SIZES ::perf::sz1080p/*, ::perf::sz4320p*/
#define MAT_TYPES CV_8UC1, CV_32FC1
#define FILE_EXTENSION String(".xml"), String(".yml")
PERF_TEST_P(Size_Mat_StrType, fs_text,
testing::Combine(testing::Values(MAT_SIZES),
testing::Values(MAT_TYPES),
testing::Values(FILE_EXTENSION))
)
{
Size size = get<0>(GetParam());
int type = get<1>(GetParam());
String ext = get<2>(GetParam());
Mat src(size.height, size.width, type);
Mat dst = src.clone();
declare.in(src, WARMUP_RNG).out(dst);
cv::String file_name = cv::tempfile(ext.c_str());
cv::String key = "test_mat";
TEST_CYCLE_MULTIRUN(4)
{
{
FileStorage fs(file_name, cv::FileStorage::WRITE);
fs << key << src;
fs.release();
}
{
FileStorage fs(file_name, cv::FileStorage::READ);
fs[key] >> dst;
fs.release();
}
}
remove(file_name.c_str());
SANITY_CHECK(dst, 1);
}
PERF_TEST_P(Size_Mat_StrType, fs_base64,
testing::Combine(testing::Values(MAT_SIZES),
testing::Values(MAT_TYPES),
testing::Values(FILE_EXTENSION))
)
{
Size size = get<0>(GetParam());
int type = get<1>(GetParam());
String ext = get<2>(GetParam());
Mat src(size.height, size.width, type);
Mat dst = src.clone();
cv::String file_name = cv::tempfile(ext.c_str());
cv::String key = "test_mat";
declare.in(src, WARMUP_RNG).out(dst);
TEST_CYCLE_MULTIRUN(4)
{
{
FileStorage fs(file_name, cv::FileStorage::WRITE_BASE64);
fs << key << src;
fs.release();
}
{
FileStorage fs(file_name, cv::FileStorage::READ);
fs[key] >> dst;
fs.release();
}
}
remove(file_name.c_str());
SANITY_CHECK(dst, 1);
}

705
modules/core/src/persistence.cpp
Unescape View File

@ -243,6 +243,17 @@ typedef struct CvFileStorage
std::deque<char>* outbuf;
base64::Base64Writer * base64_writer;
bool is_default_using_base64;
enum Base64State {
Uncertain,
NotUse,
InUse,
} state_of_writing_base64; /**< used in WriteRawData only */
bool is_write_struct_delayed;
char* delayed_struct_key;
int delayed_struct_flags;
char* delayed_type_name;
bool is_opened;
}
@ -270,21 +281,23 @@ namespace base64
bool base64_valid (uint8_t const * src, size_t off, size_t cnt);
bool base64_valid ( char const * src, size_t off = 0U, size_t cnt = 0U);
size_t base64_encode_buffer_size(size_t cnt);
size_t base64_encode_buffer_size(size_t cnt, bool is_end_with_zero = true);
size_t base64_decode_buffer_size(size_t cnt);
size_t base64_decode_buffer_size(size_t cnt, bool is_end_with_zero = true);
size_t base64_decode_buffer_size(size_t cnt, char const * src, bool is_end_with_zero = true);
size_t base64_decode_buffer_size(size_t cnt, uchar const * src, bool is_end_with_zero = true);
/* binary */
template<typename _uint_t> inline size_t to_binary(_uint_t val, uchar * cur);
template<> inline size_t to_binary(double val, uchar * cur);
template<> inline size_t to_binary(float val, uchar * cur);
template<typename _uint_t> inline size_t to_binary(_uint_t val, uchar * cur);
template<> inline size_t to_binary(double val, uchar * cur);
template<> inline size_t to_binary(float val, uchar * cur);
template<typename _primitive_t> inline size_t to_binary(uchar const * val, uchar * cur);
template<typename _uint_t> inline size_t binary_to(uchar const * cur, _uint_t & val);
template<> inline size_t binary_to(uchar const * cur, double & val);
template<> inline size_t binary_to(uchar const * cur, float & val);
template<typename _primitive_t> inline size_t binary_to(uchar const * cur, uchar * val);
template<typename _uint_t> inline size_t binary_to(uchar const * cur, _uint_t & val);
template<> inline size_t binary_to(uchar const * cur, double & val);
template<> inline size_t binary_to(uchar const * cur, float & val);
template<typename _primitive_t> inline size_t binary_to(uchar const * cur, uchar * val);
class MatToBinaryConvertor;
class RawDataToBinaryConvertor;
@ -313,22 +326,37 @@ namespace base64
class Base64ContextEmitter;
class Base64Writer
{
public:
Base64Writer(::CvFileStorage * fs);
~Base64Writer();
void write(const void* _data, size_t len, const char* dt);
template<typename _to_binary_convertor_t> void write(_to_binary_convertor_t & convertor, const char* dt);
private:
void check_dt(const char* dt);
private:
::CvFileStorage * file_storage;
Base64ContextEmitter * emitter;
std::string data_type_string;
};
/* other */
std::string make_base64_header(int byte_size, const char * dt);
std::string make_base64_header(const char * dt);
bool read_base64_header(std::string const & header, int & byte_size, std::string & dt);
bool read_base64_header(std::vector<char> const & header, std::string & dt);
void make_seq(void * binary_data, int elem_cnt, const char * dt, CvSeq & seq);
/* sample */
void cvStartWriteRawData_Base64(::CvFileStorage * fs, const char* name, int len, const char* dt);
void cvWriteRawData_Base64(::CvFileStorage * fs, const void* _data, int len);
void cvEndWriteRawData_Base64(::CvFileStorage * fs);
void cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt);
void cvWriteRawData_Base64(::cv::FileStorage & fs, const void* _data, int len, const char* dt);
void cvWriteMat_Base64(CvFileStorage * fs, const char * name, ::cv::Mat const & mat);
void cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Mat const & mat);
}
@ -1031,6 +1059,184 @@ static double icv_strtod( CvFileStorage* fs, char* ptr, char** endptr )
return fval;
}
static std::vector<std::string> analyze_file_name( std::string const & file_name )
{
static const char parameter_begin = '?';
static const char parameter_separator = '&';
std::vector<std::string> result;
size_t beg = file_name.find_last_of(parameter_begin);
size_t end = file_name.size();
result.push_back(file_name.substr(0U, beg));
if ( beg != std::string::npos )
{
beg ++;
for ( size_t param_beg = beg, param_end = beg;
param_end < end;
param_beg = param_end + 1U )
{
param_end = file_name.find_first_of( parameter_separator, param_beg );
if ( (param_end == std::string::npos || param_end != param_beg) && param_beg + 1U < end )
{
result.push_back( file_name.substr( param_beg, param_end - param_beg ) );
}
}
}
return result;
}
static bool is_param_exist( std::vector<std::string> & const params, std::string & const param )
{
if ( params.size() < 2U )
return false;
return std::find(params.begin(), params.end(), param) != params.end();
}
static void switch_to_Base64_state( CvFileStorage* fs, CvFileStorage::Base64State state )
{
const char * err_unkonwn_state = "Unexpected error, unable to determine the Base64 state.";
const char * err_unable_to_switch = "Unexpected error, unable to switch to this state.";
/* like a finite state machine */
switch (fs->state_of_writing_base64)
{
case CvFileStorage::Base64State::Uncertain:
switch (state)
{
case CvFileStorage::Base64State::InUse:
CV_DbgAssert( fs->base64_writer == 0 );
fs->base64_writer = new base64::Base64Writer( fs );
break;
case CvFileStorage::Base64State::Uncertain:
break;
case CvFileStorage::Base64State::NotUse:
break;
default:
CV_Error( CV_StsError, err_unkonwn_state );
break;
}
break;
case CvFileStorage::Base64State::InUse:
switch (state)
{
case CvFileStorage::Base64State::InUse:
case CvFileStorage::Base64State::NotUse:
CV_Error( CV_StsError, err_unable_to_switch );
break;
case CvFileStorage::Base64State::Uncertain:
delete fs->base64_writer;
fs->base64_writer = 0;
break;
default:
CV_Error( CV_StsError, err_unkonwn_state );
break;
}
break;
case CvFileStorage::Base64State::NotUse:
switch (state)
{
case CvFileStorage::Base64State::InUse:
case CvFileStorage::Base64State::NotUse:
CV_Error( CV_StsError, err_unable_to_switch );
break;
case CvFileStorage::Base64State::Uncertain:
break;
default:
CV_Error( CV_StsError, err_unkonwn_state );
break;
}
break;
default:
CV_Error( CV_StsError, err_unkonwn_state );
break;
}
fs->state_of_writing_base64 = state;
}
static void check_if_write_struct_is_delayed( CvFileStorage* fs, bool change_type_to_base64 = false )
{
if ( fs->is_write_struct_delayed )
{
/* save data to prevent recursive call errors */
char * struct_key = 0;
char * type_name = 0;
int struct_flags = fs->delayed_struct_flags;
if ( fs->delayed_struct_key != 0 )
{
struct_key = new char[strlen(fs->delayed_struct_key) + 1U];
strcpy(struct_key, fs->delayed_struct_key);
}
if ( fs->delayed_type_name != 0 )
{
type_name = new char[strlen(type_name) + 1U];
strcpy(type_name, fs->delayed_type_name);
}
/* reset */
delete fs->delayed_struct_key;
delete fs->delayed_type_name;
fs->delayed_struct_key = 0;
fs->delayed_struct_flags = 0;
fs->delayed_type_name = 0;
fs->is_write_struct_delayed = false;
/* call */
if ( change_type_to_base64 )
{
fs->start_write_struct( fs, struct_key, struct_flags, "binary");
if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain )
switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain );
switch_to_Base64_state( fs, CvFileStorage::Base64State::InUse );
}
else
{
fs->start_write_struct( fs, struct_key, struct_flags, type_name);
if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain )
switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain );
switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse );
}
delete struct_key;
delete type_name;
}
}
static void make_write_struct_delayed(
CvFileStorage* fs,
const char* key,
int struct_flags,
const char* type_name )
{
CV_Assert( fs->is_write_struct_delayed == false );
CV_DbgAssert( fs->delayed_struct_key == 0 );
CV_DbgAssert( fs->delayed_struct_flags == 0 );
CV_DbgAssert( fs->delayed_type_name == 0 );
fs->delayed_struct_flags = struct_flags;
if ( key != 0 )
{
fs->delayed_struct_key = new char[strlen(key) + 1U];
strcpy(fs->delayed_struct_key, key);
}
if ( type_name != 0 )
{
fs->delayed_type_name = new char[strlen(type_name) + 1U];
strcpy(fs->delayed_type_name, type_name);
}
fs->is_write_struct_delayed = true;
}
/****************************************************************************************\
* YAML Parser *
@ -1119,40 +1325,41 @@ static char* icvYMLParseBase64(CvFileStorage* fs, char* ptr, int indent, CvFileN
/* calc (decoded) total_byte_size from header */
std::string dt;
int total_byte_size = -1;
{
if (end - beg < static_cast<int>(base64::ENCODED_HEADER_SIZE))
CV_PARSE_ERROR("Unrecognized Base64 header");
std::vector<char> header(base64::HEADER_SIZE + 1, ' ');
base64::base64_decode(beg, header.data(), 0U, base64::ENCODED_HEADER_SIZE);
std::istringstream iss(header.data());
if (!(iss >> total_byte_size) || total_byte_size < 0)
CV_PARSE_ERROR("Cannot parse size in Base64 header");
if (!(iss >> dt) || dt.empty())
if ( !base64::read_base64_header(header, dt) || dt.empty() )
CV_PARSE_ERROR("Cannot parse dt in Base64 header");
beg += base64::ENCODED_HEADER_SIZE;
}
/* buffer for decoded data(exclude header) */
std::vector<uchar> buffer(total_byte_size + 4);
/* get all Base64 data */
std::string base64_buffer;
base64_buffer.reserve( 16U * 1024U * 1024U );
while( beg < end )
{
base64::Base64ContextParser parser(buffer.data(), total_byte_size + 4);
/* decoding */
while(beg < end)
{
/* save this part [beg, end) */
parser.read((const uchar *)beg, (const uchar *)end);
beg = end;
base64_buffer.append( beg, end );
beg = end;
icvYMLGetMultilineStringContent( fs, beg, indent, beg, end );
}
if ( !base64::base64_valid(base64_buffer.data(), 0U, base64_buffer.size()) )
CV_PARSE_ERROR( "Invalid Base64 data." );
/* find next part */
icvYMLGetMultilineStringContent(fs, beg, indent, beg, end);
}
/* buffer for decoded data(exclude header) */
std::vector<uchar> binary_buffer( base64::base64_decode_buffer_size(base64_buffer.size()) );
int total_byte_size = base64::base64_decode_buffer_size( base64_buffer.size(), base64_buffer.data(), false );
{
base64::Base64ContextParser parser(binary_buffer.data(), binary_buffer.size() );
const uchar * buffer_beg = reinterpret_cast<const uchar *>( base64_buffer.data() );
const uchar * buffer_end = buffer_beg + base64_buffer.size();
parser.read( buffer_beg, buffer_end );
parser.flush();
}
/* save as CvSeq */
int elem_size = ::icvCalcStructSize(dt.c_str(), 0);
if (total_byte_size % elem_size != 0)
@ -1160,9 +1367,10 @@ static char* icvYMLParseBase64(CvFileStorage* fs, char* ptr, int indent, CvFileN
int elem_cnt = total_byte_size / elem_size;
node->tag = CV_NODE_NONE;
int struct_flags = CV_NODE_FLOW + CV_NODE_SEQ; /* after icvFSCreateCollection, node->tag == struct_flags */
int struct_flags = CV_NODE_FLOW | CV_NODE_SEQ;
/* after icvFSCreateCollection, node->tag == struct_flags */
icvFSCreateCollection(fs, struct_flags, node);
base64::make_seq(buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
if (fs->dummy_eof) {
/* end of file */
@ -1621,6 +1829,16 @@ icvYMLParse( CvFileStorage* fs )
static void
icvYMLWrite( CvFileStorage* fs, const char* key, const char* data )
{
check_if_write_struct_is_delayed( fs );
if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain )
{
switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse );
}
else if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse )
{
CV_Error( CV_StsError, "At present, output Base64 data only." );
}
int i, keylen = 0;
int datalen = 0;
int struct_flags;
@ -2034,17 +2252,17 @@ static void icvXMLGetMultilineStringContent(CvFileStorage* fs,
ptr = icvXMLSkipSpaces(fs, ptr, CV_XML_INSIDE_TAG);
beg = ptr;
end = ptr;
if (fs->dummy_eof)
if ( fs->dummy_eof )
return ; /* end of file */
if (*beg == '<')
if ( *beg == '<' )
return; /* end of string */
/* find end */
while(cv_isprint(*ptr)) /* no check for base64 string */
while( cv_isprint(*ptr) ) /* no check for base64 string */
++ ptr;
if (*ptr == '\0')
CV_PARSE_ERROR("Unexpected end of line");
if ( *ptr == '\0' )
CV_PARSE_ERROR( "Unexpected end of line" );
end = ptr;
}
@ -2061,48 +2279,52 @@ static char* icvXMLParseBase64(CvFileStorage* fs, char* ptr, CvFileNode * node)
/* calc (decoded) total_byte_size from header */
std::string dt;
int total_byte_size = -1;
{
if (end - beg < static_cast<int>(base64::ENCODED_HEADER_SIZE))
CV_PARSE_ERROR("Unrecognized Base64 header");
std::vector<char> header(base64::HEADER_SIZE + 1, ' ');
base64::base64_decode(beg, header.data(), 0U, base64::ENCODED_HEADER_SIZE);
std::istringstream iss(header.data());
if (!(iss >> total_byte_size) || total_byte_size < 0)
CV_PARSE_ERROR("Cannot parse size in Base64 header");
if (!(iss >> dt) || dt.empty())
if ( !base64::read_base64_header(header, dt) || dt.empty() )
CV_PARSE_ERROR("Cannot parse dt in Base64 header");
beg += base64::ENCODED_HEADER_SIZE;
}
/* alloc buffer for all decoded data(include header) */
std::vector<uchar> buffer(total_byte_size + 4);
/* get all Base64 data */
std::string base64_buffer;
base64_buffer.reserve( 16U * 1024U * 1024U );
while( beg < end )
{
base64::Base64ContextParser parser(buffer.data(), total_byte_size + 4);
base64_buffer.append( beg, end );
beg = end;
icvXMLGetMultilineStringContent( fs, beg, beg, end );
}
if ( !base64::base64_valid(base64_buffer.data(), 0U, base64_buffer.size()) )
CV_PARSE_ERROR( "Invalid Base64 data." );
/* decoding */
while(beg < end)
{
/* save this part [beg, end) */
parser.read((const uchar *)beg, (const uchar *)end);
beg = end;
/* find next part */
icvXMLGetMultilineStringContent(fs, beg, beg, end);
}
/* alloc buffer for all decoded data(include header) */
std::vector<uchar> binary_buffer( base64::base64_decode_buffer_size(base64_buffer.size()) );
int total_byte_size = base64::base64_decode_buffer_size( base64_buffer.size(), base64_buffer.data(), false );
{
base64::Base64ContextParser parser(binary_buffer.data(), binary_buffer.size() );
const uchar * buffer_beg = reinterpret_cast<const uchar *>( base64_buffer.data() );
const uchar * buffer_end = buffer_beg + base64_buffer.size();
parser.read( buffer_beg, buffer_end );
parser.flush();
}
/* save as CvSeq */
int elem_size = ::icvCalcStructSize(dt.c_str(), 0);
if (total_byte_size % elem_size != 0)
CV_PARSE_ERROR("Byte size not match elememt size");
CV_PARSE_ERROR("data size not matches elememt size");
int elem_cnt = total_byte_size / elem_size;
node->tag = CV_NODE_NONE;
int struct_flags = CV_NODE_SEQ; /* after icvFSCreateCollection, node->tag == struct_flags */
int struct_flags = CV_NODE_SEQ;
/* after icvFSCreateCollection, node->tag == struct_flags */
icvFSCreateCollection(fs, struct_flags, node);
base64::make_seq(buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
if (fs->dummy_eof) {
/* end of file */
@ -2757,6 +2979,16 @@ icvXMLStartNextStream( CvFileStorage* fs )
static void
icvXMLWriteScalar( CvFileStorage* fs, const char* key, const char* data, int len )
{
check_if_write_struct_is_delayed( fs );
if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain )
{
switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse );
}
else if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse )
{
CV_Error( CV_StsError, "Currently only Base64 data is allowed." );
}
if( CV_NODE_IS_MAP(fs->struct_flags) ||
(!CV_NODE_IS_COLLECTION(fs->struct_flags) && key) )
{
@ -2964,15 +3196,24 @@ icvXMLWriteComment( CvFileStorage* fs, const char* comment, int eol_comment )
\****************************************************************************************/
CV_IMPL CvFileStorage*
cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, const char* encoding )
cvOpenFileStorage( const char* query, CvMemStorage* dststorage, int flags, const char* encoding )
{
CvFileStorage* fs = 0;
int default_block_size = 1 << 18;
bool append = (flags & 3) == CV_STORAGE_APPEND;
bool mem = (flags & CV_STORAGE_MEMORY) != 0;
bool write_mode = (flags & 3) != 0;
bool write_base64 = write_mode && (flags & CV_STORAGE_BASE64) != 0;
bool isGZ = false;
size_t fnamelen = 0;
const char * filename = 0;
std::vector<std::string> params = analyze_file_name( query );
if ( !params.empty() )
filename = params.begin()->c_str();
if (write_base64 == false && is_param_exist( params, std::string("base64") ) )
write_base64 = true;
if( !filename || filename[0] == '\0' )
{
@ -3073,6 +3314,16 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
fs->struct_flags = CV_NODE_EMPTY;
fs->buffer_start = fs->buffer = (char*)cvAlloc( buf_size + 1024 );
fs->buffer_end = fs->buffer_start + buf_size;
fs->base64_writer = 0;
fs->is_default_using_base64 = write_base64;
fs->state_of_writing_base64 = CvFileStorage::Base64State::Uncertain;
fs->is_write_struct_delayed = false;
fs->delayed_struct_key = 0;
fs->delayed_struct_flags = 0;
fs->delayed_type_name = 0;
if( fs->fmt == CV_STORAGE_FORMAT_XML )
{
size_t file_size = fs->file ? (size_t)ftell( fs->file ) : (size_t)0;
@ -3247,7 +3498,48 @@ cvStartWriteStruct( CvFileStorage* fs, const char* key, int struct_flags,
const char* type_name, CvAttrList /*attributes*/ )
{
CV_CHECK_OUTPUT_FILE_STORAGE(fs);
fs->start_write_struct( fs, key, struct_flags, type_name );
check_if_write_struct_is_delayed( fs );
if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::NotUse )
switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain );
if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain
&&
CV_NODE_IS_SEQ(struct_flags)
&&
fs->is_default_using_base64
&&
type_name == 0
)
{
/* Uncertain if output Base64 data */
make_write_struct_delayed( fs, key, struct_flags, type_name );
}
else if ( type_name && memcmp(type_name, "binary", 6) == 0 )
{
/* Must output Base64 data */
if ( !CV_NODE_IS_SEQ(struct_flags) )
CV_Error( CV_StsBadArg, "must set 'struct_flags |= CV_NODE_SEQ' if using Base64.");
else if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain )
CV_Error( CV_StsError, "function \'cvStartWriteStruct\' calls cannot be nested if using Base64.");
fs->start_write_struct( fs, key, struct_flags, type_name );
if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain )
switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain );
switch_to_Base64_state( fs, CvFileStorage::Base64State::InUse );
}
else
{
/* Won't output Base64 data */
if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse )
CV_Error( CV_StsError, "At the end of the output Base64, `cvEndWriteStruct` is needed.");
fs->start_write_struct( fs, key, struct_flags, type_name );
if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain )
switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain );
switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse );
}
}
@ -3255,6 +3547,11 @@ CV_IMPL void
cvEndWriteStruct( CvFileStorage* fs )
{
CV_CHECK_OUTPUT_FILE_STORAGE(fs);
check_if_write_struct_is_delayed( fs );
if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain )
switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain );
fs->end_write_struct( fs );
}
@ -3432,6 +3729,17 @@ icvDecodeSimpleFormat( const char* dt )
CV_IMPL void
cvWriteRawData( CvFileStorage* fs, const void* _data, int len, const char* dt )
{
if (fs->is_default_using_base64 ||
fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse )
{
base64::cvWriteRawDataBase64( fs, _data, len, dt );
return;
}
else if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain )
{
switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse );
}
const char* data0 = (const char*)_data;
int offset = 0;
int fmt_pairs[CV_FS_MAX_FMT_PAIRS*2], k, fmt_pair_count;
@ -3760,15 +4068,15 @@ icvWriteFileNode( CvFileStorage* fs, const char* name, const CvFileNode* node )
break;
case CV_NODE_SEQ:
case CV_NODE_MAP:
fs->start_write_struct( fs, name, CV_NODE_TYPE(node->tag) +
cvStartWriteStruct( fs, name, CV_NODE_TYPE(node->tag) +
(CV_NODE_SEQ_IS_SIMPLE(node->data.seq) ? CV_NODE_FLOW : 0),
node->info ? node->info->type_name : 0 );
icvWriteCollection( fs, node );
fs->end_write_struct( fs );
cvEndWriteStruct( fs );
break;
case CV_NODE_NONE:
fs->start_write_struct( fs, name, CV_NODE_SEQ, 0 );
fs->end_write_struct( fs );
cvStartWriteStruct( fs, name, CV_NODE_SEQ, 0 );
cvEndWriteStruct( fs );
break;
default:
CV_Error( CV_StsBadFlag, "Unknown type of file node" );
@ -6226,14 +6534,29 @@ bool base64::base64_valid(char const * src, size_t off, size_t cnt)
return base64_valid(reinterpret_cast<uint8_t const *>(src), off, cnt);
}
size_t base64::base64_encode_buffer_size(size_t cnt)
size_t base64::base64_encode_buffer_size(size_t cnt, bool is_end_with_zero)
{
size_t additional = static_cast<size_t>(is_end_with_zero == true);
return (cnt + 2U) / 3U * 4U + additional;
}
size_t base64::base64_decode_buffer_size(size_t cnt, bool is_end_with_zero)
{
size_t additional = static_cast<size_t>(is_end_with_zero == true);
return cnt / 4U * 3U + additional;
}
size_t base64::base64_decode_buffer_size(size_t cnt, char const * src, bool is_end_with_zero)
{
return size_t((cnt + 2U) / 3U * 4U + 1U);
return base64_decode_buffer_size(cnt, reinterpret_cast<uchar const *>(src), is_end_with_zero);
}
size_t base64::base64_decode_buffer_size(size_t cnt)
size_t base64::base64_decode_buffer_size(size_t cnt, uchar const * src, bool is_end_with_zero)
{
return size_t(cnt / 4U * 3U + 1U);
size_t padding_cnt = 0U;
for (uchar const * ptr = src + cnt - 1U; *ptr == base64_padding; ptr--)
padding_cnt ++;
return base64_decode_buffer_size(cnt, is_end_with_zero) - padding_cnt;
}
/****************************************************************************
@ -6308,13 +6631,10 @@ binary_to(uchar const * cur, uchar * val)
* others
***************************************************************************/
std::string base64::make_base64_header(int byte_size, const char * dt)
std::string base64::make_base64_header(const char * dt)
{
int size = byte_size;
std::ostringstream oss;
oss << size << ' '
<< dt << ' ';
oss << dt << ' ';
std::string buffer(oss.str());
CV_Assert(buffer.size() < HEADER_SIZE);
@ -6325,10 +6645,10 @@ std::string base64::make_base64_header(int byte_size, const char * dt)
return buffer;
}
bool base64::read_base64_header(std::string const & header, int & byte_size, std::string & dt)
bool base64::read_base64_header(std::vector<char> const & header, std::string & dt)
{
std::istringstream iss(header);
return static_cast<bool>(iss >> byte_size >> dt);
std::istringstream iss(header.data());
return static_cast<bool>(iss >> dt);
}
/****************************************************************************
@ -6351,10 +6671,9 @@ base64::Base64ContextParser::Base64ContextParser(uchar * buffer, size_t size)
base64::Base64ContextParser::~Base64ContextParser()
{
if (src_cur != src_beg) {
/* encode the rest binary data to base64 buffer */
/* encode the rest binary data to base64 buffer */
if (src_cur != src_beg)
flush();
}
}
base64::Base64ContextParser & base64::Base64ContextParser::
@ -6383,9 +6702,12 @@ read(const uchar * beg, const uchar * end)
bool base64::Base64ContextParser::flush()
{
if (!base64_valid(src_beg, 0U, src_cur - src_beg))
if ( !base64_valid(src_beg, 0U, src_cur - src_beg) )
return false;
if ( src_cur == src_beg )
return true;
uchar * buffer = binary_buffer.data();
size_t len = base64_decode(src_beg, buffer, 0U, src_cur - src_beg);
src_cur = src_beg;
@ -6514,7 +6836,7 @@ public:
private:
/* because of Base64, we must keep its length a multiple of 3 */
static const size_t BUFFER_LEN = 51U;
static const size_t BUFFER_LEN = 48U;
// static_assert(BUFFER_LEN % 3 == 0, "BUFFER_LEN is invalid");
private:
@ -6584,17 +6906,17 @@ public:
{
CV_DbgAssert(*this);
/* copy to dst */
/* [1]copy to dst */
dst += to_binary_func(row_begin + x, dst);
/* move to next */
/* [2]move to next */
x += step;
if (x >= x_max) {
/* when x arrive end, reset it and increase y */
/* when x arrive end, reset x and increase y */
x = 0U;
++ y;
if (y >= y_max) {
/* when y arrive end, reset it and increase iter */
/* when y arrive end, reset y and increase iter */
y = 0U;
++ mat_iter;
if (mat_iter == mats.end()) {
@ -6935,68 +7257,55 @@ private:
* Wapper
***************************************************************************/
class base64::Base64Writer
base64::Base64Writer::Base64Writer(::CvFileStorage * fs)
: file_storage(fs)
, emitter(new Base64ContextEmitter(fs))
, data_type_string()
{
public:
CV_CHECK_OUTPUT_FILE_STORAGE(fs);
icvFSFlush(fs);
}
Base64Writer(::CvFileStorage * fs, const char * name, int len, const char* dt)
: file_storage(fs)
, emitter(fs)
, remaining_data_length(len)
, data_type_string(dt)
{
CV_CHECK_OUTPUT_FILE_STORAGE(fs);
void base64::Base64Writer::write(const void* _data, size_t len, const char* dt)
{
check_dt(dt);
cvStartWriteStruct(fs, name, CV_NODE_SEQ, "binary");
icvFSFlush(fs);
RawDataToBinaryConvertor convertor(
_data, static_cast<int>(len), data_type_string.c_str()
);
emitter->write(convertor);
}
/* output header */
template<typename _to_binary_convertor_t> inline
void base64::Base64Writer::write(_to_binary_convertor_t & convertor, const char* dt)
{
check_dt(dt);
emitter->write(convertor);
}
base64::Base64Writer::~Base64Writer()
{
delete emitter;
}
/* total byte size(before encode) */
int size = len * ::icvCalcStructSize(dt, 0);
void base64::Base64Writer::check_dt(const char* dt)
{
if ( dt == 0 )
CV_Error( CV_StsBadArg, "Invalid \'dt\'." );
else if (data_type_string.empty()) {
data_type_string = dt;
std::string buffer = make_base64_header(size, dt);
/* output header */
std::string buffer = make_base64_header(dt);
const uchar * beg = reinterpret_cast<const uchar *>(buffer.data());
const uchar * end = beg + buffer.size();
emitter.write(beg, end);
}
void write(const void* _data, int len)
{
CV_Assert(len >= 0);
CV_Assert(remaining_data_length >= static_cast<size_t>(len));
remaining_data_length -= static_cast<size_t>(len);
RawDataToBinaryConvertor convertor(_data, len, data_type_string);
emitter.write(convertor);
}
template<typename _to_binary_convertor_t> inline
void write(_to_binary_convertor_t & convertor, int data_length_of_convertor)
{
CV_Assert(data_length_of_convertor >= 0);
CV_Assert(remaining_data_length >= static_cast<size_t>(data_length_of_convertor));
remaining_data_length -= static_cast<size_t>(data_length_of_convertor);
emitter.write(convertor);
}
~Base64Writer()
{
CV_Assert(remaining_data_length == 0U);
emitter.flush();
cvEndWriteStruct(file_storage);
icvFSFlush(file_storage);
}
emitter->write(beg, end);
} else if ( data_type_string != dt )
CV_Error( CV_StsBadArg, "\'dt\' does not match." );
}
private:
::CvFileStorage * file_storage;
Base64ContextEmitter emitter;
size_t remaining_data_length;
const char* data_type_string;
};
void base64::make_seq(void * binary, int elem_cnt, const char * dt, ::CvSeq & seq)
{
@ -7009,51 +7318,23 @@ void base64::make_seq(void * binary, int elem_cnt, const char * dt, ::CvSeq & se
}
}
void base64::cvStartWriteRawData_Base64(::CvFileStorage * fs, const char* name, int len, const char* dt)
void base64::cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt)
{
CV_Assert(fs);
CV_CHECK_OUTPUT_FILE_STORAGE(fs);
CV_Assert(fs->base64_writer == 0);
fs->base64_writer = new Base64Writer(fs, name, len, dt);
}
void base64::cvWriteRawData_Base64(::CvFileStorage * fs, const void* _data, int len)
{
CV_Assert(fs);
CV_CHECK_OUTPUT_FILE_STORAGE(fs);
CV_Assert(fs->base64_writer != 0);
fs->base64_writer->write(_data, len);
}
check_if_write_struct_is_delayed( fs, true );
void base64::cvEndWriteRawData_Base64(::CvFileStorage * fs)
{
CV_Assert(fs);
CV_CHECK_OUTPUT_FILE_STORAGE(fs);
CV_Assert(fs->base64_writer != 0);
delete fs->base64_writer;
fs->base64_writer = 0;
}
void base64::cvWriteRawData_Base64(::cv::FileStorage & fs, const void* _data, int len, const char* dt)
{
cvStartWriteStruct(*fs, fs.elname.c_str(), CV_NODE_SEQ, "binary");
if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain )
{
Base64ContextEmitter emitter(*fs);
{ /* header */
/* total byte size(before encode) */
int size = len * ::icvCalcStructSize(dt, 0);
std::string buffer = make_base64_header(size, dt);
const uchar * beg = reinterpret_cast<const uchar *>(buffer.data());
const uchar * end = beg + buffer.size();
emitter.write(beg, end);
}
{ /* body */
RawDataToBinaryConvertor convert(_data, len, dt);
emitter.write(convert);
}
switch_to_Base64_state( fs, CvFileStorage::Base64State::InUse );
}
else if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::InUse )
{
CV_Error( CV_StsError, "Base64 should not be used at present." );
}
cvEndWriteStruct(*fs);
fs->base64_writer->write(_data, len, dt);
}
void base64::cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Mat const & mat)
@ -7064,31 +7345,30 @@ void base64::cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Ma
{ /* [1]output other attr */
if (mat.dims <= 2) {
cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MAT);
::cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MAT);
cvWriteInt(fs, "rows", mat.rows );
cvWriteInt(fs, "cols", mat.cols );
::cvWriteInt(fs, "rows", mat.rows );
::cvWriteInt(fs, "cols", mat.cols );
} else {
cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MATND);
::cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MATND);
cvStartWriteStruct(fs, "sizes", CV_NODE_SEQ | CV_NODE_FLOW);
cvWriteRawData(fs, mat.size.p, mat.dims, "i");
cvEndWriteStruct(fs);
::cvStartWriteStruct(fs, "sizes", CV_NODE_SEQ | CV_NODE_FLOW);
::cvWriteRawData(fs, mat.size.p, mat.dims, "i");
::cvEndWriteStruct(fs);
}
cvWriteString(fs, "dt", ::icvEncodeFormat(CV_MAT_TYPE(mat.type()), dt ), 0 );
::cvWriteString(fs, "dt", ::icvEncodeFormat(CV_MAT_TYPE(mat.type()), dt ), 0 );
}
{ /* [2]deal with matrix's data */
int len = static_cast<int>(mat.total());
MatToBinaryConvertor convertor(mat);
cvStartWriteRawData_Base64(fs, "data", len, dt);
fs->base64_writer->write(convertor, len);
cvEndWriteRawData_Base64(fs);
::cvStartWriteStruct(fs, "data", CV_NODE_SEQ, "binary");
fs->base64_writer->write(convertor, dt);
::cvEndWriteStruct(fs);
}
{ /* [3]output end */
cvEndWriteStruct(fs);
::cvEndWriteStruct(fs);
}
}
@ -7096,36 +7376,21 @@ void base64::cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Ma
* Interface
***************************************************************************/
namespace cv
CV_IMPL void cvWriteMatBase64(::CvFileStorage* fs, const char* name, const ::CvMat* mat)
{
void cvWriteMat_Base64(::CvFileStorage* fs, const char* name, const ::CvMat* mat)
{
::cv::Mat holder = ::cv::cvarrToMat(mat);
::base64::cvWriteMat_Base64(fs, name, holder);
}
void cvWriteMatND_Base64(::CvFileStorage* fs, const char* name, const ::CvMatND* mat)
{
::cv::Mat holder = ::cv::cvarrToMat(mat);
::base64::cvWriteMat_Base64(fs, name, holder);
}
void cvStartWriteRawData_Base64(::CvFileStorage * fs, const char* name, int len, const char* dt)
{
::base64::cvStartWriteRawData_Base64(fs, name, len, dt);
}
void cvWriteRawData_Base64(::CvFileStorage * fs, const void* _data, int len)
{
::base64::cvWriteRawData_Base64(fs, _data, len);
}
void cvEndWriteRawData_Base64(::CvFileStorage * fs)
{
::base64::cvEndWriteRawData_Base64(fs);
}
::cv::Mat holder = ::cv::cvarrToMat(mat);
::base64::cvWriteMat_Base64(fs, name, holder);
}
CV_IMPL void cvWriteMatNDBase64(::CvFileStorage* fs, const char* name, const ::CvMatND* mat)
{
::cv::Mat holder = ::cv::cvarrToMat(mat);
::base64::cvWriteMat_Base64(fs, name, holder);
}
CV_IMPL void cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt)
{
::base64::cvWriteRawDataBase64(fs, _data, len, dt);
}
/* End of file. */

147
modules/core/test/test_io.cpp
Unescape View File

@ -593,150 +593,3 @@ TEST(Core_InputOutput, FileStorageSpaces)
ASSERT_STREQ(values[i].c_str(), valuesRead[i].c_str());
}
}
TEST(Core_InputOutput, filestorage_yml_compatibility)
{
// TODO:
}
class CV_Base64IOTest : public cvtest::BaseTest
{
private:
std::string file_name;
struct data_t
{
uchar u1, u2;
int i1, i2, i3;
double d1, d2;
int i4;
};
public:
CV_Base64IOTest(std::string const & test_file_name)
: file_name(test_file_name) {}
~CV_Base64IOTest() {}
protected:
void run(int)
{
try
{
std::vector<data_t> rawdata;
cv::Mat _em_out, _em_in;
cv::Mat _2d_out, _2d_in;
cv::Mat _nd_out, _nd_in;
{ /* init */
/* normal mat */
_2d_out = cv::Mat(100, 100, CV_8UC3, cvScalar(1U, 2U, 127U));
for (int i = 0; i < _2d_out.rows; ++i)
for (int j = 0; j < _2d_out.cols; ++j)
_2d_out.at<cv::Vec3b>(i, j)[1] = (i + j) % 256;
/* 4d mat */
const int Size[] = {4, 4, 4, 4};
cv::Mat _4d(4, Size, CV_64FC4, cvScalar(0.888, 0.111, 0.666, 0.444));
const cv::Range ranges[] = {
cv::Range(0, 2),
cv::Range(0, 2),
cv::Range(1, 2),
cv::Range(0, 2) };
_nd_out = _4d(ranges);
/* raw data */
for (int i = 0; i < 1000; i++) {
data_t tmp;
tmp.u1 = 1;
tmp.u2 = 2;
tmp.i1 = 1;
tmp.i2 = 2;
tmp.i3 = 3;
tmp.d1 = 0.1;
tmp.d2 = 0.2;
tmp.i4 = i;
rawdata.push_back(tmp);
}
}
{ /* write */
cv::FileStorage fs(file_name, cv::FileStorage::WRITE);
CvMat holder = _2d_out;
cv::cvWriteMat_Base64(*fs, "normal_2d_mat", &holder);
CvMatND holder_nd = _nd_out;
cv::cvWriteMatND_Base64(*fs, "normal_nd_mat", &holder_nd);
holder = _em_out;
cv::cvWriteMat_Base64(*fs, "empty_2d_mat", &holder);
cv::cvStartWriteRawData_Base64(*fs, "rawdata", static_cast<int>(rawdata.size()), "2u3i2di");
for (int i = 0; i < 10; i++)
cv::cvWriteRawData_Base64(*fs, rawdata.data() + i * 100, 100);
cv::cvEndWriteRawData_Base64(*fs);
fs.release();
}
{ /* read */
cv::FileStorage fs(file_name, cv::FileStorage::READ);
/* mat */
fs["empty_2d_mat"] >> _em_in;
fs["normal_2d_mat"] >> _2d_in;
fs["normal_nd_mat"] >> _nd_in;
/* raw data */
std::vector<data_t>(1000).swap(rawdata);
cvReadRawData(*fs, fs["rawdata"].node, rawdata.data(), "2u3i2di");
fs.release();
}
for (int i = 0; i < 1000; i++) {
// TODO: Solve this bug in `cvReadRawData`
//EXPECT_EQ(rawdata[i].u1, 1);
//EXPECT_EQ(rawdata[i].u2, 2);
//EXPECT_EQ(rawdata[i].i1, 1);
//EXPECT_EQ(rawdata[i].i2, 2);
//EXPECT_EQ(rawdata[i].i3, 3);
//EXPECT_EQ(rawdata[i].d1, 0.1);
//EXPECT_EQ(rawdata[i].d2, 0.2);
//EXPECT_EQ(rawdata[i].i4, i);
}
EXPECT_EQ(_em_in.rows , _em_out.rows);
EXPECT_EQ(_em_in.cols , _em_out.cols);
EXPECT_EQ(_em_in.dims , _em_out.dims);
EXPECT_EQ(_em_in.depth(), _em_out.depth());
EXPECT_TRUE(_em_in.empty());
EXPECT_EQ(_2d_in.rows , _2d_in.rows);
EXPECT_EQ(_2d_in.cols , _2d_in.cols);
EXPECT_EQ(_2d_in.dims , _2d_in.dims);
EXPECT_EQ(_2d_in.depth(), _2d_in.depth());
for(int i = 0; i < _2d_in.rows; ++i)
for (int j = 0; j < _2d_in.cols; ++j)
EXPECT_EQ(_2d_in.at<cv::Vec3b>(i, j), _2d_out.at<cv::Vec3b>(i, j));
EXPECT_EQ(_nd_in.rows , _nd_in.rows);
EXPECT_EQ(_nd_in.cols , _nd_in.cols);
EXPECT_EQ(_nd_in.dims , _nd_in.dims);
EXPECT_EQ(_nd_in.depth(), _nd_in.depth());
EXPECT_EQ(cv::countNonZero(cv::mean(_nd_in != _nd_out)), 0);
}
catch(...)
{
ts->set_failed_test_info(cvtest::TS::FAIL_MISMATCH);
}
}
};
TEST(Core_InputOutput, filestorage_yml_base64)
{
CV_Base64IOTest test("base64_test_tmp_file.yml"); test.safe_run();
}
TEST(Core_InputOutput, filestorage_xml_base64)
{
CV_Base64IOTest test("base64_test_tmp_file.xml"); test.safe_run();
}

270
modules/core/test/test_io_base64.cpp
Unescape View File

@ -0,0 +1,270 @@
#include "test_precomp.hpp"
using namespace cv;
using namespace std;
struct data_t
{
typedef uchar u;
typedef char b;
typedef ushort w;
typedef short s;
typedef int i;
typedef float f;
typedef double d;
u u1 ;u u2 ; i i1 ;
i i2 ;i i3 ;
d d1 ;
d d2 ;
i i4 ;
static inline const char * signature() { return "2u3i2di"; }
};
TEST(Core_InputOutput_Base64, basic)
{
char const * filenames[] = {
"core_io_base64_basic_test.yml",
"core_io_base64_basic_test.xml",
0
};
for (char const ** ptr = filenames; *ptr; ptr++)
{
char const * name = *ptr;
std::vector<data_t> rawdata;
cv::Mat _em_out, _em_in;
cv::Mat _2d_out, _2d_in;
cv::Mat _nd_out, _nd_in;
cv::Mat _rd_out(64, 64, CV_64FC1), _rd_in;
{ /* init */
/* a normal mat */
_2d_out = cv::Mat(100, 100, CV_8UC3, cvScalar(1U, 2U, 127U));
for (int i = 0; i < _2d_out.rows; ++i)
for (int j = 0; j < _2d_out.cols; ++j)
_2d_out.at<cv::Vec3b>(i, j)[1] = (i + j) % 256;
/* a 4d mat */
const int Size[] = {4, 4, 4, 4};
cv::Mat _4d(4, Size, CV_64FC4, cvScalar(0.888, 0.111, 0.666, 0.444));
const cv::Range ranges[] = {
cv::Range(0, 2),
cv::Range(0, 2),
cv::Range(1, 2),
cv::Range(0, 2) };
_nd_out = _4d(ranges);
/* a random mat */
cv::randu(_rd_out, cv::Scalar(0.0), cv::Scalar(1.0));
/* raw data */
for (int i = 0; i < 1000; i++) {
data_t tmp;
tmp.u1 = 1;
tmp.u2 = 2;
tmp.i1 = 1;
tmp.i2 = 2;
tmp.i3 = 3;
tmp.d1 = 0.1;
tmp.d2 = 0.2;
tmp.i4 = i;
rawdata.push_back(tmp);
}
}
{ /* write */
cv::FileStorage fs(name, cv::FileStorage::WRITE_BASE64);
fs << "normal_2d_mat" << _2d_out;
fs << "normal_nd_mat" << _nd_out;
fs << "empty_2d_mat" << _em_out;
fs << "random_mat" << _rd_out;
cvStartWriteStruct( *fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW, "binary" );
for (int i = 0; i < 10; i++)
cvWriteRawDataBase64(*fs, rawdata.data() + i * 100, 100, data_t::signature());
cvEndWriteStruct( *fs );
fs.release();
}
{ /* read */
cv::FileStorage fs(name, cv::FileStorage::READ);
/* mat */
fs["empty_2d_mat"] >> _em_in;
fs["normal_2d_mat"] >> _2d_in;
fs["normal_nd_mat"] >> _nd_in;
fs["random_mat"] >> _rd_in;
/* raw data */
std::vector<data_t>(1000).swap(rawdata);
cvReadRawData(*fs, fs["rawdata"].node, rawdata.data(), data_t::signature());
fs.release();
}
for (int i = 0; i < 1000; i++) {
// TODO: Solve this bug in `cvReadRawData`
//EXPECT_EQ(rawdata[i].u1, 1);
//EXPECT_EQ(rawdata[i].u2, 2);
//EXPECT_EQ(rawdata[i].i1, 1);
//EXPECT_EQ(rawdata[i].i2, 2);
//EXPECT_EQ(rawdata[i].i3, 3);
//EXPECT_EQ(rawdata[i].d1, 0.1);
//EXPECT_EQ(rawdata[i].d2, 0.2);
//EXPECT_EQ(rawdata[i].i4, i);
}
EXPECT_EQ(_em_in.rows , _em_out.rows);
EXPECT_EQ(_em_in.cols , _em_out.cols);
EXPECT_EQ(_em_in.dims , _em_out.dims);
EXPECT_EQ(_em_in.depth(), _em_out.depth());
EXPECT_TRUE(_em_in.empty());
EXPECT_EQ(_2d_in.rows , _2d_out.rows);
EXPECT_EQ(_2d_in.cols , _2d_out.cols);
EXPECT_EQ(_2d_in.dims , _2d_out.dims);
EXPECT_EQ(_2d_in.depth(), _2d_out.depth());
for(int i = 0; i < _2d_out.rows; ++i)
for (int j = 0; j < _2d_out.cols; ++j)
EXPECT_EQ(_2d_in.at<cv::Vec3b>(i, j), _2d_out.at<cv::Vec3b>(i, j));
EXPECT_EQ(_nd_in.rows , _nd_out.rows);
EXPECT_EQ(_nd_in.cols , _nd_out.cols);
EXPECT_EQ(_nd_in.dims , _nd_out.dims);
EXPECT_EQ(_nd_in.depth(), _nd_out.depth());
EXPECT_EQ(cv::countNonZero(cv::mean(_nd_in != _nd_out)), 0);
EXPECT_EQ(_rd_in.rows , _rd_out.rows);
EXPECT_EQ(_rd_in.cols , _rd_out.cols);
EXPECT_EQ(_rd_in.dims , _rd_out.dims);
EXPECT_EQ(_rd_in.depth(), _rd_out.depth());
EXPECT_EQ(cv::countNonZero(cv::mean(_rd_in != _rd_out)), 0);
remove(name);
}
}
TEST(Core_InputOutput_Base64, valid)
{
char const * filenames[] = {
"core_io_base64_other_test.yml",
"core_io_base64_other_test.xml",
"core_io_base64_other_test.yml?base64",
"core_io_base64_other_test.xml?base64",
0
};
char const * real_name[] = {
"core_io_base64_other_test.yml",
"core_io_base64_other_test.xml",
"core_io_base64_other_test.yml",
"core_io_base64_other_test.xml",
0
};
std::vector<int> rawdata(10, static_cast<int>(0x00010203));
cv::String str_out = "test_string";
for (char const ** ptr = filenames; *ptr; ptr++)
{
char const * name = *ptr;
EXPECT_NO_THROW(
{
cv::FileStorage fs(name, cv::FileStorage::WRITE_BASE64);
cvStartWriteStruct(*fs, "manydata", CV_NODE_SEQ);
cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW);
for (int i = 0; i < 10; i++)
cvWriteRawData(*fs, rawdata.data(), rawdata.size(), "i");
cvEndWriteStruct(*fs);
cvWriteString(*fs, 0, str_out.c_str(), 1);
cvEndWriteStruct(*fs);
fs.release();
});
{
cv::FileStorage fs(name, cv::FileStorage::READ);
std::vector<int> data_in(rawdata.size());
fs["manydata"][0].readRaw("i", (uchar *)data_in.data(), data_in.size());
EXPECT_TRUE(fs["manydata"][0].isSeq());
EXPECT_TRUE(std::equal(rawdata.begin(), rawdata.end(), data_in.begin()));
cv::String str_in;
fs["manydata"][1] >> str_in;
EXPECT_TRUE(fs["manydata"][1].isString());
EXPECT_EQ(str_in, str_out);
fs.release();
}
EXPECT_NO_THROW(
{
cv::FileStorage fs(name, cv::FileStorage::WRITE);
cvStartWriteStruct(*fs, "manydata", CV_NODE_SEQ);
cvWriteString(*fs, 0, str_out.c_str(), 1);
cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW, "binary");
for (int i = 0; i < 10; i++)
cvWriteRawData(*fs, rawdata.data(), rawdata.size(), "i");
cvEndWriteStruct(*fs);
cvEndWriteStruct(*fs);
fs.release();
});
{
cv::FileStorage fs(name, cv::FileStorage::READ);
cv::String str_in;
fs["manydata"][0] >> str_in;
EXPECT_TRUE(fs["manydata"][0].isString());
EXPECT_EQ(str_in, str_out);
std::vector<int> data_in(rawdata.size());
fs["manydata"][1].readRaw("i", (uchar *)data_in.data(), data_in.size());
EXPECT_TRUE(fs["manydata"][1].isSeq());
EXPECT_TRUE(std::equal(rawdata.begin(), rawdata.end(), data_in.begin()));
fs.release();
}
remove(real_name[ptr - filenames]);
}
}
TEST(Core_InputOutput_Base64, invalid)
{
char const * filenames[] = {
"core_io_base64_other_test.yml",
"core_io_base64_other_test.xml",
0
};
for (char const ** ptr = filenames; *ptr; ptr++)
{
char const * name = *ptr;
EXPECT_ANY_THROW({
cv::FileStorage fs(name, cv::FileStorage::WRITE);
cvStartWriteStruct(*fs, "rawdata", CV_NODE_SEQ, "binary");
cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW);
});
EXPECT_ANY_THROW({
cv::FileStorage fs(name, cv::FileStorage::WRITE);
cvStartWriteStruct(*fs, "rawdata", CV_NODE_SEQ);
cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW);
cvWriteRawDataBase64(*fs, name, 1, "u");
});
remove(name);
}
}
TEST(Core_InputOutput_Base64, TODO_compatibility)
{
// TODO:
}

6
modules/ml/test/test_save_load.cpp
Unescape View File

@ -65,11 +65,11 @@ int CV_SLMLTest::run_test_case( int testCaseIdx )
{
get_test_error( testCaseIdx, &test_resps1 );
fname1 = tempfile(".yml.gz");
save( fname1.c_str() );
save( (fname1 + "?base64").c_str() );
load( fname1.c_str() );
get_test_error( testCaseIdx, &test_resps2 );
fname2 = tempfile(".yml.gz");
save( fname2.c_str() );
save( (fname2 + "?base64").c_str() );
}
else
ts->printf( cvtest::TS::LOG, "model can not be trained" );
@ -280,7 +280,7 @@ TEST(DISABLED_ML_SVM, linear_save_load)
svm1 = Algorithm::load<SVM>("SVM45_X_38-1.xml");
svm2 = Algorithm::load<SVM>("SVM45_X_38-2.xml");
string tname = tempfile("a.xml");
svm2->save(tname);
svm2->save(tname + "?base64");
svm3 = Algorithm::load<SVM>(tname);
ASSERT_EQ(svm1->getVarCount(), svm2->getVarCount());

Write Preview
Loading…
Cancel
Save