Chiebot-Mirror
/
opencv
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

backported API for storing OpenCV data structures to text string and reading them back (r8481, r8516, r8518, r8522)

Browse Source
pull/13383/head
Andrey Kamaev 13 years ago
parent 4e4b96e3d9
commit 9ad470ba5c
6 changed files with 248 additions and 99 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 15
      modules/core/doc/xml_yaml_persistence.rst
  2. 11
      modules/core/include/opencv2/core/core.hpp
  3. 5
      modules/core/include/opencv2/core/types_c.h
  4. 284
      modules/core/src/persistence.cpp
  5. 15
      modules/core/test/test_io.cpp
  6. 17
      modules/core/test/test_misc.cpp

15
modules/core/doc/xml_yaml_persistence.rst
Unescape View File

@ -156,9 +156,9 @@ The constructors.
.. ocv:function:: FileStorage::FileStorage()
.. ocv:function:: FileStorage::FileStorage(const string& filename, int flags, const string& encoding=string())
.. ocv:function:: FileStorage::FileStorage(const string& source, int flags, const string& encoding=string())
:param filename: Name of the file to open. Extension of the file (``.xml`` or ``.yml``/``.yaml``) determines its format (XML or YAML respectively). Also you can append ``.gz`` to work with compressed files, for example ``myHugeMatrix.xml.gz``.
:param source: Name of the file to open or the text string to read the data from. Extension of the file (``.xml`` or ``.yml``/``.yaml``) determines its format (XML or YAML respectively). Also you can append ``.gz`` to work with compressed files, for example ``myHugeMatrix.xml.gz``. If both ``FileStorage::WRITE`` and ``FileStorage::MEMORY`` flags are specified, ``source`` is used just to specify the output file format (e.g. ``mydata.xml``, ``.yml`` etc.).
:param flags: Mode of operation. Possible values are:
@ -167,6 +167,8 @@ The constructors.
* **FileStorage::WRITE** Open the file for writing.
* **FileStorage::APPEND** Open the file for appending.
* **FileStorage::MEMORY** Read data from ``source`` or write data to the internal buffer (which is returned by ``FileStorage::release``)
:param encoding: Encoding of the file. Note that UTF-16 XML encoding is not supported currently and you should use 8-bit encoding instead of it.
@ -202,6 +204,15 @@ Closes the file and releases all the memory buffers.
Call this method after all I/O operations with the storage are finished.
FileStorage::releaseAndGetString
--------------------------------
Closes the file and releases all the memory buffers.
.. ocv:function:: string FileStorage::releaseAndGetString()
Call this method after all I/O operations with the storage are finished. If the storage was opened for writing data and ``FileStorage::WRITE`` was specified
FileStorage::getFirstTopLevelNode
---------------------------------
Returns the first element of the top-level mapping.

11
modules/core/include/opencv2/core/core.hpp
Unescape View File

@ -3941,7 +3941,12 @@ public:
{
READ=0, //! read mode
WRITE=1, //! write mode
APPEND=2 //! append mode
APPEND=2, //! append mode
MEMORY=4,
FORMAT_MASK=(7<<3),
FORMAT_AUTO=0,
FORMAT_XML=(1<<3),
FORMAT_YAML=(2<<3)
};
enum
{
@ -3953,7 +3958,7 @@ public:
//! the default constructor
CV_WRAP FileStorage();
//! the full constructor that opens file storage for reading or writing
CV_WRAP FileStorage(const string& filename, int flags, const string& encoding=string());
CV_WRAP FileStorage(const string& source, int flags, const string& encoding=string());
//! the constructor that takes pointer to the C FileStorage structure
FileStorage(CvFileStorage* fs);
//! the destructor. calls release()
@ -3965,6 +3970,8 @@ public:
CV_WRAP virtual bool isOpened() const;
//! closes the file and releases all the memory buffers
CV_WRAP virtual void release();
//! closes the file, releases all the memory buffers and returns the text string
CV_WRAP string releaseAndGetString();
//! returns the first element of the top-level mapping
CV_WRAP FileNode getFirstTopLevelNode() const;

5
modules/core/include/opencv2/core/types_c.h
Unescape View File

@ -1740,6 +1740,11 @@ typedef struct CvFileStorage CvFileStorage;
#define CV_STORAGE_WRITE_TEXT CV_STORAGE_WRITE
#define CV_STORAGE_WRITE_BINARY CV_STORAGE_WRITE
#define CV_STORAGE_APPEND 2
#define CV_STORAGE_MEMORY 4
#define CV_STORAGE_FORMAT_MASK (7<<3)
#define CV_STORAGE_FORMAT_AUTO 0
#define CV_STORAGE_FORMAT_XML 8
#define CV_STORAGE_FORMAT_YAML 16
/* List of attributes: */
typedef struct CvAttrList

284
modules/core/src/persistence.cpp
Unescape View File

@ -41,7 +41,10 @@
//M*/
#include "precomp.hpp"
#include <ctype.h>
#include <deque>
#include <iterator>
#include <wchar.h>
#include <zlib.h>
@ -208,7 +211,7 @@ typedef void (*CvStartNextStream)( struct CvFileStorage* fs );
typedef struct CvFileStorage
{
int flags;
int is_xml;
int fmt;
int write_mode;
int is_first;
CvMemStorage* memstorage;
@ -240,52 +243,86 @@ typedef struct CvFileStorage
CvWriteString write_string;
CvWriteComment write_comment;
CvStartNextStream start_next_stream;
//CvParse parse;
const char* strbuf;
size_t strbufsize, strbufpos;
std::deque<char>* outbuf;
bool is_opened;
}
CvFileStorage;
static void icvPuts( CvFileStorage* fs, const char* str )
{
CV_Assert( fs->file || fs->gzfile );
if( fs->file )
if( fs->outbuf )
std::copy(str, str + strlen(str), std::back_inserter(*fs->outbuf));
else if( fs->file )
fputs( str, fs->file );
else
else if( fs->gzfile )
gzputs( fs->gzfile, str );
else
CV_Error( CV_StsError, "The storage is not opened" );
}
static char* icvGets( CvFileStorage* fs, char* str, int maxCount )
{
CV_Assert( fs->file || fs->gzfile );
if( fs->strbuf )
{
size_t i = fs->strbufpos, len = fs->strbufsize;
int j = 0;
const char* instr = fs->strbuf;
while( i < len && j < maxCount-1 )
{
char c = instr[i++];
if( c == '\0' )
break;
str[j++] = c;
if( c == '\n' )
break;
}
str[j++] = '\0';
fs->strbufpos = i;
return j > 1 ? str : 0;
}
if( fs->file )
return fgets( str, maxCount, fs->file );
return gzgets( fs->gzfile, str, maxCount );
if( fs->gzfile )
return gzgets( fs->gzfile, str, maxCount );
CV_Error( CV_StsError, "The storage is not opened" );
return 0;
}
static int icvEof( CvFileStorage* fs )
{
CV_Assert( fs->file || fs->gzfile );
if( fs->strbuf )
return fs->strbufpos >= fs->strbufsize;
if( fs->file )
return feof(fs->file);
return gzeof(fs->gzfile);
if( fs->gzfile )
return gzeof(fs->gzfile);
return false;
}
static void icvClose( CvFileStorage* fs )
static void icvCloseFile( CvFileStorage* fs )
{
if( fs->file )
fclose( fs->file );
if( fs->gzfile )
else if( fs->gzfile )
gzclose( fs->gzfile );
fs->file = 0;
fs->gzfile = 0;
fs->strbuf = 0;
fs->strbufpos = 0;
fs->is_opened = false;
}
static void icvRewind( CvFileStorage* fs )
{
CV_Assert( fs->file || fs->gzfile );
if( fs->file )
rewind(fs->file);
else
else if( fs->gzfile )
gzrewind(fs->gzfile);
fs->strbufpos = 0;
}
#define CV_YML_INDENT 3
@ -373,7 +410,7 @@ icvFSCreateCollection( CvFileStorage* fs, int tag, CvFileNode* collection )
{
if( collection->tag != CV_NODE_NONE )
{
assert( fs->is_xml != 0 );
assert( fs->fmt == CV_STORAGE_FORMAT_XML );
CV_PARSE_ERROR( "Sequence element should not have name (use <_></_>)" );
}
@ -477,19 +514,18 @@ icvFSFlush( CvFileStorage* fs )
}
/* closes file storage and deallocates buffers */
CV_IMPL void
cvReleaseFileStorage( CvFileStorage** p_fs )
static void
icvClose( CvFileStorage* fs, std::string* out )
{
if( !p_fs )
if( out )
out->clear();
if( !fs )
CV_Error( CV_StsNullPtr, "NULL double pointer to file storage" );
if( *p_fs )
if( fs->is_opened )
{
CvFileStorage* fs = *p_fs;
*p_fs = 0;
if( fs->write_mode && (fs->file || fs->gzfile) )
if( fs->write_mode && (fs->file || fs->gzfile || fs->outbuf) )
{
if( fs->write_stack )
{
@ -497,19 +533,42 @@ cvReleaseFileStorage( CvFileStorage** p_fs )
cvEndWriteStruct(fs);
}
icvFSFlush(fs);
if( fs->is_xml )
if( fs->fmt == CV_STORAGE_FORMAT_XML )
icvPuts( fs, "</opencv_storage>\n" );
}
icvCloseFile(fs);
}
if( fs->outbuf && out )
{
out->resize(fs->outbuf->size());
std::copy(fs->outbuf->begin(), fs->outbuf->end(), out->begin());
}
}
//icvFSReleaseCollection( fs->roots ); // delete all the user types recursively
icvClose(fs);
cvReleaseMemStorage( &fs->strstorage );
/* closes file storage and deallocates buffers */
CV_IMPL void
cvReleaseFileStorage( CvFileStorage** p_fs )
{
if( !p_fs )
CV_Error( CV_StsNullPtr, "NULL double pointer to file storage" );
if( *p_fs )
{
CvFileStorage* fs = *p_fs;
*p_fs = 0;
icvClose(fs, 0);
cvReleaseMemStorage( &fs->strstorage );
cvFree( &fs->buffer_start );
cvReleaseMemStorage( &fs->memstorage );
if( fs->outbuf )
delete fs->outbuf;
memset( fs, 0, sizeof(*fs) );
cvFree( &fs );
}
@ -2601,10 +2660,22 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
char* xml_buf = 0;
int default_block_size = 1 << 18;
bool append = (flags & 3) == CV_STORAGE_APPEND;
bool mem = (flags & CV_STORAGE_MEMORY) != 0;
bool write_mode = (flags & 3) != 0;
bool isGZ = false;
size_t fnamelen = 0;
if( !filename )
CV_Error( CV_StsNullPtr, "NULL filename" );
if( !filename || filename[0] == '\0' )
{
if( !write_mode )
CV_Error( CV_StsNullPtr, mem ? "NULL or empty filename" : "NULL or empty buffer" );
mem = true;
}
else
fnamelen = strlen(filename);
if( mem && append )
CV_Error( CV_StsBadFlag, "CV_STORAGE_APPEND and CV_STORAGE_MEMORY are not currently compatible" );
fs = (CvFileStorage*)cvAlloc( sizeof(*fs) );
memset( fs, 0, sizeof(*fs));
@ -2612,44 +2683,43 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
fs->memstorage = cvCreateMemStorage( default_block_size );
fs->dststorage = dststorage ? dststorage : fs->memstorage;
int fnamelen = (int)strlen(filename);
if( !fnamelen )
CV_Error( CV_StsError, "Empty filename" );
fs->filename = (char*)cvMemStorageAlloc( fs->memstorage, fnamelen+1 );
strcpy( fs->filename, filename );
char* dot_pos = strrchr(fs->filename, '.');
char compression = '\0';
if( dot_pos && dot_pos[1] == 'g' && dot_pos[2] == 'z' &&
(dot_pos[3] == '\0' || (cv_isdigit(dot_pos[3]) && dot_pos[4] == '\0')) )
fs->flags = CV_FILE_STORAGE;
fs->write_mode = write_mode;
if( !mem )
{
if( append )
CV_Error(CV_StsNotImplemented, "Appending data to compressed file is not implemented" );
isGZ = true;
compression = dot_pos[3];
if( compression )
dot_pos[3] = '\0', fnamelen--;
}
fs->filename = (char*)cvMemStorageAlloc( fs->memstorage, fnamelen+1 );
strcpy( fs->filename, filename );
fs->flags = CV_FILE_STORAGE;
fs->write_mode = (flags & 3) != 0;
char* dot_pos = strrchr(fs->filename, '.');
char compression = '\0';
if( !isGZ )
{
fs->file = fopen(fs->filename, !fs->write_mode ? "rt" : !append ? "wt" : "a+t" );
if( !fs->file )
goto _exit_;
}
else
{
char mode[] = { fs->write_mode ? 'w' : 'r', 'b', compression ? compression : '3', '\0' };
fs->gzfile = gzopen(fs->filename, mode);
if( !fs->gzfile )
goto _exit_;
}
if( dot_pos && dot_pos[1] == 'g' && dot_pos[2] == 'z' &&
(dot_pos[3] == '\0' || (cv_isdigit(dot_pos[3]) && dot_pos[4] == '\0')) )
{
if( append )
CV_Error(CV_StsNotImplemented, "Appending data to compressed file is not implemented" );
isGZ = true;
compression = dot_pos[3];
if( compression )
dot_pos[3] = '\0', fnamelen--;
}
if( !isGZ )
{
fs->file = fopen(fs->filename, !fs->write_mode ? "rt" : !append ? "wt" : "a+t" );
if( !fs->file )
goto _exit_;
}
else
{
char mode[] = { fs->write_mode ? 'w' : 'r', 'b', compression ? compression : '3', '\0' };
fs->gzfile = gzopen(fs->filename, mode);
if( !fs->gzfile )
goto _exit_;
}
}
fs->roots = 0;
fs->struct_indent = 0;
fs->struct_flags = 0;
@ -2657,27 +2727,38 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
if( fs->write_mode )
{
int fmt = flags & CV_STORAGE_FORMAT_MASK;
if( mem )
fs->outbuf = new std::deque<char>;
if( fmt == CV_STORAGE_FORMAT_AUTO && filename )
{
const char* dot_pos = filename + fnamelen - (isGZ ? 7 : 4);
fs->fmt = (dot_pos >= filename && (memcmp( dot_pos, ".xml", 4) == 0 ||
memcmp(dot_pos, ".XML", 4) == 0 || memcmp(dot_pos, ".Xml", 4) == 0)) ?
CV_STORAGE_FORMAT_XML : CV_STORAGE_FORMAT_YAML;
}
else
fs->fmt = fmt != CV_STORAGE_FORMAT_AUTO ? fmt : CV_STORAGE_FORMAT_XML;
// we use factor=6 for XML (the longest characters (' and ") are encoded with 6 bytes (&apos; and &quot;)
// and factor=4 for YAML ( as we use 4 bytes for non ASCII characters (e.g. \xAB))
int buf_size = CV_FS_MAX_LEN*(fs->is_xml ? 6 : 4) + 1024;
dot_pos = fs->filename + fnamelen - (isGZ ? 7 : 4);
fs->is_xml = dot_pos > fs->filename && (memcmp( dot_pos, ".xml", 4) == 0 ||
memcmp(dot_pos, ".XML", 4) == 0 || memcmp(dot_pos, ".Xml", 4) == 0);
int buf_size = CV_FS_MAX_LEN*(fs->fmt == CV_STORAGE_FORMAT_XML ? 6 : 4) + 1024;
if( append )
fseek( fs->file, 0, SEEK_END );
fs->write_stack = cvCreateSeq( 0, sizeof(CvSeq), fs->is_xml ?
fs->write_stack = cvCreateSeq( 0, sizeof(CvSeq), fs->fmt == CV_STORAGE_FORMAT_XML ?
sizeof(CvXMLStackRecord) : sizeof(int), fs->memstorage );
fs->is_first = 1;
fs->struct_indent = 0;
fs->struct_flags = CV_NODE_EMPTY;
fs->buffer_start = fs->buffer = (char*)cvAlloc( buf_size + 1024 );
fs->buffer_end = fs->buffer_start + buf_size;
if( fs->is_xml )
if( fs->fmt == CV_STORAGE_FORMAT_XML )
{
int file_size = fs->file ? (int)ftell( fs->file ) : 0;
size_t file_size = fs->file ? ftell( fs->file ) : (size_t)0;
fs->strstorage = cvCreateChildMemStorage( fs->memstorage );
if( !append || file_size == 0 )
{
@ -2702,7 +2783,7 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
int xml_buf_size = 1 << 10;
char substr[] = "</opencv_storage>";
int last_occurence = -1;
xml_buf_size = MIN(xml_buf_size, file_size);
xml_buf_size = MIN(xml_buf_size, int(file_size));
fseek( fs->file, -xml_buf_size, SEEK_END );
xml_buf = (char*)cvAlloc( xml_buf_size+2 );
// find the last occurence of </opencv_storage>
@ -2724,7 +2805,7 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
}
if( last_occurence < 0 )
CV_Error( CV_StsError, "Could not find </opencv_storage> in the end of file.\n" );
icvClose( fs );
icvCloseFile( fs );
fs->file = fopen( fs->filename, "r+t" );
fseek( fs->file, last_occurence, SEEK_SET );
// replace the last "</opencv_storage>" with " <!-- resumed -->", which has the same length
@ -2757,18 +2838,30 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
}
else
{
int buf_size = 1 << 20;
if( mem )
{
fs->strbuf = filename;
fs->strbufsize = fnamelen;
}
size_t buf_size = 1 << 20;
const char* yaml_signature = "%YAML:";
char buf[16];
icvGets( fs, buf, sizeof(buf)-2 );
fs->is_xml = strncmp( buf, yaml_signature, strlen(yaml_signature) ) != 0;
fs->fmt = strncmp( buf, yaml_signature, strlen(yaml_signature) ) == 0 ?
CV_STORAGE_FORMAT_YAML : CV_STORAGE_FORMAT_XML;
if( !isGZ )
{
fseek( fs->file, 0, SEEK_END );
buf_size = ftell( fs->file );
buf_size = MIN( buf_size, (1 << 20) );
buf_size = MAX( buf_size, CV_FS_MAX_LEN*2 + 1024 );
if( !mem )
{
fseek( fs->file, 0, SEEK_END );
buf_size = ftell( fs->file );
}
else
buf_size = fs->strbufsize;
buf_size = MIN( buf_size, (size_t)(1 << 20) );
buf_size = MAX( buf_size, (size_t)(CV_FS_MAX_LEN*2 + 1024) );
}
icvRewind(fs);
@ -2785,7 +2878,7 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
//mode = cvGetErrMode();
//cvSetErrMode( CV_ErrModeSilent );
if( fs->is_xml )
if( fs->fmt == CV_STORAGE_FORMAT_XML )
icvXMLParse( fs );
else
icvYMLParse( fs );
@ -2795,16 +2888,21 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
cvFree( &fs->buffer_start );
fs->buffer = fs->buffer_end = 0;
}
fs->is_opened = true;
_exit_:
if( fs )
{
if( cvGetErrStatus() < 0 || (!fs->file && !fs->gzfile) )
if( cvGetErrStatus() < 0 || (!fs->file && !fs->gzfile && !fs->outbuf && !fs->strbuf) )
{
cvReleaseFileStorage( &fs );
}
else if( !fs->write_mode )
{
icvClose(fs);
icvCloseFile(fs);
// we close the file since it's not needed anymore. But icvCloseFile() resets is_opened,
// which may be misleading. Since we restore the value of is_opened.
fs->is_opened = true;
}
}
@ -3057,7 +3155,7 @@ cvWriteRawData( CvFileStorage* fs, const void* _data, int len, const char* dt )
return;
}
if( fs->is_xml )
if( fs->fmt == CV_STORAGE_FORMAT_XML )
{
int buf_len = (int)strlen(ptr);
icvXMLWriteScalar( fs, 0, ptr, buf_len );
@ -5054,7 +5152,7 @@ bool FileStorage::open(const string& filename, int flags, const string& encoding
bool FileStorage::isOpened() const
{
return !fs.empty();
return !fs.empty() && fs.obj->is_opened;
}
void FileStorage::release()
@ -5064,6 +5162,16 @@ void FileStorage::release()
state = UNDEFINED;
}
string FileStorage::releaseAndGetString()
{
string buf;
if( fs.obj && fs.obj->outbuf )
icvClose(fs.obj, &buf);
release();
return buf;
}
FileNode FileStorage::root(int streamidx) const
{
return isOpened() ? FileNode(fs, cvGetRootFileNode(fs, streamidx)) : FileNode();

15
modules/core/test/test_io.cpp
Unescape View File

@ -91,7 +91,7 @@ protected:
{-1000000, 1000000}, {-10, 10}, {-10, 10}};
RNG& rng = ts->get_rng();
RNG rng0;
test_case_count = 2;
test_case_count = 4;
int progress = 0;
MemStorage storage(cvCreateMemStorage(0));
@ -102,9 +102,10 @@ protected:
cvClearMemStorage(storage);
bool mem = (idx % 4) >= 2;
string filename = tempfile(idx % 2 ? ".yml" : ".xml");
FileStorage fs(filename.c_str(), FileStorage::WRITE);
FileStorage fs(filename, FileStorage::WRITE + (mem ? FileStorage::MEMORY : 0));
int test_int = (int)cvtest::randInt(rng);
double test_real = (cvtest::randInt(rng)%2?1:-1)*exp(cvtest::randReal(rng)*18-9);
@ -179,11 +180,11 @@ protected:
fs.writeObj("test_graph",graph);
CvGraph* graph2 = (CvGraph*)cvClone(graph);
fs.release();
string content = fs.releaseAndGetString();
if(!fs.open(filename.c_str(), FileStorage::READ))
if(!fs.open(mem ? content : filename, FileStorage::READ + (mem ? FileStorage::MEMORY : 0)))
{
ts->printf( cvtest::TS::LOG, "filename %s can not be read\n", filename.c_str() );
ts->printf( cvtest::TS::LOG, "filename %s can not be read\n", !mem ? filename.c_str() : content.c_str());
ts->set_failed_test_info( cvtest::TS::FAIL_MISSING_TEST_DATA );
return;
}
@ -310,7 +311,6 @@ protected:
int real_width = (int)tm["width"];
int real_height = (int)tm["height"];
int real_lbp_val = 0;
FileNodeIterator it;
it = tm_lbp.begin();
@ -370,7 +370,8 @@ protected:
}
fs.release();
remove(filename.c_str());
if( !mem )
remove(filename.c_str());
}
}
};

17
modules/core/test/test_misc.cpp
Unescape View File

@ -22,4 +22,21 @@ TEST(Core_Drawing, _914)
int pixelsDrawn = rows*cols - countNonZero(img);
ASSERT_EQ( (3*rows + cols)*3 - 3*9, pixelsDrawn);
}
TEST(Core_OutputArraySreate, _1997)
{
struct local {
static void create(OutputArray arr, Size submatSize, int type)
{
int sizes[] = {submatSize.width, submatSize.height};
arr.create(sizeof(sizes)/sizeof(sizes[0]), sizes, type);
}
};
Mat mat(Size(512, 512), CV_8U);
Size submatSize = Size(256, 256);
ASSERT_NO_THROW(local::create( mat(Rect(Point(), submatSize)), submatSize, mat.type() ));
}
Write Preview
Loading…
Cancel
Save