Skip to content

O
opencv
Commit 002904e4 authored by Alexander Alekhin's avatar Alexander Alekhin
Browse files
Options

Merge pull request #15050 from alalek:core_fix_base64_packed_struct

parents e0f8bb83 4ea8526e
Hide whitespace changes
Inline Side-by-side
Showing with 173 additions and 118 deletions
modules/core/include/opencv2/core/persistence.hpp
View file @ 002904e4
......@@ -597,8 +597,8 @@ public:
Usually it is more convenient to use operator `>>` instead of this method.
@param fmt Specification of each array element. See @ref format_spec "format specification"
@param vec Pointer to the destination array.
@param len Number of elements to read. If it is greater than number of remaining elements then all
of them will be read.
@param len Number of bytes to read (buffer size limit). If it is greater than number of
remaining elements then all of them will be read.
*/
void readRaw( const String& fmt, uchar* vec, size_t len ) const;
......@@ -668,11 +668,12 @@ public:
Usually it is more convenient to use operator `>>` instead of this method.
@param fmt Specification of each array element. See @ref format_spec "format specification"
@param vec Pointer to the destination array.
@param maxCount Number of elements to read. If it is greater than number of remaining elements then
all of them will be read.
@param len Number of bytes to read (buffer size limit). If it is greater than number of
remaining elements then all of them will be read.
*/
FileNodeIterator& readRaw( const String& fmt, uchar* vec,
size_t maxCount=(size_t)INT_MAX );
size_t len=(size_t)INT_MAX );
struct SeqReader
{
......
modules/core/src/persistence.cpp
View file @ 002904e4
......@@ -145,7 +145,7 @@ CvGenericHash* cvCreateMap( int flags, int header_size, int elem_size, CvMemStor
return map;
}
void icvParseError( CvFileStorage* fs, const char* func_name,
void icvParseError(const CvFileStorage* fs, const char* func_name,
const char* err_msg, const char* source_file, int source_line )
{
cv::String msg = cv::format("%s(%d): %s", fs->filename, fs->lineno, err_msg);
......
modules/core/src/persistence.hpp
View file @ 002904e4
......@@ -55,7 +55,7 @@ size_t base64_decode_buffer_size(size_t cnt, char const * src, bool is_end_with
size_t base64_decode_buffer_size(size_t cnt, uchar const * src, bool is_end_with_zero = true);
std::string make_base64_header(const char * dt);
bool read_base64_header(std::vector<char> const & header, std::string & dt);
void make_seq(void * binary_data, int elem_cnt, const char * dt, CvSeq & seq);
void make_seq(::CvFileStorage* fs, const uchar* binary_data, size_t elem_cnt, const char * dt, CvSeq & seq);
void cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt);
class Base64ContextEmitter;
......@@ -262,7 +262,7 @@ void icvFSCreateCollection( CvFileStorage* fs, int tag, CvFileNode* collection )
char* icvFSResizeWriteBuffer( CvFileStorage* fs, char* ptr, int len );
int icvCalcStructSize( const char* dt, int initial_size );
int icvCalcElemSize( const char* dt, int initial_size );
void CV_NORETURN icvParseError( CvFileStorage* fs, const char* func_name, const char* err_msg, const char* source_file, int source_line );
void CV_NORETURN icvParseError(const CvFileStorage* fs, const char* func_name, const char* err_msg, const char* source_file, int source_line);
char* icvEncodeFormat( int elem_type, char* dt );
int icvDecodeFormat( const char* dt, int* fmt_pairs, int max_len );
int icvDecodeSimpleFormat( const char* dt );
......
modules/core/src/persistence_base64.cpp
View file @ 002904e4
......@@ -5,6 +5,8 @@
#include "precomp.hpp"
#include "persistence.hpp"
#include <opencv2/core/utils/logger.hpp>
#include <opencv2/core/utils/configuration.private.hpp>
namespace base64 {
......@@ -555,7 +557,7 @@ public:
CV_Assert(len > 0);
/* calc step and to_binary_funcs */
make_to_binary_funcs(dt);
step_packed = make_to_binary_funcs(dt);
end = beg;
cur = beg;
......@@ -570,10 +572,10 @@ public:
for (size_t i = 0U, n = to_binary_funcs.size(); i < n; i++) {
elem_to_binary_t & pack = to_binary_funcs[i];
pack.func(cur + pack.offset, dst + pack.offset);
pack.func(cur + pack.offset, dst + pack.offset_packed);
}
cur += step;
dst += step;
dst += step_packed;
return *this;
}
......@@ -588,14 +590,16 @@ private:
struct elem_to_binary_t
{
size_t offset;
size_t offset_packed;
to_binary_t func;
};
private:
void make_to_binary_funcs(const std::string &dt)
size_t make_to_binary_funcs(const std::string &dt)
{
size_t cnt = 0;
size_t offset = 0;
size_t offset_packed = 0;
char type = '\0';
std::istringstream iss(dt);
......@@ -646,11 +650,15 @@ private:
pack.offset = offset;
offset += size;
pack.offset_packed = offset_packed;
offset_packed += size;
to_binary_funcs.push_back(pack);
}
}
CV_Assert(iss.eof());
return offset_packed;
}
private:
......@@ -659,27 +667,26 @@ private:
const uchar * end;
size_t step;
size_t step_packed;
std::vector<elem_to_binary_t> to_binary_funcs;
};
class BinaryToCvSeqConvertor
{
public:
BinaryToCvSeqConvertor(const void* src, int len, const char* dt)
: cur(reinterpret_cast<const uchar *>(src))
, beg(reinterpret_cast<const uchar *>(src))
, end(reinterpret_cast<const uchar *>(src))
BinaryToCvSeqConvertor(CvFileStorage* fs, const uchar* src, size_t total_byte_size, const char* dt)
: cur(src)
, end(src + total_byte_size)
{
CV_Assert(src);
CV_Assert(dt);
CV_Assert(len >= 0);
CV_Assert(total_byte_size > 0);
/* calc binary_to_funcs */
make_funcs(dt);
step = make_funcs(dt); // calc binary_to_funcs
functor_iter = binary_to_funcs.begin();
step = ::icvCalcStructSize(dt, 0);
end = beg + step * static_cast<size_t>(len);
if (total_byte_size % step != 0)
CV_PARSE_ERROR("Total byte size not match elememt size");
}
inline BinaryToCvSeqConvertor & operator >> (CvFileNode & dst)
......@@ -699,7 +706,7 @@ public:
double d;
} buffer; /* for GCC -Wstrict-aliasing */
std::memset(buffer.mem, 0, sizeof(buffer));
functor_iter->func(cur + functor_iter->offset, buffer.mem);
functor_iter->func(cur + functor_iter->offset_packed, buffer.mem);
/* set node::data */
switch (functor_iter->cv_type)
......@@ -746,16 +753,17 @@ private:
struct binary_to_filenode_t
{
size_t cv_type;
size_t offset;
size_t offset_packed;
binary_to_t func;
};
private:
void make_funcs(const char* dt)
size_t make_funcs(const char* dt)
{
size_t cnt = 0;
char type = '\0';
size_t offset = 0;
size_t offset_packed = 0;
std::istringstream iss(dt);
while (!iss.eof()) {
......@@ -803,9 +811,28 @@ private:
}; // need a better way for outputting error.
offset = static_cast<size_t>(cvAlign(static_cast<int>(offset), static_cast<int>(size)));
pack.offset = offset;
if (offset != offset_packed)
{
static bool skip_message = cv::utils::getConfigurationParameterBool("OPENCV_PERSISTENCE_SKIP_PACKED_STRUCT_WARNING",
#ifdef _DEBUG
false
#else
true
#endif
);
if (!skip_message)
{
CV_LOG_WARNING(NULL, "Binary converter: struct storage layout has been changed in OpenCV 3.4.7. Alignment gaps has been removed from the storage containers. "
"Details: https://github.com/opencv/opencv/pull/15050"
);
skip_message = true;
}
}
offset += size;
pack.offset_packed = offset_packed;
offset_packed += size;
/* set type */
switch (type)
{
......@@ -827,12 +854,13 @@ private:
CV_Assert(iss.eof());
CV_Assert(binary_to_funcs.size());
return offset_packed;
}
private:
const uchar * cur;
const uchar * beg;
const uchar * end;
size_t step;
......@@ -889,11 +917,13 @@ void Base64Writer::check_dt(const char* dt)
}
void make_seq(void * binary, int elem_cnt, const char * dt, ::CvSeq & seq)
void make_seq(CvFileStorage* fs, const uchar* binary, size_t total_byte_size, const char * dt, ::CvSeq & seq)
{
if (total_byte_size == 0)
return;
::CvFileNode node;
node.info = 0;
BinaryToCvSeqConvertor convertor(binary, elem_cnt, dt);
BinaryToCvSeqConvertor convertor(fs, binary, total_byte_size, dt);
while (convertor) {
convertor >> node;
cvSeqPush(&seq, &node);
......
modules/core/src/persistence_cpp.cpp
View file @ 002904e4
......@@ -11,21 +11,6 @@
namespace cv
{
static void getElemSize( const String& fmt, size_t& elemSize, size_t& cn )
{
const char* dt = fmt.c_str();
cn = 1;
if( cv_isdigit(dt[0]) )
{
cn = dt[0] - '0';
dt++;
}
char c = dt[0];
elemSize = cn*(c == 'u' || c == 'c' ? sizeof(uchar) : c == 'w' || c == 's' ? sizeof(ushort) :
c == 'i' ? sizeof(int) : c == 'f' ? sizeof(float) : c == 'd' ? sizeof(double) :
c == 'r' ? sizeof(void*) : (size_t)0);
}
FileStorage::FileStorage()
{
state = UNDEFINED;
......@@ -164,8 +149,8 @@ void FileStorage::writeRaw( const String& fmt, const uchar* vec, size_t len )
{
if( !isOpened() )
return;
size_t elemSize, cn;
getElemSize( fmt, elemSize, cn );
CV_Assert(!fmt.empty());
size_t elemSize = ::icvCalcStructSize(fmt.c_str(), 0);
CV_Assert( len % elemSize == 0 );
cvWriteRawData( fs, vec, (int)(len/elemSize), fmt.c_str());
}
......@@ -412,19 +397,30 @@ FileNodeIterator& FileNodeIterator::operator -= (int ofs)
}
FileNodeIterator& FileNodeIterator::readRaw( const String& fmt, uchar* vec, size_t maxCount )
FileNodeIterator& FileNodeIterator::readRaw(const String& fmt, uchar* vec, size_t len)
{
if( fs && container && remaining > 0 )
CV_Assert(!fmt.empty());
if( fs && container && remaining > 0 && len > 0)
{
size_t elem_size, cn;
getElemSize( fmt, elem_size, cn );
CV_Assert( elem_size > 0 );
size_t count = std::min(remaining, maxCount);
if( reader.seq )
if (reader.seq)
{
cvReadRawDataSlice( fs, (CvSeqReader*)&reader, (int)count, vec, fmt.c_str() );
remaining -= count*cn;
size_t step = ::icvCalcStructSize(fmt.c_str(), 0);
if (len % step && len != (size_t)INT_MAX) // TODO remove compatibility hack
{
CV_PARSE_ERROR("readRaw: total byte size not match elememt size");
}
size_t maxCount = len / step;
int fmt_pairs[CV_FS_MAX_FMT_PAIRS*2] = {};
int fmt_pair_count = icvDecodeFormat(fmt.c_str(), fmt_pairs, CV_FS_MAX_FMT_PAIRS);
int vecElems = 0;
for (int k = 0; k < fmt_pair_count; k++)
{
vecElems += fmt_pairs[k*2];
}
CV_Assert(vecElems > 0);
size_t count = std::min((size_t)remaining, (size_t)maxCount * vecElems);
cvReadRawDataSlice(fs, (CvSeqReader*)&reader, (int)count, vec, fmt.c_str());
remaining -= count;
}
else
{
......
modules/core/src/persistence_json.cpp
View file @ 002904e4
......@@ -259,15 +259,9 @@ static char* icvJSONParseValue( CvFileStorage* fs, char* ptr, CvFileNode* node )
parser.flush();
}
/* save as CvSeq */
int elem_size = ::icvCalcStructSize(dt.c_str(), 0);
if (total_byte_size % elem_size != 0)
CV_PARSE_ERROR("Byte size not match elememt size");
int elem_cnt = total_byte_size / elem_size;
/* after icvFSCreateCollection, node->tag == struct_flags */
icvFSCreateCollection(fs, CV_NODE_FLOW | CV_NODE_SEQ, node);
base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
base64::make_seq(fs, binary_buffer.data(), total_byte_size, dt.c_str(), *node->data.seq);
}
else
{
......
modules/core/src/persistence_xml.cpp
View file @ 002904e4
......@@ -167,17 +167,11 @@ static char* icvXMLParseBase64(CvFileStorage* fs, char* ptr, CvFileNode * node)
parser.flush();
}
/* save as CvSeq */
int elem_size = ::icvCalcStructSize(dt.c_str(), 0);
if (total_byte_size % elem_size != 0)
CV_PARSE_ERROR("data size not matches elememt size");
int elem_cnt = total_byte_size / elem_size;
node->tag = CV_NODE_NONE;
int struct_flags = CV_NODE_SEQ;
/* after icvFSCreateCollection, node->tag == struct_flags */
icvFSCreateCollection(fs, struct_flags, node);
base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
base64::make_seq(fs, binary_buffer.data(), total_byte_size, dt.c_str(), *node->data.seq);
if (fs->dummy_eof) {
/* end of file */
......
modules/core/src/persistence_yml.cpp
View file @ 002904e4
......@@ -130,17 +130,11 @@ static char* icvYMLParseBase64(CvFileStorage* fs, char* ptr, int indent, CvFileN
parser.flush();
}
/* save as CvSeq */
int elem_size = ::icvCalcStructSize(dt.c_str(), 0);
if (total_byte_size % elem_size != 0)
CV_PARSE_ERROR("Byte size not match elememt size");
int elem_cnt = total_byte_size / elem_size;
node->tag = CV_NODE_NONE;
int struct_flags = CV_NODE_FLOW | CV_NODE_SEQ;
/* after icvFSCreateCollection, node->tag == struct_flags */
icvFSCreateCollection(fs, struct_flags, node);
base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
base64::make_seq(fs, binary_buffer.data(), total_byte_size, dt.c_str(), *node->data.seq);
if (fs->dummy_eof) {
/* end of file */
......
modules/core/test/test_io.cpp
View file @ 002904e4
......@@ -659,38 +659,29 @@ struct data_t
}
};
TEST(Core_InputOutput, filestorage_base64_basic)
static void test_filestorage_basic(int write_flags, const char* suffix_name, bool testReadWrite, bool useMemory = false)
{
const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info();
std::string basename = (test_info == 0)
? "filestorage_base64_valid_call"
: (std::string(test_info->test_case_name()) + "--" + test_info->name());
char const * filenames[] = {
"core_io_base64_basic_test.yml",
"core_io_base64_basic_test.xml",
"core_io_base64_basic_test.json",
0
};
CV_Assert(test_info);
std::string name = (std::string(test_info->test_case_name()) + "--" + test_info->name() + suffix_name);
if (!testReadWrite)
name = string(cvtest::TS::ptr()->get_data_path()) + "io/" + name;
for (char const ** ptr = filenames; *ptr; ptr++)
{
char const * suffix_name = *ptr;
std::string name = basename + '_' + suffix_name;
const size_t rawdata_N = 40;
std::vector<data_t> rawdata;
cv::Mat _em_out, _em_in;
cv::Mat _2d_out, _2d_in;
cv::Mat _nd_out, _nd_in;
cv::Mat _rd_out(64, 64, CV_64FC1), _rd_in;
cv::Mat _rd_out(8, 16, CV_64FC1), _rd_in;
bool no_type_id = true;
{ /* init */
/* a normal mat */
_2d_out = cv::Mat(100, 100, CV_8UC3, cvScalar(1U, 2U, 127U));
_2d_out = cv::Mat(10, 20, CV_8UC3, cvScalar(1U, 2U, 127U));
for (int i = 0; i < _2d_out.rows; ++i)
for (int j = 0; j < _2d_out.cols; ++j)
_2d_out.at<cv::Vec3b>(i, j)[1] = (i + j) % 256;
......@@ -709,7 +700,7 @@ TEST(Core_InputOutput, filestorage_base64_basic)
cv::randu(_rd_out, cv::Scalar(0.0), cv::Scalar(1.0));
/* raw data */
for (int i = 0; i < 1000; i++) {
for (int i = 0; i < (int)rawdata_N; i++) {
data_t tmp;
tmp.u1 = 1;
tmp.u2 = 2;
......@@ -722,24 +713,41 @@ TEST(Core_InputOutput, filestorage_base64_basic)
rawdata.push_back(tmp);
}
}
{ /* write */
cv::FileStorage fs(name, cv::FileStorage::WRITE_BASE64);
#ifdef GENERATE_TEST_DATA
#else
if (testReadWrite || useMemory)
#endif
{
cv::FileStorage fs(name, write_flags + (useMemory ? cv::FileStorage::MEMORY : 0));
fs << "normal_2d_mat" << _2d_out;
fs << "normal_nd_mat" << _nd_out;
fs << "empty_2d_mat" << _em_out;
fs << "random_mat" << _rd_out;
cvStartWriteStruct( *fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW, "binary" );
for (int i = 0; i < 10; i++)
cvWriteRawDataBase64(*fs, rawdata.data() + i * 100, 100, data_t::signature());
cvEndWriteStruct( *fs );
fs << "rawdata" << "[:";
for (int i = 0; i < (int)rawdata_N/10; i++)
fs.writeRaw(data_t::signature(), (const uchar*)&rawdata[i * 10], sizeof(data_t) * 10);
fs << "]";
fs.release();
size_t sz = 0;
if (useMemory)
{
name = fs.releaseAndGetString();
sz = name.size();
}
else
{
fs.release();
std::ifstream f(name.c_str(), std::ios::in|std::ios::binary);
f.seekg(0, std::fstream::end);
sz = (size_t)f.tellg();
f.close();
}
std::cout << "Storage size: " << sz << std::endl;
EXPECT_LE(sz, (size_t)6000);
}
{ /* read */
cv::FileStorage fs(name, cv::FileStorage::READ);
cv::FileStorage fs(name, cv::FileStorage::READ + (useMemory ? cv::FileStorage::MEMORY : 0));
/* mat */
fs["empty_2d_mat"] >> _em_in;
......@@ -754,14 +762,14 @@ TEST(Core_InputOutput, filestorage_base64_basic)
no_type_id = false;
/* raw data */
std::vector<data_t>(1000).swap(rawdata);
cvReadRawData(*fs, fs["rawdata"].node, rawdata.data(), data_t::signature());
std::vector<data_t>(rawdata_N).swap(rawdata);
fs["rawdata"].readRaw(data_t::signature(), (uchar*)&rawdata[0], rawdata.size() * sizeof(data_t));
fs.release();
}
int errors = 0;
for (int i = 0; i < 1000; i++)
for (int i = 0; i < (int)rawdata_N; i++)
{
EXPECT_EQ((int)rawdata[i].u1, 1);
EXPECT_EQ((int)rawdata[i].u2, 2);
......@@ -815,18 +823,54 @@ TEST(Core_InputOutput, filestorage_base64_basic)
EXPECT_EQ(_nd_in.cols , _nd_out.cols);
EXPECT_EQ(_nd_in.dims , _nd_out.dims);
EXPECT_EQ(_nd_in.depth(), _nd_out.depth());
EXPECT_EQ(cv::countNonZero(cv::mean(_nd_in != _nd_out)), 0);
EXPECT_EQ(0, cv::norm(_nd_in, _nd_out, NORM_INF));
EXPECT_EQ(_rd_in.rows , _rd_out.rows);
EXPECT_EQ(_rd_in.cols , _rd_out.cols);
EXPECT_EQ(_rd_in.dims , _rd_out.dims);
EXPECT_EQ(_rd_in.depth(), _rd_out.depth());
EXPECT_EQ(cv::countNonZero(cv::mean(_rd_in != _rd_out)), 0);
remove(name.c_str());
EXPECT_EQ(0, cv::norm(_rd_in, _rd_out, NORM_INF));
}
}
TEST(Core_InputOutput, filestorage_base64_basic_read_XML)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", false);
}
TEST(Core_InputOutput, filestorage_base64_basic_read_YAML)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", false);
}
TEST(Core_InputOutput, filestorage_base64_basic_read_JSON)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", false);
}
TEST(Core_InputOutput, filestorage_base64_basic_rw_XML)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true);
}
TEST(Core_InputOutput, filestorage_base64_basic_rw_YAML)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true);
}
TEST(Core_InputOutput, filestorage_base64_basic_rw_JSON)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true);
}
TEST(Core_InputOutput, filestorage_base64_basic_memory_XML)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true, true);
}
TEST(Core_InputOutput, filestorage_base64_basic_memory_YAML)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true, true);
}
TEST(Core_InputOutput, filestorage_base64_basic_memory_JSON)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true, true);
}
TEST(Core_InputOutput, filestorage_base64_valid_call)
{
const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info();
......@@ -856,10 +900,12 @@ TEST(Core_InputOutput, filestorage_base64_valid_call)
std::vector<int> rawdata(10, static_cast<int>(0x00010203));
cv::String str_out = "test_string";
for (char const ** ptr = filenames; *ptr; ptr++)
for (int n = 0; n < 6; n++)
{
char const * suffix_name = *ptr;
char const* suffix_name = filenames[n];
SCOPED_TRACE(suffix_name);
std::string name = basename + '_' + suffix_name;
std::string file_name = basename + '_' + real_name[n];
EXPECT_NO_THROW(
{
......@@ -877,9 +923,9 @@ TEST(Core_InputOutput, filestorage_base64_valid_call)
});
{
cv::FileStorage fs(name, cv::FileStorage::READ);
cv::FileStorage fs(file_name, cv::FileStorage::READ);
std::vector<int> data_in(rawdata.size());
fs["manydata"][0].readRaw("i", (uchar *)data_in.data(), data_in.size());
fs["manydata"][0].readRaw("i", (uchar *)data_in.data(), data_in.size() * sizeof(data_in[0]));
EXPECT_TRUE(fs["manydata"][0].isSeq());
EXPECT_TRUE(std::equal(rawdata.begin(), rawdata.end(), data_in.begin()));
cv::String str_in;
......@@ -905,19 +951,19 @@ TEST(Core_InputOutput, filestorage_base64_valid_call)
});
{
cv::FileStorage fs(name, cv::FileStorage::READ);
cv::FileStorage fs(file_name, cv::FileStorage::READ);
cv::String str_in;
fs["manydata"][0] >> str_in;
EXPECT_TRUE(fs["manydata"][0].isString());
EXPECT_EQ(str_in, str_out);
std::vector<int> data_in(rawdata.size());
fs["manydata"][1].readRaw("i", (uchar *)data_in.data(), data_in.size());
fs["manydata"][1].readRaw("i", (uchar *)data_in.data(), data_in.size() * sizeof(data_in[0]));
EXPECT_TRUE(fs["manydata"][1].isSeq());
EXPECT_TRUE(std::equal(rawdata.begin(), rawdata.end(), data_in.begin()));
fs.release();
}
remove((basename + '_' + real_name[ptr - filenames]).c_str());
EXPECT_EQ(0, remove(file_name.c_str()));
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment