Skip to content

O
opencv
Commit c3b838b7 authored by Alexander Alekhin's avatar Alexander Alekhin Committed by Alexander Alekhin
Browse files
Options

core(persistence): struct storage layout without alignment gaps

parent 054c7962
Hide whitespace changes
Inline Side-by-side
Showing with 137 additions and 79 deletions
modules/core/src/persistence.hpp
View file @ c3b838b7
......@@ -55,7 +55,7 @@ size_t base64_decode_buffer_size(size_t cnt, char const * src, bool is_end_with
size_t base64_decode_buffer_size(size_t cnt, uchar const * src, bool is_end_with_zero = true);
std::string make_base64_header(const char * dt);
bool read_base64_header(std::vector<char> const & header, std::string & dt);
void make_seq(void * binary_data, int elem_cnt, const char * dt, CvSeq & seq);
void make_seq(::CvFileStorage* fs, const uchar* binary_data, size_t elem_cnt, const char * dt, CvSeq & seq);
void cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt);
class Base64ContextEmitter;
......
modules/core/src/persistence_base64.cpp
View file @ c3b838b7
......@@ -5,6 +5,8 @@
#include "precomp.hpp"
#include "persistence.hpp"
#include <opencv2/core/utils/logger.hpp>
#include <opencv2/core/utils/configuration.private.hpp>
namespace base64 {
......@@ -555,7 +557,7 @@ public:
CV_Assert(len > 0);
/* calc step and to_binary_funcs */
make_to_binary_funcs(dt);
step_packed = make_to_binary_funcs(dt);
end = beg;
cur = beg;
......@@ -570,10 +572,10 @@ public:
for (size_t i = 0U, n = to_binary_funcs.size(); i < n; i++) {
elem_to_binary_t & pack = to_binary_funcs[i];
pack.func(cur + pack.offset, dst + pack.offset);
pack.func(cur + pack.offset, dst + pack.offset_packed);
}
cur += step;
dst += step;
dst += step_packed;
return *this;
}
......@@ -588,14 +590,16 @@ private:
struct elem_to_binary_t
{
size_t offset;
size_t offset_packed;
to_binary_t func;
};
private:
void make_to_binary_funcs(const std::string &dt)
size_t make_to_binary_funcs(const std::string &dt)
{
size_t cnt = 0;
size_t offset = 0;
size_t offset_packed = 0;
char type = '\0';
std::istringstream iss(dt);
......@@ -646,11 +650,15 @@ private:
pack.offset = offset;
offset += size;
pack.offset_packed = offset_packed;
offset_packed += size;
to_binary_funcs.push_back(pack);
}
}
CV_Assert(iss.eof());
return offset_packed;
}
private:
......@@ -659,27 +667,26 @@ private:
const uchar * end;
size_t step;
size_t step_packed;
std::vector<elem_to_binary_t> to_binary_funcs;
};
class BinaryToCvSeqConvertor
{
public:
BinaryToCvSeqConvertor(const void* src, int len, const char* dt)
: cur(reinterpret_cast<const uchar *>(src))
, beg(reinterpret_cast<const uchar *>(src))
, end(reinterpret_cast<const uchar *>(src))
BinaryToCvSeqConvertor(CvFileStorage* fs, const uchar* src, size_t total_byte_size, const char* dt)
: cur(src)
, end(src + total_byte_size)
{
CV_Assert(src);
CV_Assert(dt);
CV_Assert(len >= 0);
CV_Assert(total_byte_size > 0);
/* calc binary_to_funcs */
make_funcs(dt);
step = make_funcs(dt); // calc binary_to_funcs
functor_iter = binary_to_funcs.begin();
step = ::icvCalcStructSize(dt, 0);
end = beg + step * static_cast<size_t>(len);
if (total_byte_size % step != 0)
CV_PARSE_ERROR("Total byte size not match elememt size");
}
inline BinaryToCvSeqConvertor & operator >> (CvFileNode & dst)
......@@ -699,7 +706,7 @@ public:
double d;
} buffer; /* for GCC -Wstrict-aliasing */
std::memset(buffer.mem, 0, sizeof(buffer));
functor_iter->func(cur + functor_iter->offset, buffer.mem);
functor_iter->func(cur + functor_iter->offset_packed, buffer.mem);
/* set node::data */
switch (functor_iter->cv_type)
......@@ -746,16 +753,17 @@ private:
struct binary_to_filenode_t
{
size_t cv_type;
size_t offset;
size_t offset_packed;
binary_to_t func;
};
private:
void make_funcs(const char* dt)
size_t make_funcs(const char* dt)
{
size_t cnt = 0;
char type = '\0';
size_t offset = 0;
size_t offset_packed = 0;
std::istringstream iss(dt);
while (!iss.eof()) {
......@@ -803,9 +811,28 @@ private:
}; // need a better way for outputting error.
offset = static_cast<size_t>(cvAlign(static_cast<int>(offset), static_cast<int>(size)));
pack.offset = offset;
if (offset != offset_packed)
{
static bool skip_message = cv::utils::getConfigurationParameterBool("OPENCV_PERSISTENCE_SKIP_PACKED_STRUCT_WARNING",
#ifdef _DEBUG
false
#else
true
#endif
);
if (!skip_message)
{
CV_LOG_WARNING(NULL, "Binary converter: struct storage layout has been changed in OpenCV 3.4.7. Alignment gaps has been removed from the storage containers. "
"Details: https://github.com/opencv/opencv/pull/15050"
);
skip_message = true;
}
}
offset += size;
pack.offset_packed = offset_packed;
offset_packed += size;
/* set type */
switch (type)
{
......@@ -827,12 +854,13 @@ private:
CV_Assert(iss.eof());
CV_Assert(binary_to_funcs.size());
return offset_packed;
}
private:
const uchar * cur;
const uchar * beg;
const uchar * end;
size_t step;
......@@ -889,11 +917,13 @@ void Base64Writer::check_dt(const char* dt)
}
void make_seq(void * binary, int elem_cnt, const char * dt, ::CvSeq & seq)
void make_seq(CvFileStorage* fs, const uchar* binary, size_t total_byte_size, const char * dt, ::CvSeq & seq)
{
if (total_byte_size == 0)
return;
::CvFileNode node;
node.info = 0;
BinaryToCvSeqConvertor convertor(binary, elem_cnt, dt);
BinaryToCvSeqConvertor convertor(fs, binary, total_byte_size, dt);
while (convertor) {
convertor >> node;
cvSeqPush(&seq, &node);
......
modules/core/src/persistence_json.cpp
View file @ c3b838b7
......@@ -259,15 +259,9 @@ static char* icvJSONParseValue( CvFileStorage* fs, char* ptr, CvFileNode* node )
parser.flush();
}
/* save as CvSeq */
int elem_size = ::icvCalcStructSize(dt.c_str(), 0);
if (total_byte_size % elem_size != 0)
CV_PARSE_ERROR("Byte size not match elememt size");
int elem_cnt = total_byte_size / elem_size;
/* after icvFSCreateCollection, node->tag == struct_flags */
icvFSCreateCollection(fs, CV_NODE_FLOW | CV_NODE_SEQ, node);
base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
base64::make_seq(fs, binary_buffer.data(), total_byte_size, dt.c_str(), *node->data.seq);
}
else
{
......
modules/core/src/persistence_xml.cpp
View file @ c3b838b7
......@@ -167,17 +167,11 @@ static char* icvXMLParseBase64(CvFileStorage* fs, char* ptr, CvFileNode * node)
parser.flush();
}
/* save as CvSeq */
int elem_size = ::icvCalcStructSize(dt.c_str(), 0);
if (total_byte_size % elem_size != 0)
CV_PARSE_ERROR("data size not matches elememt size");
int elem_cnt = total_byte_size / elem_size;
node->tag = CV_NODE_NONE;
int struct_flags = CV_NODE_SEQ;
/* after icvFSCreateCollection, node->tag == struct_flags */
icvFSCreateCollection(fs, struct_flags, node);
base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
base64::make_seq(fs, binary_buffer.data(), total_byte_size, dt.c_str(), *node->data.seq);
if (fs->dummy_eof) {
/* end of file */
......
modules/core/src/persistence_yml.cpp
View file @ c3b838b7
......@@ -130,17 +130,11 @@ static char* icvYMLParseBase64(CvFileStorage* fs, char* ptr, int indent, CvFileN
parser.flush();
}
/* save as CvSeq */
int elem_size = ::icvCalcStructSize(dt.c_str(), 0);
if (total_byte_size % elem_size != 0)
CV_PARSE_ERROR("Byte size not match elememt size");
int elem_cnt = total_byte_size / elem_size;
node->tag = CV_NODE_NONE;
int struct_flags = CV_NODE_FLOW | CV_NODE_SEQ;
/* after icvFSCreateCollection, node->tag == struct_flags */
icvFSCreateCollection(fs, struct_flags, node);
base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
base64::make_seq(fs, binary_buffer.data(), total_byte_size, dt.c_str(), *node->data.seq);
if (fs->dummy_eof) {
/* end of file */
......
modules/core/test/test_io.cpp
View file @ c3b838b7
......@@ -659,38 +659,29 @@ struct data_t
}
};
TEST(Core_InputOutput, filestorage_base64_basic)
static void test_filestorage_basic(int write_flags, const char* suffix_name, bool testReadWrite, bool useMemory = false)
{
const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info();
std::string basename = (test_info == 0)
? "filestorage_base64_valid_call"
: (std::string(test_info->test_case_name()) + "--" + test_info->name());
char const * filenames[] = {
"core_io_base64_basic_test.yml",
"core_io_base64_basic_test.xml",
"core_io_base64_basic_test.json",
0
};
CV_Assert(test_info);
std::string name = (std::string(test_info->test_case_name()) + "--" + test_info->name() + suffix_name);
if (!testReadWrite)
name = string(cvtest::TS::ptr()->get_data_path()) + "io/" + name;
for (char const ** ptr = filenames; *ptr; ptr++)
{
char const * suffix_name = *ptr;
std::string name = basename + '_' + suffix_name;
const size_t rawdata_N = 40;
std::vector<data_t> rawdata;
cv::Mat _em_out, _em_in;
cv::Mat _2d_out, _2d_in;
cv::Mat _nd_out, _nd_in;
cv::Mat _rd_out(64, 64, CV_64FC1), _rd_in;
cv::Mat _rd_out(8, 16, CV_64FC1), _rd_in;
bool no_type_id = true;
{ /* init */
/* a normal mat */
_2d_out = cv::Mat(100, 100, CV_8UC3, cvScalar(1U, 2U, 127U));
_2d_out = cv::Mat(10, 20, CV_8UC3, cvScalar(1U, 2U, 127U));
for (int i = 0; i < _2d_out.rows; ++i)
for (int j = 0; j < _2d_out.cols; ++j)
_2d_out.at<cv::Vec3b>(i, j)[1] = (i + j) % 256;
......@@ -709,7 +700,7 @@ TEST(Core_InputOutput, filestorage_base64_basic)
cv::randu(_rd_out, cv::Scalar(0.0), cv::Scalar(1.0));
/* raw data */
for (int i = 0; i < 1000; i++) {
for (int i = 0; i < (int)rawdata_N; i++) {
data_t tmp;
tmp.u1 = 1;
tmp.u2 = 2;
......@@ -722,24 +713,41 @@ TEST(Core_InputOutput, filestorage_base64_basic)
rawdata.push_back(tmp);
}
}
{ /* write */
cv::FileStorage fs(name, cv::FileStorage::WRITE_BASE64);
#ifdef GENERATE_TEST_DATA
#else
if (testReadWrite || useMemory)
#endif
{
cv::FileStorage fs(name, write_flags + (useMemory ? cv::FileStorage::MEMORY : 0));
fs << "normal_2d_mat" << _2d_out;
fs << "normal_nd_mat" << _nd_out;
fs << "empty_2d_mat" << _em_out;
fs << "random_mat" << _rd_out;
cvStartWriteStruct( *fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW, "binary" );
for (int i = 0; i < 10; i++)
cvWriteRawDataBase64(*fs, rawdata.data() + i * 100, 100, data_t::signature());
for (int i = 0; i < (int)rawdata_N/10; i++)
cvWriteRawData(*fs, rawdata.data() + i * 10, 10, data_t::signature());
cvEndWriteStruct( *fs );
fs.release();
size_t sz = 0;
if (useMemory)
{
name = fs.releaseAndGetString();
sz = name.size();
}
else
{
fs.release();
std::ifstream f(name.c_str(), std::ios::in|std::ios::binary);
f.seekg(0, std::fstream::end);
sz = (size_t)f.tellg();
f.close();
}
std::cout << "Storage size: " << sz << std::endl;
EXPECT_LE(sz, (size_t)6000);
}
{ /* read */
cv::FileStorage fs(name, cv::FileStorage::READ);
cv::FileStorage fs(name, cv::FileStorage::READ + (useMemory ? cv::FileStorage::MEMORY : 0));
/* mat */
fs["empty_2d_mat"] >> _em_in;
......@@ -754,14 +762,14 @@ TEST(Core_InputOutput, filestorage_base64_basic)
no_type_id = false;
/* raw data */
std::vector<data_t>(1000).swap(rawdata);
std::vector<data_t>(rawdata_N).swap(rawdata);
cvReadRawData(*fs, fs["rawdata"].node, rawdata.data(), data_t::signature());
fs.release();
}
int errors = 0;
for (int i = 0; i < 1000; i++)
for (int i = 0; i < (int)rawdata_N; i++)
{
EXPECT_EQ((int)rawdata[i].u1, 1);
EXPECT_EQ((int)rawdata[i].u2, 2);
......@@ -815,18 +823,54 @@ TEST(Core_InputOutput, filestorage_base64_basic)
EXPECT_EQ(_nd_in.cols , _nd_out.cols);
EXPECT_EQ(_nd_in.dims , _nd_out.dims);
EXPECT_EQ(_nd_in.depth(), _nd_out.depth());
EXPECT_EQ(cv::countNonZero(cv::mean(_nd_in != _nd_out)), 0);
EXPECT_EQ(0, cv::norm(_nd_in, _nd_out, NORM_INF));
EXPECT_EQ(_rd_in.rows , _rd_out.rows);
EXPECT_EQ(_rd_in.cols , _rd_out.cols);
EXPECT_EQ(_rd_in.dims , _rd_out.dims);
EXPECT_EQ(_rd_in.depth(), _rd_out.depth());
EXPECT_EQ(cv::countNonZero(cv::mean(_rd_in != _rd_out)), 0);
remove(name.c_str());
EXPECT_EQ(0, cv::norm(_rd_in, _rd_out, NORM_INF));
}
}
TEST(Core_InputOutput, filestorage_base64_basic_read_XML)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", false);
}
TEST(Core_InputOutput, filestorage_base64_basic_read_YAML)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", false);
}
TEST(Core_InputOutput, filestorage_base64_basic_read_JSON)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", false);
}
TEST(Core_InputOutput, filestorage_base64_basic_rw_XML)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true);
}
TEST(Core_InputOutput, filestorage_base64_basic_rw_YAML)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true);
}
TEST(Core_InputOutput, filestorage_base64_basic_rw_JSON)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true);
}
TEST(Core_InputOutput, filestorage_base64_basic_memory_XML)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true, true);
}
TEST(Core_InputOutput, filestorage_base64_basic_memory_YAML)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true, true);
}
TEST(Core_InputOutput, filestorage_base64_basic_memory_JSON)
{
test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true, true);
}
TEST(Core_InputOutput, filestorage_base64_valid_call)
{
const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info();
......@@ -856,10 +900,12 @@ TEST(Core_InputOutput, filestorage_base64_valid_call)
std::vector<int> rawdata(10, static_cast<int>(0x00010203));
cv::String str_out = "test_string";
for (char const ** ptr = filenames; *ptr; ptr++)
for (int n = 0; n < 6; n++)
{
char const * suffix_name = *ptr;
char const* suffix_name = filenames[n];
SCOPED_TRACE(suffix_name);
std::string name = basename + '_' + suffix_name;
std::string file_name = basename + '_' + real_name[n];
EXPECT_NO_THROW(
{
......@@ -877,7 +923,7 @@ TEST(Core_InputOutput, filestorage_base64_valid_call)
});
{
cv::FileStorage fs(name, cv::FileStorage::READ);
cv::FileStorage fs(file_name, cv::FileStorage::READ);
std::vector<int> data_in(rawdata.size());
fs["manydata"][0].readRaw("i", (uchar *)data_in.data(), data_in.size());
EXPECT_TRUE(fs["manydata"][0].isSeq());
......@@ -905,7 +951,7 @@ TEST(Core_InputOutput, filestorage_base64_valid_call)
});
{
cv::FileStorage fs(name, cv::FileStorage::READ);
cv::FileStorage fs(file_name, cv::FileStorage::READ);
cv::String str_in;
fs["manydata"][0] >> str_in;
EXPECT_TRUE(fs["manydata"][0].isString());
......@@ -917,7 +963,7 @@ TEST(Core_InputOutput, filestorage_base64_valid_call)
fs.release();
}
remove((basename + '_' + real_name[ptr - filenames]).c_str());
EXPECT_EQ(0, remove(file_name.c_str()));
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment