Merge pull request #7088 from wiryls:FileStorageJSON

modules/core/include/opencv2/core/core_c.h

@@ -1976,7 +1976,7 @@ CVAPI(void) cvSetIPLAllocators( Cv_iplCreateImageHeader create_header,
 
 The function opens file storage for reading or writing data. In the latter case, a new file is
 created or an existing file is rewritten. The type of the read or written file is determined by the
-filename extension: .xml for XML and .yml or .yaml for YAML.
+filename extension: .xml for XML, .yml or .yaml for YAML and .json for JSON.
 
 At the same time, it also supports adding parameters like "example.xml?base64". The three ways
 are the same:
@@ -2031,7 +2031,8 @@ One and only one of the two above flags must be specified
 @param type_name Optional parameter - the object type name. In
     case of XML it is written as a type_id attribute of the structure opening tag. In the case of
     YAML it is written after a colon following the structure name (see the example in
-    CvFileStorage description). Mainly it is used with user objects. When the storage is read, the
+    CvFileStorage description). In case of JSON it is written as a name/value pair.
+    Mainly it is used with user objects. When the storage is read, the
     encoded type name is used to determine the object type (see CvTypeInfo and cvFindType ).
 @param attributes This parameter is not used in the current implementation
  */
@@ -2499,7 +2500,7 @@ CVAPI(void) cvReadRawData( const CvFileStorage* fs, const CvFileNode* src,
 /** @brief Writes a file node to another file storage.
 
 The function writes a copy of a file node to file storage. Possible applications of the function are
-merging several file storages into one and conversion between XML and YAML formats.
+merging several file storages into one and conversion between XML, YAML and JSON formats.
 @param fs Destination file storage
 @param new_node_name New name of the file node in the destination file storage. To keep the
 existing name, use cvcvGetFileNodeName

modules/core/include/opencv2/core/persistence.hpp

@@ -57,8 +57,9 @@ Several functions that are described below take CvFileStorage\* as inputs and al
 save or to load hierarchical collections that consist of scalar values, standard CXCore objects
 (such as matrices, sequences, graphs), and user-defined objects.
 
-OpenCV can read and write data in XML (<http://www.w3c.org/XML>) or YAML (<http://www.yaml.org>)
-formats. Below is an example of 3x3 floating-point identity matrix A, stored in XML and YAML files
+OpenCV can read and write data in XML (<http://www.w3c.org/XML>), YAML (<http://www.yaml.org>) or
+JSON (<http://www.json.org/>) formats. Below is an example of 3x3 floating-point identity matrix A,
+stored in XML and YAML files
 using CXCore functions:
 XML:
 @code{.xml}
@@ -85,7 +86,8 @@ As it can be seen from the examples, XML uses nested tags to represent hierarchy
 indentation for that purpose (similar to the Python programming language).
 
 The same functions can read and write data in both formats; the particular format is determined by
-the extension of the opened file, ".xml" for XML files and ".yml" or ".yaml" for YAML.
+the extension of the opened file, ".xml" for XML files, ".yml" or ".yaml" for YAML and ".json" for
+JSON.
  */
 typedef struct CvFileStorage CvFileStorage;
 typedef struct CvFileNode CvFileNode;
@@ -101,20 +103,20 @@ namespace cv {
 
 /** @addtogroup core_xml
 
-XML/YAML file storages.     {#xml_storage}
+XML/YAML/JSON file storages.     {#xml_storage}
 =======================
 Writing to a file storage.
 --------------------------
-You can store and then restore various OpenCV data structures to/from XML (<http://www.w3c.org/XML>)
-or YAML (<http://www.yaml.org>) formats. Also, it is possible store and load arbitrarily complex
-data structures, which include OpenCV data structures, as well as primitive data types (integer and
-floating-point numbers and text strings) as their elements.
+You can store and then restore various OpenCV data structures to/from XML (<http://www.w3c.org/XML>),
+YAML (<http://www.yaml.org>) or JSON (<http://www.json.org/>) formats. Also, it is possible store
+and load arbitrarily complex data structures, which include OpenCV data structures, as well as
+primitive data types (integer and floating-point numbers and text strings) as their elements.
 
-Use the following procedure to write something to XML or YAML:
+Use the following procedure to write something to XML, YAML or JSON:
 -# Create new FileStorage and open it for writing. It can be done with a single call to
 FileStorage::FileStorage constructor that takes a filename, or you can use the default constructor
-and then call FileStorage::open. Format of the file (XML or YAML) is determined from the filename
-extension (".xml" and ".yml"/".yaml", respectively)
+and then call FileStorage::open. Format of the file (XML, YAML or JSON) is determined from the filename
+extension (".xml", ".yml"/".yaml" and ".json", respectively)
 -# Write all the data you want using the streaming operator `<<`, just like in the case of STL
 streams.
 -# Close the file using FileStorage::release. FileStorage destructor also closes the file.
@@ -177,19 +179,19 @@ features:
    - { x:344, y:158, lbp:[ 1, 1, 0, 0, 0, 0, 1, 0 ] }
 @endcode
 
-As an exercise, you can replace ".yml" with ".xml" in the sample above and see, how the
+As an exercise, you can replace ".yml" with ".xml" or ".json" in the sample above and see, how the
 corresponding XML file will look like.
 
 Several things can be noted by looking at the sample code and the output:
 
--   The produced YAML (and XML) consists of heterogeneous collections that can be nested. There are 2
-    types of collections: named collections (mappings) and unnamed collections (sequences). In mappings
+-   The produced YAML (and XML/JSON) consists of heterogeneous collections that can be nested. There are
+    2 types of collections: named collections (mappings) and unnamed collections (sequences). In mappings
     each element has a name and is accessed by name. This is similar to structures and std::map in
     C/C++ and dictionaries in Python. In sequences elements do not have names, they are accessed by
     indices. This is similar to arrays and std::vector in C/C++ and lists, tuples in Python.
     "Heterogeneous" means that elements of each single collection can have different types.
 
-    Top-level collection in YAML/XML is a mapping. Each matrix is stored as a mapping, and the matrix
+    Top-level collection in YAML/XML/JSON is a mapping. Each matrix is stored as a mapping, and the matrix
     elements are stored as a sequence. Then, there is a sequence of features, where each feature is
     represented a mapping, and lbp value in a nested sequence.
 
@@ -205,7 +207,7 @@ Several things can be noted by looking at the sample code and the output:
 -   To write a sequence, you first write the special string `[`, then write the elements, then
     write the closing `]`.
 
--   In YAML (but not XML), mappings and sequences can be written in a compact Python-like inline
+-   In YAML/JSON (but not XML), mappings and sequences can be written in a compact Python-like inline
     form. In the sample above matrix elements, as well as each feature, including its lbp value, is
     stored in such inline form. To store a mapping/sequence in a compact form, put `:` after the
     opening character, e.g. use `{:` instead of `{` and `[:` instead of `[`. When the
@@ -213,7 +215,7 @@ Several things can be noted by looking at the sample code and the output:
 
 Reading data from a file storage.
 ---------------------------------
-To read the previously written XML or YAML file, do the following:
+To read the previously written XML, YAML or JSON file, do the following:
 -#  Open the file storage using FileStorage::FileStorage constructor or FileStorage::open method.
     In the current implementation the whole file is parsed and the whole representation of file
     storage is built in memory as a hierarchy of file nodes (see FileNode)
@@ -294,8 +296,8 @@ A complete example using the FileStorage interface
 class CV_EXPORTS FileNode;
 class CV_EXPORTS FileNodeIterator;
 
-/** @brief XML/YAML file storage class that encapsulates all the information necessary for writing or reading
-data to/from a file.
+/** @brief XML/YAML/JSON file storage class that encapsulates all the information necessary for writing or
+reading data to/from a file.
  */
 class CV_EXPORTS_W FileStorage
 {
@@ -312,6 +314,7 @@ public:
         FORMAT_AUTO = 0,      //!< flag, auto format
         FORMAT_XML  = (1<<3), //!< flag, XML format
         FORMAT_YAML = (2<<3), //!< flag, YAML format
+        FORMAT_JSON = (3<<3), //!< flag, JSON format
 
         BASE64      = 64,     //!< flag, write rawdata in Base64 by default. (consider using WRITE_BASE64)
         WRITE_BASE64 = BASE64 | WRITE, //!< flag, enable both WRITE and BASE64
@@ -333,9 +336,9 @@ public:
 
     /** @overload
     @param source Name of the file to open or the text string to read the data from. Extension of the
-    file (.xml or .yml/.yaml) determines its format (XML or YAML respectively). Also you can append .gz
-    to work with compressed files, for example myHugeMatrix.xml.gz. If both FileStorage::WRITE and
-    FileStorage::MEMORY flags are specified, source is used just to specify the output file format (e.g.
+    file (.xml, .yml/.yaml, or .json) determines its format (XML, YAML or JSON respectively). Also you can
+    append .gz to work with compressed files, for example myHugeMatrix.xml.gz. If both FileStorage::WRITE
+    and FileStorage::MEMORY flags are specified, source is used just to specify the output file format (e.g.
     mydata.xml, .yml etc.).
     @param flags Mode of operation. See  FileStorage::Mode
     @param encoding Encoding of the file. Note that UTF-16 XML encoding is not supported currently and
@@ -354,12 +357,12 @@ public:
     See description of parameters in FileStorage::FileStorage. The method calls FileStorage::release
     before opening the file.
     @param filename Name of the file to open or the text string to read the data from.
-       Extension of the file (.xml or .yml/.yaml) determines its format (XML or YAML respectively).
-        Also you can append .gz to work with compressed files, for example myHugeMatrix.xml.gz. If both
+       Extension of the file (.xml, .yml/.yaml or .json) determines its format (XML, YAML or JSON
+        respectively). Also you can append .gz to work with compressed files, for example myHugeMatrix.xml.gz. If both
         FileStorage::WRITE and FileStorage::MEMORY flags are specified, source is used just to specify
         the output file format (e.g. mydata.xml, .yml etc.). A file name can also contain parameters.
-        You can use this format, "*?base64" (e.g. "file.xml?base64"), as an alternative to
-        FileStorage::BASE64 flag. Note: it is case sensitive.
+        You can use this format, "*?base64" (e.g. "file.json?base64" (case sensitive)), as an alternative to
+        FileStorage::BASE64 flag.
     @param flags Mode of operation. One of FileStorage::Mode
     @param encoding Encoding of the file. Note that UTF-16 XML encoding is not supported currently and
     you should use 8-bit encoding instead of it.

modules/core/include/opencv2/core/types_c.h

@@ -1669,6 +1669,7 @@ typedef struct CvFileStorage CvFileStorage;
 #define CV_STORAGE_FORMAT_AUTO   0
 #define CV_STORAGE_FORMAT_XML    8
 #define CV_STORAGE_FORMAT_YAML  16
+#define CV_STORAGE_FORMAT_JSON  24
 #define CV_STORAGE_BASE64       64
 #define CV_STORAGE_WRITE_BASE64  (CV_STORAGE_BASE64 | CV_STORAGE_WRITE)
 

modules/core/perf/perf_io_base64.cpp

@@ -11,7 +11,7 @@ typedef TestBaseWithParam<Size_MatType_Str_t> Size_Mat_StrType;
 
 #define MAT_SIZES      ::perf::sz1080p/*, ::perf::sz4320p*/
 #define MAT_TYPES      CV_8UC1, CV_32FC1
-#define FILE_EXTENSION String(".xml"), String(".yml")
+#define FILE_EXTENSION String(".xml"), String(".yml"), String(".json")
 
 
 PERF_TEST_P(Size_Mat_StrType, fs_text,

modules/core/test/test_io.cpp

@@ -91,9 +91,10 @@ protected:
             {-1000000, 1000000}, {-10, 10}, {-10, 10}};
         RNG& rng = ts->get_rng();
         RNG rng0;
-        test_case_count = 4;
         int progress = 0;
         MemStorage storage(cvCreateMemStorage(0));
+        const char * suffixs[3] = {".yml", ".xml", ".json" };
+        test_case_count = 6;
 
         for( int idx = 0; idx < test_case_count; idx++ )
         {
@@ -102,8 +103,8 @@ protected:
 
             cvClearMemStorage(storage);
 
-            bool mem = (idx % 4) >= 2;
-            string filename = tempfile(idx % 2 ? ".yml" : ".xml");
+            bool mem = (idx % test_case_count) >= (test_case_count >> 1);
+            string filename = tempfile(suffixs[idx % (test_case_count >> 1)]);
 
             FileStorage fs(filename, FileStorage::WRITE + (mem ? FileStorage::MEMORY : 0));
 
@@ -430,82 +431,91 @@ public:
 protected:
     void run(int)
     {
-        try
-        {
-            string fname = cv::tempfile(".xml");
-            vector<int> mi, mi2, mi3, mi4;
-            vector<Mat> mv, mv2, mv3, mv4;
-            vector<UserDefinedType> vudt, vudt2, vudt3, vudt4;
-            Mat m(10, 9, CV_32F);
-            Mat empty;
-            UserDefinedType udt = { 8, 3.3f };
-            randu(m, 0, 1);
-            mi3.push_back(5);
-            mv3.push_back(m);
-            vudt3.push_back(udt);
-            Point_<float> p1(1.1f, 2.2f), op1;
-            Point3i p2(3, 4, 5), op2;
-            Size s1(6, 7), os1;
-            Complex<int> c1(9, 10), oc1;
-            Rect r1(11, 12, 13, 14), or1;
-            Vec<int, 5> v1(15, 16, 17, 18, 19), ov1;
-            Scalar sc1(20.0, 21.1, 22.2, 23.3), osc1;
-            Range g1(7, 8), og1;
-
-            FileStorage fs(fname, FileStorage::WRITE);
-            fs << "mi" << mi;
-            fs << "mv" << mv;
-            fs << "mi3" << mi3;
-            fs << "mv3" << mv3;
-            fs << "vudt" << vudt;
-            fs << "vudt3" << vudt3;
-            fs << "empty" << empty;
-            fs << "p1" << p1;
-            fs << "p2" << p2;
-            fs << "s1" << s1;
-            fs << "c1" << c1;
-            fs << "r1" << r1;
-            fs << "v1" << v1;
-            fs << "sc1" << sc1;
-            fs << "g1" << g1;
-            fs.release();
+        const char * suffix[3] = {
+            ".yml",
+            ".xml",
+            ".json"
+        };
 
-            fs.open(fname, FileStorage::READ);
-            fs["mi"] >> mi2;
-            fs["mv"] >> mv2;
-            fs["mi3"] >> mi4;
-            fs["mv3"] >> mv4;
-            fs["vudt"] >> vudt2;
-            fs["vudt3"] >> vudt4;
-            fs["empty"] >> empty;
-            fs["p1"] >> op1;
-            fs["p2"] >> op2;
-            fs["s1"] >> os1;
-            fs["c1"] >> oc1;
-            fs["r1"] >> or1;
-            fs["v1"] >> ov1;
-            fs["sc1"] >> osc1;
-            fs["g1"] >> og1;
-            CV_Assert( mi2.empty() );
-            CV_Assert( mv2.empty() );
-            CV_Assert( cvtest::norm(Mat(mi3), Mat(mi4), CV_C) == 0 );
-            CV_Assert( mv4.size() == 1 );
-            double n = cvtest::norm(mv3[0], mv4[0], CV_C);
-            CV_Assert( vudt2.empty() );
-            CV_Assert( vudt3 == vudt4 );
-            CV_Assert( n == 0 );
-            CV_Assert( op1 == p1 );
-            CV_Assert( op2 == p2 );
-            CV_Assert( os1 == s1 );
-            CV_Assert( oc1 == c1 );
-            CV_Assert( or1 == r1 );
-            CV_Assert( ov1 == v1 );
-            CV_Assert( osc1 == sc1 );
-            CV_Assert( og1 == g1 );
-        }
-        catch(...)
+        for ( size_t i = 0u; i < 3u; i++ )
         {
-            ts->set_failed_test_info(cvtest::TS::FAIL_MISMATCH);
+            try
+            {
+                string fname = cv::tempfile(suffix[i]);
+                vector<int> mi, mi2, mi3, mi4;
+                vector<Mat> mv, mv2, mv3, mv4;
+                vector<UserDefinedType> vudt, vudt2, vudt3, vudt4;
+                Mat m(10, 9, CV_32F);
+                Mat empty;
+                UserDefinedType udt = { 8, 3.3f };
+                randu(m, 0, 1);
+                mi3.push_back(5);
+                mv3.push_back(m);
+                vudt3.push_back(udt);
+                Point_<float> p1(1.1f, 2.2f), op1;
+                Point3i p2(3, 4, 5), op2;
+                Size s1(6, 7), os1;
+                Complex<int> c1(9, 10), oc1;
+                Rect r1(11, 12, 13, 14), or1;
+                Vec<int, 5> v1(15, 16, 17, 18, 19), ov1;
+                Scalar sc1(20.0, 21.1, 22.2, 23.3), osc1;
+                Range g1(7, 8), og1;
+
+                FileStorage fs(fname, FileStorage::WRITE);
+                fs << "mi" << mi;
+                fs << "mv" << mv;
+                fs << "mi3" << mi3;
+                fs << "mv3" << mv3;
+                fs << "vudt" << vudt;
+                fs << "vudt3" << vudt3;
+                fs << "empty" << empty;
+                fs << "p1" << p1;
+                fs << "p2" << p2;
+                fs << "s1" << s1;
+                fs << "c1" << c1;
+                fs << "r1" << r1;
+                fs << "v1" << v1;
+                fs << "sc1" << sc1;
+                fs << "g1" << g1;
+                fs.release();
+
+                fs.open(fname, FileStorage::READ);
+                fs["mi"] >> mi2;
+                fs["mv"] >> mv2;
+                fs["mi3"] >> mi4;
+                fs["mv3"] >> mv4;
+                fs["vudt"] >> vudt2;
+                fs["vudt3"] >> vudt4;
+                fs["empty"] >> empty;
+                fs["p1"] >> op1;
+                fs["p2"] >> op2;
+                fs["s1"] >> os1;
+                fs["c1"] >> oc1;
+                fs["r1"] >> or1;
+                fs["v1"] >> ov1;
+                fs["sc1"] >> osc1;
+                fs["g1"] >> og1;
+                CV_Assert( mi2.empty() );
+                CV_Assert( mv2.empty() );
+                CV_Assert( cvtest::norm(Mat(mi3), Mat(mi4), CV_C) == 0 );
+                CV_Assert( mv4.size() == 1 );
+                double n = cvtest::norm(mv3[0], mv4[0], CV_C);
+                CV_Assert( vudt2.empty() );
+                CV_Assert( vudt3 == vudt4 );
+                CV_Assert( n == 0 );
+                CV_Assert( op1 == p1 );
+                CV_Assert( op2 == p2 );
+                CV_Assert( os1 == s1 );
+                CV_Assert( oc1 == c1 );
+                CV_Assert( or1 == r1 );
+                CV_Assert( ov1 == v1 );
+                CV_Assert( osc1 == sc1 );
+                CV_Assert( og1 == g1 );
+            }
+            catch(...)
+            {
+                ts->set_failed_test_info(cvtest::TS::FAIL_MISMATCH);
+            }
         }
     }
 };
@@ -618,6 +628,7 @@ TEST(Core_InputOutput, filestorage_base64_basic)
     char const * filenames[] = {
         "core_io_base64_basic_test.yml",
         "core_io_base64_basic_test.xml",
+        "core_io_base64_basic_test.json",
         0
     };
 
@@ -632,6 +643,8 @@ TEST(Core_InputOutput, filestorage_base64_basic)
         cv::Mat _nd_out, _nd_in;
         cv::Mat _rd_out(64, 64, CV_64FC1), _rd_in;
 
+        bool no_type_id = true;
+
         {   /* init */
 
             /* a normal mat */
@@ -692,6 +705,12 @@ TEST(Core_InputOutput, filestorage_base64_basic)
             fs["normal_nd_mat"] >> _nd_in;
             fs["random_mat"]    >> _rd_in;
 
+            if ( !fs["empty_2d_mat"]["type_id"].empty() ||
+                !fs["normal_2d_mat"]["type_id"].empty() ||
+                !fs["normal_nd_mat"]["type_id"].empty() ||
+                !fs[   "random_mat"]["type_id"].empty() )
+                no_type_id = false;
+
             /* raw data */
             std::vector<data_t>(1000).swap(rawdata);
             cvReadRawData(*fs, fs["rawdata"].node, rawdata.data(), data_t::signature());
@@ -711,6 +730,8 @@ TEST(Core_InputOutput, filestorage_base64_basic)
             //EXPECT_EQ(rawdata[i].i4, i);
         }
 
+        EXPECT_TRUE(no_type_id);
+
         EXPECT_EQ(_em_in.rows   , _em_out.rows);
         EXPECT_EQ(_em_in.cols   , _em_out.cols);
         EXPECT_EQ(_em_in.depth(), _em_out.depth());
@@ -745,15 +766,19 @@ TEST(Core_InputOutput, filestorage_base64_valid_call)
     char const * filenames[] = {
         "core_io_base64_other_test.yml",
         "core_io_base64_other_test.xml",
+        "core_io_base64_other_test.json",
         "core_io_base64_other_test.yml?base64",
         "core_io_base64_other_test.xml?base64",
+        "core_io_base64_other_test.json?base64",
         0
     };
     char const * real_name[] = {
         "core_io_base64_other_test.yml",
         "core_io_base64_other_test.xml",
+        "core_io_base64_other_test.json",
         "core_io_base64_other_test.yml",
         "core_io_base64_other_test.xml",
+        "core_io_base64_other_test.json",
         0
     };
 
@@ -829,6 +854,7 @@ TEST(Core_InputOutput, filestorage_base64_invalid_call)
     char const * filenames[] = {
         "core_io_base64_other_test.yml",
         "core_io_base64_other_test.xml",
+        "core_io_base64_other_test.json",
         0
     };
 
@@ -877,3 +903,28 @@ TEST(Core_InputOutput, filestorage_yml_vec2i)
 
     remove(file_name.c_str());
 }
+
+TEST(Core_InputOutput, filestorage_json_comment)
+{
+    String mem_str =
+        "{ /* comment */\n"
+        "  \"key\": \"value\"\n"
+        "  /************\n"
+        "   * multiline comment\n"
+        "   ************/\n"
+        "  // 233\n"
+        "  // \n"
+        "}\n"
+        ;
+
+    String str;
+
+    EXPECT_NO_THROW(
+    {
+        cv::FileStorage fs(mem_str, cv::FileStorage::READ | cv::FileStorage::MEMORY);
+        fs["key"] >> str;
+        fs.release();
+    });
+
+    EXPECT_EQ(str, String("value"));
+}

modules/ml/test/test_save_load.cpp

@@ -64,11 +64,11 @@ int CV_SLMLTest::run_test_case( int testCaseIdx )
         if( code == cvtest::TS::OK )
         {
             get_test_error( testCaseIdx, &test_resps1 );
-            fname1 = tempfile(".yml.gz");
+            fname1 = tempfile(".json.gz");
             save( (fname1 + "?base64").c_str() );
             load( fname1.c_str() );
             get_test_error( testCaseIdx, &test_resps2 );
-            fname2 = tempfile(".yml.gz");
+            fname2 = tempfile(".json.gz");
             save( (fname2 + "?base64").c_str() );
         }
         else
@@ -279,7 +279,7 @@ TEST(DISABLED_ML_SVM, linear_save_load)
 
     svm1 = Algorithm::load<SVM>("SVM45_X_38-1.xml");
     svm2 = Algorithm::load<SVM>("SVM45_X_38-2.xml");
-    string tname = tempfile("a.xml");
+    string tname = tempfile("a.json");
     svm2->save(tname + "?base64");
     svm3 = Algorithm::load<SVM>(tname);