Added .proto parsing and convertion to .fbs.

Bug: 15777858
Change-Id: Iabef9b8c8044e593bb89510feebdee00d2f1840b
Tested: on Linux and Windows.
parent 18cf19f8
...@@ -16,6 +16,7 @@ set(FlatBuffers_Compiler_SRCS ...@@ -16,6 +16,7 @@ set(FlatBuffers_Compiler_SRCS
src/idl_gen_general.cpp src/idl_gen_general.cpp
src/idl_gen_go.cpp src/idl_gen_go.cpp
src/idl_gen_text.cpp src/idl_gen_text.cpp
src/idl_gen_fbs.cpp
src/flatc.cpp src/flatc.cpp
) )
...@@ -25,6 +26,7 @@ set(FlatBuffers_Tests_SRCS ...@@ -25,6 +26,7 @@ set(FlatBuffers_Tests_SRCS
include/flatbuffers/util.h include/flatbuffers/util.h
src/idl_parser.cpp src/idl_parser.cpp
src/idl_gen_text.cpp src/idl_gen_text.cpp
src/idl_gen_fbs.cpp
tests/test.cpp tests/test.cpp
# file generate by running compiler on tests/monster_test.fbs # file generate by running compiler on tests/monster_test.fbs
${CMAKE_CURRENT_BINARY_DIR}/tests/monster_test_generated.h ${CMAKE_CURRENT_BINARY_DIR}/tests/monster_test_generated.h
......
...@@ -266,6 +266,7 @@ ...@@ -266,6 +266,7 @@
<ClInclude Include="..\..\include\flatbuffers\flatbuffers.h" /> <ClInclude Include="..\..\include\flatbuffers\flatbuffers.h" />
<ClInclude Include="..\..\include\flatbuffers\idl.h" /> <ClInclude Include="..\..\include\flatbuffers\idl.h" />
<ClInclude Include="..\..\include\flatbuffers\util.h" /> <ClInclude Include="..\..\include\flatbuffers\util.h" />
<ClCompile Include="..\..\src\idl_gen_fbs.cpp" />
<ClCompile Include="..\..\src\idl_gen_general.cpp" /> <ClCompile Include="..\..\src\idl_gen_general.cpp" />
<ClCompile Include="..\..\src\idl_gen_go.cpp"> <ClCompile Include="..\..\src\idl_gen_go.cpp">
<WarningLevel Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Level4</WarningLevel> <WarningLevel Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Level4</WarningLevel>
......
...@@ -267,6 +267,7 @@ ...@@ -267,6 +267,7 @@
<ClInclude Include="..\..\include\flatbuffers\idl.h" /> <ClInclude Include="..\..\include\flatbuffers\idl.h" />
<ClInclude Include="..\..\include\flatbuffers\util.h" /> <ClInclude Include="..\..\include\flatbuffers\util.h" />
<ClInclude Include="..\..\tests\monster_test_generated.h" /> <ClInclude Include="..\..\tests\monster_test_generated.h" />
<ClCompile Include="..\..\src\idl_gen_fbs.cpp" />
<ClCompile Include="..\..\src\idl_parser.cpp" /> <ClCompile Include="..\..\src\idl_parser.cpp" />
<ClCompile Include="..\..\src\idl_gen_text.cpp" /> <ClCompile Include="..\..\src\idl_gen_text.cpp" />
<ClCompile Include="..\..\tests\test.cpp" /> <ClCompile Include="..\..\tests\test.cpp" />
......
...@@ -61,13 +61,16 @@ $(document).ready(function(){initNavTree('md__compiler.html','');}); ...@@ -61,13 +61,16 @@ $(document).ready(function(){initNavTree('md__compiler.html','');});
<ul> <ul>
<li><code>-c</code> : Generate a C++ header for all definitions in this file (as <code>filename_generated.h</code>). Skipped for data.</li> <li><code>-c</code> : Generate a C++ header for all definitions in this file (as <code>filename_generated.h</code>). Skipped for data.</li>
<li><code>-j</code> : Generate Java classes. Skipped for data.</li> <li><code>-j</code> : Generate Java classes. Skipped for data.</li>
<li><code>-n</code> : Generate C# classes. Skipped for data.</li>
<li><code>-g</code> : Generate Go classes. Skipped for data.</li>
<li><code>-b</code> : If data is contained in this file, generate a <code>filename.bin</code> containing the binary flatbuffer.</li> <li><code>-b</code> : If data is contained in this file, generate a <code>filename.bin</code> containing the binary flatbuffer.</li>
<li><code>-t</code> : If data is contained in this file, generate a <code>filename.json</code> representing the data in the flatbuffer.</li> <li><code>-t</code> : If data is contained in this file, generate a <code>filename.json</code> representing the data in the flatbuffer.</li>
<li><code>-o PATH</code> : Output all generated files to PATH (either absolute, or relative to the current directory). If omitted, PATH will be the current directory. PATH should end in your systems path separator, e.g. <code>/</code> or <code>\</code>.</li> <li><code>-o PATH</code> : Output all generated files to PATH (either absolute, or relative to the current directory). If omitted, PATH will be the current directory. PATH should end in your systems path separator, e.g. <code>/</code> or <code>\</code>.</li>
<li><code>-I PATH</code> : when encountering <code>include</code> statements, attempt to load the files from this path. Paths will be tried in the order given, and if all fail (or none are specified) it will try to load relative to the path of the schema file being parsed.</li> <li><code>-I PATH</code> : when encountering <code>include</code> statements, attempt to load the files from this path. Paths will be tried in the order given, and if all fail (or none are specified) it will try to load relative to the path of the schema file being parsed.</li>
<li><code>-S</code> : Generate strict JSON (field names are enclosed in quotes). By default, no quotes are generated.</li> <li><code>--strict-json</code> : Generate strict JSON (field names are enclosed in quotes). By default, no quotes are generated.</li>
<li><code>-P</code> : Don't prefix enum values in generated C++ by their enum type.</li> <li><code>--no-prefix</code> : Don't prefix enum values in generated C++ by their enum type.</li>
<li><code>-H</code> : Generate include statements for included schemas the generated file depends on (C++). </li> <li><code>--gen-includes</code> : Generate include statements for included schemas the generated file depends on (C++).</li>
<li><code>--proto</code>: Expect input files to be .proto files (protocol buffers). Output the corresponding .fbs file. Currently supports: <code>package</code>, <code>message</code>, <code>enum</code>. Does not support, but will skip without error: <code>import</code>, <code>option</code>. Does not support, will generate error: <code>service</code>, <code>extend</code>, <code>extensions</code>, <code>oneof</code>, <code>group</code>, custom options, nested declarations. </li>
</ul> </ul>
</div></div><!-- contents --> </div></div><!-- contents -->
</div><!-- doc-content --> </div><!-- doc-content -->
......
...@@ -21,6 +21,10 @@ be generated for each file processed: ...@@ -21,6 +21,10 @@ be generated for each file processed:
- `-j` : Generate Java classes. Skipped for data. - `-j` : Generate Java classes. Skipped for data.
- `-n` : Generate C# classes. Skipped for data.
- `-g` : Generate Go classes. Skipped for data.
- `-b` : If data is contained in this file, generate a - `-b` : If data is contained in this file, generate a
`filename.bin` containing the binary flatbuffer. `filename.bin` containing the binary flatbuffer.
...@@ -37,10 +41,18 @@ be generated for each file processed: ...@@ -37,10 +41,18 @@ be generated for each file processed:
fail (or none are specified) it will try to load relative to the path of fail (or none are specified) it will try to load relative to the path of
the schema file being parsed. the schema file being parsed.
- `-S` : Generate strict JSON (field names are enclosed in quotes). - `--strict-json` : Generate strict JSON (field names are enclosed in quotes).
By default, no quotes are generated. By default, no quotes are generated.
- `-P` : Don't prefix enum values in generated C++ by their enum type. - `--no-prefix` : Don't prefix enum values in generated C++ by their enum
type.
- `--gen-includes` : Generate include statements for included schemas the
generated file depends on (C++).
- `-H` : Generate include statements for included schemas the generated file - `--proto`: Expect input files to be .proto files (protocol buffers).
depends on (C++). Output the corresponding .fbs file.
Currently supports: `package`, `message`, `enum`.
Does not support, but will skip without error: `import`, `option`.
Does not support, will generate error: `service`, `extend`, `extensions`,
`oneof`, `group`, custom options, nested declarations.
...@@ -260,11 +260,12 @@ struct EnumDef : public Definition { ...@@ -260,11 +260,12 @@ struct EnumDef : public Definition {
class Parser { class Parser {
public: public:
Parser() : Parser(bool proto_mode = false) :
root_struct_def(nullptr), root_struct_def(nullptr),
source_(nullptr), source_(nullptr),
cursor_(nullptr), cursor_(nullptr),
line_(1) { line_(1),
proto_mode_(proto_mode) {
// Just in case none are declared: // Just in case none are declared:
namespaces_.push_back(new Namespace()); namespaces_.push_back(new Namespace());
} }
...@@ -298,6 +299,7 @@ class Parser { ...@@ -298,6 +299,7 @@ class Parser {
void Next(); void Next();
bool IsNext(int t); bool IsNext(int t);
void Expect(int t); void Expect(int t);
void ParseTypeIdent(Type &type);
void ParseType(Type &type); void ParseType(Type &type);
FieldDef &AddField(StructDef &struct_def, FieldDef &AddField(StructDef &struct_def,
const std::string &name, const std::string &name,
...@@ -314,7 +316,11 @@ class Parser { ...@@ -314,7 +316,11 @@ class Parser {
int64_t ParseIntegerFromString(Type &type); int64_t ParseIntegerFromString(Type &type);
StructDef *LookupCreateStruct(const std::string &name); StructDef *LookupCreateStruct(const std::string &name);
void ParseEnum(bool is_union); void ParseEnum(bool is_union);
void ParseNamespace();
StructDef &StartStruct();
void ParseDecl(); void ParseDecl();
void ParseProtoDecl();
Type ParseTypeFromProtoType();
public: public:
SymbolTable<StructDef> structs_; SymbolTable<StructDef> structs_;
...@@ -333,6 +339,7 @@ class Parser { ...@@ -333,6 +339,7 @@ class Parser {
const char *source_, *cursor_; const char *source_, *cursor_;
int line_; // the current line being parsed int line_; // the current line being parsed
int token_; int token_;
bool proto_mode_;
std::string attribute_; std::string attribute_;
std::vector<std::string> doc_comment_; std::vector<std::string> doc_comment_;
...@@ -415,6 +422,16 @@ extern bool GenerateGeneral(const Parser &parser, ...@@ -415,6 +422,16 @@ extern bool GenerateGeneral(const Parser &parser,
const std::string &file_name, const std::string &file_name,
const GeneratorOptions &opts); const GeneratorOptions &opts);
// Generate a schema file from the internal representation, useful after
// parsing a .proto schema.
extern std::string GenerateFBS(const Parser &parser,
const std::string &file_name,
const GeneratorOptions &opts);
extern bool GenerateFBS(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions &opts);
} // namespace flatbuffers } // namespace flatbuffers
#endif // FLATBUFFERS_IDL_H_ #endif // FLATBUFFERS_IDL_H_
......
...@@ -60,29 +60,29 @@ struct Generator { ...@@ -60,29 +60,29 @@ struct Generator {
const std::string &path, const std::string &path,
const std::string &file_name, const std::string &file_name,
const flatbuffers::GeneratorOptions &opts); const flatbuffers::GeneratorOptions &opts);
const char *extension; const char *opt;
const char *name; const char *name;
flatbuffers::GeneratorOptions::Language lang; flatbuffers::GeneratorOptions::Language lang;
const char *help; const char *help;
}; };
const Generator generators[] = { const Generator generators[] = {
{ flatbuffers::GenerateBinary, "b", "binary", { flatbuffers::GenerateBinary, "-b", "binary",
flatbuffers::GeneratorOptions::kMAX, flatbuffers::GeneratorOptions::kMAX,
"Generate wire format binaries for any data definitions" }, "Generate wire format binaries for any data definitions" },
{ flatbuffers::GenerateTextFile, "t", "text", { flatbuffers::GenerateTextFile, "-t", "text",
flatbuffers::GeneratorOptions::kMAX, flatbuffers::GeneratorOptions::kMAX,
"Generate text output for any data definitions" }, "Generate text output for any data definitions" },
{ flatbuffers::GenerateCPP, "c", "C++", { flatbuffers::GenerateCPP, "-c", "C++",
flatbuffers::GeneratorOptions::kMAX, flatbuffers::GeneratorOptions::kMAX,
"Generate C++ headers for tables/structs" }, "Generate C++ headers for tables/structs" },
{ flatbuffers::GenerateGo, "g", "Go", { flatbuffers::GenerateGo, "-g", "Go",
flatbuffers::GeneratorOptions::kMAX, flatbuffers::GeneratorOptions::kMAX,
"Generate Go files for tables/structs" }, "Generate Go files for tables/structs" },
{ flatbuffers::GenerateGeneral, "j", "Java", { flatbuffers::GenerateGeneral, "-j", "Java",
flatbuffers::GeneratorOptions::kJava, flatbuffers::GeneratorOptions::kJava,
"Generate Java classes for tables/structs" }, "Generate Java classes for tables/structs" },
{ flatbuffers::GenerateGeneral, "n", "C#", { flatbuffers::GenerateGeneral, "-n", "C#",
flatbuffers::GeneratorOptions::kCSharp, flatbuffers::GeneratorOptions::kCSharp,
"Generate C# classes for tables/structs" } "Generate C# classes for tables/structs" }
}; };
...@@ -98,31 +98,33 @@ static void Error(const char *err, const char *obj, bool usage, ...@@ -98,31 +98,33 @@ static void Error(const char *err, const char *obj, bool usage,
if (usage) { if (usage) {
printf("usage: %s [OPTION]... FILE... [-- FILE...]\n", program_name); printf("usage: %s [OPTION]... FILE... [-- FILE...]\n", program_name);
for (size_t i = 0; i < sizeof(generators) / sizeof(generators[0]); ++i) for (size_t i = 0; i < sizeof(generators) / sizeof(generators[0]); ++i)
printf(" -%s %s.\n", generators[i].extension, generators[i].help); printf(" %s %s.\n", generators[i].opt, generators[i].help);
printf(" -o PATH Prefix PATH to all generated files.\n" printf(
" -I PATH Search for includes in the specified path.\n" " -o PATH Prefix PATH to all generated files.\n"
" -S Strict JSON: add quotes to field names.\n" " -I PATH Search for includes in the specified path.\n"
" -P Don\'t prefix enum values with the enum name in C++.\n" " --strict-json Strict JSON: add quotes to field names.\n"
" -H Generate include statements for included schemas the\n" " --no-prefix Don\'t prefix enum values with the enum type in C++.\n"
" generated file depends on (C++).\n" " --gen-includes Generate include statements for included schemas the\n"
"FILEs may depend on declarations in earlier files.\n" " generated file depends on (C++).\n"
"FILEs after the -- must be binary flatbuffer format files.\n" " --proto Input is a .proto, translate to .fbs.\n"
"Output files are named using the base file name of the input," "FILEs may depend on declarations in earlier files.\n"
"and written to the current directory or the path given by -o.\n" "FILEs after the -- must be binary flatbuffer format files.\n"
"example: %s -c -b schema1.fbs schema2.fbs data.json\n", "Output files are named using the base file name of the input,"
program_name); "and written to the current directory or the path given by -o.\n"
"example: %s -c -b schema1.fbs schema2.fbs data.json\n",
program_name);
} }
exit(1); exit(1);
} }
int main(int argc, const char *argv[]) { int main(int argc, const char *argv[]) {
program_name = argv[0]; program_name = argv[0];
flatbuffers::Parser parser;
flatbuffers::GeneratorOptions opts; flatbuffers::GeneratorOptions opts;
std::string output_path; std::string output_path;
const size_t num_generators = sizeof(generators) / sizeof(generators[0]); const size_t num_generators = sizeof(generators) / sizeof(generators[0]);
bool generator_enabled[num_generators] = { false }; bool generator_enabled[num_generators] = { false };
bool any_generator = false; bool any_generator = false;
bool proto_mode = false;
std::vector<std::string> filenames; std::vector<std::string> filenames;
std::vector<const char *> include_directories; std::vector<const char *> include_directories;
size_t binary_files_from = std::numeric_limits<size_t>::max(); size_t binary_files_from = std::numeric_limits<size_t>::max();
...@@ -131,40 +133,34 @@ int main(int argc, const char *argv[]) { ...@@ -131,40 +133,34 @@ int main(int argc, const char *argv[]) {
if (arg[0] == '-') { if (arg[0] == '-') {
if (filenames.size() && arg[1] != '-') if (filenames.size() && arg[1] != '-')
Error("invalid option location", arg, true); Error("invalid option location", arg, true);
if (strlen(arg) != 2) std::string opt = arg;
Error("invalid commandline argument", arg, true); if (opt == "-o") {
switch (arg[1]) { if (++i >= argc) Error("missing path following", arg, true);
case 'o': output_path = flatbuffers::ConCatPathFileName(argv[i], "");
if (++i >= argc) Error("missing path following", arg, true); } else if(opt == "-I") {
output_path = flatbuffers::ConCatPathFileName(argv[i], ""); if (++i >= argc) Error("missing path following", arg, true);
break; include_directories.push_back(argv[i]);
case 'I': } else if(opt == "--strict-json") {
if (++i >= argc) Error("missing path following", arg, true); opts.strict_json = true;
include_directories.push_back(argv[i]); } else if(opt == "--no-prefix") {
break; opts.prefixed_enums = false;
case 'S': } else if(opt == "--gen-includes") {
opts.strict_json = true; opts.include_dependence_headers = true;
break; } else if(opt == "--") { // Separator between text and binary inputs.
case 'P': binary_files_from = filenames.size();
opts.prefixed_enums = false; } else if(opt == "--proto") {
break; proto_mode = true;
case 'H': any_generator = true;
opts.include_dependence_headers = true; } else {
break; for (size_t i = 0; i < num_generators; ++i) {
case '-': // Separator between text and binary input files. if(opt == generators[i].opt) {
binary_files_from = filenames.size(); generator_enabled[i] = true;
break; any_generator = true;
default: goto found;
for (size_t i = 0; i < num_generators; ++i) {
if(!strcmp(arg+1, generators[i].extension)) {
generator_enabled[i] = true;
any_generator = true;
goto found;
}
} }
Error("unknown commandline argument", arg, true); }
found: Error("unknown commandline argument", arg, true);
break; found:;
} }
} else { } else {
filenames.push_back(argv[i]); filenames.push_back(argv[i]);
...@@ -178,6 +174,7 @@ int main(int argc, const char *argv[]) { ...@@ -178,6 +174,7 @@ int main(int argc, const char *argv[]) {
"specify one of -c -g -j -t -b etc.", true); "specify one of -c -g -j -t -b etc.", true);
// Now process the files: // Now process the files:
flatbuffers::Parser parser(proto_mode);
for (auto file_it = filenames.begin(); for (auto file_it = filenames.begin();
file_it != filenames.end(); file_it != filenames.end();
++file_it) { ++file_it) {
...@@ -219,6 +216,8 @@ int main(int argc, const char *argv[]) { ...@@ -219,6 +216,8 @@ int main(int argc, const char *argv[]) {
} }
} }
if (proto_mode) GenerateFBS(parser, output_path, filebase, opts);
// We do not want to generate code for the definitions in this file // We do not want to generate code for the definitions in this file
// in any files coming up next. // in any files coming up next.
parser.MarkGenerated(); parser.MarkGenerated();
......
...@@ -168,7 +168,7 @@ static void GenEnum(EnumDef &enum_def, std::string *code_ptr, ...@@ -168,7 +168,7 @@ static void GenEnum(EnumDef &enum_def, std::string *code_ptr,
// on the wrong type. // on the wrong type.
auto signature = "inline bool Verify" + enum_def.name + auto signature = "inline bool Verify" + enum_def.name +
"(flatbuffers::Verifier &verifier, " + "(flatbuffers::Verifier &verifier, " +
"const void *union_obj, uint8_t type)"; "const void *union_obj, " + enum_def.name + " type)";
code += signature + ";\n\n"; code += signature + ";\n\n";
code_post += signature + " {\n switch (type) {\n"; code_post += signature + " {\n switch (type) {\n";
for (auto it = enum_def.vals.vec.begin(); for (auto it = enum_def.vals.vec.begin();
...@@ -201,7 +201,7 @@ std::string GenUnderlyingCast(const Parser &parser, const FieldDef &field, ...@@ -201,7 +201,7 @@ std::string GenUnderlyingCast(const Parser &parser, const FieldDef &field,
// Generate an accessor struct, builder structs & function for a table. // Generate an accessor struct, builder structs & function for a table.
static void GenTable(const Parser &parser, StructDef &struct_def, static void GenTable(const Parser &parser, StructDef &struct_def,
std::string *code_ptr) { const GeneratorOptions &opts, std::string *code_ptr) {
if (struct_def.generated) return; if (struct_def.generated) return;
std::string &code = *code_ptr; std::string &code = *code_ptr;
...@@ -359,13 +359,13 @@ static void GenTable(const Parser &parser, StructDef &struct_def, ...@@ -359,13 +359,13 @@ static void GenTable(const Parser &parser, StructDef &struct_def,
code += ",\n " + GenTypeWire(parser, field.value.type, " ", true); code += ",\n " + GenTypeWire(parser, field.value.type, " ", true);
code += field.name + " = "; code += field.name + " = ";
if (field.value.type.enum_def && IsScalar(field.value.type.base_type)) { if (field.value.type.enum_def && IsScalar(field.value.type.base_type)) {
auto ed = field.value.type.enum_def->ReverseLookup( auto ev = field.value.type.enum_def->ReverseLookup(
StringToInt(field.value.constant.c_str()), false); static_cast<int>(StringToInt(field.value.constant.c_str())), false);
if (ed) { if (ev) {
code += WrapInNameSpace(parser, code += WrapInNameSpace(parser,
field.value.type.enum_def->defined_namespace, field.value.type.enum_def->defined_namespace,
field.value.type.enum_def->name + "_" + GenEnumVal(*field.value.type.enum_def, *ev,
ed->name); opts));
} else { } else {
code += GenUnderlyingCast(parser, field, true, field.value.constant); code += GenUnderlyingCast(parser, field, true, field.value.constant);
} }
...@@ -561,7 +561,7 @@ std::string GenerateCPP(const Parser &parser, ...@@ -561,7 +561,7 @@ std::string GenerateCPP(const Parser &parser,
} }
for (auto it = parser.structs_.vec.begin(); for (auto it = parser.structs_.vec.begin();
it != parser.structs_.vec.end(); ++it) { it != parser.structs_.vec.end(); ++it) {
if (!(**it).fixed) GenTable(parser, **it, &decl_code); if (!(**it).fixed) GenTable(parser, **it, opts, &decl_code);
} }
// Only output file-level code if there were any declarations. // Only output file-level code if there were any declarations.
......
/*
* Copyright 2014 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// independent from idl_parser, since this code is not needed for most clients
#include "flatbuffers/flatbuffers.h"
#include "flatbuffers/idl.h"
#include "flatbuffers/util.h"
namespace flatbuffers {
static std::string GenType(const Type &type) {
switch (type.base_type) {
case BASE_TYPE_STRUCT: return type.struct_def->name;
case BASE_TYPE_UNION: return type.enum_def->name;
case BASE_TYPE_VECTOR: return "[" + GenType(type.VectorType()) + "]";
default: return kTypeNames[type.base_type];
}
}
// Generate a flatbuffer schema from the Parser's internal representation.
std::string GenerateFBS(const Parser &parser, const std::string &file_name,
const GeneratorOptions &opts) {
std::string schema;
schema += "// Generated from " + file_name + ".proto\n\n";
if (opts.include_dependence_headers) {
int num_includes = 0;
for (auto it = parser.included_files_.begin();
it != parser.included_files_.end(); ++it) {
auto basename = flatbuffers::StripPath(
flatbuffers::StripExtension(it->first));
if (basename != file_name) {
schema += "include \"" + basename + ".fbs\";\n";
num_includes++;
}
}
if (num_includes) schema += "\n";
}
schema += "namespace ";
auto name_space = parser.namespaces_.back();
for (auto it = name_space->components.begin();
it != name_space->components.end(); ++it) {
if (it != name_space->components.begin()) schema += ".";
schema += *it;
}
schema += ";\n\n";
// Generate code for all the enum declarations.
for (auto it = parser.enums_.vec.begin();
it != parser.enums_.vec.end(); ++it) {
EnumDef &enum_def = **it;
schema += "enum " + enum_def.name + " : ";
schema += GenType(enum_def.underlying_type) + " {\n";
for (auto it = enum_def.vals.vec.begin();
it != enum_def.vals.vec.end(); ++it) {
auto &ev = **it;
schema += " " + ev.name + " = " + NumToString(ev.value) + ",\n";
}
schema += "}\n\n";
}
// Generate code for all structs/tables.
for (auto it = parser.structs_.vec.begin();
it != parser.structs_.vec.end(); ++it) {
StructDef &struct_def = **it;
schema += "table " + struct_def.name + " {\n";
for (auto it = struct_def.fields.vec.begin();
it != struct_def.fields.vec.end(); ++it) {
auto &field = **it;
schema += " " + field.name + ":" + GenType(field.value.type);
if (field.value.constant != "0") schema += " = " + field.value.constant;
if (field.required) schema += " (required)";
schema += ";\n";
}
schema += "}\n\n";
}
return schema;
}
bool GenerateFBS(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions &opts) {
return SaveFile((path + file_name + ".fbs").c_str(),
GenerateFBS(parser, file_name, opts), false);
}
} // namespace flatbuffers
...@@ -280,20 +280,24 @@ void Parser::Expect(int t) { ...@@ -280,20 +280,24 @@ void Parser::Expect(int t) {
Next(); Next();
} }
void Parser::ParseTypeIdent(Type &type) {
auto enum_def = enums_.Lookup(attribute_);
if (enum_def) {
type = enum_def->underlying_type;
if (enum_def->is_union) type.base_type = BASE_TYPE_UNION;
} else {
type.base_type = BASE_TYPE_STRUCT;
type.struct_def = LookupCreateStruct(attribute_);
}
}
// Parse any IDL type. // Parse any IDL type.
void Parser::ParseType(Type &type) { void Parser::ParseType(Type &type) {
if (token_ >= kTokenBOOL && token_ <= kTokenSTRING) { if (token_ >= kTokenBOOL && token_ <= kTokenSTRING) {
type.base_type = static_cast<BaseType>(token_ - kTokenNONE); type.base_type = static_cast<BaseType>(token_ - kTokenNONE);
} else { } else {
if (token_ == kTokenIdentifier) { if (token_ == kTokenIdentifier) {
auto enum_def = enums_.Lookup(attribute_); ParseTypeIdent(type);
if (enum_def) {
type = enum_def->underlying_type;
if (enum_def->is_union) type.base_type = BASE_TYPE_UNION;
} else {
type.base_type = BASE_TYPE_STRUCT;
type.struct_def = LookupCreateStruct(attribute_);
}
} else if (token_ == '[') { } else if (token_ == '[') {
Next(); Next();
Type subtype; Type subtype;
...@@ -374,7 +378,8 @@ void Parser::ParseField(StructDef &struct_def) { ...@@ -374,7 +378,8 @@ void Parser::ParseField(StructDef &struct_def) {
IsScalar(type.base_type) && IsScalar(type.base_type) &&
!struct_def.fixed && !struct_def.fixed &&
!type.enum_def->attributes.Lookup("bit_flags") && !type.enum_def->attributes.Lookup("bit_flags") &&
!type.enum_def->ReverseLookup(StringToInt(field.value.constant.c_str()))) !type.enum_def->ReverseLookup(static_cast<int>(
StringToInt(field.value.constant.c_str()))))
Error("enum " + type.enum_def->name + Error("enum " + type.enum_def->name +
" does not have a declaration for this field\'s default of " + " does not have a declaration for this field\'s default of " +
field.value.constant); field.value.constant);
...@@ -717,14 +722,18 @@ void Parser::ParseEnum(bool is_union) { ...@@ -717,14 +722,18 @@ void Parser::ParseEnum(bool is_union) {
enum_def.underlying_type.base_type = BASE_TYPE_UTYPE; enum_def.underlying_type.base_type = BASE_TYPE_UTYPE;
enum_def.underlying_type.enum_def = &enum_def; enum_def.underlying_type.enum_def = &enum_def;
} else { } else {
// Give specialized error message, since this type spec used to if (proto_mode_) {
// be optional in the first FlatBuffers release. enum_def.underlying_type.base_type = BASE_TYPE_SHORT;
if (!IsNext(':')) Error("must specify the underlying integer type for this" } else {
" enum (e.g. \': short\', which was the default)."); // Give specialized error message, since this type spec used to
// Specify the integer type underlying this enum. // be optional in the first FlatBuffers release.
ParseType(enum_def.underlying_type); if (!IsNext(':')) Error("must specify the underlying integer type for this"
if (!IsInteger(enum_def.underlying_type.base_type)) " enum (e.g. \': short\', which was the default).");
Error("underlying enum type must be integral"); // Specify the integer type underlying this enum.
ParseType(enum_def.underlying_type);
if (!IsInteger(enum_def.underlying_type.base_type))
Error("underlying enum type must be integral");
}
// Make this type refer back to the enum it was derived from. // Make this type refer back to the enum it was derived from.
enum_def.underlying_type.enum_def = &enum_def; enum_def.underlying_type.enum_def = &enum_def;
} }
...@@ -752,7 +761,7 @@ void Parser::ParseEnum(bool is_union) { ...@@ -752,7 +761,7 @@ void Parser::ParseEnum(bool is_union) {
if (prevsize && enum_def.vals.vec[prevsize - 1]->value >= ev.value) if (prevsize && enum_def.vals.vec[prevsize - 1]->value >= ev.value)
Error("enum values must be specified in ascending order"); Error("enum values must be specified in ascending order");
} }
} while (IsNext(',') && token_ != '}'); } while (IsNext(proto_mode_ ? ';' : ',') && token_ != '}');
Expect('}'); Expect('}');
if (enum_def.attributes.Lookup("bit_flags")) { if (enum_def.attributes.Lookup("bit_flags")) {
for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end();
...@@ -765,22 +774,27 @@ void Parser::ParseEnum(bool is_union) { ...@@ -765,22 +774,27 @@ void Parser::ParseEnum(bool is_union) {
} }
} }
void Parser::ParseDecl() { StructDef &Parser::StartStruct() {
std::vector<std::string> dc = doc_comment_;
bool fixed = IsNext(kTokenStruct);
if (!fixed) Expect(kTokenTable);
std::string name = attribute_; std::string name = attribute_;
Expect(kTokenIdentifier); Expect(kTokenIdentifier);
auto &struct_def = *LookupCreateStruct(name); auto &struct_def = *LookupCreateStruct(name);
if (!struct_def.predecl) Error("datatype already exists: " + name); if (!struct_def.predecl) Error("datatype already exists: " + name);
struct_def.predecl = false; struct_def.predecl = false;
struct_def.name = name; struct_def.name = name;
struct_def.doc_comment = dc;
struct_def.fixed = fixed;
// Move this struct to the back of the vector just in case it was predeclared, // Move this struct to the back of the vector just in case it was predeclared,
// to preserve declartion order. // to preserve declaration order.
remove(structs_.vec.begin(), structs_.vec.end(), &struct_def); remove(structs_.vec.begin(), structs_.vec.end(), &struct_def);
structs_.vec.back() = &struct_def; structs_.vec.back() = &struct_def;
return struct_def;
}
void Parser::ParseDecl() {
std::vector<std::string> dc = doc_comment_;
bool fixed = IsNext(kTokenStruct);
if (!fixed) Expect(kTokenTable);
auto &struct_def = StartStruct();
struct_def.doc_comment = dc;
struct_def.fixed = fixed;
ParseMetaData(struct_def); ParseMetaData(struct_def);
struct_def.sortbysize = struct_def.sortbysize =
struct_def.attributes.Lookup("original_order") == nullptr && !fixed; struct_def.attributes.Lookup("original_order") == nullptr && !fixed;
...@@ -875,6 +889,119 @@ void Parser::MarkGenerated() { ...@@ -875,6 +889,119 @@ void Parser::MarkGenerated() {
} }
} }
void Parser::ParseNamespace() {
Next();
auto ns = new Namespace();
namespaces_.push_back(ns);
for (;;) {
ns->components.push_back(attribute_);
Expect(kTokenIdentifier);
if (!IsNext('.')) break;
}
Expect(';');
}
// Best effort parsing of .proto declarations, with the aim to turn them
// in the closest corresponding FlatBuffer equivalent.
// We parse everything as identifiers instead of keywords, since we don't
// want protobuf keywords to become invalid identifiers in FlatBuffers.
void Parser::ParseProtoDecl() {
if (attribute_ == "package") {
// These are identical in syntax to FlatBuffer's namespace decl.
ParseNamespace();
} else if (attribute_ == "message") {
Next();
auto &struct_def = StartStruct();
Expect('{');
while (token_ != '}') {
// Parse the qualifier.
bool required = false;
bool repeated = false;
if (attribute_ == "optional") {
// This is the default.
} else if (attribute_ == "required") {
required = true;
} else if (attribute_ == "repeated") {
repeated = true;
} else {
Error("expecting optional/required/repeated, got: " + attribute_);
}
Type type = ParseTypeFromProtoType();
// Repeated elements get mapped to a vector.
if (repeated) {
type.element = type.base_type;
type.base_type = BASE_TYPE_VECTOR;
}
std::string name = attribute_;
Expect(kTokenIdentifier);
// Parse the field id. Since we're just translating schemas, not
// any kind of binary compatibility, we can safely ignore these, and
// assign our own.
Expect('=');
Expect(kTokenIntegerConstant);
auto &field = AddField(struct_def, name, type);
field.required = required;
// See if there's a default specified.
if (IsNext('[')) {
if (attribute_ != "default") Error("\'default\' expected");
Next();
Expect('=');
field.value.constant = attribute_;
Next();
Expect(']');
}
Expect(';');
}
Next();
} else if (attribute_ == "enum") {
// These are almost the same, just with different terminator:
ParseEnum(false);
} else if (attribute_ == "import") {
Next();
included_files_[attribute_] = true;
Expect(kTokenStringConstant);
Expect(';');
} else if (attribute_ == "option") { // Skip these.
Next();
Expect(kTokenIdentifier);
Expect('=');
Next(); // Any single token.
Expect(';');
} else {
Error("don\'t know how to parse .proto declaration starting with " +
attribute_);
}
}
// Parse a protobuf type, and map it to the corresponding FlatBuffer one.
Type Parser::ParseTypeFromProtoType() {
Expect(kTokenIdentifier);
struct type_lookup { const char *proto_type; BaseType fb_type; };
static type_lookup lookup[] = {
{ "float", BASE_TYPE_FLOAT }, { "double", BASE_TYPE_DOUBLE },
{ "int32", BASE_TYPE_INT }, { "int64", BASE_TYPE_LONG },
{ "uint32", BASE_TYPE_UINT }, { "uint64", BASE_TYPE_ULONG },
{ "sint32", BASE_TYPE_INT }, { "sint64", BASE_TYPE_LONG },
{ "fixed32", BASE_TYPE_UINT }, { "fixed64", BASE_TYPE_ULONG },
{ "sfixed32", BASE_TYPE_INT }, { "sfixed64", BASE_TYPE_LONG },
{ "bool", BASE_TYPE_BOOL },
{ "string", BASE_TYPE_STRING },
{ "bytes", BASE_TYPE_STRING },
{ nullptr, BASE_TYPE_NONE }
};
Type type;
for (auto tl = lookup; tl->proto_type; tl++) {
if (attribute_ == tl->proto_type) {
type.base_type = tl->fb_type;
Next();
return type;
}
}
ParseTypeIdent(type);
Expect(kTokenIdentifier);
return type;
}
bool Parser::Parse(const char *source, const char **include_paths, bool Parser::Parse(const char *source, const char **include_paths,
const char *source_filename) { const char *source_filename) {
if (source_filename) included_files_[source_filename] = true; if (source_filename) included_files_[source_filename] = true;
...@@ -922,16 +1049,10 @@ bool Parser::Parse(const char *source, const char **include_paths, ...@@ -922,16 +1049,10 @@ bool Parser::Parse(const char *source, const char **include_paths,
} }
// Now parse all other kinds of declarations: // Now parse all other kinds of declarations:
while (token_ != kTokenEof) { while (token_ != kTokenEof) {
if (token_ == kTokenNameSpace) { if (proto_mode_) {
Next(); ParseProtoDecl();
auto ns = new Namespace(); } else if (token_ == kTokenNameSpace) {
namespaces_.push_back(ns); ParseNamespace();
for (;;) {
ns->components.push_back(attribute_);
Expect(kTokenIdentifier);
if (!IsNext('.')) break;
}
Expect(';');
} else if (token_ == '{') { } else if (token_ == '{') {
if (!root_struct_def) Error("no root type set to parse json with"); if (!root_struct_def) Error("no root type set to parse json with");
if (builder_.GetSize()) { if (builder_.GetSize()) {
......
...@@ -139,7 +139,8 @@ func (rcv *Monster) TestarrayofstringLength() int { ...@@ -139,7 +139,8 @@ func (rcv *Monster) TestarrayofstringLength() int {
return 0 return 0
} }
/// an example documentation comment: this will end up in the generated code multiline too /// an example documentation comment: this will end up in the generated code
/// multiline too
func (rcv *Monster) Testarrayoftables(obj *Monster, j int) bool { func (rcv *Monster) Testarrayoftables(obj *Monster, j int) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(26)) o := flatbuffers.UOffsetT(rcv._tab.Offset(26))
if o != 0 { if o != 0 {
......
...@@ -44,7 +44,7 @@ inline const char **EnumNamesAny() { ...@@ -44,7 +44,7 @@ inline const char **EnumNamesAny() {
inline const char *EnumNameAny(Any e) { return EnumNamesAny()[e]; } inline const char *EnumNameAny(Any e) { return EnumNamesAny()[e]; }
inline bool VerifyAny(flatbuffers::Verifier &verifier, const void *union_obj, uint8_t type); inline bool VerifyAny(flatbuffers::Verifier &verifier, const void *union_obj, Any type);
MANUALLY_ALIGNED_STRUCT(2) Test { MANUALLY_ALIGNED_STRUCT(2) Test {
private: private:
...@@ -228,7 +228,7 @@ inline flatbuffers::Offset<Monster> CreateMonster(flatbuffers::FlatBufferBuilder ...@@ -228,7 +228,7 @@ inline flatbuffers::Offset<Monster> CreateMonster(flatbuffers::FlatBufferBuilder
return builder_.Finish(); return builder_.Finish();
} }
inline bool VerifyAny(flatbuffers::Verifier &verifier, const void *union_obj, uint8_t type) { inline bool VerifyAny(flatbuffers::Verifier &verifier, const void *union_obj, Any type) {
switch (type) { switch (type) {
case Any_NONE: return true; case Any_NONE: return true;
case Any_Monster: return verifier.VerifyTable(reinterpret_cast<const Monster *>(union_obj)); case Any_Monster: return verifier.VerifyTable(reinterpret_cast<const Monster *>(union_obj));
......
// Generated from test.proto
namespace proto.test;
enum ProtoEnum : short {
FOO = 1,
BAR = 5,
}
table OtherMessage {
a:double;
b:float = 3.14149;
}
table ProtoMessage {
c:int = 16;
d:long;
p:uint;
e:ulong;
f:int = -1;
g:long;
h:uint;
q:ulong;
i:int;
j:long;
k:bool;
l:string (required);
m:string;
n:OtherMessage;
o:[string];
}
// Sample .proto file that we can translate to the corresponding .fbs.
package proto.test;
option some_option = is_ignored;
import "some_other_schema.proto";
enum ProtoEnum {
FOO = 1;
BAR = 5;
}
message OtherMessage {
optional double a = 26;
optional float b = 32 [default = 3.14149];
}
message ProtoMessage {
optional int32 c = 12 [default = 16];
optional int64 d = 1 [default = 0];
optional uint32 p = 1;
optional uint64 e = 2;
optional sint32 f = 3 [default = -1];
optional sint64 g = 4;
optional fixed32 h = 5;
optional fixed64 q = 6;
optional sfixed32 i = 7;
optional sfixed64 j = 8;
optional bool k = 9;
required string l = 10;
optional bytes m = 11;
optional OtherMessage n = 12;
repeated string o = 14;
}
...@@ -182,7 +182,6 @@ void AccessFlatBufferTest(const std::string &flatbuf) { ...@@ -182,7 +182,6 @@ void AccessFlatBufferTest(const std::string &flatbuf) {
// example of parsing text straight into a buffer, and generating // example of parsing text straight into a buffer, and generating
// text back from it: // text back from it:
void ParseAndGenerateTextTest() { void ParseAndGenerateTextTest() {
// load FlatBuffer schema (.fbs) and JSON from disk // load FlatBuffer schema (.fbs) and JSON from disk
std::string schemafile; std::string schemafile;
std::string jsonfile; std::string jsonfile;
...@@ -216,6 +215,34 @@ void ParseAndGenerateTextTest() { ...@@ -216,6 +215,34 @@ void ParseAndGenerateTextTest() {
} }
} }
// Parse a .proto schema, output as .fbs
void ParseProtoTest() {
// load the .proto and the golden file from disk
std::string protofile;
std::string goldenfile;
TEST_EQ(flatbuffers::LoadFile(
"tests/prototest/test.proto", false, &protofile), true);
TEST_EQ(flatbuffers::LoadFile(
"tests/prototest/test.golden", false, &goldenfile), true);
// Parse proto.
flatbuffers::Parser parser(true);
TEST_EQ(parser.Parse(protofile.c_str(), nullptr), true);
// Generate fbs.
flatbuffers::GeneratorOptions opts;
auto fbs = flatbuffers::GenerateFBS(parser, "test", opts);
// Ensure generated file is parsable.
flatbuffers::Parser parser2;
TEST_EQ(parser2.Parse(fbs.c_str(), nullptr), true);
if (fbs != goldenfile) {
printf("%s----------------\n%s", fbs.c_str(), goldenfile.c_str());
TEST_NOTNULL(NULL);
}
}
template<typename T> void CompareTableFieldValue(flatbuffers::Table *table, template<typename T> void CompareTableFieldValue(flatbuffers::Table *table,
flatbuffers::voffset_t voffset, flatbuffers::voffset_t voffset,
T val) { T val) {
...@@ -544,6 +571,7 @@ int main(int /*argc*/, const char * /*argv*/[]) { ...@@ -544,6 +571,7 @@ int main(int /*argc*/, const char * /*argv*/[]) {
#ifndef __ANDROID__ // requires file access #ifndef __ANDROID__ // requires file access
ParseAndGenerateTextTest(); ParseAndGenerateTextTest();
ParseProtoTest();
#endif #endif
FuzzTest1(); FuzzTest1();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment