Skip to content

F
flatbuffers
Commit 40a33b1d authored by Wouter van Oortmerssen's avatar Wouter van Oortmerssen
Browse files
Options

Replaced exception handling in the parser with error checking. 

This to allow the code to run on a greater range of build
configurations (that don't allow exceptions/RTTI).

If anyone ever doubts the usefulness of exception handling,
please show them this commit.

Change-Id: If7190babdde93c3f9cd97b8e1ab447bf0c81696d
Tested: on Linux.
parent a9909a0c
Hide whitespace changes
Inline Side-by-side
Showing with 704 additions and 534 deletions
CMakeLists.txt
View file @ 40a33b1d
...@@ -81,7 +81,7 @@ if(APPLE) ...@@ -81,7 +81,7 @@ if(APPLE)
"${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -Wall -pedantic -Werror -Wextra") "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -Wall -pedantic -Werror -Wextra")
elseif(CMAKE_COMPILER_IS_GNUCXX) elseif(CMAKE_COMPILER_IS_GNUCXX)
set(CMAKE_CXX_FLAGS set(CMAKE_CXX_FLAGS
"${CMAKE_CXX_FLAGS} -std=c++0x -Wall -pedantic -Werror -Wextra -Werror=shadow") "${CMAKE_CXX_FLAGS} -std=c++0x -Wall -pedantic -Werror -Wextra -Werror=shadow -Wunused-result -Werror=unused-result")
elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
set(CMAKE_CXX_FLAGS set(CMAKE_CXX_FLAGS
"${CMAKE_CXX_FLAGS} -std=c++0x -stdlib=libc++ -Wall -pedantic -Werror -Wextra") "${CMAKE_CXX_FLAGS} -std=c++0x -stdlib=libc++ -Wall -pedantic -Werror -Wextra")
......
include/flatbuffers/idl.h
View file @ 40a33b1d
...@@ -340,6 +340,46 @@ struct IDLOptions { ...@@ -340,6 +340,46 @@ struct IDLOptions {
lang(IDLOptions::kJava) {} lang(IDLOptions::kJava) {}
}; };
// A way to make error propagation less error prone by requiring values to be
// checked.
// Once you create a value of this type you must either:
// - Call Check() on it.
// - Copy or assign it to another value.
// Failure to do so leads to an assert.
// This guarantees that this as return value cannot be ignored.
class CheckedError {
public:
explicit CheckedError(bool error)
: is_error_(error), has_been_checked_(false) {}
CheckedError &operator=(const CheckedError &other) {
is_error_ = other.is_error_;
has_been_checked_ = false;
other.has_been_checked_ = true;
return *this;
}
CheckedError(const CheckedError &other) {
*this = other; // Use assignment operator.
}
~CheckedError() { assert(has_been_checked_); }
bool Check() { has_been_checked_ = true; return is_error_; }
private:
bool is_error_;
mutable bool has_been_checked_;
};
// Additionally, in GCC we can get these errors statically, for additional
// assurance:
#ifdef __GNUC__
#define CHECKED_ERROR CheckedError __attribute__((warn_unused_result))
#else
#define CHECKED_ERROR CheckedError
#endif
class Parser { class Parser {
public: public:
explicit Parser(const IDLOptions &options = IDLOptions()) explicit Parser(const IDLOptions &options = IDLOptions())
...@@ -395,44 +435,51 @@ class Parser { ...@@ -395,44 +435,51 @@ class Parser {
// See reflection/reflection.fbs // See reflection/reflection.fbs
void Serialize(); void Serialize();
private: CHECKED_ERROR CheckBitsFit(int64_t val, size_t bits);
int64_t ParseHexNum(int nibbles);
void Next(); private:
bool IsNext(int t); CHECKED_ERROR Error(const std::string &msg);
void Expect(int t); CHECKED_ERROR ParseHexNum(int nibbles, int64_t *val);
CHECKED_ERROR Next();
bool Is(int t);
CHECKED_ERROR Expect(int t);
std::string TokenToStringId(int t); std::string TokenToStringId(int t);
EnumDef *LookupEnum(const std::string &id); EnumDef *LookupEnum(const std::string &id);
void ParseNamespacing(std::string *id, std::string *last); CHECKED_ERROR ParseNamespacing(std::string *id, std::string *last);
void ParseTypeIdent(Type &type); CHECKED_ERROR ParseTypeIdent(Type &type);
void ParseType(Type &type); CHECKED_ERROR ParseType(Type &type);
FieldDef &AddField(StructDef &struct_def, CHECKED_ERROR AddField(StructDef &struct_def, const std::string &name,
const std::string &name, const Type &type, FieldDef **dest);
const Type &type); CHECKED_ERROR ParseField(StructDef &struct_def);
void ParseField(StructDef &struct_def); CHECKED_ERROR ParseAnyValue(Value &val, FieldDef *field, size_t parent_fieldn);
void ParseAnyValue(Value &val, FieldDef *field, size_t parent_fieldn); CHECKED_ERROR ParseTable(const StructDef &struct_def, std::string *value,
uoffset_t ParseTable(const StructDef &struct_def, std::string *value); uoffset_t *ovalue);
void SerializeStruct(const StructDef &struct_def, const Value &val); void SerializeStruct(const StructDef &struct_def, const Value &val);
void AddVector(bool sortbysize, int count); void AddVector(bool sortbysize, int count);
uoffset_t ParseVector(const Type &type); CHECKED_ERROR ParseVector(const Type &type, uoffset_t *ovalue);
void ParseMetaData(Definition &def); CHECKED_ERROR ParseMetaData(Definition &def);
bool TryTypedValue(int dtoken, bool check, Value &e, BaseType req); CHECKED_ERROR TryTypedValue(int dtoken, bool check, Value &e, BaseType req,
void ParseHash(Value &e, FieldDef* field); bool *destmatch);
void ParseSingleValue(Value &e); CHECKED_ERROR ParseHash(Value &e, FieldDef* field);
int64_t ParseIntegerFromString(Type &type); CHECKED_ERROR ParseSingleValue(Value &e);
CHECKED_ERROR ParseIntegerFromString(Type &type, int64_t *result);
StructDef *LookupCreateStruct(const std::string &name, StructDef *LookupCreateStruct(const std::string &name,
bool create_if_new = true, bool create_if_new = true,
bool definition = false); bool definition = false);
EnumDef &ParseEnum(bool is_union); CHECKED_ERROR ParseEnum(bool is_union, EnumDef **dest);
void ParseNamespace(); CHECKED_ERROR ParseNamespace();
StructDef &StartStruct(const std::string &name); CHECKED_ERROR StartStruct(const std::string &name, StructDef **dest);
void ParseDecl(); CHECKED_ERROR ParseDecl();
void ParseProtoFields(StructDef *struct_def, bool isextend, CHECKED_ERROR ParseProtoFields(StructDef *struct_def, bool isextend,
bool inside_oneof); bool inside_oneof);
void ParseProtoOption(); CHECKED_ERROR ParseProtoOption();
void ParseProtoKey(); CHECKED_ERROR ParseProtoKey();
void ParseProtoDecl(); CHECKED_ERROR ParseProtoDecl();
void ParseProtoCurliesOrIdent(); CHECKED_ERROR ParseProtoCurliesOrIdent();
Type ParseTypeFromProtoType(); CHECKED_ERROR ParseTypeFromProtoType(Type *type);
CHECKED_ERROR DoParse(const char *_source, const char **include_paths,
const char *source_filename);
public: public:
SymbolTable<StructDef> structs_; SymbolTable<StructDef> structs_;
...@@ -454,7 +501,7 @@ class Parser { ...@@ -454,7 +501,7 @@ class Parser {
const char *source_, *cursor_; const char *source_, *cursor_;
int line_; // the current line being parsed int line_; // the current line being parsed
int token_; int token_;
std::string files_being_parsed_; std::string file_being_parsed_;
std::string attribute_; std::string attribute_;
std::vector<std::string> doc_comment_; std::vector<std::string> doc_comment_;
......
src/idl_parser.cpp
View file @ 40a33b1d
...@@ -43,38 +43,68 @@ static_assert(BASE_TYPE_UNION == ...@@ -43,38 +43,68 @@ static_assert(BASE_TYPE_UNION ==
static_cast<BaseType>(reflection::Union), static_cast<BaseType>(reflection::Union),
"enums don't match"); "enums don't match");
static void Error(const std::string &msg) { #define ECHECK(call) { auto ce = (call); if (ce.Check()) return ce; }
throw msg; #define NEXT() ECHECK(Next())
#define EXPECT(tok) ECHECK(Expect(tok))
CheckedError Parser::Error(const std::string &msg) {
error_ = file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : "";
#ifdef _WIN32
error_ += "(" + NumToString(line_) + ")"; // MSVC alike
#else
if (file_being_parsed_.length()) error_ += ":";
error_ += NumToString(line_) + ":0"; // gcc alike
#endif
error_ += ": error: " + msg;
return CheckedError(true);
} }
inline CheckedError NoError() { return CheckedError(false); }
// Ensure that integer values we parse fit inside the declared integer type. // Ensure that integer values we parse fit inside the declared integer type.
static void CheckBitsFit(int64_t val, size_t bits) { CheckedError Parser::CheckBitsFit(int64_t val, size_t bits) {
// Bits we allow to be used. // Bits we allow to be used.
auto mask = static_cast<int64_t>((1ull << bits) - 1); auto mask = static_cast<int64_t>((1ull << bits) - 1);
if (bits < 64 && if (bits < 64 &&
(val & ~mask) != 0 && // Positive or unsigned. (val & ~mask) != 0 && // Positive or unsigned.
(val | mask) != -1) // Negative. (val | mask) != -1) // Negative.
Error("constant does not fit in a " + NumToString(bits) + "-bit field"); return Error("constant does not fit in a " + NumToString(bits) +
"-bit field");
return NoError();
} }
// atot: templated version of atoi/atof: convert a string to an instance of T. // atot: templated version of atoi/atof: convert a string to an instance of T.
template<typename T> inline T atot(const char *s) { template<typename T> inline CheckedError atot(const char *s, Parser &parser,
auto val = StringToInt(s); T *val) {
CheckBitsFit(val, sizeof(T) * 8); int64_t i = StringToInt(s);
return (T)val; ECHECK(parser.CheckBitsFit(i, sizeof(T) * 8));
*val = (T)i;
return NoError();
} }
template<> inline bool atot<bool>(const char *s) { template<> inline CheckedError atot<bool>(const char *s, Parser &parser,
return 0 != atoi(s); bool *val) {
(void)parser;
*val = 0 != atoi(s);
return NoError();
} }
template<> inline float atot<float>(const char *s) { template<> inline CheckedError atot<float>(const char *s, Parser &parser,
return static_cast<float>(strtod(s, nullptr)); float *val) {
(void)parser;
*val = static_cast<float>(strtod(s, nullptr));
return NoError();
} }
template<> inline double atot<double>(const char *s) { template<> inline CheckedError atot<double>(const char *s, Parser &parser,
return strtod(s, nullptr); double *val) {
(void)parser;
*val = strtod(s, nullptr);
return NoError();
} }
template<> inline Offset<void> atot<Offset<void>>(const char *s) { template<> inline CheckedError atot<Offset<void>>(const char *s, Parser &parser,
return Offset<void>(atoi(s)); Offset<void> *val) {
(void)parser;
*val = Offset<void>(atoi(s));
return NoError();
} }
std::string Namespace::GetFullyQualifiedName(const std::string &name, std::string Namespace::GetFullyQualifiedName(const std::string &name,
...@@ -153,18 +183,18 @@ std::string Parser::TokenToStringId(int t) { ...@@ -153,18 +183,18 @@ std::string Parser::TokenToStringId(int t) {
} }
// Parses exactly nibbles worth of hex digits into a number, or error. // Parses exactly nibbles worth of hex digits into a number, or error.
int64_t Parser::ParseHexNum(int nibbles) { CheckedError Parser::ParseHexNum(int nibbles, int64_t *val) {
for (int i = 0; i < nibbles; i++) for (int i = 0; i < nibbles; i++)
if (!isxdigit(cursor_[i])) if (!isxdigit(cursor_[i]))
Error("escape code must be followed by " + NumToString(nibbles) + return Error("escape code must be followed by " + NumToString(nibbles) +
" hex digits"); " hex digits");
std::string target(cursor_, cursor_ + nibbles); std::string target(cursor_, cursor_ + nibbles);
auto val = StringToUInt(target.c_str(), 16); *val = StringToUInt(target.c_str(), 16);
cursor_ += nibbles; cursor_ += nibbles;
return val; return NoError();
} }
void Parser::Next() { CheckedError Parser::Next() {
doc_comment_.clear(); doc_comment_.clear();
bool seen_newline = false; bool seen_newline = false;
attribute_.clear(); attribute_.clear();
...@@ -172,20 +202,19 @@ void Parser::Next() { ...@@ -172,20 +202,19 @@ void Parser::Next() {
char c = *cursor_++; char c = *cursor_++;
token_ = c; token_ = c;
switch (c) { switch (c) {
case '\0': cursor_--; token_ = kTokenEof; return; case '\0': cursor_--; token_ = kTokenEof; return NoError();
case ' ': case '\r': case '\t': break; case ' ': case '\r': case '\t': break;
case '\n': line_++; seen_newline = true; break; case '\n': line_++; seen_newline = true; break;
case '{': case '}': case '(': case ')': case '[': case ']': return; case '{': case '}': case '(': case ')': case '[': case ']':
case ',': case ':': case ';': case '=': return; case ',': case ':': case ';': case '=': return NoError();
case '.': case '.':
if(!isdigit(*cursor_)) return; if(!isdigit(*cursor_)) return NoError();
Error("floating point constant can\'t start with \".\""); return Error("floating point constant can\'t start with \".\"");
break;
case '\"': case '\"':
case '\'': case '\'':
while (*cursor_ != c) { while (*cursor_ != c) {
if (*cursor_ < ' ' && *cursor_ >= 0) if (*cursor_ < ' ' && *cursor_ >= 0)
Error("illegal character in string constant"); return Error("illegal character in string constant");
if (*cursor_ == '\\') { if (*cursor_ == '\\') {
cursor_++; cursor_++;
switch (*cursor_) { switch (*cursor_) {
...@@ -200,15 +229,19 @@ void Parser::Next() { ...@@ -200,15 +229,19 @@ void Parser::Next() {
case '/': attribute_ += '/'; cursor_++; break; case '/': attribute_ += '/'; cursor_++; break;
case 'x': { // Not in the JSON standard case 'x': { // Not in the JSON standard
cursor_++; cursor_++;
attribute_ += static_cast<char>(ParseHexNum(2)); int64_t val;
ECHECK(ParseHexNum(2, &val));
attribute_ += static_cast<char>(val);
break; break;
} }
case 'u': { case 'u': {
cursor_++; cursor_++;
ToUTF8(static_cast<int>(ParseHexNum(4)), &attribute_); int64_t val;
ECHECK(ParseHexNum(4, &val));
ToUTF8(static_cast<int>(val), &attribute_);
break; break;
} }
default: Error("unknown escape code in string constant"); break; default: return Error("unknown escape code in string constant");
} }
} else { // printable chars + UTF-8 bytes } else { // printable chars + UTF-8 bytes
attribute_ += *cursor_++; attribute_ += *cursor_++;
...@@ -216,14 +249,15 @@ void Parser::Next() { ...@@ -216,14 +249,15 @@ void Parser::Next() {
} }
cursor_++; cursor_++;
token_ = kTokenStringConstant; token_ = kTokenStringConstant;
return; return NoError();
case '/': case '/':
if (*cursor_ == '/') { if (*cursor_ == '/') {
const char *start = ++cursor_; const char *start = ++cursor_;
while (*cursor_ && *cursor_ != '\n' && *cursor_ != '\r') cursor_++; while (*cursor_ && *cursor_ != '\n' && *cursor_ != '\r') cursor_++;
if (*start == '/') { // documentation comment if (*start == '/') { // documentation comment
if (cursor_ != source_ && !seen_newline) if (cursor_ != source_ && !seen_newline)
Error("a documentation comment should be on a line on its own"); return Error(
"a documentation comment should be on a line on its own");
doc_comment_.push_back(std::string(start + 1, cursor_)); doc_comment_.push_back(std::string(start + 1, cursor_));
} }
break; break;
...@@ -231,7 +265,7 @@ void Parser::Next() { ...@@ -231,7 +265,7 @@ void Parser::Next() {
cursor_++; cursor_++;
// TODO: make nested. // TODO: make nested.
while (*cursor_ != '*' || cursor_[1] != '/') { while (*cursor_ != '*' || cursor_[1] != '/') {
if (!*cursor_) Error("end of file in comment"); if (!*cursor_) return Error("end of file in comment");
cursor_++; cursor_++;
} }
cursor_ += 2; cursor_ += 2;
...@@ -251,7 +285,7 @@ void Parser::Next() { ...@@ -251,7 +285,7 @@ void Parser::Next() {
PTYPE) \ PTYPE) \
if (attribute_ == IDLTYPE) { \ if (attribute_ == IDLTYPE) { \
token_ = kToken ## ENUM; \ token_ = kToken ## ENUM; \
return; \ return NoError(); \
} }
FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
#undef FLATBUFFERS_TD #undef FLATBUFFERS_TD
...@@ -260,28 +294,52 @@ void Parser::Next() { ...@@ -260,28 +294,52 @@ void Parser::Next() {
if (attribute_ == "true" || attribute_ == "false") { if (attribute_ == "true" || attribute_ == "false") {
attribute_ = NumToString(attribute_ == "true"); attribute_ = NumToString(attribute_ == "true");
token_ = kTokenIntegerConstant; token_ = kTokenIntegerConstant;
return; return NoError();
} }
// Check for declaration keywords: // Check for declaration keywords:
if (attribute_ == "table") { token_ = kTokenTable; return; } if (attribute_ == "table") {
if (attribute_ == "struct") { token_ = kTokenStruct; return; } token_ = kTokenTable;
if (attribute_ == "enum") { token_ = kTokenEnum; return; } return NoError();
if (attribute_ == "union") { token_ = kTokenUnion; return; } }
if (attribute_ == "namespace") { token_ = kTokenNameSpace; return; } if (attribute_ == "struct") {
if (attribute_ == "root_type") { token_ = kTokenRootType; return; } token_ = kTokenStruct;
if (attribute_ == "include") { token_ = kTokenInclude; return; } return NoError();
if (attribute_ == "attribute") { token_ = kTokenAttribute; return; } }
if (attribute_ == "enum") {
token_ = kTokenEnum;
return NoError();
}
if (attribute_ == "union") {
token_ = kTokenUnion;
return NoError();
}
if (attribute_ == "namespace") {
token_ = kTokenNameSpace;
return NoError();
}
if (attribute_ == "root_type") {
token_ = kTokenRootType;
return NoError();
}
if (attribute_ == "include") {
token_ = kTokenInclude;
return NoError();
}
if (attribute_ == "attribute") {
token_ = kTokenAttribute;
return NoError();
}
if (attribute_ == "file_identifier") { if (attribute_ == "file_identifier") {
token_ = kTokenFileIdentifier; token_ = kTokenFileIdentifier;
return; return NoError();
} }
if (attribute_ == "file_extension") { if (attribute_ == "file_extension") {
token_ = kTokenFileExtension; token_ = kTokenFileExtension;
return; return NoError();
} }
// If not, it is a user-defined identifier: // If not, it is a user-defined identifier:
token_ = kTokenIdentifier; token_ = kTokenIdentifier;
return; return NoError();
} else if (isdigit(static_cast<unsigned char>(c)) || c == '-') { } else if (isdigit(static_cast<unsigned char>(c)) || c == '-') {
const char *start = cursor_ - 1; const char *start = cursor_ - 1;
if (c == '0' && (*cursor_ == 'x' || *cursor_ == 'X')) { if (c == '0' && (*cursor_ == 'x' || *cursor_ == 'X')) {
...@@ -290,7 +348,7 @@ void Parser::Next() { ...@@ -290,7 +348,7 @@ void Parser::Next() {
attribute_.append(start + 2, cursor_); attribute_.append(start + 2, cursor_);
attribute_ = NumToString(StringToUInt(attribute_.c_str(), 16)); attribute_ = NumToString(StringToUInt(attribute_.c_str(), 16));
token_ = kTokenIntegerConstant; token_ = kTokenIntegerConstant;
return; return NoError();
} }
while (isdigit(static_cast<unsigned char>(*cursor_))) cursor_++; while (isdigit(static_cast<unsigned char>(*cursor_))) cursor_++;
if (*cursor_ == '.' || *cursor_ == 'e' || *cursor_ == 'E') { if (*cursor_ == '.' || *cursor_ == 'e' || *cursor_ == 'E') {
...@@ -310,56 +368,57 @@ void Parser::Next() { ...@@ -310,56 +368,57 @@ void Parser::Next() {
token_ = kTokenIntegerConstant; token_ = kTokenIntegerConstant;
} }
attribute_.append(start, cursor_); attribute_.append(start, cursor_);
return; return NoError();
} }
std::string ch; std::string ch;
ch = c; ch = c;
if (c < ' ' || c > '~') ch = "code: " + NumToString(c); if (c < ' ' || c > '~') ch = "code: " + NumToString(c);
Error("illegal character: " + ch); return Error("illegal character: " + ch);
break;
} }
} }
} }
// Check if a given token is next, if so, consume it as well. // Check if a given token is next.
bool Parser::IsNext(int t) { bool Parser::Is(int t) {
bool isnext = t == token_; return t == token_;
if (isnext) Next();
return isnext;
} }
// Expect a given token to be next, consume it, or error if not present. // Expect a given token to be next, consume it, or error if not present.
void Parser::Expect(int t) { CheckedError Parser::Expect(int t) {
if (t != token_) { if (t != token_) {
Error("expecting: " + TokenToString(t) + " instead got: " + return Error("expecting: " + TokenToString(t) + " instead got: " +
TokenToStringId(token_)); TokenToStringId(token_));
} }
Next(); NEXT();
return NoError();
} }
void Parser::ParseNamespacing(std::string *id, std::string *last) { CheckedError Parser::ParseNamespacing(std::string *id, std::string *last) {
while (IsNext('.')) { while (Is('.')) {
NEXT();
*id += "."; *id += ".";
*id += attribute_; *id += attribute_;
if (last) *last = attribute_; if (last) *last = attribute_;
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
} }
return NoError();
} }
EnumDef *Parser::LookupEnum(const std::string &id) { EnumDef *Parser::LookupEnum(const std::string &id) {
// Search thru parent namespaces. // Search thru parent namespaces.
for (int components = static_cast<int>(namespaces_.back()->components.size()); for (int components = static_cast<int>(namespaces_.back()->components.size());
components >= 0; components--) { components >= 0; components--) {
auto ed = enums_.Lookup(namespaces_.back()->GetFullyQualifiedName(id, components)); auto ed = enums_.Lookup(
namespaces_.back()->GetFullyQualifiedName(id, components));
if (ed) return ed; if (ed) return ed;
} }
return nullptr; return nullptr;
} }
void Parser::ParseTypeIdent(Type &type) { CheckedError Parser::ParseTypeIdent(Type &type) {
std::string id = attribute_; std::string id = attribute_;
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
ParseNamespacing(&id, nullptr); ECHECK(ParseNamespacing(&id, nullptr));
auto enum_def = LookupEnum(id); auto enum_def = LookupEnum(id);
if (enum_def) { if (enum_def) {
type = enum_def->underlying_type; type = enum_def->underlying_type;
...@@ -368,41 +427,45 @@ void Parser::ParseTypeIdent(Type &type) { ...@@ -368,41 +427,45 @@ void Parser::ParseTypeIdent(Type &type) {
type.base_type = BASE_TYPE_STRUCT; type.base_type = BASE_TYPE_STRUCT;
type.struct_def = LookupCreateStruct(id); type.struct_def = LookupCreateStruct(id);
} }
return NoError();
} }
// Parse any IDL type. // Parse any IDL type.
void Parser::ParseType(Type &type) { CheckedError Parser::ParseType(Type &type) {
if (token_ >= kTokenBOOL && token_ <= kTokenSTRING) { if (token_ >= kTokenBOOL && token_ <= kTokenSTRING) {
type.base_type = static_cast<BaseType>(token_ - kTokenNONE); type.base_type = static_cast<BaseType>(token_ - kTokenNONE);
Next(); NEXT();
} else { } else {
if (token_ == kTokenIdentifier) { if (token_ == kTokenIdentifier) {
ParseTypeIdent(type); ECHECK(ParseTypeIdent(type));
} else if (token_ == '[') { } else if (token_ == '[') {
Next(); NEXT();
Type subtype; Type subtype;
ParseType(subtype); ECHECK(ParseType(subtype));
if (subtype.base_type == BASE_TYPE_VECTOR) { if (subtype.base_type == BASE_TYPE_VECTOR) {
// We could support this, but it will complicate things, and it's // We could support this, but it will complicate things, and it's
// easier to work around with a struct around the inner vector. // easier to work around with a struct around the inner vector.
Error("nested vector types not supported (wrap in table first)."); return Error(
"nested vector types not supported (wrap in table first).");
} }
if (subtype.base_type == BASE_TYPE_UNION) { if (subtype.base_type == BASE_TYPE_UNION) {
// We could support this if we stored a struct of 2 elements per // We could support this if we stored a struct of 2 elements per
// union element. // union element.
Error("vector of union types not supported (wrap in table first)."); return Error(
"vector of union types not supported (wrap in table first).");
} }
type = Type(BASE_TYPE_VECTOR, subtype.struct_def, subtype.enum_def); type = Type(BASE_TYPE_VECTOR, subtype.struct_def, subtype.enum_def);
type.element = subtype.base_type; type.element = subtype.base_type;
Expect(']'); EXPECT(']');
} else { } else {
Error("illegal type syntax"); return Error("illegal type syntax");
} }
} }
return NoError();
} }
FieldDef &Parser::AddField(StructDef &struct_def, const std::string &name, CheckedError Parser::AddField(StructDef &struct_def, const std::string &name,
const Type &type) { const Type &type, FieldDef **dest) {
auto &field = *new FieldDef(); auto &field = *new FieldDef();
field.value.offset = field.value.offset =
FieldIndexToOffset(static_cast<voffset_t>(struct_def.fields.vec.size())); FieldIndexToOffset(static_cast<voffset_t>(struct_def.fields.vec.size()));
...@@ -420,36 +483,38 @@ FieldDef &Parser::AddField(StructDef &struct_def, const std::string &name, ...@@ -420,36 +483,38 @@ FieldDef &Parser::AddField(StructDef &struct_def, const std::string &name,
struct_def.bytesize += size; struct_def.bytesize += size;
} }
if (struct_def.fields.Add(name, &field)) if (struct_def.fields.Add(name, &field))
Error("field already exists: " + name); return Error("field already exists: " + name);
return field; *dest = &field;
return NoError();
} }
void Parser::ParseField(StructDef &struct_def) { CheckedError Parser::ParseField(StructDef &struct_def) {
std::string name = attribute_; std::string name = attribute_;
std::vector<std::string> dc = doc_comment_; std::vector<std::string> dc = doc_comment_;
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
Expect(':'); EXPECT(':');
Type type; Type type;
ParseType(type); ECHECK(ParseType(type));
if (struct_def.fixed && !IsScalar(type.base_type) && !IsStruct(type)) if (struct_def.fixed && !IsScalar(type.base_type) && !IsStruct(type))
Error("structs_ may contain only scalar or struct fields"); return Error("structs_ may contain only scalar or struct fields");
FieldDef *typefield = nullptr; FieldDef *typefield = nullptr;
if (type.base_type == BASE_TYPE_UNION) { if (type.base_type == BASE_TYPE_UNION) {
// For union fields, add a second auto-generated field to hold the type, // For union fields, add a second auto-generated field to hold the type,
// with _type appended as the name. // with _type appended as the name.
typefield = &AddField(struct_def, name + "_type", ECHECK(AddField(struct_def, name + "_type", type.enum_def->underlying_type,
type.enum_def->underlying_type); &typefield));
} }
auto &field = AddField(struct_def, name, type); FieldDef *field;
ECHECK(AddField(struct_def, name, type, &field));
if (token_ == '=') { if (token_ == '=') {
Next(); NEXT();
if (!IsScalar(type.base_type)) if (!IsScalar(type.base_type))
Error("default values currently only supported for scalars"); return Error("default values currently only supported for scalars");
ParseSingleValue(field.value); ECHECK(ParseSingleValue(field->value));
} }
if (type.enum_def && if (type.enum_def &&
...@@ -457,59 +522,62 @@ void Parser::ParseField(StructDef &struct_def) { ...@@ -457,59 +522,62 @@ void Parser::ParseField(StructDef &struct_def) {
!struct_def.fixed && !struct_def.fixed &&
!type.enum_def->attributes.Lookup("bit_flags") && !type.enum_def->attributes.Lookup("bit_flags") &&
!type.enum_def->ReverseLookup(static_cast<int>( !type.enum_def->ReverseLookup(static_cast<int>(
StringToInt(field.value.constant.c_str())))) StringToInt(field->value.constant.c_str()))))
Error("enum " + type.enum_def->name + return Error("enum " + type.enum_def->name +
" does not have a declaration for this field\'s default of " + " does not have a declaration for this field\'s default of " +
field.value.constant); field->value.constant);
field.doc_comment = dc; field->doc_comment = dc;
ParseMetaData(field); ECHECK(ParseMetaData(*field));
field.deprecated = field.attributes.Lookup("deprecated") != nullptr; field->deprecated = field->attributes.Lookup("deprecated") != nullptr;
auto hash_name = field.attributes.Lookup("hash"); auto hash_name = field->attributes.Lookup("hash");
if (hash_name) { if (hash_name) {
switch (type.base_type) { switch (type.base_type) {
case BASE_TYPE_INT: case BASE_TYPE_INT:
case BASE_TYPE_UINT: { case BASE_TYPE_UINT: {
if (FindHashFunction32(hash_name->constant.c_str()) == nullptr) if (FindHashFunction32(hash_name->constant.c_str()) == nullptr)
Error("Unknown hashing algorithm for 32 bit types: " + return Error("Unknown hashing algorithm for 32 bit types: " +
hash_name->constant); hash_name->constant);
break; break;
} }
case BASE_TYPE_LONG: case BASE_TYPE_LONG:
case BASE_TYPE_ULONG: { case BASE_TYPE_ULONG: {
if (FindHashFunction64(hash_name->constant.c_str()) == nullptr) if (FindHashFunction64(hash_name->constant.c_str()) == nullptr)
Error("Unknown hashing algorithm for 64 bit types: " + return Error("Unknown hashing algorithm for 64 bit types: " +
hash_name->constant); hash_name->constant);
break; break;
} }
default: default:
Error("only int, uint, long and ulong data types support hashing."); return Error(
"only int, uint, long and ulong data types support hashing.");
} }
} }
if (field.deprecated && struct_def.fixed) if (field->deprecated && struct_def.fixed)
Error("can't deprecate fields in a struct"); return Error("can't deprecate fields in a struct");
field.required = field.attributes.Lookup("required") != nullptr; field->required = field->attributes.Lookup("required") != nullptr;
if (field.required && (struct_def.fixed || if (field->required && (struct_def.fixed ||
IsScalar(field.value.type.base_type))) IsScalar(field->value.type.base_type)))
Error("only non-scalar fields in tables may be 'required'"); return Error("only non-scalar fields in tables may be 'required'");
field.key = field.attributes.Lookup("key") != nullptr; field->key = field->attributes.Lookup("key") != nullptr;
if (field.key) { if (field->key) {
if (struct_def.has_key) if (struct_def.has_key)
Error("only one field may be set as 'key'"); return Error("only one field may be set as 'key'");
struct_def.has_key = true; struct_def.has_key = true;
if (!IsScalar(field.value.type.base_type)) { if (!IsScalar(field->value.type.base_type)) {
field.required = true; field->required = true;
if (field.value.type.base_type != BASE_TYPE_STRING) if (field->value.type.base_type != BASE_TYPE_STRING)
Error("'key' field must be string or scalar type"); return Error("'key' field must be string or scalar type");
} }
} }
auto nested = field.attributes.Lookup("nested_flatbuffer"); auto nested = field->attributes.Lookup("nested_flatbuffer");
if (nested) { if (nested) {
if (nested->type.base_type != BASE_TYPE_STRING) if (nested->type.base_type != BASE_TYPE_STRING)
Error("nested_flatbuffer attribute must be a string (the root type)"); return Error(
if (field.value.type.base_type != BASE_TYPE_VECTOR || "nested_flatbuffer attribute must be a string (the root type)");
field.value.type.element != BASE_TYPE_UCHAR) if (field->value.type.base_type != BASE_TYPE_VECTOR ||
Error("nested_flatbuffer attribute may only apply to a vector of ubyte"); field->value.type.element != BASE_TYPE_UCHAR)
return Error(
"nested_flatbuffer attribute may only apply to a vector of ubyte");
// This will cause an error if the root type of the nested flatbuffer // This will cause an error if the root type of the nested flatbuffer
// wasn't defined elsewhere. // wasn't defined elsewhere.
LookupCreateStruct(nested->constant); LookupCreateStruct(nested->constant);
...@@ -518,7 +586,7 @@ void Parser::ParseField(StructDef &struct_def) { ...@@ -518,7 +586,7 @@ void Parser::ParseField(StructDef &struct_def) {
if (typefield) { if (typefield) {
// If this field is a union, and it has a manually assigned id, // If this field is a union, and it has a manually assigned id,
// the automatically added type field should have an id as well (of N - 1). // the automatically added type field should have an id as well (of N - 1).
auto attr = field.attributes.Lookup("id"); auto attr = field->attributes.Lookup("id");
if (attr) { if (attr) {
auto id = atoi(attr->constant.c_str()); auto id = atoi(attr->constant.c_str());
auto val = new Value(); auto val = new Value();
...@@ -528,35 +596,41 @@ void Parser::ParseField(StructDef &struct_def) { ...@@ -528,35 +596,41 @@ void Parser::ParseField(StructDef &struct_def) {
} }
} }
Expect(';'); EXPECT(';');
return NoError();
} }
void Parser::ParseAnyValue(Value &val, FieldDef *field, size_t parent_fieldn) { CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
size_t parent_fieldn) {
switch (val.type.base_type) { switch (val.type.base_type) {
case BASE_TYPE_UNION: { case BASE_TYPE_UNION: {
assert(field); assert(field);
if (!parent_fieldn || if (!parent_fieldn ||
field_stack_.back().second->value.type.base_type != BASE_TYPE_UTYPE) field_stack_.back().second->value.type.base_type != BASE_TYPE_UTYPE)
Error("missing type field before this union value: " + field->name); return Error("missing type field before this union value: " +
auto enum_idx = atot<unsigned char>( field->name);
field_stack_.back().first.constant.c_str()); uint8_t enum_idx;
ECHECK(atot(field_stack_.back().first.constant.c_str(), *this,
&enum_idx));
auto enum_val = val.type.enum_def->ReverseLookup(enum_idx); auto enum_val = val.type.enum_def->ReverseLookup(enum_idx);
if (!enum_val) Error("illegal type id for: " + field->name); if (!enum_val) return Error("illegal type id for: " + field->name);
ParseTable(*enum_val->struct_def, &val.constant); ECHECK(ParseTable(*enum_val->struct_def, &val.constant, nullptr));
break; break;
} }
case BASE_TYPE_STRUCT: case BASE_TYPE_STRUCT:
ParseTable(*val.type.struct_def, &val.constant); ECHECK(ParseTable(*val.type.struct_def, &val.constant, nullptr));
break; break;
case BASE_TYPE_STRING: { case BASE_TYPE_STRING: {
auto s = attribute_; auto s = attribute_;
Expect(kTokenStringConstant); EXPECT(kTokenStringConstant);
val.constant = NumToString(builder_.CreateString(s).o); val.constant = NumToString(builder_.CreateString(s).o);
break; break;
} }
case BASE_TYPE_VECTOR: { case BASE_TYPE_VECTOR: {
Expect('['); EXPECT('[');
val.constant = NumToString(ParseVector(val.type.VectorType())); uoffset_t off;
ECHECK(ParseVector(val.type.VectorType(), &off));
val.constant = NumToString(off);
break; break;
} }
case BASE_TYPE_INT: case BASE_TYPE_INT:
...@@ -565,16 +639,17 @@ void Parser::ParseAnyValue(Value &val, FieldDef *field, size_t parent_fieldn) { ...@@ -565,16 +639,17 @@ void Parser::ParseAnyValue(Value &val, FieldDef *field, size_t parent_fieldn) {
case BASE_TYPE_ULONG: { case BASE_TYPE_ULONG: {
if (field && field->attributes.Lookup("hash") && if (field && field->attributes.Lookup("hash") &&
(token_ == kTokenIdentifier || token_ == kTokenStringConstant)) { (token_ == kTokenIdentifier || token_ == kTokenStringConstant)) {
ParseHash(val, field); ECHECK(ParseHash(val, field));
} else { } else {
ParseSingleValue(val); ECHECK(ParseSingleValue(val));
} }
break; break;
} }
default: default:
ParseSingleValue(val); ECHECK(ParseSingleValue(val));
break; break;
} }
return NoError();
} }
void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) { void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) {
...@@ -585,35 +660,39 @@ void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) { ...@@ -585,35 +660,39 @@ void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) {
builder_.AddStructOffset(val.offset, builder_.GetSize()); builder_.AddStructOffset(val.offset, builder_.GetSize());
} }
uoffset_t Parser::ParseTable(const StructDef &struct_def, std::string *value) { CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
Expect('{'); uoffset_t *ovalue) {
EXPECT('{');
size_t fieldn = 0; size_t fieldn = 0;
for (;;) { for (;;) {
if ((!opts.strict_json || !fieldn) && IsNext('}')) break; if ((!opts.strict_json || !fieldn) && Is('}')) { NEXT(); break; }
std::string name = attribute_; std::string name = attribute_;
if (!IsNext(kTokenStringConstant)) if (Is(kTokenStringConstant)) {
Expect(opts.strict_json ? kTokenStringConstant : kTokenIdentifier); NEXT();
} else {
EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier);
}
auto field = struct_def.fields.Lookup(name); auto field = struct_def.fields.Lookup(name);
if (!field) Error("unknown field: " + name); if (!field) return Error("unknown field: " + name);
Expect(':'); EXPECT(':');
Value val = field->value; Value val = field->value;
ParseAnyValue(val, field, fieldn); ECHECK(ParseAnyValue(val, field, fieldn));
size_t i = field_stack_.size(); size_t i = field_stack_.size();
// Hardcoded insertion-sort with error-check. // Hardcoded insertion-sort with error-check.
// If fields are specified in order, then this loop exits immediately. // If fields are specified in order, then this loop exits immediately.
for (; i > field_stack_.size() - fieldn; i--) { for (; i > field_stack_.size() - fieldn; i--) {
auto existing_field = field_stack_[i - 1].second; auto existing_field = field_stack_[i - 1].second;
if (existing_field == field) if (existing_field == field)
Error("field set more than once: " + field->name); return Error("field set more than once: " + field->name);
if (existing_field->value.offset < field->value.offset) break; if (existing_field->value.offset < field->value.offset) break;
} }
field_stack_.insert(field_stack_.begin() + i, std::make_pair(val, field)); field_stack_.insert(field_stack_.begin() + i, std::make_pair(val, field));
fieldn++; fieldn++;
if (IsNext('}')) break; if (Is('}')) { NEXT(); break; }
Expect(','); EXPECT(',');
} }
if (struct_def.fixed && fieldn != struct_def.fields.vec.size()) if (struct_def.fixed && fieldn != struct_def.fields.vec.size())
Error("struct: wrong number of initializers: " + struct_def.name); return Error("struct: wrong number of initializers: " + struct_def.name);
auto start = struct_def.fixed auto start = struct_def.fixed
? builder_.StartStruct(struct_def.minalign) ? builder_.StartStruct(struct_def.minalign)
...@@ -627,18 +706,22 @@ uoffset_t Parser::ParseTable(const StructDef &struct_def, std::string *value) { ...@@ -627,18 +706,22 @@ uoffset_t Parser::ParseTable(const StructDef &struct_def, std::string *value) {
it != field_stack_.rbegin() + fieldn; ++it) { it != field_stack_.rbegin() + fieldn; ++it) {
auto &field_value = it->first; auto &field_value = it->first;
auto field = it->second; auto field = it->second;
if (!struct_def.sortbysize || size == SizeOf(field_value.type.base_type)) { if (!struct_def.sortbysize ||
size == SizeOf(field_value.type.base_type)) {
switch (field_value.type.base_type) { switch (field_value.type.base_type) {
#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \ #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \
PTYPE) \ PTYPE) \
case BASE_TYPE_ ## ENUM: \ case BASE_TYPE_ ## ENUM: \
builder_.Pad(field->padding); \ builder_.Pad(field->padding); \
if (struct_def.fixed) { \ if (struct_def.fixed) { \
builder_.PushElement(atot<CTYPE>(field_value.constant.c_str())); \ CTYPE val; \
ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
builder_.PushElement(val); \
} else { \ } else { \
builder_.AddElement(field_value.offset, \ CTYPE val, valdef; \
atot<CTYPE>( field_value.constant.c_str()), \ ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
atot<CTYPE>(field->value.constant.c_str())); \ ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \
builder_.AddElement(field_value.offset, val, valdef); \
} \ } \
break; break;
FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD); FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD);
...@@ -650,8 +733,9 @@ uoffset_t Parser::ParseTable(const StructDef &struct_def, std::string *value) { ...@@ -650,8 +733,9 @@ uoffset_t Parser::ParseTable(const StructDef &struct_def, std::string *value) {
if (IsStruct(field->value.type)) { \ if (IsStruct(field->value.type)) { \
SerializeStruct(*field->value.type.struct_def, field_value); \ SerializeStruct(*field->value.type.struct_def, field_value); \
} else { \ } else { \
builder_.AddOffset(field_value.offset, \ CTYPE val; \
atot<CTYPE>(field_value.constant.c_str())); \ ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
builder_.AddOffset(field_value.offset, val); \
} \ } \
break; break;
FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD); FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD);
...@@ -672,27 +756,27 @@ uoffset_t Parser::ParseTable(const StructDef &struct_def, std::string *value) { ...@@ -672,27 +756,27 @@ uoffset_t Parser::ParseTable(const StructDef &struct_def, std::string *value) {
reinterpret_cast<const char *>(builder_.GetCurrentBufferPointer()), reinterpret_cast<const char *>(builder_.GetCurrentBufferPointer()),
struct_def.bytesize); struct_def.bytesize);
builder_.PopBytes(struct_def.bytesize); builder_.PopBytes(struct_def.bytesize);
return 0xFFFFFFFF; // Value not used by the caller. assert(!ovalue);
} else { } else {
auto off = builder_.EndTable( auto val = builder_.EndTable(start,
start, static_cast<voffset_t>(struct_def.fields.vec.size()));
static_cast<voffset_t>(struct_def.fields.vec.size())); if (ovalue) *ovalue = val;
if (value) *value = NumToString(off); if (value) *value = NumToString(val);
return off;
} }
return NoError();
} }
uoffset_t Parser::ParseVector(const Type &type) { CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) {
int count = 0; int count = 0;
for (;;) { for (;;) {
if ((!opts.strict_json || !count) && IsNext(']')) break; if ((!opts.strict_json || !count) && Is(']')) { NEXT(); break; }
Value val; Value val;
val.type = type; val.type = type;
ParseAnyValue(val, nullptr, 0); ECHECK(ParseAnyValue(val, nullptr, 0));
field_stack_.push_back(std::make_pair(val, nullptr)); field_stack_.push_back(std::make_pair(val, nullptr));
count++; count++;
if (IsNext(']')) break; if (Is(']')) { NEXT(); break; }
Expect(','); EXPECT(',');
} }
builder_.StartVector(count * InlineSize(type) / InlineAlignment(type), builder_.StartVector(count * InlineSize(type) / InlineAlignment(type),
...@@ -704,7 +788,11 @@ uoffset_t Parser::ParseVector(const Type &type) { ...@@ -704,7 +788,11 @@ uoffset_t Parser::ParseVector(const Type &type) {
#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \
case BASE_TYPE_ ## ENUM: \ case BASE_TYPE_ ## ENUM: \
if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \ if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \
else builder_.PushElement(atot<CTYPE>(val.constant.c_str())); \ else { \
CTYPE elem; \
ECHECK(atot(val.constant.c_str(), *this, &elem)); \
builder_.PushElement(elem); \
} \
break; break;
FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
#undef FLATBUFFERS_TD #undef FLATBUFFERS_TD
...@@ -713,51 +801,55 @@ uoffset_t Parser::ParseVector(const Type &type) { ...@@ -713,51 +801,55 @@ uoffset_t Parser::ParseVector(const Type &type) {
} }
builder_.ClearOffsets(); builder_.ClearOffsets();
return builder_.EndVector(count); *ovalue = builder_.EndVector(count);
return NoError();
} }
void Parser::ParseMetaData(Definition &def) { CheckedError Parser::ParseMetaData(Definition &def) {
if (IsNext('(')) { if (Is('(')) {
NEXT();
for (;;) { for (;;) {
auto name = attribute_; auto name = attribute_;
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
if (known_attributes_.find(name) == known_attributes_.end()) if (known_attributes_.find(name) == known_attributes_.end())
Error("user define attributes must be declared before use: " + name); return Error("user define attributes must be declared before use: " +
name);
auto e = new Value(); auto e = new Value();
def.attributes.Add(name, e); def.attributes.Add(name, e);
if (IsNext(':')) { if (Is(':')) {
ParseSingleValue(*e); NEXT();
ECHECK(ParseSingleValue(*e));
} }
if (IsNext(')')) break; if (Is(')')) { NEXT(); break; }
Expect(','); EXPECT(',');
} }
} }
return NoError();
} }
bool Parser::TryTypedValue(int dtoken, CheckedError Parser::TryTypedValue(int dtoken, bool check, Value &e,
bool check, BaseType req, bool *destmatch) {
Value &e,
BaseType req) {
bool match = dtoken == token_; bool match = dtoken == token_;
if (match) { if (match) {
*destmatch = true;
e.constant = attribute_; e.constant = attribute_;
if (!check) { if (!check) {
if (e.type.base_type == BASE_TYPE_NONE) { if (e.type.base_type == BASE_TYPE_NONE) {
e.type.base_type = req; e.type.base_type = req;
} else { } else {
Error(std::string("type mismatch: expecting: ") + return Error(std::string("type mismatch: expecting: ") +
kTypeNames[e.type.base_type] + kTypeNames[e.type.base_type] +
", found: " + ", found: " +
kTypeNames[req]); kTypeNames[req]);
} }
} }
Next(); NEXT();
} }
return match; return NoError();
} }
int64_t Parser::ParseIntegerFromString(Type &type) { CheckedError Parser::ParseIntegerFromString(Type &type, int64_t *result) {
int64_t result = 0; *result = 0;
// Parse one or more enum identifiers, separated by spaces. // Parse one or more enum identifiers, separated by spaces.
const char *next = attribute_.c_str(); const char *next = attribute_.c_str();
do { do {
...@@ -773,29 +865,30 @@ int64_t Parser::ParseIntegerFromString(Type &type) { ...@@ -773,29 +865,30 @@ int64_t Parser::ParseIntegerFromString(Type &type) {
if (type.enum_def) { // The field has an enum type if (type.enum_def) { // The field has an enum type
auto enum_val = type.enum_def->vals.Lookup(word); auto enum_val = type.enum_def->vals.Lookup(word);
if (!enum_val) if (!enum_val)
Error("unknown enum value: " + word + return Error("unknown enum value: " + word +
", for enum: " + type.enum_def->name); ", for enum: " + type.enum_def->name);
result |= enum_val->value; *result |= enum_val->value;
} else { // No enum type, probably integral field. } else { // No enum type, probably integral field.
if (!IsInteger(type.base_type)) if (!IsInteger(type.base_type))
Error("not a valid value for this field: " + word); return Error("not a valid value for this field: " + word);
// TODO: could check if its a valid number constant here. // TODO: could check if its a valid number constant here.
const char *dot = strrchr(word.c_str(), '.'); const char *dot = strrchr(word.c_str(), '.');
if (!dot) Error("enum values need to be qualified by an enum type"); if (!dot)
return Error("enum values need to be qualified by an enum type");
std::string enum_def_str(word.c_str(), dot); std::string enum_def_str(word.c_str(), dot);
std::string enum_val_str(dot + 1, word.c_str() + word.length()); std::string enum_val_str(dot + 1, word.c_str() + word.length());
auto enum_def = LookupEnum(enum_def_str); auto enum_def = LookupEnum(enum_def_str);
if (!enum_def) Error("unknown enum: " + enum_def_str); if (!enum_def) return Error("unknown enum: " + enum_def_str);
auto enum_val = enum_def->vals.Lookup(enum_val_str); auto enum_val = enum_def->vals.Lookup(enum_val_str);
if (!enum_val) Error("unknown enum value: " + enum_val_str); if (!enum_val) return Error("unknown enum value: " + enum_val_str);
result |= enum_val->value; *result |= enum_val->value;
} }
} while(*next); } while(*next);
return result; return NoError();
} }
void Parser::ParseHash(Value &e, FieldDef* field) { CheckedError Parser::ParseHash(Value &e, FieldDef* field) {
assert(field); assert(field);
Value *hash_name = field->attributes.Lookup("hash"); Value *hash_name = field->attributes.Lookup("hash");
switch (e.type.base_type) { switch (e.type.base_type) {
...@@ -816,31 +909,41 @@ void Parser::ParseHash(Value &e, FieldDef* field) { ...@@ -816,31 +909,41 @@ void Parser::ParseHash(Value &e, FieldDef* field) {
default: default:
assert(0); assert(0);
} }
Next(); NEXT();
return NoError();
} }
void Parser::ParseSingleValue(Value &e) { CheckedError Parser::ParseSingleValue(Value &e) {
// First check if this could be a string/identifier enum value: // First check if this could be a string/identifier enum value:
if (e.type.base_type != BASE_TYPE_STRING && if (e.type.base_type != BASE_TYPE_STRING &&
e.type.base_type != BASE_TYPE_NONE && e.type.base_type != BASE_TYPE_NONE &&
(token_ == kTokenIdentifier || token_ == kTokenStringConstant)) { (token_ == kTokenIdentifier || token_ == kTokenStringConstant)) {
e.constant = NumToString(ParseIntegerFromString(e.type)); int64_t val;
Next(); ECHECK(ParseIntegerFromString(e.type, &val));
} else if (TryTypedValue(kTokenIntegerConstant, e.constant = NumToString(val);
IsScalar(e.type.base_type), NEXT();
e,
BASE_TYPE_INT) ||
TryTypedValue(kTokenFloatConstant,
IsFloat(e.type.base_type),
e,
BASE_TYPE_FLOAT) ||
TryTypedValue(kTokenStringConstant,
e.type.base_type == BASE_TYPE_STRING,
e,
BASE_TYPE_STRING)) {
} else { } else {
Error("cannot parse value starting with: " + TokenToStringId(token_)); bool match = false;
ECHECK(TryTypedValue(kTokenIntegerConstant,
IsScalar(e.type.base_type),
e,
BASE_TYPE_INT,
&match));
ECHECK(TryTypedValue(kTokenFloatConstant,
IsFloat(e.type.base_type),
e,
BASE_TYPE_FLOAT,
&match));
ECHECK(TryTypedValue(kTokenStringConstant,
e.type.base_type == BASE_TYPE_STRING,
e,
BASE_TYPE_STRING,
&match));
if (!match)
return Error("cannot parse value starting with: " +
TokenToStringId(token_));
} }
return NoError();
} }
StructDef *Parser::LookupCreateStruct(const std::string &name, StructDef *Parser::LookupCreateStruct(const std::string &name,
...@@ -885,20 +988,20 @@ StructDef *Parser::LookupCreateStruct(const std::string &name, ...@@ -885,20 +988,20 @@ StructDef *Parser::LookupCreateStruct(const std::string &name,
return struct_def; return struct_def;
} }
EnumDef &Parser::ParseEnum(bool is_union) { CheckedError Parser::ParseEnum(bool is_union, EnumDef **dest) {
std::vector<std::string> enum_comment = doc_comment_; std::vector<std::string> enum_comment = doc_comment_;
Next(); NEXT();
std::string enum_name = attribute_; std::string enum_name = attribute_;
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
auto &enum_def = *new EnumDef(); auto &enum_def = *new EnumDef();
enum_def.name = enum_name; enum_def.name = enum_name;
enum_def.file = files_being_parsed_; enum_def.file = file_being_parsed_;
enum_def.doc_comment = enum_comment; enum_def.doc_comment = enum_comment;
enum_def.is_union = is_union; enum_def.is_union = is_union;
enum_def.defined_namespace = namespaces_.back(); enum_def.defined_namespace = namespaces_.back();
if (enums_.Add(namespaces_.back()->GetFullyQualifiedName(enum_name), if (enums_.Add(namespaces_.back()->GetFullyQualifiedName(enum_name),
&enum_def)) &enum_def))
Error("enum already exists: " + enum_name); return Error("enum already exists: " + enum_name);
if (is_union) { if (is_union) {
enum_def.underlying_type.base_type = BASE_TYPE_UTYPE; enum_def.underlying_type.base_type = BASE_TYPE_UTYPE;
enum_def.underlying_type.enum_def = &enum_def; enum_def.underlying_type.enum_def = &enum_def;
...@@ -908,107 +1011,119 @@ EnumDef &Parser::ParseEnum(bool is_union) { ...@@ -908,107 +1011,119 @@ EnumDef &Parser::ParseEnum(bool is_union) {
} else { } else {
// Give specialized error message, since this type spec used to // Give specialized error message, since this type spec used to
// be optional in the first FlatBuffers release. // be optional in the first FlatBuffers release.
if (!IsNext(':')) Error("must specify the underlying integer type for this" if (!Is(':')) {
" enum (e.g. \': short\', which was the default)."); return Error("must specify the underlying integer type for this"
" enum (e.g. \': short\', which was the default).");
} else {
NEXT();
}
// Specify the integer type underlying this enum. // Specify the integer type underlying this enum.
ParseType(enum_def.underlying_type); ECHECK(ParseType(enum_def.underlying_type));
if (!IsInteger(enum_def.underlying_type.base_type)) if (!IsInteger(enum_def.underlying_type.base_type))
Error("underlying enum type must be integral"); return Error("underlying enum type must be integral");
} }
// Make this type refer back to the enum it was derived from. // Make this type refer back to the enum it was derived from.
enum_def.underlying_type.enum_def = &enum_def; enum_def.underlying_type.enum_def = &enum_def;
} }
ParseMetaData(enum_def); ECHECK(ParseMetaData(enum_def));
Expect('{'); EXPECT('{');
if (is_union) enum_def.vals.Add("NONE", new EnumVal("NONE", 0)); if (is_union) enum_def.vals.Add("NONE", new EnumVal("NONE", 0));
do { for (;;) {
if (opts.proto_mode && attribute_ == "option") { if (opts.proto_mode && attribute_ == "option") {
ParseProtoOption(); ECHECK(ParseProtoOption());
} else { } else {
auto value_name = attribute_; auto value_name = attribute_;
auto full_name = value_name; auto full_name = value_name;
std::vector<std::string> value_comment = doc_comment_; std::vector<std::string> value_comment = doc_comment_;
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
if (is_union) ParseNamespacing(&full_name, &value_name); if (is_union) ECHECK(ParseNamespacing(&full_name, &value_name));
auto prevsize = enum_def.vals.vec.size(); auto prevsize = enum_def.vals.vec.size();
auto value = enum_def.vals.vec.size() auto value = enum_def.vals.vec.size()
? enum_def.vals.vec.back()->value + 1 ? enum_def.vals.vec.back()->value + 1
: 0; : 0;
auto &ev = *new EnumVal(value_name, value); auto &ev = *new EnumVal(value_name, value);
if (enum_def.vals.Add(value_name, &ev)) if (enum_def.vals.Add(value_name, &ev))
Error("enum value already exists: " + value_name); return Error("enum value already exists: " + value_name);
ev.doc_comment = value_comment; ev.doc_comment = value_comment;
if (is_union) { if (is_union) {
ev.struct_def = LookupCreateStruct(full_name); ev.struct_def = LookupCreateStruct(full_name);
} }
if (IsNext('=')) { if (Is('=')) {
NEXT();
ev.value = atoi(attribute_.c_str()); ev.value = atoi(attribute_.c_str());
Expect(kTokenIntegerConstant); EXPECT(kTokenIntegerConstant);
if (!opts.proto_mode && prevsize && if (!opts.proto_mode && prevsize &&
enum_def.vals.vec[prevsize - 1]->value >= ev.value) enum_def.vals.vec[prevsize - 1]->value >= ev.value)
Error("enum values must be specified in ascending order"); return Error("enum values must be specified in ascending order");
} }
if (opts.proto_mode && IsNext('[')) { if (opts.proto_mode && Is('[')) {
NEXT();
// ignore attributes on enums. // ignore attributes on enums.
while (token_ != ']') Next(); while (token_ != ']') NEXT();
Next(); NEXT();
} }
} }
} while (IsNext(opts.proto_mode ? ';' : ',') && token_ != '}'); if (!Is(opts.proto_mode ? ';' : ',')) break;
Expect('}'); NEXT();
if (Is('}')) break;
}
EXPECT('}');
if (enum_def.attributes.Lookup("bit_flags")) { if (enum_def.attributes.Lookup("bit_flags")) {
for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end();
++it) { ++it) {
if (static_cast<size_t>((*it)->value) >= if (static_cast<size_t>((*it)->value) >=
SizeOf(enum_def.underlying_type.base_type) * 8) SizeOf(enum_def.underlying_type.base_type) * 8)
Error("bit flag out of range of underlying integral type"); return Error("bit flag out of range of underlying integral type");
(*it)->value = 1LL << (*it)->value; (*it)->value = 1LL << (*it)->value;
} }
} }
return enum_def; if (dest) *dest = &enum_def;
return NoError();
} }
StructDef &Parser::StartStruct(const std::string &name) { CheckedError Parser::StartStruct(const std::string &name, StructDef **dest) {
auto &struct_def = *LookupCreateStruct(name, true, true); auto &struct_def = *LookupCreateStruct(name, true, true);
if (!struct_def.predecl) Error("datatype already exists: " + name); if (!struct_def.predecl) return Error("datatype already exists: " + name);
struct_def.predecl = false; struct_def.predecl = false;
struct_def.name = name; struct_def.name = name;
struct_def.file = files_being_parsed_; struct_def.file = file_being_parsed_;
// Move this struct to the back of the vector just in case it was predeclared, // Move this struct to the back of the vector just in case it was predeclared,
// to preserve declaration order. // to preserve declaration order.
*remove(structs_.vec.begin(), structs_.vec.end(), &struct_def) = &struct_def; *remove(structs_.vec.begin(), structs_.vec.end(), &struct_def) = &struct_def;
return struct_def; *dest = &struct_def;
return NoError();
} }
void Parser::ParseDecl() { CheckedError Parser::ParseDecl() {
std::vector<std::string> dc = doc_comment_; std::vector<std::string> dc = doc_comment_;
bool fixed = IsNext(kTokenStruct); bool fixed = Is(kTokenStruct);
if (!fixed) Expect(kTokenTable); if (fixed) NEXT() else EXPECT(kTokenTable);
std::string name = attribute_; std::string name = attribute_;
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
auto &struct_def = StartStruct(name); StructDef *struct_def;
struct_def.doc_comment = dc; ECHECK(StartStruct(name, &struct_def));
struct_def.fixed = fixed; struct_def->doc_comment = dc;
ParseMetaData(struct_def); struct_def->fixed = fixed;
struct_def.sortbysize = ECHECK(ParseMetaData(*struct_def));
struct_def.attributes.Lookup("original_order") == nullptr && !fixed; struct_def->sortbysize =
Expect('{'); struct_def->attributes.Lookup("original_order") == nullptr && !fixed;
while (token_ != '}') ParseField(struct_def); EXPECT('{');
auto force_align = struct_def.attributes.Lookup("force_align"); while (token_ != '}') ECHECK(ParseField(*struct_def));
auto force_align = struct_def->attributes.Lookup("force_align");
if (fixed && force_align) { if (fixed && force_align) {
auto align = static_cast<size_t>(atoi(force_align->constant.c_str())); auto align = static_cast<size_t>(atoi(force_align->constant.c_str()));
if (force_align->type.base_type != BASE_TYPE_INT || if (force_align->type.base_type != BASE_TYPE_INT ||
align < struct_def.minalign || align < struct_def->minalign ||
align > 16 || align > 16 ||
align & (align - 1)) align & (align - 1))
Error("force_align must be a power of two integer ranging from the" return Error("force_align must be a power of two integer ranging from the"
"struct\'s natural alignment to 16"); "struct\'s natural alignment to 16");
struct_def.minalign = align; struct_def->minalign = align;
} }
struct_def.PadLastField(struct_def.minalign); struct_def->PadLastField(struct_def->minalign);
// Check if this is a table that has manual id assignments // Check if this is a table that has manual id assignments
auto &fields = struct_def.fields.vec; auto &fields = struct_def->fields.vec;
if (!struct_def.fixed && fields.size()) { if (!struct_def->fixed && fields.size()) {
size_t num_id_fields = 0; size_t num_id_fields = 0;
for (auto it = fields.begin(); it != fields.end(); ++it) { for (auto it = fields.begin(); it != fields.end(); ++it) {
if ((*it)->attributes.Lookup("id")) num_id_fields++; if ((*it)->attributes.Lookup("id")) num_id_fields++;
...@@ -1017,7 +1132,8 @@ void Parser::ParseDecl() { ...@@ -1017,7 +1132,8 @@ void Parser::ParseDecl() {
if (num_id_fields) { if (num_id_fields) {
// Then all fields must have them. // Then all fields must have them.
if (num_id_fields != fields.size()) if (num_id_fields != fields.size())
Error("either all fields or no fields must have an 'id' attribute"); return Error(
"either all fields or no fields must have an 'id' attribute");
// Simply sort by id, then the fields are the same as if no ids had // Simply sort by id, then the fields are the same as if no ids had
// been specified. // been specified.
std::sort(fields.begin(), fields.end(), std::sort(fields.begin(), fields.end(),
...@@ -1029,7 +1145,7 @@ void Parser::ParseDecl() { ...@@ -1029,7 +1145,7 @@ void Parser::ParseDecl() {
// Verify we have a contiguous set, and reassign vtable offsets. // Verify we have a contiguous set, and reassign vtable offsets.
for (int i = 0; i < static_cast<int>(fields.size()); i++) { for (int i = 0; i < static_cast<int>(fields.size()); i++) {
if (i != atoi(fields[i]->attributes.Lookup("id")->constant.c_str())) if (i != atoi(fields[i]->attributes.Lookup("id")->constant.c_str()))
Error("field id\'s must be consecutive from 0, id " + return Error("field id\'s must be consecutive from 0, id " +
NumToString(i) + " missing or set twice"); NumToString(i) + " missing or set twice");
fields[i]->value.offset = FieldIndexToOffset(static_cast<voffset_t>(i)); fields[i]->value.offset = FieldIndexToOffset(static_cast<voffset_t>(i));
} }
...@@ -1039,30 +1155,32 @@ void Parser::ParseDecl() { ...@@ -1039,30 +1155,32 @@ void Parser::ParseDecl() {
// This is not an ideal situation, but should occur very infrequently, // This is not an ideal situation, but should occur very infrequently,
// and allows us to keep using very readable names for type & length fields // and allows us to keep using very readable names for type & length fields
// without inducing compile errors. // without inducing compile errors.
auto CheckClash = [&fields, &struct_def](const char *suffix, auto CheckClash = [&fields, &struct_def, this](const char *suffix,
BaseType basetype) { BaseType basetype) -> CheckedError {
auto len = strlen(suffix); auto len = strlen(suffix);
for (auto it = fields.begin(); it != fields.end(); ++it) { for (auto it = fields.begin(); it != fields.end(); ++it) {
auto &fname = (*it)->name; auto &fname = (*it)->name;
if (fname.length() > len && if (fname.length() > len &&
fname.compare(fname.length() - len, len, suffix) == 0 && fname.compare(fname.length() - len, len, suffix) == 0 &&
(*it)->value.type.base_type != BASE_TYPE_UTYPE) { (*it)->value.type.base_type != BASE_TYPE_UTYPE) {
auto field = struct_def.fields.Lookup( auto field = struct_def->fields.Lookup(
fname.substr(0, fname.length() - len)); fname.substr(0, fname.length() - len));
if (field && field->value.type.base_type == basetype) if (field && field->value.type.base_type == basetype)
Error("Field " + fname + return Error("Field " + fname +
" would clash with generated functions for field " + " would clash with generated functions for field " +
field->name); field->name);
} }
} }
return NoError();
}; };
CheckClash("_type", BASE_TYPE_UNION); ECHECK(CheckClash("_type", BASE_TYPE_UNION));
CheckClash("Type", BASE_TYPE_UNION); ECHECK(CheckClash("Type", BASE_TYPE_UNION));
CheckClash("_length", BASE_TYPE_VECTOR); ECHECK(CheckClash("_length", BASE_TYPE_VECTOR));
CheckClash("Length", BASE_TYPE_VECTOR); ECHECK(CheckClash("Length", BASE_TYPE_VECTOR));
CheckClash("_byte_vector", BASE_TYPE_STRING); ECHECK(CheckClash("_byte_vector", BASE_TYPE_STRING));
CheckClash("ByteVector", BASE_TYPE_STRING); ECHECK(CheckClash("ByteVector", BASE_TYPE_STRING));
Expect('}'); EXPECT('}');
return NoError();
} }
bool Parser::SetRootType(const char *name) { bool Parser::SetRootType(const char *name) {
...@@ -1087,44 +1205,46 @@ void Parser::MarkGenerated() { ...@@ -1087,44 +1205,46 @@ void Parser::MarkGenerated() {
} }
} }
void Parser::ParseNamespace() { CheckedError Parser::ParseNamespace() {
Next(); NEXT();
auto ns = new Namespace(); auto ns = new Namespace();
namespaces_.push_back(ns); namespaces_.push_back(ns);
if (token_ != ';') { if (token_ != ';') {
for (;;) { for (;;) {
ns->components.push_back(attribute_); ns->components.push_back(attribute_);
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
if (!IsNext('.')) break; if (Is('.')) NEXT() else break;
} }
} }
Expect(';'); EXPECT(';');
return NoError();
} }
// Best effort parsing of .proto declarations, with the aim to turn them // Best effort parsing of .proto declarations, with the aim to turn them
// in the closest corresponding FlatBuffer equivalent. // in the closest corresponding FlatBuffer equivalent.
// We parse everything as identifiers instead of keywords, since we don't // We parse everything as identifiers instead of keywords, since we don't
// want protobuf keywords to become invalid identifiers in FlatBuffers. // want protobuf keywords to become invalid identifiers in FlatBuffers.
void Parser::ParseProtoDecl() { CheckedError Parser::ParseProtoDecl() {
bool isextend = attribute_ == "extend"; bool isextend = attribute_ == "extend";
if (attribute_ == "package") { if (attribute_ == "package") {
// These are identical in syntax to FlatBuffer's namespace decl. // These are identical in syntax to FlatBuffer's namespace decl.
ParseNamespace(); ECHECK(ParseNamespace());
} else if (attribute_ == "message" || isextend) { } else if (attribute_ == "message" || isextend) {
std::vector<std::string> struct_comment = doc_comment_; std::vector<std::string> struct_comment = doc_comment_;
Next(); NEXT();
StructDef *struct_def = nullptr; StructDef *struct_def = nullptr;
if (isextend) { if (isextend) {
IsNext('.'); // qualified names may start with a . ? if (Is('.')) NEXT(); // qualified names may start with a . ?
auto id = attribute_; auto id = attribute_;
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
ParseNamespacing(&id, nullptr); ECHECK(ParseNamespacing(&id, nullptr));
struct_def = LookupCreateStruct(id, false); struct_def = LookupCreateStruct(id, false);
if (!struct_def) Error("cannot extend unknown message type: " + id); if (!struct_def)
return Error("cannot extend unknown message type: " + id);
} else { } else {
std::string name = attribute_; std::string name = attribute_;
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
struct_def = &StartStruct(name); ECHECK(StartStruct(name, &struct_def));
// Since message definitions can be nested, we create a new namespace. // Since message definitions can be nested, we create a new namespace.
auto ns = new Namespace(); auto ns = new Namespace();
// Copy of current namespace. // Copy of current namespace.
...@@ -1134,7 +1254,7 @@ void Parser::ParseProtoDecl() { ...@@ -1134,7 +1254,7 @@ void Parser::ParseProtoDecl() {
namespaces_.push_back(ns); namespaces_.push_back(ns);
} }
struct_def->doc_comment = struct_comment; struct_def->doc_comment = struct_comment;
ParseProtoFields(struct_def, isextend, false); ECHECK(ParseProtoFields(struct_def, isextend, false));
if (!isextend) { if (!isextend) {
// We have to remove the nested namespace, but we can't just throw it // We have to remove the nested namespace, but we can't just throw it
// away, so put it at the beginning of the vector. // away, so put it at the beginning of the vector.
...@@ -1142,13 +1262,14 @@ void Parser::ParseProtoDecl() { ...@@ -1142,13 +1262,14 @@ void Parser::ParseProtoDecl() {
namespaces_.pop_back(); namespaces_.pop_back();
namespaces_.insert(namespaces_.begin(), ns); namespaces_.insert(namespaces_.begin(), ns);
} }
IsNext(';'); if (Is(';')) NEXT();
} else if (attribute_ == "enum") { } else if (attribute_ == "enum") {
// These are almost the same, just with different terminator: // These are almost the same, just with different terminator:
auto &enum_def = ParseEnum(false); EnumDef *enum_def;
IsNext(';'); ECHECK(ParseEnum(false, &enum_def));
if (Is(';')) NEXT();
// Protobuf allows them to be specified in any order, so sort afterwards. // Protobuf allows them to be specified in any order, so sort afterwards.
auto &v = enum_def.vals.vec; auto &v = enum_def->vals.vec;
std::sort(v.begin(), v.end(), [](const EnumVal *a, const EnumVal *b) { std::sort(v.begin(), v.end(), [](const EnumVal *a, const EnumVal *b) {
return a->value < b->value; return a->value < b->value;
}); });
...@@ -1158,46 +1279,48 @@ void Parser::ParseProtoDecl() { ...@@ -1158,46 +1279,48 @@ void Parser::ParseProtoDecl() {
else ++it; else ++it;
} }
} else if (attribute_ == "syntax") { // Skip these. } else if (attribute_ == "syntax") { // Skip these.
Next(); NEXT();
Expect('='); EXPECT('=');
Expect(kTokenStringConstant); EXPECT(kTokenStringConstant);
Expect(';'); EXPECT(';');
} else if (attribute_ == "option") { // Skip these. } else if (attribute_ == "option") { // Skip these.
ParseProtoOption(); ECHECK(ParseProtoOption());
Expect(';'); EXPECT(';');
} else if (attribute_ == "service") { // Skip these. } else if (attribute_ == "service") { // Skip these.
Next(); NEXT();
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
ParseProtoCurliesOrIdent(); ECHECK(ParseProtoCurliesOrIdent());
} else { } else {
Error("don\'t know how to parse .proto declaration starting with " + return Error("don\'t know how to parse .proto declaration starting with " +
TokenToStringId(token_)); TokenToStringId(token_));
} }
return NoError();
} }
void Parser::ParseProtoFields(StructDef *struct_def, bool isextend, CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
bool inside_oneof) { bool inside_oneof) {
Expect('{'); EXPECT('{');
while (token_ != '}') { while (token_ != '}') {
if (attribute_ == "message" || attribute_ == "extend" || if (attribute_ == "message" || attribute_ == "extend" ||
attribute_ == "enum") { attribute_ == "enum") {
// Nested declarations. // Nested declarations.
ParseProtoDecl(); ECHECK(ParseProtoDecl());
} else if (attribute_ == "extensions") { // Skip these. } else if (attribute_ == "extensions") { // Skip these.
Next(); NEXT();
Expect(kTokenIntegerConstant); EXPECT(kTokenIntegerConstant);
if (IsNext(kTokenIdentifier)) { // to if (Is(kTokenIdentifier)) {
Next(); // num NEXT(); // to
NEXT(); // num
} }
Expect(';'); EXPECT(';');
} else if (attribute_ == "option") { // Skip these. } else if (attribute_ == "option") { // Skip these.
ParseProtoOption(); ECHECK(ParseProtoOption());
Expect(';'); EXPECT(';');
} else if (attribute_ == "reserved") { // Skip these. } else if (attribute_ == "reserved") { // Skip these.
Next(); NEXT();
Expect(kTokenIntegerConstant); EXPECT(kTokenIntegerConstant);
while (IsNext(',')) Expect(kTokenIntegerConstant); while (Is(',')) { NEXT(); EXPECT(kTokenIntegerConstant); }
Expect(';'); EXPECT(';');
} else { } else {
std::vector<std::string> field_comment = doc_comment_; std::vector<std::string> field_comment = doc_comment_;
// Parse the qualifier. // Parse the qualifier.
...@@ -1207,16 +1330,16 @@ void Parser::ParseProtoFields(StructDef *struct_def, bool isextend, ...@@ -1207,16 +1330,16 @@ void Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
if (!inside_oneof) { if (!inside_oneof) {
if (attribute_ == "optional") { if (attribute_ == "optional") {
// This is the default. // This is the default.
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
} else if (attribute_ == "required") { } else if (attribute_ == "required") {
required = true; required = true;
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
} else if (attribute_ == "repeated") { } else if (attribute_ == "repeated") {
repeated = true; repeated = true;
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
} else if (attribute_ == "oneof") { } else if (attribute_ == "oneof") {
oneof = true; oneof = true;
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
} else { } else {
// can't error, proto3 allows decls without any of the above. // can't error, proto3 allows decls without any of the above.
} }
...@@ -1224,12 +1347,12 @@ void Parser::ParseProtoFields(StructDef *struct_def, bool isextend, ...@@ -1224,12 +1347,12 @@ void Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
StructDef *anonymous_struct = nullptr; StructDef *anonymous_struct = nullptr;
Type type; Type type;
if (attribute_ == "group" || oneof) { if (attribute_ == "group" || oneof) {
if (!oneof) Expect(kTokenIdentifier); if (!oneof) EXPECT(kTokenIdentifier);
auto name = "Anonymous" + NumToString(anonymous_counter++); auto name = "Anonymous" + NumToString(anonymous_counter++);
anonymous_struct = &StartStruct(name); ECHECK(StartStruct(name, &anonymous_struct));
type = Type(BASE_TYPE_STRUCT, anonymous_struct); type = Type(BASE_TYPE_STRUCT, anonymous_struct);
} else { } else {
type = ParseTypeFromProtoType(); ECHECK(ParseTypeFromProtoType(&type));
} }
// Repeated elements get mapped to a vector. // Repeated elements get mapped to a vector.
if (repeated) { if (repeated) {
...@@ -1238,93 +1361,100 @@ void Parser::ParseProtoFields(StructDef *struct_def, bool isextend, ...@@ -1238,93 +1361,100 @@ void Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
} }
std::string name = attribute_; std::string name = attribute_;
// Protos may use our keywords "attribute" & "namespace" as an identifier. // Protos may use our keywords "attribute" & "namespace" as an identifier.
if (IsNext(kTokenAttribute) || IsNext(kTokenNameSpace)) { if (Is(kTokenAttribute) || Is(kTokenNameSpace)) {
NEXT();
// TODO: simpler to just not make these keywords? // TODO: simpler to just not make these keywords?
name += "_"; // Have to make it not a keyword. name += "_"; // Have to make it not a keyword.
} else { } else {
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
} }
if (!oneof) { if (!oneof) {
// Parse the field id. Since we're just translating schemas, not // Parse the field id. Since we're just translating schemas, not
// any kind of binary compatibility, we can safely ignore these, and // any kind of binary compatibility, we can safely ignore these, and
// assign our own. // assign our own.
Expect('='); EXPECT('=');
Expect(kTokenIntegerConstant); EXPECT(kTokenIntegerConstant);
} }
FieldDef *existing_field = nullptr; FieldDef *field = nullptr;
if (isextend) { if (isextend) {
// We allow a field to be re-defined when extending. // We allow a field to be re-defined when extending.
// TODO: are there situations where that is problematic? // TODO: are there situations where that is problematic?
existing_field = struct_def->fields.Lookup(name); field = struct_def->fields.Lookup(name);
} }
auto &field = existing_field if (!field) ECHECK(AddField(*struct_def, name, type, &field));
? *existing_field field->doc_comment = field_comment;
: AddField(*struct_def, name, type); if (!IsScalar(type.base_type)) field->required = required;
field.doc_comment = field_comment;
if (!IsScalar(type.base_type)) field.required = required;
// See if there's a default specified. // See if there's a default specified.
if (IsNext('[')) { if (Is('[')) {
do { NEXT();
for (;;) {
auto key = attribute_; auto key = attribute_;
ParseProtoKey(); ECHECK(ParseProtoKey());
Expect('='); EXPECT('=');
auto val = attribute_; auto val = attribute_;
ParseProtoCurliesOrIdent(); ECHECK(ParseProtoCurliesOrIdent());
if (key == "default") { if (key == "default") {
// Temp: skip non-numeric defaults (enums). // Temp: skip non-numeric defaults (enums).
auto numeric = strpbrk(val.c_str(), "0123456789-+."); auto numeric = strpbrk(val.c_str(), "0123456789-+.");
if (IsScalar(type.base_type) && numeric == val.c_str()) if (IsScalar(type.base_type) && numeric == val.c_str())
field.value.constant = val; field->value.constant = val;
} else if (key == "deprecated") { } else if (key == "deprecated") {
field.deprecated = val == "true"; field->deprecated = val == "true";
} }
} while (IsNext(',')); if (!Is(',')) break;
Expect(']'); NEXT();
}
EXPECT(']');
} }
if (anonymous_struct) { if (anonymous_struct) {
ParseProtoFields(anonymous_struct, false, oneof); ECHECK(ParseProtoFields(anonymous_struct, false, oneof));
IsNext(';'); if (Is(';')) NEXT();
} else { } else {
Expect(';'); EXPECT(';');
} }
} }
} }
Next(); NEXT();
return NoError();
} }
void Parser::ParseProtoKey() { CheckedError Parser::ParseProtoKey() {
if (token_ == '(') { if (token_ == '(') {
Next(); NEXT();
// Skip "(a.b)" style custom attributes. // Skip "(a.b)" style custom attributes.
while (token_ == '.' || token_ == kTokenIdentifier) Next(); while (token_ == '.' || token_ == kTokenIdentifier) NEXT();
Expect(')'); EXPECT(')');
while (IsNext('.')) Expect(kTokenIdentifier); while (Is('.')) { NEXT(); EXPECT(kTokenIdentifier); }
} else { } else {
Expect(kTokenIdentifier); EXPECT(kTokenIdentifier);
} }
return NoError();
} }
void Parser::ParseProtoCurliesOrIdent() { CheckedError Parser::ParseProtoCurliesOrIdent() {
if (IsNext('{')) { if (Is('{')) {
NEXT();
for (int nesting = 1; nesting; ) { for (int nesting = 1; nesting; ) {
if (token_ == '{') nesting++; if (token_ == '{') nesting++;
else if (token_ == '}') nesting--; else if (token_ == '}') nesting--;
Next(); NEXT();
} }
} else { } else {
Next(); // Any single token. NEXT(); // Any single token.
} }
return NoError();
} }
void Parser::ParseProtoOption() { CheckedError Parser::ParseProtoOption() {
Next(); NEXT();
ParseProtoKey(); ECHECK(ParseProtoKey());
Expect('='); EXPECT('=');
ParseProtoCurliesOrIdent(); ECHECK(ParseProtoCurliesOrIdent());
return NoError();
} }
// Parse a protobuf type, and map it to the corresponding FlatBuffer one. // Parse a protobuf type, and map it to the corresponding FlatBuffer one.
Type Parser::ParseTypeFromProtoType() { CheckedError Parser::ParseTypeFromProtoType(Type *type) {
struct type_lookup { const char *proto_type; BaseType fb_type; }; struct type_lookup { const char *proto_type; BaseType fb_type; };
static type_lookup lookup[] = { static type_lookup lookup[] = {
{ "float", BASE_TYPE_FLOAT }, { "double", BASE_TYPE_DOUBLE }, { "float", BASE_TYPE_FLOAT }, { "double", BASE_TYPE_DOUBLE },
...@@ -1338,22 +1468,26 @@ Type Parser::ParseTypeFromProtoType() { ...@@ -1338,22 +1468,26 @@ Type Parser::ParseTypeFromProtoType() {
{ "bytes", BASE_TYPE_STRING }, { "bytes", BASE_TYPE_STRING },
{ nullptr, BASE_TYPE_NONE } { nullptr, BASE_TYPE_NONE }
}; };
Type type;
for (auto tl = lookup; tl->proto_type; tl++) { for (auto tl = lookup; tl->proto_type; tl++) {
if (attribute_ == tl->proto_type) { if (attribute_ == tl->proto_type) {
type.base_type = tl->fb_type; type->base_type = tl->fb_type;
Next(); NEXT();
return type; return NoError();
} }
} }
IsNext('.'); // qualified names may start with a . ? if (Is('.')) NEXT(); // qualified names may start with a . ?
ParseTypeIdent(type); ECHECK(ParseTypeIdent(*type));
return type; return NoError();
} }
bool Parser::Parse(const char *source, const char **include_paths, bool Parser::Parse(const char *source, const char **include_paths,
const char *source_filename) { const char *source_filename) {
files_being_parsed_ = source_filename ? source_filename : ""; return !DoParse(source, include_paths, source_filename).Check();
}
CheckedError Parser::DoParse(const char *source, const char **include_paths,
const char *source_filename) {
file_being_parsed_ = source_filename ? source_filename : "";
if (source_filename && if (source_filename &&
included_files_.find(source_filename) == included_files_.end()) { included_files_.find(source_filename) == included_files_.end()) {
included_files_[source_filename] = true; included_files_[source_filename] = true;
...@@ -1369,141 +1503,130 @@ bool Parser::Parse(const char *source, const char **include_paths, ...@@ -1369,141 +1503,130 @@ bool Parser::Parse(const char *source, const char **include_paths,
builder_.Clear(); builder_.Clear();
// Start with a blank namespace just in case this file doesn't have one. // Start with a blank namespace just in case this file doesn't have one.
namespaces_.push_back(new Namespace()); namespaces_.push_back(new Namespace());
try { NEXT();
Next(); // Includes must come before type declarations:
// Includes must come before type declarations: for (;;) {
for (;;) { // Parse pre-include proto statements if any:
// Parse pre-include proto statements if any: if (opts.proto_mode &&
if (opts.proto_mode && (attribute_ == "option" || attribute_ == "syntax" ||
(attribute_ == "option" || attribute_ == "syntax" || attribute_ == "package")) {
attribute_ == "package")) { ECHECK(ParseProtoDecl());
ParseProtoDecl(); } else if (Is(kTokenInclude) ||
} else if (IsNext(kTokenInclude) || (opts.proto_mode &&
(opts.proto_mode && attribute_ == "import" &&
attribute_ == "import" && Is(kTokenIdentifier))) {
IsNext(kTokenIdentifier))) { NEXT();
if (opts.proto_mode && attribute_ == "public") Next(); if (opts.proto_mode && attribute_ == "public") NEXT();
auto name = attribute_; auto name = attribute_;
Expect(kTokenStringConstant); EXPECT(kTokenStringConstant);
// Look for the file in include_paths. // Look for the file in include_paths.
std::string filepath; std::string filepath;
for (auto paths = include_paths; paths && *paths; paths++) { for (auto paths = include_paths; paths && *paths; paths++) {
filepath = flatbuffers::ConCatPathFileName(*paths, name); filepath = flatbuffers::ConCatPathFileName(*paths, name);
if(FileExists(filepath.c_str())) break; if(FileExists(filepath.c_str())) break;
}
if (filepath.empty())
Error("unable to locate include file: " + name);
if (source_filename)
files_included_per_file_[source_filename].insert(filepath);
if (included_files_.find(filepath) == included_files_.end()) {
// We found an include file that we have not parsed yet.
// Load it and parse it.
std::string contents;
if (!LoadFile(filepath.c_str(), true, &contents))
Error("unable to load include file: " + name);
if (!Parse(contents.c_str(), include_paths, filepath.c_str())) {
// Any errors, we're done.
return false;
}
// We generally do not want to output code for any included files:
if (!opts.generate_all) MarkGenerated();
// This is the easiest way to continue this file after an include:
// instead of saving and restoring all the state, we simply start the
// file anew. This will cause it to encounter the same include statement
// again, but this time it will skip it, because it was entered into
// included_files_.
// This is recursive, but only go as deep as the number of include
// statements.
return Parse(source, include_paths, source_filename);
}
Expect(';');
} else {
break;
} }
} if (filepath.empty())
// Now parse all other kinds of declarations: return Error("unable to locate include file: " + name);
while (token_ != kTokenEof) { if (source_filename)
if (opts.proto_mode) { files_included_per_file_[source_filename].insert(filepath);
ParseProtoDecl(); if (included_files_.find(filepath) == included_files_.end()) {
} else if (token_ == kTokenNameSpace) { // We found an include file that we have not parsed yet.
ParseNamespace(); // Load it and parse it.
} else if (token_ == '{') { std::string contents;
if (!root_struct_def_) Error("no root type set to parse json with"); if (!LoadFile(filepath.c_str(), true, &contents))
if (builder_.GetSize()) { return Error("unable to load include file: " + name);
Error("cannot have more than one json object in a file"); ECHECK(DoParse(contents.c_str(), include_paths, filepath.c_str()));
} // We generally do not want to output code for any included files:
builder_.Finish(Offset<Table>(ParseTable(*root_struct_def_, nullptr)), if (!opts.generate_all) MarkGenerated();
file_identifier_.length() ? file_identifier_.c_str() : nullptr); // This is the easiest way to continue this file after an include:
} else if (token_ == kTokenEnum) { // instead of saving and restoring all the state, we simply start the
ParseEnum(false); // file anew. This will cause it to encounter the same include
} else if (token_ == kTokenUnion) { // statement again, but this time it will skip it, because it was
ParseEnum(true); // entered into included_files_.
} else if (token_ == kTokenRootType) { // This is recursive, but only go as deep as the number of include
Next(); // statements.
auto root_type = attribute_; return DoParse(source, include_paths, source_filename);
Expect(kTokenIdentifier);
ParseNamespacing(&root_type, nullptr);
if (!SetRootType(root_type.c_str()))
Error("unknown root type: " + root_type);
if (root_struct_def_->fixed)
Error("root type must be a table");
Expect(';');
} else if (token_ == kTokenFileIdentifier) {
Next();
file_identifier_ = attribute_;
Expect(kTokenStringConstant);
if (file_identifier_.length() !=
FlatBufferBuilder::kFileIdentifierLength)
Error("file_identifier must be exactly " +
NumToString(FlatBufferBuilder::kFileIdentifierLength) +
" characters");
Expect(';');
} else if (token_ == kTokenFileExtension) {
Next();
file_extension_ = attribute_;
Expect(kTokenStringConstant);
Expect(';');
} else if(token_ == kTokenInclude) {
Error("includes must come before declarations");
} else if(token_ == kTokenAttribute) {
Next();
auto name = attribute_;
Expect(kTokenStringConstant);
Expect(';');
known_attributes_.insert(name);
} else {
ParseDecl();
} }
EXPECT(';');
} else {
break;
} }
for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) { }
if ((*it)->predecl) { // Now parse all other kinds of declarations:
Error("type referenced but not defined: " + (*it)->name); while (token_ != kTokenEof) {
if (opts.proto_mode) {
ECHECK(ParseProtoDecl());
} else if (token_ == kTokenNameSpace) {
ECHECK(ParseNamespace());
} else if (token_ == '{') {
if (!root_struct_def_)
return Error("no root type set to parse json with");
if (builder_.GetSize()) {
return Error("cannot have more than one json object in a file");
} }
uoffset_t toff;
ECHECK(ParseTable(*root_struct_def_, nullptr, &toff));
builder_.Finish(Offset<Table>(toff),
file_identifier_.length() ? file_identifier_.c_str() : nullptr);
} else if (token_ == kTokenEnum) {
ECHECK(ParseEnum(false, nullptr));
} else if (token_ == kTokenUnion) {
ECHECK(ParseEnum(true, nullptr));
} else if (token_ == kTokenRootType) {
NEXT();
auto root_type = attribute_;
EXPECT(kTokenIdentifier);
ECHECK(ParseNamespacing(&root_type, nullptr));
if (!SetRootType(root_type.c_str()))
return Error("unknown root type: " + root_type);
if (root_struct_def_->fixed)
return Error("root type must be a table");
EXPECT(';');
} else if (token_ == kTokenFileIdentifier) {
NEXT();
file_identifier_ = attribute_;
EXPECT(kTokenStringConstant);
if (file_identifier_.length() !=
FlatBufferBuilder::kFileIdentifierLength)
return Error("file_identifier must be exactly " +
NumToString(FlatBufferBuilder::kFileIdentifierLength) +
" characters");
EXPECT(';');
} else if (token_ == kTokenFileExtension) {
NEXT();
file_extension_ = attribute_;
EXPECT(kTokenStringConstant);
EXPECT(';');
} else if(token_ == kTokenInclude) {
return Error("includes must come before declarations");
} else if(token_ == kTokenAttribute) {
NEXT();
auto name = attribute_;
EXPECT(kTokenStringConstant);
EXPECT(';');
known_attributes_.insert(name);
} else {
ECHECK(ParseDecl());
} }
for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) { }
auto &enum_def = **it; for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) {
if (enum_def.is_union) { if ((*it)->predecl) {
for (auto val_it = enum_def.vals.vec.begin(); return Error("type referenced but not defined: " + (*it)->name);
val_it != enum_def.vals.vec.end(); }
++val_it) { }
auto &val = **val_it; for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
if (val.struct_def && val.struct_def->fixed) auto &enum_def = **it;
Error("only tables can be union elements: " + val.name); if (enum_def.is_union) {
} for (auto val_it = enum_def.vals.vec.begin();
val_it != enum_def.vals.vec.end();
++val_it) {
auto &val = **val_it;
if (val.struct_def && val.struct_def->fixed)
return Error("only tables can be union elements: " + val.name);
} }
} }
} catch (const std::string &msg) {
error_ = source_filename ? AbsolutePath(source_filename) : "";
#ifdef _WIN32
error_ += "(" + NumToString(line_) + ")"; // MSVC alike
#else
if (source_filename) error_ += ":";
error_ += NumToString(line_) + ":0"; // gcc alike
#endif
error_ += ": error: " + msg;
return false;
} }
return true; return NoError();
} }
std::set<std::string> Parser::GetIncludedFilesRecursive( std::set<std::string> Parser::GetIncludedFilesRecursive(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment