Skip to content

F
flatbuffers
Commit 40a33b1d authored by Wouter van Oortmerssen's avatar Wouter van Oortmerssen
Browse files
Options

Replaced exception handling in the parser with error checking. 

This to allow the code to run on a greater range of build
configurations (that don't allow exceptions/RTTI).

If anyone ever doubts the usefulness of exception handling,
please show them this commit.

Change-Id: If7190babdde93c3f9cd97b8e1ab447bf0c81696d
Tested: on Linux.
parent a9909a0c
Hide whitespace changes
Inline Side-by-side
Showing with 704 additions and 534 deletions
CMakeLists.txt
View file @ 40a33b1d
......@@ -81,7 +81,7 @@ if(APPLE)
"${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -Wall -pedantic -Werror -Wextra")
elseif(CMAKE_COMPILER_IS_GNUCXX)
set(CMAKE_CXX_FLAGS
"${CMAKE_CXX_FLAGS} -std=c++0x -Wall -pedantic -Werror -Wextra -Werror=shadow")
"${CMAKE_CXX_FLAGS} -std=c++0x -Wall -pedantic -Werror -Wextra -Werror=shadow -Wunused-result -Werror=unused-result")
elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
set(CMAKE_CXX_FLAGS
"${CMAKE_CXX_FLAGS} -std=c++0x -stdlib=libc++ -Wall -pedantic -Werror -Wextra")
......
include/flatbuffers/idl.h
View file @ 40a33b1d
......@@ -340,6 +340,46 @@ struct IDLOptions {
lang(IDLOptions::kJava) {}
};
// A way to make error propagation less error prone by requiring values to be
// checked.
// Once you create a value of this type you must either:
// - Call Check() on it.
// - Copy or assign it to another value.
// Failure to do so leads to an assert.
// This guarantees that this as return value cannot be ignored.
class CheckedError {
public:
explicit CheckedError(bool error)
: is_error_(error), has_been_checked_(false) {}
CheckedError &operator=(const CheckedError &other) {
is_error_ = other.is_error_;
has_been_checked_ = false;
other.has_been_checked_ = true;
return *this;
}
CheckedError(const CheckedError &other) {
*this = other; // Use assignment operator.
}
~CheckedError() { assert(has_been_checked_); }
bool Check() { has_been_checked_ = true; return is_error_; }
private:
bool is_error_;
mutable bool has_been_checked_;
};
// Additionally, in GCC we can get these errors statically, for additional
// assurance:
#ifdef __GNUC__
#define CHECKED_ERROR CheckedError __attribute__((warn_unused_result))
#else
#define CHECKED_ERROR CheckedError
#endif
class Parser {
public:
explicit Parser(const IDLOptions &options = IDLOptions())
......@@ -395,44 +435,51 @@ class Parser {
// See reflection/reflection.fbs
void Serialize();
private:
int64_t ParseHexNum(int nibbles);
void Next();
bool IsNext(int t);
void Expect(int t);
CHECKED_ERROR CheckBitsFit(int64_t val, size_t bits);
private:
CHECKED_ERROR Error(const std::string &msg);
CHECKED_ERROR ParseHexNum(int nibbles, int64_t *val);
CHECKED_ERROR Next();
bool Is(int t);
CHECKED_ERROR Expect(int t);
std::string TokenToStringId(int t);
EnumDef *LookupEnum(const std::string &id);
void ParseNamespacing(std::string *id, std::string *last);
void ParseTypeIdent(Type &type);
void ParseType(Type &type);
FieldDef &AddField(StructDef &struct_def,
const std::string &name,
const Type &type);
void ParseField(StructDef &struct_def);
void ParseAnyValue(Value &val, FieldDef *field, size_t parent_fieldn);
uoffset_t ParseTable(const StructDef &struct_def, std::string *value);
CHECKED_ERROR ParseNamespacing(std::string *id, std::string *last);
CHECKED_ERROR ParseTypeIdent(Type &type);
CHECKED_ERROR ParseType(Type &type);
CHECKED_ERROR AddField(StructDef &struct_def, const std::string &name,
const Type &type, FieldDef **dest);
CHECKED_ERROR ParseField(StructDef &struct_def);
CHECKED_ERROR ParseAnyValue(Value &val, FieldDef *field, size_t parent_fieldn);
CHECKED_ERROR ParseTable(const StructDef &struct_def, std::string *value,
uoffset_t *ovalue);
void SerializeStruct(const StructDef &struct_def, const Value &val);
void AddVector(bool sortbysize, int count);
uoffset_t ParseVector(const Type &type);
void ParseMetaData(Definition &def);
bool TryTypedValue(int dtoken, bool check, Value &e, BaseType req);
void ParseHash(Value &e, FieldDef* field);
void ParseSingleValue(Value &e);
int64_t ParseIntegerFromString(Type &type);
CHECKED_ERROR ParseVector(const Type &type, uoffset_t *ovalue);
CHECKED_ERROR ParseMetaData(Definition &def);
CHECKED_ERROR TryTypedValue(int dtoken, bool check, Value &e, BaseType req,
bool *destmatch);
CHECKED_ERROR ParseHash(Value &e, FieldDef* field);
CHECKED_ERROR ParseSingleValue(Value &e);
CHECKED_ERROR ParseIntegerFromString(Type &type, int64_t *result);
StructDef *LookupCreateStruct(const std::string &name,
bool create_if_new = true,
bool definition = false);
EnumDef &ParseEnum(bool is_union);
void ParseNamespace();
StructDef &StartStruct(const std::string &name);
void ParseDecl();
void ParseProtoFields(StructDef *struct_def, bool isextend,
bool inside_oneof);
void ParseProtoOption();
void ParseProtoKey();
void ParseProtoDecl();
void ParseProtoCurliesOrIdent();
Type ParseTypeFromProtoType();
CHECKED_ERROR ParseEnum(bool is_union, EnumDef **dest);
CHECKED_ERROR ParseNamespace();
CHECKED_ERROR StartStruct(const std::string &name, StructDef **dest);
CHECKED_ERROR ParseDecl();
CHECKED_ERROR ParseProtoFields(StructDef *struct_def, bool isextend,
bool inside_oneof);
CHECKED_ERROR ParseProtoOption();
CHECKED_ERROR ParseProtoKey();
CHECKED_ERROR ParseProtoDecl();
CHECKED_ERROR ParseProtoCurliesOrIdent();
CHECKED_ERROR ParseTypeFromProtoType(Type *type);
CHECKED_ERROR DoParse(const char *_source, const char **include_paths,
const char *source_filename);
public:
SymbolTable<StructDef> structs_;
......@@ -454,7 +501,7 @@ class Parser {
const char *source_, *cursor_;
int line_; // the current line being parsed
int token_;
std::string files_being_parsed_;
std::string file_being_parsed_;
std::string attribute_;
std::vector<std::string> doc_comment_;
......
src/idl_parser.cpp
View file @ 40a33b1d
......@@ -43,38 +43,68 @@ static_assert(BASE_TYPE_UNION ==
static_cast<BaseType>(reflection::Union),
"enums don't match");
static void Error(const std::string &msg) {
throw msg;
#define ECHECK(call) { auto ce = (call); if (ce.Check()) return ce; }
#define NEXT() ECHECK(Next())
#define EXPECT(tok) ECHECK(Expect(tok))
CheckedError Parser::Error(const std::string &msg) {
error_ = file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : "";
#ifdef _WIN32
error_ += "(" + NumToString(line_) + ")"; // MSVC alike
#else
if (file_being_parsed_.length()) error_ += ":";
error_ += NumToString(line_) + ":0"; // gcc alike
#endif
error_ += ": error: " + msg;
return CheckedError(true);
}
inline CheckedError NoError() { return CheckedError(false); }
// Ensure that integer values we parse fit inside the declared integer type.
static void CheckBitsFit(int64_t val, size_t bits) {
CheckedError Parser::CheckBitsFit(int64_t val, size_t bits) {
// Bits we allow to be used.
auto mask = static_cast<int64_t>((1ull << bits) - 1);
if (bits < 64 &&
(val & ~mask) != 0 && // Positive or unsigned.
(val | mask) != -1) // Negative.
Error("constant does not fit in a " + NumToString(bits) + "-bit field");
return Error("constant does not fit in a " + NumToString(bits) +
"-bit field");
return NoError();
}
// atot: templated version of atoi/atof: convert a string to an instance of T.
template<typename T> inline T atot(const char *s) {
auto val = StringToInt(s);
CheckBitsFit(val, sizeof(T) * 8);
return (T)val;
template<typename T> inline CheckedError atot(const char *s, Parser &parser,
T *val) {
int64_t i = StringToInt(s);
ECHECK(parser.CheckBitsFit(i, sizeof(T) * 8));
*val = (T)i;
return NoError();
}
template<> inline bool atot<bool>(const char *s) {
return 0 != atoi(s);
template<> inline CheckedError atot<bool>(const char *s, Parser &parser,
bool *val) {
(void)parser;
*val = 0 != atoi(s);
return NoError();
}
template<> inline float atot<float>(const char *s) {
return static_cast<float>(strtod(s, nullptr));
template<> inline CheckedError atot<float>(const char *s, Parser &parser,
float *val) {
(void)parser;
*val = static_cast<float>(strtod(s, nullptr));
return NoError();
}
template<> inline double atot<double>(const char *s) {
return strtod(s, nullptr);
template<> inline CheckedError atot<double>(const char *s, Parser &parser,
double *val) {
(void)parser;
*val = strtod(s, nullptr);
return NoError();
}
template<> inline Offset<void> atot<Offset<void>>(const char *s) {
return Offset<void>(atoi(s));
template<> inline CheckedError atot<Offset<void>>(const char *s, Parser &parser,
Offset<void> *val) {
(void)parser;
*val = Offset<void>(atoi(s));
return NoError();
}
std::string Namespace::GetFullyQualifiedName(const std::string &name,
......@@ -153,18 +183,18 @@ std::string Parser::TokenToStringId(int t) {
}
// Parses exactly nibbles worth of hex digits into a number, or error.
int64_t Parser::ParseHexNum(int nibbles) {
CheckedError Parser::ParseHexNum(int nibbles, int64_t *val) {
for (int i = 0; i < nibbles; i++)
if (!isxdigit(cursor_[i]))
Error("escape code must be followed by " + NumToString(nibbles) +
" hex digits");
return Error("escape code must be followed by " + NumToString(nibbles) +
" hex digits");
std::string target(cursor_, cursor_ + nibbles);
auto val = StringToUInt(target.c_str(), 16);
*val = StringToUInt(target.c_str(), 16);
cursor_ += nibbles;
return val;
return NoError();
}
void Parser::Next() {
CheckedError Parser::Next() {
doc_comment_.clear();
bool seen_newline = false;
attribute_.clear();
......@@ -172,20 +202,19 @@ void Parser::Next() {
char c = *cursor_++;
token_ = c;
switch (c) {
case '\0': cursor_--; token_ = kTokenEof; return;
case '\0': cursor_--; token_ = kTokenEof; return NoError();
case ' ': case '\r': case '\t': break;
case '\n': line_++; seen_newline = true; break;
case '{': case '}': case '(': case ')': case '[': case ']': return;
case ',': case ':': case ';': case '=': return;
case '{': case '}': case '(': case ')': case '[': case ']':
case ',': case ':': case ';': case '=': return NoError();
case '.':
if(!isdigit(*cursor_)) return;
Error("floating point constant can\'t start with \".\"");
break;
if(!isdigit(*cursor_)) return NoError();
return Error("floating point constant can\'t start with \".\"");
case '\"':
case '\'':
while (*cursor_ != c) {
if (*cursor_ < ' ' && *cursor_ >= 0)
Error("illegal character in string constant");
return Error("illegal character in string constant");
if (*cursor_ == '\\') {
cursor_++;
switch (*cursor_) {
......@@ -200,15 +229,19 @@ void Parser::Next() {
case '/': attribute_ += '/'; cursor_++; break;
case 'x': { // Not in the JSON standard
cursor_++;
attribute_ += static_cast<char>(ParseHexNum(2));
int64_t val;
ECHECK(ParseHexNum(2, &val));
attribute_ += static_cast<char>(val);
break;
}
case 'u': {
cursor_++;
ToUTF8(static_cast<int>(ParseHexNum(4)), &attribute_);
int64_t val;
ECHECK(ParseHexNum(4, &val));
ToUTF8(static_cast<int>(val), &attribute_);
break;
}
default: Error("unknown escape code in string constant"); break;
default: return Error("unknown escape code in string constant");
}
} else { // printable chars + UTF-8 bytes
attribute_ += *cursor_++;
......@@ -216,14 +249,15 @@ void Parser::Next() {
}
cursor_++;
token_ = kTokenStringConstant;
return;
return NoError();
case '/':
if (*cursor_ == '/') {
const char *start = ++cursor_;
while (*cursor_ && *cursor_ != '\n' && *cursor_ != '\r') cursor_++;
if (*start == '/') { // documentation comment
if (cursor_ != source_ && !seen_newline)
Error("a documentation comment should be on a line on its own");
return Error(
"a documentation comment should be on a line on its own");
doc_comment_.push_back(std::string(start + 1, cursor_));
}
break;
......@@ -231,7 +265,7 @@ void Parser::Next() {
cursor_++;
// TODO: make nested.
while (*cursor_ != '*' || cursor_[1] != '/') {
if (!*cursor_) Error("end of file in comment");
if (!*cursor_) return Error("end of file in comment");
cursor_++;
}
cursor_ += 2;
......@@ -251,7 +285,7 @@ void Parser::Next() {
PTYPE) \
if (attribute_ == IDLTYPE) { \
token_ = kToken ## ENUM; \
return; \
return NoError(); \
}
FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
#undef FLATBUFFERS_TD
......@@ -260,28 +294,52 @@ void Parser::Next() {
if (attribute_ == "true" || attribute_ == "false") {
attribute_ = NumToString(attribute_ == "true");
token_ = kTokenIntegerConstant;
return;
return NoError();
}
// Check for declaration keywords:
if (attribute_ == "table") { token_ = kTokenTable; return; }
if (attribute_ == "struct") { token_ = kTokenStruct; return; }
if (attribute_ == "enum") { token_ = kTokenEnum; return; }
if (attribute_ == "union") { token_ = kTokenUnion; return; }
if (attribute_ == "namespace") { token_ = kTokenNameSpace; return; }
if (attribute_ == "root_type") { token_ = kTokenRootType; return; }
if (attribute_ == "include") { token_ = kTokenInclude; return; }
if (attribute_ == "attribute") { token_ = kTokenAttribute; return; }
if (attribute_ == "table") {
token_ = kTokenTable;
return NoError();
}
if (attribute_ == "struct") {
token_ = kTokenStruct;
return NoError();
}
if (attribute_ == "enum") {
token_ = kTokenEnum;
return NoError();
}
if (attribute_ == "union") {
token_ = kTokenUnion;
return NoError();
}
if (attribute_ == "namespace") {
token_ = kTokenNameSpace;
return NoError();
}
if (attribute_ == "root_type") {
token_ = kTokenRootType;
return NoError();
}
if (attribute_ == "include") {
token_ = kTokenInclude;
return NoError();
}
if (attribute_ == "attribute") {
token_ = kTokenAttribute;
return NoError();
}
if (attribute_ == "file_identifier") {
token_ = kTokenFileIdentifier;
return;
return NoError();
}
if (attribute_ == "file_extension") {
token_ = kTokenFileExtension;
return;
return NoError();
}
// If not, it is a user-defined identifier:
token_ = kTokenIdentifier;
return;
return NoError();
} else if (isdigit(static_cast<unsigned char>(c)) || c == '-') {
const char *start = cursor_ - 1;
if (c == '0' && (*cursor_ == 'x' || *cursor_ == 'X')) {
......@@ -290,7 +348,7 @@ void Parser::Next() {
attribute_.append(start + 2, cursor_);
attribute_ = NumToString(StringToUInt(attribute_.c_str(), 16));
token_ = kTokenIntegerConstant;
return;
return NoError();
}
while (isdigit(static_cast<unsigned char>(*cursor_))) cursor_++;
if (*cursor_ == '.' || *cursor_ == 'e' || *cursor_ == 'E') {
......@@ -310,56 +368,57 @@ void Parser::Next() {
token_ = kTokenIntegerConstant;
}
attribute_.append(start, cursor_);
return;
return NoError();
}
std::string ch;
ch = c;
if (c < ' ' || c > '~') ch = "code: " + NumToString(c);
Error("illegal character: " + ch);
break;
return Error("illegal character: " + ch);
}
}
}
// Check if a given token is next, if so, consume it as well.
bool Parser::IsNext(int t) {
bool isnext = t == token_;
if (isnext) Next();
return isnext;
// Check if a given token is next.
bool Parser::Is(int t) {
return t == token_;
}
// Expect a given token to be next, consume it, or error if not present.
void Parser::Expect(int t) {
CheckedError Parser::Expect(int t) {
if (t != token_) {
Error("expecting: " + TokenToString(t) + " instead got: " +
TokenToStringId(token_));
return Error("expecting: " + TokenToString(t) + " instead got: " +
TokenToStringId(token_));
}
Next();
NEXT();
return NoError();
}
void Parser::ParseNamespacing(std::string *id, std::string *last) {
while (IsNext('.')) {
CheckedError Parser::ParseNamespacing(std::string *id, std::string *last) {
while (Is('.')) {
NEXT();
*id += ".";
*id += attribute_;
if (last) *last = attribute_;
Expect(kTokenIdentifier);
EXPECT(kTokenIdentifier);
}
return NoError();
}
EnumDef *Parser::LookupEnum(const std::string &id) {
// Search thru parent namespaces.
for (int components = static_cast<int>(namespaces_.back()->components.size());
components >= 0; components--) {
auto ed = enums_.Lookup(namespaces_.back()->GetFullyQualifiedName(id, components));
auto ed = enums_.Lookup(
namespaces_.back()->GetFullyQualifiedName(id, components));
if (ed) return ed;
}
return nullptr;
}
void Parser::ParseTypeIdent(Type &type) {
CheckedError Parser::ParseTypeIdent(Type &type) {
std::string id = attribute_;
Expect(kTokenIdentifier);
ParseNamespacing(&id, nullptr);
EXPECT(kTokenIdentifier);
ECHECK(ParseNamespacing(&id, nullptr));
auto enum_def = LookupEnum(id);
if (enum_def) {
type = enum_def->underlying_type;
......@@ -368,41 +427,45 @@ void Parser::ParseTypeIdent(Type &type) {
type.base_type = BASE_TYPE_STRUCT;
type.struct_def = LookupCreateStruct(id);
}
return NoError();
}
// Parse any IDL type.
void Parser::ParseType(Type &type) {
CheckedError Parser::ParseType(Type &type) {
if (token_ >= kTokenBOOL && token_ <= kTokenSTRING) {
type.base_type = static_cast<BaseType>(token_ - kTokenNONE);
Next();
NEXT();
} else {
if (token_ == kTokenIdentifier) {
ParseTypeIdent(type);
ECHECK(ParseTypeIdent(type));
} else if (token_ == '[') {
Next();
NEXT();
Type subtype;
ParseType(subtype);
ECHECK(ParseType(subtype));
if (subtype.base_type == BASE_TYPE_VECTOR) {
// We could support this, but it will complicate things, and it's
// easier to work around with a struct around the inner vector.
Error("nested vector types not supported (wrap in table first).");
return Error(
"nested vector types not supported (wrap in table first).");
}
if (subtype.base_type == BASE_TYPE_UNION) {
// We could support this if we stored a struct of 2 elements per
// union element.
Error("vector of union types not supported (wrap in table first).");
return Error(
"vector of union types not supported (wrap in table first).");
}
type = Type(BASE_TYPE_VECTOR, subtype.struct_def, subtype.enum_def);
type.element = subtype.base_type;
Expect(']');
EXPECT(']');
} else {
Error("illegal type syntax");
return Error("illegal type syntax");
}
}
return NoError();
}
FieldDef &Parser::AddField(StructDef &struct_def, const std::string &name,
const Type &type) {
CheckedError Parser::AddField(StructDef &struct_def, const std::string &name,
const Type &type, FieldDef **dest) {
auto &field = *new FieldDef();
field.value.offset =
FieldIndexToOffset(static_cast<voffset_t>(struct_def.fields.vec.size()));
......@@ -420,36 +483,38 @@ FieldDef &Parser::AddField(StructDef &struct_def, const std::string &name,
struct_def.bytesize += size;
}
if (struct_def.fields.Add(name, &field))
Error("field already exists: " + name);
return field;
return Error("field already exists: " + name);
*dest = &field;
return NoError();
}
void Parser::ParseField(StructDef &struct_def) {
CheckedError Parser::ParseField(StructDef &struct_def) {
std::string name = attribute_;
std::vector<std::string> dc = doc_comment_;
Expect(kTokenIdentifier);
Expect(':');
EXPECT(kTokenIdentifier);
EXPECT(':');
Type type;
ParseType(type);
ECHECK(ParseType(type));
if (struct_def.fixed && !IsScalar(type.base_type) && !IsStruct(type))
Error("structs_ may contain only scalar or struct fields");
return Error("structs_ may contain only scalar or struct fields");
FieldDef *typefield = nullptr;
if (type.base_type == BASE_TYPE_UNION) {
// For union fields, add a second auto-generated field to hold the type,
// with _type appended as the name.
typefield = &AddField(struct_def, name + "_type",
type.enum_def->underlying_type);
ECHECK(AddField(struct_def, name + "_type", type.enum_def->underlying_type,
&typefield));
}
auto &field = AddField(struct_def, name, type);
FieldDef *field;
ECHECK(AddField(struct_def, name, type, &field));
if (token_ == '=') {
Next();
NEXT();
if (!IsScalar(type.base_type))
Error("default values currently only supported for scalars");
ParseSingleValue(field.value);
return Error("default values currently only supported for scalars");
ECHECK(ParseSingleValue(field->value));
}
if (type.enum_def &&
......@@ -457,59 +522,62 @@ void Parser::ParseField(StructDef &struct_def) {
!struct_def.fixed &&
!type.enum_def->attributes.Lookup("bit_flags") &&
!type.enum_def->ReverseLookup(static_cast<int>(
StringToInt(field.value.constant.c_str()))))
Error("enum " + type.enum_def->name +
StringToInt(field->value.constant.c_str()))))
return Error("enum " + type.enum_def->name +
" does not have a declaration for this field\'s default of " +
field.value.constant);
field->value.constant);
field.doc_comment = dc;
ParseMetaData(field);
field.deprecated = field.attributes.Lookup("deprecated") != nullptr;
auto hash_name = field.attributes.Lookup("hash");
field->doc_comment = dc;
ECHECK(ParseMetaData(*field));
field->deprecated = field->attributes.Lookup("deprecated") != nullptr;
auto hash_name = field->attributes.Lookup("hash");
if (hash_name) {
switch (type.base_type) {
case BASE_TYPE_INT:
case BASE_TYPE_UINT: {
if (FindHashFunction32(hash_name->constant.c_str()) == nullptr)
Error("Unknown hashing algorithm for 32 bit types: " +
return Error("Unknown hashing algorithm for 32 bit types: " +
hash_name->constant);
break;
}
case BASE_TYPE_LONG:
case BASE_TYPE_ULONG: {
if (FindHashFunction64(hash_name->constant.c_str()) == nullptr)
Error("Unknown hashing algorithm for 64 bit types: " +
return Error("Unknown hashing algorithm for 64 bit types: " +
hash_name->constant);
break;
}
default:
Error("only int, uint, long and ulong data types support hashing.");
return Error(
"only int, uint, long and ulong data types support hashing.");
}
}
if (field.deprecated && struct_def.fixed)
Error("can't deprecate fields in a struct");
field.required = field.attributes.Lookup("required") != nullptr;
if (field.required && (struct_def.fixed ||
IsScalar(field.value.type.base_type)))
Error("only non-scalar fields in tables may be 'required'");
field.key = field.attributes.Lookup("key") != nullptr;
if (field.key) {
if (field->deprecated && struct_def.fixed)
return Error("can't deprecate fields in a struct");
field->required = field->attributes.Lookup("required") != nullptr;
if (field->required && (struct_def.fixed ||
IsScalar(field->value.type.base_type)))
return Error("only non-scalar fields in tables may be 'required'");
field->key = field->attributes.Lookup("key") != nullptr;
if (field->key) {
if (struct_def.has_key)
Error("only one field may be set as 'key'");
return Error("only one field may be set as 'key'");
struct_def.has_key = true;
if (!IsScalar(field.value.type.base_type)) {
field.required = true;
if (field.value.type.base_type != BASE_TYPE_STRING)
Error("'key' field must be string or scalar type");
if (!IsScalar(field->value.type.base_type)) {
field->required = true;
if (field->value.type.base_type != BASE_TYPE_STRING)
return Error("'key' field must be string or scalar type");
}
}
auto nested = field.attributes.Lookup("nested_flatbuffer");
auto nested = field->attributes.Lookup("nested_flatbuffer");
if (nested) {
if (nested->type.base_type != BASE_TYPE_STRING)
Error("nested_flatbuffer attribute must be a string (the root type)");
if (field.value.type.base_type != BASE_TYPE_VECTOR ||
field.value.type.element != BASE_TYPE_UCHAR)
Error("nested_flatbuffer attribute may only apply to a vector of ubyte");
return Error(
"nested_flatbuffer attribute must be a string (the root type)");
if (field->value.type.base_type != BASE_TYPE_VECTOR ||
field->value.type.element != BASE_TYPE_UCHAR)
return Error(
"nested_flatbuffer attribute may only apply to a vector of ubyte");
// This will cause an error if the root type of the nested flatbuffer
// wasn't defined elsewhere.
LookupCreateStruct(nested->constant);
......@@ -518,7 +586,7 @@ void Parser::ParseField(StructDef &struct_def) {
if (typefield) {
// If this field is a union, and it has a manually assigned id,
// the automatically added type field should have an id as well (of N - 1).
auto attr = field.attributes.Lookup("id");
auto attr = field->attributes.Lookup("id");
if (attr) {
auto id = atoi(attr->constant.c_str());
auto val = new Value();
......@@ -528,35 +596,41 @@ void Parser::ParseField(StructDef &struct_def) {
}
}
Expect(';');
EXPECT(';');
return NoError();
}
void Parser::ParseAnyValue(Value &val, FieldDef *field, size_t parent_fieldn) {
CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
size_t parent_fieldn) {
switch (val.type.base_type) {
case BASE_TYPE_UNION: {
assert(field);
if (!parent_fieldn ||
field_stack_.back().second->value.type.base_type != BASE_TYPE_UTYPE)
Error("missing type field before this union value: " + field->name);
auto enum_idx = atot<unsigned char>(
field_stack_.back().first.constant.c_str());
return Error("missing type field before this union value: " +
field->name);
uint8_t enum_idx;
ECHECK(atot(field_stack_.back().first.constant.c_str(), *this,
&enum_idx));
auto enum_val = val.type.enum_def->ReverseLookup(enum_idx);
if (!enum_val) Error("illegal type id for: " + field->name);
ParseTable(*enum_val->struct_def, &val.constant);
if (!enum_val) return Error("illegal type id for: " + field->name);
ECHECK(ParseTable(*enum_val->struct_def, &val.constant, nullptr));
break;
}
case BASE_TYPE_STRUCT:
ParseTable(*val.type.struct_def, &val.constant);
ECHECK(ParseTable(*val.type.struct_def, &val.constant, nullptr));
break;
case BASE_TYPE_STRING: {
auto s = attribute_;
Expect(kTokenStringConstant);
EXPECT(kTokenStringConstant);
val.constant = NumToString(builder_.CreateString(s).o);
break;
}
case BASE_TYPE_VECTOR: {
Expect('[');
val.constant = NumToString(ParseVector(val.type.VectorType()));
EXPECT('[');
uoffset_t off;
ECHECK(ParseVector(val.type.VectorType(), &off));
val.constant = NumToString(off);
break;
}
case BASE_TYPE_INT:
......@@ -565,16 +639,17 @@ void Parser::ParseAnyValue(Value &val, FieldDef *field, size_t parent_fieldn) {
case BASE_TYPE_ULONG: {
if (field && field->attributes.Lookup("hash") &&
(token_ == kTokenIdentifier || token_ == kTokenStringConstant)) {
ParseHash(val, field);
ECHECK(ParseHash(val, field));
} else {
ParseSingleValue(val);
ECHECK(ParseSingleValue(val));
}
break;
}
default:
ParseSingleValue(val);
ECHECK(ParseSingleValue(val));
break;
}
return NoError();
}
void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) {
......@@ -585,35 +660,39 @@ void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) {
builder_.AddStructOffset(val.offset, builder_.GetSize());
}
uoffset_t Parser::ParseTable(const StructDef &struct_def, std::string *value) {
Expect('{');
CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
uoffset_t *ovalue) {
EXPECT('{');
size_t fieldn = 0;
for (;;) {
if ((!opts.strict_json || !fieldn) && IsNext('}')) break;
if ((!opts.strict_json || !fieldn) && Is('}')) { NEXT(); break; }
std::string name = attribute_;
if (!IsNext(kTokenStringConstant))
Expect(opts.strict_json ? kTokenStringConstant : kTokenIdentifier);
if (Is(kTokenStringConstant)) {
NEXT();
} else {
EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier);
}
auto field = struct_def.fields.Lookup(name);
if (!field) Error("unknown field: " + name);
Expect(':');
if (!field) return Error("unknown field: " + name);
EXPECT(':');
Value val = field->value;
ParseAnyValue(val, field, fieldn);
ECHECK(ParseAnyValue(val, field, fieldn));
size_t i = field_stack_.size();
// Hardcoded insertion-sort with error-check.
// If fields are specified in order, then this loop exits immediately.
for (; i > field_stack_.size() - fieldn; i--) {
auto existing_field = field_stack_[i - 1].second;
if (existing_field == field)
Error("field set more than once: " + field->name);
return Error("field set more than once: " + field->name);
if (existing_field->value.offset < field->value.offset) break;
}
field_stack_.insert(field_stack_.begin() + i, std::make_pair(val, field));
fieldn++;
if (IsNext('}')) break;
Expect(',');
if (Is('}')) { NEXT(); break; }
EXPECT(',');
}
if (struct_def.fixed && fieldn != struct_def.fields.vec.size())
Error("struct: wrong number of initializers: " + struct_def.name);
return Error("struct: wrong number of initializers: " + struct_def.name);
auto start = struct_def.fixed
? builder_.StartStruct(struct_def.minalign)
......@@ -627,18 +706,22 @@ uoffset_t Parser::ParseTable(const StructDef &struct_def, std::string *value) {
it != field_stack_.rbegin() + fieldn; ++it) {
auto &field_value = it->first;
auto field = it->second;
if (!struct_def.sortbysize || size == SizeOf(field_value.type.base_type)) {
if (!struct_def.sortbysize ||
size == SizeOf(field_value.type.base_type)) {
switch (field_value.type.base_type) {
#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \
PTYPE) \
case BASE_TYPE_ ## ENUM: \
builder_.Pad(field->padding); \
if (struct_def.fixed) { \
builder_.PushElement(atot<CTYPE>(field_value.constant.c_str())); \
CTYPE val; \
ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
builder_.PushElement(val); \
} else { \
builder_.AddElement(field_value.offset, \
atot<CTYPE>( field_value.constant.c_str()), \
atot<CTYPE>(field->value.constant.c_str())); \
CTYPE val, valdef; \
ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \
builder_.AddElement(field_value.offset, val, valdef); \
} \
break;
FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD);
......@@ -650,8 +733,9 @@ uoffset_t Parser::ParseTable(const StructDef &struct_def, std::string *value) {
if (IsStruct(field->value.type)) { \
SerializeStruct(*field->value.type.struct_def, field_value); \
} else { \
builder_.AddOffset(field_value.offset, \
atot<CTYPE>(field_value.constant.c_str())); \
CTYPE val; \
ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
builder_.AddOffset(field_value.offset, val); \
} \
break;
FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD);
......@@ -672,27 +756,27 @@ uoffset_t Parser::ParseTable(const StructDef &struct_def, std::string *value) {
reinterpret_cast<const char *>(builder_.GetCurrentBufferPointer()),
struct_def.bytesize);
builder_.PopBytes(struct_def.bytesize);
return 0xFFFFFFFF; // Value not used by the caller.
assert(!ovalue);
} else {
auto off = builder_.EndTable(
start,
static_cast<voffset_t>(struct_def.fields.vec.size()));
if (value) *value = NumToString(off);
return off;
auto val = builder_.EndTable(start,
static_cast<voffset_t>(struct_def.fields.vec.size()));
if (ovalue) *ovalue = val;
if (value) *value = NumToString(val);
}
return NoError();
}
uoffset_t Parser::ParseVector(const Type &type) {
CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) {
int count = 0;
for (;;) {
if ((!opts.strict_json || !count) && IsNext(']')) break;
if ((!opts.strict_json || !count) && Is(']')) { NEXT(); break; }
Value val;
val.type = type;
ParseAnyValue(val, nullptr, 0);
ECHECK(ParseAnyValue(val, nullptr, 0));
field_stack_.push_back(std::make_pair(val, nullptr));
count++;
if (IsNext(']')) break;
Expect(',');
if (Is(']')) { NEXT(); break; }
EXPECT(',');
}
builder_.StartVector(count * InlineSize(type) / InlineAlignment(type),
......@@ -704,7 +788,11 @@ uoffset_t Parser::ParseVector(const Type &type) {
#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \
case BASE_TYPE_ ## ENUM: \
if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \
else builder_.PushElement(atot<CTYPE>(val.constant.c_str())); \
else { \
CTYPE elem; \
ECHECK(atot(val.constant.c_str(), *this, &elem)); \
builder_.PushElement(elem); \
} \
break;
FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
#undef FLATBUFFERS_TD
......@@ -713,51 +801,55 @@ uoffset_t Parser::ParseVector(const Type &type) {
}
builder_.ClearOffsets();
return builder_.EndVector(count);
*ovalue = builder_.EndVector(count);
return NoError();
}
void Parser::ParseMetaData(Definition &def) {
if (IsNext('(')) {
CheckedError Parser::ParseMetaData(Definition &def) {
if (Is('(')) {
NEXT();
for (;;) {
auto name = attribute_;
Expect(kTokenIdentifier);
EXPECT(kTokenIdentifier);
if (known_attributes_.find(name) == known_attributes_.end())
Error("user define attributes must be declared before use: " + name);
return Error("user define attributes must be declared before use: " +
name);
auto e = new Value();
def.attributes.Add(name, e);
if (IsNext(':')) {
ParseSingleValue(*e);
if (Is(':')) {
NEXT();
ECHECK(ParseSingleValue(*e));
}
if (IsNext(')')) break;
Expect(',');
if (Is(')')) { NEXT(); break; }
EXPECT(',');
}
}
return NoError();
}
bool Parser::TryTypedValue(int dtoken,
bool check,
Value &e,
BaseType req) {
CheckedError Parser::TryTypedValue(int dtoken, bool check, Value &e,
BaseType req, bool *destmatch) {
bool match = dtoken == token_;
if (match) {
*destmatch = true;
e.constant = attribute_;
if (!check) {
if (e.type.base_type == BASE_TYPE_NONE) {
e.type.base_type = req;
} else {
Error(std::string("type mismatch: expecting: ") +
kTypeNames[e.type.base_type] +
", found: " +
kTypeNames[req]);
return Error(std::string("type mismatch: expecting: ") +
kTypeNames[e.type.base_type] +
", found: " +
kTypeNames[req]);
}
}
Next();
NEXT();
}
return match;
return NoError();
}
int64_t Parser::ParseIntegerFromString(Type &type) {
int64_t result = 0;
CheckedError Parser::ParseIntegerFromString(Type &type, int64_t *result) {
*result = 0;
// Parse one or more enum identifiers, separated by spaces.
const char *next = attribute_.c_str();
do {
......@@ -773,29 +865,30 @@ int64_t Parser::ParseIntegerFromString(Type &type) {
if (type.enum_def) { // The field has an enum type
auto enum_val = type.enum_def->vals.Lookup(word);
if (!enum_val)
Error("unknown enum value: " + word +
return Error("unknown enum value: " + word +
", for enum: " + type.enum_def->name);
result |= enum_val->value;
*result |= enum_val->value;
} else { // No enum type, probably integral field.
if (!IsInteger(type.base_type))
Error("not a valid value for this field: " + word);
return Error("not a valid value for this field: " + word);
// TODO: could check if its a valid number constant here.
const char *dot = strrchr(word.c_str(), '.');
if (!dot) Error("enum values need to be qualified by an enum type");
if (!dot)
return Error("enum values need to be qualified by an enum type");
std::string enum_def_str(word.c_str(), dot);
std::string enum_val_str(dot + 1, word.c_str() + word.length());
auto enum_def = LookupEnum(enum_def_str);
if (!enum_def) Error("unknown enum: " + enum_def_str);
if (!enum_def) return Error("unknown enum: " + enum_def_str);
auto enum_val = enum_def->vals.Lookup(enum_val_str);
if (!enum_val) Error("unknown enum value: " + enum_val_str);
result |= enum_val->value;
if (!enum_val) return Error("unknown enum value: " + enum_val_str);
*result |= enum_val->value;
}
} while(*next);
return result;
return NoError();
}
void Parser::ParseHash(Value &e, FieldDef* field) {
CheckedError Parser::ParseHash(Value &e, FieldDef* field) {
assert(field);
Value *hash_name = field->attributes.Lookup("hash");
switch (e.type.base_type) {
......@@ -816,31 +909,41 @@ void Parser::ParseHash(Value &e, FieldDef* field) {
default:
assert(0);
}
Next();
NEXT();
return NoError();
}
void Parser::ParseSingleValue(Value &e) {
CheckedError Parser::ParseSingleValue(Value &e) {
// First check if this could be a string/identifier enum value:
if (e.type.base_type != BASE_TYPE_STRING &&
e.type.base_type != BASE_TYPE_NONE &&
(token_ == kTokenIdentifier || token_ == kTokenStringConstant)) {
e.constant = NumToString(ParseIntegerFromString(e.type));
Next();
} else if (TryTypedValue(kTokenIntegerConstant,
IsScalar(e.type.base_type),
e,
BASE_TYPE_INT) ||
TryTypedValue(kTokenFloatConstant,
IsFloat(e.type.base_type),
e,
BASE_TYPE_FLOAT) ||
TryTypedValue(kTokenStringConstant,
e.type.base_type == BASE_TYPE_STRING,
e,
BASE_TYPE_STRING)) {
int64_t val;
ECHECK(ParseIntegerFromString(e.type, &val));
e.constant = NumToString(val);
NEXT();
} else {
Error("cannot parse value starting with: " + TokenToStringId(token_));
bool match = false;
ECHECK(TryTypedValue(kTokenIntegerConstant,
IsScalar(e.type.base_type),
e,
BASE_TYPE_INT,
&match));
ECHECK(TryTypedValue(kTokenFloatConstant,
IsFloat(e.type.base_type),
e,
BASE_TYPE_FLOAT,
&match));
ECHECK(TryTypedValue(kTokenStringConstant,
e.type.base_type == BASE_TYPE_STRING,
e,
BASE_TYPE_STRING,
&match));
if (!match)
return Error("cannot parse value starting with: " +
TokenToStringId(token_));
}
return NoError();
}
StructDef *Parser::LookupCreateStruct(const std::string &name,
......@@ -885,20 +988,20 @@ StructDef *Parser::LookupCreateStruct(const std::string &name,
return struct_def;
}
EnumDef &Parser::ParseEnum(bool is_union) {
CheckedError Parser::ParseEnum(bool is_union, EnumDef **dest) {
std::vector<std::string> enum_comment = doc_comment_;
Next();
NEXT();
std::string enum_name = attribute_;
Expect(kTokenIdentifier);
EXPECT(kTokenIdentifier);
auto &enum_def = *new EnumDef();
enum_def.name = enum_name;
enum_def.file = files_being_parsed_;
enum_def.file = file_being_parsed_;
enum_def.doc_comment = enum_comment;
enum_def.is_union = is_union;
enum_def.defined_namespace = namespaces_.back();
if (enums_.Add(namespaces_.back()->GetFullyQualifiedName(enum_name),
&enum_def))
Error("enum already exists: " + enum_name);
return Error("enum already exists: " + enum_name);
if (is_union) {
enum_def.underlying_type.base_type = BASE_TYPE_UTYPE;
enum_def.underlying_type.enum_def = &enum_def;
......@@ -908,107 +1011,119 @@ EnumDef &Parser::ParseEnum(bool is_union) {
} else {
// Give specialized error message, since this type spec used to
// be optional in the first FlatBuffers release.
if (!IsNext(':')) Error("must specify the underlying integer type for this"
" enum (e.g. \': short\', which was the default).");
if (!Is(':')) {
return Error("must specify the underlying integer type for this"
" enum (e.g. \': short\', which was the default).");
} else {
NEXT();
}
// Specify the integer type underlying this enum.
ParseType(enum_def.underlying_type);
ECHECK(ParseType(enum_def.underlying_type));
if (!IsInteger(enum_def.underlying_type.base_type))
Error("underlying enum type must be integral");
return Error("underlying enum type must be integral");
}
// Make this type refer back to the enum it was derived from.
enum_def.underlying_type.enum_def = &enum_def;
}
ParseMetaData(enum_def);
Expect('{');
ECHECK(ParseMetaData(enum_def));
EXPECT('{');
if (is_union) enum_def.vals.Add("NONE", new EnumVal("NONE", 0));
do {
for (;;) {
if (opts.proto_mode && attribute_ == "option") {
ParseProtoOption();
ECHECK(ParseProtoOption());
} else {
auto value_name = attribute_;
auto full_name = value_name;
std::vector<std::string> value_comment = doc_comment_;
Expect(kTokenIdentifier);
if (is_union) ParseNamespacing(&full_name, &value_name);
EXPECT(kTokenIdentifier);
if (is_union) ECHECK(ParseNamespacing(&full_name, &value_name));
auto prevsize = enum_def.vals.vec.size();
auto value = enum_def.vals.vec.size()
? enum_def.vals.vec.back()->value + 1
: 0;
auto &ev = *new EnumVal(value_name, value);
if (enum_def.vals.Add(value_name, &ev))
Error("enum value already exists: " + value_name);
return Error("enum value already exists: " + value_name);
ev.doc_comment = value_comment;
if (is_union) {
ev.struct_def = LookupCreateStruct(full_name);
}
if (IsNext('=')) {
if (Is('=')) {
NEXT();
ev.value = atoi(attribute_.c_str());
Expect(kTokenIntegerConstant);
EXPECT(kTokenIntegerConstant);
if (!opts.proto_mode && prevsize &&
enum_def.vals.vec[prevsize - 1]->value >= ev.value)
Error("enum values must be specified in ascending order");
return Error("enum values must be specified in ascending order");
}
if (opts.proto_mode && IsNext('[')) {
if (opts.proto_mode && Is('[')) {
NEXT();
// ignore attributes on enums.
while (token_ != ']') Next();
Next();
while (token_ != ']') NEXT();
NEXT();
}
}
} while (IsNext(opts.proto_mode ? ';' : ',') && token_ != '}');
Expect('}');
if (!Is(opts.proto_mode ? ';' : ',')) break;
NEXT();
if (Is('}')) break;
}
EXPECT('}');
if (enum_def.attributes.Lookup("bit_flags")) {
for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end();
++it) {
if (static_cast<size_t>((*it)->value) >=
SizeOf(enum_def.underlying_type.base_type) * 8)
Error("bit flag out of range of underlying integral type");
return Error("bit flag out of range of underlying integral type");
(*it)->value = 1LL << (*it)->value;
}
}
return enum_def;
if (dest) *dest = &enum_def;
return NoError();
}
StructDef &Parser::StartStruct(const std::string &name) {
CheckedError Parser::StartStruct(const std::string &name, StructDef **dest) {
auto &struct_def = *LookupCreateStruct(name, true, true);
if (!struct_def.predecl) Error("datatype already exists: " + name);
if (!struct_def.predecl) return Error("datatype already exists: " + name);
struct_def.predecl = false;
struct_def.name = name;
struct_def.file = files_being_parsed_;
struct_def.file = file_being_parsed_;
// Move this struct to the back of the vector just in case it was predeclared,
// to preserve declaration order.
*remove(structs_.vec.begin(), structs_.vec.end(), &struct_def) = &struct_def;
return struct_def;
*dest = &struct_def;
return NoError();
}
void Parser::ParseDecl() {
CheckedError Parser::ParseDecl() {
std::vector<std::string> dc = doc_comment_;
bool fixed = IsNext(kTokenStruct);
if (!fixed) Expect(kTokenTable);
bool fixed = Is(kTokenStruct);
if (fixed) NEXT() else EXPECT(kTokenTable);
std::string name = attribute_;
Expect(kTokenIdentifier);
auto &struct_def = StartStruct(name);
struct_def.doc_comment = dc;
struct_def.fixed = fixed;
ParseMetaData(struct_def);
struct_def.sortbysize =
struct_def.attributes.Lookup("original_order") == nullptr && !fixed;
Expect('{');
while (token_ != '}') ParseField(struct_def);
auto force_align = struct_def.attributes.Lookup("force_align");
EXPECT(kTokenIdentifier);
StructDef *struct_def;
ECHECK(StartStruct(name, &struct_def));
struct_def->doc_comment = dc;
struct_def->fixed = fixed;
ECHECK(ParseMetaData(*struct_def));
struct_def->sortbysize =
struct_def->attributes.Lookup("original_order") == nullptr && !fixed;
EXPECT('{');
while (token_ != '}') ECHECK(ParseField(*struct_def));
auto force_align = struct_def->attributes.Lookup("force_align");
if (fixed && force_align) {
auto align = static_cast<size_t>(atoi(force_align->constant.c_str()));
if (force_align->type.base_type != BASE_TYPE_INT ||
align < struct_def.minalign ||
align < struct_def->minalign ||
align > 16 ||
align & (align - 1))
Error("force_align must be a power of two integer ranging from the"
return Error("force_align must be a power of two integer ranging from the"
"struct\'s natural alignment to 16");
struct_def.minalign = align;
struct_def->minalign = align;
}
struct_def.PadLastField(struct_def.minalign);
struct_def->PadLastField(struct_def->minalign);
// Check if this is a table that has manual id assignments
auto &fields = struct_def.fields.vec;
if (!struct_def.fixed && fields.size()) {
auto &fields = struct_def->fields.vec;
if (!struct_def->fixed && fields.size()) {
size_t num_id_fields = 0;
for (auto it = fields.begin(); it != fields.end(); ++it) {
if ((*it)->attributes.Lookup("id")) num_id_fields++;
......@@ -1017,7 +1132,8 @@ void Parser::ParseDecl() {
if (num_id_fields) {
// Then all fields must have them.
if (num_id_fields != fields.size())
Error("either all fields or no fields must have an 'id' attribute");
return Error(
"either all fields or no fields must have an 'id' attribute");
// Simply sort by id, then the fields are the same as if no ids had
// been specified.
std::sort(fields.begin(), fields.end(),
......@@ -1029,7 +1145,7 @@ void Parser::ParseDecl() {
// Verify we have a contiguous set, and reassign vtable offsets.
for (int i = 0; i < static_cast<int>(fields.size()); i++) {
if (i != atoi(fields[i]->attributes.Lookup("id")->constant.c_str()))
Error("field id\'s must be consecutive from 0, id " +
return Error("field id\'s must be consecutive from 0, id " +
NumToString(i) + " missing or set twice");
fields[i]->value.offset = FieldIndexToOffset(static_cast<voffset_t>(i));
}
......@@ -1039,30 +1155,32 @@ void Parser::ParseDecl() {
// This is not an ideal situation, but should occur very infrequently,
// and allows us to keep using very readable names for type & length fields
// without inducing compile errors.
auto CheckClash = [&fields, &struct_def](const char *suffix,
BaseType basetype) {
auto CheckClash = [&fields, &struct_def, this](const char *suffix,
BaseType basetype) -> CheckedError {
auto len = strlen(suffix);
for (auto it = fields.begin(); it != fields.end(); ++it) {
auto &fname = (*it)->name;
if (fname.length() > len &&
fname.compare(fname.length() - len, len, suffix) == 0 &&
(*it)->value.type.base_type != BASE_TYPE_UTYPE) {
auto field = struct_def.fields.Lookup(
auto field = struct_def->fields.Lookup(
fname.substr(0, fname.length() - len));
if (field && field->value.type.base_type == basetype)
Error("Field " + fname +
return Error("Field " + fname +
" would clash with generated functions for field " +
field->name);
}
}
return NoError();
};
CheckClash("_type", BASE_TYPE_UNION);
CheckClash("Type", BASE_TYPE_UNION);
CheckClash("_length", BASE_TYPE_VECTOR);
CheckClash("Length", BASE_TYPE_VECTOR);
CheckClash("_byte_vector", BASE_TYPE_STRING);
CheckClash("ByteVector", BASE_TYPE_STRING);
Expect('}');
ECHECK(CheckClash("_type", BASE_TYPE_UNION));
ECHECK(CheckClash("Type", BASE_TYPE_UNION));
ECHECK(CheckClash("_length", BASE_TYPE_VECTOR));
ECHECK(CheckClash("Length", BASE_TYPE_VECTOR));
ECHECK(CheckClash("_byte_vector", BASE_TYPE_STRING));
ECHECK(CheckClash("ByteVector", BASE_TYPE_STRING));
EXPECT('}');
return NoError();
}
bool Parser::SetRootType(const char *name) {
......@@ -1087,44 +1205,46 @@ void Parser::MarkGenerated() {
}
}
void Parser::ParseNamespace() {
Next();
CheckedError Parser::ParseNamespace() {
NEXT();
auto ns = new Namespace();
namespaces_.push_back(ns);
if (token_ != ';') {
for (;;) {
ns->components.push_back(attribute_);
Expect(kTokenIdentifier);
if (!IsNext('.')) break;
EXPECT(kTokenIdentifier);
if (Is('.')) NEXT() else break;
}
}
Expect(';');
EXPECT(';');
return NoError();
}
// Best effort parsing of .proto declarations, with the aim to turn them
// in the closest corresponding FlatBuffer equivalent.
// We parse everything as identifiers instead of keywords, since we don't
// want protobuf keywords to become invalid identifiers in FlatBuffers.
void Parser::ParseProtoDecl() {
CheckedError Parser::ParseProtoDecl() {
bool isextend = attribute_ == "extend";
if (attribute_ == "package") {
// These are identical in syntax to FlatBuffer's namespace decl.
ParseNamespace();
ECHECK(ParseNamespace());
} else if (attribute_ == "message" || isextend) {
std::vector<std::string> struct_comment = doc_comment_;
Next();
NEXT();
StructDef *struct_def = nullptr;
if (isextend) {
IsNext('.'); // qualified names may start with a . ?
if (Is('.')) NEXT(); // qualified names may start with a . ?
auto id = attribute_;
Expect(kTokenIdentifier);
ParseNamespacing(&id, nullptr);
EXPECT(kTokenIdentifier);
ECHECK(ParseNamespacing(&id, nullptr));
struct_def = LookupCreateStruct(id, false);
if (!struct_def) Error("cannot extend unknown message type: " + id);
if (!struct_def)
return Error("cannot extend unknown message type: " + id);
} else {
std::string name = attribute_;
Expect(kTokenIdentifier);
struct_def = &StartStruct(name);
EXPECT(kTokenIdentifier);
ECHECK(StartStruct(name, &struct_def));
// Since message definitions can be nested, we create a new namespace.
auto ns = new Namespace();
// Copy of current namespace.
......@@ -1134,7 +1254,7 @@ void Parser::ParseProtoDecl() {
namespaces_.push_back(ns);
}
struct_def->doc_comment = struct_comment;
ParseProtoFields(struct_def, isextend, false);
ECHECK(ParseProtoFields(struct_def, isextend, false));
if (!isextend) {
// We have to remove the nested namespace, but we can't just throw it
// away, so put it at the beginning of the vector.
......@@ -1142,13 +1262,14 @@ void Parser::ParseProtoDecl() {
namespaces_.pop_back();
namespaces_.insert(namespaces_.begin(), ns);
}
IsNext(';');
if (Is(';')) NEXT();
} else if (attribute_ == "enum") {
// These are almost the same, just with different terminator:
auto &enum_def = ParseEnum(false);
IsNext(';');
EnumDef *enum_def;
ECHECK(ParseEnum(false, &enum_def));
if (Is(';')) NEXT();
// Protobuf allows them to be specified in any order, so sort afterwards.
auto &v = enum_def.vals.vec;
auto &v = enum_def->vals.vec;
std::sort(v.begin(), v.end(), [](const EnumVal *a, const EnumVal *b) {
return a->value < b->value;
});
......@@ -1158,46 +1279,48 @@ void Parser::ParseProtoDecl() {
else ++it;
}
} else if (attribute_ == "syntax") { // Skip these.
Next();
Expect('=');
Expect(kTokenStringConstant);
Expect(';');
NEXT();
EXPECT('=');
EXPECT(kTokenStringConstant);
EXPECT(';');
} else if (attribute_ == "option") { // Skip these.
ParseProtoOption();
Expect(';');
ECHECK(ParseProtoOption());
EXPECT(';');
} else if (attribute_ == "service") { // Skip these.
Next();
Expect(kTokenIdentifier);
ParseProtoCurliesOrIdent();
NEXT();
EXPECT(kTokenIdentifier);
ECHECK(ParseProtoCurliesOrIdent());
} else {
Error("don\'t know how to parse .proto declaration starting with " +
return Error("don\'t know how to parse .proto declaration starting with " +
TokenToStringId(token_));
}
return NoError();
}
void Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
bool inside_oneof) {
Expect('{');
CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
bool inside_oneof) {
EXPECT('{');
while (token_ != '}') {
if (attribute_ == "message" || attribute_ == "extend" ||
attribute_ == "enum") {
// Nested declarations.
ParseProtoDecl();
ECHECK(ParseProtoDecl());
} else if (attribute_ == "extensions") { // Skip these.
Next();
Expect(kTokenIntegerConstant);
if (IsNext(kTokenIdentifier)) { // to
Next(); // num
NEXT();
EXPECT(kTokenIntegerConstant);
if (Is(kTokenIdentifier)) {
NEXT(); // to
NEXT(); // num
}
Expect(';');
EXPECT(';');
} else if (attribute_ == "option") { // Skip these.
ParseProtoOption();
Expect(';');
ECHECK(ParseProtoOption());
EXPECT(';');
} else if (attribute_ == "reserved") { // Skip these.
Next();
Expect(kTokenIntegerConstant);
while (IsNext(',')) Expect(kTokenIntegerConstant);
Expect(';');
NEXT();
EXPECT(kTokenIntegerConstant);
while (Is(',')) { NEXT(); EXPECT(kTokenIntegerConstant); }
EXPECT(';');
} else {
std::vector<std::string> field_comment = doc_comment_;
// Parse the qualifier.
......@@ -1207,16 +1330,16 @@ void Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
if (!inside_oneof) {
if (attribute_ == "optional") {
// This is the default.
Expect(kTokenIdentifier);
EXPECT(kTokenIdentifier);
} else if (attribute_ == "required") {
required = true;
Expect(kTokenIdentifier);
EXPECT(kTokenIdentifier);
} else if (attribute_ == "repeated") {
repeated = true;
Expect(kTokenIdentifier);
EXPECT(kTokenIdentifier);
} else if (attribute_ == "oneof") {
oneof = true;
Expect(kTokenIdentifier);
EXPECT(kTokenIdentifier);
} else {
// can't error, proto3 allows decls without any of the above.
}
......@@ -1224,12 +1347,12 @@ void Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
StructDef *anonymous_struct = nullptr;
Type type;
if (attribute_ == "group" || oneof) {
if (!oneof) Expect(kTokenIdentifier);
if (!oneof) EXPECT(kTokenIdentifier);
auto name = "Anonymous" + NumToString(anonymous_counter++);
anonymous_struct = &StartStruct(name);
ECHECK(StartStruct(name, &anonymous_struct));
type = Type(BASE_TYPE_STRUCT, anonymous_struct);
} else {
type = ParseTypeFromProtoType();
ECHECK(ParseTypeFromProtoType(&type));
}
// Repeated elements get mapped to a vector.
if (repeated) {
......@@ -1238,93 +1361,100 @@ void Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
}
std::string name = attribute_;
// Protos may use our keywords "attribute" & "namespace" as an identifier.
if (IsNext(kTokenAttribute) || IsNext(kTokenNameSpace)) {
if (Is(kTokenAttribute) || Is(kTokenNameSpace)) {
NEXT();
// TODO: simpler to just not make these keywords?
name += "_"; // Have to make it not a keyword.
} else {
Expect(kTokenIdentifier);
EXPECT(kTokenIdentifier);
}
if (!oneof) {
// Parse the field id. Since we're just translating schemas, not
// any kind of binary compatibility, we can safely ignore these, and
// assign our own.
Expect('=');
Expect(kTokenIntegerConstant);
EXPECT('=');
EXPECT(kTokenIntegerConstant);
}
FieldDef *existing_field = nullptr;
FieldDef *field = nullptr;
if (isextend) {
// We allow a field to be re-defined when extending.
// TODO: are there situations where that is problematic?
existing_field = struct_def->fields.Lookup(name);
field = struct_def->fields.Lookup(name);
}
auto &field = existing_field
? *existing_field
: AddField(*struct_def, name, type);
field.doc_comment = field_comment;
if (!IsScalar(type.base_type)) field.required = required;
if (!field) ECHECK(AddField(*struct_def, name, type, &field));
field->doc_comment = field_comment;
if (!IsScalar(type.base_type)) field->required = required;
// See if there's a default specified.
if (IsNext('[')) {
do {
if (Is('[')) {
NEXT();
for (;;) {
auto key = attribute_;
ParseProtoKey();
Expect('=');
ECHECK(ParseProtoKey());
EXPECT('=');
auto val = attribute_;
ParseProtoCurliesOrIdent();
ECHECK(ParseProtoCurliesOrIdent());
if (key == "default") {
// Temp: skip non-numeric defaults (enums).
auto numeric = strpbrk(val.c_str(), "0123456789-+.");
if (IsScalar(type.base_type) && numeric == val.c_str())
field.value.constant = val;
field->value.constant = val;
} else if (key == "deprecated") {
field.deprecated = val == "true";
field->deprecated = val == "true";
}
} while (IsNext(','));
Expect(']');
if (!Is(',')) break;
NEXT();
}
EXPECT(']');
}
if (anonymous_struct) {
ParseProtoFields(anonymous_struct, false, oneof);
IsNext(';');
ECHECK(ParseProtoFields(anonymous_struct, false, oneof));
if (Is(';')) NEXT();
} else {
Expect(';');
EXPECT(';');
}
}
}
Next();
NEXT();
return NoError();
}
void Parser::ParseProtoKey() {
CheckedError Parser::ParseProtoKey() {
if (token_ == '(') {
Next();
NEXT();
// Skip "(a.b)" style custom attributes.
while (token_ == '.' || token_ == kTokenIdentifier) Next();
Expect(')');
while (IsNext('.')) Expect(kTokenIdentifier);
while (token_ == '.' || token_ == kTokenIdentifier) NEXT();
EXPECT(')');
while (Is('.')) { NEXT(); EXPECT(kTokenIdentifier); }
} else {
Expect(kTokenIdentifier);
EXPECT(kTokenIdentifier);
}
return NoError();
}
void Parser::ParseProtoCurliesOrIdent() {
if (IsNext('{')) {
CheckedError Parser::ParseProtoCurliesOrIdent() {
if (Is('{')) {
NEXT();
for (int nesting = 1; nesting; ) {
if (token_ == '{') nesting++;
else if (token_ == '}') nesting--;
Next();
NEXT();
}
} else {
Next(); // Any single token.
NEXT(); // Any single token.
}
return NoError();
}
void Parser::ParseProtoOption() {
Next();
ParseProtoKey();
Expect('=');
ParseProtoCurliesOrIdent();
CheckedError Parser::ParseProtoOption() {
NEXT();
ECHECK(ParseProtoKey());
EXPECT('=');
ECHECK(ParseProtoCurliesOrIdent());
return NoError();
}
// Parse a protobuf type, and map it to the corresponding FlatBuffer one.
Type Parser::ParseTypeFromProtoType() {
CheckedError Parser::ParseTypeFromProtoType(Type *type) {
struct type_lookup { const char *proto_type; BaseType fb_type; };
static type_lookup lookup[] = {
{ "float", BASE_TYPE_FLOAT }, { "double", BASE_TYPE_DOUBLE },
......@@ -1338,22 +1468,26 @@ Type Parser::ParseTypeFromProtoType() {
{ "bytes", BASE_TYPE_STRING },
{ nullptr, BASE_TYPE_NONE }
};
Type type;
for (auto tl = lookup; tl->proto_type; tl++) {
if (attribute_ == tl->proto_type) {
type.base_type = tl->fb_type;
Next();
return type;
type->base_type = tl->fb_type;
NEXT();
return NoError();
}
}
IsNext('.'); // qualified names may start with a . ?
ParseTypeIdent(type);
return type;
if (Is('.')) NEXT(); // qualified names may start with a . ?
ECHECK(ParseTypeIdent(*type));
return NoError();
}
bool Parser::Parse(const char *source, const char **include_paths,
const char *source_filename) {
files_being_parsed_ = source_filename ? source_filename : "";
return !DoParse(source, include_paths, source_filename).Check();
}
CheckedError Parser::DoParse(const char *source, const char **include_paths,
const char *source_filename) {
file_being_parsed_ = source_filename ? source_filename : "";
if (source_filename &&
included_files_.find(source_filename) == included_files_.end()) {
included_files_[source_filename] = true;
......@@ -1369,141 +1503,130 @@ bool Parser::Parse(const char *source, const char **include_paths,
builder_.Clear();
// Start with a blank namespace just in case this file doesn't have one.
namespaces_.push_back(new Namespace());
try {
Next();
// Includes must come before type declarations:
for (;;) {
// Parse pre-include proto statements if any:
if (opts.proto_mode &&
(attribute_ == "option" || attribute_ == "syntax" ||
attribute_ == "package")) {
ParseProtoDecl();
} else if (IsNext(kTokenInclude) ||
(opts.proto_mode &&
attribute_ == "import" &&
IsNext(kTokenIdentifier))) {
if (opts.proto_mode && attribute_ == "public") Next();
auto name = attribute_;
Expect(kTokenStringConstant);
// Look for the file in include_paths.
std::string filepath;
for (auto paths = include_paths; paths && *paths; paths++) {
filepath = flatbuffers::ConCatPathFileName(*paths, name);
if(FileExists(filepath.c_str())) break;
}
if (filepath.empty())
Error("unable to locate include file: " + name);
if (source_filename)
files_included_per_file_[source_filename].insert(filepath);
if (included_files_.find(filepath) == included_files_.end()) {
// We found an include file that we have not parsed yet.
// Load it and parse it.
std::string contents;
if (!LoadFile(filepath.c_str(), true, &contents))
Error("unable to load include file: " + name);
if (!Parse(contents.c_str(), include_paths, filepath.c_str())) {
// Any errors, we're done.
return false;
}
// We generally do not want to output code for any included files:
if (!opts.generate_all) MarkGenerated();
// This is the easiest way to continue this file after an include:
// instead of saving and restoring all the state, we simply start the
// file anew. This will cause it to encounter the same include statement
// again, but this time it will skip it, because it was entered into
// included_files_.
// This is recursive, but only go as deep as the number of include
// statements.
return Parse(source, include_paths, source_filename);
}
Expect(';');
} else {
break;
NEXT();
// Includes must come before type declarations:
for (;;) {
// Parse pre-include proto statements if any:
if (opts.proto_mode &&
(attribute_ == "option" || attribute_ == "syntax" ||
attribute_ == "package")) {
ECHECK(ParseProtoDecl());
} else if (Is(kTokenInclude) ||
(opts.proto_mode &&
attribute_ == "import" &&
Is(kTokenIdentifier))) {
NEXT();
if (opts.proto_mode && attribute_ == "public") NEXT();
auto name = attribute_;
EXPECT(kTokenStringConstant);
// Look for the file in include_paths.
std::string filepath;
for (auto paths = include_paths; paths && *paths; paths++) {
filepath = flatbuffers::ConCatPathFileName(*paths, name);
if(FileExists(filepath.c_str())) break;
}
}
// Now parse all other kinds of declarations:
while (token_ != kTokenEof) {
if (opts.proto_mode) {
ParseProtoDecl();
} else if (token_ == kTokenNameSpace) {
ParseNamespace();
} else if (token_ == '{') {
if (!root_struct_def_) Error("no root type set to parse json with");
if (builder_.GetSize()) {
Error("cannot have more than one json object in a file");
}
builder_.Finish(Offset<Table>(ParseTable(*root_struct_def_, nullptr)),
file_identifier_.length() ? file_identifier_.c_str() : nullptr);
} else if (token_ == kTokenEnum) {
ParseEnum(false);
} else if (token_ == kTokenUnion) {
ParseEnum(true);
} else if (token_ == kTokenRootType) {
Next();
auto root_type = attribute_;
Expect(kTokenIdentifier);
ParseNamespacing(&root_type, nullptr);
if (!SetRootType(root_type.c_str()))
Error("unknown root type: " + root_type);
if (root_struct_def_->fixed)
Error("root type must be a table");
Expect(';');
} else if (token_ == kTokenFileIdentifier) {
Next();
file_identifier_ = attribute_;
Expect(kTokenStringConstant);
if (file_identifier_.length() !=
FlatBufferBuilder::kFileIdentifierLength)
Error("file_identifier must be exactly " +
NumToString(FlatBufferBuilder::kFileIdentifierLength) +
" characters");
Expect(';');
} else if (token_ == kTokenFileExtension) {
Next();
file_extension_ = attribute_;
Expect(kTokenStringConstant);
Expect(';');
} else if(token_ == kTokenInclude) {
Error("includes must come before declarations");
} else if(token_ == kTokenAttribute) {
Next();
auto name = attribute_;
Expect(kTokenStringConstant);
Expect(';');
known_attributes_.insert(name);
} else {
ParseDecl();
if (filepath.empty())
return Error("unable to locate include file: " + name);
if (source_filename)
files_included_per_file_[source_filename].insert(filepath);
if (included_files_.find(filepath) == included_files_.end()) {
// We found an include file that we have not parsed yet.
// Load it and parse it.
std::string contents;
if (!LoadFile(filepath.c_str(), true, &contents))
return Error("unable to load include file: " + name);
ECHECK(DoParse(contents.c_str(), include_paths, filepath.c_str()));
// We generally do not want to output code for any included files:
if (!opts.generate_all) MarkGenerated();
// This is the easiest way to continue this file after an include:
// instead of saving and restoring all the state, we simply start the
// file anew. This will cause it to encounter the same include
// statement again, but this time it will skip it, because it was
// entered into included_files_.
// This is recursive, but only go as deep as the number of include
// statements.
return DoParse(source, include_paths, source_filename);
}
EXPECT(';');
} else {
break;
}
for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) {
if ((*it)->predecl) {
Error("type referenced but not defined: " + (*it)->name);
}
// Now parse all other kinds of declarations:
while (token_ != kTokenEof) {
if (opts.proto_mode) {
ECHECK(ParseProtoDecl());
} else if (token_ == kTokenNameSpace) {
ECHECK(ParseNamespace());
} else if (token_ == '{') {
if (!root_struct_def_)
return Error("no root type set to parse json with");
if (builder_.GetSize()) {
return Error("cannot have more than one json object in a file");
}
uoffset_t toff;
ECHECK(ParseTable(*root_struct_def_, nullptr, &toff));
builder_.Finish(Offset<Table>(toff),
file_identifier_.length() ? file_identifier_.c_str() : nullptr);
} else if (token_ == kTokenEnum) {
ECHECK(ParseEnum(false, nullptr));
} else if (token_ == kTokenUnion) {
ECHECK(ParseEnum(true, nullptr));
} else if (token_ == kTokenRootType) {
NEXT();
auto root_type = attribute_;
EXPECT(kTokenIdentifier);
ECHECK(ParseNamespacing(&root_type, nullptr));
if (!SetRootType(root_type.c_str()))
return Error("unknown root type: " + root_type);
if (root_struct_def_->fixed)
return Error("root type must be a table");
EXPECT(';');
} else if (token_ == kTokenFileIdentifier) {
NEXT();
file_identifier_ = attribute_;
EXPECT(kTokenStringConstant);
if (file_identifier_.length() !=
FlatBufferBuilder::kFileIdentifierLength)
return Error("file_identifier must be exactly " +
NumToString(FlatBufferBuilder::kFileIdentifierLength) +
" characters");
EXPECT(';');
} else if (token_ == kTokenFileExtension) {
NEXT();
file_extension_ = attribute_;
EXPECT(kTokenStringConstant);
EXPECT(';');
} else if(token_ == kTokenInclude) {
return Error("includes must come before declarations");
} else if(token_ == kTokenAttribute) {
NEXT();
auto name = attribute_;
EXPECT(kTokenStringConstant);
EXPECT(';');
known_attributes_.insert(name);
} else {
ECHECK(ParseDecl());
}
for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
auto &enum_def = **it;
if (enum_def.is_union) {
for (auto val_it = enum_def.vals.vec.begin();
val_it != enum_def.vals.vec.end();
++val_it) {
auto &val = **val_it;
if (val.struct_def && val.struct_def->fixed)
Error("only tables can be union elements: " + val.name);
}
}
for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) {
if ((*it)->predecl) {
return Error("type referenced but not defined: " + (*it)->name);
}
}
for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
auto &enum_def = **it;
if (enum_def.is_union) {
for (auto val_it = enum_def.vals.vec.begin();
val_it != enum_def.vals.vec.end();
++val_it) {
auto &val = **val_it;
if (val.struct_def && val.struct_def->fixed)
return Error("only tables can be union elements: " + val.name);
}
}
} catch (const std::string &msg) {
error_ = source_filename ? AbsolutePath(source_filename) : "";
#ifdef _WIN32
error_ += "(" + NumToString(line_) + ")"; // MSVC alike
#else
if (source_filename) error_ += ":";
error_ += NumToString(line_) + ":0"; // gcc alike
#endif
error_ += ": error: " + msg;
return false;
}
return true;
return NoError();
}
std::set<std::string> Parser::GetIncludedFilesRecursive(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment