Skip to content

F
flatbuffers
Commit 026c6ddb authored by Wouter van Oortmerssen's avatar Wouter van Oortmerssen
Browse files
Options

Merge branch 'master' of https://github.com/google/flatbuffers

parents 3eebba78 b36bd67b
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 338 additions and 91 deletions
include/flatbuffers/idl.h
View file @ 026c6ddb
......@@ -348,6 +348,7 @@ struct IDLOptions {
bool escape_proto_identifiers;
bool generate_object_based_api;
bool union_value_namespacing;
bool allow_non_utf8;
// Possible options for the more general generator below.
enum Language { kJava, kCSharp, kGo, kMAX };
......@@ -370,6 +371,7 @@ struct IDLOptions {
escape_proto_identifiers(false),
generate_object_based_api(false),
union_value_namespacing(true),
allow_non_utf8(false),
lang(IDLOptions::kJava) {}
};
......
include/flatbuffers/util.h
View file @ 026c6ddb
......@@ -276,6 +276,10 @@ inline int FromUTF8(const char **in) {
}
if ((**in << len) & 0x80) return -1; // Bit after leading 1's must be 0.
if (!len) return *(*in)++;
// UTF-8 encoded values with a length are between 2 and 4 bytes.
if (len < 2 || len > 4) {
return -1;
}
// Grab initial bits of the code.
int ucc = *(*in)++ & ((1 << (7 - len)) - 1);
for (int i = 0; i < len - 1; i++) {
......@@ -283,6 +287,32 @@ inline int FromUTF8(const char **in) {
ucc <<= 6;
ucc |= *(*in)++ & 0x3F; // Grab 6 more bits of the code.
}
// UTF-8 cannot encode values between 0xD800 and 0xDFFF (reserved for
// UTF-16 surrogate pairs).
if (ucc >= 0xD800 && ucc <= 0xDFFF) {
return -1;
}
// UTF-8 must represent code points in their shortest possible encoding.
switch (len) {
case 2:
// Two bytes of UTF-8 can represent code points from U+0080 to U+07FF.
if (ucc < 0x0080 || ucc > 0x07FF) {
return -1;
}
break;
case 3:
// Three bytes of UTF-8 can represent code points from U+0800 to U+FFFF.
if (ucc < 0x0800 || ucc > 0xFFFF) {
return -1;
}
break;
case 4:
// Four bytes of UTF-8 can represent code points from U+10000 to U+10FFFF.
if (ucc < 0x10000 || ucc > 0x10FFFF) {
return -1;
}
break;
}
return ucc;
}
......
src/flatc.cpp
View file @ 026c6ddb
......@@ -106,6 +106,9 @@ static void Error(const std::string &err, bool usage, bool show_exe_name) {
" --version Print the version number of flatc and exit.\n"
" --strict-json Strict JSON: field names must be / will be quoted,\n"
" no trailing commas in tables/vectors.\n"
" --allow-non-utf8 Pass non-UTF-8 input through parser and emit nonstandard\n"
" \\x escapes in JSON. (Default is to raise parse error on\n"
" non-UTF-8 input.)\n"
" --defaults-json Output fields whose value is the default when\n"
" writing JSON\n"
" --unknown-json Allow fields in JSON that are not defined in the\n"
......@@ -184,6 +187,8 @@ int main(int argc, const char *argv[]) {
conform_to_schema = argv[argi];
} else if(arg == "--strict-json") {
opts.strict_json = true;
} else if(arg == "--allow-non-utf8") {
opts.allow_non_utf8 = true;
} else if(arg == "--no-js-exports") {
opts.skip_js_exports = true;
} else if(arg == "--defaults-json") {
......
src/idl_gen_go.cpp
View file @ 026c6ddb
......@@ -126,7 +126,7 @@ static void NewRootTypeFromBuffer(const StructDef &struct_def,
code += " {\n";
code += "\tn := flatbuffers.GetUOffsetT(buf[offset:])\n";
code += "\tx := &" + struct_def.name + "{}\n";
code += "\tx.Init(buf, n + offset)\n";
code += "\tx.Init(buf, n+offset)\n";
code += "\treturn x\n";
code += "}\n\n";
}
......@@ -178,9 +178,10 @@ static void GetScalarFieldOfStruct(const StructDef &struct_def,
std::string getter = GenGetter(field.value.type);
GenReceiver(struct_def, code_ptr);
code += " " + MakeCamel(field.name);
code += "() " + TypeName(field) + " { return " + getter;
code += "() " + TypeName(field) + " {\n";
code +="\treturn " + getter;
code += "(rcv._tab.Pos + flatbuffers.UOffsetT(";
code += NumToString(field.value.offset) + ")) }\n";
code += NumToString(field.value.offset) + "))\n}\n";
}
// Get the value of a table's scalar.
......@@ -212,7 +213,7 @@ static void GetStructFieldOfStruct(const StructDef &struct_def,
code += "\tif obj == nil {\n";
code += "\t\tobj = new(" + TypeName(field) + ")\n";
code += "\t}\n";
code += "\tobj.Init(rcv._tab.Bytes, rcv._tab.Pos + ";
code += "\tobj.Init(rcv._tab.Bytes, rcv._tab.Pos+";
code += NumToString(field.value.offset) + ")";
code += "\n\treturn obj\n";
code += "}\n";
......@@ -287,9 +288,9 @@ static void GetMemberOfVectorOfStruct(const StructDef &struct_def,
if (!(vectortype.struct_def->fixed)) {
code += "\t\tx = rcv._tab.Indirect(x)\n";
}
code += "\tif obj == nil {\n";
code += "\t\tobj = new(" + TypeName(field) + ")\n";
code += "\t}\n";
code += "\t\tif obj == nil {\n";
code += "\t\t\tobj = new(" + TypeName(field) + ")\n";
code += "\t\t}\n";
code += "\t\tobj.Init(rcv._tab.Bytes, x)\n";
code += "\t\treturn true\n\t}\n";
code += "\treturn false\n";
......@@ -310,7 +311,7 @@ static void GetMemberOfVectorOfNonStruct(const StructDef &struct_def,
code += OffsetPrefix(field);
code += "\t\ta := rcv._tab.Vector(o)\n";
code += "\t\treturn " + GenGetter(field.value.type) + "(";
code += "a + flatbuffers.UOffsetT(j * ";
code += "a + flatbuffers.UOffsetT(j*";
code += NumToString(InlineSize(vectortype)) + "))\n";
code += "\t}\n";
if (vectortype.base_type == BASE_TYPE_STRING) {
......@@ -326,7 +327,10 @@ static void BeginBuilderArgs(const StructDef &struct_def,
std::string *code_ptr) {
std::string &code = *code_ptr;
code += "\n";
if (code.substr(code.length() - 2) != "\n\n") {
// a previous mutate has not put an extra new line
code += "\n";
}
code += "func Create" + struct_def.name;
code += "(builder *flatbuffers.Builder";
}
......@@ -368,20 +372,20 @@ static void StructBuilderBody(const StructDef &struct_def,
const char *nameprefix,
std::string *code_ptr) {
std::string &code = *code_ptr;
code += " builder.Prep(" + NumToString(struct_def.minalign) + ", ";
code += "\tbuilder.Prep(" + NumToString(struct_def.minalign) + ", ";
code += NumToString(struct_def.bytesize) + ")\n";
for (auto it = struct_def.fields.vec.rbegin();
it != struct_def.fields.vec.rend();
++it) {
auto &field = **it;
if (field.padding)
code += " builder.Pad(" + NumToString(field.padding) + ")\n";
code += "\tbuilder.Pad(" + NumToString(field.padding) + ")\n";
if (IsStruct(field.value.type)) {
StructBuilderBody(*field.value.type.struct_def,
(nameprefix + (field.name + "_")).c_str(),
code_ptr);
} else {
code += " builder.Prepend" + GenMethod(field) + "(";
code += "\tbuilder.Prepend" + GenMethod(field) + "(";
code += nameprefix + MakeCamel(field.name, false) + ")\n";
}
}
......@@ -389,7 +393,7 @@ static void StructBuilderBody(const StructDef &struct_def,
static void EndBuilderBody(std::string *code_ptr) {
std::string &code = *code_ptr;
code += " return builder.Offset()\n";
code += "\treturn builder.Offset()\n";
code += "}\n";
}
......@@ -398,10 +402,10 @@ static void GetStartOfTable(const StructDef &struct_def,
std::string *code_ptr) {
std::string &code = *code_ptr;
code += "func " + struct_def.name + "Start";
code += "(builder *flatbuffers.Builder) { ";
code += "builder.StartObject(";
code += "(builder *flatbuffers.Builder) {\n";
code += "\tbuilder.StartObject(";
code += NumToString(struct_def.fields.vec.size());
code += ") }\n";
code += ")\n}\n";
}
// Set the value of a table's field.
......@@ -418,8 +422,8 @@ static void BuildFieldOfTable(const StructDef &struct_def,
} else {
code += GenTypeBasic(field.value.type);
}
code += ") ";
code += "{ builder.Prepend";
code += ") {\n";
code += "\tbuilder.Prepend";
code += GenMethod(field) + "Slot(";
code += NumToString(offset) + ", ";
if (!IsScalar(field.value.type.base_type) && (!struct_def.fixed)) {
......@@ -430,7 +434,7 @@ static void BuildFieldOfTable(const StructDef &struct_def,
code += MakeCamel(field.name, false);
}
code += ", " + field.value.constant;
code += ") }\n";
code += ")\n}\n";
}
// Set the value of one of the members of a table's vector.
......@@ -441,7 +445,7 @@ static void BuildVectorOfTable(const StructDef &struct_def,
code += "func " + struct_def.name + "Start";
code += MakeCamel(field.name);
code += "Vector(builder *flatbuffers.Builder, numElems int) ";
code += "flatbuffers.UOffsetT { return builder.StartVector(";
code += "flatbuffers.UOffsetT {\n\treturn builder.StartVector(";
auto vector_type = field.value.type.VectorType();
auto alignment = InlineAlignment(vector_type);
auto elem_size = InlineSize(vector_type);
......@@ -456,7 +460,7 @@ static void GetEndOffsetOnTable(const StructDef &struct_def,
std::string &code = *code_ptr;
code += "func " + struct_def.name + "End";
code += "(builder *flatbuffers.Builder) flatbuffers.UOffsetT ";
code += "{ return builder.EndObject() }\n";
code += "{\n\treturn builder.EndObject()\n}\n";
}
// Generate the receiver for function signatures.
......@@ -521,9 +525,9 @@ static void MutateScalarFieldOfStruct(const StructDef &struct_def,
std::string setter = "rcv._tab.Mutate" + type;
GenReceiver(struct_def, code_ptr);
code += " Mutate" + MakeCamel(field.name);
code += "(n " + TypeName(field) + ") bool { return " + setter;
code += "(rcv._tab.Pos + flatbuffers.UOffsetT(";
code += NumToString(field.value.offset) + "), n) }\n\n";
code += "(n " + TypeName(field) + ") bool {\n\treturn " + setter;
code += "(rcv._tab.Pos+flatbuffers.UOffsetT(";
code += NumToString(field.value.offset) + "), n)\n}\n\n";
}
// Mutate the value of a table's scalar.
......@@ -732,7 +736,7 @@ class GoGenerator : public BaseGenerator {
if (needs_imports) {
code += "import (\n";
code += "\tflatbuffers \"github.com/google/flatbuffers/go\"\n";
code += ")\n";
code += ")\n\n";
}
}
......
src/idl_gen_text.cpp
View file @ 026c6ddb
......@@ -93,7 +93,7 @@ template<typename T> void PrintVector(const Vector<T> &v, Type type,
text += "]";
}
static void EscapeString(const String &s, std::string *_text) {
static void EscapeString(const String &s, std::string *_text, const IDLOptions& opts) {
std::string &text = *_text;
text += "\"";
for (uoffset_t i = 0; i < s.size(); i++) {
......@@ -113,17 +113,32 @@ static void EscapeString(const String &s, std::string *_text) {
// Not printable ASCII data. Let's see if it's valid UTF-8 first:
const char *utf8 = s.c_str() + i;
int ucc = FromUTF8(&utf8);
if (ucc >= 0x80 && ucc <= 0xFFFF) {
// Parses as Unicode within JSON's \uXXXX range, so use that.
text += "\\u";
text += IntToStringHex(ucc, 4);
if (ucc < 0) {
if (opts.allow_non_utf8) {
text += "\\x";
text += IntToStringHex(static_cast<uint8_t>(c), 2);
} else {
// We previously checked for non-UTF-8 and returned a parse error,
// so we shouldn't reach here.
assert(0);
}
} else {
if (ucc <= 0xFFFF) {
// Parses as Unicode within JSON's \uXXXX range, so use that.
text += "\\u";
text += IntToStringHex(ucc, 4);
} else if (ucc <= 0x10FFFF) {
// Encode Unicode SMP values to a surrogate pair using two \u escapes.
uint32_t base = ucc - 0x10000;
uint16_t highSurrogate = (base >> 10) + 0xD800;
uint16_t lowSurrogate = (base & 0x03FF) + 0xDC00;
text += "\\u";
text += IntToStringHex(highSurrogate, 4);
text += "\\u";
text += IntToStringHex(lowSurrogate, 4);
}
// Skip past characters recognized.
i = static_cast<uoffset_t>(utf8 - s.c_str() - 1);
} else {
// It's either unprintable ASCII, arbitrary binary, or Unicode data
// that doesn't fit \uXXXX, so use \xXX escape code instead.
text += "\\x";
text += IntToStringHex(static_cast<uint8_t>(c), 2);
}
}
break;
......@@ -157,7 +172,7 @@ template<> void Print<const void *>(const void *val,
_text);
break;
case BASE_TYPE_STRING: {
EscapeString(*reinterpret_cast<const String *>(val), _text);
EscapeString(*reinterpret_cast<const String *>(val), _text, opts);
break;
}
case BASE_TYPE_VECTOR:
......
src/idl_parser.cpp
View file @ 026c6ddb
......@@ -61,6 +61,17 @@ static_assert(BASE_TYPE_UNION ==
#define NEXT() ECHECK(Next())
#define EXPECT(tok) ECHECK(Expect(tok))
static bool ValidateUTF8(const std::string &str) {
const char *s = &str[0];
const char * const sEnd = s + str.length();
while (s < sEnd) {
if (FromUTF8(&s) < 0) {
return false;
}
}
return true;
}
CheckedError Parser::Error(const std::string &msg) {
error_ = file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : "";
#ifdef _WIN32
......@@ -320,6 +331,9 @@ CheckedError Parser::Next() {
"illegal Unicode sequence (unpaired high surrogate)");
}
cursor_++;
if (!opts.allow_non_utf8 && !ValidateUTF8(attribute_)) {
return Error("illegal UTF-8 sequence");
}
token_ = kTokenStringConstant;
return NoError();
}
......
tests/GoTest.sh
View file @ 026c6ddb
......@@ -29,9 +29,9 @@ mkdir -p ${go_src}/MyGame/Example
mkdir -p ${go_src}/github.com/google/flatbuffers/go
mkdir -p ${go_src}/flatbuffers_test
cp -u MyGame/Example/*.go ./go_gen/src/MyGame/Example/
cp -u ../go/* ./go_gen/src/github.com/google/flatbuffers/go
cp -u ./go_test.go ./go_gen/src/flatbuffers_test/
cp -a MyGame/Example/*.go ./go_gen/src/MyGame/Example/
cp -a ../go/* ./go_gen/src/github.com/google/flatbuffers/go
cp -a ./go_test.go ./go_gen/src/flatbuffers_test/
# Run tests with necessary flags.
# Developers may wish to see more detail by appending the verbosity flag
......@@ -50,6 +50,18 @@ GOPATH=${go_path} go test flatbuffers_test \
--fuzz_fields=4 \
--fuzz_objects=10000
GO_TEST_RESULT=$?
rm -rf ${go_path}/{pkg,src}
if [[ $GO_TEST_RESULT == 0 ]]; then
echo "OK: Go tests passed."
else
echo "KO: Go tests failed."
exit 1
fi
echo "OK: Go tests passed."
NOT_FMT_FILES=$(gofmt -l MyGame)
if [[ ${NOT_FMT_FILES} != "" ]]; then
echo "These files are not well gofmt'ed:\n\n${NOT_FMT_FILES}"
# enable this when enums are properly formated
# exit 1
fi
tests/MyGame/Example/Monster.go
View file @ 026c6ddb
This diff is collapsed.
tests/MyGame/Example/Stat.go
View file @ 026c6ddb
......@@ -5,6 +5,7 @@ package Example
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Stat struct {
_tab flatbuffers.Table
}
......@@ -12,7 +13,7 @@ type Stat struct {
func GetRootAsStat(buf []byte, offset flatbuffers.UOffsetT) *Stat {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Stat{}
x.Init(buf, n + offset)
x.Init(buf, n+offset)
return x
}
......@@ -53,8 +54,18 @@ func (rcv *Stat) MutateCount(n uint16) bool {
return rcv._tab.MutateUint16Slot(8, n)
}
func StatStart(builder *flatbuffers.Builder) { builder.StartObject(3) }
func StatAddId(builder *flatbuffers.Builder, id flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(id), 0) }
func StatAddVal(builder *flatbuffers.Builder, val int64) { builder.PrependInt64Slot(1, val, 0) }
func StatAddCount(builder *flatbuffers.Builder, count uint16) { builder.PrependUint16Slot(2, count, 0) }
func StatEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() }
func StatStart(builder *flatbuffers.Builder) {
builder.StartObject(3)
}
func StatAddId(builder *flatbuffers.Builder, id flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(id), 0)
}
func StatAddVal(builder *flatbuffers.Builder, val int64) {
builder.PrependInt64Slot(1, val, 0)
}
func StatAddCount(builder *flatbuffers.Builder, count uint16) {
builder.PrependUint16Slot(2, count, 0)
}
func StatEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
tests/MyGame/Example/Test.go
View file @ 026c6ddb
......@@ -5,6 +5,7 @@ package Example
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Test struct {
_tab flatbuffers.Struct
}
......@@ -14,17 +15,24 @@ func (rcv *Test) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Pos = i
}
func (rcv *Test) A() int16 { return rcv._tab.GetInt16(rcv._tab.Pos + flatbuffers.UOffsetT(0)) }
func (rcv *Test) MutateA(n int16) bool { return rcv._tab.MutateInt16(rcv._tab.Pos + flatbuffers.UOffsetT(0), n) }
func (rcv *Test) B() int8 { return rcv._tab.GetInt8(rcv._tab.Pos + flatbuffers.UOffsetT(2)) }
func (rcv *Test) MutateB(n int8) bool { return rcv._tab.MutateInt8(rcv._tab.Pos + flatbuffers.UOffsetT(2), n) }
func (rcv *Test) A() int16 {
return rcv._tab.GetInt16(rcv._tab.Pos + flatbuffers.UOffsetT(0))
}
func (rcv *Test) MutateA(n int16) bool {
return rcv._tab.MutateInt16(rcv._tab.Pos+flatbuffers.UOffsetT(0), n)
}
func (rcv *Test) B() int8 {
return rcv._tab.GetInt8(rcv._tab.Pos + flatbuffers.UOffsetT(2))
}
func (rcv *Test) MutateB(n int8) bool {
return rcv._tab.MutateInt8(rcv._tab.Pos+flatbuffers.UOffsetT(2), n)
}
func CreateTest(builder *flatbuffers.Builder, a int16, b int8) flatbuffers.UOffsetT {
builder.Prep(2, 4)
builder.Pad(1)
builder.PrependInt8(b)
builder.PrependInt16(a)
return builder.Offset()
builder.Prep(2, 4)
builder.Pad(1)
builder.PrependInt8(b)
builder.PrependInt16(a)
return builder.Offset()
}
tests/MyGame/Example/TestSimpleTableWithEnum.go
View file @ 026c6ddb
......@@ -5,6 +5,7 @@ package Example
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type TestSimpleTableWithEnum struct {
_tab flatbuffers.Table
}
......@@ -12,7 +13,7 @@ type TestSimpleTableWithEnum struct {
func GetRootAsTestSimpleTableWithEnum(buf []byte, offset flatbuffers.UOffsetT) *TestSimpleTableWithEnum {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &TestSimpleTableWithEnum{}
x.Init(buf, n + offset)
x.Init(buf, n+offset)
return x
}
......@@ -33,6 +34,12 @@ func (rcv *TestSimpleTableWithEnum) MutateColor(n int8) bool {
return rcv._tab.MutateInt8Slot(4, n)
}
func TestSimpleTableWithEnumStart(builder *flatbuffers.Builder) { builder.StartObject(1) }
func TestSimpleTableWithEnumAddColor(builder *flatbuffers.Builder, color int8) { builder.PrependInt8Slot(0, color, 2) }
func TestSimpleTableWithEnumEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() }
func TestSimpleTableWithEnumStart(builder *flatbuffers.Builder) {
builder.StartObject(1)
}
func TestSimpleTableWithEnumAddColor(builder *flatbuffers.Builder, color int8) {
builder.PrependInt8Slot(0, color, 2)
}
func TestSimpleTableWithEnumEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
tests/MyGame/Example/Vec3.go
View file @ 026c6ddb
......@@ -5,6 +5,7 @@ package Example
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Vec3 struct {
_tab flatbuffers.Struct
}
......@@ -14,42 +15,62 @@ func (rcv *Vec3) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Pos = i
}
func (rcv *Vec3) X() float32 { return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(0)) }
func (rcv *Vec3) MutateX(n float32) bool { return rcv._tab.MutateFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(0), n) }
func (rcv *Vec3) X() float32 {
return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(0))
}
func (rcv *Vec3) MutateX(n float32) bool {
return rcv._tab.MutateFloat32(rcv._tab.Pos+flatbuffers.UOffsetT(0), n)
}
func (rcv *Vec3) Y() float32 { return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(4)) }
func (rcv *Vec3) MutateY(n float32) bool { return rcv._tab.MutateFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(4), n) }
func (rcv *Vec3) Y() float32 {
return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(4))
}
func (rcv *Vec3) MutateY(n float32) bool {
return rcv._tab.MutateFloat32(rcv._tab.Pos+flatbuffers.UOffsetT(4), n)
}
func (rcv *Vec3) Z() float32 { return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(8)) }
func (rcv *Vec3) MutateZ(n float32) bool { return rcv._tab.MutateFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(8), n) }
func (rcv *Vec3) Z() float32 {
return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(8))
}
func (rcv *Vec3) MutateZ(n float32) bool {
return rcv._tab.MutateFloat32(rcv._tab.Pos+flatbuffers.UOffsetT(8), n)
}
func (rcv *Vec3) Test1() float64 { return rcv._tab.GetFloat64(rcv._tab.Pos + flatbuffers.UOffsetT(16)) }
func (rcv *Vec3) MutateTest1(n float64) bool { return rcv._tab.MutateFloat64(rcv._tab.Pos + flatbuffers.UOffsetT(16), n) }
func (rcv *Vec3) Test1() float64 {
return rcv._tab.GetFloat64(rcv._tab.Pos + flatbuffers.UOffsetT(16))
}
func (rcv *Vec3) MutateTest1(n float64) bool {
return rcv._tab.MutateFloat64(rcv._tab.Pos+flatbuffers.UOffsetT(16), n)
}
func (rcv *Vec3) Test2() int8 { return rcv._tab.GetInt8(rcv._tab.Pos + flatbuffers.UOffsetT(24)) }
func (rcv *Vec3) MutateTest2(n int8) bool { return rcv._tab.MutateInt8(rcv._tab.Pos + flatbuffers.UOffsetT(24), n) }
func (rcv *Vec3) Test2() int8 {
return rcv._tab.GetInt8(rcv._tab.Pos + flatbuffers.UOffsetT(24))
}
func (rcv *Vec3) MutateTest2(n int8) bool {
return rcv._tab.MutateInt8(rcv._tab.Pos+flatbuffers.UOffsetT(24), n)
}
func (rcv *Vec3) Test3(obj *Test) *Test {
if obj == nil {
obj = new(Test)
}
obj.Init(rcv._tab.Bytes, rcv._tab.Pos + 26)
obj.Init(rcv._tab.Bytes, rcv._tab.Pos+26)
return obj
}
func CreateVec3(builder *flatbuffers.Builder, x float32, y float32, z float32, test1 float64, test2 int8, test3_a int16, test3_b int8) flatbuffers.UOffsetT {
builder.Prep(16, 32)
builder.Pad(2)
builder.Prep(2, 4)
builder.Pad(1)
builder.PrependInt8(test3_b)
builder.PrependInt16(test3_a)
builder.Pad(1)
builder.PrependInt8(test2)
builder.PrependFloat64(test1)
builder.Pad(4)
builder.PrependFloat32(z)
builder.PrependFloat32(y)
builder.PrependFloat32(x)
return builder.Offset()
builder.Prep(16, 32)
builder.Pad(2)
builder.Prep(2, 4)
builder.Pad(1)
builder.PrependInt8(test3_b)
builder.PrependInt16(test3_a)
builder.Pad(1)
builder.PrependInt8(test2)
builder.PrependFloat64(test1)
builder.Pad(4)
builder.PrependFloat32(z)
builder.PrependFloat32(y)
builder.PrependFloat32(x)
return builder.Offset()
}
tests/MyGame/Example2/Monster.go
View file @ 026c6ddb
......@@ -5,6 +5,7 @@ package Example2
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Monster struct {
_tab flatbuffers.Table
}
......@@ -12,7 +13,7 @@ type Monster struct {
func GetRootAsMonster(buf []byte, offset flatbuffers.UOffsetT) *Monster {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Monster{}
x.Init(buf, n + offset)
x.Init(buf, n+offset)
return x
}
......@@ -21,5 +22,9 @@ func (rcv *Monster) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Pos = i
}
func MonsterStart(builder *flatbuffers.Builder) { builder.StartObject(0) }
func MonsterEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() }
func MonsterStart(builder *flatbuffers.Builder) {
builder.StartObject(0)
}
func MonsterEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
tests/test.cpp
View file @ 026c6ddb
......@@ -978,15 +978,36 @@ void IntegerOutOfRangeTest() {
void UnicodeTest() {
flatbuffers::Parser parser;
// Without setting allow_non_utf8 = true, we treat \x sequences as byte sequences
// which are then validated as UTF-8.
TEST_EQ(parser.Parse("table T { F:string; }"
"root_type T;"
"{ F:\"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC"
"\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\" }"), true);
"\\u5225\\u30B5\\u30A4\\u30C8\\xE2\\x82\\xAC\\u0080\\uD83D\\uDE0E\" }"),
true);
std::string jsongen;
parser.opts.indent_step = -1;
GenerateText(parser, parser.builder_.GetBufferPointer(), &jsongen);
TEST_EQ(jsongen == "{F: \"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC"
"\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\"}", true);
TEST_EQ(jsongen,
std::string(
"{F: \"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC"
"\\u5225\\u30B5\\u30A4\\u30C8\\u20AC\\u0080\\uD83D\\uDE0E\"}"));
}
void UnicodeTestAllowNonUTF8() {
flatbuffers::Parser parser;
parser.opts.allow_non_utf8 = true;
TEST_EQ(parser.Parse("table T { F:string; }"
"root_type T;"
"{ F:\"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC"
"\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\\u0080\\uD83D\\uDE0E\" }"), true);
std::string jsongen;
parser.opts.indent_step = -1;
GenerateText(parser, parser.builder_.GetBufferPointer(), &jsongen);
TEST_EQ(jsongen,
std::string(
"{F: \"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC"
"\\u5225\\u30B5\\u30A4\\u30C8\\u0001\\x80\\u0080\\uD83D\\uDE0E\"}"));
}
void UnicodeSurrogatesTest() {
......@@ -1027,6 +1048,96 @@ void UnicodeInvalidSurrogatesTest() {
"{ F:\"\\uDC00\"}", "unpaired low surrogate");
}
void InvalidUTF8Test() {
// "1 byte" pattern, under min length of 2 bytes
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\x80\"}", "illegal UTF-8 sequence");
// 2 byte pattern, string too short
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\xDF\"}", "illegal UTF-8 sequence");
// 3 byte pattern, string too short
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\xEF\xBF\"}", "illegal UTF-8 sequence");
// 4 byte pattern, string too short
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\xF7\xBF\xBF\"}", "illegal UTF-8 sequence");
// "5 byte" pattern, string too short
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\xFB\xBF\xBF\xBF\"}", "illegal UTF-8 sequence");
// "6 byte" pattern, string too short
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\xFD\xBF\xBF\xBF\xBF\"}", "illegal UTF-8 sequence");
// "7 byte" pattern, string too short
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\xFE\xBF\xBF\xBF\xBF\xBF\"}", "illegal UTF-8 sequence");
// "5 byte" pattern, over max length of 4 bytes
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\xFB\xBF\xBF\xBF\xBF\"}", "illegal UTF-8 sequence");
// "6 byte" pattern, over max length of 4 bytes
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\xFD\xBF\xBF\xBF\xBF\xBF\"}", "illegal UTF-8 sequence");
// "7 byte" pattern, over max length of 4 bytes
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\xFE\xBF\xBF\xBF\xBF\xBF\xBF\"}", "illegal UTF-8 sequence");
// Three invalid encodings for U+000A (\n, aka NEWLINE)
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\xC0\x8A\"}", "illegal UTF-8 sequence");
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\xE0\x80\x8A\"}", "illegal UTF-8 sequence");
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\xF0\x80\x80\x8A\"}", "illegal UTF-8 sequence");
// Two invalid encodings for U+00A9 (COPYRIGHT SYMBOL)
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\xE0\x81\xA9\"}", "illegal UTF-8 sequence");
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\xF0\x80\x81\xA9\"}", "illegal UTF-8 sequence");
// Invalid encoding for U+20AC (EURO SYMBOL)
TestError(
"table T { F:string; }"
"root_type T;"
"{ F:\"\xF0\x82\x82\xAC\"}", "illegal UTF-8 sequence");
// UTF-16 surrogate values between U+D800 and U+DFFF cannot be encoded in UTF-8
TestError(
"table T { F:string; }"
"root_type T;"
// U+10400 "encoded" as U+D801 U+DC00
"{ F:\"\xED\xA0\x81\xED\xB0\x80\"}", "illegal UTF-8 sequence");
}
void UnknownFieldsTest() {
flatbuffers::IDLOptions opts;
opts.skip_unexpected_fields_in_json = true;
......@@ -1105,8 +1216,10 @@ int main(int /*argc*/, const char * /*argv*/[]) {
EnumStringsTest();
IntegerOutOfRangeTest();
UnicodeTest();
UnicodeTestAllowNonUTF8();
UnicodeSurrogatesTest();
UnicodeInvalidSurrogatesTest();
InvalidUTF8Test();
UnknownFieldsTest();
ParseUnionTest();
ConformTest();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment