Trimmed vtables of trailing zeroes.

This is something the format supports, but none of the builders
were doing. Can save 10-20% on FlatBuffer binary size!

Also fixed the Go tests.

Change-Id: I616c56ce9bbcfcaee23aa24f0532fcb60b6a8c75
Tested: on Linux.
parent 513958ea
......@@ -110,6 +110,11 @@ func (b *Builder) WriteVtable() (n UOffsetT) {
objectOffset := b.Offset()
existingVtable := UOffsetT(0)
// Trim vtable of trailing zeroes.
i := len(b.vtable) - 1;
for ; i >= 0 && b.vtable[i] == 0; i-- {}
b.vtable = b.vtable[:i + 1];
// Search backwards through existing vtables, because similar vtables
// are likely to have been recently appended. See
// BenchmarkVtableDeduplication for a case in which this heuristic
......
......@@ -704,8 +704,8 @@ class FlatBufferBuilder
explicit FlatBufferBuilder(size_t initial_size = 1024,
Allocator *allocator = nullptr,
bool own_allocator = false)
: buf_(initial_size, allocator, own_allocator), nested(false),
finished(false), minalign_(1), force_defaults_(false),
: buf_(initial_size, allocator, own_allocator), max_voffset_(0),
nested(false), finished(false), minalign_(1), force_defaults_(false),
dedup_vtables_(true), string_pool(nullptr) {
offsetbuf_.reserve(16); // Avoid first few reallocs.
vtables_.reserve(16);
......@@ -725,7 +725,7 @@ class FlatBufferBuilder
/// to construct another buffer.
void Clear() {
buf_.clear();
offsetbuf_.clear();
ClearOffsets();
nested = false;
finished = false;
vtables_.clear();
......@@ -839,6 +839,7 @@ class FlatBufferBuilder
void TrackField(voffset_t field, uoffset_t off) {
FieldLoc fl = { off, field };
offsetbuf_.push_back(fl);
max_voffset_ = (std::max)(max_voffset_, field);
}
// Like PushElement, but additionally tracks the field this represents.
......@@ -899,7 +900,7 @@ class FlatBufferBuilder
// This finishes one serialized object by generating the vtable if it's a
// table, comparing it against existing vtables, and writing the
// resulting vtable offset.
uoffset_t EndTable(uoffset_t start, voffset_t numfields) {
uoffset_t EndTable(uoffset_t start) {
// If you get this assert, a corresponding StartTable wasn't called.
assert(nested);
// Write the vtable offset, which is the start of any Table.
......@@ -908,11 +909,17 @@ class FlatBufferBuilder
// Write a vtable, which consists entirely of voffset_t elements.
// It starts with the number of offsets, followed by a type id, followed
// by the offsets themselves. In reverse:
buf_.fill_big(numfields * sizeof(voffset_t));
// Include space for the last offset and ensure empty tables have a
// minimum size.
max_voffset_ = (std::max)(static_cast<voffset_t>(max_voffset_ +
sizeof(voffset_t)),
FieldIndexToOffset(0));
buf_.fill_big(max_voffset_);
auto table_object_size = vtableoffsetloc - start;
assert(table_object_size < 0x10000); // Vtable use 16bit offsets.
PushElement<voffset_t>(static_cast<voffset_t>(table_object_size));
PushElement<voffset_t>(FieldIndexToOffset(numfields));
WriteScalar<voffset_t>(buf_.data() + sizeof(voffset_t),
static_cast<voffset_t>(table_object_size));
WriteScalar<voffset_t>(buf_.data(), max_voffset_);
// Write the offsets into the table
for (auto field_location = offsetbuf_.begin();
field_location != offsetbuf_.end();
......@@ -922,7 +929,7 @@ class FlatBufferBuilder
assert(!ReadScalar<voffset_t>(buf_.data() + field_location->id));
WriteScalar<voffset_t>(buf_.data() + field_location->id, pos);
}
offsetbuf_.clear();
ClearOffsets();
auto vt1 = reinterpret_cast<voffset_t *>(buf_.data());
auto vt1_size = ReadScalar<voffset_t>(vt1);
auto vt_use = GetSize();
......@@ -955,6 +962,11 @@ class FlatBufferBuilder
return vtableoffsetloc;
}
// DEPRECATED: call the version above instead.
uoffset_t EndTable(uoffset_t start, voffset_t /*numfields*/) {
return EndTable(start);
}
// This checks a required field has been set in a given table that has
// just been constructed.
template<typename T> void Required(Offset<T> table, voffset_t field) {
......@@ -973,7 +985,10 @@ class FlatBufferBuilder
uoffset_t EndStruct() { return GetSize(); }
void ClearOffsets() { offsetbuf_.clear(); }
void ClearOffsets() {
offsetbuf_.clear();
max_voffset_ = 0;
}
// Aligns such that when "len" bytes are written, an object can be written
// after it with "alignment" without padding.
......@@ -1510,6 +1525,9 @@ class FlatBufferBuilder
// Accumulating offsets of table members while it is being built.
std::vector<FieldLoc> offsetbuf_;
// Track how much of the vtable is in use, so we can output the most compact
// possible vtable.
voffset_t max_voffset_;
// Ensure objects are not nested.
bool nested;
......
......@@ -42,6 +42,29 @@ enum BaseType {
Union = 16
};
inline BaseType (&EnumValuesBaseType())[17] {
static BaseType values[] = {
None,
UType,
Bool,
Byte,
UByte,
Short,
UShort,
Int,
UInt,
Long,
ULong,
Float,
Double,
String,
Vector,
Obj,
Union
};
return values;
}
inline const char **EnumNamesBaseType() {
static const char *names[] = {
"None",
......@@ -113,7 +136,7 @@ struct TypeBuilder {
}
TypeBuilder &operator=(const TypeBuilder &);
flatbuffers::Offset<Type> Finish() {
const auto end = fbb_.EndTable(start_, 3);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Type>(end);
return o;
}
......@@ -173,7 +196,7 @@ struct KeyValueBuilder {
}
KeyValueBuilder &operator=(const KeyValueBuilder &);
flatbuffers::Offset<KeyValue> Finish() {
const auto end = fbb_.EndTable(start_, 2);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<KeyValue>(end);
fbb_.Required(o, KeyValue::VT_KEY);
return o;
......@@ -266,7 +289,7 @@ struct EnumValBuilder {
}
EnumValBuilder &operator=(const EnumValBuilder &);
flatbuffers::Offset<EnumVal> Finish() {
const auto end = fbb_.EndTable(start_, 4);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<EnumVal>(end);
fbb_.Required(o, EnumVal::VT_NAME);
return o;
......@@ -381,7 +404,7 @@ struct EnumBuilder {
}
EnumBuilder &operator=(const EnumBuilder &);
flatbuffers::Offset<Enum> Finish() {
const auto end = fbb_.EndTable(start_, 6);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Enum>(end);
fbb_.Required(o, Enum::VT_NAME);
fbb_.Required(o, Enum::VT_VALUES);
......@@ -544,7 +567,7 @@ struct FieldBuilder {
}
FieldBuilder &operator=(const FieldBuilder &);
flatbuffers::Offset<Field> Finish() {
const auto end = fbb_.EndTable(start_, 11);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Field>(end);
fbb_.Required(o, Field::VT_NAME);
fbb_.Required(o, Field::VT_TYPE);
......@@ -695,7 +718,7 @@ struct ObjectBuilder {
}
ObjectBuilder &operator=(const ObjectBuilder &);
flatbuffers::Offset<Object> Finish() {
const auto end = fbb_.EndTable(start_, 7);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Object>(end);
fbb_.Required(o, Object::VT_NAME);
fbb_.Required(o, Object::VT_FIELDS);
......@@ -808,7 +831,7 @@ struct SchemaBuilder {
}
SchemaBuilder &operator=(const SchemaBuilder &);
flatbuffers::Offset<Schema> Finish() {
const auto end = fbb_.EndTable(start_, 5);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Schema>(end);
fbb_.Required(o, Schema::VT_OBJECTS);
fbb_.Required(o, Schema::VT_ENUMS);
......
......@@ -478,7 +478,7 @@ public class FlatBufferBuilder {
obj.sortTables(offsets, bb);
return createVectorOfTables(offsets);
}
/**
* Encode the string `s` in the buffer using UTF-8. If {@code s} is
* already a {@link CharBuffer}, this method is allocation free.
......@@ -744,7 +744,11 @@ public class FlatBufferBuilder {
addInt(0);
int vtableloc = offset();
// Write out the current vtable.
for (int i = vtable_in_use - 1; i >= 0 ; i--) {
int i = vtable_in_use - 1;
// Trim trailing zeroes.
for (; i >= 0 && vtable[i] == 0; i--) {}
int trimmed_size = i + 1;
for (; i >= 0 ; i--) {
// Offset relative to the start of the table.
short off = (short)(vtable[i] != 0 ? vtableloc - vtable[i] : 0);
addShort(off);
......@@ -752,12 +756,12 @@ public class FlatBufferBuilder {
final int standard_fields = 2; // The fields below:
addShort((short)(vtableloc - object_start));
addShort((short)((vtable_in_use + standard_fields) * SIZEOF_SHORT));
addShort((short)((trimmed_size + standard_fields) * SIZEOF_SHORT));
// Search for an existing vtable that matches the current one.
int existing_vtable = 0;
outer_loop:
for (int i = 0; i < num_vtables; i++) {
for (i = 0; i < num_vtables; i++) {
int vt1 = bb.capacity() - vtables[i];
int vt2 = space;
short len = bb.getShort(vt1);
......
......@@ -604,23 +604,28 @@ flatbuffers.Builder.prototype.endObject = function() {
this.addInt32(0);
var vtableloc = this.offset();
// Trim trailing zeroes.
var i = this.vtable_in_use - 1;
for (; i >= 0 && this.vtable[i] == 0; i--) {}
var trimmed_size = i + 1;
// Write out the current vtable.
for (var i = this.vtable_in_use - 1; i >= 0; i--) {
for (; i >= 0; i--) {
// Offset relative to the start of the table.
this.addInt16(this.vtable[i] != 0 ? vtableloc - this.vtable[i] : 0);
}
var standard_fields = 2; // The fields below:
this.addInt16(vtableloc - this.object_start);
this.addInt16((this.vtable_in_use + standard_fields) * flatbuffers.SIZEOF_SHORT);
var len = (trimmed_size + standard_fields) * flatbuffers.SIZEOF_SHORT;
this.addInt16(len);
// Search for an existing vtable that matches the current one.
var existing_vtable = 0;
var vt1 = this.space;
outer_loop:
for (var i = 0; i < this.vtables.length; i++) {
var vt1 = this.bb.capacity() - this.vtables[i];
var vt2 = this.space;
var len = this.bb.readInt16(vt1);
for (i = 0; i < this.vtables.length; i++) {
var vt2 = this.bb.capacity() - this.vtables[i];
if (len == this.bb.readInt16(vt2)) {
for (var j = flatbuffers.SIZEOF_SHORT; j < len; j += flatbuffers.SIZEOF_SHORT) {
if (this.bb.readInt16(vt1 + j) != this.bb.readInt16(vt2 + j)) {
......
......@@ -500,7 +500,11 @@ namespace FlatBuffers
AddInt((int)0);
var vtableloc = Offset;
// Write out the current vtable.
for (int i = _vtableSize - 1; i >= 0 ; i--) {
int i = _vtableSize - 1;
// Trim trailing zeroes.
for (; i >= 0 && _vtable[i] == 0; i--) {}
int trimmedSize = i + 1;
for (; i >= 0 ; i--) {
// Offset relative to the start of the table.
short off = (short)(_vtable[i] != 0
? vtableloc - _vtable[i]
......@@ -513,12 +517,12 @@ namespace FlatBuffers
const int standardFields = 2; // The fields below:
AddShort((short)(vtableloc - _objectStart));
AddShort((short)((_vtableSize + standardFields) *
AddShort((short)((trimmedSize + standardFields) *
sizeof(short)));
// Search for an existing vtable that matches the current one.
int existingVtable = 0;
for (int i = 0; i < _numVtables; i++) {
for (i = 0; i < _numVtables; i++) {
int vt1 = _bb.Length - _vtables[i];
int vt2 = _space;
short len = _bb.GetShort(vt1);
......
......@@ -596,7 +596,7 @@ class FlatbufferBuilder
if (function_exists('mb_detect_encoding')) {
return (bool) mb_detect_encoding($bytes, 'UTF-8', true);
}
$len = strlen($bytes);
if ($len < 1) {
/* NOTE: always return 1 when passed string is null */
......@@ -812,14 +812,18 @@ class FlatbufferBuilder
$this->addInt(0);
$vtableloc = $this->offset();
for ($i = $this->vtable_in_use -1; $i >= 0; $i--) {
$i = $this->vtable_in_use -1;
// Trim trailing zeroes.
for (; $i >= 0 && $this->vtable[$i] == 0; $i--) {}
$trimmed_size = $i + 1;
for (; $i >= 0; $i--) {
$off = ($this->vtable[$i] != 0) ? $vtableloc - $this->vtable[$i] : 0;
$this->addShort($off);
}
$standard_fields = 2; // the fields below
$this->addShort($vtableloc - $this->object_start);
$this->addShort(($this->vtable_in_use + $standard_fields) * Constants::SIZEOF_SHORT);
$this->addShort(($trimmed_size + $standard_fields) * Constants::SIZEOF_SHORT);
// search for an existing vtable that matches the current one.
$existing_vtable = 0;
......
......@@ -193,6 +193,10 @@ class Builder(object):
objectOffset = self.Offset()
existingVtable = None
# Trim trailing 0 offsets.
while self.current_vtable and self.current_vtable[-1] == 0:
self.current_vtable.pop()
# Search backwards through existing vtables, because similar vtables
# are likely to have been recently appended. See
# BenchmarkVtableDeduplication for a case in which this heuristic
......
......@@ -316,7 +316,7 @@ struct MonsterBuilder {
}
MonsterBuilder &operator=(const MonsterBuilder &);
flatbuffers::Offset<Monster> Finish() {
const auto end = fbb_.EndTable(start_, 10);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Monster>(end);
return o;
}
......@@ -426,7 +426,7 @@ struct WeaponBuilder {
}
WeaponBuilder &operator=(const WeaponBuilder &);
flatbuffers::Offset<Weapon> Finish() {
const auto end = fbb_.EndTable(start_, 2);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Weapon>(end);
return o;
}
......
......@@ -1531,9 +1531,8 @@ class CppGenerator : public BaseGenerator {
"(const {{STRUCT_NAME}}Builder &);";
// Finish() function.
auto num_fields = NumToString(struct_def.fields.vec.size());
code_ += " flatbuffers::Offset<{{STRUCT_NAME}}> Finish() {";
code_ += " const auto end = fbb_.EndTable(start_, " + num_fields + ");";
code_ += " const auto end = fbb_.EndTable(start_);";
code_ += " auto o = flatbuffers::Offset<{{STRUCT_NAME}}>(end);";
for (auto it = struct_def.fields.vec.begin();
......
......@@ -1071,8 +1071,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
builder_.PopBytes(struct_def.bytesize);
assert(!ovalue);
} else {
auto val = builder_.EndTable(start,
static_cast<voffset_t>(struct_def.fields.vec.size()));
auto val = builder_.EndTable(start);
if (ovalue) *ovalue = val;
if (value) *value = NumToString(val);
}
......
......@@ -481,7 +481,7 @@ Offset<const Table *> CopyTable(FlatBufferBuilder &fbb,
fbb.ClearOffsets();
return fbb.EndStruct();
} else {
return fbb.EndTable(start, static_cast<voffset_t>(fielddefs->size()));
return fbb.EndTable(start);
}
}
......
......@@ -205,11 +205,11 @@ namespace FlatBuffers.Test
builder.EndObject();
Assert.ArrayEqual(new byte[]
{
0, 0, 0, 0, 0, 0, // padding to 16 bytes
6, 0, // vtable bytes
// No padding.
4, 0, // vtable bytes
4, 0, // end of object from here
0, 0, // entry 0 is empty (default value)
6, 0, 0, 0, // int32 offset for start of vtable
// entry 0 is not stored (trimmed end of vtable)
4, 0, 0, 0, // int32 offset for start of vtable
},
builder.DataBuffer.Data);
}
......
......@@ -30,6 +30,9 @@ mkdir -p ${go_src}/github.com/google/flatbuffers/go
mkdir -p ${go_src}/flatbuffers_test
cp -a MyGame/Example/*.go ./go_gen/src/MyGame/Example/
# do not compile the gRPC generated files, which are not tested by go_test.go
# below, but have their own test.
rm ./go_gen/src/MyGame/Example/*_grpc.go
cp -a ../go/* ./go_gen/src/github.com/google/flatbuffers/go
cp -a ./go_test.go ./go_gen/src/flatbuffers_test/
......
......@@ -715,10 +715,10 @@ func CheckByteLayout(fail func(string, ...interface{})) {
b.PrependBoolSlot(0, false, false)
b.EndObject()
check([]byte{
6, 0, // vtable bytes
4, 0, // vtable bytes
4, 0, // end of object from here
0, 0, // entry 1 is zero
6, 0, 0, 0, // offset for start of vtable (int32)
// entry 1 is zero and not stored.
4, 0, 0, 0, // offset for start of vtable (int32)
})
// test 10: vtable with one int16
......@@ -1085,7 +1085,6 @@ func CheckManualBuild(fail func(string, ...interface{})) ([]byte, flatbuffers.UO
b.PrependByteSlot(7, 1, 0)
b.PrependUOffsetTSlot(8, mon2, 0)
b.PrependUOffsetTSlot(9, test4, 0)
b.PrependUOffsetTSlot(9, test5, 0)
mon := b.EndObject()
b.Finish(mon)
......
No preview for this file type
No preview for this file type
......@@ -345,7 +345,7 @@ struct MonsterBuilder {
}
MonsterBuilder &operator=(const MonsterBuilder &);
flatbuffers::Offset<Monster> Finish() {
const auto end = fbb_.EndTable(start_, 0);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Monster>(end);
return o;
}
......@@ -404,7 +404,7 @@ struct TestSimpleTableWithEnumBuilder {
}
TestSimpleTableWithEnumBuilder &operator=(const TestSimpleTableWithEnumBuilder &);
flatbuffers::Offset<TestSimpleTableWithEnum> Finish() {
const auto end = fbb_.EndTable(start_, 1);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<TestSimpleTableWithEnum>(end);
return o;
}
......@@ -487,7 +487,7 @@ struct StatBuilder {
}
StatBuilder &operator=(const StatBuilder &);
flatbuffers::Offset<Stat> Finish() {
const auto end = fbb_.EndTable(start_, 3);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Stat>(end);
return o;
}
......@@ -1015,7 +1015,7 @@ struct MonsterBuilder {
}
MonsterBuilder &operator=(const MonsterBuilder &);
flatbuffers::Offset<Monster> Finish() {
const auto end = fbb_.EndTable(start_, 34);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Monster>(end);
fbb_.Required(o, Monster::VT_NAME);
return o;
......@@ -1352,7 +1352,7 @@ struct TypeAliasesBuilder {
}
TypeAliasesBuilder &operator=(const TypeAliasesBuilder &);
flatbuffers::Offset<TypeAliases> Finish() {
const auto end = fbb_.EndTable(start_, 12);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<TypeAliases>(end);
return o;
}
......
......@@ -105,7 +105,7 @@ struct TableInNestedNSBuilder {
}
TableInNestedNSBuilder &operator=(const TableInNestedNSBuilder &);
flatbuffers::Offset<TableInNestedNS> Finish() {
const auto end = fbb_.EndTable(start_, 1);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<TableInNestedNS>(end);
return o;
}
......
......@@ -76,7 +76,7 @@ struct TableInFirstNSBuilder {
}
TableInFirstNSBuilder &operator=(const TableInFirstNSBuilder &);
flatbuffers::Offset<TableInFirstNS> Finish() {
const auto end = fbb_.EndTable(start_, 3);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<TableInFirstNS>(end);
return o;
}
......@@ -140,7 +140,7 @@ struct TableInCBuilder {
}
TableInCBuilder &operator=(const TableInCBuilder &);
flatbuffers::Offset<TableInC> Finish() {
const auto end = fbb_.EndTable(start_, 2);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<TableInC>(end);
return o;
}
......@@ -190,7 +190,7 @@ struct SecondTableInABuilder {
}
SecondTableInABuilder &operator=(const SecondTableInABuilder &);
flatbuffers::Offset<SecondTableInA> Finish() {
const auto end = fbb_.EndTable(start_, 1);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<SecondTableInA>(end);
return o;
}
......
......@@ -490,10 +490,10 @@ class TestByteLayout(unittest.TestCase):
b.PrependBoolSlot(0, False, False)
b.EndObject()
self.assertBuilderEquals(b, [
6, 0, # vtable bytes
4, 0, # vtable bytes
4, 0, # end of object from here
0, 0, # entry 1 is zero
6, 0, 0, 0, # offset for start of vtable (int32)
# entry 1 is zero and not stored
4, 0, 0, 0, # offset for start of vtable (int32)
])
def test_vtable_with_one_int16(self):
......
......@@ -863,7 +863,7 @@ void FuzzTest1() {
case 10: builder.AddElement<double >(off, double_val, 0); break;
}
}
objects[i] = builder.EndTable(start, fields_per_object);
objects[i] = builder.EndTable(start);
}
builder.PreAlign<flatbuffers::largest_scalar_t>(0); // Align whole buffer.
......
......@@ -196,7 +196,7 @@ struct AttackerBuilder {
}
AttackerBuilder &operator=(const AttackerBuilder &);
flatbuffers::Offset<Attacker> Finish() {
const auto end = fbb_.EndTable(start_, 1);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Attacker>(end);
return o;
}
......@@ -309,7 +309,7 @@ struct MovieBuilder {
}
MovieBuilder &operator=(const MovieBuilder &);
flatbuffers::Offset<Movie> Finish() {
const auto end = fbb_.EndTable(start_, 4);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Movie>(end);
return o;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment