[Python] (scalar) vector reading speedup via numpy (#4390)

* Add numpy accessor to python flatbuffers scalar vectors

* Update python tests to test numpy vector accessor

* Update appveyor CI to run Python tests, save generated code as artifact

* Update example generated python code

* Add numpy info to python usage docs

* Update test schema and python tests w/ multi-byte vector

* did not mean to push profiling code

* adding float64 numpy tests

appveyor.yml

@@ -5,6 +5,12 @@ branches:
 os: Visual Studio 2015
 
 environment:
+
+  global:
+    # Workaround for https://github.com/conda/conda-build/issues/636
+    PYTHONIOENCODING: UTF-8
+    CONDA_INSTALL_LOCN: "C:\\Miniconda35-x64"
+
   matrix:
     - CMAKE_VS_VERSION: "10 2010"
     - CMAKE_VS_VERSION: "14 2015"
@@ -26,17 +32,39 @@ build:
   project: ALL_BUILD.vcxproj
   verbosity: minimal
 
+install:
+  - set PATH=%CONDA_INSTALL_LOCN%;%CONDA_INSTALL_LOCN%\scripts;%PATH%;
+
 test_script:
+  - "cd tests"
+  - rem "Building all code"
+  - generate_code.bat -b %CONFIGURATION%
+  - 7z a GeneratedMyGameCode.zip MyGame\
   - rem "---------------- C++ -----------------"
+  - "cd .."
   - "%CONFIGURATION%\\flattests.exe"
-  - rem "---------------- Java -----------------"
   - "cd tests"
+  - rem "---------------- Java -----------------"
   - "java -version"
   - "JavaTest.bat"
   - rem "---------------- JS -----------------"
   - "node --version"
   - "..\\%CONFIGURATION%\\flatc -b -I include_test monster_test.fbs unicode_test.json"
   - "node JavaScriptTest ./monster_test_generated"
+  - rem "-------------- Python ---------------"
+  - where python
+  - python --version
+  - where pip
+  - pip --version
+  - where conda
+  - conda --version
+  - rem "installing flatbuffers python library"
+  - pip install ../python
+  - rem "testing without installing Numpy"
+  - python py_test.py 0 0 0
+  - rem "testing after installing Numpy"
+  - conda install --yes numpy
+  - python py_test.py 0 0 0
   - rem "---------------- C# -----------------"
   # Have to compile this here rather than in "build" above because AppVeyor only
   # supports building one project??
@@ -47,5 +75,7 @@ test_script:
   - "cd ..\\.."
 
 artifacts:
-  - path: $(CONFIGURATION)\\flatc.exe
+  - path: $(CONFIGURATION)\flatc.exe
     name: flatc.exe
+  - path: tests\GeneratedMyGameCode.zip
+    name: GeneratedMyGameCode.zip

docs/source/PythonUsage.md

@@ -64,6 +64,33 @@ Now you can access values like this:
     pos = monster.Pos()
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+## Support for Numpy arrays
+
+The Flatbuffers python library also has support for accessing scalar
+vectors as numpy arrays. This can be orders of magnitude faster than
+iterating over the vector one element at a time, and is particularly
+useful when unpacking large nested flatbuffers. The generated code for
+a scalar vector will have a method `<vector name>AsNumpy()`. In the
+case of the Monster example, you could access the inventory vector
+like this:
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.py}
+    inventory = monster.InventoryAsNumpy()
+    # inventory is a numpy array of type np.dtype('uint8')
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+instead of
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.py}
+    inventory = []
+    for i in range(monster.InventoryLength()):
+        inventory.append(int(monster.Inventory(i)))
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Numpy is not a requirement. If numpy is not installed on your system,
+then attempting to access one of the `*asNumpy()` methods will result
+in a `NumpyRequiredForThisFeature` exception.
+
 ## Text Parsing
 
 There currently is no support for parsing text (Schema's and JSON) directly

python/flatbuffers/compat.py

@@ -12,9 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-""" A tiny version of `six` to help with backwards compability. """
+""" A tiny version of `six` to help with backwards compability. Also includes
+ compatibility helpers for numpy. """
 
 import sys
+import imp
 
 PY2 = sys.version_info[0] == 2
 PY26 = sys.version_info[0:2] == (2, 6)
@@ -43,4 +45,37 @@ else:
         memoryview_type = memoryview
         struct_bool_decl = "?"
 
+# Helper functions to facilitate making numpy optional instead of required
+
+def import_numpy():
+    """
+    Returns the numpy module if it exists on the system,
+    otherwise returns None.
+    """
+    try:
+        imp.find_module('numpy')
+        numpy_exists = True
+    except ImportError:
+        numpy_exists = False
+
+    if numpy_exists:
+        # We do this outside of try/except block in case numpy exists
+        # but is not installed correctly. We do not want to catch an
+        # incorrect installation which would manifest as an
+        # ImportError.
+        import numpy as np
+    else:
+        np = None
+
+    return np
+
+
+class NumpyRequiredForThisFeature(RuntimeError):
+    """
+    Error raised when user tries to use a feature that
+    requires numpy without having numpy installed.
+    """
+    pass
+
+
 # NOTE: Future Jython support may require code here (look at `six`).

python/flatbuffers/encode.py

@@ -15,13 +15,26 @@
 from . import number_types as N
 from . import packer
 from .compat import memoryview_type
+from .compat import import_numpy, NumpyRequiredForThisFeature
 
+np = import_numpy()
 
 def Get(packer_type, buf, head):
-    """ Get decodes a value at buf[head:] using `packer_type`. """
+    """ Get decodes a value at buf[head] using `packer_type`. """
     return packer_type.unpack_from(memoryview_type(buf), head)[0]
 
 
+def GetVectorAsNumpy(numpy_type, buf, count, offset):
+    """ GetVecAsNumpy decodes values starting at buf[head] as
+    `numpy_type`, where `numpy_type` is a numpy dtype. """
+    if np is not None:
+        # TODO: could set .flags.writeable = False to make users jump through
+        #       hoops before modifying...
+        return np.frombuffer(buf, dtype=numpy_type, count=count, offset=offset)
+    else:
+        raise NumpyRequiredForThisFeature('Numpy was not found.')
+
+
 def Write(packer_type, buf, head, n):
-    """ Write encodes `n` at buf[head:] using `packer_type`. """
+    """ Write encodes `n` at buf[head] using `packer_type`. """
     packer_type.pack_into(buf, head, n)

python/flatbuffers/number_types.py

@@ -16,7 +16,9 @@ import collections
 import struct
 
 from . import packer
+from .compat import import_numpy, NumpyRequiredForThisFeature
 
+np = import_numpy()
 
 # For reference, see:
 # https://docs.python.org/2/library/ctypes.html#ctypes-fundamental-data-types-2
@@ -170,3 +172,10 @@ def uint64_to_float64(n):
     packed = struct.pack("<1Q", n)
     (unpacked,) = struct.unpack("<1d", packed)
     return unpacked
+
+
+def to_numpy_type(number_type):
+    if np is not None:
+        return np.dtype(number_type.name).newbyteorder('<')
+    else:
+        raise NumpyRequiredForThisFeature('Numpy was not found.')

python/flatbuffers/table.py

@@ -101,6 +101,18 @@ class Table(object):
             return d
         return self.Get(validator_flags, self.Pos + off)
 
+    def GetVectorAsNumpy(self, flags, off):
+        """
+        GetVectorAsNumpy returns the vector that starts at `Vector(off)`
+        as a numpy array with the type specified by `flags`. The array is
+        a `view` into Bytes, so modifying the returned array will
+        modify Bytes in place.
+        """
+        offset = self.Vector(off)
+        length = self.VectorLen(off) # TODO: length accounts for bytewidth, right?
+        numpy_dtype = N.to_numpy_type(flags)
+        return encode.GetVectorAsNumpy(numpy_dtype, self.Bytes, length, offset)
+
     def GetVOffsetTSlot(self, slot, d):
         """
         GetVOffsetTSlot retrieves the VOffsetT that the given vtable location

src/idl_gen_python.cpp

@@ -274,6 +274,38 @@ static void GetMemberOfVectorOfNonStruct(const StructDef &struct_def,
   code += "\n";
 }
 
+// Returns a non-struct vector as a numpy array. Much faster
+// than iterating over the vector element by element.
+static void GetVectorOfNonStructAsNumpy(const StructDef &struct_def,
+                                        const FieldDef &field,
+                                        std::string *code_ptr) {
+  std::string &code = *code_ptr;
+  auto vectortype = field.value.type.VectorType();
+
+  // Currently, we only support accessing as numpy array if
+  // the vector type is a scalar.
+  if (!(IsScalar(vectortype.base_type))) {
+    return;
+  }
+
+  GenReceiver(struct_def, code_ptr);
+  code += MakeCamel(field.name) + "AsNumpy(self):";
+  code += OffsetPrefix(field);
+
+  code += Indent + Indent + Indent;
+  code += "return ";
+  code += "self._tab.GetVectorAsNumpy(flatbuffers.number_types.";
+  code += MakeCamel(GenTypeGet(field.value.type));
+  code += "Flags, o)\n";
+
+  if (vectortype.base_type == BASE_TYPE_STRING) {
+    code += Indent + Indent + "return \"\"\n";
+  } else {
+    code += Indent + Indent + "return 0\n";
+  }
+  code += "\n";
+}
+
 // Begin the creator function signature.
 static void BeginBuilderArgs(const StructDef &struct_def,
                              std::string *code_ptr) {
@@ -440,6 +472,7 @@ static void GenStructAccessor(const StructDef &struct_def,
           GetMemberOfVectorOfStruct(struct_def, field, code_ptr);
         } else {
           GetMemberOfVectorOfNonStruct(struct_def, field, code_ptr);
+          GetVectorOfNonStructAsNumpy(struct_def, field, code_ptr);
         }
         break;
       }

tests/MyGame/Example/Monster.cs

@@ -89,8 +89,16 @@ public struct Monster : IFlatbufferObject
   public bool MutateFlex(int j, byte flex) { int o = __p.__offset(64); if (o != 0) { __p.bb.Put(__p.__vector(o) + j * 1, flex); return true; } else { return false; } }
   public Test? Test5(int j) { int o = __p.__offset(66); return o != 0 ? (Test?)(new Test()).__assign(__p.__vector(o) + j * 4, __p.bb) : null; }
   public int Test5Length { get { int o = __p.__offset(66); return o != 0 ? __p.__vector_len(o) : 0; } }
+  public long VectorOfLongs(int j) { int o = __p.__offset(68); return o != 0 ? __p.bb.GetLong(__p.__vector(o) + j * 8) : (long)0; }
+  public int VectorOfLongsLength { get { int o = __p.__offset(68); return o != 0 ? __p.__vector_len(o) : 0; } }
+  public ArraySegment<byte>? GetVectorOfLongsBytes() { return __p.__vector_as_arraysegment(68); }
+  public bool MutateVectorOfLongs(int j, long vector_of_longs) { int o = __p.__offset(68); if (o != 0) { __p.bb.PutLong(__p.__vector(o) + j * 8, vector_of_longs); return true; } else { return false; } }
+  public double VectorOfDoubles(int j) { int o = __p.__offset(70); return o != 0 ? __p.bb.GetDouble(__p.__vector(o) + j * 8) : (double)0; }
+  public int VectorOfDoublesLength { get { int o = __p.__offset(70); return o != 0 ? __p.__vector_len(o) : 0; } }
+  public ArraySegment<byte>? GetVectorOfDoublesBytes() { return __p.__vector_as_arraysegment(70); }
+  public bool MutateVectorOfDoubles(int j, double vector_of_doubles) { int o = __p.__offset(70); if (o != 0) { __p.bb.PutDouble(__p.__vector(o) + j * 8, vector_of_doubles); return true; } else { return false; } }
 
-  public static void StartMonster(FlatBufferBuilder builder) { builder.StartObject(32); }
+  public static void StartMonster(FlatBufferBuilder builder) { builder.StartObject(34); }
   public static void AddPos(FlatBufferBuilder builder, Offset<Vec3> posOffset) { builder.AddStruct(0, posOffset.Value, 0); }
   public static void AddMana(FlatBufferBuilder builder, short mana) { builder.AddShort(1, mana, 150); }
   public static void AddHp(FlatBufferBuilder builder, short hp) { builder.AddShort(2, hp, 100); }
@@ -139,6 +147,12 @@ public struct Monster : IFlatbufferObject
   public static void StartFlexVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(1, numElems, 1); }
   public static void AddTest5(FlatBufferBuilder builder, VectorOffset test5Offset) { builder.AddOffset(31, test5Offset.Value, 0); }
   public static void StartTest5Vector(FlatBufferBuilder builder, int numElems) { builder.StartVector(4, numElems, 2); }
+  public static void AddVectorOfLongs(FlatBufferBuilder builder, VectorOffset vectorOfLongsOffset) { builder.AddOffset(32, vectorOfLongsOffset.Value, 0); }
+  public static VectorOffset CreateVectorOfLongsVector(FlatBufferBuilder builder, long[] data) { builder.StartVector(8, data.Length, 8); for (int i = data.Length - 1; i >= 0; i--) builder.AddLong(data[i]); return builder.EndVector(); }
+  public static void StartVectorOfLongsVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(8, numElems, 8); }
+  public static void AddVectorOfDoubles(FlatBufferBuilder builder, VectorOffset vectorOfDoublesOffset) { builder.AddOffset(33, vectorOfDoublesOffset.Value, 0); }
+  public static VectorOffset CreateVectorOfDoublesVector(FlatBufferBuilder builder, double[] data) { builder.StartVector(8, data.Length, 8); for (int i = data.Length - 1; i >= 0; i--) builder.AddDouble(data[i]); return builder.EndVector(); }
+  public static void StartVectorOfDoublesVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(8, numElems, 8); }
   public static Offset<Monster> EndMonster(FlatBufferBuilder builder) {
     int o = builder.EndObject();
     builder.Required(o, 10);  // name

tests/MyGame/Example/Monster.go

@@ -482,8 +482,42 @@ func (rcv *Monster) Test5Length() int {
 	return 0
 }
 
+func (rcv *Monster) VectorOfLongs(j int) int64 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(68))
+	if o != 0 {
+		a := rcv._tab.Vector(o)
+		return rcv._tab.GetInt64(a + flatbuffers.UOffsetT(j*8))
+	}
+	return 0
+}
+
+func (rcv *Monster) VectorOfLongsLength() int {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(68))
+	if o != 0 {
+		return rcv._tab.VectorLen(o)
+	}
+	return 0
+}
+
+func (rcv *Monster) VectorOfDoubles(j int) float64 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(70))
+	if o != 0 {
+		a := rcv._tab.Vector(o)
+		return rcv._tab.GetFloat64(a + flatbuffers.UOffsetT(j*8))
+	}
+	return 0
+}
+
+func (rcv *Monster) VectorOfDoublesLength() int {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(70))
+	if o != 0 {
+		return rcv._tab.VectorLen(o)
+	}
+	return 0
+}
+
 func MonsterStart(builder *flatbuffers.Builder) {
-	builder.StartObject(32)
+	builder.StartObject(34)
 }
 func MonsterAddPos(builder *flatbuffers.Builder, pos flatbuffers.UOffsetT) {
 	builder.PrependStructSlot(0, flatbuffers.UOffsetT(pos), 0)
@@ -608,6 +642,18 @@ func MonsterAddTest5(builder *flatbuffers.Builder, test5 flatbuffers.UOffsetT) {
 func MonsterStartTest5Vector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
 	return builder.StartVector(4, numElems, 2)
 }
+func MonsterAddVectorOfLongs(builder *flatbuffers.Builder, vectorOfLongs flatbuffers.UOffsetT) {
+	builder.PrependUOffsetTSlot(32, flatbuffers.UOffsetT(vectorOfLongs), 0)
+}
+func MonsterStartVectorOfLongsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+	return builder.StartVector(8, numElems, 8)
+}
+func MonsterAddVectorOfDoubles(builder *flatbuffers.Builder, vectorOfDoubles flatbuffers.UOffsetT) {
+	builder.PrependUOffsetTSlot(33, flatbuffers.UOffsetT(vectorOfDoubles), 0)
+}
+func MonsterStartVectorOfDoublesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+	return builder.StartVector(8, numElems, 8)
+}
 func MonsterEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
 	return builder.EndObject()
 }

tests/MyGame/Example/Monster.java

@@ -98,8 +98,16 @@ public final class Monster extends Table {
   public Test test5(int j) { return test5(new Test(), j); }
   public Test test5(Test obj, int j) { int o = __offset(66); return o != 0 ? obj.__assign(__vector(o) + j * 4, bb) : null; }
   public int test5Length() { int o = __offset(66); return o != 0 ? __vector_len(o) : 0; }
+  public long vectorOfLongs(int j) { int o = __offset(68); return o != 0 ? bb.getLong(__vector(o) + j * 8) : 0; }
+  public int vectorOfLongsLength() { int o = __offset(68); return o != 0 ? __vector_len(o) : 0; }
+  public ByteBuffer vectorOfLongsAsByteBuffer() { return __vector_as_bytebuffer(68, 8); }
+  public boolean mutateVectorOfLongs(int j, long vector_of_longs) { int o = __offset(68); if (o != 0) { bb.putLong(__vector(o) + j * 8, vector_of_longs); return true; } else { return false; } }
+  public double vectorOfDoubles(int j) { int o = __offset(70); return o != 0 ? bb.getDouble(__vector(o) + j * 8) : 0; }
+  public int vectorOfDoublesLength() { int o = __offset(70); return o != 0 ? __vector_len(o) : 0; }
+  public ByteBuffer vectorOfDoublesAsByteBuffer() { return __vector_as_bytebuffer(70, 8); }
+  public boolean mutateVectorOfDoubles(int j, double vector_of_doubles) { int o = __offset(70); if (o != 0) { bb.putDouble(__vector(o) + j * 8, vector_of_doubles); return true; } else { return false; } }
 
-  public static void startMonster(FlatBufferBuilder builder) { builder.startObject(32); }
+  public static void startMonster(FlatBufferBuilder builder) { builder.startObject(34); }
   public static void addPos(FlatBufferBuilder builder, int posOffset) { builder.addStruct(0, posOffset, 0); }
   public static void addMana(FlatBufferBuilder builder, short mana) { builder.addShort(1, mana, 150); }
   public static void addHp(FlatBufferBuilder builder, short hp) { builder.addShort(2, hp, 100); }
@@ -148,6 +156,12 @@ public final class Monster extends Table {
   public static void startFlexVector(FlatBufferBuilder builder, int numElems) { builder.startVector(1, numElems, 1); }
   public static void addTest5(FlatBufferBuilder builder, int test5Offset) { builder.addOffset(31, test5Offset, 0); }
   public static void startTest5Vector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 2); }
+  public static void addVectorOfLongs(FlatBufferBuilder builder, int vectorOfLongsOffset) { builder.addOffset(32, vectorOfLongsOffset, 0); }
+  public static int createVectorOfLongsVector(FlatBufferBuilder builder, long[] data) { builder.startVector(8, data.length, 8); for (int i = data.length - 1; i >= 0; i--) builder.addLong(data[i]); return builder.endVector(); }
+  public static void startVectorOfLongsVector(FlatBufferBuilder builder, int numElems) { builder.startVector(8, numElems, 8); }
+  public static void addVectorOfDoubles(FlatBufferBuilder builder, int vectorOfDoublesOffset) { builder.addOffset(33, vectorOfDoublesOffset, 0); }
+  public static int createVectorOfDoublesVector(FlatBufferBuilder builder, double[] data) { builder.startVector(8, data.length, 8); for (int i = data.length - 1; i >= 0; i--) builder.addDouble(data[i]); return builder.endVector(); }
+  public static void startVectorOfDoublesVector(FlatBufferBuilder builder, int numElems) { builder.startVector(8, numElems, 8); }
   public static int endMonster(FlatBufferBuilder builder) {
     int o = builder.endObject();
     builder.required(o, 10);  // name

tests/MyGame/Example/Monster.php

@@ -444,22 +444,60 @@ class Monster extends Table
         return $o != 0 ? $this->__vector_len($o) : 0;
     }
 
+    /**
+     * @param int offset
+     * @return long
+     */
+    public function getVectorOfLongs($j)
+    {
+        $o = $this->__offset(68);
+        return $o != 0 ? $this->bb->getLong($this->__vector($o) + $j * 8) : 0;
+    }
+
+    /**
+     * @return int
+     */
+    public function getVectorOfLongsLength()
+    {
+        $o = $this->__offset(68);
+        return $o != 0 ? $this->__vector_len($o) : 0;
+    }
+
+    /**
+     * @param int offset
+     * @return double
+     */
+    public function getVectorOfDoubles($j)
+    {
+        $o = $this->__offset(70);
+        return $o != 0 ? $this->bb->getDouble($this->__vector($o) + $j * 8) : 0;
+    }
+
+    /**
+     * @return int
+     */
+    public function getVectorOfDoublesLength()
+    {
+        $o = $this->__offset(70);
+        return $o != 0 ? $this->__vector_len($o) : 0;
+    }
+
     /**
      * @param FlatBufferBuilder $builder
      * @return void
      */
     public static function startMonster(FlatBufferBuilder $builder)
     {
-        $builder->StartObject(32);
+        $builder->StartObject(34);
     }
 
     /**
      * @param FlatBufferBuilder $builder
      * @return Monster
      */
-    public static function createMonster(FlatBufferBuilder $builder, $pos, $mana, $hp, $name, $inventory, $color, $test_type, $test, $test4, $testarrayofstring, $testarrayoftables, $enemy, $testnestedflatbuffer, $testempty, $testbool, $testhashs32_fnv1, $testhashu32_fnv1, $testhashs64_fnv1, $testhashu64_fnv1, $testhashs32_fnv1a, $testhashu32_fnv1a, $testhashs64_fnv1a, $testhashu64_fnv1a, $testarrayofbools, $testf, $testf2, $testf3, $testarrayofstring2, $testarrayofsortedstruct, $flex, $test5)
+    public static function createMonster(FlatBufferBuilder $builder, $pos, $mana, $hp, $name, $inventory, $color, $test_type, $test, $test4, $testarrayofstring, $testarrayoftables, $enemy, $testnestedflatbuffer, $testempty, $testbool, $testhashs32_fnv1, $testhashu32_fnv1, $testhashs64_fnv1, $testhashu64_fnv1, $testhashs32_fnv1a, $testhashu32_fnv1a, $testhashs64_fnv1a, $testhashu64_fnv1a, $testarrayofbools, $testf, $testf2, $testf3, $testarrayofstring2, $testarrayofsortedstruct, $flex, $test5, $vector_of_longs, $vector_of_doubles)
     {
-        $builder->startObject(32);
+        $builder->startObject(34);
         self::addPos($builder, $pos);
         self::addMana($builder, $mana);
         self::addHp($builder, $hp);
@@ -491,6 +529,8 @@ class Monster extends Table
         self::addTestarrayofsortedstruct($builder, $testarrayofsortedstruct);
         self::addFlex($builder, $flex);
         self::addTest5($builder, $test5);
+        self::addVectorOfLongs($builder, $vector_of_longs);
+        self::addVectorOfDoubles($builder, $vector_of_doubles);
         $o = $builder->endObject();
         $builder->required($o, 10);  // name
         return $o;
@@ -1041,6 +1081,74 @@ class Monster extends Table
         $builder->startVector(4, $numElems, 2);
     }
 
+    /**
+     * @param FlatBufferBuilder $builder
+     * @param VectorOffset
+     * @return void
+     */
+    public static function addVectorOfLongs(FlatBufferBuilder $builder, $vectorOfLongs)
+    {
+        $builder->addOffsetX(32, $vectorOfLongs, 0);
+    }
+
+    /**
+     * @param FlatBufferBuilder $builder
+     * @param array offset array
+     * @return int vector offset
+     */
+    public static function createVectorOfLongsVector(FlatBufferBuilder $builder, array $data)
+    {
+        $builder->startVector(8, count($data), 8);
+        for ($i = count($data) - 1; $i >= 0; $i--) {
+            $builder->addLong($data[$i]);
+        }
+        return $builder->endVector();
+    }
+
+    /**
+     * @param FlatBufferBuilder $builder
+     * @param int $numElems
+     * @return void
+     */
+    public static function startVectorOfLongsVector(FlatBufferBuilder $builder, $numElems)
+    {
+        $builder->startVector(8, $numElems, 8);
+    }
+
+    /**
+     * @param FlatBufferBuilder $builder
+     * @param VectorOffset
+     * @return void
+     */
+    public static function addVectorOfDoubles(FlatBufferBuilder $builder, $vectorOfDoubles)
+    {
+        $builder->addOffsetX(33, $vectorOfDoubles, 0);
+    }
+
+    /**
+     * @param FlatBufferBuilder $builder
+     * @param array offset array
+     * @return int vector offset
+     */
+    public static function createVectorOfDoublesVector(FlatBufferBuilder $builder, array $data)
+    {
+        $builder->startVector(8, count($data), 8);
+        for ($i = count($data) - 1; $i >= 0; $i--) {
+            $builder->addDouble($data[$i]);
+        }
+        return $builder->endVector();
+    }
+
+    /**
+     * @param FlatBufferBuilder $builder
+     * @param int $numElems
+     * @return void
+     */
+    public static function startVectorOfDoublesVector(FlatBufferBuilder $builder, $numElems)
+    {
+        $builder->startVector(8, $numElems, 8);
+    }
+
     /**
      * @param FlatBufferBuilder $builder
      * @return int table offset

tests/MyGame/Example/Monster.py

@@ -59,6 +59,13 @@ class Monster(object):
             return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
         return 0
 
+    # Monster
+    def InventoryAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o)
+        return 0
+
     # Monster
     def InventoryLength(self):
         o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
@@ -165,6 +172,13 @@ class Monster(object):
             return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
         return 0
 
+    # Monster
+    def TestnestedflatbufferAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(30))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o)
+        return 0
+
     # Monster
     def TestnestedflatbufferLength(self):
         o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(30))
@@ -254,6 +268,13 @@ class Monster(object):
             return self._tab.Get(flatbuffers.number_types.BoolFlags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
         return 0
 
+    # Monster
+    def TestarrayofboolsAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(52))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.BoolFlags, o)
+        return 0
+
     # Monster
     def TestarrayofboolsLength(self):
         o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(52))
@@ -324,6 +345,13 @@ class Monster(object):
             return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
         return 0
 
+    # Monster
+    def FlexAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(64))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o)
+        return 0
+
     # Monster
     def FlexLength(self):
         o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(64))
@@ -350,7 +378,51 @@ class Monster(object):
             return self._tab.VectorLen(o)
         return 0
 
-def MonsterStart(builder): builder.StartObject(32)
+    # Monster
+    def VectorOfLongs(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(68))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Int64Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8))
+        return 0
+
+    # Monster
+    def VectorOfLongsAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(68))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int64Flags, o)
+        return 0
+
+    # Monster
+    def VectorOfLongsLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(68))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+    # Monster
+    def VectorOfDoubles(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(70))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Float64Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8))
+        return 0
+
+    # Monster
+    def VectorOfDoublesAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(70))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o)
+        return 0
+
+    # Monster
+    def VectorOfDoublesLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(70))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def MonsterStart(builder): builder.StartObject(34)
 def MonsterAddPos(builder, pos): builder.PrependStructSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(pos), 0)
 def MonsterAddMana(builder, mana): builder.PrependInt16Slot(1, mana, 150)
 def MonsterAddHp(builder, hp): builder.PrependInt16Slot(2, hp, 100)
@@ -392,4 +464,8 @@ def MonsterAddFlex(builder, flex): builder.PrependUOffsetTRelativeSlot(30, flatb
 def MonsterStartFlexVector(builder, numElems): return builder.StartVector(1, numElems, 1)
 def MonsterAddTest5(builder, test5): builder.PrependUOffsetTRelativeSlot(31, flatbuffers.number_types.UOffsetTFlags.py_type(test5), 0)
 def MonsterStartTest5Vector(builder, numElems): return builder.StartVector(4, numElems, 2)
+def MonsterAddVectorOfLongs(builder, vectorOfLongs): builder.PrependUOffsetTRelativeSlot(32, flatbuffers.number_types.UOffsetTFlags.py_type(vectorOfLongs), 0)
+def MonsterStartVectorOfLongsVector(builder, numElems): return builder.StartVector(8, numElems, 8)
+def MonsterAddVectorOfDoubles(builder, vectorOfDoubles): builder.PrependUOffsetTRelativeSlot(33, flatbuffers.number_types.UOffsetTFlags.py_type(vectorOfDoubles), 0)
+def MonsterStartVectorOfDoublesVector(builder, numElems): return builder.StartVector(8, numElems, 8)
 def MonsterEnd(builder): return builder.EndObject()

tests/monster_test.fbs

@@ -75,6 +75,8 @@ table Monster {
   testf2:float = 3 (id:26);
   testf3:float (id:27);
   flex:[ubyte] (id:30, flexbuffer);
+  vector_of_longs:[long] (id:32);
+  vector_of_doubles:[double] (id:33);
 }
 
 rpc_service MonsterStorage {

tests/monster_test.schema.json

@@ -109,7 +109,9 @@
         "testarrayofstring2" : { "type" : "array", "items" : { "type" : "string" } },
         "testarrayofsortedstruct" : { "type" : "array", "items" : { "$ref" : "#/definitions/MyGame_Example_Ability" } },
         "flex" : { "type" : "array", "items" : { "type" : "number" } },
-        "test5" : { "type" : "array", "items" : { "$ref" : "#/definitions/MyGame_Example_Test" } }
+        "test5" : { "type" : "array", "items" : { "$ref" : "#/definitions/MyGame_Example_Test" } },
+        "vector_of_longs" : { "type" : "array", "items" : { "type" : "number" } },
+        "vector_of_doubles" : { "type" : "array", "items" : { "type" : "number" } }
       },
       "required" : [ "name"]
     }

tests/monster_test_generated.js

@@ -1314,11 +1314,53 @@ MyGame.Example.Monster.prototype.test5Length = function() {
   return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
 };
 
+/**
+ * @param {number} index
+ * @returns {flatbuffers.Long}
+ */
+MyGame.Example.Monster.prototype.vectorOfLongs = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 68);
+  return offset ? this.bb.readInt64(this.bb.__vector(this.bb_pos + offset) + index * 8) : this.bb.createLong(0, 0);
+};
+
+/**
+ * @returns {number}
+ */
+MyGame.Example.Monster.prototype.vectorOfLongsLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 68);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+MyGame.Example.Monster.prototype.vectorOfDoubles = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 70);
+  return offset ? this.bb.readFloat64(this.bb.__vector(this.bb_pos + offset) + index * 8) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+MyGame.Example.Monster.prototype.vectorOfDoublesLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 70);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Float64Array}
+ */
+MyGame.Example.Monster.prototype.vectorOfDoublesArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 70);
+  return offset ? new Float64Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
 /**
  * @param {flatbuffers.Builder} builder
  */
 MyGame.Example.Monster.startMonster = function(builder) {
-  builder.startObject(32);
+  builder.startObject(34);
 };
 
 /**
@@ -1740,6 +1782,64 @@ MyGame.Example.Monster.startTest5Vector = function(builder, numElems) {
   builder.startVector(4, numElems, 2);
 };
 
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} vectorOfLongsOffset
+ */
+MyGame.Example.Monster.addVectorOfLongs = function(builder, vectorOfLongsOffset) {
+  builder.addFieldOffset(32, vectorOfLongsOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<flatbuffers.Long>} data
+ * @returns {flatbuffers.Offset}
+ */
+MyGame.Example.Monster.createVectorOfLongsVector = function(builder, data) {
+  builder.startVector(8, data.length, 8);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addInt64(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+MyGame.Example.Monster.startVectorOfLongsVector = function(builder, numElems) {
+  builder.startVector(8, numElems, 8);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} vectorOfDoublesOffset
+ */
+MyGame.Example.Monster.addVectorOfDoubles = function(builder, vectorOfDoublesOffset) {
+  builder.addFieldOffset(33, vectorOfDoublesOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+MyGame.Example.Monster.createVectorOfDoublesVector = function(builder, data) {
+  builder.startVector(8, data.length, 8);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addFloat64(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+MyGame.Example.Monster.startVectorOfDoublesVector = function(builder, numElems) {
+  builder.startVector(8, numElems, 8);
+};
+
 /**
  * @param {flatbuffers.Builder} builder
  * @returns {flatbuffers.Offset}

tests/monsterdata_test.json

@@ -19,6 +19,18 @@
     3,
     4
   ],
+  vector_of_longs: [
+      1,
+      100,
+      10000,
+      1000000,
+      100000000
+  ],
+  vector_of_doubles: [
+      -1.7976931348623157e+308,
+      0,
+      1.7976931348623157e+308
+  ],
   test_type: Monster,
   test: {
     name: "Fred",

tests/py_test.py

@@ -15,6 +15,7 @@
 
 import os.path
 import sys
+import imp
 PY_VERSION = sys.version_info[:2]
 
 import ctypes
@@ -25,6 +26,7 @@ import unittest
 
 from flatbuffers import compat
 from flatbuffers.compat import range_func as compat_range
+from flatbuffers.compat import NumpyRequiredForThisFeature
 
 import flatbuffers
 from flatbuffers import number_types as N
@@ -130,6 +132,40 @@ def CheckReadBuffer(buf, offset):
         invsum += int(v)
     asserter(invsum == 10)
 
+    for i in range(5):
+        asserter(monster.VectorOfLongs(i) == 10 ** (i * 2))
+
+    asserter(([-1.7976931348623157e+308, 0, 1.7976931348623157e+308]
+              == [monster.VectorOfDoubles(i)
+                  for i in range(monster.VectorOfDoublesLength())]))
+
+    try:
+        imp.find_module('numpy')
+        # if numpy exists, then we should be able to get the
+        # vector as a numpy array
+        import numpy as np
+
+        asserter(monster.InventoryAsNumpy().sum() == 10)
+        asserter(monster.InventoryAsNumpy().dtype == np.dtype('uint8'))
+
+        VectorOfLongs = monster.VectorOfLongsAsNumpy()
+        asserter(VectorOfLongs.dtype == np.dtype('int64'))
+        for i in range(5):
+            asserter(VectorOfLongs[i] == 10 ** (i * 2))
+
+        VectorOfDoubles = monster.VectorOfDoublesAsNumpy()
+        asserter(VectorOfDoubles.dtype == np.dtype('float64'))
+        asserter(VectorOfDoubles[0] == np.finfo('float64').min)
+        asserter(VectorOfDoubles[1] == 0.0)
+        asserter(VectorOfDoubles[2] == np.finfo('float64').max)
+
+    except ImportError:
+        # If numpy does not exist, trying to get vector as numpy
+        # array should raise NumpyRequiredForThisFeature. The way
+        # assertRaises has been implemented prevents us from
+        # asserting this error is raised outside of a test case.
+        pass
+
     asserter(monster.Test4Length() == 2)
 
     # create a 'Test' object and populate it:
@@ -794,6 +830,20 @@ def make_monster_from_generated_code():
     b.PrependUOffsetTRelative(test1)
     testArrayOfString = b.EndVector(2)
 
+    MyGame.Example.Monster.MonsterStartVectorOfLongsVector(b, 5)
+    b.PrependInt64(100000000)
+    b.PrependInt64(1000000)
+    b.PrependInt64(10000)
+    b.PrependInt64(100)
+    b.PrependInt64(1)
+    VectorOfLongs = b.EndVector(5)
+
+    MyGame.Example.Monster.MonsterStartVectorOfDoublesVector(b, 3)
+    b.PrependFloat64(1.7976931348623157e+308)
+    b.PrependFloat64(0)
+    b.PrependFloat64(-1.7976931348623157e+308)
+    VectorOfDoubles = b.EndVector(3)
+
     MyGame.Example.Monster.MonsterStart(b)
 
     pos = MyGame.Example.Vec3.CreateVec3(b, 1.0, 2.0, 3.0, 3.0, 2, 5, 6)
@@ -806,6 +856,8 @@ def make_monster_from_generated_code():
     MyGame.Example.Monster.MonsterAddTest(b, mon2)
     MyGame.Example.Monster.MonsterAddTest4(b, test4)
     MyGame.Example.Monster.MonsterAddTestarrayofstring(b, testArrayOfString)
+    MyGame.Example.Monster.MonsterAddVectorOfLongs(b, VectorOfLongs)
+    MyGame.Example.Monster.MonsterAddVectorOfDoubles(b, VectorOfDoubles)
     mon = MyGame.Example.Monster.MonsterEnd(b)
 
     b.Finish(mon)
@@ -962,6 +1014,15 @@ class TestAllCodePathsOfExampleSchema(unittest.TestCase):
         self.assertEqual(0, mon2.Testnestedflatbuffer(0))
         self.assertEqual(2, mon2.Testnestedflatbuffer(1))
         self.assertEqual(4, mon2.Testnestedflatbuffer(2))
+        try:
+            imp.find_module('numpy')
+            # if numpy exists, then we should be able to get the
+            # vector as a numpy array
+            self.assertEqual([0, 2, 4], mon2.TestnestedflatbufferAsNumpy().tolist())
+        except ImportError:
+            assertRaises(self,
+                         lambda: mon2.TestnestedflatbufferAsNumpy(),
+                         NumpyRequiredForThisFeature)
 
     def test_nondefault_monster_testempty(self):
         b = flatbuffers.Builder(0)