C# support for directly reading and writting to memory other than byte[]. For…

C# support for directly reading and writting to memory other than byte[].  For example, ByteBuffer can be initialized with a custom allocator which uses shared memory / memory mapped files. (#4886)

Public access to the backing buffer uses Span<T> instead of ArraySegment<T>.

Writing to the buffer now supports Span<T> in addition to T[].

To maintain backwards compatibility ENABLE_SPAN_T must be defined.

net/FlatBuffers/FlatBufferBuilder.cs

@@ -62,6 +62,17 @@ namespace FlatBuffers
             _bb = new ByteBuffer(initialSize);
         }
 
+        /// <summary>
+        /// Create a FlatBufferBuilder backed by the pased in ByteBuffer
+        /// </summary>
+        /// <param name="buffer">The ByteBuffer to write to</param>
+        public FlatBufferBuilder(ByteBuffer buffer)
+        {
+            _bb = buffer;
+            _space = buffer.Length;
+            buffer.Reset();
+        }
+
         /// <summary>
         /// Reset the FlatBufferBuilder by purging all data that it holds.
         /// </summary>
@@ -191,6 +202,20 @@ namespace FlatBuffers
             _space = _bb.Put(_space, x);
         }
 
+#if ENABLE_SPAN_T
+        /// <summary>
+        /// Puts a span of type T into this builder at the 
+        /// current offset
+        /// </summary>
+        /// <typeparam name="T">The type of the input data </typeparam>
+        /// <param name="x">The span to copy data from</param>
+        public void Put<T>(Span<T> x)
+            where T : struct
+        {
+            _space = _bb.Put(_space, x);
+        }
+#endif
+
         public void PutDouble(double x)
         {
             _bb.PutDouble(_space -= sizeof(double), x);
@@ -288,6 +313,28 @@ namespace FlatBuffers
             Put(x);
         }
 
+#if ENABLE_SPAN_T
+        /// <summary>
+        /// Add a span of type T to the buffer (aligns the data and grows if necessary).
+        /// </summary>
+        /// <typeparam name="T">The type of the input data</typeparam>
+        /// <param name="x">The span to copy data from</param>
+        public void Add<T>(Span<T> x)
+            where T : struct
+        {
+            if (!ByteBuffer.IsSupportedType<T>())
+            {
+                throw new ArgumentException("Cannot add this Type array to the builder");
+            }
+
+            int size = ByteBuffer.SizeOf<T>();
+            // Need to prep on size (for data alignment) and then we pass the
+            // rest of the length (minus 1) as additional bytes
+            Prep(size, size * (x.Length - 1));
+            Put(x);
+        }
+#endif
+
         /// <summary>
         /// Add a `double` to the buffer (aligns the data and grows if necessary).
         /// </summary>
@@ -511,6 +558,27 @@ namespace FlatBuffers
             return new StringOffset(EndVector().Value);
         }
 
+
+#if ENABLE_SPAN_T
+        /// <summary>
+        /// Creates a string in the buffer from a Span containing
+        /// a UTF8 string.
+        /// </summary>
+        /// <param name="chars">the UTF8 string to add to the buffer</param>
+        /// <returns>
+        /// The offset in the buffer where the encoded string starts.
+        /// </returns>
+        public StringOffset CreateUTF8String(Span<byte> chars)
+        {
+            NotNested();
+            AddByte(0);
+            var utf8StringLen = chars.Length;
+            StartVector(1, utf8StringLen, 1);
+            _space = _bb.Put(_space, chars);
+            return new StringOffset(EndVector().Value);
+        }
+#endif
+
         /// @cond FLATBUFFERS_INTERNAL
         // Structs are stored inline, so nothing additional is being added.
         // `d` is always 0.

net/FlatBuffers/FlatBuffers.csproj

@@ -10,4 +10,7 @@
     <PackageLicenseUrl>https://github.com/google/flatbuffers/blob/master/LICENSE.txt</PackageLicenseUrl>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="System.Memory" Version="4.5.1" />
+  </ItemGroup>
 </Project>
\ No newline at end of file

net/FlatBuffers/Table.cs

@@ -78,6 +78,23 @@ namespace FlatBuffers
             return offset + bb.GetInt(offset) + sizeof(int);  // data starts after the length
         }
 
+#if ENABLE_SPAN_T
+        // Get the data of a vector whoses offset is stored at "offset" in this object as an
+        // Spant<byte>. If the vector is not present in the ByteBuffer,
+        // then an empty span will be returned.
+        public Span<byte> __vector_as_span(int offset)
+        {
+            var o = this.__offset(offset);
+            if (0 == o)
+            {
+                return new Span<byte>();
+            }
+
+            var pos = this.__vector(o);
+            var len = this.__vector_len(o);
+            return bb.ToSpan(pos, len);
+        }
+#else
         // Get the data of a vector whoses offset is stored at "offset" in this object as an
         // ArraySegment&lt;byte&gt;. If the vector is not present in the ByteBuffer,
         // then a null value will be returned.
@@ -93,6 +110,7 @@ namespace FlatBuffers
             var len = this.__vector_len(o);
             return bb.ToArraySegment(pos, len);
         }
+#endif
 
         // Get the data of a vector whoses offset is stored at "offset" in this object as an
         // T[]. If the vector is not present in the ByteBuffer, then a null value will be

src/idl_gen_general.cpp

@@ -1113,12 +1113,21 @@ class GeneralGenerator : public BaseGenerator {
             code += "); }\n";
             break;
           case IDLOptions::kCSharp:
+            code += "#if ENABLE_SPAN_T\n";
+            code += "  public Span<byte> Get";
+            code += MakeCamel(field.name, lang_.first_camel_upper);
+            code += "Bytes() { return ";
+            code += lang_.accessor_prefix + "__vector_as_span(";
+            code += NumToString(field.value.offset);
+            code += "); }\n";
+            code += "#else\n";
             code += "  public ArraySegment<byte>? Get";
             code += MakeCamel(field.name, lang_.first_camel_upper);
             code += "Bytes() { return ";
             code += lang_.accessor_prefix + "__vector_as_arraysegment(";
             code += NumToString(field.value.offset);
             code += "); }\n";
+            code += "#endif\n";
 
             // For direct blockcopying the data into a typed array
             code += "  public ";

tests/FlatBuffers.Test/FlatBuffersExampleTests.cs

@@ -110,14 +110,19 @@ namespace FlatBuffers.Test
                 Monster.FinishMonsterBuffer(fbb, mon);
             }
 
-
             // Dump to output directory so we can inspect later, if needed
+#if ENABLE_SPAN_T
+            var data = fbb.DataBuffer.ToSizedArray();
+            string filename = @"Resources/monsterdata_cstest" + (sizePrefix ? "_sp" : "") + ".mon";
+            File.WriteAllBytes(filename, data);
+#else
             using (var ms = fbb.DataBuffer.ToMemoryStream(fbb.DataBuffer.Position, fbb.Offset))
             {
                 var data = ms.ToArray();
                 string filename = @"Resources/monsterdata_cstest" + (sizePrefix ? "_sp" : "") + ".mon";
                 File.WriteAllBytes(filename, data);
             }
+#endif
 
             // Remove the size prefix if necessary for further testing
             ByteBuffer dataBuffer = fbb.DataBuffer;
@@ -243,6 +248,19 @@ namespace FlatBuffers.Test
 
             Assert.AreEqual(false, monster.Testbool);
 
+#if ENABLE_SPAN_T
+            var nameBytes = monster.GetNameBytes();
+            Assert.AreEqual("MyMonster", Encoding.UTF8.GetString(nameBytes.ToArray(), 0, nameBytes.Length));
+
+            if (0 == monster.TestarrayofboolsLength)
+            {
+                Assert.IsFalse(monster.GetTestarrayofboolsBytes().Length != 0);
+            }
+            else
+            {
+                Assert.IsTrue(monster.GetTestarrayofboolsBytes().Length == 0);
+            }
+#else
             var nameBytes = monster.GetNameBytes().Value;
             Assert.AreEqual("MyMonster", Encoding.UTF8.GetString(nameBytes.Array, nameBytes.Offset, nameBytes.Count));
 
@@ -254,6 +272,7 @@ namespace FlatBuffers.Test
             {
                 Assert.IsTrue(monster.GetTestarrayofboolsBytes().HasValue);
             }
+#endif
         }
 
         [FlatBuffersTestMethod]

tests/MyGame/Example/Stat.cs

@@ -18,7 +18,11 @@ public struct Stat : IFlatbufferObject
   public Stat __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; }
 
   public string Id { get { int o = __p.__offset(4); return o != 0 ? __p.__string(o + __p.bb_pos) : null; } }
+#if ENABLE_SPAN_T
+  public Span<byte> GetIdBytes() { return __p.__vector_as_span(4); }
+#else
   public ArraySegment<byte>? GetIdBytes() { return __p.__vector_as_arraysegment(4); }
+#endif
   public byte[] GetIdArray() { return __p.__vector_as_array<byte>(4); }
   public long Val { get { int o = __p.__offset(6); return o != 0 ? __p.bb.GetLong(o + __p.bb_pos) : (long)0; } }
   public bool MutateVal(long val) { int o = __p.__offset(6); if (o != 0) { __p.bb.PutLong(o + __p.bb_pos, val); return true; } else { return false; } }

tests/MyGame/Example/TypeAliases.cs

@@ -39,12 +39,20 @@ public struct TypeAliases : IFlatbufferObject
   public bool MutateF64(double f64) { int o = __p.__offset(22); if (o != 0) { __p.bb.PutDouble(o + __p.bb_pos, f64); return true; } else { return false; } }
   public sbyte V8(int j) { int o = __p.__offset(24); return o != 0 ? __p.bb.GetSbyte(__p.__vector(o) + j * 1) : (sbyte)0; }
   public int V8Length { get { int o = __p.__offset(24); return o != 0 ? __p.__vector_len(o) : 0; } }
+#if ENABLE_SPAN_T
+  public Span<byte> GetV8Bytes() { return __p.__vector_as_span(24); }
+#else
   public ArraySegment<byte>? GetV8Bytes() { return __p.__vector_as_arraysegment(24); }
+#endif
   public sbyte[] GetV8Array() { return __p.__vector_as_array<sbyte>(24); }
   public bool MutateV8(int j, sbyte v8) { int o = __p.__offset(24); if (o != 0) { __p.bb.PutSbyte(__p.__vector(o) + j * 1, v8); return true; } else { return false; } }
   public double Vf64(int j) { int o = __p.__offset(26); return o != 0 ? __p.bb.GetDouble(__p.__vector(o) + j * 8) : (double)0; }
   public int Vf64Length { get { int o = __p.__offset(26); return o != 0 ? __p.__vector_len(o) : 0; } }
+#if ENABLE_SPAN_T
+  public Span<byte> GetVf64Bytes() { return __p.__vector_as_span(26); }
+#else
   public ArraySegment<byte>? GetVf64Bytes() { return __p.__vector_as_arraysegment(26); }
+#endif
   public double[] GetVf64Array() { return __p.__vector_as_array<double>(26); }
   public bool MutateVf64(int j, double vf64) { int o = __p.__offset(26); if (o != 0) { __p.bb.PutDouble(__p.__vector(o) + j * 8, vf64); return true; } else { return false; } }
 

tests/union_vector/Movie.cs

@@ -19,7 +19,12 @@ public struct Movie : IFlatbufferObject
   public TTable? MainCharacter<TTable>() where TTable : struct, IFlatbufferObject { int o = __p.__offset(6); return o != 0 ? (TTable?)__p.__union<TTable>(o) : null; }
   public Character CharactersType(int j) { int o = __p.__offset(8); return o != 0 ? (Character)__p.bb.Get(__p.__vector(o) + j * 1) : (Character)0; }
   public int CharactersTypeLength { get { int o = __p.__offset(8); return o != 0 ? __p.__vector_len(o) : 0; } }
+#if ENABLE_SPAN_T
+  public Span<byte> GetCharactersTypeBytes() { return __p.__vector_as_span(8); }
+#else
   public ArraySegment<byte>? GetCharactersTypeBytes() { return __p.__vector_as_arraysegment(8); }
+#endif
+  public Character[] GetCharactersTypeArray() { return __p.__vector_as_array<Character>(8); }
   public TTable? Characters<TTable>(int j) where TTable : struct, IFlatbufferObject { int o = __p.__offset(10); return o != 0 ? (TTable?)__p.__union<TTable>(__p.__vector(o) + j * 4) : null; }
   public int CharactersLength { get { int o = __p.__offset(10); return o != 0 ? __p.__vector_len(o) : 0; } }
 
@@ -41,9 +46,11 @@ public struct Movie : IFlatbufferObject
   public static void AddMainCharacter(FlatBufferBuilder builder, int mainCharacterOffset) { builder.AddOffset(1, mainCharacterOffset, 0); }
   public static void AddCharactersType(FlatBufferBuilder builder, VectorOffset charactersTypeOffset) { builder.AddOffset(2, charactersTypeOffset.Value, 0); }
   public static VectorOffset CreateCharactersTypeVector(FlatBufferBuilder builder, Character[] data) { builder.StartVector(1, data.Length, 1); for (int i = data.Length - 1; i >= 0; i--) builder.AddByte((byte)data[i]); return builder.EndVector(); }
+  public static VectorOffset CreateCharactersTypeVectorBlock(FlatBufferBuilder builder, Character[] data) { builder.StartVector(1, data.Length, 1); builder.Add(data); return builder.EndVector(); }
   public static void StartCharactersTypeVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(1, numElems, 1); }
   public static void AddCharacters(FlatBufferBuilder builder, VectorOffset charactersOffset) { builder.AddOffset(3, charactersOffset.Value, 0); }
   public static VectorOffset CreateCharactersVector(FlatBufferBuilder builder, int[] data) { builder.StartVector(4, data.Length, 4); for (int i = data.Length - 1; i >= 0; i--) builder.AddOffset(data[i]); return builder.EndVector(); }
+  public static VectorOffset CreateCharactersVectorBlock(FlatBufferBuilder builder, int[] data) { builder.StartVector(4, data.Length, 4); builder.Add(data); return builder.EndVector(); }
   public static void StartCharactersVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(4, numElems, 4); }
   public static Offset<Movie> EndMovie(FlatBufferBuilder builder) {
     int o = builder.EndObject();