Fix rust crate for big-endian targets (#5229)

Thanks for tackling this, @tymcauley !

* big endian docker test -- wip

* tweaks

* tweaks

* tweaks

* docker tweaks

* fix conditional compilation issues

* reactivate other docker tests

* try some more cross-platform config (from tymcauley)

* Update tests/docker/languages/Dockerfile.testing.rust.big_endian.1_30_1
Co-Authored-By: rw <rw@users.noreply.github.com>

* Update tests/docker/languages/Dockerfile.testing.rust.big_endian.1_30_1
Co-Authored-By: rw <rw@users.noreply.github.com>

* Update tests/docker/languages/Dockerfile.testing.rust.big_endian.1_30_1
Co-Authored-By: rw <rw@users.noreply.github.com>

* Resolved Rust warnings during big-endian builds.

* Unify Rust test suites for x86 and MIPS builds.

Note that I had to add four extra packages to the MIPS `Dockerfile`:
`libexpat1`, `libmagic1`, `libmpdec2`, and `libreadline7`. For a reason
I couldn't identify, even the simplest Rust MIPS binaries run with
`qemu-mips` would fail with a segfault when run through this
`Dockerfile`. After installing the `gdb-multiarch` package to attempt to
debug the issue, the binaries ran successfully. I pared down the
packages installed by `gdb-multiarch`, and these four packages are the
minimum subset necessary to get Rust MIPS binaries running under
`qemu-mips`.

* Changed Rust tests to use `Vector`s instead of direct-slice-access.

The direct-slice-access method is not available on big-endian targets,
but `flatbuffers::Vector`s provide an array interface that is available
on all platforms.

* Resolved FooStruct endianness issues using explicit struct constructor.

This more closely resembles how FlatBuffers structs are constructed in
generated Rust code.

* Added explanation of how `FooStruct` parallels generated struct code.

Also collected duplicate implementations of `FooStruct` into a common
location.

rust/flatbuffers/src/endian_scalar.rs

@@ -88,7 +88,7 @@ impl EndianScalar for f32 {
         }
         #[cfg(not(target_endian = "little"))]
         {
-            byte_swap_f32(&self)
+            byte_swap_f32(self)
         }
     }
     /// Convert f32 from little-endian to host endian-ness.
@@ -100,7 +100,7 @@ impl EndianScalar for f32 {
         }
         #[cfg(not(target_endian = "little"))]
         {
-            byte_swap_f32(&self)
+            byte_swap_f32(self)
         }
     }
 }
@@ -115,7 +115,7 @@ impl EndianScalar for f64 {
         }
         #[cfg(not(target_endian = "little"))]
         {
-            byte_swap_f64(&self)
+            byte_swap_f64(self)
         }
     }
     /// Convert f64 from little-endian to host endian-ness.
@@ -127,7 +127,7 @@ impl EndianScalar for f64 {
         }
         #[cfg(not(target_endian = "little"))]
         {
-            byte_swap_f64(&self)
+            byte_swap_f64(self)
         }
     }
 }

rust/flatbuffers/src/vector.rs

@@ -19,7 +19,9 @@ use std::mem::size_of;
 use std::slice::from_raw_parts;
 use std::str::from_utf8_unchecked;
 
-use endian_scalar::{EndianScalar, read_scalar};
+#[cfg(target_endian = "little")]
+use endian_scalar::EndianScalar;
+use endian_scalar::read_scalar;
 use follow::Follow;
 use primitives::*;
 
@@ -85,6 +87,7 @@ mod le_safe_slice_impls {
     impl super::SafeSliceAccess for f64 {}
 }
 
+#[cfg(target_endian = "little")]
 pub use self::le_safe_slice_impls::*;
 
 pub fn follow_cast_ref<'a, T: Sized + 'a>(buf: &'a [u8], loc: usize) -> &'a T {
@@ -104,6 +107,7 @@ impl<'a> Follow<'a> for &'a str {
     }
 }
 
+#[cfg(target_endian = "little")]
 fn follow_slice_helper<T>(buf: &[u8], loc: usize) -> &[T] {
     let sz = size_of::<T>();
     debug_assert!(sz > 0);

tests/RustTest.sh

@@ -15,8 +15,14 @@ set -e
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+if [[ "$1" == "mips-unknown-linux-gnu" ]]; then
+    TARGET_FLAG="--target mips-unknown-linux-gnu"
+    export CARGO_TARGET_MIPS_UNKNOWN_LINUX_GNU_LINKER=mips-linux-gnu-gcc
+    export CARGO_TARGET_MIPS_UNKNOWN_LINUX_GNU_RUNNER="qemu-mips -L /usr/mips-linux-gnu"
+fi
+
 cd ./rust_usage_test
-cargo test -- --quiet
+cargo test $TARGET_FLAG -- --quiet
 TEST_RESULT=$?
 if [[ $TEST_RESULT  == 0 ]]; then
     echo "OK: Rust tests passed."
@@ -25,7 +31,7 @@ else
     exit 1
 fi
 
-cargo run --bin=alloc_check
+cargo run $TARGET_FLAG --bin=alloc_check
 TEST_RESULT=$?
 if [[ $TEST_RESULT  == 0 ]]; then
     echo "OK: Rust heap alloc test passed."
@@ -34,4 +40,4 @@ else
     exit 1
 fi
 
-cargo bench
+cargo bench $TARGET_FLAG

tests/docker/languages/Dockerfile.testing.rust.big_endian.1_30_1

+FROM rust:1.30.1-slim-stretch as base
+RUN apt -qq update -y && apt -qq install -y \
+    gcc-mips-linux-gnu \
+    libexpat1 \
+    libmagic1 \
+    libmpdec2 \
+    libreadline7 \
+    qemu-user
+RUN rustup target add mips-unknown-linux-gnu
+WORKDIR /code
+ADD . .
+RUN cp flatc_debian_stretch flatc
+WORKDIR /code/tests
+RUN rustc --version
+RUN ./RustTest.sh mips-unknown-linux-gnu