Utility for checking the encoding and line ending of source files (#5188)

* Add utility for checking the encoding of source files

- accept source files with ASCII or UTF-8 without BOM
- accept only CRLF line ending

* Fix non-ascii symbol in idl_parcer.cpp

* Remove BOM from test.cpp

.travis.yml

@@ -89,6 +89,7 @@ matrix:
       - if [ "$TRAVIS_OS_NAME" == "linux" ]; then sudo ln -s -v -f $(which gcc-$GCC_VERSION) /usr/bin/gcc; fi
 
       script:
+      - bash .travis/check-sources.sh
       - bash grpc/build_grpc.sh
       - cmake . 
         -DCMAKE_BUILD_TYPE=$BUILD_TYPE 

.travis/check-sources.sh

+#!/bin/bash
+#
+# Copyright 2018 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -e
+
+if [ -n "$1" ]; then
+  scan_dir="$1"
+else
+  scan_dir="$( pwd )"
+fi
+
+py_checker="$0.py"
+
+echo "scan root directory = '$scan_dir'"
+python3 --version
+# Scan recursively and search all *.cpp and *.h files using regex patterns.
+# Assume that script running from a root of Flatbuffers working dir.
+python3 $py_checker "ascii" "$scan_dir/include" "\.h$"
+python3 $py_checker "ascii" "$scan_dir/src"     "\.cpp$"
+python3 $py_checker "ascii" "$scan_dir/tests"   "\.h$"
+python3 $py_checker "utf-8" "$scan_dir/tests"   "\.cpp$"

.travis/check-sources.sh.py

+import os
+import re
+import sys
+
+def check_encoding(encoding, scan_dir, regex_pattern):
+  fname = None
+  try:
+    assert encoding in ['ascii', 'utf-8'], "unexpected encoding"
+    cmp = re.compile(regex_pattern)
+    for root, dirs, files in os.walk(scan_dir):
+      fname = root
+      cmp_list = [f for f in files if cmp.search(f) is not None]
+      for f in cmp_list:
+        fname = os.path.join(root, f)
+        with open(fname, mode='rb') as test_file:
+          btext = test_file.read()
+        # check encoding
+        btext.decode(encoding=encoding, errors="strict")
+        if encoding == "utf-8" and btext.startswith(b'\xEF\xBB\xBF'):
+          raise ValueError("unexpected BOM in file")
+        # check strict CRLF line-ending
+        LF = btext.count(b'\r')
+        CRLF = btext.count(b'\r\n')
+        assert LF >= CRLF, "CRLF logic error"
+        if CRLF != LF:
+          raise ValueError("CRLF violation: found {} LF characters".format(LF - CRLF))
+  except Exception as err:
+    print("ERROR with [{}]: {}".format(fname, err))
+    return -1
+  else:
+    return 0
+
+if __name__ == "__main__":
+  # python check-sources.sh.py 'ascii' '.' '.*\.(cpp|h)$'
+  res = check_encoding(sys.argv[1], sys.argv[2], sys.argv[3])
+  sys.exit(0 if res == 0 else -1)

src/idl_parser.cpp

@@ -428,7 +428,7 @@ CheckedError Parser::Next() {
 
         auto dot_lvl = (c == '.') ? 0 : 1;  // dot_lvl==0 <=> exactly one '.' seen
         if (!dot_lvl && !is_digit(*cursor_)) return NoError(); // enum?
-        // Parser accepts hexadecimal-floating-literal (see C++ 5.13.4).
+        // Parser accepts hexadecimal-floating-literal (see C++ 5.13.4).
         if (is_digit(c) || has_sign || !dot_lvl) {
           const auto start = cursor_ - 1;
           auto start_digits = !is_digit(c) ? cursor_ : cursor_ - 1;

tests/test.cpp

-/*
+/*
  * Copyright 2014 Google Inc. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");