Utility for checking the encoding and line ending of source files (#5188)

* Add utility for checking the encoding of source files

- accept source files with ASCII or UTF-8 without BOM
- accept only CRLF line ending

* Fix non-ascii symbol in idl_parcer.cpp

* Remove BOM from test.cpp

2 files changed, 69 insertions, 0 deletions

diff --git a/.travis/check-sources.sh b/.travis/check-sources.sh
new file mode 100644
index 00000000..3e6dbf16
--- /dev/null
+++ b/.travis/check-sources.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# Copyright 2018 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -e
+
+if [ -n "$1" ]; then
+  scan_dir="$1"
+else
+  scan_dir="$( pwd )"
+fi
+
+py_checker="$0.py"
+
+echo "scan root directory = '$scan_dir'"
+python3 --version
+# Scan recursively and search all *.cpp and *.h files using regex patterns.
+# Assume that script running from a root of Flatbuffers working dir.
+python3 $py_checker "ascii" "$scan_dir/include" "\.h$"
+python3 $py_checker "ascii" "$scan_dir/src"     "\.cpp$"
+python3 $py_checker "ascii" "$scan_dir/tests"   "\.h$"
+python3 $py_checker "utf-8" "$scan_dir/tests"   "\.cpp$"
diff --git a/.travis/check-sources.sh.py b/.travis/check-sources.sh.py
new file mode 100644
index 00000000..5ad060cd
--- /dev/null
+++ b/.travis/check-sources.sh.py
@@ -0,0 +1,36 @@
+import os
+import re
+import sys
+
+def check_encoding(encoding, scan_dir, regex_pattern):
+  fname = None
+  try:
+    assert encoding in ['ascii', 'utf-8'], "unexpected encoding"
+    cmp = re.compile(regex_pattern)
+    for root, dirs, files in os.walk(scan_dir):
+      fname = root
+      cmp_list = [f for f in files if cmp.search(f) is not None]
+      for f in cmp_list:
+        fname = os.path.join(root, f)
+        with open(fname, mode='rb') as test_file:
+          btext = test_file.read()
+        # check encoding
+        btext.decode(encoding=encoding, errors="strict")
+        if encoding == "utf-8" and btext.startswith(b'\xEF\xBB\xBF'):
+          raise ValueError("unexpected BOM in file")
+        # check strict CRLF line-ending
+        LF = btext.count(b'\r')
+        CRLF = btext.count(b'\r\n')
+        assert LF >= CRLF, "CRLF logic error"
+        if CRLF != LF:
+          raise ValueError("CRLF violation: found {} LF characters".format(LF - CRLF))
+  except Exception as err:
+    print("ERROR with [{}]: {}".format(fname, err))
+    return -1
+  else:
+    return 0
+
+if __name__ == "__main__":
+  # python check-sources.sh.py 'ascii' '.' '.*\.(cpp|h)$'
+  res = check_encoding(sys.argv[1], sys.argv[2], sys.argv[3])
+  sys.exit(0 if res == 0 else -1)