diff options
Diffstat (limited to 'infra/command')
-rw-r--r-- | infra/command/build-docker-image | 46 | ||||
-rw-r--r-- | infra/command/format | 198 | ||||
-rw-r--r-- | infra/command/gen-coverage-report | 78 | ||||
-rw-r--r-- | infra/command/install-githooks | 15 | ||||
-rw-r--r-- | infra/command/pylint | 32 |
5 files changed, 369 insertions, 0 deletions
diff --git a/infra/command/build-docker-image b/infra/command/build-docker-image new file mode 100644 index 000000000..5b1dbb5e1 --- /dev/null +++ b/infra/command/build-docker-image @@ -0,0 +1,46 @@ +#!/bin/bash + +function Usage() +{ + echo "Usage: $0 $(basename ${BASH_SOURCE[0]}) [OPTIONS]" + echo "" + echo "Options:" + echo " --tizen Build docker image for tizen build" + echo "Options can use as docker build option:" + docker build --help +} + +DOCKER_FILE_RPATH="infra/docker/Dockerfile" +DOCKER_BUILD_ARGS=() + +# Handle argument for this script +# Set default docker image name, tag +for i in "$@" +do + case ${i} in + -h|--help|help) + Usage + exit 1 + ;; + esac +done + +DOCKER_BUILD_ARGS+="-t ${DOCKER_IMAGE_NAME:-nnas}" + +# Argument for docker build commands +for i in "$@" +do + case ${i} in + -h|--help|help) + # Already handled argument + ;; + *) + DOCKER_BUILD_ARGS+=(${i}) + ;; + esac +done + +docker build --build-arg http_proxy="${http_proxy}" \ + --build-arg https_proxy="${https_proxy}" \ + ${DOCKER_BUILD_ARGS[@]} \ + - < ${NNAS_PROJECT_PATH}/${DOCKER_FILE_RPATH} diff --git a/infra/command/format b/infra/command/format new file mode 100644 index 000000000..1015c4473 --- /dev/null +++ b/infra/command/format @@ -0,0 +1,198 @@ +#!/bin/bash + +INVALID_EXIT=0 +FILES_TO_CHECK=() +DIRECTORIES_NOT_TO_BE_TESTED=() + +function pushd () { + command pushd "$@" > /dev/null +} + +function popd () { + command popd "$@" > /dev/null +} + +function command_exists() { + command -v $1 > /dev/null 2>&1 +} + +function check_newline() { + # Check all files (CMakeLists.txt, *.cl, ... not only for C++, Python) + if [[ ${#FILES_TO_CHECK} -ne 0 ]]; then + CRCHECK=$(file ${FILES_TO_CHECK} | grep 'with CR') + fi + FILES_TO_FIX=($(echo "$CRCHECK" | grep "with CRLF line" | cut -d':' -f1)) + for f in ${FILES_TO_FIX[@]}; do + tr -d '\r' < $f > $f.fixed && cat $f.fixed > $f && rm $f.fixed + done + FILES_TO_FIX=($(echo "${CRCHECK}" | grep "with CR line" | cut -d':' -f1)) + for f in ${FILES_TO_FIX[@]}; do + tr '\r' '\n' < $f > $f.fixed && cat $f.fixed > $f && rm $f.fixed + done +} + +function check_permission() { + # Check all files except script + FILES_TO_CHECK_PERMISSION=() + for f in ${FILES_TO_CHECK[@]}; do + # Manually ignore permission checking + if [[ ${f} == !(nnas|nnfw|nncc|*.sh|*.py) ]]; then + FILES_TO_CHECK_PERMISSION+=("${f}") + fi + done + + if [[ ${#FILES_TO_CHECK_PERMISSION} -eq 0 ]]; then + return + fi + for FILE_TO_CHECK in ${FILES_TO_CHECK_PERMISSION[@]}; do + RESULT=$(stat -c '%A' ${FILE_TO_CHECK} | grep 'x') + if [ "${RESULT}" != "" ]; then + chmod a-x ${FILE_TO_CHECK} + fi + done +} + +function check_cpp_files() { + if [[ ${__Check_CPP} -eq 0 ]]; then + echo "[SKIPPED] C/C++ check is skipped" + return + fi + + CLANG_FORMAT_CANDIDATES=() + CLANG_FORMAT_CANDIDATES+=("clang-format") + CLANG_FORMAT_CANDIDATES+=("clang-format-3.9") + + for CLANG_FORMAT_CANDIDATE in ${CLANG_FORMAT_CANDIDATES[@]}; do + if command_exists ${CLANG_FORMAT_CANDIDATE} ; then + CLANG_FORMAT="${CLANG_FORMAT_CANDIDATE}" + fi + done + + if [[ -z ${CLANG_FORMAT} ]]; then + echo "[ERROR] clang-format is unavailable" + echo + echo "Please install clang-format before running format check" + exit 1 + fi + + # Check c++ files + FILES_TO_CHECK_CPP=() + for f in ${FILES_TO_CHECK[@]}; do + # Manually ignore style checking + if [[ ${f} == +(*/NeuralNetworks.h|*/NeuralNetworksExtensions.h) ]]; then + continue + fi + + # File extension to check + if [[ ${f} == +(*.h|*.hpp|*.cpp|*.cc|*.c|*.cl) ]]; then + FILES_TO_CHECK_CPP+=("${f}") + fi + done + + # Skip by '.FORMATDENY' file + for s in ${DIRECTORIES_NOT_TO_BE_TESTED[@]}; do + FILES_TO_CHECK_CPP=(${FILES_TO_CHECK_CPP[*]/$s*/}) + done + + if [[ ${#FILES_TO_CHECK_CPP} -ne 0 ]]; then + clang-format-3.9 -i ${FILES_TO_CHECK_CPP[@]} + EXIT_CODE=$? + if [[ ${EXIT_CODE} -ne 0 ]]; then + INVALID_EXIT=${EXIT_CODE} + fi + fi +} + +function check_python_files() { + if [[ ${__Check_PYTHON} -eq 0 ]]; then + echo "[SKIPPED] Python check is skipped" + return + fi + + if ! command_exists yapf; then + echo "[ERROR] yapf is unavailable" + echo " Please install yapf." + exit 1 + fi + + # Check python files + FILES_TO_CHECK_PYTHON=() + for f in ${FILES_TO_CHECK[@]}; do + # File extension to check + if [[ ${f} == *.py ]]; then + FILES_TO_CHECK_PYTHON+=("${f}") + fi + done + for s in ${DIRECTORIES_NOT_TO_BE_TESTED[@]}; do + skip=${s#'.'/}/ + FILES_TO_CHECK_PYTHON=(${FILES_TO_CHECK_PYTHON[*]/$skip*/}) + done + + if [[ ${#FILES_TO_CHECK_PYTHON} -ne 0 ]]; then + yapf -i --style='{based_on_style: pep8, column_limit: 90}' ${FILES_TO_CHECK_PYTHON[@]} + EXIT_CODE=$? + if [[ ${EXIT_CODE} -ne 0 ]]; then + INVALID_EXIT=${EXIT_CODE} + fi + fi +} + +pushd ${NNAS_PROJECT_PATH} + +if [ -n "$(git diff)" ]; then + echo "[WARNING] Commit all the changes before running format check" + echo " format.patch file will contain unstaged files" +fi + +__Check_CPP=${CHECK_CPP:-"1"} +__Check_PYTHON=${CHECK_PYTHON:-"1"} + +FILES_TO_CHECK=$(git ls-files -co --exclude-standard) +if [[ "${CHECK_DIFF_ONLY}" = "1" ]]; then + MASTER_EXIST=$(git rev-parse --verify master) + CURRENT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2-) + DIFF_COMMITS=`git log --graph --oneline master..HEAD | wc -l` + if [[ -z "${MASTER_EXIST}" ]]; then + echo "Cannot found local master branch" + elif [[ "${CURRENT_BRANCH}" = "master" ]]; then + echo "Current branch is master" + else + FILES_TO_CHECK=$(git diff --name-only --diff-filter=d HEAD~${DIFF_COMMITS}..HEAD) + fi +fi + +for DIR_NOT_TO_BE_TESTED in $(git ls-files -co --exclude-standard '*/.FORMATDENY'); do + DIRECTORIES_NOT_TO_BE_TESTED+=($(dirname "${DIR_NOT_TO_BE_TESTED}")) +done + +check_newline +check_permission +check_cpp_files +check_python_files + +DIFF=$(git diff | tee format.patch) + +popd + +if [[ -z "${CRCHECK}" ]] && [[ ! -n "${DIFF}" ]] && [[ ${INVALID_EXIT} -eq 0 ]]; then + echo "[PASSED] Format checker succeed." + return +fi + +# Something went wrong + +if [[ ! -z "${CRCHECK}" ]]; then + echo "[FAILED] Please use LF for newline for following files." + echo "${CRCHECK}" +fi + +if [[ ${PATCHFILE_SIZE} -ne 0 ]]; then + echo "[FAILED] Format checker failed and update code to follow convention." + echo " You can find changes in format.patch" +fi + +if [[ ${INVALID_EXIT} -ne 0 ]]; then + echo "[[FAILED] Invalid format checker exit." +fi + +exit 1 diff --git a/infra/command/gen-coverage-report b/infra/command/gen-coverage-report new file mode 100644 index 000000000..5f928eca5 --- /dev/null +++ b/infra/command/gen-coverage-report @@ -0,0 +1,78 @@ +#!/bin/bash + +import "build.configuration" + +BUILD_WORKSPACE_PATH="${NNAS_PROJECT_PATH}/${BUILD_WORKSPACE_RPATH}" + +LCOV_PATH=$(command -v lcov) +GENHTML_PATH=$(command -v genhtml) + +if [[ -z "${LCOV_PATH}" ]]; then + echo "ERROR: 'lcov' is not found" + exit 255 +fi + +if [[ -z "${GENHTML_PATH}" ]]; then + echo "ERROR: 'genhtml' is not found" + exit 255 +fi + +if [[ -z "${GCOV_PATH}" ]]; then + GCOV_PATH=$(command -v gcov) + if [[ -z "${GCOV_PATH}" ]]; then + echo "ERROR: 'gcov' is not found" + exit 255 + fi +fi + +OUTPUT_TAG="${NNAS_COVERAGE:-coverage}" +OUTPUT_PATH="${NNAS_COVERAGE_PATH:-${NNAS_PROJECT_PATH}/${OUTPUT_TAG}}" + +if [[ -e "${OUTPUT_PATH}" ]]; then + echo "ERROR: '${OUTPUT_PATH}' already exists" + exit 255 +fi + +mkdir -p "${OUTPUT_PATH}" + +RAW_BASE_COVERAGE_INFO_PATH="${OUTPUT_PATH}/coverage.base.raw.info" +RAW_TEST_COVERAGE_INFO_PATH="${OUTPUT_PATH}/coverage.test.raw.info" +RAW_COVERAGE_INFO_PATH="${OUTPUT_PATH}/coverage.raw.info" +EXTRACTED_COVERAGE_INFO_PATH="${OUTPUT_PATH}/coverage.extracted.info" +EXCLUDED_COVERAGE_INFO_PATH="${OUTPUT_PATH}/coverage.excluded.info" +COVERAGE_INFO_PATH="${OUTPUT_PATH}/coverage.info" +HTML_PATH="${OUTPUT_PATH}/html" + +CANDIDATES=() + +for CANDIDATE in "$@"; +do + CANDIDATES+=("${NNAS_PROJECT_PATH}/${CANDIDATE}/*") +done + +# Capture initial zero coverage data +"${LCOV_PATH}" -c -i -d "${BUILD_WORKSPACE_PATH}" --gcov-tool ${GCOV_PATH} -o "${RAW_BASE_COVERAGE_INFO_PATH}" + +# Capture tests coverage data +"${LCOV_PATH}" -c -d "${BUILD_WORKSPACE_PATH}" --gcov-tool ${GCOV_PATH} -o "${RAW_TEST_COVERAGE_INFO_PATH}" + +# Append zero coverage data and tests coverage data +"${LCOV_PATH}" -o "${RAW_COVERAGE_INFO_PATH}" \ + -a "${RAW_BASE_COVERAGE_INFO_PATH}" \ + -a "${RAW_TEST_COVERAGE_INFO_PATH}" + +# Extract data for particular pathes +"${LCOV_PATH}" -e "${RAW_COVERAGE_INFO_PATH}" -o "${EXTRACTED_COVERAGE_INFO_PATH}" \ + "${CANDIDATES[@]}" + +# Exclude *.test.cpp files from coverage report +"${LCOV_PATH}" -r "${EXTRACTED_COVERAGE_INFO_PATH}" -o "${EXCLUDED_COVERAGE_INFO_PATH}" \ + '*.test.cpp' + +# Final coverage data +cp -v ${EXCLUDED_COVERAGE_INFO_PATH} ${COVERAGE_INFO_PATH} + +# Gen html +"${GENHTML_PATH}" "${EXCLUDED_COVERAGE_INFO_PATH}" \ + --prefix "${NNAS_PROJECT_PATH}" \ + --output-directory "${HTML_PATH}" diff --git a/infra/command/install-githooks b/infra/command/install-githooks new file mode 100644 index 000000000..e624aa6d7 --- /dev/null +++ b/infra/command/install-githooks @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +REPO_HOOKS_PATH=$NNAS_PROJECT_PATH/infra/git-hooks +GIT_HOOKS_PATH=$NNAS_PROJECT_PATH/.git/hooks + +# Create symbolic links to hooks dir +if [ -e $GIT_HOOKS_PATH/pre-push ]; then + echo "Backup old $GIT_HOOKS_PATH/pre-push to $GIT_HOOKS_PATH/pre-push~" + mv -v $GIT_HOOKS_PATH/pre-push $GIT_HOOKS_PATH/pre-push~ +elif [ -h $GIT_HOOKS_PATH/pre-push ]; then + ls -l $GIT_HOOKS_PATH/pre-push + echo "Remove broken symlink $GIT_HOOKS_PATH/pre-push" + rm -v $GIT_HOOKS_PATH/pre-push +fi +ln -sv $REPO_HOOKS_PATH/pre-push.sh $GIT_HOOKS_PATH/pre-push diff --git a/infra/command/pylint b/infra/command/pylint new file mode 100644 index 000000000..d20f89169 --- /dev/null +++ b/infra/command/pylint @@ -0,0 +1,32 @@ +#!/bin/bash + +INVALID_EXIT=0 + +__Check_PYLINT=${CHECK_PYLINT:-"1"} + +DIRECTORIES_NOT_TO_BE_TESTED=() + +for DIR_NOT_TO_BE_TESTED in $(find -name '.FORMATDENY' -exec dirname {} \;); do + DIRECTORIES_NOT_TO_BE_TESTED+=("$DIR_NOT_TO_BE_TESTED") +done + +PYTHON_FILES_TO_CHECK=$(git ls-files '*.py') +ARR=($PYTHON_FILES_TO_CHECK) +for s in ${DIRECTORIES_NOT_TO_BE_TESTED[@]}; do + skip=${s#'.'/}/ + ARR=(${ARR[*]//$skip*/}) +done +PYTHON_FILES_TO_CHECK=${ARR[*]} +if [[ ${#PYTHON_FILES_TO_CHECK} -ne 0 ]]; then + pylint --disable=all --enable="E0110, E0203, W1401, W1402, W0221, W0199, W0111, E0237, E1111, E1128, E0701, E0703, E1300, W1302, W1300, W1501, E0012, E0111, W0211, E1310, E1003, W0702, W0711, W1502, E0712, W0640, W0124, E0116, W0102, W1505, W0402, E0108, E0241, W0705, W0109, W0123, W0122, W0106, W1305, E1303, E0102, W0604, W0603, W0602, W0601, W0406, E0240, E0239, E0100, E0604, W1307, E0303, E1139, E1126, E1127, E0238, E0236, E0113, E1130, W1202, E1201, W1201, E1206, E1205, E1200, W0150, E0202, E0704, W0410, W1303, W1306, E1304, E1125, E1302, W0223, E0213, W0233, E0107, E0115, E0117, E1134, E1133, E0103, E0711, W0104, W0105, E0702, E0710, W0623, W0622, E1124, W0404, E0402, E1132, E0101, E0104, W0222, E0114, W0231, E0001, E1306, E1305, E1121, E0112, E1301, E0632, E0603, E0602, W0108, W0107, W0101, E0011, E1137, E1131, E1138, E1135, W1304, W1301, E0601, W0120, E1700" $PYTHON_FILES_TO_CHECK + EXIT_CODE=$? + if [[ $EXIT_CODE -ne 0 ]]; then + INVALID_EXIT=$EXIT_CODE + fi +fi + +if [[ $INVALID_EXIT -eq 0 ]]; then + echo "[PASSED] Format checker succeed." + return +fi +exit 1 |