path: root/tools/test_driver/test_driver.sh

#!/bin/bash
#
# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e
# NOTE: Supposed that this script would be executed with an artifact path.
#       The artifact path has tests/(test suite) and Product/
#       Reference this PR(https://github.sec.samsung.net/STAR/nnfw/pull/375).

function Usage()
{
    echo "Usage: ./$0 --artifactpath=.    # run all tests"
    echo "Usage: ./$0 --artifactpath=/home/dragon/nnfw --frameworktest --verification --benchmark    # run fw test & verfication and benchmark"
    echo ""
    echo "--artifactpath        - (should be passed) path that has tests/ and Product/"
    echo ""
    echo "Following three options are needed when you want to tests of specific types. If you don't pass any one, unittest and verification will be run"
    echo "--unittest            - (default=on) run unit test"
    echo "--frameworktest       - (default=off)run framework test"
    echo "--verification        - (default=on) run verification"
    echo ""
    echo "Following option is only needed when you want to test benchmark."
    echo "--benchmark           - (default=off) run benchmark"
    echo "--benchmark_acl       - (default=off) run benchmark-acl"
    echo ""
    echo "etc."
    echo "--framework_driverbin           - (default=../../Product/out/bin/tflite_run) runner for runnning framework tests"
    echo "--verification_driverbin        - (default=../../Product/out/bin/nnapi_test) runner for runnning verification tests"
    echo "--benchmark_driverbin           - (default=../../Product/out/bin/tflite_benchmark) runner for runnning benchmark"
    echo "--runtestsh           - (default=\$ARTIFACT_PATH/tests/framework/run_test.sh) run_test.sh with path where it is for framework test and verification"
    echo "--unittestdir         - (default=\$ARTIFACT_PATH/Product/out/unittest) directory that has unittest binaries for unit test"
    echo ""
    echo "--ldlibrarypath       - (default=\$ARTIFACT_PATH/Product/out/lib) path that you want to include libraries"
    echo "--usennapi            - (default=on)  declare USE_NNAPI=1"
    echo "--nousennapi          - (default=off) declare nothing about USE_NNAPI"
    echo "--acl_envon           - (default=off) declare envs for ACL"
    echo ""
}

ARTIFACT_PATH=""
FRAMEWORK_DRIVER_BIN=""
VERIFICATION_DRIVER_BIN=""
BENCHMARK_DRIVER_BIN=""
RUN_TEST_SH=""
UNIT_TEST_DIR=""
LD_LIBRARY_PATH_IN_SHELL=""
USE_NNAPI="USE_NNAPI=1"
ALLTEST_ON="true"
UNITTEST_ON="false"
FRAMEWORKTEST_ON="false"
VERIFICATION_ON="false"
BENCHMARK_ON="false"
BENCHMARK_ACL_ON="false"
MODEL_LIST="inceptionv3/inception_nonslim inceptionv3/inception_slim"
ACL_ENV_ON="false"

for i in "$@"
do
    case $i in
        -h|--help|help)
            Usage
            exit 1
            ;;
        --artifactpath=*)
            ARTIFACT_PATH=${i#*=}
            ;;
        --framework_driverbin=*)
            FRAMEWORK_DRIVER_BIN=${i#*=}
            ;;
        --verification_driverbin=*)
            VERIFICATION_DRIVER_BIN=${i#*=}
            ;;
        --benchmark_driverbin=*)
            BENCHMARK_DRIVER_BIN=${i#*=}
            ;;
        --runtestsh=*)
            RUN_TEST_SH=${i#*=}
            ;;
        --unittestdir=*)
            UNIT_TEST_DIR=${i#*=}
            ;;
        --ldlibrarypath=*)
            LD_LIBRARY_PATH_IN_SHELL=${i#*=}
            ;;
        --usennapi)
            USE_NNAPI="USE_NNAPI=1"
            ;;
        --nousennapi)
            USE_NNAPI=""
            ;;
        --unittest)
            ALLTEST_ON="false"
            UNITTEST_ON="true"
            ;;
        --frameworktest)
            ALLTEST_ON="false"
            FRAMEWORKTEST_ON="true"
            ;;
        --verification)
            ALLTEST_ON="false"
            VERIFICATION_ON="true"
            ;;
        --benchmark)
            ALLTEST_ON="false"
            BENCHMARK_ON="true"
            ;;
        --benchmark_acl)
            ALLTEST_ON="false"
            BENCHMARK_ACL_ON="true"
            ;;
        --acl_envon)
            ACL_ENV_ON="true"
            ;;
        *)
            # Be careful that others params are handled as $ARTIFACT_PATH
            ARTIFACT_PATH="$i"
            ;;
    esac
    shift
done

if [ ! -e "$ARTIFACT_PATH" ]; then
    echo "Pass on with proper ARTIFACT_PATH"
    exit 1
fi
ARTIFACT_PATH="$(readlink -f $ARTIFACT_PATH)"

if [ -z "$RUN_TEST_SH" ]; then
    RUN_TEST_SH=$ARTIFACT_PATH/tests/framework/run_test.sh
fi

if [ ! -e "$RUN_TEST_SH" ]; then
    echo "Cannot find $RUN_TEST_SH"
    exit 1
fi

if [ -z "$UNIT_TEST_DIR" ]; then
    UNIT_TEST_DIR=$ARTIFACT_PATH/Product/out/unittest
fi

if [ -z "$LD_LIBRARY_PATH_IN_SHELL" ]; then
    LD_LIBRARY_PATH="$ARTIFACT_PATH/Product/out/lib:$LD_LIBRARY_PATH"
else
    LD_LIBRARY_PATH="$LD_LIBRARY_PATH_IN_SHELL:$LD_LIBRARY_PATH"
fi

# Set env
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH
if [ -n "$USE_NNAPI" ]; then
    export "$USE_NNAPI"
fi

function switch_nnfw_kernel_env()
{
    local switch=$1  # "ON" or "OFF"
    local mode=$2    # "acl" or "neon" or ""
    local NNFW_KERNEL_ENV_FILE=$ARTIFACT_PATH/tools/test_driver/nnfw_kernel_env_list.txt

    for ENV in $(cat $NNFW_KERNEL_ENV_FILE); do
        if [[ "$switch" == "ON" ]]; then
            export "$ENV=$mode"
        else
            unset "$ENV"
        fi
    done
}

if [ "$ACL_ENV_ON" == "true" ]; then
    switch_nnfw_kernel_env "ON" "acl"
fi

# Run unittest in each part such as Runtime, ACL
if [ "$ALLTEST_ON" == "true" ] || [ "$UNITTEST_ON" == "true" ]; then
    if [ ! -e "$ARTIFACT_PATH/report" ]; then
        mkdir -p $ARTIFACT_PATH/report
    fi

    echo ""
    echo "============================================"
    echo "Unittest start"
    echo "============================================"

    num_unittest=0
    for TEST_BIN in `ls $UNIT_TEST_DIR`; do
        num_unittest=$((num_unittest+1))
        echo "============================================"
        echo "Starting set $num_unittest: $TEST_BIN..."
        echo "============================================"
        $UNIT_TEST_DIR/$TEST_BIN --gtest_output=xml:$ARTIFACT_PATH/report/$TEST_BIN.xml
        echo "============================================"
        echo "Finishing set $num_unittest: $TEST_BIN..."
        echo "============================================"
    done
    echo "============================================"
    echo "Completed total $num_unittest set of unittest"
    echo "Unittest end"
    echo "============================================"
fi

# Run tflite_run with various tflite models
if [ "$FRAMEWORKTEST_ON" == "true" ]; then
    if [ -z "$FRAMEWORK_DRIVER_BIN" ]; then
        FRAMEWORK_DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_run
    fi

    if [ ! -e "$ARTIFACT_PATH/report" ]; then
        mkdir -p $ARTIFACT_PATH/report
    fi

    echo ""
    echo "============================================"
    echo "Framework Test with tflite_run..."
    $RUN_TEST_SH --driverbin=$FRAMEWORK_DRIVER_BIN \
        --reportdir=$ARTIFACT_PATH/report \
        --tapname=framework_test.tap \
        > $ARTIFACT_PATH/report/framework_test.log 2>&1
    echo "============================================"
    cat $ARTIFACT_PATH/report/framework_test.tap
    echo "============================================"
    echo ""
fi

# Run nnapi_test with various tflite models
if [ "$ALLTEST_ON" == "true" ] || [ "$VERIFICATION_ON" == "true" ]; then
    if [ -z "$VERIFICATION_DRIVER_BIN" ]; then
        VERIFICATION_DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/nnapi_test
    fi

    if [ ! -e "$ARTIFACT_PATH/report" ]; then
        mkdir -p $ARTIFACT_PATH/report
    fi

    echo ""
    echo "============================================"
    echo "Verification with nnapi_test..."
    $RUN_TEST_SH --driverbin=$VERIFICATION_DRIVER_BIN \
        --reportdir=$ARTIFACT_PATH/report \
        --tapname=verification.tap \
        > $ARTIFACT_PATH/report/verification.log 2>&1
    echo "============================================"
    cat $ARTIFACT_PATH/report/verification.tap
    echo "============================================"
    echo ""
fi

# Benchmark test
function get_result_of_benchmark_test()
{
    local RUN_TEST_SH=$1
    local DRIVER_BIN=$2
    local MODEL=$3
    local LOG_FILE=$4

    $RUN_TEST_SH --driverbin=$DRIVER_BIN $MODEL > $LOG_FILE 2>&1

    local RESULT=`grep -E '^Mean:' $LOG_FILE | sed -e 's/ms//g' | awk '{print $2}'`
    echo "$RESULT"
}

function print_result_of_benchmark_test()
{
    local NAME=$1
    local RESULT=$2
    local RESULT_FILE=$3

    echo "$NAME $RESULT" > $RESULT_FILE
}

function run_benchmark_test()
{
    local DRIVER_BIN=
    local LOG_FILE=
    local RESULT_FILE=
    local RESULT=
    local REPORT_MODEL_DIR=

    if [ -z "$BENCHMARK_DRIVER_BIN" ]; then
        DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_benchmark
    else
        DRIVER_BIN=$BENCHMARK_DRIVER_BIN
    fi

    export COUNT=5
    echo "============================================"
    local i=0
    for MODEL in $MODEL_LIST; do
        echo "Benchmark test with tflite_benchmark & `echo $MODEL`"
        echo $MODEL >> $MODELS_FILE

        REPORT_MODEL_DIR=$ARTIFACT_PATH/report/benchmark/$MODEL
        mkdir -p $REPORT_MODEL_DIR

        # TFLite+CPU
        LOG_FILE=$REPORT_MODEL_DIR/tflite_cpu.txt
        RESULT_FILE=$REPORT_MODEL_DIR/tflite_cpu.result
        echo -n "TFLite + CPU................... "
        unset USE_NNAPI
        RESULT=$(get_result_of_benchmark_test $RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE)
        echo "$RESULT ms"
        print_result_of_benchmark_test "TFLite_CPU" $RESULT $RESULT_FILE
        

        # TFLite+NNAPI(CPU fallback)
        LOG_FILE=$REPORT_MODEL_DIR/tflite_nnapi_cpu.txt
        RESULT_FILE=$REPORT_MODEL_DIR/tflite_nnapi_cpu.result
        echo -n "TFLite + NNAPI(CPU)............ "
        export USE_NNAPI=1
        RESULT=$(get_result_of_benchmark_test $RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE)
        echo "$RESULT ms"
        print_result_of_benchmark_test "TFLite_NNAPI_CPU" $RESULT $RESULT_FILE

        # TFLite+NNAPI(ACL)
        LOG_FILE=$REPORT_MODEL_DIR/tflite_nnapi_acl.txt
        RESULT_FILE=$REPORT_MODEL_DIR/tflite_nnapi_acl.result
        echo -n "TFLite + NNAPI(ACL)............ "
        switch_nnfw_kernel_env "ON" "acl"
        RESULT=$(get_result_of_benchmark_test $RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE)
        echo "$RESULT ms"
        print_result_of_benchmark_test "TFLite_NNAPI_ACL" $RESULT $RESULT_FILE
        unset USE_NNAPI
        switch_nnfw_kernel_env "OFF"

        if [[ $i -ne $(echo $MODEL_LIST | wc -w)-1 ]]; then
            echo ""
        fi
        i=$((i+1))
    done
    echo "============================================"
    unset COUNT
}

function run_benchmark_acl()
{
    local REPORT_DIR=$ARTIFACT_PATH/report/benchmark
    local DRIVER_DIR=$ARTIFACT_PATH/Product/out/bin
    local LOG_FILE=""
    local RESULT_FILE=""
    local RESULT=""

    export COUNT=5
    echo "============================================"
    local i=0
    for BENCHMARK_ACL_BIN in $(ls $DRIVER_DIR/benchmark_*); do
        BENCHMARK_ACL_BIN_BASENAME=$(basename $BENCHMARK_ACL_BIN)
        mkdir -p $REPORT_DIR/$BENCHMARK_ACL_BIN_BASENAME
        echo "Benchmark/acl test by $BENCHMARK_ACL_BIN_BASENAME"
        echo $BENCHMARK_ACL_BIN_BASENAME >> $MODELS_FILE

        # ACL(NEON)
        LOG_FILE=$REPORT_DIR/$BENCHMARK_ACL_BIN_BASENAME/acl_neon.txt
        RESULT_FILE=$REPORT_DIR/$BENCHMARK_ACL_BIN_BASENAME/acl_neon.result
        echo -n "ACL(NEON)...... "
        $BENCHMARK_ACL_BIN 0 > $LOG_FILE 2>&1
        RESULT=`grep -E '^Mean:' $LOG_FILE | sed -e 's/ms//g' | awk '{print $2}'`
        echo "$RESULT ms"
        echo "ACL(NEON)" $RESULT > $RESULT_FILE

        # ACL(OpenCL)
        LOG_FILE=$REPORT_DIR/$BENCHMARK_ACL_BIN_BASENAME/acl_opencl.txt
        RESULT_FILE=$REPORT_DIR/$BENCHMARK_ACL_BIN_BASENAME/acl_opencl.result
        echo -n "ACL(OpenCL).... "
        $BENCHMARK_ACL_BIN 1 > $LOG_FILE 2>&1
        RESULT=`grep -E '^Mean:' $LOG_FILE | sed -e 's/ms//g' | awk '{print $2}'`
        echo "$RESULT ms"
        echo "ACL(OpenCL)" $RESULT > $RESULT_FILE

        if [[ $i -ne $(ls $DRIVER_DIR/benchmark_* | wc -w)-1 ]]; then
            echo ""
        fi
        i=$((i+1))
    done
    echo "============================================"
    unset COUNT
}

if [ "$BENCHMARK_ON" == "true" ] || [ "$BENCHMARK_ACL_ON" == "true" ]; then
    if [ ! -e "$ARTIFACT_PATH/report" ]; then
        mkdir -p $ARTIFACT_PATH/report
    fi
    REPORT_MODEL_DIR=$ARTIFACT_PATH/report/benchmark
    mkdir -p $REPORT_MODEL_DIR
    MODELS_FILE=$ARTIFACT_PATH/report/benchmark/benchmark_models.txt
    rm -f $MODELS_FILE
fi

# Run tflite_benchmark with Iv3_non_slim & Iv3_slim
if [ "$BENCHMARK_ON" == "true" ]; then
    echo ""
    run_benchmark_test
    echo ""
fi

# Run benchmark/acl/benchmark_googlenet,mobilenet and inception_v3
if [ "$BENCHMARK_ACL_ON" == "true" ]; then
    echo ""
    run_benchmark_acl
    echo ""
fi

# make json file
if [ "$BENCHMARK_ON" == "true" ] || [ "$BENCHMARK_ACL_ON" == "true" ]; then
    # functions to fill json with benchmark results
    source $ARTIFACT_PATH/tools/test_driver/print_to_json.sh
    print_to_json
fi