diff options
Diffstat (limited to 'compiler/circle-part-value-test')
21 files changed, 428 insertions, 0 deletions
diff --git a/compiler/circle-part-value-test/CMakeLists.txt b/compiler/circle-part-value-test/CMakeLists.txt new file mode 100644 index 000000000..d75b17d1f --- /dev/null +++ b/compiler/circle-part-value-test/CMakeLists.txt @@ -0,0 +1,99 @@ +# +# this project validates partitioned models produced by circle-partitioner +# with circle-part-driver and two scripts; part_eval_all.sh and part_eval_one.py +# + +if(NOT ENABLE_TEST) + return() +endif(NOT ENABLE_TEST) + +get_target_property(ARTIFACTS_BIN_PATH testDataGenerator BINARY_DIR) + +unset(RECIPE_LIST) +unset(PARTITION_LIST) +unset(TEST_DEPS) + +macro(add RECIPE_NAME PARTITION_NAME) + list(APPEND RECIPE_LIST ${RECIPE_NAME}) + list(APPEND PARTITION_LIST ${PARTITION_NAME}) +endmacro(add) + +# Read "test.lst" +include("test.lst") + +list(LENGTH RECIPE_LIST RECIPE_LENGTH) +math(EXPR RECIPE_LENGTH_M1 "${RECIPE_LENGTH} - 1") + +foreach(IDX RANGE ${RECIPE_LENGTH_M1}) + list(GET RECIPE_LIST ${IDX} RECIPE_NAME) + list(GET PARTITION_LIST ${IDX} PARTITION_NAME) + + # NOTE about the name: + # Use '.recipe' name for source tflite and circle files + # Use '.part' name for actual test folder and test files + + # Output to a folder + set(PARTITIONER_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${PARTITION_NAME}") + + add_custom_command(OUTPUT ${PARTITIONER_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E make_directory "${PARTITIONER_OUTPUT_PATH}" + COMMENT "Make directory ${PARTITIONER_OUTPUT_PATH}" + ) + + # Copy tflite + set(TFLITE_SRC_PATH "${ARTIFACTS_BIN_PATH}/${RECIPE_NAME}.tflite") + set(TFLITE_DST_PATH "${PARTITIONER_OUTPUT_PATH}/${PARTITION_NAME}.tflite") + + add_custom_command(OUTPUT ${TFLITE_DST_PATH} + COMMAND ${CMAKE_COMMAND} -E copy "${TFLITE_SRC_PATH}" "${TFLITE_DST_PATH}" + DEPENDS ${TFLITE_SRC_PATH} + COMMENT "Copy ${RECIPE_NAME}.tflite" + ) + list(APPEND TEST_DEPS ${TFLITE_DST_PATH}) + + # Copy circle + set(CIRCLE_SRC_PATH "${ARTIFACTS_BIN_PATH}/${RECIPE_NAME}.circle") + set(CIRCLE_DST_PATH "${PARTITIONER_OUTPUT_PATH}/${PARTITION_NAME}.circle") + + add_custom_command(OUTPUT ${CIRCLE_DST_PATH} + COMMAND ${CMAKE_COMMAND} -E copy "${CIRCLE_SRC_PATH}" "${CIRCLE_DST_PATH}" + DEPENDS ${CIRCLE_SRC_PATH} + COMMENT "Copy ${RECIPE_NAME}.circle" + ) + list(APPEND TEST_DEPS ${CIRCLE_DST_PATH}) + + # Copy .part + set(PART_FILE "${PARTITION_NAME}.part") + set(PART_SRC_PATH "${CMAKE_CURRENT_SOURCE_DIR}/parts/${PART_FILE}") + set(PART_DST_PATH "${PARTITIONER_OUTPUT_PATH}/${PART_FILE}") + + add_custom_command(OUTPUT ${PART_DST_PATH} + COMMAND ${CMAKE_COMMAND} -E copy "${PART_SRC_PATH}" "${PART_DST_PATH}" + DEPENDS ${PART_SRC_PATH} + COMMENT "Copy ${PART_FILE}" + ) + list(APPEND TEST_DEPS ${PART_DST_PATH}) + + # Partition connection file to generate + set(PARTITIONER_CONN_JSON "${PARTITIONER_OUTPUT_PATH}/${PARTITION_NAME}.conn.json") + + # Run partitioner + add_custom_command(OUTPUT ${PARTITIONER_CONN_JSON} + COMMAND circle_partitioner "${PART_FILE}" "${PARTITION_NAME}.circle" "${PARTITIONER_OUTPUT_PATH}" + DEPENDS circle_partitioner ${PART_DST_PATH} ${CIRCLE_DST_PATH} + COMMENT "Parition ${RECIPE_NAME}.circle with ${PART_FILE}" + ) + list(APPEND TEST_DEPS ${PARTITIONER_CONN_JSON}) +endforeach(IDX) + +add_custom_target(circle_part_value_test_prepare ALL DEPENDS ${TEST_DEPS}) +add_dependencies(circle_part_value_test_prepare common_artifacts_deps) + +# run evaluation +add_test(NAME circle_part_value_test + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/part_eval_all.sh" + "${CMAKE_CURRENT_BINARY_DIR}" + "${NNCC_OVERLAY_DIR}/venv_2_3_0" + "$<TARGET_FILE:circle_part_driver>" + ${PARTITION_LIST} +) diff --git a/compiler/circle-part-value-test/README.md b/compiler/circle-part-value-test/README.md new file mode 100644 index 000000000..6322b0791 --- /dev/null +++ b/compiler/circle-part-value-test/README.md @@ -0,0 +1,15 @@ +# circle-part-value-test + +_circle-part-value-test_ evaluates partitioned models produced by circle-partitioner. + +### Process of evaluation + +Evaluation process is like how _luci-value-test_ does. + +1) generates random input and stores to reference input file(s) +2) executes tflite file from common-artifacts for reference output +3) partitions circle file with .part file and produces into output folder +4) executes produced partitioned circle models with reference input file(s) +5) saves output(s) of circle models to file(s) +6) compares reference output with saved output file(s) +7) fail test if values differ diff --git a/compiler/circle-part-value-test/part_eval_all.sh b/compiler/circle-part-value-test/part_eval_all.sh new file mode 100755 index 000000000..ae8ae4731 --- /dev/null +++ b/compiler/circle-part-value-test/part_eval_all.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# This script verifies the basic behavior of circle-partitioner +# +# HOW TO USE +# +# ./part_eval_all.sh <path/to/work_dir> <path/to/venv_dir> <path/to/driver> <TEST 1> <TEST 2> ... +# +# bin_dir : build directory of circle-part-value-test (ex: build/compiler/circle-part-value-test) +# work_dir : artifacts directoy where test materials exist +# venv_dir : python virtual environment home directory + +VERIFY_SOURCE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VERIFY_SCRIPT_PATH="${VERIFY_SOURCE_PATH}/part_eval_one.py" +WORKDIR="$1"; shift +VIRTUALENV="$1"; shift +CIRCLE_PART_DRIVER_PATH="$1"; shift + +TESTED=() +PASSED=() +FAILED=() + +for TESTCASE in "$@"; do + TESTED+=("${TESTCASE}") + + # for simplicity, folder uses same ${TESTCASE} + TESTCASE_FOLDER="${WORKDIR}/${TESTCASE}" + + PASSED_TAG="${TESTCASE_FOLDER}.passed" + rm -f "${PASSED_TAG}" + + cat > "${TESTCASE_FOLDER}.log" <( + exec 2>&1 + set -ex + + # chdir into the folder as ini has relative filename of the model + pushd ${TESTCASE_FOLDER} + + source "${VIRTUALENV}/bin/activate" + "${VIRTUALENV}/bin/python" "${VERIFY_SCRIPT_PATH}" \ + --driver "${CIRCLE_PART_DRIVER_PATH}" \ + --name "${TESTCASE}" + + if [[ $? -eq 0 ]]; then + touch "${PASSED_TAG}" + fi + + popd + ) + + if [[ -f "${PASSED_TAG}" ]]; then + PASSED+=("${TESTCASE}") + else + FAILED+=("${TESTCASE}") + fi +done + +if [[ ${#TESTED[@]} -ne ${#PASSED[@]} ]]; then + echo "FAILED" + for TEST in "${FAILED[@]}" + do + echo "- ${TEST}" + done + exit 255 +fi + +echo "PASSED" +exit 0 diff --git a/compiler/circle-part-value-test/part_eval_one.py b/compiler/circle-part-value-test/part_eval_one.py new file mode 100755 index 000000000..b0b65fd74 --- /dev/null +++ b/compiler/circle-part-value-test/part_eval_one.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +import numpy as np +import tensorflow as tf +import subprocess +import argparse +import traceback + +# +# This script compares the execution result of TFLite interpreter and +# partitioned model(s) from a circle model +# +# Basic usage for example: +# part_eval_one.py \ +# --driver build/compiler/circle-part-driver/circle-part-driver \ +# --name test_file +# +parser = argparse.ArgumentParser() +parser.add_argument('--driver', type=str, required=True) +parser.add_argument('--name', type=str, required=True) +args = parser.parse_args() + +driver = args.driver +tflite_model = args.name + ".tflite" +circle_model = args.name + ".circle" +partition_conn_ini = args.name + ".conn.ini" + +# Build TFLite interpreter. +interpreter = tf.lite.Interpreter(tflite_model) +interpreter.allocate_tensors() + +# Generate random input data. +num_inputs = len(interpreter.get_input_details()) +for i in range(num_inputs): + input_details = interpreter.get_input_details()[i] + if input_details["dtype"] == np.float32: + input_data = np.array( + np.random.random_sample(input_details["shape"]), input_details["dtype"]) + elif input_details["dtype"] == np.uint8: + input_data = np.array( + np.random.randint(0, 256, size=input_details["shape"]), + input_details["dtype"]) + elif input_details["dtype"] == np.bool_: + input_data = np.array( + np.random.choice(a=[True, False], size=input_details["shape"]), + input_details["dtype"]) + else: + raise SystemExit("Unsupported input dtype") + + interpreter.set_tensor(input_details["index"], input_data) + input_data.tofile(circle_model + ".input" + str(i)) + +# Do inference +interpreter.invoke() + +# Execute circle-part-driver. +partition_command = [ + driver, partition_conn_ini, + str(num_inputs), circle_model + ".input", circle_model + ".output" +] +print("Run: ") +for arg in partition_command: + print(" ", arg, "\\") +print("", flush=True) + +subprocess.run(partition_command, check=True) + +# Compare the results. +for idx in range(len(interpreter.get_output_details())): + output_details = interpreter.get_output_details()[idx] + output_data = np.fromfile(circle_model + ".output" + str(idx), + output_details["dtype"]) + shape_file = open(circle_model + ".output" + str(idx) + ".shape", 'r') + output_shape = [int(i) for i in shape_file.read().split(',')] + luci_output_data = np.reshape(output_data, output_shape) + try: + if output_details["dtype"] == np.uint8: + if np.allclose( + luci_output_data, + interpreter.get_tensor( + interpreter.get_output_details()[idx]["index"]), + rtol=0, + atol=0) == False: + raise SystemExit("Execution result of " + tflite_model + + " does not match with " + circle_model) + elif output_details["dtype"] == np.float32: + if np.allclose( + luci_output_data, + interpreter.get_tensor( + interpreter.get_output_details()[idx]["index"]), + rtol=1.e-5, + atol=1.e-5) == False: + raise SystemExit("Execution result of " + tflite_model + + " does not match with " + circle_model) + elif output_details["dtype"] == np.int64: + if np.allclose( + luci_output_data, + interpreter.get_tensor( + interpreter.get_output_details()[idx]["index"]), + rtol=0, + atol=0) == False: + raise SystemExit("Execution result of " + tflite_model + + " does not match with " + circle_model) + elif output_details["dtype"] == np.int32: + if np.allclose( + luci_output_data, + interpreter.get_tensor( + interpreter.get_output_details()[idx]["index"]), + rtol=0, + atol=0) == False: + raise SystemExit("Execution result of " + tflite_model + + " does not match with " + circle_model) + else: + raise SystemExit("Unsupported data type: ", output_details["dtype"]) + except: + print(traceback.format_exc()) + quit(255) + +quit(0) diff --git a/compiler/circle-part-value-test/parts/Net_InstanceNorm_003.001.part b/compiler/circle-part-value-test/parts/Net_InstanceNorm_003.001.part new file mode 100644 index 000000000..01b8c704e --- /dev/null +++ b/compiler/circle-part-value-test/parts/Net_InstanceNorm_003.001.part @@ -0,0 +1,7 @@ +[partition] +backends=cpu,acl_cl +default=cpu +comply=opcode + +[OPCODE] +ADD=acl_cl diff --git a/compiler/circle-part-value-test/parts/Net_InstanceNorm_003.002.part b/compiler/circle-part-value-test/parts/Net_InstanceNorm_003.002.part new file mode 100644 index 000000000..dc378a448 --- /dev/null +++ b/compiler/circle-part-value-test/parts/Net_InstanceNorm_003.002.part @@ -0,0 +1,8 @@ +[partition] +backends=cpu,acl_cl +default=cpu +comply=opcode + +[OPCODE] +SUB=acl_cl +DIV=acl_cl diff --git a/compiler/circle-part-value-test/parts/Net_InstanceNorm_003.part b/compiler/circle-part-value-test/parts/Net_InstanceNorm_003.part new file mode 100644 index 000000000..d4d439d27 --- /dev/null +++ b/compiler/circle-part-value-test/parts/Net_InstanceNorm_003.part @@ -0,0 +1,7 @@ +[partition] +backends=cpu,acl_cl +default=cpu +comply=opcode + +[OPCODE] +DIV=acl_cl diff --git a/compiler/circle-part-value-test/parts/Part_Add_Sqrt_000.part b/compiler/circle-part-value-test/parts/Part_Add_Sqrt_000.part new file mode 100644 index 000000000..402af87e9 --- /dev/null +++ b/compiler/circle-part-value-test/parts/Part_Add_Sqrt_000.part @@ -0,0 +1,7 @@ +[partition] +backends=cpu,acl_cl +default=cpu +comply=opcode + +[OPCODE] +SQRT=acl_cl diff --git a/compiler/circle-part-value-test/parts/Part_Add_Sqrt_Rsqrt_000.part b/compiler/circle-part-value-test/parts/Part_Add_Sqrt_Rsqrt_000.part new file mode 100644 index 000000000..c6dba9f94 --- /dev/null +++ b/compiler/circle-part-value-test/parts/Part_Add_Sqrt_Rsqrt_000.part @@ -0,0 +1,7 @@ +[partition] +backends=cpu,acl_cl +default=cpu +comply=opcode + +[OPCODE] +RSQRT=acl_cl diff --git a/compiler/circle-part-value-test/parts/Part_Add_Sub_000.part b/compiler/circle-part-value-test/parts/Part_Add_Sub_000.part new file mode 100644 index 000000000..905137ce7 --- /dev/null +++ b/compiler/circle-part-value-test/parts/Part_Add_Sub_000.part @@ -0,0 +1,7 @@ +[partition] +backends=cpu,acl_cl +default=cpu +comply=opcode + +[OPCODE] +SUB=acl_cl diff --git a/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_000.part b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_000.part new file mode 100644 index 000000000..402af87e9 --- /dev/null +++ b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_000.part @@ -0,0 +1,7 @@ +[partition] +backends=cpu,acl_cl +default=cpu +comply=opcode + +[OPCODE] +SQRT=acl_cl diff --git a/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_001.part b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_001.part new file mode 100644 index 000000000..402af87e9 --- /dev/null +++ b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_001.part @@ -0,0 +1,7 @@ +[partition] +backends=cpu,acl_cl +default=cpu +comply=opcode + +[OPCODE] +SQRT=acl_cl diff --git a/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_002.part b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_002.part new file mode 100644 index 000000000..402af87e9 --- /dev/null +++ b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_002.part @@ -0,0 +1,7 @@ +[partition] +backends=cpu,acl_cl +default=cpu +comply=opcode + +[OPCODE] +SQRT=acl_cl diff --git a/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_003.part b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_003.part new file mode 100644 index 000000000..402af87e9 --- /dev/null +++ b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_003.part @@ -0,0 +1,7 @@ +[partition] +backends=cpu,acl_cl +default=cpu +comply=opcode + +[OPCODE] +SQRT=acl_cl diff --git a/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_Add_000.part b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_Add_000.part new file mode 100644 index 000000000..402af87e9 --- /dev/null +++ b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_Add_000.part @@ -0,0 +1,7 @@ +[partition] +backends=cpu,acl_cl +default=cpu +comply=opcode + +[OPCODE] +SQRT=acl_cl diff --git a/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_Add_001.part b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_Add_001.part new file mode 100644 index 000000000..402af87e9 --- /dev/null +++ b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_Add_001.part @@ -0,0 +1,7 @@ +[partition] +backends=cpu,acl_cl +default=cpu +comply=opcode + +[OPCODE] +SQRT=acl_cl diff --git a/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_Add_002.part b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_Add_002.part new file mode 100644 index 000000000..402af87e9 --- /dev/null +++ b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_Add_002.part @@ -0,0 +1,7 @@ +[partition] +backends=cpu,acl_cl +default=cpu +comply=opcode + +[OPCODE] +SQRT=acl_cl diff --git a/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_Add_003.part b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_Add_003.part new file mode 100644 index 000000000..0ec264c94 --- /dev/null +++ b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_Add_003.part @@ -0,0 +1,7 @@ +[partition] +backends=cpu,acl_cl +default=cpu +comply=opcode + +[OPCODE] +WWW=acl_cl diff --git a/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_Add_004.part b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_Add_004.part new file mode 100644 index 000000000..febab2246 --- /dev/null +++ b/compiler/circle-part-value-test/parts/Part_Sqrt_Rsqrt_Add_004.part @@ -0,0 +1,6 @@ +[partition] +backends=cpu,acl_cl +default=cpu +comply=opcode + +[OPCODE] diff --git a/compiler/circle-part-value-test/requires.cmake b/compiler/circle-part-value-test/requires.cmake new file mode 100644 index 000000000..a9301f947 --- /dev/null +++ b/compiler/circle-part-value-test/requires.cmake @@ -0,0 +1,3 @@ +require("common-artifacts") +require("circle-partitioner") +require("circle-part-driver") diff --git a/compiler/circle-part-value-test/test.lst b/compiler/circle-part-value-test/test.lst new file mode 100644 index 000000000..8316560f0 --- /dev/null +++ b/compiler/circle-part-value-test/test.lst @@ -0,0 +1,20 @@ +# Add recipe names from /res/TensorFlowLiteRecipes to test. +# Only add items exist in common-artifacts test: tflite/circle files are copied as source. +# +# add(RECIPE_NAME PARTITION_NAME) + +add(Part_Add_Sub_000 Part_Add_Sub_000) +add(Part_Sqrt_Rsqrt_000 Part_Sqrt_Rsqrt_000) +add(Part_Sqrt_Rsqrt_001 Part_Sqrt_Rsqrt_001) +add(Part_Sqrt_Rsqrt_002 Part_Sqrt_Rsqrt_002) +add(Part_Sqrt_Rsqrt_003 Part_Sqrt_Rsqrt_003) +add(Part_Sqrt_Rsqrt_Add_000 Part_Sqrt_Rsqrt_Add_000) +add(Part_Sqrt_Rsqrt_Add_001 Part_Sqrt_Rsqrt_Add_001) +add(Part_Sqrt_Rsqrt_Add_002 Part_Sqrt_Rsqrt_Add_002) +add(Part_Sqrt_Rsqrt_Add_003 Part_Sqrt_Rsqrt_Add_003) +add(Part_Sqrt_Rsqrt_Add_004 Part_Sqrt_Rsqrt_Add_004) +add(Part_Add_Sqrt_000 Part_Add_Sqrt_000) +add(Part_Add_Sqrt_Rsqrt_000 Part_Add_Sqrt_Rsqrt_000) +add(Net_InstanceNorm_003 Net_InstanceNorm_003) +add(Net_InstanceNorm_003 Net_InstanceNorm_003.001) +add(Net_InstanceNorm_003 Net_InstanceNorm_003.002) |