diff options
Diffstat (limited to 'infra')
39 files changed, 438 insertions, 232 deletions
diff --git a/infra/cmake/packages/ARMComputeSourceConfig.cmake b/infra/cmake/packages/ARMComputeSourceConfig.cmake index 51a235a35..adec1f91b 100644 --- a/infra/cmake/packages/ARMComputeSourceConfig.cmake +++ b/infra/cmake/packages/ARMComputeSourceConfig.cmake @@ -8,7 +8,7 @@ function(_ARMComputeSource_import) nnas_include(OptionTools) envoption(EXTERNAL_DOWNLOAD_SERVER "https://github.com") - set(ARMCOMPUTE_URL ${EXTERNAL_DOWNLOAD_SERVER}/ARM-software/ComputeLibrary/archive/v19.11.1.tar.gz) + set(ARMCOMPUTE_URL ${EXTERNAL_DOWNLOAD_SERVER}/ARM-software/ComputeLibrary/archive/v20.05.tar.gz) ExternalSource_Download(ARMCOMPUTE ${ARMCOMPUTE_URL}) set(ARMComputeSource_DIR ${ARMCOMPUTE_SOURCE_DIR} PARENT_SCOPE) diff --git a/infra/cmake/packages/FarmhashSourceConfig.cmake b/infra/cmake/packages/FarmhashSourceConfig.cmake index 8a9a384df..a19c8b992 100644 --- a/infra/cmake/packages/FarmhashSourceConfig.cmake +++ b/infra/cmake/packages/FarmhashSourceConfig.cmake @@ -9,7 +9,7 @@ function(_FarmhashSource_import) # NOTE TensorFlow 1.12 downloads farmhash from the following URL # TensorFlow 1.13.1 downloads farmhash from the following URL - # TensorFlow 2.3-rc0 downloads farmhash from the following URL + # TensorFlow 2.3.0 downloads farmhash from the following URL envoption(FARMHASH_1_12_URL https://github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz) ExternalSource_Download(FARMHASH ${FARMHASH_1_12_URL}) diff --git a/infra/cmake/packages/FlatBuffersConfig.cmake b/infra/cmake/packages/FlatBuffersConfig.cmake index ab0b7708f..da084e7d3 100644 --- a/infra/cmake/packages/FlatBuffersConfig.cmake +++ b/infra/cmake/packages/FlatBuffersConfig.cmake @@ -25,7 +25,8 @@ function(_FlatBuffers_build) BUILD_DIR ${CMAKE_BINARY_DIR}/externals/FLATBUFFERS/build INSTALL_DIR ${EXT_OVERLAY_DIR} BUILD_FLAGS ${ADDITIONAL_CXX_FLAGS} - IDENTIFIER "1.10-fix1" + IDENTIFIER "1.10-fix2" + EXTRA_OPTS "-DFLATBUFFERS_BUILD_TESTS:BOOL=OFF" PKG_NAME "FLATBUFFERS") endfunction(_FlatBuffers_build) diff --git a/infra/cmake/packages/HDF5Config.cmake b/infra/cmake/packages/HDF5Config.cmake index e282e0bc9..19803f1ea 100644 --- a/infra/cmake/packages/HDF5Config.cmake +++ b/infra/cmake/packages/HDF5Config.cmake @@ -27,6 +27,7 @@ _HDF5_build() find_path(HDF5_CONFIG_DIR "hdf5-config.cmake" PATHS ${EXT_OVERLAY_DIR} PATH_SUFFIXES + cmake share/cmake share/cmake/hdf5 cmake/hdf5 diff --git a/infra/cmake/packages/Pybind11Config.cmake b/infra/cmake/packages/Pybind11Config.cmake new file mode 100644 index 000000000..b6d500496 --- /dev/null +++ b/infra/cmake/packages/Pybind11Config.cmake @@ -0,0 +1,22 @@ +function(_Pybind11_import) + nnas_find_package(Pybind11Source QUIET) + + if(NOT Pybind11Source_FOUND) + set(Pybind11_FOUND FALSE PARENT_SCOPE) + return() + endif(NOT Pybind11Source_FOUND) + + nnas_include(ExternalBuildTools) + ExternalBuild_CMake(CMAKE_DIR ${Pybind11Source_DIR} + BUILD_DIR ${CMAKE_BINARY_DIR}/externals/PYBIND11/build + INSTALL_DIR ${EXT_OVERLAY_DIR} + IDENTIFIER "2.5.0" + PKG_NAME "PYBIND11" + EXTRA_OPTS "-DPYBIND11_TEST:BOOL=OFF") + + find_path(Pybind11_INCLUDE_DIRS NAMES pybind11.h PATHS ${EXT_OVERLAY_DIR} PATH_SUFFIXES include/pybind11) + + set(Pybind11_FOUND TRUE PARENT_SCOPE) +endfunction(_Pybind11_import) + +_Pybind11_import() diff --git a/infra/cmake/packages/Pybind11SourceConfig.cmake b/infra/cmake/packages/Pybind11SourceConfig.cmake new file mode 100644 index 000000000..76f51e4d3 --- /dev/null +++ b/infra/cmake/packages/Pybind11SourceConfig.cmake @@ -0,0 +1,18 @@ +function(_Pybind11Source_import) + if(NOT DOWNLOAD_PYBIND11) + set(Pybind11Source_FOUND FALSE PARENT_SCOPE) + return() + endif(NOT DOWNLOAD_PYBIND11) + + nnas_include(ExternalSourceTools) + nnas_include(OptionTools) + + envoption(PYBIND11_URL https://github.com/pybind/pybind11/archive/v2.5.0.tar.gz) + + ExternalSource_Download(PYBIND11 ${PYBIND11_URL}) + + set(Pybind11Source_DIR ${PYBIND11_SOURCE_DIR} PARENT_SCOPE) + set(Pybind11Source_FOUND TRUE PARENT_SCOPE) +endfunction(_Pybind11Source_import) + +_Pybind11Source_import() diff --git a/infra/cmake/packages/TensorFlowEigenSource-2.3.0/TensorFlowEigenSourceConfig.cmake b/infra/cmake/packages/TensorFlowEigenSource-2.3.0/TensorFlowEigenSourceConfig.cmake new file mode 100644 index 000000000..d50d04508 --- /dev/null +++ b/infra/cmake/packages/TensorFlowEigenSource-2.3.0/TensorFlowEigenSourceConfig.cmake @@ -0,0 +1,21 @@ +function(_TensorFlowEigenSource_import) + if(NOT DOWNLOAD_EIGEN) + set(TensorFlowEigenSource_FOUND FALSE PARENT_SCOPE) + return() + endif(NOT DOWNLOAD_EIGEN) + + nnas_include(ExternalSourceTools) + nnas_include(OptionTools) + + # Exact version used by TensorFlow v2.3.0. + # See tensorflow/tensorflow/workspace.bzl. + envoption(EXTERNAL_DOWNLOAD_SERVER "https://gitlab.com") + envoption(TENSORFLOW_2_3_0_EIGEN_URL ${EXTERNAL_DOWNLOAD_SERVER}/libeigen/eigen/-/archive/386d809bde475c65b7940f290efe80e6a05878c4/eigen-386d809bde475c65b7940f290efe80e6a05878c4.tar.gz) + + ExternalSource_Download(EIGEN DIRNAME TENSORFLOW-2.3.0-EIGEN ${TENSORFLOW_2_3_0_EIGEN_URL}) + + set(TensorFlowEigenSource_DIR ${EIGEN_SOURCE_DIR} PARENT_SCOPE) + set(TensorFlowEigenSource_FOUND TRUE PARENT_SCOPE) +endfunction(_TensorFlowEigenSource_import) + +_TensorFlowEigenSource_import() diff --git a/infra/cmake/packages/TensorFlowEigenSource-2.3.0/TensorFlowEigenSourceConfigVersion.cmake b/infra/cmake/packages/TensorFlowEigenSource-2.3.0/TensorFlowEigenSourceConfigVersion.cmake new file mode 100644 index 000000000..04df5eb6d --- /dev/null +++ b/infra/cmake/packages/TensorFlowEigenSource-2.3.0/TensorFlowEigenSourceConfigVersion.cmake @@ -0,0 +1,10 @@ +set(PACKAGE_VERSION "2.3.0") +set(PACKAGE_VERSION_EXACT FALSE) +set(PACKAGE_VERSION_COMPATIBLE FALSE) +set(PACKAGE_VERSION_UNSUITABLE TRUE) + +if(PACKAGE_FIND_VERSION VERSION_EQUAL PACKAGE_VERSION) + set(PACKAGE_VERSION_EXACT TRUE) + set(PACKAGE_VERSION_COMPATIBLE TRUE) + set(PACKAGE_VERSION_UNSUITABLE FALSE) +endif(PACKAGE_FIND_VERSION VERSION_EQUAL PACKAGE_VERSION) diff --git a/infra/cmake/packages/TensorFlowSource-2.3.0/TensorFlowSourceConfig.cmake b/infra/cmake/packages/TensorFlowSource-2.3.0/TensorFlowSourceConfig.cmake new file mode 100644 index 000000000..5c3a0f8cc --- /dev/null +++ b/infra/cmake/packages/TensorFlowSource-2.3.0/TensorFlowSourceConfig.cmake @@ -0,0 +1,18 @@ +function(_TensorFlowSource_import) + if(NOT DOWNLOAD_TENSORFLOW) + set(TensorFlowSource_FOUND FALSE PARENT_SCOPE) + return() + endif(NOT DOWNLOAD_TENSORFLOW) + + nnas_include(ExternalSourceTools) + nnas_include(OptionTools) + + envoption(TENSORFLOW_2_3_0_URL https://github.com/tensorflow/tensorflow/archive/v2.3.0.tar.gz) + + ExternalSource_Download(TENSORFLOW DIRNAME TENSORFLOW-2.3.0 ${TENSORFLOW_2_3_0_URL}) + + set(TensorFlowSource_DIR ${TENSORFLOW_SOURCE_DIR} PARENT_SCOPE) + set(TensorFlowSource_FOUND TRUE PARENT_SCOPE) +endfunction(_TensorFlowSource_import) + +_TensorFlowSource_import() diff --git a/infra/cmake/packages/TensorFlowSource-2.3.0/TensorFlowSourceConfigVersion.cmake b/infra/cmake/packages/TensorFlowSource-2.3.0/TensorFlowSourceConfigVersion.cmake new file mode 100644 index 000000000..04df5eb6d --- /dev/null +++ b/infra/cmake/packages/TensorFlowSource-2.3.0/TensorFlowSourceConfigVersion.cmake @@ -0,0 +1,10 @@ +set(PACKAGE_VERSION "2.3.0") +set(PACKAGE_VERSION_EXACT FALSE) +set(PACKAGE_VERSION_COMPATIBLE FALSE) +set(PACKAGE_VERSION_UNSUITABLE TRUE) + +if(PACKAGE_FIND_VERSION VERSION_EQUAL PACKAGE_VERSION) + set(PACKAGE_VERSION_EXACT TRUE) + set(PACKAGE_VERSION_COMPATIBLE TRUE) + set(PACKAGE_VERSION_UNSUITABLE FALSE) +endif(PACKAGE_FIND_VERSION VERSION_EQUAL PACKAGE_VERSION) diff --git a/infra/docker/Dockerfile b/infra/docker/Dockerfile index e675b53ad..052cc4fb6 100644 --- a/infra/docker/Dockerfile +++ b/infra/docker/Dockerfile @@ -1,8 +1,6 @@ FROM ubuntu:16.04 ARG UBUNTU_MIRROR -ENV http_proxy $http_proxy -ENV https_proxy $https_proxy RUN if [ -n "$http_proxy" ] ; then echo "Acquire::http::proxy \"${http_proxy}\";" >> /etc/apt/apt.conf ; fi RUN if [ -n "$https_proxy" ] ; then echo "Acquire::https::proxy \"${https_proxy}\";" >> /etc/apt/apt.conf ; fi @@ -22,6 +20,7 @@ RUN apt-get update && apt-get -qqy install libprotobuf-dev protobuf-compiler # Additonal tools RUN apt-get update && apt-get -qqy install doxygen graphviz wget unzip clang-format-3.9 python3 python3-pip python3-venv hdf5-tools pylint +RUN pip3 install --upgrade pip RUN pip3 install yapf==0.22.0 numpy # Install google test (source) diff --git a/infra/docker/Dockerfile.1804 b/infra/docker/Dockerfile.1804 index fc6fc9a1a..cc31bba1f 100644 --- a/infra/docker/Dockerfile.1804 +++ b/infra/docker/Dockerfile.1804 @@ -1,12 +1,6 @@ FROM ubuntu:18.04 ARG UBUNTU_MIRROR -ENV http_proxy $http_proxy -ENV https_proxy $https_proxy - -RUN if [ -n "$http_proxy" ] ; then echo "Acquire::http::proxy \"${http_proxy}\";" >> /etc/apt/apt.conf ; fi -RUN if [ -n "$https_proxy" ] ; then echo "Acquire::https::proxy \"${https_proxy}\";" >> /etc/apt/apt.conf ; fi -RUN if [ -n "$UBUNTU_MIRROR" ] ; then sed "s/archive.ubuntu.com/${UBUNTU_MIRROR}/g" -i /etc/apt/sources.list ; fi # Install 'add-apt-repository' RUN apt-get update && apt-get -qqy install software-properties-common @@ -22,6 +16,7 @@ RUN apt-get update && apt-get -qqy install libprotobuf-dev protobuf-compiler # Additonal tools RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qqy install doxygen graphviz wget unzip clang-format-3.9 python3 python3-pip python3-venv hdf5-tools pylint +RUN pip3 install --upgrade pip RUN pip3 install yapf==0.22.0 numpy # Install google test (source) diff --git a/infra/nncc/CMakeLists.txt b/infra/nncc/CMakeLists.txt index 3ac6680de..0be6885e2 100644 --- a/infra/nncc/CMakeLists.txt +++ b/infra/nncc/CMakeLists.txt @@ -98,6 +98,7 @@ option(DOWNLOAD_CAFFE "Download Caffe source" ON) option(DOWNLOAD_PYTORCH "Download Pytorch source" ON) option(DOWNLOAD_ONNX "Download ONNX source" ON) option(DOWNLOAD_ABSEIL "Download Abseil-cpp source" ON) +option(DOWNLOAD_PYBIND11 "Download Pybind11 source" ON) option(DOWNLOAD_GTEST "Download Google Test source" ON) option(BUILD_GTEST "Build Google Test from the downloaded source" ON) diff --git a/infra/nncc/command/utcount b/infra/nncc/command/utcount index d4610e3f0..d06c5c9de 100644 --- a/infra/nncc/command/utcount +++ b/infra/nncc/command/utcount @@ -13,7 +13,7 @@ BUILD_ITEMS="angkor cwrap pepper-str pepper-strcast pp stdex \ oops pepper-assert \ hermes hermes-std \ loco locop locomotiv logo-core logo \ -foder souschef arser \ +foder souschef arser vconone \ safemain mio-circle mio-tflite \ tflite2circle \ luci \ diff --git a/infra/nnfw/cmake/CfgOptionFlags.cmake b/infra/nnfw/cmake/CfgOptionFlags.cmake index d1395f871..3c6b7d960 100644 --- a/infra/nnfw/cmake/CfgOptionFlags.cmake +++ b/infra/nnfw/cmake/CfgOptionFlags.cmake @@ -23,7 +23,7 @@ CMAKE_DEPENDENT_OPTION(BUILD_RUNTIME_NNAPI_TEST "Build Runtime NN API Generated OFF) option(BUILD_RUNTIME_NNFW_API_TEST "Build Runtime NNFW API Tests" ON) option(BUILD_TFLITE_RUN "Build tflite-run" ON) -option(BUILD_TFLITE_RUN_2_2_0 "Build tflite-run 2.2.0" OFF) +option(BUILD_TFLITE_VANILLA_RUN "Build tflite-vanilla-run" OFF) option(BUILD_TFLITE_BENCHMARK_MODEL "Build tflite benchmark model" OFF) option(BUILD_NNAPI_TEST "Build nnapi_test" ON) option(BUILD_NNPACKAGE_RUN "Build nnpackge_run" ON) @@ -70,7 +70,7 @@ option(DOWNLOAD_BOOST "Download boost source" OFF) option(DOWNLOAD_RUY "Download ruy source" ON) option(BUILD_BOOST "Build boost source" OFF) option(BUILD_TENSORFLOW_LITE "Build TensorFlow Lite from the downloaded source" ON) -option(BUILD_TENSORFLOW_LITE_2_2_0 "Build TensorFlow Lite from the downloaded source" OFF) +option(BUILD_TENSORFLOW_LITE_2_3_0 "Build TensorFlow Lite 2.3.0 from the downloaded source" OFF) option(BUILD_GTEST "Download and build Google Test" ON) option(BUILD_ARMCOMPUTE "Build ARM Compute from the downloaded source" ON) option(BUILD_RUY "Build ruy library from the downloaded source" ON) diff --git a/infra/nnfw/cmake/packages/EigenConfig.cmake b/infra/nnfw/cmake/packages/EigenConfig.cmake index f37d653cf..e71830a16 100644 --- a/infra/nnfw/cmake/packages/EigenConfig.cmake +++ b/infra/nnfw/cmake/packages/EigenConfig.cmake @@ -1,5 +1,5 @@ function(_Eigen_import) - nnas_find_package(TensorFlowEigenSource-2.3.0-rc0 QUIET) + nnas_find_package(TensorFlowEigenSource EXACT 2.3.0 QUIET) if(NOT TensorFlowEigenSource_FOUND) set(Eigen_FOUND FALSE PARENT_SCOPE) diff --git a/infra/nnfw/cmake/packages/TensorFlowLite-2.2.0Config.cmake b/infra/nnfw/cmake/packages/TensorFlowLite-2.2.0Config.cmake deleted file mode 100644 index e698235f4..000000000 --- a/infra/nnfw/cmake/packages/TensorFlowLite-2.2.0Config.cmake +++ /dev/null @@ -1,92 +0,0 @@ -if(BUILD_TENSORFLOW_LITE_2_2_0) - macro(return_unless VAR) - if(NOT ${VAR}) - message("${VAR} NOT TRUE") - set(TensorFlowLite_2_2_0_FOUND PARENT_SCOPE) - return() - endif(NOT ${VAR}) - endmacro(return_unless) - - nnas_include(ExternalSourceTools) - nnas_include(OptionTools) - - # Below urls come from https://github.com/tensorflow/tensorflow/blob/v2.2.0/tensorflow/lite/tools/make/Makefile - - set(absl_url "https://github.com/abseil/abseil-cpp/archive/43ef2148c0936ebf7cb4be6b19927a9d9d145b8f.tar.gz") - ExternalSource_Download("tflite220_Absl" ${absl_url}) - set(TFLite220AbslSource_DIR "${tflite220_Absl_SOURCE_DIR}") - if (NOT TFLite220AbslSource_DIR STREQUAL "") - set(TFLite220AbslSource_FOUND TRUE) - endif() - return_unless(TFLite220AbslSource_FOUND) - - set(eigen_url "https://gitlab.com/libeigen/eigen/-/archive/52a2fbbb008a47c5e3fb8ac1c65c2feecb0c511c/eigen-52a2fbbb008a47c5e3fb8ac1c65c2feecb0c511c.tar.gz") - ExternalSource_Download("tflite220_Eigen" ${eigen_url}) - set(TFLite220EigenSource_DIR "${tflite220_Eigen_SOURCE_DIR}") - if (NOT TFLite220EigenSource_DIR STREQUAL "") - set(TFLite220EigenSource_FOUND TRUE) - endif() - return_unless(TFLite220EigenSource_FOUND) - - set(farmhash_url "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz") - ExternalSource_Download("tflite220_Farmhash" ${farmhash_url}) - set(TFLite220FarmhashSource_DIR "${tflite220_Farmhash_SOURCE_DIR}") - if (NOT TFLite220FarmhashSource_DIR STREQUAL "") - set(TFLite220FarmhashSource_FOUND TRUE) - endif() - return_unless(TFLite220FarmhashSource_FOUND) - - set(fft2d_url "https://storage.googleapis.com/mirror.tensorflow.org/www.kurims.kyoto-u.ac.jp/~ooura/fft2d.tgz") - ExternalSource_Download("tflite220_FFT2D" ${fft2d_url}) - set(TFLite220FFT2DSource_DIR "${tflite220_FFT2D_SOURCE_DIR}") - if (NOT TFLite220FFT2DSource_DIR STREQUAL "") - set(TFLite220FFT2DSource_FOUND TRUE) - endif() - return_unless(TFLite220FFT2DSource_FOUND) - - set(flatbuffers_url "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/flatbuffers/archive/v1.11.0.tar.gz") - ExternalSource_Download("tflite220_FlatBuffers" ${flatbuffers_url}) - set(TFLite220FlatBuffersSource_DIR "${tflite220_FlatBuffers_SOURCE_DIR}") - if (NOT TFLite220FlatBuffersSource_DIR STREQUAL "") - set(TFLite220FlatBuffersSource_FOUND TRUE) - endif() - return_unless(TFLite220FlatBuffersSource_FOUND) - - set(fp16_url "https://github.com/Maratyszcza/FP16/archive/febbb1c163726b5db24bed55cc9dc42529068997.zip") - ExternalSource_Download("tflite220_FP16" ${fp16_url}) - set(TFLite220FP16Source_DIR "${tflite220_FP16_SOURCE_DIR}") - if (NOT TFLite220FP16Source_DIR STREQUAL "") - set(TFLite220FP16Source_FOUND TRUE) - endif() - return_unless(TFLite220FP16Source_FOUND) - - set(gemmlowp_url "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/gemmlowp/archive/12fed0cd7cfcd9e169bf1925bc3a7a58725fdcc3.zip") - ExternalSource_Download("tflite220_GEMMLowp" ${gemmlowp_url}) - set(TFLite220GEMMLowpSource_DIR "${tflite220_GEMMLowp_SOURCE_DIR}") - if (NOT TFLite220GEMMLowpSource_DIR STREQUAL "") - set(TFLite220GEMMLowpSource_FOUND TRUE) - endif() - return_unless(TFLite220GEMMLowpSource_FOUND) - - set(neon2sse_url "https://github.com/intel/ARM_NEON_2_x86_SSE/archive/master.zip") - ExternalSource_Download("tflite220_NEON2SSE" ${neon2sse_url}) - set(TFLite220NEON2SSESource_DIR "${tflite220_NEON2SSE_SOURCE_DIR}") - if (NOT TFLite220NEON2SSESource_DIR STREQUAL "") - set(TFLite220NEON2SSESource_FOUND TRUE) - endif() - return_unless(TFLite220NEON2SSESource_FOUND) - - set(tensorflow_url "https://github.com/tensorflow/tensorflow/archive/v2.2.0.tar.gz") - ExternalSource_Download("tflite220_TensorFlow" ${tensorflow_url}) - set(TFLite220TensorFlowSource_DIR "${tflite220_TensorFlow_SOURCE_DIR}") - if (NOT TFLite220TensorFlowSource_DIR STREQUAL "") - set(TFLite220TensorFlowSource_FOUND TRUE) - endif() - return_unless(TFLite220TensorFlowSource_FOUND) - - nnas_include(ExternalProjectTools) - add_extdirectory("${CMAKE_CURRENT_LIST_DIR}/TensorFlowLite-2.2.0" tflite-2.2.0) - - set(TensorFlowLite_2_2_0_FOUND TRUE) - return() -endif() diff --git a/infra/nnfw/cmake/packages/TensorFlowLite-2.2.0/CMakeLists.txt b/infra/nnfw/cmake/packages/TensorFlowLite-2.3.0/CMakeLists.txt index 8e7f78eb1..20547b92d 100644 --- a/infra/nnfw/cmake/packages/TensorFlowLite-2.2.0/CMakeLists.txt +++ b/infra/nnfw/cmake/packages/TensorFlowLite-2.3.0/CMakeLists.txt @@ -1,8 +1,8 @@ -# Reference: https://github.com/tensorflow/tensorflow/blob/v2.2.0/tensorflow/lite/tools/make/Makefile +# Reference: https://github.com/tensorflow/tensorflow/blob/v2.3.0/tensorflow/lite/tools/make/Makefile # -# Tensorflow Lite library 2.2.0 +# Tensorflow Lite library 2.3.0 # -set(TENSORFLOW_LITE_BASE ${TFLite220TensorFlowSource_DIR}/tensorflow/lite) +set(TENSORFLOW_LITE_BASE ${TFLiteVanillaTensorFlowSource_DIR}/tensorflow/lite) file(GLOB TFLITE_CORE_SRCS "${TENSORFLOW_LITE_BASE}/*.c" "${TENSORFLOW_LITE_BASE}/*.cc" @@ -18,8 +18,7 @@ file(GLOB TFLITE_API_SRCS "${TENSORFLOW_LITE_BASE}/core/api/*.c" list(APPEND TFLITE_PROFILING_SRCS "${TENSORFLOW_LITE_BASE}/profiling/memory_info.cc") list(APPEND TFLITE_PROFILING_SRCS "${TENSORFLOW_LITE_BASE}/profiling/time.cc") -file(GLOB TFLITE_EXPERIMENTAL_SRCS "${TENSORFLOW_LITE_BASE}/experimental/resource/*.cc" - "${TENSORFLOW_LITE_BASE}/experimental/ruy/*.cc") +file(GLOB TFLITE_EXPERIMENTAL_SRCS "${TENSORFLOW_LITE_BASE}/experimental/resource/*.cc") file(GLOB TFLITE_SPARSITY_SRCS "${TENSORFLOW_LITE_BASE}/tools/optimize/sparsity/*.cc") @@ -32,24 +31,37 @@ list(APPEND TFLITE_SRCS ${TFLITE_EXPERIMENTAL_SRCS}) list(APPEND TFLITE_SRCS ${TFLITE_SPARSITY_SRCS}) # externals -list(APPEND TFLITE_SRCS "${TFLite220FarmhashSource_DIR}/src/farmhash.cc") -list(APPEND TFLITE_SRCS "${TFLite220FFT2DSource_DIR}/fftsg.c") -list(APPEND TFLITE_SRCS "${TFLite220FFT2DSource_DIR}/fftsg2d.c") -list(APPEND TFLITE_SRCS "${TFLite220FlatBuffersSource_DIR}/src/util.cpp") +list(APPEND TFLITE_SRCS "${TFLiteVanillaFarmhashSource_DIR}/src/farmhash.cc") +list(APPEND TFLITE_SRCS "${TFLiteVanillaFFT2DSource_DIR}/fftsg.c") +list(APPEND TFLITE_SRCS "${TFLiteVanillaFFT2DSource_DIR}/fftsg2d.c") +list(APPEND TFLITE_SRCS "${TFLiteVanillaFlatBuffersSource_DIR}/src/util.cpp") # externals - absl -file(GLOB_RECURSE ABSL_SRCS "${TFLite220AbslSource_DIR}/absl/*.cc") -file(GLOB_RECURSE ABSL_EXCLS "${TFLite220AbslSource_DIR}/absl/*test*.cc" - "${TFLite220AbslSource_DIR}/absl/*benchmark*.cc" - "${TFLite220AbslSource_DIR}/absl/synchronization/*.cc" - "${TFLite220AbslSource_DIR}/absl/debugging/*.cc" - "${TFLite220AbslSource_DIR}/absl/hash/*.cc" - "${TFLite220AbslSource_DIR}/absl/flags/*.cc") +file(GLOB_RECURSE ABSL_SRCS "${TFLiteVanillaAbslSource_DIR}/absl/*.cc") +file(GLOB_RECURSE ABSL_EXCLS "${TFLiteVanillaAbslSource_DIR}/absl/*test*.cc" + "${TFLiteVanillaAbslSource_DIR}/absl/*benchmark*.cc" + "${TFLiteVanillaAbslSource_DIR}/absl/synchronization/*.cc" + "${TFLiteVanillaAbslSource_DIR}/absl/debugging/*.cc" + "${TFLiteVanillaAbslSource_DIR}/absl/hash/*.cc" + "${TFLiteVanillaAbslSource_DIR}/absl/flags/*.cc" + "${TFLiteVanillaAbslSource_DIR}/absl/random/*.cc") list(REMOVE_ITEM ABSL_SRCS ${ABSL_EXCLS}) list(APPEND TFLITE_SRCS ${ABSL_SRCS}) +# externals - ruy +file(GLOB RUY_SRCS "${TFLiteVanillaRuySource_DIR}/ruy/*.cc") +file(GLOB_RECURSE RUY_EXCLS "${TFLiteVanillaRuySource_DIR}/ruy/*test*.cc" + "${TFLiteVanillaRuySource_DIR}/ruy/*benchmark*.cc" + "${TFLiteVanillaRuySource_DIR}/ruy/*example*.cc") +list(REMOVE_ITEM RUY_SRCS ${RUY_EXCLS}) +# Temporary fix for ruy compilation error. +# TODO(b/158800055): Remove this hack once the ruy version is correctly bumped. +list(REMOVE_ITEM RUY_SRCS "${TFLiteVanillaRuySource_DIR}/ruy/prepare_packed_matrices.cc") +list(APPEND TFLITE_SRCS ${RUY_SRCS}) + + # Build with mmap? true -# caution: v2.2.0's Makefile has wrong code on this part. This is fixed on master branch. +# caution: v2.3.0's Makefile has wrong code on this part. This is fixed on master branch. set(BUILD_WITH_MMAP TRUE) if(${BUILD_WITH_MMAP}) list(REMOVE_ITEM TFLITE_SRCS "${TENSORFLOW_LITE_BASE}/mmap_allocation_disabled.cc") @@ -86,25 +98,26 @@ file(GLOB_RECURSE TFLITE_EXCLS "${TENSORFLOW_LITE_BASE}/*test*.cc" list(REMOVE_ITEM TFLITE_SRCS ${TFLITE_EXCLS}) # include headers -list(APPEND TFLITE_INCLUDES "${TFLite220TensorFlowSource_DIR}") -list(APPEND TFLITE_INCLUDES "${TFLite220EigenSource_DIR}") -list(APPEND TFLITE_INCLUDES "${TFLite220AbslSource_DIR}") -list(APPEND TFLITE_INCLUDES "${TFLite220GEMMLowpSource_DIR}") -list(APPEND TFLITE_INCLUDES "${TFLite220NEON2SSESource_DIR}") -list(APPEND TFLITE_INCLUDES "${TFLite220FarmhashSource_DIR}/src") -list(APPEND TFLITE_INCLUDES "${TFLite220FlatBuffersSource_DIR}/include") -list(APPEND TFLITE_INCLUDES "${TFLite220FP16Source_DIR}/include") - -add_library(tensorflow-lite-2.2.0 STATIC ${TFLITE_SRCS}) -target_include_directories(tensorflow-lite-2.2.0 SYSTEM PUBLIC ${TFLITE_INCLUDES}) -target_compile_definitions(tensorflow-lite-2.2.0 PUBLIC "GEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK -DTFLITE_WITH_RUY -DTFLITE_WITH_RUY_GEMV") -set_property(TARGET tensorflow-lite-2.2.0 PROPERTY POSITION_INDEPENDENT_CODE ON) -target_link_libraries(tensorflow-lite-2.2.0 eigen ${LIB_PTHREAD} dl) -if(${BUILD_WITH_NNAPI}) - target_link_libraries(tensorflow-lite-2.2.0 rt) +list(APPEND TFLITE_INCLUDES "${TFLiteVanillaTensorFlowSource_DIR}") +list(APPEND TFLITE_INCLUDES "${TFLiteVanillaEigenSource_DIR}") +list(APPEND TFLITE_INCLUDES "${TFLiteVanillaAbslSource_DIR}") +list(APPEND TFLITE_INCLUDES "${TFLiteVanillaGEMMLowpSource_DIR}") +list(APPEND TFLITE_INCLUDES "${TFLiteVanillaNEON2SSESource_DIR}") +list(APPEND TFLITE_INCLUDES "${TFLiteVanillaFarmhashSource_DIR}/src") +list(APPEND TFLITE_INCLUDES "${TFLiteVanillaFlatBuffersSource_DIR}/include") +list(APPEND TFLITE_INCLUDES "${TFLiteVanillaFP16Source_DIR}/include") +list(APPEND TFLITE_INCLUDES "${TFLiteVanillaRuySource_DIR}") + +add_library(tensorflow-lite-2.3.0 STATIC ${TFLITE_SRCS}) +target_include_directories(tensorflow-lite-2.3.0 SYSTEM PUBLIC ${TFLITE_INCLUDES}) +target_compile_definitions(tensorflow-lite-2.3.0 PUBLIC "GEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK -DTFLITE_WITH_RUY -DTFLITE_WITH_RUY_GEMV") +set_property(TARGET tensorflow-lite-2.3.0 PROPERTY POSITION_INDEPENDENT_CODE ON) +target_link_libraries(tensorflow-lite-2.3.0 eigen ${LIB_PTHREAD} dl) +if(NOT ANDROID AND ${BUILD_WITH_NNAPI}) + target_link_libraries(tensorflow-lite-2.3.0 rt) endif() if(ANDROID) - target_link_libraries(tensorflow-lite-2.2.0 log) - target_include_directories(tensorflow-lite-2.2.0 PUBLIC "${NDK_DIR}/..") + target_link_libraries(tensorflow-lite-2.3.0 log) + target_include_directories(tensorflow-lite-2.3.0 PUBLIC "${NDK_DIR}/..") endif() diff --git a/infra/nnfw/cmake/packages/TensorFlowLite-2.3.0Config.cmake b/infra/nnfw/cmake/packages/TensorFlowLite-2.3.0Config.cmake new file mode 100644 index 000000000..d00ca96a6 --- /dev/null +++ b/infra/nnfw/cmake/packages/TensorFlowLite-2.3.0Config.cmake @@ -0,0 +1,100 @@ +if(BUILD_TENSORFLOW_LITE_2_3_0) + macro(return_unless VAR) + if(NOT ${VAR}) + message("${VAR} NOT TRUE") + set(TensorFlowLite_2_3_0_FOUND PARENT_SCOPE) + return() + endif(NOT ${VAR}) + endmacro(return_unless) + + nnas_include(ExternalSourceTools) + nnas_include(OptionTools) + + # Below urls come from https://github.com/tensorflow/tensorflow/blob/v2.3.0/tensorflow/lite/tools/make/Makefile + + set(absl_url "https://github.com/abseil/abseil-cpp/archive/df3ea785d8c30a9503321a3d35ee7d35808f190d.tar.gz") + ExternalSource_Download("TFLiteVanilla_Absl" ${absl_url}) + set(TFLiteVanillaAbslSource_DIR "${TFLiteVanilla_Absl_SOURCE_DIR}") + if (NOT TFLiteVanillaAbslSource_DIR STREQUAL "") + set(TFLiteVanillaAbslSource_FOUND TRUE) + endif() + return_unless(TFLiteVanillaAbslSource_FOUND) + + set(eigen_url "https://gitlab.com/libeigen/eigen/-/archive/386d809bde475c65b7940f290efe80e6a05878c4/eigen-386d809bde475c65b7940f290efe80e6a05878c4.tar.gz") + ExternalSource_Download("TFLiteVanilla_Eigen" ${eigen_url}) + set(TFLiteVanillaEigenSource_DIR "${TFLiteVanilla_Eigen_SOURCE_DIR}") + if (NOT TFLiteVanillaEigenSource_DIR STREQUAL "") + set(TFLiteVanillaEigenSource_FOUND TRUE) + endif() + return_unless(TFLiteVanillaEigenSource_FOUND) + + set(farmhash_url "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz") + ExternalSource_Download("TFLiteVanilla_Farmhash" ${farmhash_url}) + set(TFLiteVanillaFarmhashSource_DIR "${TFLiteVanilla_Farmhash_SOURCE_DIR}") + if (NOT TFLiteVanillaFarmhashSource_DIR STREQUAL "") + set(TFLiteVanillaFarmhashSource_FOUND TRUE) + endif() + return_unless(TFLiteVanillaFarmhashSource_FOUND) + + set(fft2d_url "https://storage.googleapis.com/mirror.tensorflow.org/github.com/petewarden/OouraFFT/archive/v1.0.tar.gz") + ExternalSource_Download("TFLiteVanilla_FFT2D" ${fft2d_url}) + set(TFLiteVanillaFFT2DSource_DIR "${TFLiteVanilla_FFT2D_SOURCE_DIR}") + if (NOT TFLiteVanillaFFT2DSource_DIR STREQUAL "") + set(TFLiteVanillaFFT2DSource_FOUND TRUE) + endif() + return_unless(TFLiteVanillaFFT2DSource_FOUND) + + set(flatbuffers_url "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/flatbuffers/archive/v1.12.0.tar.gz") + ExternalSource_Download("TFLiteVanilla_FlatBuffers" ${flatbuffers_url}) + set(TFLiteVanillaFlatBuffersSource_DIR "${TFLiteVanilla_FlatBuffers_SOURCE_DIR}") + if (NOT TFLiteVanillaFlatBuffersSource_DIR STREQUAL "") + set(TFLiteVanillaFlatBuffersSource_FOUND TRUE) + endif() + return_unless(TFLiteVanillaFlatBuffersSource_FOUND) + + set(fp16_url "https://github.com/Maratyszcza/FP16/archive/4dfe081cf6bcd15db339cf2680b9281b8451eeb3.zip") + ExternalSource_Download("TFLiteVanilla_FP16" ${fp16_url}) + set(TFLiteVanillaFP16Source_DIR "${TFLiteVanilla_FP16_SOURCE_DIR}") + if (NOT TFLiteVanillaFP16Source_DIR STREQUAL "") + set(TFLiteVanillaFP16Source_FOUND TRUE) + endif() + return_unless(TFLiteVanillaFP16Source_FOUND) + + set(gemmlowp_url "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/gemmlowp/archive/fda83bdc38b118cc6b56753bd540caa49e570745.zip") + ExternalSource_Download("TFLiteVanilla_GEMMLowp" ${gemmlowp_url}) + set(TFLiteVanillaGEMMLowpSource_DIR "${TFLiteVanilla_GEMMLowp_SOURCE_DIR}") + if (NOT TFLiteVanillaGEMMLowpSource_DIR STREQUAL "") + set(TFLiteVanillaGEMMLowpSource_FOUND TRUE) + endif() + return_unless(TFLiteVanillaGEMMLowpSource_FOUND) + + set(neon2sse_url "https://github.com/intel/ARM_NEON_2_x86_SSE/archive/1200fe90bb174a6224a525ee60148671a786a71f.tar.gz") + ExternalSource_Download("TFLiteVanilla_NEON2SSE" ${neon2sse_url}) + set(TFLiteVanillaNEON2SSESource_DIR "${TFLiteVanilla_NEON2SSE_SOURCE_DIR}") + if (NOT TFLiteVanillaNEON2SSESource_DIR STREQUAL "") + set(TFLiteVanillaNEON2SSESource_FOUND TRUE) + endif() + return_unless(TFLiteVanillaNEON2SSESource_FOUND) + + set(tensorflow_url "https://github.com/tensorflow/tensorflow/archive/v2.3.0.tar.gz") + ExternalSource_Download("TFLiteVanilla_TensorFlow" ${tensorflow_url}) + set(TFLiteVanillaTensorFlowSource_DIR "${TFLiteVanilla_TensorFlow_SOURCE_DIR}") + if (NOT TFLiteVanillaTensorFlowSource_DIR STREQUAL "") + set(TFLiteVanillaTensorFlowSource_FOUND TRUE) + endif() + return_unless(TFLiteVanillaTensorFlowSource_FOUND) + + set(ruy_url "https://github.com/google/ruy/archive/34ea9f4993955fa1ff4eb58e504421806b7f2e8f.zip") + ExternalSource_Download("TFLiteVanilla_Ruy" ${ruy_url}) + set(TFLiteVanillaRuySource_DIR "${TFLiteVanilla_Ruy_SOURCE_DIR}") + if (NOT TFLiteVanillaRuySource_DIR STREQUAL "") + set(TFLiteVanillaRuySource_FOUND TRUE) + endif() + return_unless(TFLiteVanillaRuySource_FOUND) + + nnas_include(ExternalProjectTools) + add_extdirectory("${CMAKE_CURRENT_LIST_DIR}/TensorFlowLite-2.3.0" tflite-2.3.0) + + set(TensorFlowLite_2_3_0_FOUND TRUE) + return() +endif() diff --git a/infra/nnfw/config/gbs.conf b/infra/nnfw/config/gbs.conf index 515cadaba..bad9eb204 100644 --- a/infra/nnfw/config/gbs.conf +++ b/infra/nnfw/config/gbs.conf @@ -5,7 +5,7 @@ profile = profile.tizen [profile.tizen] user=obs_viewer obs = obs.tizen -repos = repo.tizen_base,repo.tizen_mobile +repos = repo.tizen_one,repo.tizen_base,repo.tizen_mobile buildroot = /home/GBS-ROOT/ [obs.tizen] @@ -15,6 +15,8 @@ url = http://api.tizen.org url = http://download.tizen.org/snapshots/tizen/unified/latest/repos/standard/packages/ [repo.tizen_base] -url = http://download.tizen.org/snapshots/tizen/base/latest/repos/standard/packages/ +url = http://download.tizen.org/snapshots/tizen/base/latest/repos/standard/packages/ +[repo.tizen_one] +url = http://nnfw.mooo.com/archive/tizen/ diff --git a/infra/packaging/build b/infra/packaging/build index 036c2d575..e941a724b 100644 --- a/infra/packaging/build +++ b/infra/packaging/build @@ -85,7 +85,8 @@ function join_by # Invoke "preset_configure" function that the preset provides preset_configure -NPROC=$(cat /proc/cpuinfo | grep -c processor) +NPROC=${NPROC:-$(cat /proc/cpuinfo | grep -c processor)} +echo "[BUILD] \"make\" with -j${NPROC} option. You can specify the number of jobs by defining NPROC" cmake --build . -- -j$((NPROC/2)) all cmake --build . -- install # Install NN Package tools diff --git a/infra/packaging/preset/20200630 b/infra/packaging/preset/20200630 index e1599357a..5d1635809 100644 --- a/infra/packaging/preset/20200630 +++ b/infra/packaging/preset/20200630 @@ -14,6 +14,7 @@ function preset_configure() REQUIRED_UNITS+=("souschef") REQUIRED_UNITS+=("safemain") REQUIRED_UNITS+=("arser") + REQUIRED_UNITS+=("vconone") # Hermes Logging Framework REQUIRED_UNITS+=("hermes" "hermes-std") # loco IR and related utilities @@ -27,12 +28,16 @@ function preset_configure() REQUIRED_UNITS+=("tf2tfliteV2" "luci-interpreter" "circle-verify") REQUIRED_UNITS+=("record-minmax" "circle-quantizer") REQUIRED_UNITS+=("one-cmds") + REQUIRED_UNITS+=("bcq-tools") + + NPROC=${NPROC:-$(cat /proc/cpuinfo | grep -c processor)} # TODO Use "nncc configure" and "nncc build" cmake \ -DCMAKE_INSTALL_PREFIX="${NNCC_INSTALL_PREFIX}" \ -DCMAKE_BUILD_TYPE=release \ -DBUILD_WHITELIST=$(join_by ";" "${REQUIRED_UNITS[@]}") \ + -DEXTERNALS_BUILD_THREADS=$((NPROC/2)) \ ${EXTRA_OPTIONS[@]} \ "${NNAS_PROJECT_PATH}/infra/nncc" } @@ -44,14 +49,4 @@ function preset_install() # Install tf2nnpkg install -T -m 755 -D "${SCRIPT_PATH}/res/tf2nnpkg.${PRESET}" "${NNAS_INSTALL_PREFIX}/bin/tf2nnpkg" - - # Create python virtual enviornment - python3 -m venv "${NNAS_INSTALL_PREFIX}/bin/venv" - - # Install tensorflow - source "${NNAS_INSTALL_PREFIX}/bin/venv/bin/activate" - python -m pip --default-timeout=1000 --trusted-host pypi.org --trusted-host files.pythonhost.org \ - install -U pip setuptools - python -m pip --default-timeout=1000 --trusted-host pypi.org --trusted-host files.pythonhost.org \ - install tensorflow-cpu==2.3.0rc0 } diff --git a/infra/packaging/preset/20200731_windows b/infra/packaging/preset/20200731_windows new file mode 100644 index 000000000..65d179eaf --- /dev/null +++ b/infra/packaging/preset/20200731_windows @@ -0,0 +1,65 @@ +#!/bin/bash + +function preset_configure() +{ + REQUIRED_UNITS=() + # Common Libraries + REQUIRED_UNITS+=("angkor" "cwrap" "pepper-str" "pepper-strcast" "pp" "stdex") + REQUIRED_UNITS+=("oops" "pepper-assert" "foder") + REQUIRED_UNITS+=("souschef") + REQUIRED_UNITS+=("safemain") + REQUIRED_UNITS+=("arser") + REQUIRED_UNITS+=("vconone") + # Hermes Logging Framework + REQUIRED_UNITS+=("hermes" "hermes-std") + # loco IR and related utilities + REQUIRED_UNITS+=("loco" "locop" "locomotiv" "logo-core" "logo") + # Flatbuffer I/O + REQUIRED_UNITS+=("mio-tflite" "mio-circle") + # Circle compiler library (.circle -> .circle) + REQUIRED_UNITS+=("luci") + # Tools + REQUIRED_UNITS+=("tflite2circle" "circle2circle" "tflchef" "circlechef") + REQUIRED_UNITS+=("tf2tfliteV2" "luci-interpreter" "circle-verify") + REQUIRED_UNITS+=("record-minmax" "circle-quantizer") + REQUIRED_UNITS+=("one-cmds") + + NPROC=$(cat /proc/cpuinfo | grep -c processor) + + # TODO Use "nncc configure" and "nncc build" + cmake \ + -G "MSYS Makefiles" \ + -DTF2NNPKG_FOR_WINDOWS=ON \ + -DUSE_PROTOBUF_LEGACY_IMPORT=ON \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--allow-multiple-definition" \ + -DCMAKE_SHARED_LINKER_FLAGS="-Wl,--allow-multiple-definition" \ + -DENABLE_TEST=OFF \ + -DDOWNLOAD_GTEST=OFF \ + -DBUILD_GTEST=OFF \ + -DCMAKE_C_COMPILER=gcc \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_INSTALL_PREFIX="${NNCC_INSTALL_PREFIX}" \ + -DCMAKE_BUILD_TYPE=release \ + -DBUILD_WHITELIST=$(join_by ";" "${REQUIRED_UNITS[@]}") \ + -DEXTERNALS_BUILD_THREADS=$((NPROC/2)) \ + ${EXTRA_OPTIONS[@]} \ + "${NNAS_PROJECT_PATH}/infra/nncc" +} + +function preset_install() +{ + # Install libraries to bin/ for Windows release + mv ${NNCC_INSTALL_PREFIX}/lib/*.dll ${NNCC_INSTALL_PREFIX}/bin + rm -rf ${NNCC_INSTALL_PREFIX}/lib + + install -t "${NNPKG_INSTALL_PREFIX}/bin" -D \ + "${NNAS_PROJECT_PATH}/tools/nnpackage_tool/model2nnpkg/model2nnpkg.sh" + + # Install tf2nnpkg + install -T -m 755 -D "${SCRIPT_PATH}/res/tf2nnpkg.20200630" "${NNAS_INSTALL_PREFIX}/bin/tf2nnpkg" + + # Though you have to install tensorflow to run 'tf2tfliteV2', + # tensorflow can't be installed in mingw. First, You can install tensorflow + # from Window native CMD(run as administrator) with python virtual environment. + # And, you must copy it to "${NNAS_INSTALL_PREFIX}/bin/venv" +} diff --git a/infra/packaging/res/tf2nnpkg.20200630 b/infra/packaging/res/tf2nnpkg.20200630 index 9101f8273..7846fd388 100644 --- a/infra/packaging/res/tf2nnpkg.20200630 +++ b/infra/packaging/res/tf2nnpkg.20200630 @@ -14,10 +14,16 @@ command_exists() { usage() { echo "Convert TensorFlow model to nnpackage." - echo "Usage: tf2nnpkg --info <path/to/info> --graphdef <path/to/pb> [OPTION] -o <path/to/nnpkg/directory>" - exit 0 + echo "Usage: tf2nnpkg" + echo " --info <path/to/info>" + echo " --graphdef <path/to/pb>" + echo " -o <path/to/nnpkg/directory>" + echo " --v2 (optional) Use TF 2.x interface" + exit 255 } +TF_INTERFACE="--v1" + # Parse command-line arguments # while [ "$#" -ne 0 ]; do @@ -39,6 +45,10 @@ while [ "$#" -ne 0 ]; do export OUTPUT_DIR="$2" shift 2 ;; + '--v2') + TF_INTERFACE="--v2" + shift + ;; *) echo "${CUR}" shift @@ -83,10 +93,7 @@ OUTPUT=$(awk -F, '/^output/ { print $2 }' ${INFO_FILE} | cut -d: -f1 | tr -d ' ' INPUT_SHAPES=$(grep ^input ${INFO_FILE} | cut -d "[" -f2 | cut -d "]" -f1 | tr -d ' ' | xargs | tr ' ' ':') # generate tflite file -python "${ROOT}/bin/tf2tfliteV2.py" --v2 --input_path ${GRAPHDEF_FILE} \ ---output_path "${TMPDIR}/${MODEL_NAME}.tflite" \ ---input_arrays ${INPUT} --output_arrays ${OUTPUT} || \ -python "${ROOT}/bin/tf2tfliteV2.py" --v1 --input_path ${GRAPHDEF_FILE} \ +python "${ROOT}/bin/tf2tfliteV2.py" ${TF_INTERFACE} --input_path ${GRAPHDEF_FILE} \ --output_path "${TMPDIR}/${MODEL_NAME}.tflite" \ --input_arrays ${INPUT} --input_shapes ${INPUT_SHAPES} \ --output_arrays ${OUTPUT} diff --git a/infra/scripts/build-tcm.sh b/infra/scripts/build-tcm.sh new file mode 100755 index 000000000..22fb33558 --- /dev/null +++ b/infra/scripts/build-tcm.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# STEP 1 +# Download latest TCM tool from +# https://github.sec.samsung.net/RS-TCM/tca-standalone/releases/download/v0.0.8/tca-standalone-0.0.8.jar +# +# STEP 2 +# Create symbolic link `./src` for source directory to be analyzed which has `.ahub` configuration. +# +# STEP 3 +# run this `build-tcm.sh` script. +# +# See the following link for additional details. +# https://github.sec.samsung.net/RS-TCM/tca-standalone/wiki/Tutorials-CPP-Gtest +# + +echo ${PROJECT_DIR:=${PWD}} + +java -jar $PROJECT_DIR/tca-standalone-0.0.8.jar \ + --outdir=$PROJECT_DIR/tcm-output \ + --config=$PROJECT_DIR/.ahub/tcchecker-tca/config.yaml \ + --local=$PROJECT_DIR/src \ + --logfile=$PROJECT_DIR/tcm-output/tcm.log \ + --debug diff --git a/infra/scripts/common.sh b/infra/scripts/common.sh index 28aa213ec..a10aac271 100755 --- a/infra/scripts/common.sh +++ b/infra/scripts/common.sh @@ -15,15 +15,18 @@ # TFLiteModelVerification $1 $2 $3 # Run ./tests/scripts/test-driver.sh script verification test # -# Unittests $1 $2 $3 -# Run ./tests/scripts/test-driver.sh script unittest +# NNAPIGTest $1 $2 $3 +# Run [INSTALL_PATH]/test/onert-test unittest command for nnapi gtest # # NNPackageTest $1 $2 -# Run ./tests/scripts/nnpkg_test.sh script nnpackage test +# Run [INSTALL_PATH]/test/onert-test nnpkg-test command CURRENT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_PATH="$(cd ${CURRENT_PATH}/../../ && pwd)" +# Install path on CI +INSTALL_PATH=$ROOT_PATH/Product/out + function CheckTestPrepared() { # Model download server setting @@ -47,16 +50,12 @@ function TFLiteModelVerification() export BACKENDS=$1 if [[ "$2" == "" ]]; then - ./tests/scripts/test-driver.sh \ - --reportdir=$ROOT_PATH/$3 \ - --verification \ - . + $INSTALL_PATH/test/onert-test verify-tflite --api=nnapi \ + --reportdir=$ROOT_PATH/$3 else - ./tests/scripts/test-driver.sh \ - --frameworktest_list_file=$2 \ - --reportdir=$ROOT_PATH/$3 \ - --verification \ - . + $INSTALL_PATH/test/onert-test verify-tflite --api=nnapi \ + --list=$2 \ + --reportdir=$ROOT_PATH/$3 fi unset BACKENDS @@ -64,10 +63,10 @@ function TFLiteModelVerification() } # $1: (required) backend -# $2: (required) unittest skiplist file relative path from nnfw root directory +# $2: (required) nnapi gtest skiplist file relative path from nnfw root directory # pass empty string if there is no test list # $3: (required) relative path for report from nnfw root directory -function Unittests() +function NNAPIGTest() { [[ $# -ne 3 ]] && echo "Invalid function argument setting" && exit 1 @@ -75,7 +74,7 @@ function Unittests() # Backup original nnapi_gtest.skip # TODO Pass skiplist to test-driver.sh - SKIPLIST_FILE="${ROOT_PATH}/Product/out/unittest/nnapi_gtest.skip" + SKIPLIST_FILE="${INSTALL_PATH}/unittest/nnapi_gtest.skip" BACKUP_FILE="${SKIPLIST_FILE}.backup" if [[ "$2" != "" ]]; then cp ${SKIPLIST_FILE} ${BACKUP_FILE} @@ -83,10 +82,9 @@ function Unittests() fi export BACKENDS=$1 - ./tests/scripts/test-driver.sh \ + $INSTALL_PATH/test/onert-test unittest \ --reportdir=$ROOT_PATH/$3 \ - --unittest \ - . + --unittestdir=$INSTALL_PATH/unittest unset BACKENDS # TODO Pass skiplist to test-driver.sh @@ -115,7 +113,7 @@ function NNPackageTest() do for entry in "nnpkg-tcs"/$f; do if [ -e $entry ]; then - BACKENDS="$1" tests/scripts/nnpkg_test.sh -d -i nnpkg-tcs $(basename "$entry") + BACKENDS="$1" $INSTALL_PATH/test/onert-test nnpkg-test -d -i nnpkg-tcs $(basename "$entry") fi done EXITCODE_F=$? @@ -144,16 +142,11 @@ function TFLiteLoaderTest() export BACKENDS=$1 if [[ "$2" == "" ]]; then - ./tests/scripts/test-driver.sh \ - --frameworktest \ - --framework_driverbin="$ROOT_PATH/Product/out/bin/tflite_loader_test_tool" \ + $INSTALL_PATH/test/onert-test verify-tflite --api=loader \ --reportdir=$ROOT_PATH/$3 - . else - ./tests/scripts/test-driver.sh \ - --frameworktest \ - --framework_driverbin="$ROOT_PATH/Product/out/bin/tflite_loader_test_tool" \ - --frameworktest_list_file=tests/scripts/list/tflite_loader_list.${TEST_ARCH}.txt \ + $INSTALL_PATH/test/onert-test verify-tflite --api=loader \ + --list=$2 \ --reportdir=$ROOT_PATH/$3 fi unset BACKENDS diff --git a/infra/scripts/compiler_modules.sh b/infra/scripts/compiler_modules.sh index d436e8a1f..a0323e0a0 100644 --- a/infra/scripts/compiler_modules.sh +++ b/infra/scripts/compiler_modules.sh @@ -7,7 +7,7 @@ DEBUG_BUILD_ITEMS="angkor;cwrap;pepper-str;pepper-strcast;pp;stdex" DEBUG_BUILD_ITEMS+=";oops;pepper-assert" DEBUG_BUILD_ITEMS+=";hermes;hermes-std" DEBUG_BUILD_ITEMS+=";loco;locop;locomotiv;logo-core;logo" -DEBUG_BUILD_ITEMS+=";foder;souschef;arser" +DEBUG_BUILD_ITEMS+=";foder;souschef;arser;vconone" DEBUG_BUILD_ITEMS+=";safemain;mio-circle;mio-tflite" DEBUG_BUILD_ITEMS+=";tflite2circle" DEBUG_BUILD_ITEMS+=";luci" diff --git a/infra/scripts/docker_build_cross_aarch64_runtime.sh b/infra/scripts/docker_build_cross_aarch64_runtime.sh index 7da673601..011d14c18 100755 --- a/infra/scripts/docker_build_cross_aarch64_runtime.sh +++ b/infra/scripts/docker_build_cross_aarch64_runtime.sh @@ -6,7 +6,7 @@ CURRENT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_PATH="$CURRENT_PATH/../../" # prepare rootfs -if [ ! -d $ROOTFS_DIR ]; then +if [ -z "$ROOTFS_DIR" ] || [ ! -d $ROOTFS_DIR ]; then echo "It will use default rootfs path" else DOCKER_VOLUMES+=" -v $ROOTFS_DIR:/opt/rootfs" diff --git a/infra/scripts/docker_build_cross_arm_runtime.sh b/infra/scripts/docker_build_cross_arm_runtime.sh index f1f666aa3..551fb5700 100755 --- a/infra/scripts/docker_build_cross_arm_runtime.sh +++ b/infra/scripts/docker_build_cross_arm_runtime.sh @@ -6,7 +6,7 @@ CURRENT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_PATH="$CURRENT_PATH/../../" # prepare rootfs -if [ ! -d $ROOTFS_DIR ]; then +if [ -z "$ROOTFS_DIR" ] || [ ! -d $ROOTFS_DIR ]; then echo "It will use default rootfs path" else DOCKER_VOLUMES+=" -v $ROOTFS_DIR:/opt/rootfs" diff --git a/infra/scripts/docker_build_cross_arm_runtime_release.sh b/infra/scripts/docker_build_cross_arm_runtime_release.sh index ea66f1774..876f318f4 100755 --- a/infra/scripts/docker_build_cross_arm_runtime_release.sh +++ b/infra/scripts/docker_build_cross_arm_runtime_release.sh @@ -6,7 +6,7 @@ CURRENT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_PATH="$CURRENT_PATH/../../" # prepare rootfs -if [ ! -d $ROOTFS_DIR ]; then +if [ -z "$ROOTFS_DIR" ] || [ ! -d $ROOTFS_DIR ]; then echo "It will use default rootfs path" else DOCKER_VOLUMES+=" -v $ROOTFS_DIR:/opt/rootfs" diff --git a/infra/scripts/docker_build_cross_coverage.sh b/infra/scripts/docker_build_cross_coverage.sh index 08244e5d8..f42251baa 100755 --- a/infra/scripts/docker_build_cross_coverage.sh +++ b/infra/scripts/docker_build_cross_coverage.sh @@ -6,7 +6,7 @@ CURRENT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_PATH="$CURRENT_PATH/../../" # prepare rootfs -if [ ! -d $ROOTFS_DIR ]; then +if [ -z "$ROOTFS_DIR" ] || [ ! -d $ROOTFS_DIR ]; then echo "It will use default rootfs path" else DOCKER_VOLUMES+=" -v $ROOTFS_DIR:/opt/rootfs" diff --git a/infra/scripts/docker_build_nncc.sh b/infra/scripts/docker_build_nncc.sh index 418b50dfe..046bc8a4c 100755 --- a/infra/scripts/docker_build_nncc.sh +++ b/infra/scripts/docker_build_nncc.sh @@ -54,7 +54,18 @@ pushd $ROOT_PATH > /dev/null mkdir -p ${NNCC_INSTALL_PREFIX} ./nncc docker-run ./nnas create-package --prefix "${PWD}/${NNCC_INSTALL_PREFIX}" -- "${CONFIG_OPTIONS}" +# create python virtual environment +./nncc docker-run python3 -m venv "${NNCC_INSTALL_PREFIX}/bin/venv" + +./nncc docker-run "${NNCC_INSTALL_PREFIX}/bin/venv/bin/python" \ + -m pip --default-timeout=1000 --trusted-host pypi.org --trusted-host files.pythonhost.org \ + install -U pip setuptools +./nncc docker-run "${NNCC_INSTALL_PREFIX}/bin/venv/bin/python" \ + -m pip --default-timeout=1000 --trusted-host pypi.org --trusted-host files.pythonhost.org \ + install tensorflow-cpu==2.3.0 + mkdir -p ${ARCHIVE_PATH} -tar -zcf ${ARCHIVE_PATH}/nncc-package.tar.gz -C ${NNCC_INSTALL_PREFIX} ./ +tar -zcf ${ARCHIVE_PATH}/nncc-package.tar.gz -C ${NNCC_INSTALL_PREFIX} --exclude "bin/venv" ./ +tar -zcf ${ARCHIVE_PATH}/nncc-venv-package.tar.gz -C ${NNCC_INSTALL_PREFIX} bin/venv popd > /dev/null diff --git a/infra/scripts/docker_build_tizen_cross.sh b/infra/scripts/docker_build_tizen_cross.sh index 18809ad07..ee0f183f1 100755 --- a/infra/scripts/docker_build_tizen_cross.sh +++ b/infra/scripts/docker_build_tizen_cross.sh @@ -6,7 +6,7 @@ CURRENT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_PATH="$CURRENT_PATH/../../" # prepare rootfs -if [ ! -d $ROOTFS_DIR ]; then +if [ -z "$ROOTFS_DIR" ] || [ ! -d $ROOTFS_DIR ]; then echo "It will use default rootfs path" else DOCKER_VOLUMES+=" -v $ROOTFS_DIR:/opt/rootfs" diff --git a/infra/scripts/docker_collect_nnpkg_resources.sh b/infra/scripts/docker_collect_nnpkg_resources.sh index 556c5bd74..55adaa15d 100755 --- a/infra/scripts/docker_collect_nnpkg_resources.sh +++ b/infra/scripts/docker_collect_nnpkg_resources.sh @@ -60,7 +60,7 @@ pushd $ROOT_PATH > /dev/null REQUIRED_UNITS=() # Common Libraries REQUIRED_UNITS+=("angkor" "cwrap" "pepper-str" "pepper-strcast" "pp" "stdex") -REQUIRED_UNITS+=("oops" "safemain" "foder" "arser" "oops") +REQUIRED_UNITS+=("oops" "safemain" "foder" "arser" "vconone") # Hermes Logging Framework REQUIRED_UNITS+=("hermes" "hermes-std") # loco IR and related utilities diff --git a/infra/scripts/test_arm_nnpkg.sh b/infra/scripts/test_arm_nnpkg.sh index 68adaf913..d00eb730f 100755 --- a/infra/scripts/test_arm_nnpkg.sh +++ b/infra/scripts/test_arm_nnpkg.sh @@ -7,10 +7,10 @@ BACKENDS=("acl_cl" "acl_neon" "cpu") for BACKEND in "${BACKENDS[@]}"; do - NNPackageTest ${BACKEND} "tests/scripts/list/nnpkg_test_list.armv7l-linux.${BACKEND}" + NNPackageTest ${BACKEND} "Product/out/test/list/nnpkg_test_list.armv7l-linux.${BACKEND}" done # Interpreter test export DISABLE_COMPILE=1 -NNPackageTest "interp" "tests/scripts/list/nnpkg_test_list.noarch.interp" +NNPackageTest "interp" "Product/out/test/list/nnpkg_test_list.noarch.interp" unset DISABLE_COMPILE diff --git a/infra/scripts/test_coverage.sh b/infra/scripts/test_coverage.sh index c3dc04821..12a9942ab 100755 --- a/infra/scripts/test_coverage.sh +++ b/infra/scripts/test_coverage.sh @@ -32,7 +32,7 @@ export GCOV_PREFIX_STRIP=`cat $ROOT_PATH/tests/scripts/build_path_depth.txt` TENSOR_LOGGING=trace_log.txt ONERT_LOG_ENABLE=1 GRAPH_DOT_DUMP=1 ./infra/scripts/test_ubuntu_runtime_mixed.sh # Enable trace event (acl_cl default backend) export TRACE_FILEPATH=trace.json -TFLiteModelVerification "acl_cl" "tests/scripts/list/frameworktest_list.armv7l.acl_cl.txt" "report/acl_cl/trace" +TFLiteModelVerification "acl_cl" "Product/out/test/list/frameworktest_list.armv7l.acl_cl.txt" "report/acl_cl/trace" unset TRACE_FILEPATH # Interpreter diff --git a/infra/scripts/test_ubuntu_runtime.sh b/infra/scripts/test_ubuntu_runtime.sh index 76e567a29..f250df5a0 100755 --- a/infra/scripts/test_ubuntu_runtime.sh +++ b/infra/scripts/test_ubuntu_runtime.sh @@ -68,7 +68,7 @@ else fi UNITTEST_SKIPLIST="Product/out/unittest/nnapi_gtest.skip.${TEST_PLATFORM}.${BACKEND}" -FRAMEWORK_TESTLIST="tests/scripts/list/frameworktest_list.${TEST_ARCH}.${BACKEND}.txt" +FRAMEWORK_TESTLIST="Product/out/test/list/frameworktest_list.${TEST_ARCH}.${BACKEND}.txt" REPORT_BASE="report/${BACKEND}" EXECUTORS=("Linear" "Dataflow" "Parallel") @@ -91,7 +91,7 @@ do export EXECUTOR="${EXECUTOR}" fi - Unittests "${BACKEND}" "${UNITTEST_SKIPLIST}" "${REPORT_PATH}" + NNAPIGTest "${BACKEND}" "${UNITTEST_SKIPLIST}" "${REPORT_PATH}" TFLiteModelVerification "${BACKEND}" "${FRAMEWORK_TESTLIST}" "${REPORT_PATH}" if [ $EXECUTOR = "Interpreter" ]; then @@ -103,12 +103,7 @@ done # Current support acl_cl backend testlist only # TODO Support more backends -TFLITE_LOADER_TESTLIST="tests/scripts/list/tflite_loader_list.${TEST_ARCH}.txt" +TFLITE_LOADER_TESTLIST="Product/out/test/list/tflite_loader_list.${TEST_ARCH}.txt" if [[ $TFLITE_LOADER = "1" ]]; then TFLiteLoaderTest "${BACKEND}" "${TFLITE_LOADER_TESTLIST}" "${REPORT_BASE}/loader/${EXECUTOR}" - - # Test custom op - pushd ${ROOT_PATH} > /dev/null - ./Product/out/tests/FillFrom_runner - popd > /dev/null fi diff --git a/infra/scripts/test_ubuntu_runtime_mixed.sh b/infra/scripts/test_ubuntu_runtime_mixed.sh index 265a2acff..24fde8896 100755 --- a/infra/scripts/test_ubuntu_runtime_mixed.sh +++ b/infra/scripts/test_ubuntu_runtime_mixed.sh @@ -14,32 +14,26 @@ TEST_OS="linux" # This test requires test model installation pushd ${ROOT_PATH} > /dev/null -echo -echo "==== Run nnfw_api_gtest begin ====" -echo -NNFW_API_TEST_MODEL_INSTALLER=tests/scripts/nnfw_api_gtest/install_nnfw_api_gtest_nnpackages.sh -TEST_BIN=Product/out/unittest_standalone/nnfw_api_gtest -$NNFW_API_TEST_MODEL_INSTALLER --install-dir ${TEST_BIN}_models -${TEST_BIN} -echo -echo "==== Run nnfw_api_gtest end ====" -echo +echo "" +echo "==== Run standalone unittest begin ====" +echo "" +Product/out/test/onert-test prepare-model --model=nnpackage +Product/out/test/onert-test unittest --unittestdir=Product/out/unittest_standalone +echo "" +echo "==== Run standalone unittest end ====" +echo "" + +# Test custom op +pushd ${ROOT_PATH} > /dev/null +./Product/out/test/FillFrom_runner popd > /dev/null -Product/out/unittest_standalone/test_compute -Product/out/unittest_standalone/test_onert -Product/out/unittest_standalone/test_onert_backend_cpu_common -Product/out/unittest_standalone/test_onert_frontend_nnapi -Product/out/unittest_standalone/tflite_test - -pushd ${ROOT_PATH} - # NOTE Fixed backend assignment by type of operation # TODO Enhance this with randomized test BACKENDS=(acl_cl acl_neon cpu) # Get the intersect of framework test list files -TESTLIST_PREFIX="tests/scripts/list/frameworktest_list.${TEST_ARCH}" +TESTLIST_PREFIX="Product/out/test/list/frameworktest_list.${TEST_ARCH}" SKIPLIST_PREFIX="Product/out/unittest/nnapi_gtest.skip.${TEST_ARCH}-${TEST_OS}" sort $TESTLIST_PREFIX.${BACKENDS[0]}.txt > $TESTLIST_PREFIX.intersect.txt sort $SKIPLIST_PREFIX.${BACKENDS[0]} > $SKIPLIST_PREFIX.union @@ -65,5 +59,5 @@ export OP_BACKEND_Conv2D="cpu" export OP_BACKEND_MaxPool2D="acl_cl" export OP_BACKEND_AvgPool2D="acl_neon" export ACL_LAYOUT="NCHW" -Unittests "acl_cl;acl_neon;cpu" "Product/out/unittest/nnapi_gtest.skip.${TEST_ARCH}-${TEST_OS}.union" "report/mixed" +NNAPIGTest "acl_cl;acl_neon;cpu" "Product/out/unittest/nnapi_gtest.skip.${TEST_ARCH}-${TEST_OS}.union" "report/mixed" TFLiteModelVerification "acl_cl;acl_neon;cpu" "${TESTLIST_PREFIX}.intersect.txt" "report/mixed" diff --git a/infra/scripts/tizen_xu4_test.sh b/infra/scripts/tizen_xu4_test.sh index 5521b5fdc..8f9e86fb0 100755 --- a/infra/scripts/tizen_xu4_test.sh +++ b/infra/scripts/tizen_xu4_test.sh @@ -23,19 +23,21 @@ function install_model() { # download tflite model files pushd $HOST_HOME - tests/scripts/framework/run_test.sh --download=on + tests/scripts/models/run_test.sh --download=on --run=off # TODO Since this command removes model file(.zip), # We must always download the file unlike model file(.tflite). # Because caching applies only to tflite file. find tests -name "*.zip" -exec rm {} \; - tar -zcf cache.tar.gz tests/scripts/framework/cache + tar -zcf cache.tar.gz -C tests/scripts/models cache $SDB_CMD push cache.tar.gz $TEST_ROOT/. rm -rf cache.tar.gz - $SDB_CMD shell tar -zxf $TEST_ROOT/cache.tar.gz -C $TEST_ROOT + $SDB_CMD shell tar -zxf $TEST_ROOT/cache.tar.gz -C $TEST_ROOT/Product/out/test/models # download api test model file for nnfw_api_gtest MODEL_CACHE_DIR=$(mktemp -d) - tests/scripts/nnfw_api_gtest/install_nnfw_api_gtest_nnpackages.sh --install-dir $MODEL_CACHE_DIR + tests/scripts/models/run_test.sh --download=on --run=off \ + --configdir=test/scripts/nnfw_api_gtest/models \ + --cachedir=$MODEL_CACHE_DIR tar -zcf $MODEL_CACHE_DIR/api_model_test.tar.gz -C $MODEL_CACHE_DIR . $SDB_CMD push $MODEL_CACHE_DIR/api_model_test.tar.gz $TEST_ROOT/Product/out/unittest_standalone/nnfw_api_gtest_models/ $SDB_CMD shell tar -zxf $TEST_ROOT/Product/out/unittest_standalone/nnfw_api_gtest_models/api_model_test.tar.gz \ @@ -157,7 +159,7 @@ else rm -rf ${GCOV_DIR}/* pushd ${GCOV_DIR} - sdb pull ${TEST_ROOT}/tests/scripts/build_path.txt + sdb pull ${TEST_ROOT}/Product/out/test/build_path.txt SRC_PREFIX=`cat build_path.txt` GCOV_PREFIX_STRIP=`echo "${SRC_PREFIX}" | grep -o '/' | wc -l` GCOV_DATA_PATH="/opt/usr/nnfw-gcov" |