diff options
Diffstat (limited to 'tests/tools')
36 files changed, 806 insertions, 2346 deletions
diff --git a/tests/tools/nnapi_test/CMakeLists.txt b/tests/tools/nnapi_test/CMakeLists.txt index eac649b15..b52f4f34b 100644 --- a/tests/tools/nnapi_test/CMakeLists.txt +++ b/tests/tools/nnapi_test/CMakeLists.txt @@ -1,14 +1,5 @@ -if(NOT BUILD_NNAPI_TEST) - return() -endif(NOT BUILD_NNAPI_TEST) - list(APPEND SOURCES "src/nnapi_test.cc") -list(APPEND SOURCES "src/args.cc") - -nnfw_find_package(Boost REQUIRED program_options) add_executable(nnapi_test ${SOURCES}) -target_include_directories(nnapi_test PRIVATE ${Boost_INCLUDE_DIRS}) target_link_libraries(nnapi_test nnfw_lib_tflite) -target_link_libraries(nnapi_test ${Boost_PROGRAM_OPTIONS_LIBRARY}) install(TARGETS nnapi_test DESTINATION bin) diff --git a/tests/tools/nnapi_test/src/args.cc b/tests/tools/nnapi_test/src/args.cc deleted file mode 100644 index 420e092c0..000000000 --- a/tests/tools/nnapi_test/src/args.cc +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "args.h" - -#include <iostream> - -namespace nnapi_test -{ - -Args::Args(const int argc, char **argv) -{ - Initialize(); - try - { - Parse(argc, argv); - } - catch (const std::exception &e) - { - std::cerr << "The argments that cannot be parsed: " << e.what() << '\n'; - print(argv); - exit(255); - } -} - -void Args::print(char **argv) -{ - std::cout << "nnapi_test\n\n"; - std::cout << "Usage: " << argv[0] << " <.tflite> [<options>]\n\n"; - std::cout << _options; - std::cout << "\n"; -} - -void Args::Initialize(void) -{ - // General options - po::options_description general("General options", 100); - - // clang-format off - general.add_options() - ("help,h", "Print available options") - ("tflite", po::value<std::string>()->required()) - ("seed", po::value<int>()->default_value(0), "The seed of random inputs") - ("num_runs", po::value<int>()->default_value(2), "The number of runs") - ; - // clang-format on - - _options.add(general); - _positional.add("tflite", 1); - _positional.add("seed", 2); -} - -void Args::Parse(const int argc, char **argv) -{ - po::variables_map vm; - po::store(po::command_line_parser(argc, argv).options(_options).positional(_positional).run(), - vm); - - if (vm.count("help")) - { - print(argv); - - exit(0); - } - - po::notify(vm); - if (vm.count("tflite")) - { - _tflite_filename = vm["tflite"].as<std::string>(); - - if (_tflite_filename.empty()) - { - std::cerr << "Please specify tflite file.\n"; - print(argv); - exit(255); - } - else - { - if (access(_tflite_filename.c_str(), F_OK) == -1) - { - std::cerr << "tflite file not found: " << _tflite_filename << "\n"; - exit(255); - } - } - } - - if (vm.count("seed")) - { - _seed = vm["seed"].as<int>(); - } - - if (vm.count("num_runs")) - { - _num_runs = vm["num_runs"].as<int>(); - if (_num_runs < 0) - { - std::cerr << "num_runs value must be greater than 0.\n"; - exit(255); - } - } -} - -} // end of namespace nnapi_test diff --git a/tests/tools/nnapi_test/src/args.h b/tests/tools/nnapi_test/src/args.h deleted file mode 100644 index 486fbefd5..000000000 --- a/tests/tools/nnapi_test/src/args.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __NNAPI_TEST_ARGS_H__ -#define __NNAPI_TEST_ARGS_H__ - -#include <boost/program_options.hpp> -#include <string> - -namespace po = boost::program_options; - -namespace nnapi_test -{ - -class Args -{ -public: - Args(const int argc, char **argv); - void print(char **argv); - - const std::string &getTfliteFilename(void) const { return _tflite_filename; } - const int getSeed(void) const { return _seed; } - const int getNumRuns(void) const { return _num_runs; } - -private: - void Initialize(); - void Parse(const int argc, char **argv); - -private: - po::positional_options_description _positional; - po::options_description _options; - - std::string _tflite_filename; - int _seed; - int _num_runs; -}; - -} // end of namespace nnapi_test - -#endif // __NNAPI_TEST_ARGS_H__ diff --git a/tests/tools/nnapi_test/src/nnapi_test.cc b/tests/tools/nnapi_test/src/nnapi_test.cc index 921d0dc42..799188f66 100644 --- a/tests/tools/nnapi_test/src/nnapi_test.cc +++ b/tests/tools/nnapi_test/src/nnapi_test.cc @@ -18,42 +18,34 @@ #include "tensorflow/lite/model.h" #include "tflite/interp/FlatBufferBuilder.h" -#include "tflite/RandomTestRunner.h" +#include "tflite/Diff.h" #include <iostream> #include <stdexcept> -#include "args.h" - using namespace tflite; using namespace nnfw::tflite; -using namespace nnapi_test; int main(const int argc, char **argv) { - Args args(argc, argv); + if (argc < 2) + { + std::cerr << "nnapi_test\n\n"; + std::cerr << "Usage: " << argv[0] << " <.tflite>\n\n"; + return 1; + } - const auto filename = args.getTfliteFilename(); + const auto filename = argv[1]; StderrReporter error_reporter; - auto model = FlatBufferModel::BuildFromFile(filename.c_str(), &error_reporter); - - if (model == nullptr) - { - // error_reporter must have shown the error message already - return 1; - } + auto model = FlatBufferModel::BuildFromFile(filename, &error_reporter); const nnfw::tflite::FlatBufferBuilder builder(*model); try { - const auto seed = static_cast<uint32_t>(args.getSeed()); - auto runner = nnfw::tflite::RandomTestRunner::make(seed); - const auto num_runs = static_cast<size_t>(args.getNumRuns()); - runner.compile(builder); - return runner.run(num_runs); + return RandomTestRunner::make(0).run(builder); } catch (const std::exception &e) { diff --git a/tests/tools/nnpackage_run/CMakeLists.txt b/tests/tools/nnpackage_run/CMakeLists.txt index afbcbeaca..5fd7136cc 100644 --- a/tests/tools/nnpackage_run/CMakeLists.txt +++ b/tests/tools/nnpackage_run/CMakeLists.txt @@ -2,44 +2,31 @@ if(NOT BUILD_NNPACKAGE_RUN) return() endif(NOT BUILD_NNPACKAGE_RUN) -if(NOT BUILD_ONERT) +if(NOT BUILD_NEURUN) return() -endif(NOT BUILD_ONERT) +endif(NOT BUILD_NEURUN) + +nnfw_find_package(HDF5 QUIET) +if(NOT HDF5_FOUND) + message(WARNING "HDF5 NOT found. Install libhdf5-dev to build nnpackage_run.") + return() +endif(NOT HDF5_FOUND) list(APPEND NNPACKAGE_RUN_SRCS "src/nnpackage_run.cc") list(APPEND NNPACKAGE_RUN_SRCS "src/args.cc") -list(APPEND NNPACKAGE_RUN_SRCS "src/nnfw_util.cc") -list(APPEND NNPACKAGE_RUN_SRCS "src/randomgen.cc") +list(APPEND NNPACKAGE_RUN_SRCS "src/tensor_dumper.cc") -nnfw_find_package(Boost REQUIRED program_options) -nnfw_find_package(Ruy QUIET) -nnfw_find_package(HDF5 QUIET) - -if (HDF5_FOUND) - list(APPEND NNPACKAGE_RUN_SRCS "src/h5formatter.cc") -endif() +nnfw_find_package(Boost REQUIRED) add_executable(nnpackage_run ${NNPACKAGE_RUN_SRCS}) - -if (HDF5_FOUND) - target_compile_definitions(nnpackage_run PRIVATE ONERT_HAVE_HDF5=1) - target_include_directories(nnpackage_run PRIVATE ${HDF5_INCLUDE_DIRS}) - target_link_libraries(nnpackage_run ${HDF5_CXX_LIBRARIES}) -else() - message(WARNING "HDF5 NOT found. Install libhdf5-dev or set EXT_HDF5_DIR to support load/dump in nnpackage_run.") -endif(HDF5_FOUND) - target_include_directories(nnpackage_run PRIVATE src) target_include_directories(nnpackage_run PRIVATE ${Boost_INCLUDE_DIRS}) +target_include_directories(nnpackage_run PRIVATE ${HDF5_INCLUDE_DIRS}) -target_link_libraries(nnpackage_run tflite_loader) -target_link_libraries(nnpackage_run nnfw_lib_tflite jsoncpp) +target_link_libraries(nnpackage_run neurun_core neurun tflite_loader) +target_link_libraries(nnpackage_run tensorflow-lite ${LIB_PTHREAD} dl nnfw_lib_tflite) target_link_libraries(nnpackage_run nnfw-dev) -target_link_libraries(nnpackage_run ${Boost_PROGRAM_OPTIONS_LIBRARY}) -target_link_libraries(nnpackage_run nnfw_lib_benchmark) -if(Ruy_FOUND AND PROFILE_RUY) - target_link_libraries(nnpackage_run ruy_instrumentation) - target_link_libraries(nnpackage_run ruy_profiler) -endif(Ruy_FOUND AND PROFILE_RUY) +target_link_libraries(nnpackage_run boost_program_options boost_system boost_filesystem) +target_link_libraries(nnpackage_run ${HDF5_LIBRARIES}) install(TARGETS nnpackage_run DESTINATION bin) diff --git a/tests/tools/nnpackage_run/README.md b/tests/tools/nnpackage_run/README.md index 898cc84cf..c7167b2d4 100644 --- a/tests/tools/nnpackage_run/README.md +++ b/tests/tools/nnpackage_run/README.md @@ -17,6 +17,6 @@ $ ./nnpackage_run path_to_nnpackage_directory Output would look like: ``` -nnfw_prepare takes 425.235 ms -nnfw_run takes 2.525 ms +nnfw_prepare takes 425.235 sec +nnfw_run takes 2.525 sec ``` diff --git a/tests/tools/nnpackage_run/src/allocation.h b/tests/tools/nnpackage_run/src/allocation.h deleted file mode 100644 index ea4672f9a..000000000 --- a/tests/tools/nnpackage_run/src/allocation.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __NNPACKAGE_RUN_ALLOCATION_H__ -#define __NNPACKAGE_RUN_ALLOCATION_H__ - -#include <cstdlib> -#include <cstdint> - -namespace nnpkg_run -{ -class Allocation -{ -public: - Allocation() : data_(nullptr) {} - ~Allocation() { free(data_); } - void *data() const { return data_; } - void *alloc(uint64_t sz) { return data_ = malloc(sz); } -private: - void *data_; -}; -} // end of namespace - -#endif // __NNPACKAGE_RUN_ALLOCATION_H__ diff --git a/tests/tools/nnpackage_run/src/args.cc b/tests/tools/nnpackage_run/src/args.cc index c0f937797..a7593fabb 100644 --- a/tests/tools/nnpackage_run/src/args.cc +++ b/tests/tools/nnpackage_run/src/args.cc @@ -16,89 +16,13 @@ #include "args.h" -#include <functional> #include <iostream> -#include <json/json.h> +#include <boost/filesystem.hpp> -namespace +namespace NNPackageRun { -// This function parses a json object and returns as a vector of integers -// For example, -// [0, [1, 2, 3, 4], 3, 40, 4, []] in JSON -// is converted to: -// { -// 0 -> [1, 2, 3, 4] -// 3 -> 40 -// 4 -> [] -// } in std::unordered_map. Note that the value type is still Json::Value. -std::unordered_map<uint32_t, Json::Value> argArrayToMap(const Json::Value &jsonval) -{ - if (!jsonval.isArray() || (jsonval.size() % 2 != 0)) - { - std::cerr << "JSON argument must be an even-sized array in JSON\n"; - exit(1); - } - - std::unordered_map<uint32_t, Json::Value> ret; - for (uint32_t i = 0; i < jsonval.size(); i += 2) - { - if (!jsonval[i].isUInt()) - { - std::cerr << "Key values(values in even indices) must be unsigned integers\n"; - exit(1); - } - uint32_t key = jsonval[i].asUInt(); - Json::Value val = jsonval[i + 1]; - ret[key] = jsonval[i + 1]; - } - return ret; -} - -// param shape_str is a form of, e.g., "[1, [2, 3], 3, []]" or "h5" -void handleShapeJsonParam(nnpkg_run::TensorShapeMap &shape_map, const std::string &shape_str) -{ - Json::Value root; - Json::Reader reader; - if (!reader.parse(shape_str, root, false)) - { - std::cerr << "Invalid JSON format for output_sizes \"" << shape_str << "\"\n"; - exit(1); - } - - auto arg_map = argArrayToMap(root); - for (auto &pair : arg_map) - { - uint32_t key = pair.first; - Json::Value &shape_json = pair.second; - if (!shape_json.isArray()) - { - std::cerr << "All the values must be list: " << shape_str << "\n"; - exit(1); - } - - std::vector<int> shape; - for (auto &dim_json : shape_json) - { - if (!dim_json.isUInt()) - { - std::cerr << "All the dims should be dim >= 0: " << shape_str << "\n"; - exit(1); - } - - shape.emplace_back(dim_json.asUInt64()); - } - - shape_map[key] = shape; - } -} - -} // namespace - -namespace nnpkg_run -{ - -Args::Args(const int argc, char **argv) +Args::Args(const int argc, char **argv) noexcept { Initialize(); Parse(argc, argv); @@ -106,135 +30,15 @@ Args::Args(const int argc, char **argv) void Args::Initialize(void) { - auto process_nnpackage = [&](const std::string &package_filename) { - _package_filename = package_filename; - - std::cerr << "Package Filename " << _package_filename << std::endl; - if (_package_filename.empty()) - { - // TODO Print usage instead of the below message - std::cerr << "Please specify nnpackage file. Run with `--help` for usage." - << "\n"; - - exit(1); - } - else - { - if (access(_package_filename.c_str(), F_OK) == -1) - { - std::cerr << "nnpackage not found: " << _package_filename << "\n"; - } - } - }; - - auto process_output_sizes = [&](const std::string &output_sizes_json_str) { - Json::Value root; - Json::Reader reader; - if (!reader.parse(output_sizes_json_str, root, false)) - { - std::cerr << "Invalid JSON format for output_sizes \"" << output_sizes_json_str << "\"\n"; - exit(1); - } - - auto arg_map = argArrayToMap(root); - for (auto &pair : arg_map) - { - uint32_t key = pair.first; - Json::Value &val_json = pair.second; - if (!val_json.isUInt()) - { - std::cerr << "All the values in `output_sizes` must be unsigned integers\n"; - exit(1); - } - uint32_t val = val_json.asUInt(); - _output_sizes[key] = val; - } - }; - - auto process_shape_prepare = [&](const std::string &shape_str) { -#if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1 - if (shape_str == "H5" || shape_str == "h5") - { - _when_to_use_h5_shape = WhenToUseH5Shape::PREPARE; - return; - } -#endif - try - { - handleShapeJsonParam(_shape_prepare, shape_str); - } - catch (const std::exception &e) - { - std::cerr << "error with '--shape_prepare' option: " << shape_str << std::endl; - exit(1); - } - }; - - auto process_shape_run = [&](const std::string &shape_str) { -#if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1 - if (shape_str == "H5" || shape_str == "h5") - { - _when_to_use_h5_shape = WhenToUseH5Shape::RUN; - return; - } -#endif - try - { - handleShapeJsonParam(_shape_run, shape_str); - } - catch (const std::exception &e) - { - std::cerr << "error with '--shape_run' option: " << shape_str << std::endl; - exit(1); - } - }; - // General options - po::options_description general("General options", 100); + po::options_description general("General options"); // clang-format off general.add_options() - ("help,h", "Print available options") - ("version", "Print version and exit immediately") - ("nnpackage", po::value<std::string>()->required()->notifier(process_nnpackage)) -#if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1 - ("dump,d", po::value<std::string>()->default_value("")->notifier([&](const auto &v) { _dump_filename = v; }), "Output filename") - ("load,l", po::value<std::string>()->default_value("")->notifier([&](const auto &v) { _load_filename = v; }), "Input filename") -#endif - ("output_sizes", po::value<std::string>()->notifier(process_output_sizes), - "The output buffer size in JSON 1D array\n" - "If not given, the model's output sizes are used\n" - "e.g. '[0, 40, 2, 80]' to set 0th tensor to 40 and 2nd tensor to 80.\n") - ("num_runs,r", po::value<int>()->default_value(1)->notifier([&](const auto &v) { _num_runs = v; }), "The number of runs") - ("warmup_runs,w", po::value<int>()->default_value(0)->notifier([&](const auto &v) { _warmup_runs = v; }), "The number of warmup runs") - ("run_delay,t", po::value<int>()->default_value(-1)->notifier([&](const auto &v) { _run_delay = v; }), "Delay time(ms) between runs (as default no delay") - ("gpumem_poll,g", po::value<bool>()->default_value(false)->notifier([&](const auto &v) { _gpumem_poll = v; }), "Check gpu memory polling separately") - ("mem_poll,m", po::value<bool>()->default_value(false)->notifier([&](const auto &v) { _mem_poll = v; }), "Check memory polling") - ("write_report,p", po::value<bool>()->default_value(false)->notifier([&](const auto &v) { _write_report = v; }), - "Write report\n" - "{exec}-{nnpkg}-{backend}.csv will be generated.\n" - "e.g. nnpackage_run-UNIT_Add_000-acl_cl.csv.\n" - "{nnpkg} name may be changed to realpath if you use symbolic-link.") - ("shape_prepare", po::value<std::string>()->default_value("[]")->notifier(process_shape_prepare), - "Please refer to the description of 'shape_run'") - ("shape_run", po::value<std::string>()->default_value("[]")->notifier(process_shape_run), - "'--shape_prepare: set shape of tensors before compilation (before calling nnfw_prepare()).\n" - "'--shape_run: set shape of tensors before running (before calling nnfw_run()).\n" - "Allowed value:.\n" - "'[0, [1, 2], 2, []]': set 0th tensor to [1, 2] and 2nd tensor to [] (scalar).\n" -#if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1 - "'h5': read shape(s) from H5 input file. '--load' should also be provided.\n" - "if '--load' option is provided but '--shape_prepare' or '--shape_run' is not provided,\n" - "'--shape_run h5' will be used by default.\n" -#endif - "For detailed description, please consutl the description of nnfw_set_input_tensorinfo()\n" - ) - ("verbose_level,v", po::value<int>()->default_value(0)->notifier([&](const auto &v) { _verbose_level = v; }), - "Verbose level\n" - "0: prints the only result. Messages btw run don't print\n" - "1: prints result and message btw run\n" - "2: prints all of messages to print\n") - ; + ("help,h", "Display available options") + ("nnpackage", po::value<std::string>()->required()) + ("dump,d", po::value<std::string>()->default_value(""), "Output filename") + ("load,l", po::value<std::string>()->default_value(""), "Input filename"); // clang-format on _options.add(general); @@ -255,11 +59,6 @@ void Args::Parse(const int argc, char **argv) "' cannot be given at once."); } }; - - // calling, e.g., "nnpackage_run .. -- shape_prepare .. --shape_run .." should theoretically - // work but allowing both options together on command line makes the usage and implemenation - // of nnpackage_run too complicated. Therefore let's not allow those option together. - conflicting_options("shape_prepare", "shape_run"); } if (vm.count("help")) @@ -272,45 +71,38 @@ void Args::Parse(const int argc, char **argv) exit(0); } - if (vm.count("version")) - { - _print_version = true; - return; - } + po::notify(vm); - try + if (vm.count("dump")) { - po::notify(vm); + _dump_filename = vm["dump"].as<std::string>(); } - catch (const std::bad_cast &e) + + if (vm.count("load")) { - std::cerr << "Bad cast error - " << e.what() << '\n'; - exit(1); + _load_filename = vm["load"].as<std::string>(); } - // This must be run after `notify` as `_warm_up_runs` must have been processed before. - if (vm.count("mem_poll")) + if (vm.count("nnpackage")) { - // Instead of EXECUTE to avoid overhead, memory polling runs on WARMUP - if (_mem_poll && _warmup_runs == 0) + _package_filename = vm["nnpackage"].as<std::string>(); + + if (_package_filename.empty()) { - _warmup_runs = 1; + // TODO Print usage instead of the below message + std::cerr << "Please specify nnpackage file. Run with `--help` for usage." + << "\n"; + + exit(1); + } + else + { + if (!boost::filesystem::exists(_package_filename)) + { + std::cerr << "nnpackage not found: " << _package_filename << "\n"; + } } } } -bool Args::shapeParamProvided() -{ - bool provided = false; -#if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1 - // "--shape_run h5" or "--shape_prepare h5" was provided - provided = (getWhenToUseH5Shape() != WhenToUseH5Shape::NOT_PROVIDED); -#endif - // specific shape was provided - // e.g., "--shape_run '[0, [10, 1]]'" or "--shape_prepare '[0, [10, 1]]'" - provided |= (!getShapeMapForPrepare().empty()) || (!getShapeMapForRun().empty()); - - return provided; -} - -} // end of namespace nnpkg_run +} // end of namespace NNPackageRun diff --git a/tests/tools/nnpackage_run/src/args.h b/tests/tools/nnpackage_run/src/args.h index 11fd00023..d064d77e7 100644 --- a/tests/tools/nnpackage_run/src/args.h +++ b/tests/tools/nnpackage_run/src/args.h @@ -18,53 +18,22 @@ #define __NNPACKAGE_RUN_ARGS_H__ #include <string> -#include <unordered_map> -#include <vector> #include <boost/program_options.hpp> -#include "types.h" - namespace po = boost::program_options; -namespace nnpkg_run +namespace NNPackageRun { -using TensorShapeMap = std::unordered_map<uint32_t, TensorShape>; - -#if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1 -enum class WhenToUseH5Shape -{ - NOT_PROVIDED, // Param not provided - PREPARE, // read shapes in h5 file and set them as inputs' shape before calling nnfw_prepare() - RUN, // read shapes in h5 file and set them as inputs' shape before calling nnfw_run() -}; -#endif - class Args { public: - Args(const int argc, char **argv); + Args(const int argc, char **argv) noexcept; void print(void); const std::string &getPackageFilename(void) const { return _package_filename; } -#if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1 const std::string &getDumpFilename(void) const { return _dump_filename; } const std::string &getLoadFilename(void) const { return _load_filename; } - WhenToUseH5Shape getWhenToUseH5Shape(void) const { return _when_to_use_h5_shape; } -#endif - const int getNumRuns(void) const { return _num_runs; } - const int getWarmupRuns(void) const { return _warmup_runs; } - const int getRunDelay(void) const { return _run_delay; } - std::unordered_map<uint32_t, uint32_t> getOutputSizes(void) const { return _output_sizes; } - const bool getGpuMemoryPoll(void) const { return _gpumem_poll; } - const bool getMemoryPoll(void) const { return _mem_poll; } - const bool getWriteReport(void) const { return _write_report; } - const bool printVersion(void) const { return _print_version; } - TensorShapeMap &getShapeMapForPrepare() { return _shape_prepare; } - TensorShapeMap &getShapeMapForRun() { return _shape_run; } - /// @brief Return true if "--shape_run" or "--shape_prepare" is provided - bool shapeParamProvided(); - const int getVerboseLevel(void) const { return _verbose_level; } private: void Initialize(); @@ -75,24 +44,10 @@ private: po::options_description _options; std::string _package_filename; -#if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1 std::string _dump_filename; std::string _load_filename; - WhenToUseH5Shape _when_to_use_h5_shape = WhenToUseH5Shape::NOT_PROVIDED; -#endif - TensorShapeMap _shape_prepare; - TensorShapeMap _shape_run; - int _num_runs; - int _warmup_runs; - int _run_delay; - std::unordered_map<uint32_t, uint32_t> _output_sizes; - bool _gpumem_poll; - bool _mem_poll; - bool _write_report; - bool _print_version = false; - int _verbose_level; }; -} // end of namespace nnpkg_run +} // end of namespace NNPackageRun #endif // __NNPACKAGE_RUN_ARGS_H__ diff --git a/tests/tools/nnpackage_run/src/h5formatter.cc b/tests/tools/nnpackage_run/src/h5formatter.cc deleted file mode 100644 index 3929c8d90..000000000 --- a/tests/tools/nnpackage_run/src/h5formatter.cc +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "h5formatter.h" -#include "nnfw.h" -#include "nnfw_util.h" - -#include <iostream> -#include <stdexcept> -#include <H5Cpp.h> - -namespace -{ -nnpkg_run::TensorShape getShape(H5::DataSet &data_set) -{ - std::vector<hsize_t> h5_shape; // hsize_t is unsigned long long - H5::DataSpace data_space = data_set.getSpace(); - int rank = data_space.getSimpleExtentNdims(); - h5_shape.resize(rank); - - // read shape info from H5 file - data_space.getSimpleExtentDims(h5_shape.data(), NULL); - - nnpkg_run::TensorShape shape; - for (auto dim : h5_shape) - shape.emplace_back(static_cast<int>(dim)); - - return shape; -} -} // namespace - -namespace nnpkg_run -{ -static const char *h5_value_grpname = "value"; - -std::vector<TensorShape> H5Formatter::readTensorShapes(const std::string &filename) -{ - uint32_t num_inputs; - NNPR_ENSURE_STATUS(nnfw_input_size(session_, &num_inputs)); - std::vector<TensorShape> tensor_shapes; - - try - { - H5::Exception::dontPrint(); - - H5::H5File file(filename, H5F_ACC_RDONLY); - H5::Group value_group = file.openGroup(h5_value_grpname); - - // Constraints: if there are n data set names, they should be unique and - // one of [ "0", "1", .. , "n-1" ] - for (uint32_t i = 0; i < num_inputs; ++i) - { - H5::DataSet data_set = value_group.openDataSet(std::to_string(i)); - H5::DataType type = data_set.getDataType(); - auto shape = getShape(data_set); - - tensor_shapes.emplace_back(shape); - } - - return tensor_shapes; - } - catch (const H5::Exception &e) - { - H5::Exception::printErrorStack(); - std::exit(-1); - } - catch (const std::exception &e) - { - std::cerr << e.what() << std::endl; - std::exit(-1); - } -} - -void H5Formatter::loadInputs(const std::string &filename, std::vector<Allocation> &inputs) -{ - uint32_t num_inputs; - NNPR_ENSURE_STATUS(nnfw_input_size(session_, &num_inputs)); - try - { - // Turn off the automatic error printing. - H5::Exception::dontPrint(); - - H5::H5File file(filename, H5F_ACC_RDONLY); - H5::Group value_group = file.openGroup(h5_value_grpname); - for (uint32_t i = 0; i < num_inputs; ++i) - { - nnfw_tensorinfo ti; - NNPR_ENSURE_STATUS(nnfw_input_tensorinfo(session_, i, &ti)); - - // TODO Add Assert(nnfw shape, h5 file shape size) - - // allocate memory for data - auto bufsz = bufsize_for(&ti); - inputs[i].alloc(bufsz); - - H5::DataSet data_set = value_group.openDataSet(std::to_string(i)); - H5::DataType type = data_set.getDataType(); - switch (ti.dtype) - { - case NNFW_TYPE_TENSOR_FLOAT32: - if (type == H5::PredType::IEEE_F32BE || type == H5::PredType::IEEE_F32LE) - data_set.read(inputs[i].data(), H5::PredType::NATIVE_FLOAT); - else - throw std::runtime_error("model input type is f32. But h5 data type is different."); - break; - case NNFW_TYPE_TENSOR_INT32: - if (type == H5::PredType::STD_I32BE || type == H5::PredType::STD_I32LE) - data_set.read(inputs[i].data(), H5::PredType::NATIVE_INT32); - else - throw std::runtime_error("model input type is i32. But h5 data type is different."); - break; - case NNFW_TYPE_TENSOR_INT64: - if (type == H5::PredType::STD_I64BE || type == H5::PredType::STD_I64LE) - data_set.read(inputs[i].data(), H5::PredType::NATIVE_INT64); - else - throw std::runtime_error("model input type is i64. But h5 data type is different."); - break; - case NNFW_TYPE_TENSOR_QUANT8_ASYMM: - case NNFW_TYPE_TENSOR_BOOL: - case NNFW_TYPE_TENSOR_UINT8: - if (type == H5::PredType::STD_U8BE || type == H5::PredType::STD_U8LE) - data_set.read(inputs[i].data(), H5::PredType::NATIVE_UINT8); - else - throw std::runtime_error( - "model input type is qasymm8, bool or uint8. But h5 data type is different."); - break; - default: - throw std::runtime_error("nnpkg_run can load f32, i32, qasymm8, bool and uint8."); - } - NNPR_ENSURE_STATUS(nnfw_set_input(session_, i, ti.dtype, inputs[i].data(), bufsz)); - NNPR_ENSURE_STATUS(nnfw_set_input_layout(session_, i, NNFW_LAYOUT_CHANNELS_LAST)); - } - } - catch (const H5::Exception &e) - { - H5::Exception::printErrorStack(); - std::exit(-1); - } - catch (const std::exception &e) - { - std::cerr << e.what() << std::endl; - std::exit(-1); - } -}; - -void H5Formatter::dumpOutputs(const std::string &filename, std::vector<Allocation> &outputs) -{ - uint32_t num_outputs; - NNPR_ENSURE_STATUS(nnfw_output_size(session_, &num_outputs)); - try - { - // Turn off the automatic error printing. - H5::Exception::dontPrint(); - - H5::H5File file(filename, H5F_ACC_TRUNC); - H5::Group value_group = file.createGroup(h5_value_grpname); - for (uint32_t i = 0; i < num_outputs; i++) - { - nnfw_tensorinfo ti; - NNPR_ENSURE_STATUS(nnfw_output_tensorinfo(session_, i, &ti)); - std::vector<hsize_t> dims(ti.rank); - for (uint32_t j = 0; j < ti.rank; ++j) - { - if (ti.dims[j] >= 0) - dims[j] = static_cast<hsize_t>(ti.dims[j]); - else - { - std::cerr << "Negative dimension in output tensor" << std::endl; - exit(-1); - } - } - H5::DataSpace data_space(ti.rank, dims.data()); - switch (ti.dtype) - { - case NNFW_TYPE_TENSOR_FLOAT32: - { - H5::DataSet data_set = - value_group.createDataSet(std::to_string(i), H5::PredType::IEEE_F32BE, data_space); - data_set.write(outputs[i].data(), H5::PredType::NATIVE_FLOAT); - break; - } - case NNFW_TYPE_TENSOR_INT32: - { - H5::DataSet data_set = - value_group.createDataSet(std::to_string(i), H5::PredType::STD_I32LE, data_space); - data_set.write(outputs[i].data(), H5::PredType::NATIVE_INT32); - break; - } - case NNFW_TYPE_TENSOR_INT64: - { - H5::DataSet data_set = - value_group.createDataSet(std::to_string(i), H5::PredType::STD_I64LE, data_space); - data_set.write(outputs[i].data(), H5::PredType::NATIVE_INT64); - break; - } - case NNFW_TYPE_TENSOR_UINT8: - case NNFW_TYPE_TENSOR_QUANT8_ASYMM: - { - H5::DataSet data_set = - value_group.createDataSet(std::to_string(i), H5::PredType::STD_U8BE, data_space); - data_set.write(outputs[i].data(), H5::PredType::NATIVE_UINT8); - break; - } - case NNFW_TYPE_TENSOR_BOOL: - { - H5::DataSet data_set = - value_group.createDataSet(std::to_string(i), H5::PredType::STD_U8LE, data_space); - data_set.write(outputs[i].data(), H5::PredType::NATIVE_INT8); - break; - } - default: - throw std::runtime_error("nnpkg_run can dump f32, i32, qasymm8, bool and uint8."); - } - } - } - catch (const H5::Exception &e) - { - H5::Exception::printErrorStack(); - std::exit(-1); - } - catch (const std::runtime_error &e) - { - std::cerr << "Error during dumpOutputs on nnpackage_run : " << e.what() << std::endl; - std::exit(-1); - } -}; - -} // end of namespace nnpkg_run diff --git a/tests/tools/nnpackage_run/src/h5formatter.h b/tests/tools/nnpackage_run/src/h5formatter.h deleted file mode 100644 index 203ba0e72..000000000 --- a/tests/tools/nnpackage_run/src/h5formatter.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __NNPACKAGE_RUN_H5FORMATTER_H__ -#define __NNPACKAGE_RUN_H5FORMATTER_H__ - -#include <string> -#include <vector> - -#include "types.h" -#include "allocation.h" - -struct nnfw_session; - -namespace nnpkg_run -{ -class H5Formatter -{ -public: - H5Formatter(nnfw_session *sess) : session_(sess) {} - std::vector<TensorShape> readTensorShapes(const std::string &filename); - void loadInputs(const std::string &filename, std::vector<Allocation> &inputs); - void dumpOutputs(const std::string &filename, std::vector<Allocation> &outputs); - -private: - nnfw_session *session_; -}; -} // end of namespace - -#endif // __NNPACKAGE_RUN_H5FORMATTER_H__ diff --git a/tests/tools/nnpackage_run/src/nnfw_util.cc b/tests/tools/nnpackage_run/src/nnfw_util.cc deleted file mode 100644 index 01e72f99e..000000000 --- a/tests/tools/nnpackage_run/src/nnfw_util.cc +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <cassert> -#include <string> -#include "nnfw.h" - -namespace nnpkg_run -{ -uint64_t num_elems(const nnfw_tensorinfo *ti) -{ - uint64_t n = 1; - for (uint32_t i = 0; i < ti->rank; ++i) - { - assert(ti->dims[i] >= 0); - n *= ti->dims[i]; - } - return n; -} - -uint64_t bufsize_for(const nnfw_tensorinfo *ti) -{ - static int elmsize[] = { - sizeof(float), /* NNFW_TYPE_TENSOR_FLOAT32 */ - sizeof(int), /* NNFW_TYPE_TENSOR_INT32 */ - sizeof(uint8_t), /* NNFW_TYPE_TENSOR_QUANT8_ASYMM */ - sizeof(bool), /* NNFW_TYPE_TENSOR_BOOL = 3 */ - sizeof(uint8_t), /* NNFW_TYPE_TENSOR_UINT8 = 4 */ - sizeof(int64_t), /* NNFW_TYPE_TENSOR_INT64 = 5 */ - - }; - return elmsize[ti->dtype] * num_elems(ti); -} - -} // end of namespace diff --git a/tests/tools/nnpackage_run/src/nnfw_util.h b/tests/tools/nnpackage_run/src/nnfw_util.h deleted file mode 100644 index 6fe547eca..000000000 --- a/tests/tools/nnpackage_run/src/nnfw_util.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __NNPACKAGE_RUN_NNFW_UTIL_H__ -#define __NNPACKAGE_RUN_NNFW_UTIL_H__ - -#include "nnfw.h" - -#define NNPR_ENSURE_STATUS(a) \ - do \ - { \ - if ((a) != NNFW_STATUS_NO_ERROR) \ - { \ - exit(-1); \ - } \ - } while (0) - -namespace nnpkg_run -{ -uint64_t num_elems(const nnfw_tensorinfo *ti); -uint64_t bufsize_for(const nnfw_tensorinfo *ti); -} // end of namespace nnpkg_run - -#endif // __NNPACKAGE_UTIL_H__ diff --git a/tests/tools/nnpackage_run/src/nnpackage_run.cc b/tests/tools/nnpackage_run/src/nnpackage_run.cc index 05632393b..97edebf4c 100644 --- a/tests/tools/nnpackage_run/src/nnpackage_run.cc +++ b/tests/tools/nnpackage_run/src/nnpackage_run.cc @@ -14,277 +14,246 @@ * limitations under the License. */ -#include "allocation.h" #include "args.h" -#include "benchmark.h" -#if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1 -#include "h5formatter.h" -#endif +#include "tflite/Diff.h" +#include "tensor_dumper.h" +#include "hdf5.h" #include "nnfw.h" -#include "nnfw_util.h" -#include "nnfw_internal.h" -#include "randomgen.h" -#ifdef RUY_PROFILER -#include "ruy/profiler/profiler.h" -#endif - -#include <cassert> -#include <chrono> -#include <cstdlib> + +#include <assert.h> #include <iostream> -#include <libgen.h> -#include <stdexcept> -#include <unordered_map> -#include <vector> -static const char *default_backend_cand = "cpu"; +#include <chrono> + +#define NNPR_ENSURE_STATUS(a) \ + do \ + { \ + if ((a) != NNFW_STATUS_NO_ERROR) \ + { \ + exit(-1); \ + } \ + } while (0) -void overwriteShapeMap(nnpkg_run::TensorShapeMap &shape_map, - std::vector<nnpkg_run::TensorShape> shapes) +uint64_t NowMicros() { - for (uint32_t i = 0; i < shapes.size(); i++) - shape_map[i] = shapes[i]; + auto time_point = std::chrono::high_resolution_clock::now(); + auto since_epoch = time_point.time_since_epoch(); + // default precision of high resolution clock is 10e-9 (nanoseconds) + return std::chrono::duration_cast<std::chrono::microseconds>(since_epoch).count(); } -int main(const int argc, char **argv) +uint64_t num_elems(const nnfw_tensorinfo *ti) { - using namespace nnpkg_run; - - try + uint64_t n = 1; + for (uint32_t i = 0; i < ti->rank; ++i) { - Args args(argc, argv); - auto nnpackage_path = args.getPackageFilename(); - if (args.printVersion()) - { - uint32_t version; - NNPR_ENSURE_STATUS(nnfw_query_info_u32(NULL, NNFW_INFO_ID_VERSION, &version)); - std::cout << "nnpkg_run (nnfw runtime: v" << (version >> 24) << "." - << ((version & 0x0000FF00) >> 8) << "." << (version & 0xFF) << ")" << std::endl; - exit(0); - } + assert(ti->dims[i] >= 0); + n *= ti->dims[i]; + } + return n; +}; -#ifdef RUY_PROFILER - ruy::profiler::ScopeProfile ruy_profile; -#endif +std::vector<float> randomData(RandomGenerator &randgen, uint64_t size) +{ + std::vector<float> vec(size); + for (uint64_t i = 0; i < size; i++) + vec[i] = randgen.generate<float>(); + return vec; +} - // TODO Apply verbose level to phases - const int verbose = args.getVerboseLevel(); - benchmark::Phases phases( - benchmark::PhaseOption{args.getMemoryPoll(), args.getGpuMemoryPoll(), args.getRunDelay()}); +static const char *h5_value_grpname = "value"; - nnfw_session *session = nullptr; - NNPR_ENSURE_STATUS(nnfw_create_session(&session)); +int main(const int argc, char **argv) +{ + NNPackageRun::Args args(argc, argv); + auto nnpackage_path = args.getPackageFilename(); - // ModelLoad - phases.run("MODEL_LOAD", [&](const benchmark::Phase &, uint32_t) { - NNPR_ENSURE_STATUS(nnfw_load_model_from_file(session, nnpackage_path.c_str())); - }); + nnfw_session *session = nullptr; + NNPR_ENSURE_STATUS(nnfw_create_session(&session)); + NNPR_ENSURE_STATUS(nnfw_load_model_from_file(session, nnpackage_path.c_str())); - char *available_backends = std::getenv("BACKENDS"); - if (available_backends) - NNPR_ENSURE_STATUS(nnfw_set_available_backends(session, available_backends)); + uint32_t num_inputs; + NNPR_ENSURE_STATUS(nnfw_input_size(session, &num_inputs)); - uint32_t num_inputs; - NNPR_ENSURE_STATUS(nnfw_input_size(session, &num_inputs)); + // verify input and output - // verify input and output + if (num_inputs == 0) + { + std::cerr << "[ ERROR ] " + << "No inputs in model => execution is not possible" << std::endl; + exit(1); + } - auto verifyInputTypes = [session]() { - uint32_t sz; - NNPR_ENSURE_STATUS(nnfw_input_size(session, &sz)); - for (uint32_t i = 0; i < sz; ++i) + auto verifyInputTypes = [session]() { + uint32_t sz; + NNPR_ENSURE_STATUS(nnfw_input_size(session, &sz)); + for (uint32_t i = 0; i < sz; ++i) + { + nnfw_tensorinfo ti; + NNPR_ENSURE_STATUS(nnfw_input_tensorinfo(session, i, &ti)); + if (ti.dtype != NNFW_TYPE_TENSOR_FLOAT32) { - nnfw_tensorinfo ti; - NNPR_ENSURE_STATUS(nnfw_input_tensorinfo(session, i, &ti)); - - if (ti.dtype < NNFW_TYPE_TENSOR_FLOAT32 || ti.dtype > NNFW_TYPE_TENSOR_INT64) - { - std::cerr << "E: not supported input type" << std::endl; - exit(-1); - } + std::cerr << "Only float 32bit is supported." << std::endl; + exit(-1); } - }; + } + }; - auto verifyOutputTypes = [session]() { - uint32_t sz; - NNPR_ENSURE_STATUS(nnfw_output_size(session, &sz)); + auto verifyOutputTypes = [session]() { + uint32_t sz; + NNPR_ENSURE_STATUS(nnfw_output_size(session, &sz)); - for (uint32_t i = 0; i < sz; ++i) + for (uint32_t i = 0; i < sz; ++i) + { + nnfw_tensorinfo ti; + NNPR_ENSURE_STATUS(nnfw_output_tensorinfo(session, i, &ti)); + if (ti.dtype != NNFW_TYPE_TENSOR_FLOAT32) { - nnfw_tensorinfo ti; - NNPR_ENSURE_STATUS(nnfw_output_tensorinfo(session, i, &ti)); - - if (ti.dtype < NNFW_TYPE_TENSOR_FLOAT32 || ti.dtype > NNFW_TYPE_TENSOR_INT64) - { - std::cerr << "E: not supported output type" << std::endl; - exit(-1); - } + std::cerr << "Only float 32bit is supported." << std::endl; + exit(-1); } - }; + } + }; - auto setTensorInfo = [session](const TensorShapeMap &tensor_shape_map) { - for (auto tensor_shape : tensor_shape_map) - { - auto ind = tensor_shape.first; - auto &shape = tensor_shape.second; - nnfw_tensorinfo ti; - // to fill dtype - NNPR_ENSURE_STATUS(nnfw_input_tensorinfo(session, ind, &ti)); - - ti.rank = shape.size(); - for (int i = 0; i < ti.rank; i++) - ti.dims[i] = shape.at(i); - NNPR_ENSURE_STATUS(nnfw_set_input_tensorinfo(session, ind, &ti)); - } - }; - - verifyInputTypes(); - verifyOutputTypes(); - -// set input shape before compilation -#if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1 - - auto fill_shape_from_h5 = [&session](const std::string &h5_file, TensorShapeMap &shape_map) { - assert(!h5_file.empty()); - auto shapes = H5Formatter(session).readTensorShapes(h5_file); - overwriteShapeMap(shape_map, shapes); - }; - - if (args.getWhenToUseH5Shape() == WhenToUseH5Shape::PREPARE) - fill_shape_from_h5(args.getLoadFilename(), args.getShapeMapForPrepare()); -#endif - setTensorInfo(args.getShapeMapForPrepare()); - - // prepare execution - - // TODO When nnfw_{prepare|run} are failed, can't catch the time - phases.run("PREPARE", [&](const benchmark::Phase &, uint32_t) { - NNPR_ENSURE_STATUS(nnfw_prepare(session)); - }); - -// set input shape after compilation and before execution -#if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1 - if (args.getWhenToUseH5Shape() == WhenToUseH5Shape::RUN || - (!args.getLoadFilename().empty() && !args.shapeParamProvided())) - fill_shape_from_h5(args.getLoadFilename(), args.getShapeMapForRun()); -#endif - setTensorInfo(args.getShapeMapForRun()); - - // prepare input - std::vector<Allocation> inputs(num_inputs); -#if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1 - if (!args.getLoadFilename().empty()) - H5Formatter(session).loadInputs(args.getLoadFilename(), inputs); - else - RandomGenerator(session).generate(inputs); -#else - RandomGenerator(session).generate(inputs); -#endif - - // prepare output - uint32_t num_outputs = 0; - NNPR_ENSURE_STATUS(nnfw_output_size(session, &num_outputs)); - std::vector<Allocation> outputs(num_outputs); - auto output_sizes = args.getOutputSizes(); - for (uint32_t i = 0; i < num_outputs; i++) + verifyInputTypes(); + verifyOutputTypes(); + + // prepare execution + + uint64_t prepare_ms = NowMicros(); + NNPR_ENSURE_STATUS(nnfw_prepare(session)); + prepare_ms = NowMicros() - prepare_ms; + + // prepare input + + std::vector<std::vector<float>> inputs(num_inputs); + + auto loadInputs = [session, num_inputs, &inputs](std::string filename) { + hid_t file_id = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); + if (file_id < 0) + { + std::cerr << "error during opening file " << filename << "." << std::endl; + exit(-1); + } + hid_t valgrp_id = H5Gopen(file_id, h5_value_grpname, H5P_DEFAULT); + if (valgrp_id < 0) + { + std::cerr << "error during opening group " << h5_value_grpname << "." << std::endl; + H5Fclose(file_id); + exit(-1); + } + for (uint32_t i = 0; i < num_inputs; ++i) { nnfw_tensorinfo ti; - uint64_t output_size_in_bytes = 0; + NNPR_ENSURE_STATUS(nnfw_input_tensorinfo(session, i, &ti)); + + hid_t dset_id = H5Dopen(valgrp_id, std::to_string(i).c_str(), H5P_DEFAULT); + if (dset_id < 0) { - auto found = output_sizes.find(i); - if (found == output_sizes.end()) - { - NNPR_ENSURE_STATUS(nnfw_output_tensorinfo(session, i, &ti)); - output_size_in_bytes = bufsize_for(&ti); - } - else - { - output_size_in_bytes = found->second; - } + std::cerr << "error during opening dataset " << std::to_string(i) << "." << std::endl; + H5Gclose(valgrp_id); + H5Fclose(file_id); + exit(-1); } - outputs[i].alloc(output_size_in_bytes); - NNPR_ENSURE_STATUS( - nnfw_set_output(session, i, ti.dtype, outputs[i].data(), output_size_in_bytes)); - NNPR_ENSURE_STATUS(nnfw_set_output_layout(session, i, NNFW_LAYOUT_CHANNELS_LAST)); - } - // NOTE: Measuring memory can't avoid taking overhead. Therefore, memory will be measured on the - // only warmup. - if (verbose == 0) - { - phases.run("WARMUP", - [&](const benchmark::Phase &, uint32_t) { NNPR_ENSURE_STATUS(nnfw_run(session)); }, - args.getWarmupRuns()); - phases.run("EXECUTE", - [&](const benchmark::Phase &, uint32_t) { NNPR_ENSURE_STATUS(nnfw_run(session)); }, - args.getNumRuns(), true); + // check type + hid_t type = H5Dget_type(dset_id); + if (!H5Tequal(type, H5T_IEEE_F32BE)) + { + std::cerr << "h5 input has non-float32 type. nnpkg_run supports float32 only." << std::endl; + H5Dclose(dset_id); + H5Gclose(valgrp_id); + H5Fclose(file_id); + exit(-1); + } + // allocate memory for data + auto sz = num_elems(&ti); + inputs[i].resize(sz); + // read data + H5Dread(dset_id, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, inputs[i].data()); + + NNPR_ENSURE_STATUS(nnfw_set_input(session, i, NNFW_TYPE_TENSOR_FLOAT32, inputs[i].data(), + sizeof(float) * num_elems(&ti))); + // clean up + H5Dclose(dset_id); } - else + H5Gclose(valgrp_id); + H5Fclose(file_id); + }; + + auto generateInputs = [session, num_inputs, &inputs]() { + // generate random data + const int seed = 1; + RandomGenerator randgen{seed, 0.0f, 2.0f}; + for (uint32_t i = 0; i < num_inputs; ++i) { - phases.run("WARMUP", - [&](const benchmark::Phase &, uint32_t) { NNPR_ENSURE_STATUS(nnfw_run(session)); }, - [&](const benchmark::Phase &phase, uint32_t nth) { - std::cout << "... " - << "warmup " << nth + 1 << " takes " << phase.time[nth] / 1e3 << " ms" - << std::endl; - }, - args.getWarmupRuns()); - phases.run("EXECUTE", - [&](const benchmark::Phase &, uint32_t) { NNPR_ENSURE_STATUS(nnfw_run(session)); }, - [&](const benchmark::Phase &phase, uint32_t nth) { - std::cout << "... " - << "run " << nth + 1 << " takes " << phase.time[nth] / 1e3 << " ms" - << std::endl; - }, - args.getNumRuns(), true); + nnfw_tensorinfo ti; + NNPR_ENSURE_STATUS(nnfw_input_tensorinfo(session, i, &ti)); + auto input_num_elements = num_elems(&ti); + inputs[i] = randomData(randgen, input_num_elements); + NNPR_ENSURE_STATUS(nnfw_set_input(session, i, NNFW_TYPE_TENSOR_FLOAT32, inputs[i].data(), + sizeof(float) * input_num_elements)); } + }; -#if defined(ONERT_HAVE_HDF5) && ONERT_HAVE_HDF5 == 1 - // dump output tensors - if (!args.getDumpFilename().empty()) - H5Formatter(session).dumpOutputs(args.getDumpFilename(), outputs); -#endif + if (!args.getLoadFilename().empty()) + loadInputs(args.getLoadFilename()); + else + generateInputs(); - NNPR_ENSURE_STATUS(nnfw_close_session(session)); + // prepare output - // TODO Apply verbose level to result + uint32_t num_outputs = 0; + NNPR_ENSURE_STATUS(nnfw_output_size(session, &num_outputs)); + std::vector<std::vector<float>> outputs(num_outputs); - // prepare result - benchmark::Result result(phases); + for (uint32_t i = 0; i < num_outputs; i++) + { + nnfw_tensorinfo ti; + NNPR_ENSURE_STATUS(nnfw_output_tensorinfo(session, i, &ti)); + auto output_num_elements = num_elems(&ti); + outputs[i].resize(output_num_elements); + NNPR_ENSURE_STATUS(nnfw_set_output(session, i, NNFW_TYPE_TENSOR_FLOAT32, outputs[i].data(), + sizeof(float) * output_num_elements)); + } - // to stdout - benchmark::printResult(result); + uint64_t run_ms = NowMicros(); + NNPR_ENSURE_STATUS(nnfw_run(session)); + run_ms = NowMicros() - run_ms; - // to csv - if (args.getWriteReport() == false) - return 0; + // dump output tensors - // prepare csv task - std::string exec_basename; - std::string nnpkg_basename; - std::string backend_name = (available_backends) ? available_backends : default_backend_cand; + auto dumpOutputs = [session, num_outputs, &outputs](std::string filename) { + hid_t file_id = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + for (uint32_t i = 0; i < num_outputs; i++) { - char buf[PATH_MAX]; - char *res = realpath(nnpackage_path.c_str(), buf); - if (res) + nnfw_tensorinfo ti; + NNPR_ENSURE_STATUS(nnfw_output_tensorinfo(session, i, &ti)); + std::vector<hsize_t> dims; + dims.resize(ti.rank); + for (uint32_t j = 0; j < ti.rank; ++j) { - nnpkg_basename = basename(buf); + assert(ti.dims[j] >= 0); + dims[j] = ti.dims[j]; } - else - { - std::cerr << "E: during getting realpath from nnpackage_path." << std::endl; - exit(-1); - } - exec_basename = basename(argv[0]); + hid_t valgrp_id = H5Gcreate(file_id, h5_value_grpname, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + hid_t dsp_id = H5Screate_simple(ti.rank, dims.data(), NULL); + hid_t dset_id = H5Dcreate2(valgrp_id, std::to_string(i).c_str(), H5T_IEEE_F32BE, dsp_id, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + H5Dwrite(dset_id, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, outputs[i].data()); + H5Dclose(dset_id); + H5Sclose(dsp_id); } + H5Fclose(file_id); + }; - benchmark::writeResult(result, exec_basename, nnpkg_basename, backend_name); + if (!args.getDumpFilename().empty()) + dumpOutputs(args.getDumpFilename()); - return 0; - } - catch (std::runtime_error &e) - { - std::cerr << "E: Fail to run by runtime error:" << e.what() << std::endl; - exit(-1); - } + std::cout << "nnfw_prepare takes " << prepare_ms / 1e3 << " sec" << std::endl; + std::cout << "nnfw_run takes " << run_ms / 1e3 << " sec" << std::endl; + + NNPR_ENSURE_STATUS(nnfw_close_session(session)); + + return 0; } diff --git a/tests/tools/nnpackage_run/src/randomgen.cc b/tests/tools/nnpackage_run/src/randomgen.cc deleted file mode 100644 index 343242081..000000000 --- a/tests/tools/nnpackage_run/src/randomgen.cc +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "randomgen.h" -#include "nnfw.h" -#include "nnfw_util.h" -#include "misc/RandomGenerator.h" - -#include <iostream> - -namespace nnpkg_run -{ - -template <class T> void randomData(nnfw::misc::RandomGenerator &randgen, void *data, uint64_t size) -{ - for (uint64_t i = 0; i < size; i++) - reinterpret_cast<T *>(data)[i] = randgen.generate<T>(); -} - -void RandomGenerator::generate(std::vector<Allocation> &inputs) -{ - // generate random data - const int seed = 1; - nnfw::misc::RandomGenerator randgen{seed, 0.0f, 2.0f}; - for (uint32_t i = 0; i < inputs.size(); ++i) - { - nnfw_tensorinfo ti; - NNPR_ENSURE_STATUS(nnfw_input_tensorinfo(session_, i, &ti)); - auto input_size_in_bytes = bufsize_for(&ti); - inputs[i].alloc(input_size_in_bytes); - switch (ti.dtype) - { - case NNFW_TYPE_TENSOR_FLOAT32: - randomData<float>(randgen, inputs[i].data(), num_elems(&ti)); - break; - case NNFW_TYPE_TENSOR_QUANT8_ASYMM: - randomData<uint8_t>(randgen, inputs[i].data(), num_elems(&ti)); - break; - case NNFW_TYPE_TENSOR_BOOL: - randomData<bool>(randgen, inputs[i].data(), num_elems(&ti)); - break; - case NNFW_TYPE_TENSOR_UINT8: - randomData<uint8_t>(randgen, inputs[i].data(), num_elems(&ti)); - break; - case NNFW_TYPE_TENSOR_INT32: - randomData<int32_t>(randgen, inputs[i].data(), num_elems(&ti)); - break; - case NNFW_TYPE_TENSOR_INT64: - randomData<int64_t>(randgen, inputs[i].data(), num_elems(&ti)); - break; - default: - std::cerr << "Not supported input type" << std::endl; - std::exit(-1); - } - NNPR_ENSURE_STATUS( - nnfw_set_input(session_, i, ti.dtype, inputs[i].data(), input_size_in_bytes)); - NNPR_ENSURE_STATUS(nnfw_set_input_layout(session_, i, NNFW_LAYOUT_CHANNELS_LAST)); - } -}; - -} // end of namespace nnpkg_run diff --git a/tests/tools/nnpackage_run/src/tensor_dumper.cc b/tests/tools/nnpackage_run/src/tensor_dumper.cc new file mode 100644 index 000000000..ae4ed9518 --- /dev/null +++ b/tests/tools/nnpackage_run/src/tensor_dumper.cc @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensor_dumper.h" + +#include <fstream> +#include <iostream> +#include <cstring> + +namespace NNPackageRun +{ +TensorDumper::TensorDumper(const std::string &filename) +{ + // TODO Handle file open/write error + file_.open(filename, std::ios::out | std::ios::binary); + dumpInt32(version); +} + +TensorDumper::~TensorDumper() { file_.close(); } + +void TensorDumper::dumpInt32(int32_t i) +{ + file_.write(reinterpret_cast<const char *>(&i), sizeof(i)); +} + +void TensorDumper::dumpSizeT(size_t i) +{ + file_.write(reinterpret_cast<const char *>(&i), sizeof(i)); +} + +void TensorDumper::dumpTensor(const nnfw_tensorinfo ti, void *buffer, size_t bytes) +{ + dumpInt32(ti.dtype); + dumpInt32(ti.rank); + for (uint i = 0; i < ti.rank; ++i) + dumpInt32(ti.dims[i]); + dumpSizeT(bytes); + file_.write(static_cast<char *>(buffer), bytes); +} + +} // end of namespace NNPackageRun
\ No newline at end of file diff --git a/tests/tools/nnpackage_run/src/randomgen.h b/tests/tools/nnpackage_run/src/tensor_dumper.h index 9ca51dd11..12cc22f18 100644 --- a/tests/tools/nnpackage_run/src/randomgen.h +++ b/tests/tools/nnpackage_run/src/tensor_dumper.h @@ -14,27 +14,34 @@ * limitations under the License. */ -#ifndef __NNPACKAGE_RUN_RANDOMGEN_H__ -#define __NNPACKAGE_RUN_RANDOMGEN_H__ +#ifndef __NNPACKAGE_RUN_TENSOR_DUMPER_H__ +#define __NNPACKAGE_RUN_TENSOR_DUMPER_H__ +#include <memory> #include <string> #include <vector> +#include <stddef.h> +#include <fstream> -#include "allocation.h" +#include "nnfw.h" -struct nnfw_session; - -namespace nnpkg_run +namespace NNPackageRun { -class RandomGenerator + +class TensorDumper { public: - RandomGenerator(nnfw_session *sess) : session_(sess) {} - void generate(std::vector<Allocation> &inputs); + TensorDumper(const std::string &filename); + void dumpTensor(const nnfw_tensorinfo ti, void *buffer, size_t bytes); + void dumpInt32(int32_t i); + void dumpSizeT(size_t i); + ~TensorDumper(); private: - nnfw_session *session_; + static constexpr int version = 1; + std::ofstream file_; }; -} // end of namespace -#endif // __NNPACKAGE_RUN_RANDOMGEN_H__ +} // end of namespace NNPackageRun + +#endif // __NNPACKAGE_RUN_TENSOR_DUMPER_H__ diff --git a/tests/tools/nnpackage_run/src/types.h b/tests/tools/nnpackage_run/src/types.h deleted file mode 100644 index 93a7ab230..000000000 --- a/tests/tools/nnpackage_run/src/types.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __NNPACKAGE_RUN_TYPES_H__ -#define __NNPACKAGE_RUN_TYPES_H__ - -namespace nnpkg_run -{ - -using TensorShape = std::vector<int>; - -} // end of namespace nnpkg_run - -#endif // __NNPACKAGE_RUN_TYPES_H__ diff --git a/tests/tools/tflite_benchmark/CMakeLists.txt b/tests/tools/tflite_benchmark/CMakeLists.txt new file mode 100644 index 000000000..634d451bc --- /dev/null +++ b/tests/tools/tflite_benchmark/CMakeLists.txt @@ -0,0 +1,10 @@ +list(APPEND SOURCES "src/tflite_benchmark.cc") + +nnfw_find_package(Boost REQUIRED) + +add_executable(tflite_benchmark ${SOURCES}) +target_link_libraries(tflite_benchmark nnfw_lib_tflite tensorflow-lite ${LIB_PTHREAD} dl nnfw_lib_misc) + +target_include_directories(tflite_benchmark PRIVATE ${Boost_INCLUDE_DIRS}) + +install(TARGETS tflite_benchmark DESTINATION bin) diff --git a/tests/tools/tflite_benchmark/src/tflite_benchmark.cc b/tests/tools/tflite_benchmark/src/tflite_benchmark.cc new file mode 100644 index 000000000..1fde0c449 --- /dev/null +++ b/tests/tools/tflite_benchmark/src/tflite_benchmark.cc @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tflite/ext/kernels/register.h" +#include "tensorflow/lite/model.h" + +#include "tflite/Assert.h" +#include "tflite/Session.h" +#include "tflite/InterpreterSession.h" +#include "tflite/NNAPISession.h" +#include "tflite/Diff.h" +#include "misc/tensor/IndexIterator.h" + +#include <boost/accumulators/accumulators.hpp> +#include <boost/accumulators/statistics/stats.hpp> +#include <boost/accumulators/statistics/min.hpp> +#include <boost/accumulators/statistics/max.hpp> +#include <boost/accumulators/statistics/mean.hpp> + +#include <chrono> +#include <iostream> +#include <thread> + +#include "misc/EnvVar.h" +#include "misc/benchmark.h" + +using namespace tflite; +using namespace nnfw::tflite; + +void help(std::ostream &out, const int argc, char **argv) +{ + std::string cmd = argv[0]; + auto pos = cmd.find_last_of("/"); + if (pos != std::string::npos) + cmd = cmd.substr(pos + 1); + + out << "use:" << std::endl << cmd << " <model file name>" << std::endl; +} + +bool checkParams(const int argc, char **argv) +{ + try + { + if (argc < 2) + { + help(std::cerr, argc, argv); + return false; + } + } + catch (const std::exception &e) + { + std::cerr << e.what() << std::endl; + + return false; + } + + return true; +} + +// Verifies whether the model is a flatbuffer file. +class BMFlatBufferVerifier : public tflite::TfLiteVerifier +{ +public: + bool Verify(const char *data, int length, tflite::ErrorReporter *reporter) override + { + + flatbuffers::Verifier verifier(reinterpret_cast<const uint8_t *>(data), length); + if (!tflite::VerifyModelBuffer(verifier)) + { + reporter->Report("The model is not a valid Flatbuffer file"); + return false; + } + return true; + } +}; + +int main(const int argc, char **argv) +{ + + if (!checkParams(argc, argv)) + { + return -1; + } + + const auto filename = argv[1]; + + const bool use_nnapi = nnfw::misc::EnvVar("USE_NNAPI").asBool(false); + const auto thread_count = nnfw::misc::EnvVar("THREAD").asInt(-1); + const auto pause = nnfw::misc::EnvVar("PAUSE").asInt(0); + + std::cout << "Num threads: " << thread_count << std::endl; + if (use_nnapi) + { + std::cout << "Use NNAPI" << std::endl; + } + + assert(pause >= 0); + if (pause > 0) + { + std::cout << "Insert " << pause << "s pause between iterations" << std::endl; + } + + StderrReporter error_reporter; + + std::unique_ptr<tflite::TfLiteVerifier> verifier{new BMFlatBufferVerifier}; + + auto model = FlatBufferModel::VerifyAndBuildFromFile(filename, verifier.get(), &error_reporter); + if (model == nullptr) + { + std::cerr << "Cannot create model" << std::endl; + return -1; + } + + BuiltinOpResolver resolver; + + InterpreterBuilder builder(*model, resolver); + + std::unique_ptr<Interpreter> interpreter; + + try + { + TFLITE_ENSURE(builder(&interpreter)); + } + catch (const std::exception &e) + { + std::cerr << e.what() << std::endl; + return 1; + } + + // Show inputs + for (uint32_t n = 0; n < interpreter->inputs().size(); ++n) + { + // TODO Print shape + auto tensor_id = interpreter->inputs().at(n); + auto tensor_ptr = interpreter->tensor(tensor_id); + + std::cout << "Input #" << n << ":" << std::endl; + std::cout << " Name: " << tensor_ptr->name << std::endl; + } + + // Show outputs + for (uint32_t n = 0; n < interpreter->outputs().size(); ++n) + { + // TODO Print shape + auto tensor_id = interpreter->outputs().at(n); + auto tensor_ptr = interpreter->tensor(tensor_id); + + std::cout << "Output #" << n << ":" << std::endl; + std::cout << " Name: " << tensor_ptr->name << std::endl; + } + + interpreter->SetNumThreads(thread_count); + + std::shared_ptr<nnfw::tflite::Session> sess; + + if (use_nnapi) + { + sess = std::make_shared<nnfw::tflite::NNAPISession>(interpreter.get()); + } + else + { + sess = std::make_shared<nnfw::tflite::InterpreterSession>(interpreter.get()); + } + + // + // Warming-up + // + for (uint32_t n = 0; n < 3; ++n) + { + std::chrono::milliseconds elapsed(0); + + sess->prepare(); + + for (const auto &id : interpreter->inputs()) + { + TfLiteTensor *tensor = interpreter->tensor(id); + if (tensor->type == kTfLiteInt32) + { + // Generate singed 32-bit integer (s32) input + auto tensor_view = nnfw::tflite::TensorView<int32_t>::make(*interpreter, id); + + int32_t value = 0; + + nnfw::misc::tensor::iterate(tensor_view.shape()) + << [&](const nnfw::misc::tensor::Index &ind) { + // TODO Generate random values + // Gather operation: index should be within input coverage. + tensor_view.at(ind) = value; + value++; + }; + } + else if (tensor->type == kTfLiteUInt8) + { + // Generate unsigned 8-bit integer input + auto tensor_view = nnfw::tflite::TensorView<uint8_t>::make(*interpreter, id); + + uint8_t value = 0; + + nnfw::misc::tensor::iterate(tensor_view.shape()) + << [&](const nnfw::misc::tensor::Index &ind) { + // TODO Generate random values + tensor_view.at(ind) = value; + value = (value + 1) & 0xFF; + }; + } + else + { + assert(tensor->type == kTfLiteFloat32); + + const int seed = 1; /* TODO Add an option for seed value */ + RandomGenerator randgen{seed, 0.0f, 0.2f}; + const float *end = reinterpret_cast<const float *>(tensor->data.raw_const + tensor->bytes); + for (float *ptr = tensor->data.f; ptr < end; ptr++) + { + *ptr = randgen.generate<float>(); + } + } + } + + nnfw::misc::benchmark::measure(elapsed) << [&](void) { + if (!sess->run()) + { + assert(0 && "run failed"); + } + }; + sess->teardown(); + + std::cout << "Warming-up " << n << ": " << elapsed.count() << "ms" << std::endl; + } + + // + // Measure + // + const auto cnt = nnfw::misc::EnvVar("COUNT").asInt(1); + + using namespace boost::accumulators; + + accumulator_set<double, stats<tag::mean, tag::min, tag::max>> acc; + + for (int n = 0; n < cnt; ++n) + { + std::chrono::milliseconds elapsed(0); + + sess->prepare(); + nnfw::misc::benchmark::measure(elapsed) << [&](void) { + if (!sess->run()) + { + assert(0 && "run failed"); + } + }; + sess->teardown(); + + acc(elapsed.count()); + + std::cout << "Iteration " << n << ": " << elapsed.count() << "ms" << std::endl; + + // Insert "pause" + if ((n != cnt - 1) && (pause > 0)) + { + std::this_thread::sleep_for(std::chrono::seconds(pause)); + } + } + + std::cout << "--------" << std::endl; + std::cout << "Min: " << min(acc) << "ms" << std::endl; + std::cout << "Max: " << max(acc) << "ms" << std::endl; + std::cout << "Mean: " << mean(acc) << "ms" << std::endl; + + return 0; +} diff --git a/tests/tools/tflite_benchmark_model/CMakeLists.txt b/tests/tools/tflite_benchmark_model/CMakeLists.txt index 017e1da57..ea4986a8c 100644 --- a/tests/tools/tflite_benchmark_model/CMakeLists.txt +++ b/tests/tools/tflite_benchmark_model/CMakeLists.txt @@ -2,7 +2,7 @@ if (NOT BUILD_TFLITE_BENCHMARK_MODEL) return() endif(NOT BUILD_TFLITE_BENCHMARK_MODEL) -nnfw_find_package(TensorFlowLite EXACT 1.13.1 REQUIRED) +nnfw_find_package(TensorFlowLite REQUIRED) # TODO Remove this target_compile_definitions command, and just check its presence. # This change is prerequisites on pre-built tensorflow-lite package support @@ -10,7 +10,7 @@ target_compile_definitions(tensorflow-lite PUBLIC "TFLITE_PROFILING_ENABLED") file(GLOB_RECURSE SOURCES "*.cc") -nnas_find_package(TensorFlowSource EXACT 1.13.1 REQUIRED) +nnfw_find_package(TensorFlowSource REQUIRED) set(TENSORFLOW_LITE_BASE "${TensorFlowSource_DIR}/tensorflow/lite") list(APPEND SOURCES "${TENSORFLOW_LITE_BASE}/tools/benchmark/benchmark_main.cc" "${TENSORFLOW_LITE_BASE}/tools/benchmark/benchmark_model.cc" diff --git a/tests/tools/tflite_benchmark_model/benchmark_tflite_model.cc b/tests/tools/tflite_benchmark_model/benchmark_tflite_model.cc index 16e85fc07..f6dda2628 100644 --- a/tests/tools/tflite_benchmark_model/benchmark_tflite_model.cc +++ b/tests/tools/tflite_benchmark_model/benchmark_tflite_model.cc @@ -39,16 +39,15 @@ limitations under the License. #include <unordered_set> #include <vector> +#ifdef TFLITE_FLEX +#include "tensorflow/lite/delegates/flex/delegate.h" +#endif // TFLITE_FLEX #include "tflite/ext/kernels/register.h" #include "tensorflow/lite/model.h" #include "tensorflow/lite/op_resolver.h" #include "tensorflow/lite/string_util.h" #include "tensorflow/lite/tools/benchmark/logging.h" -#ifdef GEMMLOWP_PROFILING -#include "gemmlowp/profiling/profiler.h" -#endif - // For profiling nnapi_delegate #include "profiling/profiling.h" #include "tflite/ext/nnapi_delegate.h" @@ -90,21 +89,6 @@ void ProfilingListener::OnSingleRunEnd() { summarizer_.ProcessProfiles(profile_events, *interpreter_); } -void GemmlowpProfilingListener::OnBenchmarkStart( - const BenchmarkParams& params) { -#ifdef GEMMLOWP_PROFILING - gemmlowp::RegisterCurrentThreadForProfiling(); - gemmlowp::StartProfiling(); -#endif -} - -void GemmlowpProfilingListener::OnBenchmarkEnd( - const BenchmarkResults& results) { -#ifdef GEMMLOWP_PROFILING - gemmlowp::FinishProfiling(); -#endif -} - namespace { std::vector<std::string> Split(const std::string& str, const char delim) { @@ -205,18 +189,7 @@ bool PopulateInputLayerInfo( return true; } -std::vector<int> TfLiteIntArrayToVector(const TfLiteIntArray* int_array) { - std::vector<int> values; - values.reserve(int_array->size); - for (size_t i = 0; i < int_array->size; i++) { - values.push_back(int_array->data[i]); - } - return values; -} - -} // namespace - -BenchmarkParams BenchmarkTfLiteModel::DefaultParams() { +BenchmarkParams GetDefaultParams() { BenchmarkParams default_params = BenchmarkModel::DefaultParams(); default_params.AddParam("graph", BenchmarkParam::Create<std::string>("")); default_params.AddParam("input_layer", @@ -227,13 +200,16 @@ BenchmarkParams BenchmarkTfLiteModel::DefaultParams() { return default_params; } +} // namespace + BenchmarkTfLiteModel::BenchmarkTfLiteModel() - : BenchmarkTfLiteModel(DefaultParams()) {} + : BenchmarkModel(GetDefaultParams()) { + AddListener(&profiling_listener_); +} BenchmarkTfLiteModel::BenchmarkTfLiteModel(BenchmarkParams params) : BenchmarkModel(std::move(params)) { AddListener(&profiling_listener_); - AddListener(&gemmlowp_profiling_listener_); } std::vector<Flag> BenchmarkTfLiteModel::GetFlags() { @@ -283,10 +259,12 @@ uint64_t BenchmarkTfLiteModel::ComputeInputBytes() { void BenchmarkTfLiteModel::PrepareInputsAndOutputs() { auto interpreter_inputs = interpreter->inputs(); // Set the values of the input tensors. - for (int j = 0; j < interpreter_inputs.size(); ++j) { + for (int j = 0; j < inputs.size(); ++j) { + const InputLayerInfo& input = inputs[j]; int i = interpreter_inputs[j]; TfLiteTensor* t = interpreter->tensor(i); - std::vector<int> sizes = TfLiteIntArrayToVector(t->dims); + std::vector<int> sizes = input.shape; + // TODO(ahentz): below we ignore the O-th dimension (number of batches). if (t->type == kTfLiteFloat32) { FillRandomValue<float>( @@ -305,17 +283,12 @@ void BenchmarkTfLiteModel::PrepareInputsAndOutputs() { interpreter->typed_tensor<uint8_t>(i), std::vector<int>(sizes.begin() + 1, sizes.end()), []() { return static_cast<uint8_t>(rand()) % 255; }); - } else if (t->type == kTfLiteInt8) { - FillRandomValue<int8_t>( - interpreter->typed_tensor<int8_t>(i), - std::vector<int>(sizes.begin() + 1, sizes.end()), - []() { return static_cast<int8_t>(rand()) % 255 - 127; }); } else if (t->type == kTfLiteString) { tflite::DynamicBuffer buffer; FillRandomString(&buffer, sizes, []() { return "we're have some friends over saturday to hang out in the yard"; }); - buffer.WriteToTensor(interpreter->tensor(i), /*new_shape=*/nullptr); + buffer.WriteToTensor(interpreter->tensor(i)); } else { TFLITE_LOG(FATAL) << "Don't know how to populate tensor " << t->name << " of type " << t->type; @@ -362,12 +335,21 @@ void BenchmarkTfLiteModel::Init() { bool use_nnapi = params_.Get<bool>("use_nnapi"); interpreter->UseNNAPI(use_nnapi); + if (use_nnapi) { - if (nnfw_delegate_.BuildGraph(&(interpreter.get()->primary_subgraph())) != kTfLiteOk) { + if (nnfw_delegate_.BuildGraph(interpreter.get()) != kTfLiteOk) { TFLITE_LOG(FATAL) << "Failed to BuildGraph!"; } } - ApplyDelegates(); + +#ifdef TFLITE_FLEX + TFLITE_LOG(INFO) << "Instantiating Flex Delegate"; + delegate_ = FlexDelegate::Create(); + if (delegate_) { + interpreter->ModifyGraphWithDelegate(delegate_.get(), + /*allow_dynamic_tensors=*/true); + } +#endif // TFLITE_FLEX auto interpreter_inputs = interpreter->inputs(); @@ -405,7 +387,7 @@ void BenchmarkTfLiteModel::Init() { void BenchmarkTfLiteModel::RunImpl() { bool use_nnapi = params_.Get<bool>("use_nnapi"); if (use_nnapi) { - if (nnfw_delegate_.Invoke(&interpreter->primary_subgraph()) != kTfLiteOk) { + if (nnfw_delegate_.Invoke(interpreter.get()) != kTfLiteOk) { TFLITE_LOG(FATAL) << "Failed to invoke!"; } } else { diff --git a/tests/tools/tflite_loader/CMakeLists.txt b/tests/tools/tflite_loader/CMakeLists.txt index 0fe1c69de..2705537d8 100644 --- a/tests/tools/tflite_loader/CMakeLists.txt +++ b/tests/tools/tflite_loader/CMakeLists.txt @@ -3,21 +3,21 @@ if(NOT BUILD_TFLITE_LOADER_TEST_TOOL) return() endif(NOT BUILD_TFLITE_LOADER_TEST_TOOL) -if(NOT BUILD_ONERT) - message("skipping tflite loader tool build: onert is not built") +if(NOT BUILD_NEURUN) + message("skipping tflite loader tool build: neurun is not built") return() -endif(NOT BUILD_ONERT) +endif(NOT BUILD_NEURUN) list(APPEND SOURCES "src/tflite_loader.cc") list(APPEND SOURCES "src/args.cc") -nnfw_find_package(Boost REQUIRED program_options system filesystem) +nnfw_find_package(Boost REQUIRED) add_executable(tflite_loader_test_tool ${SOURCES}) target_include_directories(tflite_loader_test_tool PRIVATE ${Boost_INCLUDE_DIRS}) -target_link_libraries(tflite_loader_test_tool onert_core onert tflite_loader) -target_link_libraries(tflite_loader_test_tool nnfw_lib_tflite nnfw_lib_misc) -target_link_libraries(tflite_loader_test_tool ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_FILESYSTEM_LIBRARY}) +target_link_libraries(tflite_loader_test_tool neurun_core neurun tflite_loader) +target_link_libraries(tflite_loader_test_tool nnfw_lib_tflite tensorflow-lite ${LIB_PTHREAD} dl nnfw_lib_misc) +target_link_libraries(tflite_loader_test_tool boost_program_options boost_system boost_filesystem) install(TARGETS tflite_loader_test_tool DESTINATION bin) diff --git a/tests/tools/tflite_loader/src/args.cc b/tests/tools/tflite_loader/src/args.cc index e9fb141ca..3fe1d0bf4 100644 --- a/tests/tools/tflite_loader/src/args.cc +++ b/tests/tools/tflite_loader/src/args.cc @@ -69,23 +69,14 @@ void Args::Parse(const int argc, char **argv) exit(0); } - try + if (vm.count("tflite")) { - if (vm.count("tflite")) - { - _tflite_filename = vm["tflite"].as<std::string>(); - } - - if (vm.count("data")) - { - _data_filenames = vm["data"].as<std::vector<std::string>>(); - } + _tflite_filename = vm["tflite"].as<std::string>(); } - catch (const std::bad_cast &e) + + if (vm.count("data")) { - std::cerr << e.what() << '\n'; - print(argv); - exit(1); + _data_filenames = vm["data"].as<std::vector<std::string>>(); } } diff --git a/tests/tools/tflite_loader/src/tflite_loader.cc b/tests/tools/tflite_loader/src/tflite_loader.cc index ce099210b..c2388f3cc 100644 --- a/tests/tools/tflite_loader/src/tflite_loader.cc +++ b/tests/tools/tflite_loader/src/tflite_loader.cc @@ -27,11 +27,11 @@ #include "compiler/Compiler.h" #include "exec/Execution.h" -#include "ir/Graph.h" +#include "graph/Graph.h" -#include "tflite_loader.h" +#include "loader.h" -#include <memory> +#include "cpp14/memory.h" const int RUN_FAILED = 1; @@ -63,7 +63,7 @@ std::vector<float> readData(const string &path) return vec; } -std::vector<float> randomData(nnfw::misc::RandomGenerator &randgen, const uint64_t size) +std::vector<float> randomData(RandomGenerator &randgen, const uint64_t size) { std::vector<float> vec(size); for (uint64_t i = 0; i < size; i++) @@ -73,18 +73,15 @@ std::vector<float> randomData(nnfw::misc::RandomGenerator &randgen, const uint64 return vec; } -void executeGraph(const std::shared_ptr<onert::ir::Graph> &g, +void executeGraph(const std::shared_ptr<neurun::graph::Graph> &g, const std::vector<std::vector<float>> &inputs, std::vector<std::vector<float>> &outputs) { - auto subgs = std::make_shared<onert::ir::Subgraphs>(); - subgs->push(onert::ir::SubgraphIndex{0}, g); - auto compiler = new onert::compiler::Compiler(subgs); - std::shared_ptr<onert::exec::ExecutorMap> executors; + auto compiler = new neurun::compiler::Compiler(g); // Compilation try { - executors = compiler->compile(); + compiler->compile(); } catch (const std::exception &e) { @@ -95,35 +92,37 @@ void executeGraph(const std::shared_ptr<onert::ir::Graph> &g, std::cout << "[Execution] Graph compiled!" << std::endl; - auto execution = std::make_shared<onert::exec::Execution>(executors); + std::shared_ptr<neurun::exec::IExecutor> executor; + compiler->release(executor); + auto execution = std::make_shared<neurun::exec::Execution>(executor); - // Setting IO - try + // Verify input shapes + auto num_inputs = inputs.size(); + for (size_t i = 0; i < num_inputs; i++) { - // Verify input shapes - auto num_inputs = inputs.size(); - for (size_t i = 0; i < num_inputs; i++) - { - auto input_operand_idx = g->getInputs().at(i); - auto input_shape = g->operands().at(input_operand_idx).shape(); - assert(inputs[i].size() == input_shape.num_elements()); - } + auto input_operand_idx = g->getInputs().at(i); + auto input_shape = g->operands().at(input_operand_idx).shape(); + assert(inputs[i].size() == input_shape.num_elements()); + } - // Set output shapes - auto num_outputs = g->getOutputs().size(); - outputs.resize(num_outputs); - for (uint32_t i = 0; i < num_outputs; i++) - { - auto output_operand_idx = g->getOutputs().at(i); - auto output_shape = g->operands().at(output_operand_idx).shape(); - outputs[i].resize(output_shape.num_elements()); - } + // Set output shapes + auto num_outputs = g->getOutputs().size(); + outputs.resize(num_outputs); + for (uint32_t i = 0; i < num_outputs; i++) + { + auto output_operand_idx = g->getOutputs().at(i); + auto output_shape = g->operands().at(output_operand_idx).shape(); + outputs[i].resize(output_shape.num_elements()); + } + // Setting IO + try + { for (size_t i = 0; i < num_inputs; i++) - execution->setInput(onert::ir::IOIndex(i), inputs[i].data(), + execution->setInput(neurun::model::IOIndex(i), inputs[i].data(), inputs[i].size() * sizeof(float)); for (uint32_t i = 0; i < num_outputs; i++) - execution->setOutput(onert::ir::IOIndex(i), outputs[i].data(), + execution->setOutput(neurun::model::IOIndex(i), outputs[i].data(), outputs[i].size() * sizeof(float)); } catch (const std::exception &e) @@ -133,17 +132,7 @@ void executeGraph(const std::shared_ptr<onert::ir::Graph> &g, exit(-1); } - try - { - execution->execute(); - } - catch (const std::exception &e) - { - std::cerr << "[Execution] Can't execute" << std::endl; - std::cerr << e.what() << '\n'; - exit(-1); - } - + execution->execute(); std::cout << "[Execution] Done!" << std::endl; delete compiler; @@ -163,12 +152,13 @@ int main(const int argc, char **argv) } std::cout << "[Execution] Stage start!" << std::endl; - std::shared_ptr<onert::ir::Graph> test_graph; + auto test_model = nnfw::cpp14::make_unique<neurun::model::Model>(); + auto test_graph = std::make_shared<neurun::graph::Graph>(std::move(test_model)); // Loading try { - test_graph = - onert::tflite_loader::loadModel(tflite_file.c_str())->at(onert::ir::SubgraphIndex{0}); + tflite_loader::Loader loader(*test_graph); + loader.loadFromFile(tflite_file.c_str()); } catch (std::exception &e) { @@ -182,12 +172,12 @@ int main(const int argc, char **argv) for (const auto &input_idx : test_graph->getInputs()) { const auto input_type = test_graph->operands().at(input_idx).typeInfo().type(); - assert(input_type == onert::ir::DataType::FLOAT32 && "Only FLOAT32 inputs are supported"); + assert(input_type == neurun::model::DataType::FLOAT32 && "Only FLOAT32 inputs are supported"); } for (const auto &output_idx : test_graph->getOutputs()) { const auto output_type = test_graph->operands().at(output_idx).typeInfo().type(); - assert(output_type == onert::ir::DataType::FLOAT32 && "Only FLOAT32 outputs are supported"); + assert(output_type == neurun::model::DataType::FLOAT32 && "Only FLOAT32 outputs are supported"); } std::cout << "[Execution] Model is deserialized!" << std::endl; @@ -209,29 +199,17 @@ int main(const int argc, char **argv) } const int seed = 1; /* TODO Add an option for seed value */ - nnfw::misc::RandomGenerator randgen{seed, 0.0f, 2.0f}; - try + RandomGenerator randgen{seed, 0.0f, 2.0f}; + for (uint32_t i = 0; i < num_inputs; i++) { - for (uint32_t i = 0; i < num_inputs; i++) + if (generate_data) { - if (generate_data) - { - uint64_t sz = - test_graph->operands().at(test_graph->getInputs().at(i)).shape().num_elements(); - inputs[i] = randomData(randgen, sz); - } - else /* read_data */ - inputs[i] = readData(data_files[i]); + uint64_t sz = test_graph->operands().at(test_graph->getInputs().at(i)).shape().num_elements(); + inputs[i] = randomData(randgen, sz); } + else /* read_data */ + inputs[i] = readData(data_files[i]); } - catch (std::exception &e) - { - std::cerr << "[ ERROR ] " - << "Failure during input data generation" << std::endl; - std::cerr << e.what() << std::endl; - exit(-1); - } - std::cout << "[Execution] Input data is defined!" << std::endl; std::vector<std::vector<float>> outputs; // Run graph diff --git a/tests/tools/tflite_run/CMakeLists.txt b/tests/tools/tflite_run/CMakeLists.txt index 3f30d3e32..1887d7cbf 100644 --- a/tests/tools/tflite_run/CMakeLists.txt +++ b/tests/tools/tflite_run/CMakeLists.txt @@ -7,16 +7,14 @@ list(APPEND TFLITE_RUN_SRCS "src/args.cc") list(APPEND TFLITE_RUN_SRCS "src/tensor_dumper.cc") list(APPEND TFLITE_RUN_SRCS "src/tensor_loader.cc") -nnfw_find_package(Boost REQUIRED program_options) +nnfw_find_package(Boost REQUIRED) add_executable(tflite_run ${TFLITE_RUN_SRCS}) target_include_directories(tflite_run PRIVATE src) target_include_directories(tflite_run PRIVATE ${Boost_INCLUDE_DIRS}) -target_link_libraries(tflite_run nnfw_lib_tflite) -target_link_libraries(tflite_run ${Boost_PROGRAM_OPTIONS_LIBRARY}) - -target_link_libraries(tflite_run nnfw_lib_benchmark) +target_link_libraries(tflite_run tensorflow-lite ${LIB_PTHREAD} dl nnfw_lib_tflite) +target_link_libraries(tflite_run boost_program_options boost_system boost_filesystem) install(TARGETS tflite_run DESTINATION bin) @@ -32,4 +30,4 @@ add_executable(tflite_test src/tflite_test.cc) ## Link test executable against gtest & gtest_main target_link_libraries(tflite_test gtest gtest_main ${LIB_PTHREAD}) ## install test binary for packaging -install(TARGETS tflite_test DESTINATION unittest_standalone) +install(TARGETS tflite_test DESTINATION unittest) diff --git a/tests/tools/tflite_run/src/args.cc b/tests/tools/tflite_run/src/args.cc index f8f581baf..6c85d884e 100644 --- a/tests/tools/tflite_run/src/args.cc +++ b/tests/tools/tflite_run/src/args.cc @@ -18,94 +18,35 @@ #include <iostream> +#include <boost/filesystem.hpp> + namespace TFLiteRun { Args::Args(const int argc, char **argv) noexcept { - try - { - Initialize(); - Parse(argc, argv); - } - catch (const std::exception &e) - { - std::cerr << "error during paring args" << e.what() << '\n'; - exit(1); - } + Initialize(); + Parse(argc, argv); } void Args::Initialize(void) { - auto process_input = [&](const std::string &v) { - _input_filename = v; - if (!_input_filename.empty()) - { - if (access(_input_filename.c_str(), F_OK) == -1) - { - std::cerr << "input image file not found: " << _input_filename << "\n"; - } - } - }; + // General options + po::options_description general("General options"); - auto process_tflite = [&](const std::string &v) { - _tflite_filename = v; - - if (_tflite_filename.empty()) - { - // TODO Print usage instead of the below message - std::cerr << "Please specify tflite file. Run with `--help` for usage." - << "\n"; - - exit(1); - } - else - { - if (access(_tflite_filename.c_str(), F_OK) == -1) - { - std::cerr << "tflite file not found: " << _tflite_filename << "\n"; - exit(1); - } - } - }; - - try - { - // General options - po::options_description general("General options"); - - // clang-format off + // clang-format off general.add_options() ("help,h", "Display available options") - ("input,i", po::value<std::string>()->default_value("")->notifier(process_input), "Input filename") - ("dump,d", po::value<std::string>()->default_value("")->notifier([&](const auto &v) { _dump_filename = v; }), "Output filename") - ("ishapes", po::value<std::vector<int>>()->multitoken()->notifier([&](const auto &v) { _input_shapes = v; }), "Input shapes") - ("compare,c", po::value<std::string>()->default_value("")->notifier([&](const auto &v) { _compare_filename = v; }), "filename to be compared with") - ("tflite", po::value<std::string>()->required()->notifier(process_tflite)) - ("num_runs,r", po::value<int>()->default_value(1)->notifier([&](const auto &v) { _num_runs = v; }), "The number of runs") - ("warmup_runs,w", po::value<int>()->default_value(0)->notifier([&](const auto &v) { _warmup_runs = v; }), "The number of warmup runs") - ("run_delay,t", po::value<int>()->default_value(-1)->notifier([&](const auto &v) { _run_delay = v; }), "Delay time(ms) between runs (as default no delay)") - ("gpumem_poll,g", po::value<bool>()->default_value(false)->notifier([&](const auto &v) { _gpumem_poll = v; }), "Check gpu memory polling separately") - ("mem_poll,m", po::value<bool>()->default_value(false), "Check memory polling") - ("write_report,p", po::value<bool>()->default_value(false)->notifier([&](const auto &v) { _write_report = v; }), "Write report") - ("validate", po::value<bool>()->default_value(true)->notifier([&](const auto &v) { _tflite_validate = v; }), "Validate tflite model") - ("verbose_level,v", po::value<int>()->default_value(0)->notifier([&](const auto &v) { _verbose_level = v; }), "Verbose level\n" - "0: prints the only result. Messages btw run don't print\n" - "1: prints result and message btw run\n" - "2: prints all of messages to print\n") - ; - // clang-format on - - _options.add(general); - _positional.add("tflite", 1); - } - catch (const std::bad_cast &e) - { - std::cerr << "error by bad cast during initialization of boost::program_options" << e.what() - << '\n'; - exit(1); - } + ("input,i", po::value<std::string>()->default_value(""), "Input filename") + ("dump,d", po::value<std::string>()->default_value(""), "Output filename") + ("ishapes", po::value<std::vector<int>>()->multitoken(), "Input shapes") + ("compare,c", po::value<std::string>()->default_value(""), "filename to be compared with") + ("tflite", po::value<std::string>()->required()); + // clang-format on + + _options.add(general); + _positional.add("tflite", 1); } void Args::Parse(const int argc, char **argv) @@ -138,14 +79,56 @@ void Args::Parse(const int argc, char **argv) po::notify(vm); - // This must be run after `notify` as `_warm_up_runs` must have been processed before. - if (vm.count("mem_poll")) + if (vm.count("dump")) + { + _dump_filename = vm["dump"].as<std::string>(); + } + + if (vm.count("compare")) + { + _compare_filename = vm["compare"].as<std::string>(); + } + + if (vm.count("input")) { - _mem_poll = vm["mem_poll"].as<bool>(); - // Instead of EXECUTE to avoid overhead, memory polling runs on WARMUP - if (_mem_poll && _warmup_runs == 0) + _input_filename = vm["input"].as<std::string>(); + + if (!_input_filename.empty()) { - _warmup_runs = 1; + if (!boost::filesystem::exists(_input_filename)) + { + std::cerr << "input image file not found: " << _input_filename << "\n"; + } + } + } + + if (vm.count("ishapes")) + { + _input_shapes.resize(vm["ishapes"].as<std::vector<int>>().size()); + for (auto i = 0; i < _input_shapes.size(); i++) + { + _input_shapes[i] = vm["ishapes"].as<std::vector<int>>()[i]; + } + } + + if (vm.count("tflite")) + { + _tflite_filename = vm["tflite"].as<std::string>(); + + if (_tflite_filename.empty()) + { + // TODO Print usage instead of the below message + std::cerr << "Please specify tflite file. Run with `--help` for usage." + << "\n"; + + exit(1); + } + else + { + if (!boost::filesystem::exists(_tflite_filename)) + { + std::cerr << "tflite file not found: " << _tflite_filename << "\n"; + } } } } diff --git a/tests/tools/tflite_run/src/args.h b/tests/tools/tflite_run/src/args.h index 054e47b05..25fd77a63 100644 --- a/tests/tools/tflite_run/src/args.h +++ b/tests/tools/tflite_run/src/args.h @@ -36,14 +36,6 @@ public: const std::string &getCompareFilename(void) const { return _compare_filename; } const std::string &getInputFilename(void) const { return _input_filename; } const std::vector<int> &getInputShapes(void) const { return _input_shapes; } - const int getNumRuns(void) const { return _num_runs; } - const int getWarmupRuns(void) const { return _warmup_runs; } - const int getRunDelay(void) const { return _run_delay; } - const bool getGpuMemoryPoll(void) const { return _gpumem_poll; } - const bool getMemoryPoll(void) const { return _mem_poll; } - const bool getWriteReport(void) const { return _write_report; } - const bool getModelValidate(void) const { return _tflite_validate; } - const int getVerboseLevel(void) const { return _verbose_level; } private: void Initialize(); @@ -58,14 +50,6 @@ private: std::string _compare_filename; std::string _input_filename; std::vector<int> _input_shapes; - int _num_runs; - int _warmup_runs; - int _run_delay; - bool _gpumem_poll; - bool _mem_poll; - bool _write_report; - bool _tflite_validate; - int _verbose_level; }; } // end of namespace TFLiteRun diff --git a/tests/tools/tflite_run/src/tensor_loader.cc b/tests/tools/tflite_run/src/tensor_loader.cc index 93d9e2f54..de605bacf 100644 --- a/tests/tools/tflite_run/src/tensor_loader.cc +++ b/tests/tools/tflite_run/src/tensor_loader.cc @@ -1,19 +1,3 @@ -/* - * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - #include "tensor_loader.h" #include <assert.h> diff --git a/tests/tools/tflite_run/src/tensor_loader.h b/tests/tools/tflite_run/src/tensor_loader.h index ef51e0fd4..2e671aa8a 100644 --- a/tests/tools/tflite_run/src/tensor_loader.h +++ b/tests/tools/tflite_run/src/tensor_loader.h @@ -1,19 +1,3 @@ -/* - * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - #ifndef __TFLITE_RUN_TENSOR_LOADER_H__ #define __TFLITE_RUN_TENSOR_LOADER_H__ diff --git a/tests/tools/tflite_run/src/tflite_run.cc b/tests/tools/tflite_run/src/tflite_run.cc index e72966db5..deed12856 100644 --- a/tests/tools/tflite_run/src/tflite_run.cc +++ b/tests/tools/tflite_run/src/tflite_run.cc @@ -30,90 +30,51 @@ #include "tflite/NNAPISession.h" #include "misc/tensor/IndexIterator.h" #include "misc/tensor/Object.h" -#include "benchmark.h" #include <iostream> #include <chrono> #include <algorithm> -#include <vector> - -#include <libgen.h> using namespace tflite; using namespace nnfw::tflite; using namespace std::placeholders; // for _1, _2 ... -namespace -{ - void print_max_idx(float *f, int size) { float *p = std::max_element(f, f + size); std::cout << "max:" << p - f; } -static const char *default_backend_cand = "tflite_cpu"; - -// Verifies whether the model is a flatbuffer file. -class BMFlatBufferVerifier : public tflite::TfLiteVerifier +int main(const int argc, char **argv) { -public: - bool Verify(const char *data, int length, tflite::ErrorReporter *reporter) override - { + bool use_nnapi = false; - flatbuffers::Verifier verifier(reinterpret_cast<const uint8_t *>(data), length); - if (!tflite::VerifyModelBuffer(verifier)) - { - reporter->Report("The model is not a valid Flatbuffer file"); - return false; - } - return true; + if (std::getenv("USE_NNAPI") != nullptr) + { + use_nnapi = true; } -}; - -} // namespace anonymous - -int main(const int argc, char **argv) -{ - const bool use_nnapi = nnfw::misc::EnvVar("USE_NNAPI").asBool(false); StderrReporter error_reporter; TFLiteRun::Args args(argc, argv); - std::chrono::milliseconds t_model_load(0), t_prepare(0); - - // TODO Apply verbose level to phases - const int verbose = args.getVerboseLevel(); - benchmark::Phases phases( - benchmark::PhaseOption{args.getMemoryPoll(), args.getGpuMemoryPoll(), args.getRunDelay()}); - - std::unique_ptr<FlatBufferModel> model; + auto model = FlatBufferModel::BuildFromFile(args.getTFLiteFilename().c_str(), &error_reporter); std::unique_ptr<Interpreter> interpreter; - std::unique_ptr<tflite::TfLiteVerifier> verifier{new BMFlatBufferVerifier}; + + std::chrono::milliseconds t_prepare(0); + std::chrono::milliseconds t_invoke(0); try { - phases.run("MODEL_LOAD", [&](const benchmark::Phase &, uint32_t) { - if (args.getModelValidate()) - { - model = FlatBufferModel::VerifyAndBuildFromFile(args.getTFLiteFilename().c_str(), - verifier.get(), &error_reporter); - } - else - { - model = FlatBufferModel::BuildFromFile(args.getTFLiteFilename().c_str(), &error_reporter); - } - if (model == nullptr) - { - throw std::runtime_error{"Cannot create model"}; - } - + nnfw::misc::benchmark::measure(t_prepare) << [&](void) { BuiltinOpResolver resolver; + InterpreterBuilder builder(*model, resolver); + TFLITE_ENSURE(builder(&interpreter)) + interpreter->SetNumThreads(nnfw::misc::EnvVar("THREAD").asInt(-1)); - }); + }; } catch (const std::exception &e) { @@ -132,15 +93,7 @@ int main(const int argc, char **argv) sess = std::make_shared<nnfw::tflite::InterpreterSession>(interpreter.get()); } - try - { - phases.run("PREPARE", [&](const benchmark::Phase &, uint32_t) { sess->prepare(); }); - } - catch (const std::exception &e) - { - std::cerr << e.what() << '\n'; - return 1; - } + sess->prepare(); if (args.getInputShapes().size() != 0) { @@ -194,7 +147,7 @@ int main(const int argc, char **argv) else { const int seed = 1; /* TODO Add an option for seed value */ - nnfw::misc::RandomGenerator randgen{seed, 0.0f, 2.0f}; + RandomGenerator randgen{seed, 0.0f, 2.0f}; // No input specified. So we fill the input tensors with random values. for (const auto &o : interpreter->inputs()) @@ -220,16 +173,13 @@ int main(const int argc, char **argv) // Generate unsigned 8-bit integer input auto tensor_view = nnfw::tflite::TensorView<uint8_t>::make(*interpreter, o); - auto fp = static_cast<uint8_t (nnfw::misc::RandomGenerator::*)( - const ::nnfw::misc::tensor::Shape &, const ::nnfw::misc::tensor::Index &)>( - &nnfw::misc::RandomGenerator::generate<uint8_t>); - const nnfw::misc::tensor::Object<uint8_t> data(tensor_view.shape(), - std::bind(fp, randgen, _1, _2)); + uint8_t value = 0; nnfw::misc::tensor::iterate(tensor_view.shape()) << [&](const nnfw::misc::tensor::Index &ind) { - const auto value = data.at(ind); + // TODO Generate random values tensor_view.at(ind) = value; + value = (value + 1) & 0xFF; }; } else if (tensor->type == kTfLiteBool) @@ -237,9 +187,9 @@ int main(const int argc, char **argv) // Generate bool input auto tensor_view = nnfw::tflite::TensorView<bool>::make(*interpreter, o); - auto fp = static_cast<bool (nnfw::misc::RandomGenerator::*)( - const ::nnfw::misc::tensor::Shape &, const ::nnfw::misc::tensor::Index &)>( - &nnfw::misc::RandomGenerator::generate<bool>); + auto fp = static_cast<bool (RandomGenerator::*)(const ::nnfw::misc::tensor::Shape &, + const ::nnfw::misc::tensor::Index &)>( + &RandomGenerator::generate<bool>); const nnfw::misc::tensor::Object<bool> data(tensor_view.shape(), std::bind(fp, randgen, _1, _2)); @@ -273,32 +223,12 @@ int main(const int argc, char **argv) } std::cout << "]" << std::endl; - // NOTE: Measuring memory can't avoid taking overhead. Therefore, memory will be measured on the - // only warmup. - if (verbose == 0) - { - phases.run("WARMUP", [&](const benchmark::Phase &, uint32_t) { sess->run(); }, - args.getWarmupRuns()); - phases.run("EXECUTE", [&](const benchmark::Phase &, uint32_t) { sess->run(); }, - args.getNumRuns(), true); - } - else - { - phases.run("WARMUP", [&](const benchmark::Phase &, uint32_t) { sess->run(); }, - [&](const benchmark::Phase &phase, uint32_t nth) { - std::cout << "... " - << "warmup " << nth + 1 << " takes " << phase.time[nth] / 1e3 << " ms" - << std::endl; - }, - args.getWarmupRuns()); - phases.run("EXECUTE", [&](const benchmark::Phase &, uint32_t) { sess->run(); }, - [&](const benchmark::Phase &phase, uint32_t nth) { - std::cout << "... " - << "run " << nth + 1 << " takes " << phase.time[nth] / 1e3 << " ms" - << std::endl; - }, - args.getNumRuns(), true); - } + nnfw::misc::benchmark::measure(t_invoke) << [&sess](void) { + if (!sess->run()) + { + assert(0 && "run failed!"); + } + }; sess->teardown(); @@ -316,29 +246,8 @@ int main(const int argc, char **argv) } std::cout << "]" << std::endl; - // TODO Apply verbose level to result - - // prepare result - benchmark::Result result(phases); - - // to stdout - benchmark::printResult(result); - - if (args.getWriteReport()) - { - // prepare csv task - std::string exec_basename; - std::string model_basename; - std::string backend_name = default_backend_cand; - { - std::vector<char> vpath(args.getTFLiteFilename().begin(), args.getTFLiteFilename().end() + 1); - model_basename = basename(vpath.data()); - size_t lastindex = model_basename.find_last_of("."); - model_basename = model_basename.substr(0, lastindex); - exec_basename = basename(argv[0]); - } - benchmark::writeResult(result, exec_basename, model_basename, backend_name); - } + std::cout << "Prepare takes " << t_prepare.count() / 1000.0 << " seconds" << std::endl; + std::cout << "Invoke takes " << t_invoke.count() / 1000.0 << " seconds" << std::endl; if (!args.getDumpFilename().empty()) { diff --git a/tests/tools/tflite_vanilla_run/CMakeLists.txt b/tests/tools/tflite_vanilla_run/CMakeLists.txt deleted file mode 100644 index 19e21e923..000000000 --- a/tests/tools/tflite_vanilla_run/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -if(NOT BUILD_TFLITE_VANILLA_RUN) - return() -endif() - -if(NOT BUILD_TENSORFLOW_LITE_2_3_0) - set(BUILD_TENSORFLOW_LITE_2_3_0 ON) -endif() - -nnfw_find_package(TensorFlowLite-2.3.0 REQUIRED) -nnfw_find_package(Boost REQUIRED) - -list(APPEND TFLITE_RUN_SRCS "src/tflite_vanilla_run.cc") -list(APPEND TFLITE_RUN_SRCS "src/args.cc") - -add_executable(tflite_vanilla_run ${TFLITE_RUN_SRCS}) -target_include_directories(tflite_vanilla_run PRIVATE src) -target_include_directories(tflite_vanilla_run PRIVATE ${Boost_INCLUDE_DIRS}) - -target_link_libraries(tflite_vanilla_run tensorflow-lite-2.3.0 ${LIB_PTHREAD} dl) -target_link_libraries(tflite_vanilla_run ${Boost_PROGRAM_OPTIONS_LIBRARY}) -target_link_libraries(tflite_vanilla_run nnfw_lib_benchmark nnfw_lib_misc) - -install(TARGETS tflite_vanilla_run DESTINATION bin) diff --git a/tests/tools/tflite_vanilla_run/src/args.cc b/tests/tools/tflite_vanilla_run/src/args.cc deleted file mode 100644 index dc9f250e4..000000000 --- a/tests/tools/tflite_vanilla_run/src/args.cc +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "args.h" - -#include <iostream> - -namespace TFLiteVanillaRun -{ - -Args::Args(const int argc, char **argv) noexcept -{ - try - { - Initialize(); - Parse(argc, argv); - } - catch (const std::exception &e) - { - std::cerr << "error during paring args" << e.what() << '\n'; - exit(1); - } -} - -void Args::Initialize(void) -{ - try - { - // General options - po::options_description general("General options"); - - // clang-format off - general.add_options() - ("help,h", "Display available options") - ("input,i", po::value<std::string>()->default_value(""), "Input filename") - ("dump,d", po::value<std::string>()->default_value(""), "Output filename") - ("ishapes", po::value<std::vector<int>>()->multitoken(), "Input shapes") - ("compare,c", po::value<std::string>()->default_value(""), "filename to be compared with") - ("tflite", po::value<std::string>()->required()) - ("num_runs,r", po::value<int>()->default_value(1), "The number of runs") - ("warmup_runs,w", po::value<int>()->default_value(0), "The number of warmup runs") - ("run_delay,t", po::value<int>()->default_value(-1), "Delay time(ms) between runs (as default no delay") - ("gpumem_poll,g", po::value<bool>()->default_value(false), "Check gpu memory polling separately") - ("mem_poll,m", po::value<bool>()->default_value(false), "Check memory polling") - ("write_report,p", po::value<bool>()->default_value(false), "Write report") - ("validate", po::value<bool>()->default_value(true), "Validate tflite model") - ("verbose_level,v", po::value<int>()->default_value(0), "Verbose level\n" - "0: prints the only result. Messages btw run don't print\n" - "1: prints result and message btw run\n" - "2: prints all of messages to print\n") - ; - // clang-format on - - _options.add(general); - _positional.add("tflite", 1); - } - catch (const std::bad_cast &e) - { - std::cerr << "error by bad cast during initialization of boost::program_options" << e.what() - << '\n'; - exit(1); - } -} - -void Args::Parse(const int argc, char **argv) -{ - po::variables_map vm; - po::store(po::command_line_parser(argc, argv).options(_options).positional(_positional).run(), - vm); - - { - auto conflicting_options = [&](const std::string &o1, const std::string &o2) { - if ((vm.count(o1) && !vm[o1].defaulted()) && (vm.count(o2) && !vm[o2].defaulted())) - { - throw boost::program_options::error(std::string("Two options '") + o1 + "' and '" + o2 + - "' cannot be given at once."); - } - }; - - conflicting_options("input", "compare"); - } - - if (vm.count("help")) - { - std::cout << "tflite_run\n\n"; - std::cout << "Usage: " << argv[0] << " <.tflite> [<options>]\n\n"; - std::cout << _options; - std::cout << "\n"; - - exit(0); - } - - po::notify(vm); - - if (vm.count("dump")) - { - _dump_filename = vm["dump"].as<std::string>(); - } - - if (vm.count("compare")) - { - _compare_filename = vm["compare"].as<std::string>(); - } - - if (vm.count("input")) - { - _input_filename = vm["input"].as<std::string>(); - - if (!_input_filename.empty()) - { - if (access(_input_filename.c_str(), F_OK) == -1) - { - std::cerr << "input image file not found: " << _input_filename << "\n"; - } - } - } - - if (vm.count("ishapes")) - { - _input_shapes.resize(vm["ishapes"].as<std::vector<int>>().size()); - for (auto i = 0; i < _input_shapes.size(); i++) - { - _input_shapes[i] = vm["ishapes"].as<std::vector<int>>()[i]; - } - } - - if (vm.count("tflite")) - { - _tflite_filename = vm["tflite"].as<std::string>(); - - if (_tflite_filename.empty()) - { - // TODO Print usage instead of the below message - std::cerr << "Please specify tflite file. Run with `--help` for usage." - << "\n"; - - exit(1); - } - else - { - if (access(_tflite_filename.c_str(), F_OK) == -1) - { - std::cerr << "tflite file not found: " << _tflite_filename << "\n"; - exit(1); - } - } - } - - if (vm.count("num_runs")) - { - _num_runs = vm["num_runs"].as<int>(); - } - - if (vm.count("warmup_runs")) - { - _warmup_runs = vm["warmup_runs"].as<int>(); - } - - if (vm.count("run_delay")) - { - _run_delay = vm["run_delay"].as<int>(); - } - - if (vm.count("gpumem_poll")) - { - _gpumem_poll = vm["gpumem_poll"].as<bool>(); - } - - if (vm.count("mem_poll")) - { - _mem_poll = vm["mem_poll"].as<bool>(); - // Instead of EXECUTE to avoid overhead, memory polling runs on WARMUP - if (_mem_poll && _warmup_runs == 0) - { - _warmup_runs = 1; - } - } - - if (vm.count("write_report")) - { - _write_report = vm["write_report"].as<bool>(); - } - - if (vm.count("validate")) - { - _tflite_validate = vm["validate"].as<bool>(); - } - - if (vm.count("verbose_level")) - { - _verbose_level = vm["verbose_level"].as<int>(); - } -} - -} // end of namespace TFLiteVanillaRun diff --git a/tests/tools/tflite_vanilla_run/src/args.h b/tests/tools/tflite_vanilla_run/src/args.h deleted file mode 100644 index 3605b651c..000000000 --- a/tests/tools/tflite_vanilla_run/src/args.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __TFLITE_VANILLA_RUN_ARGS_H__ -#define __TFLITE_VANILLA_RUN_ARGS_H__ - -#include <string> -#include <boost/program_options.hpp> - -namespace po = boost::program_options; - -namespace TFLiteVanillaRun -{ - -class Args -{ -public: - Args(const int argc, char **argv) noexcept; - void print(void); - - const std::string &getTFLiteFilename(void) const { return _tflite_filename; } - const std::string &getDumpFilename(void) const { return _dump_filename; } - const std::string &getCompareFilename(void) const { return _compare_filename; } - const std::string &getInputFilename(void) const { return _input_filename; } - const std::vector<int> &getInputShapes(void) const { return _input_shapes; } - const int getNumRuns(void) const { return _num_runs; } - const int getWarmupRuns(void) const { return _warmup_runs; } - const int getRunDelay(void) const { return _run_delay; } - const bool getGpuMemoryPoll(void) const { return _gpumem_poll; } - const bool getMemoryPoll(void) const { return _mem_poll; } - const bool getWriteReport(void) const { return _write_report; } - const bool getModelValidate(void) const { return _tflite_validate; } - const int getVerboseLevel(void) const { return _verbose_level; } - -private: - void Initialize(); - void Parse(const int argc, char **argv); - -private: - po::positional_options_description _positional; - po::options_description _options; - - std::string _tflite_filename; - std::string _dump_filename; - std::string _compare_filename; - std::string _input_filename; - std::vector<int> _input_shapes; - int _num_runs; - int _warmup_runs; - int _run_delay; - bool _gpumem_poll; - bool _mem_poll; - bool _write_report; - bool _tflite_validate; - int _verbose_level; -}; - -} // end of namespace TFLiteVanillaRun - -#endif // __TFLITE_VANILLA_RUN_ARGS_H__ diff --git a/tests/tools/tflite_vanilla_run/src/tensor_view.h b/tests/tools/tflite_vanilla_run/src/tensor_view.h deleted file mode 100644 index ca04a051e..000000000 --- a/tests/tools/tflite_vanilla_run/src/tensor_view.h +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @file TensorView.h - * @brief This file contains TensorView class - * @ingroup COM_AI_RUNTIME - */ - -#ifndef __TFLITE_VANILLA_RUN_TENSOR_VIEW_H__ -#define __TFLITE_VANILLA_RUN_TENSOR_VIEW_H__ - -#include "tensorflow/lite/interpreter.h" - -#include "misc/tensor/Shape.h" -#include "misc/tensor/Index.h" -#include "misc/tensor/Reader.h" -#include "misc/tensor/NonIncreasingStride.h" - -namespace TFLiteVanillaRun -{ - -/** - * @brief Class to define TensorView which is inherited from nnfw::misc::tensor::Reader<T> class - */ -template <typename T> class TensorView final : public nnfw::misc::tensor::Reader<T> -{ -public: - /** - * @brief Construct a TensorView object with base and shape informations - * @param[in] shape The shape of a tensor - * @param[in] base The base address of a tensor - */ - TensorView(const nnfw::misc::tensor::Shape &shape, T *base) : _shape{shape}, _base{base} - { - // Set 'stride' - _stride.init(_shape); - } - -public: - /** - * @brief Get shape of tensor - * @return Reference of shape - */ - const nnfw::misc::tensor::Shape &shape(void) const { return _shape; } - -public: - /** - * @brief Get value of tensor index - * @param[in] index The tensor index - * @return The value at the index - */ - T at(const nnfw::misc::tensor::Index &index) const override - { - const auto offset = _stride.offset(index); - return *(_base + offset); - } - -public: - /** - * @brief Get reference value of tensor index - * @param[in] index The tensor index - * @return The reference value at the index - */ - T &at(const nnfw::misc::tensor::Index &index) - { - const auto offset = _stride.offset(index); - return *(_base + offset); - } - -private: - nnfw::misc::tensor::Shape _shape; /**< The tensor shape */ - -public: - T *_base; /**< The base address of tensor */ - nnfw::misc::tensor::NonIncreasingStride _stride; /**< The NonIncreasingStride object */ - -public: - // TODO Introduce Operand ID class - /** - * @brief Create TensorView object using given parameters - * @param[in] interp The TfLite interpreter - * @param[in] tensor_index The tensor index - * @return The new TensorView<T> object - */ - static TensorView<T> make(::tflite::Interpreter &interp, int tensor_index) - { - auto tensor_ptr = interp.tensor(tensor_index); - - // Set 'shape' - nnfw::misc::tensor::Shape shape(tensor_ptr->dims->size); - - for (uint32_t axis = 0; axis < shape.rank(); ++axis) - { - shape.dim(axis) = tensor_ptr->dims->data[axis]; - } - - return TensorView<T>(shape, interp.typed_tensor<T>(tensor_index)); - } -}; - -} // namespace TFLiteVanillaRun - -#endif // __TFLITE_VANILLA_RUN_TENSOR_VIEW_H__ diff --git a/tests/tools/tflite_vanilla_run/src/tflite_vanilla_run.cc b/tests/tools/tflite_vanilla_run/src/tflite_vanilla_run.cc deleted file mode 100644 index d44ea60cf..000000000 --- a/tests/tools/tflite_vanilla_run/src/tflite_vanilla_run.cc +++ /dev/null @@ -1,276 +0,0 @@ -/* - * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorflow/lite/model.h" -#include "tensorflow/lite/kernels/register.h" - -#include "args.h" -#include "tensor_view.h" -#include "misc/EnvVar.h" -#include "misc/RandomGenerator.h" -#include "misc/tensor/IndexIterator.h" -#include "misc/tensor/Object.h" -#include "benchmark.h" - -#include <iostream> -#include <chrono> -#include <algorithm> -#include <vector> -#include <memory> - -using namespace std::placeholders; // for _1, _2 ... - -#define TFLITE_ENSURE(exp) \ - { \ - const TfLiteStatus status = (exp); \ - \ - if (status != kTfLiteOk) \ - { \ - std::ostringstream ss; \ - ss << #exp << " failed (" << __FILE__ << ":" << __LINE__ << ")"; \ - throw std::runtime_error{ss.str()}; \ - } \ - } - -namespace -{ - -void print_max_idx(float *f, int size) -{ - float *p = std::max_element(f, f + size); - std::cout << "max:" << p - f; -} - -static const char *default_backend_cand = "tflite_cpu"; - -// Verifies whether the model is a flatbuffer file. -class BMFlatBufferVerifier : public tflite::TfLiteVerifier -{ -public: - bool Verify(const char *data, int length, tflite::ErrorReporter *reporter) override - { - - flatbuffers::Verifier verifier(reinterpret_cast<const uint8_t *>(data), length); - if (!tflite::VerifyModelBuffer(verifier)) - { - reporter->Report("The model is not a valid Flatbuffer file"); - return false; - } - return true; - } -}; - -} // namespace anonymous - -int main(const int argc, char **argv) -{ - tflite::StderrReporter error_reporter; - - TFLiteVanillaRun::Args args(argc, argv); - - std::chrono::milliseconds t_model_load(0), t_prepare(0); - - // TODO Apply verbose level to phases - const int verbose = args.getVerboseLevel(); - benchmark::Phases phases( - benchmark::PhaseOption{args.getMemoryPoll(), args.getGpuMemoryPoll(), args.getRunDelay()}); - - std::unique_ptr<tflite::FlatBufferModel> model; - std::unique_ptr<tflite::Interpreter> interpreter; - std::unique_ptr<tflite::TfLiteVerifier> verifier{new BMFlatBufferVerifier}; - - try - { - phases.run("MODEL_LOAD", [&](const benchmark::Phase &, uint32_t) { - if (args.getModelValidate()) - { - model = tflite::FlatBufferModel::VerifyAndBuildFromFile(args.getTFLiteFilename().c_str(), - verifier.get(), &error_reporter); - } - else - { - model = tflite::FlatBufferModel::BuildFromFile(args.getTFLiteFilename().c_str(), - &error_reporter); - } - if (model == nullptr) - { - throw std::runtime_error{"Cannot create model"}; - } - - // Use tflite's resolver, not onert's one - tflite::ops::builtin::BuiltinOpResolver resolver; - tflite::InterpreterBuilder builder(*model, resolver); - TFLITE_ENSURE(builder(&interpreter)) - interpreter->SetNumThreads(nnfw::misc::EnvVar("THREAD").asInt(-1)); - }); - } - catch (const std::exception &e) - { - std::cerr << e.what() << '\n'; - return 1; - } - - const bool use_nnapi = nnfw::misc::EnvVar("USE_NNAPI").asBool(false); - - try - { - phases.run("PREPARE", [&](const benchmark::Phase &, uint32_t) { - interpreter->UseNNAPI(use_nnapi); - interpreter->AllocateTensors(); - }); - } - catch (const std::exception &e) - { - std::cerr << e.what() << '\n'; - return 1; - } - - const int seed = 1; /* TODO Add an option for seed value */ - nnfw::misc::RandomGenerator randgen{seed, 0.0f, 2.0f}; - - // No input specified. So we fill the input tensors with random values. - for (const auto &o : interpreter->inputs()) - { - TfLiteTensor *tensor = interpreter->tensor(o); - if (tensor->type == kTfLiteInt32) - { - // Generate singed 32-bit integer (s32) input - auto tensor_view = TFLiteVanillaRun::TensorView<int32_t>::make(*interpreter, o); - - int32_t value = 0; - - nnfw::misc::tensor::iterate(tensor_view.shape()) - << [&](const nnfw::misc::tensor::Index &ind) { - // TODO Generate random values - // Gather operation: index should be within input coverage. - tensor_view.at(ind) = value; - value++; - }; - } - else if (tensor->type == kTfLiteUInt8) - { - // Generate unsigned 8-bit integer input - auto tensor_view = TFLiteVanillaRun::TensorView<uint8_t>::make(*interpreter, o); - - uint8_t value = 0; - - nnfw::misc::tensor::iterate(tensor_view.shape()) - << [&](const nnfw::misc::tensor::Index &ind) { - // TODO Generate random values - tensor_view.at(ind) = value; - value = (value + 1) & 0xFF; - }; - } - else if (tensor->type == kTfLiteBool) - { - // Generate bool input - auto tensor_view = TFLiteVanillaRun::TensorView<bool>::make(*interpreter, o); - - auto fp = static_cast<bool (nnfw::misc::RandomGenerator::*)( - const ::nnfw::misc::tensor::Shape &, const ::nnfw::misc::tensor::Index &)>( - &nnfw::misc::RandomGenerator::generate<bool>); - const nnfw::misc::tensor::Object<bool> data(tensor_view.shape(), - std::bind(fp, randgen, _1, _2)); - - nnfw::misc::tensor::iterate(tensor_view.shape()) - << [&](const nnfw::misc::tensor::Index &ind) { - const auto value = data.at(ind); - tensor_view.at(ind) = value; - }; - } - else - { - assert(tensor->type == kTfLiteFloat32); - - const float *end = reinterpret_cast<const float *>(tensor->data.raw_const + tensor->bytes); - for (float *ptr = tensor->data.f; ptr < end; ptr++) - { - *ptr = randgen.generate<float>(); - } - } - } - - std::cout << "input tensor indices = ["; - for (const auto &o : interpreter->inputs()) - { - std::cout << o << ","; - } - std::cout << "]" << std::endl; - - // NOTE: Measuring memory can't avoid taking overhead. Therefore, memory will be measured on the - // only warmup. - if (verbose == 0) - { - phases.run("WARMUP", [&](const benchmark::Phase &, uint32_t) { interpreter->Invoke(); }, - args.getWarmupRuns()); - phases.run("EXECUTE", [&](const benchmark::Phase &, uint32_t) { interpreter->Invoke(); }, - args.getNumRuns(), true); - } - else - { - phases.run("WARMUP", [&](const benchmark::Phase &, uint32_t) { interpreter->Invoke(); }, - [&](const benchmark::Phase &phase, uint32_t nth) { - std::cout << "... " - << "warmup " << nth + 1 << " takes " << phase.time[nth] / 1e3 << " ms" - << std::endl; - }, - args.getWarmupRuns()); - phases.run("EXECUTE", [&](const benchmark::Phase &, uint32_t) { interpreter->Invoke(); }, - [&](const benchmark::Phase &phase, uint32_t nth) { - std::cout << "... " - << "run " << nth + 1 << " takes " << phase.time[nth] / 1e3 << " ms" - << std::endl; - }, - args.getNumRuns(), true); - } - - std::cout << "output tensor indices = ["; - for (const auto &o : interpreter->outputs()) - { - std::cout << o << "("; - - print_max_idx(interpreter->tensor(o)->data.f, interpreter->tensor(o)->bytes / sizeof(float)); - - std::cout << "),"; - } - std::cout << "]" << std::endl; - - // TODO Apply verbose level to result - - // prepare result - benchmark::Result result(phases); - - // to stdout - benchmark::printResult(result); - - if (args.getWriteReport()) - { - // prepare csv task - std::string exec_basename; - std::string model_basename; - std::string backend_name = default_backend_cand; - { - std::vector<char> vpath(args.getTFLiteFilename().begin(), args.getTFLiteFilename().end() + 1); - model_basename = basename(vpath.data()); - size_t lastindex = model_basename.find_last_of("."); - model_basename = model_basename.substr(0, lastindex); - exec_basename = basename(argv[0]); - } - benchmark::writeResult(result, exec_basename, model_basename, backend_name); - } - - return 0; -} |