diff options
Diffstat (limited to 'libs/support/tflite/src/Diff.cpp')
-rw-r--r-- | libs/support/tflite/src/Diff.cpp | 262 |
1 files changed, 262 insertions, 0 deletions
diff --git a/libs/support/tflite/src/Diff.cpp b/libs/support/tflite/src/Diff.cpp new file mode 100644 index 000000000..f382df2d6 --- /dev/null +++ b/libs/support/tflite/src/Diff.cpp @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "support/tflite/Diff.h" + +#include "util/fp32.h" + +#include "util/tensor/IndexIterator.h" +#include "util/tensor/IndexFormatter.h" +#include "util/tensor/Zipper.h" + +#include <iostream> + +class DiffSummary : public TfLiteTensorComparator::Observer +{ +public: + DiffSummary() + : max_abs_diff_index(0), max_abs_diff_value{0.0f}, + max_rel_diff_index(0), max_rel_diff_value{0.0f} + { + // DO NOTHING + } +public: + void notify(const nnfw::util::tensor::Index &index, float expected, float obtained) override; + +public: + nnfw::util::tensor::Index max_abs_diff_index; + float max_abs_diff_expected; + float max_abs_diff_obtained; + float max_abs_diff_value; + + nnfw::util::tensor::Index max_rel_diff_index; + float max_rel_diff_expected; + float max_rel_diff_obtained; + float max_rel_diff_value; +}; + +void DiffSummary::notify(const nnfw::util::tensor::Index &index, float expected, float obtained) +{ + const auto abs_diff_value = std::fabs(expected - obtained); + + if (max_abs_diff_value < abs_diff_value) + { + max_abs_diff_index = index; + max_abs_diff_value = abs_diff_value; + max_abs_diff_expected = expected; + max_abs_diff_obtained = obtained; + } + + const auto rel_diff_value = nnfw::util::fp32::relative_diff(expected, obtained); + + if (max_rel_diff_value < rel_diff_value) + { + max_rel_diff_index = index; + max_rel_diff_value = rel_diff_value; + max_rel_diff_expected = expected; + max_rel_diff_obtained = obtained; + } +} + +std::vector<TfLiteTensorDiff> +TfLiteTensorComparator::compare(const nnfw::support::tflite::TensorView<float> &expected, + const nnfw::support::tflite::TensorView<float> &obtained, + Observer *observer) const +{ + std::vector<TfLiteTensorDiff> res; + + assert(expected.shape() == obtained.shape()); + + nnfw::util::tensor::zip(expected.shape(), expected, obtained) << + [&] (const nnfw::util::tensor::Index &index, float expected_value, float obtained_value) + { + const auto relative_diff = nnfw::util::fp32::relative_diff(expected_value, obtained_value); + + if (!_compare_fn(expected_value, obtained_value)) + { + TfLiteTensorDiff diff(index); + + diff.expected = expected_value; + diff.obtained = obtained_value; + + res.emplace_back(diff); + } + + // Update max_diff_index, if necessary + if (observer != nullptr) + { + observer->notify(index, expected_value, obtained_value); + } + }; + + return res; +} + +bool TfLiteInterpMatchApp::run(::tflite::Interpreter &interp, ::tflite::Interpreter &nnapi) const +{ + assert(interp.outputs() == nnapi.outputs()); + + for (const auto &id : interp.outputs()) + { + const auto expected = nnfw::support::tflite::TensorView<float>::make(interp, id); + const auto obtained = nnfw::support::tflite::TensorView<float>::make(nnapi, id); + + DiffSummary summary; + + auto diffs = _comparator.compare(expected, obtained, &summary); + + if (diffs.size() == 0) + { + std::cout << " Tensor #" << id << ": MATCHED" << std::endl; + } + else + { + std::cout << " Tensor #" << id << ": UNMATCHED" << std::endl; + std::cout << " " << diffs.size() << " diffs are detected" << std::endl; + } + + // Print out max_diff + if (summary.max_abs_diff_value > 0) + { + std::cout << " Max absolute diff at [" << nnfw::util::tensor::IndexFormatter(summary.max_abs_diff_index) << "]" << std::endl; + std::cout << " expected: " << summary.max_abs_diff_expected << std::endl; + std::cout << " obtained: " << summary.max_abs_diff_obtained << std::endl; + std::cout << " absolute diff: " << summary.max_abs_diff_value << std::endl; + } + + if (summary.max_rel_diff_value > 0) + { + const auto tolerance_level = summary.max_rel_diff_value / FLT_EPSILON; + + std::cout << " Max relative diff at [" << nnfw::util::tensor::IndexFormatter(summary.max_rel_diff_index) << "]" << std::endl; + std::cout << " expected: " << summary.max_rel_diff_expected << std::endl; + std::cout << " obtained: " << summary.max_rel_diff_obtained << std::endl; + std::cout << " relative diff: " << summary.max_rel_diff_value << std::endl; + std::cout << " (tolerance level = " << tolerance_level << ")" << std::endl; + } + + if (diffs.size() > 0) + { + if (_verbose != 0) + { + std::cout << " ---- Details ---" << std::endl; + for (const auto &diff : diffs) + { + const auto absolute_diff = std::fabs(diff.expected - diff.obtained); + const auto relative_diff = nnfw::util::fp32::relative_diff(diff.expected, diff.obtained); + const auto tolerance_level = relative_diff / FLT_EPSILON; + + std::cout << " Diff at [" << nnfw::util::tensor::IndexFormatter(diff.index) << "]" << std::endl; + std::cout << " expected: " << diff.expected << std::endl; + std::cout << " obtained: " << diff.obtained << std::endl; + std::cout << " absolute diff: " << absolute_diff << std::endl; + std::cout << " relative diff: " << relative_diff << std::endl; + std::cout << " (tolerance level = " << tolerance_level << ")" << std::endl; + } + } + + return false; + } + } + + return true; +} + +#include "util/tensor/Object.h" + +// +// Random Test Runner +// +int RandomTestRunner::run(const nnfw::support::tflite::interp::Builder &builder) +{ + auto pure = builder.build(); + auto nnapi = builder.build(); + + pure->UseNNAPI(false); + nnapi->UseNNAPI(true); + + // Allocate Tensors + pure->AllocateTensors(); + nnapi->AllocateTensors(); + + assert(pure->inputs() == nnapi->inputs()); + + // Fill IFM with random numbers + auto ifm_gen = [this] (const nnfw::util::tensor::Shape &, const nnfw::util::tensor::Index &) + { + // TODO Allow users to set min/max and distribution + std::normal_distribution<float> dist(0.0f, 2.0f); + return dist(_rand); + }; + + for (const auto id : pure->inputs()) + { + auto pure_view = nnfw::support::tflite::TensorView<float>::make(*pure, id); + auto nnapi_view = nnfw::support::tflite::TensorView<float>::make(*nnapi, id); + + assert(pure_view.shape() == nnapi_view.shape()); + + const nnfw::util::tensor::Object<float> data(pure_view.shape(), ifm_gen); + + assert(pure_view.shape() == data.shape()); + + nnfw::util::tensor::iterate(pure_view.shape()) << [&] (const nnfw::util::tensor::Index &ind) + { + const auto value = data.at(ind); + + pure_view.at(ind) = value; + nnapi_view.at(ind) = value; + }; + } + + std::cout << "[NNAPI TEST] Run T/F Lite Interpreter without NNAPI" << std::endl; + pure->Invoke(); + + std::cout << "[NNAPI TEST] Run T/F Lite Interpreter with NNAPI" << std::endl; + nnapi->Invoke(); + + // Compare OFM + std::cout << "[NNAPI TEST] Compare the result" << std::endl; + + const auto tolerance = _param.tolerance; + + auto equals = [tolerance] (float lhs, float rhs) + { + // NOTE Hybrid approach + // TODO Allow users to set tolerance for absolute_epsilon_equal + if (nnfw::util::fp32::absolute_epsilon_equal(lhs, rhs)) + { + return true; + } + + return nnfw::util::fp32::epsilon_equal(lhs, rhs, tolerance); + }; + + TfLiteTensorComparator comparator(equals); + TfLiteInterpMatchApp app(comparator); + + app.verbose() = _param.verbose; + + bool res = app.run(*pure, *nnapi); + + if (!res) + { + return 255; + } + + std::cout << "[NNAPI TEST] PASSED" << std::endl; + return 0; +} |