summaryrefslogtreecommitdiff
path: root/libs/support/tflite/src/Diff.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'libs/support/tflite/src/Diff.cpp')
-rw-r--r--libs/support/tflite/src/Diff.cpp262
1 files changed, 262 insertions, 0 deletions
diff --git a/libs/support/tflite/src/Diff.cpp b/libs/support/tflite/src/Diff.cpp
new file mode 100644
index 000000000..f382df2d6
--- /dev/null
+++ b/libs/support/tflite/src/Diff.cpp
@@ -0,0 +1,262 @@
+/*
+ * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "support/tflite/Diff.h"
+
+#include "util/fp32.h"
+
+#include "util/tensor/IndexIterator.h"
+#include "util/tensor/IndexFormatter.h"
+#include "util/tensor/Zipper.h"
+
+#include <iostream>
+
+class DiffSummary : public TfLiteTensorComparator::Observer
+{
+public:
+ DiffSummary()
+ : max_abs_diff_index(0), max_abs_diff_value{0.0f},
+ max_rel_diff_index(0), max_rel_diff_value{0.0f}
+ {
+ // DO NOTHING
+ }
+public:
+ void notify(const nnfw::util::tensor::Index &index, float expected, float obtained) override;
+
+public:
+ nnfw::util::tensor::Index max_abs_diff_index;
+ float max_abs_diff_expected;
+ float max_abs_diff_obtained;
+ float max_abs_diff_value;
+
+ nnfw::util::tensor::Index max_rel_diff_index;
+ float max_rel_diff_expected;
+ float max_rel_diff_obtained;
+ float max_rel_diff_value;
+};
+
+void DiffSummary::notify(const nnfw::util::tensor::Index &index, float expected, float obtained)
+{
+ const auto abs_diff_value = std::fabs(expected - obtained);
+
+ if (max_abs_diff_value < abs_diff_value)
+ {
+ max_abs_diff_index = index;
+ max_abs_diff_value = abs_diff_value;
+ max_abs_diff_expected = expected;
+ max_abs_diff_obtained = obtained;
+ }
+
+ const auto rel_diff_value = nnfw::util::fp32::relative_diff(expected, obtained);
+
+ if (max_rel_diff_value < rel_diff_value)
+ {
+ max_rel_diff_index = index;
+ max_rel_diff_value = rel_diff_value;
+ max_rel_diff_expected = expected;
+ max_rel_diff_obtained = obtained;
+ }
+}
+
+std::vector<TfLiteTensorDiff>
+TfLiteTensorComparator::compare(const nnfw::support::tflite::TensorView<float> &expected,
+ const nnfw::support::tflite::TensorView<float> &obtained,
+ Observer *observer) const
+{
+ std::vector<TfLiteTensorDiff> res;
+
+ assert(expected.shape() == obtained.shape());
+
+ nnfw::util::tensor::zip(expected.shape(), expected, obtained) <<
+ [&] (const nnfw::util::tensor::Index &index, float expected_value, float obtained_value)
+ {
+ const auto relative_diff = nnfw::util::fp32::relative_diff(expected_value, obtained_value);
+
+ if (!_compare_fn(expected_value, obtained_value))
+ {
+ TfLiteTensorDiff diff(index);
+
+ diff.expected = expected_value;
+ diff.obtained = obtained_value;
+
+ res.emplace_back(diff);
+ }
+
+ // Update max_diff_index, if necessary
+ if (observer != nullptr)
+ {
+ observer->notify(index, expected_value, obtained_value);
+ }
+ };
+
+ return res;
+}
+
+bool TfLiteInterpMatchApp::run(::tflite::Interpreter &interp, ::tflite::Interpreter &nnapi) const
+{
+ assert(interp.outputs() == nnapi.outputs());
+
+ for (const auto &id : interp.outputs())
+ {
+ const auto expected = nnfw::support::tflite::TensorView<float>::make(interp, id);
+ const auto obtained = nnfw::support::tflite::TensorView<float>::make(nnapi, id);
+
+ DiffSummary summary;
+
+ auto diffs = _comparator.compare(expected, obtained, &summary);
+
+ if (diffs.size() == 0)
+ {
+ std::cout << " Tensor #" << id << ": MATCHED" << std::endl;
+ }
+ else
+ {
+ std::cout << " Tensor #" << id << ": UNMATCHED" << std::endl;
+ std::cout << " " << diffs.size() << " diffs are detected" << std::endl;
+ }
+
+ // Print out max_diff
+ if (summary.max_abs_diff_value > 0)
+ {
+ std::cout << " Max absolute diff at [" << nnfw::util::tensor::IndexFormatter(summary.max_abs_diff_index) << "]" << std::endl;
+ std::cout << " expected: " << summary.max_abs_diff_expected << std::endl;
+ std::cout << " obtained: " << summary.max_abs_diff_obtained << std::endl;
+ std::cout << " absolute diff: " << summary.max_abs_diff_value << std::endl;
+ }
+
+ if (summary.max_rel_diff_value > 0)
+ {
+ const auto tolerance_level = summary.max_rel_diff_value / FLT_EPSILON;
+
+ std::cout << " Max relative diff at [" << nnfw::util::tensor::IndexFormatter(summary.max_rel_diff_index) << "]" << std::endl;
+ std::cout << " expected: " << summary.max_rel_diff_expected << std::endl;
+ std::cout << " obtained: " << summary.max_rel_diff_obtained << std::endl;
+ std::cout << " relative diff: " << summary.max_rel_diff_value << std::endl;
+ std::cout << " (tolerance level = " << tolerance_level << ")" << std::endl;
+ }
+
+ if (diffs.size() > 0)
+ {
+ if (_verbose != 0)
+ {
+ std::cout << " ---- Details ---" << std::endl;
+ for (const auto &diff : diffs)
+ {
+ const auto absolute_diff = std::fabs(diff.expected - diff.obtained);
+ const auto relative_diff = nnfw::util::fp32::relative_diff(diff.expected, diff.obtained);
+ const auto tolerance_level = relative_diff / FLT_EPSILON;
+
+ std::cout << " Diff at [" << nnfw::util::tensor::IndexFormatter(diff.index) << "]" << std::endl;
+ std::cout << " expected: " << diff.expected << std::endl;
+ std::cout << " obtained: " << diff.obtained << std::endl;
+ std::cout << " absolute diff: " << absolute_diff << std::endl;
+ std::cout << " relative diff: " << relative_diff << std::endl;
+ std::cout << " (tolerance level = " << tolerance_level << ")" << std::endl;
+ }
+ }
+
+ return false;
+ }
+ }
+
+ return true;
+}
+
+#include "util/tensor/Object.h"
+
+//
+// Random Test Runner
+//
+int RandomTestRunner::run(const nnfw::support::tflite::interp::Builder &builder)
+{
+ auto pure = builder.build();
+ auto nnapi = builder.build();
+
+ pure->UseNNAPI(false);
+ nnapi->UseNNAPI(true);
+
+ // Allocate Tensors
+ pure->AllocateTensors();
+ nnapi->AllocateTensors();
+
+ assert(pure->inputs() == nnapi->inputs());
+
+ // Fill IFM with random numbers
+ auto ifm_gen = [this] (const nnfw::util::tensor::Shape &, const nnfw::util::tensor::Index &)
+ {
+ // TODO Allow users to set min/max and distribution
+ std::normal_distribution<float> dist(0.0f, 2.0f);
+ return dist(_rand);
+ };
+
+ for (const auto id : pure->inputs())
+ {
+ auto pure_view = nnfw::support::tflite::TensorView<float>::make(*pure, id);
+ auto nnapi_view = nnfw::support::tflite::TensorView<float>::make(*nnapi, id);
+
+ assert(pure_view.shape() == nnapi_view.shape());
+
+ const nnfw::util::tensor::Object<float> data(pure_view.shape(), ifm_gen);
+
+ assert(pure_view.shape() == data.shape());
+
+ nnfw::util::tensor::iterate(pure_view.shape()) << [&] (const nnfw::util::tensor::Index &ind)
+ {
+ const auto value = data.at(ind);
+
+ pure_view.at(ind) = value;
+ nnapi_view.at(ind) = value;
+ };
+ }
+
+ std::cout << "[NNAPI TEST] Run T/F Lite Interpreter without NNAPI" << std::endl;
+ pure->Invoke();
+
+ std::cout << "[NNAPI TEST] Run T/F Lite Interpreter with NNAPI" << std::endl;
+ nnapi->Invoke();
+
+ // Compare OFM
+ std::cout << "[NNAPI TEST] Compare the result" << std::endl;
+
+ const auto tolerance = _param.tolerance;
+
+ auto equals = [tolerance] (float lhs, float rhs)
+ {
+ // NOTE Hybrid approach
+ // TODO Allow users to set tolerance for absolute_epsilon_equal
+ if (nnfw::util::fp32::absolute_epsilon_equal(lhs, rhs))
+ {
+ return true;
+ }
+
+ return nnfw::util::fp32::epsilon_equal(lhs, rhs, tolerance);
+ };
+
+ TfLiteTensorComparator comparator(equals);
+ TfLiteInterpMatchApp app(comparator);
+
+ app.verbose() = _param.verbose;
+
+ bool res = app.run(*pure, *nnapi);
+
+ if (!res)
+ {
+ return 255;
+ }
+
+ std::cout << "[NNAPI TEST] PASSED" << std::endl;
+ return 0;
+}