path: root/libs/support/tflite/src/Diff.cpp

/*
 * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "support/tflite/Diff.h"

#include "util/fp32.h"

#include "util/tensor/IndexIterator.h"
#include "util/tensor/IndexFormatter.h"
#include "util/tensor/Zipper.h"

#include <iostream>

class DiffSummary : public TfLiteTensorComparator::Observer
{
public:
  DiffSummary()
      : max_abs_diff_index(0), max_abs_diff_value{0.0f},
        max_rel_diff_index(0), max_rel_diff_value{0.0f}
  {
    // DO NOTHING
  }
public:
  void notify(const nnfw::util::tensor::Index &index, float expected, float obtained) override;

public:
  nnfw::util::tensor::Index max_abs_diff_index;
  float max_abs_diff_expected;
  float max_abs_diff_obtained;
  float max_abs_diff_value;

  nnfw::util::tensor::Index max_rel_diff_index;
  float max_rel_diff_expected;
  float max_rel_diff_obtained;
  float max_rel_diff_value;
};

void DiffSummary::notify(const nnfw::util::tensor::Index &index, float expected, float obtained)
{
  const auto abs_diff_value = std::fabs(expected - obtained);

  if (max_abs_diff_value < abs_diff_value)
  {
    max_abs_diff_index = index;
    max_abs_diff_value = abs_diff_value;
    max_abs_diff_expected = expected;
    max_abs_diff_obtained = obtained;
  }

  const auto rel_diff_value = nnfw::util::fp32::relative_diff(expected, obtained);

  if (max_rel_diff_value < rel_diff_value)
  {
    max_rel_diff_index = index;
    max_rel_diff_value = rel_diff_value;
    max_rel_diff_expected = expected;
    max_rel_diff_obtained = obtained;
  }
}

std::vector<TfLiteTensorDiff>
TfLiteTensorComparator::compare(const nnfw::support::tflite::TensorView<float> &expected,
                                const nnfw::support::tflite::TensorView<float> &obtained,
                                Observer *observer) const
{
  std::vector<TfLiteTensorDiff> res;

  assert(expected.shape() == obtained.shape());

  nnfw::util::tensor::zip(expected.shape(), expected, obtained) <<
    [&] (const nnfw::util::tensor::Index &index, float expected_value, float obtained_value)
  {
    const auto relative_diff = nnfw::util::fp32::relative_diff(expected_value, obtained_value);

    if (!_compare_fn(expected_value, obtained_value))
    {
      TfLiteTensorDiff diff(index);

      diff.expected = expected_value;
      diff.obtained = obtained_value;

      res.emplace_back(diff);
    }

    // Update max_diff_index, if necessary
    if (observer != nullptr)
    {
      observer->notify(index, expected_value, obtained_value);
    }
  };

  return res;
}

bool TfLiteInterpMatchApp::run(::tflite::Interpreter &interp, ::tflite::Interpreter &nnapi) const
{
  assert(interp.outputs() == nnapi.outputs());

  for (const auto &id : interp.outputs())
  {
    const auto expected = nnfw::support::tflite::TensorView<float>::make(interp, id);
    const auto obtained = nnfw::support::tflite::TensorView<float>::make(nnapi, id);

    DiffSummary summary;

    auto diffs = _comparator.compare(expected, obtained, &summary);

    if (diffs.size() == 0)
    {
      std::cout << "  Tensor #" << id << ": MATCHED" << std::endl;
    }
    else
    {
      std::cout << "  Tensor #" << id << ": UNMATCHED" << std::endl;
      std::cout << "    " << diffs.size() << " diffs are detected" << std::endl;
    }

    // Print out max_diff
    if (summary.max_abs_diff_value > 0)
    {
      std::cout << "    Max absolute diff at [" << nnfw::util::tensor::IndexFormatter(summary.max_abs_diff_index) << "]" << std::endl;
      std::cout << "       expected: " << summary.max_abs_diff_expected << std::endl;
      std::cout << "       obtained: " << summary.max_abs_diff_obtained << std::endl;
      std::cout << "       absolute diff: " << summary.max_abs_diff_value << std::endl;
    }

    if (summary.max_rel_diff_value > 0)
    {
      const auto tolerance_level = summary.max_rel_diff_value / FLT_EPSILON;

      std::cout << "    Max relative diff at [" << nnfw::util::tensor::IndexFormatter(summary.max_rel_diff_index) << "]" << std::endl;
      std::cout << "       expected: " << summary.max_rel_diff_expected << std::endl;
      std::cout << "       obtained: " << summary.max_rel_diff_obtained << std::endl;
      std::cout << "       relative diff: " << summary.max_rel_diff_value << std::endl;
      std::cout << "         (tolerance level = " << tolerance_level << ")" << std::endl;
    }

    if (diffs.size() > 0)
    {
      if (_verbose != 0)
      {
        std::cout << "    ---- Details ---" << std::endl;
        for (const auto &diff : diffs)
        {
          const auto absolute_diff = std::fabs(diff.expected - diff.obtained);
          const auto relative_diff = nnfw::util::fp32::relative_diff(diff.expected, diff.obtained);
          const auto tolerance_level = relative_diff / FLT_EPSILON;

          std::cout << "    Diff at [" << nnfw::util::tensor::IndexFormatter(diff.index) << "]" << std::endl;
          std::cout << "      expected: " << diff.expected << std::endl;
          std::cout << "      obtained: " << diff.obtained << std::endl;
          std::cout << "      absolute diff: " << absolute_diff << std::endl;
          std::cout << "      relative diff: " << relative_diff << std::endl;
          std::cout << "         (tolerance level = " << tolerance_level << ")" << std::endl;
        }
      }

      return false;
    }
  }

  return true;
}

#include "util/tensor/Object.h"

//
// Random Test Runner
//
int RandomTestRunner::run(const nnfw::support::tflite::interp::Builder &builder)
{
  auto pure = builder.build();
  auto nnapi = builder.build();

  pure->UseNNAPI(false);
  nnapi->UseNNAPI(true);

  // Allocate Tensors
  pure->AllocateTensors();
  nnapi->AllocateTensors();

  assert(pure->inputs() == nnapi->inputs());

  // Fill IFM with random numbers
  auto ifm_gen = [this] (const nnfw::util::tensor::Shape &, const nnfw::util::tensor::Index &)
  {
    // TODO Allow users to set min/max and distribution
    std::normal_distribution<float> dist(0.0f, 2.0f);
    return dist(_rand);
  };

  for (const auto id : pure->inputs())
  {
    auto pure_view = nnfw::support::tflite::TensorView<float>::make(*pure, id);
    auto nnapi_view = nnfw::support::tflite::TensorView<float>::make(*nnapi, id);

    assert(pure_view.shape() == nnapi_view.shape());

    const nnfw::util::tensor::Object<float> data(pure_view.shape(), ifm_gen);

    assert(pure_view.shape() == data.shape());

    nnfw::util::tensor::iterate(pure_view.shape()) << [&] (const nnfw::util::tensor::Index &ind)
    {
      const auto value = data.at(ind);

      pure_view.at(ind) = value;
      nnapi_view.at(ind) = value;
    };
  }

  std::cout << "[NNAPI TEST] Run T/F Lite Interpreter without NNAPI" << std::endl;
  pure->Invoke();

  std::cout << "[NNAPI TEST] Run T/F Lite Interpreter with NNAPI" << std::endl;
  nnapi->Invoke();

  // Compare OFM
  std::cout << "[NNAPI TEST] Compare the result" << std::endl;

  const auto tolerance = _param.tolerance;

  auto equals = [tolerance] (float lhs, float rhs)
  {
    // NOTE Hybrid approach
    // TODO Allow users to set tolerance for absolute_epsilon_equal
    if (nnfw::util::fp32::absolute_epsilon_equal(lhs, rhs))
    {
      return true;
    }

    return nnfw::util::fp32::epsilon_equal(lhs, rhs, tolerance);
  };

  TfLiteTensorComparator comparator(equals);
  TfLiteInterpMatchApp app(comparator);

  app.verbose() = _param.verbose;

  bool res = app.run(*pure, *nnapi);

  if (!res)
  {
    return 255;
  }

  std::cout << "[NNAPI TEST] PASSED" << std::endl;
  return 0;
}