path: root/tests/nnapi/nnapi_test_generator/android-q-beta/include/TestHarness.h

/*
 * Copyright (C) 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* Header-only library for various helpers of test harness
 * See frameworks/ml/nn/runtime/test/TestGenerated.cpp for how this is used.
 */
#ifndef ANDROID_FRAMEWORKS_ML_NN_TOOLS_TEST_GENERATOR_TEST_HARNESS_H
#define ANDROID_FRAMEWORKS_ML_NN_TOOLS_TEST_GENERATOR_TEST_HARNESS_H

#include <gmock/gmock-matchers.h>
#include <gtest/gtest.h>

#include <cmath>
#include <functional>
#include <map>
#include <tuple>
#include <vector>

namespace test_helper {

constexpr const size_t gMaximumNumberOfErrorMessages = 10;

// TODO: Figure out the build dependency to make including "CpuOperationUtils.h" work.
inline void convertFloat16ToFloat32(const _Float16* input, std::vector<float>* output) {
    for (size_t i = 0; i < output->size(); ++i) {
        (*output)[i] = static_cast<float>(input[i]);
    }
}

// This class is a workaround for two issues our code relies on:
// 1. sizeof(bool) is implementation defined.
// 2. vector<bool> does not allow direct pointer access via the data() method.
class bool8 {
   public:
    bool8() : mValue() {}
    /* implicit */ bool8(bool value) : mValue(value) {}
    inline operator bool() const { return mValue != 0; }

   private:
    uint8_t mValue;
};

static_assert(sizeof(bool8) == 1, "size of bool8 must be 8 bits");

typedef std::map<int, std::vector<uint32_t>> OperandDimensions;
typedef std::map<int, std::vector<float>> Float32Operands;
typedef std::map<int, std::vector<int32_t>> Int32Operands;
typedef std::map<int, std::vector<uint8_t>> Quant8AsymmOperands;
typedef std::map<int, std::vector<int16_t>> Quant16SymmOperands;
typedef std::map<int, std::vector<_Float16>> Float16Operands;
typedef std::map<int, std::vector<bool8>> Bool8Operands;
typedef std::map<int, std::vector<int8_t>> Quant8ChannelOperands;
typedef std::map<int, std::vector<uint16_t>> Quant16AsymmOperands;
typedef std::map<int, std::vector<int8_t>> Quant8SymmOperands;
struct MixedTyped {
    static constexpr size_t kNumTypes = 9;
    OperandDimensions operandDimensions;
    Float32Operands float32Operands;
    Int32Operands int32Operands;
    Quant8AsymmOperands quant8AsymmOperands;
    Quant16SymmOperands quant16SymmOperands;
    Float16Operands float16Operands;
    Bool8Operands bool8Operands;
    Quant8ChannelOperands quant8ChannelOperands;
    Quant16AsymmOperands quant16AsymmOperands;
    Quant8SymmOperands quant8SymmOperands;
};
typedef std::pair<MixedTyped, MixedTyped> MixedTypedExampleType;

// Mixed-typed examples
typedef struct {
    MixedTypedExampleType operands;
    // Specifies the RANDOM_MULTINOMIAL distribution tolerance.
    // If set to greater than zero, the input is compared as log-probabilities
    // to the output and must be within this tolerance to pass.
    float expectedMultinomialDistributionTolerance = 0.0;
} MixedTypedExample;

// Go through all index-value pairs of a given input type
template <typename T>
inline void for_each(const std::map<int, std::vector<T>>& idx_and_data,
                     std::function<void(int, const std::vector<T>&)> execute) {
    for (auto& i : idx_and_data) {
        execute(i.first, i.second);
    }
}

// non-const variant of for_each
template <typename T>
inline void for_each(std::map<int, std::vector<T>>& idx_and_data,
                     std::function<void(int, std::vector<T>&)> execute) {
    for (auto& i : idx_and_data) {
        execute(i.first, i.second);
    }
}

// Go through all index-value pairs of a given input type
template <typename T>
inline void for_each(const std::map<int, std::vector<T>>& golden,
                     std::map<int, std::vector<T>>& test,
                     std::function<void(int, const std::vector<T>&, std::vector<T>&)> execute) {
    for_each<T>(golden, [&test, &execute](int index, const std::vector<T>& g) {
        auto& t = test[index];
        execute(index, g, t);
    });
}

// Go through all index-value pairs of a given input type
template <typename T>
inline void for_each(
        const std::map<int, std::vector<T>>& golden, const std::map<int, std::vector<T>>& test,
        std::function<void(int, const std::vector<T>&, const std::vector<T>&)> execute) {
    for_each<T>(golden, [&test, &execute](int index, const std::vector<T>& g) {
        auto t = test.find(index);
        ASSERT_NE(t, test.end());
        execute(index, g, t->second);
    });
}

// internal helper for for_all
template <typename T>
inline void for_all_internal(std::map<int, std::vector<T>>& idx_and_data,
                             std::function<void(int, void*, size_t)> execute_this) {
    for_each<T>(idx_and_data, [&execute_this](int idx, std::vector<T>& m) {
        execute_this(idx, static_cast<void*>(m.data()), m.size() * sizeof(T));
    });
}

// Go through all index-value pairs of all input types
// expects a functor that takes (int index, void *raw data, size_t sz)
inline void for_all(MixedTyped& idx_and_data,
                    std::function<void(int, void*, size_t)> execute_this) {
    for_all_internal(idx_and_data.float32Operands, execute_this);
    for_all_internal(idx_and_data.int32Operands, execute_this);
    for_all_internal(idx_and_data.quant8AsymmOperands, execute_this);
    for_all_internal(idx_and_data.quant16SymmOperands, execute_this);
    for_all_internal(idx_and_data.float16Operands, execute_this);
    for_all_internal(idx_and_data.bool8Operands, execute_this);
    for_all_internal(idx_and_data.quant8ChannelOperands, execute_this);
    for_all_internal(idx_and_data.quant16AsymmOperands, execute_this);
    for_all_internal(idx_and_data.quant8SymmOperands, execute_this);
    static_assert(9 == MixedTyped::kNumTypes,
                  "Number of types in MixedTyped changed, but for_all function wasn't updated");
}

// Const variant of internal helper for for_all
template <typename T>
inline void for_all_internal(const std::map<int, std::vector<T>>& idx_and_data,
                             std::function<void(int, const void*, size_t)> execute_this) {
    for_each<T>(idx_and_data, [&execute_this](int idx, const std::vector<T>& m) {
        execute_this(idx, static_cast<const void*>(m.data()), m.size() * sizeof(T));
    });
}

// Go through all index-value pairs (const variant)
// expects a functor that takes (int index, const void *raw data, size_t sz)
inline void for_all(const MixedTyped& idx_and_data,
                    std::function<void(int, const void*, size_t)> execute_this) {
    for_all_internal(idx_and_data.float32Operands, execute_this);
    for_all_internal(idx_and_data.int32Operands, execute_this);
    for_all_internal(idx_and_data.quant8AsymmOperands, execute_this);
    for_all_internal(idx_and_data.quant16SymmOperands, execute_this);
    for_all_internal(idx_and_data.float16Operands, execute_this);
    for_all_internal(idx_and_data.bool8Operands, execute_this);
    for_all_internal(idx_and_data.quant8ChannelOperands, execute_this);
    for_all_internal(idx_and_data.quant16AsymmOperands, execute_this);
    for_all_internal(idx_and_data.quant8SymmOperands, execute_this);
    static_assert(
            9 == MixedTyped::kNumTypes,
            "Number of types in MixedTyped changed, but const for_all function wasn't updated");
}

// Helper template - resize test output per golden
template <typename T>
inline void resize_accordingly_(const std::map<int, std::vector<T>>& golden,
                                std::map<int, std::vector<T>>& test) {
    for_each<T>(golden, test,
                [](int, const std::vector<T>& g, std::vector<T>& t) { t.resize(g.size()); });
}

template <>
inline void resize_accordingly_<uint32_t>(const OperandDimensions& golden,
                                          OperandDimensions& test) {
    for_each<uint32_t>(
            golden, test,
            [](int, const std::vector<uint32_t>& g, std::vector<uint32_t>& t) { t = g; });
}

inline void resize_accordingly(const MixedTyped& golden, MixedTyped& test) {
    resize_accordingly_(golden.operandDimensions, test.operandDimensions);
    resize_accordingly_(golden.float32Operands, test.float32Operands);
    resize_accordingly_(golden.int32Operands, test.int32Operands);
    resize_accordingly_(golden.quant8AsymmOperands, test.quant8AsymmOperands);
    resize_accordingly_(golden.quant16SymmOperands, test.quant16SymmOperands);
    resize_accordingly_(golden.float16Operands, test.float16Operands);
    resize_accordingly_(golden.bool8Operands, test.bool8Operands);
    resize_accordingly_(golden.quant8ChannelOperands, test.quant8ChannelOperands);
    resize_accordingly_(golden.quant16AsymmOperands, test.quant16AsymmOperands);
    resize_accordingly_(golden.quant8SymmOperands, test.quant8SymmOperands);
    static_assert(9 == MixedTyped::kNumTypes,
                  "Number of types in MixedTyped changed, but resize_accordingly function wasn't "
                  "updated");
}

template <typename T>
void filter_internal(const std::map<int, std::vector<T>>& golden,
                     std::map<int, std::vector<T>>* filtered, std::function<bool(int)> is_ignored) {
    for_each<T>(golden, [filtered, &is_ignored](int index, const std::vector<T>& m) {
        auto& g = *filtered;
        if (!is_ignored(index)) g[index] = m;
    });
}

inline MixedTyped filter(const MixedTyped& golden, std::function<bool(int)> is_ignored) {
    MixedTyped filtered;
    filter_internal(golden.operandDimensions, &filtered.operandDimensions, is_ignored);
    filter_internal(golden.float32Operands, &filtered.float32Operands, is_ignored);
    filter_internal(golden.int32Operands, &filtered.int32Operands, is_ignored);
    filter_internal(golden.quant8AsymmOperands, &filtered.quant8AsymmOperands, is_ignored);
    filter_internal(golden.quant16SymmOperands, &filtered.quant16SymmOperands, is_ignored);
    filter_internal(golden.float16Operands, &filtered.float16Operands, is_ignored);
    filter_internal(golden.bool8Operands, &filtered.bool8Operands, is_ignored);
    filter_internal(golden.quant8ChannelOperands, &filtered.quant8ChannelOperands, is_ignored);
    filter_internal(golden.quant16AsymmOperands, &filtered.quant16AsymmOperands, is_ignored);
    filter_internal(golden.quant8SymmOperands, &filtered.quant8SymmOperands, is_ignored);
    static_assert(9 == MixedTyped::kNumTypes,
                  "Number of types in MixedTyped changed, but compare function wasn't updated");
    return filtered;
}

// Compare results
template <typename T>
void compare_(const std::map<int, std::vector<T>>& golden,
              const std::map<int, std::vector<T>>& test, std::function<void(T, T)> cmp) {
    for_each<T>(golden, test, [&cmp](int index, const std::vector<T>& g, const std::vector<T>& t) {
        for (unsigned int i = 0; i < g.size(); i++) {
            SCOPED_TRACE(testing::Message()
                         << "When comparing output " << index << " element " << i);
            cmp(g[i], t[i]);
        }
    });
}

// TODO: Allow passing accuracy criteria from spec.
// Currently we only need relaxed accuracy criteria on mobilenet tests, so we return the quant8
// tolerance simply based on the current test name.
inline int getQuant8AllowedError() {
    const ::testing::TestInfo* const testInfo =
            ::testing::UnitTest::GetInstance()->current_test_info();
    const std::string testCaseName = testInfo->test_case_name();
    const std::string testName = testInfo->name();
    // We relax the quant8 precision for all tests with mobilenet:
    // - CTS/VTS GeneratedTest and DynamicOutputShapeTest with mobilenet
    // - VTS CompilationCachingTest and CompilationCachingSecurityTest except for TOCTOU tests
    if (testName.find("mobilenet") != std::string::npos ||
        (testCaseName.find("CompilationCaching") != std::string::npos &&
         testName.find("TOCTOU") == std::string::npos)) {
        return 2;
    } else {
        return 1;
    }
}

inline void compare(const MixedTyped& golden, const MixedTyped& test, float fpAtol = 1e-5f,
                    float fpRtol = 1e-5f) {
    int quant8AllowedError = getQuant8AllowedError();
    for_each<uint32_t>(
            golden.operandDimensions, test.operandDimensions,
            [](int index, const std::vector<uint32_t>& g, const std::vector<uint32_t>& t) {
                SCOPED_TRACE(testing::Message()
                             << "When comparing dimensions for output " << index);
                EXPECT_EQ(g, t);
            });
    size_t totalNumberOfErrors = 0;
    compare_<float>(golden.float32Operands, test.float32Operands,
                    [&totalNumberOfErrors, fpAtol, fpRtol](float expected, float actual) {
                        // Compute the range based on both absolute tolerance and relative tolerance
                        float fpRange = fpAtol + fpRtol * std::abs(expected);
                        if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
                            EXPECT_NEAR(expected, actual, fpRange);
                        }
                        if (std::abs(expected - actual) > fpRange) {
                            totalNumberOfErrors++;
                        }
                    });
    compare_<int32_t>(golden.int32Operands, test.int32Operands,
                      [&totalNumberOfErrors](int32_t expected, int32_t actual) {
                          if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
                              EXPECT_EQ(expected, actual);
                          }
                          if (expected != actual) {
                              totalNumberOfErrors++;
                          }
                      });
    compare_<uint8_t>(golden.quant8AsymmOperands, test.quant8AsymmOperands,
                      [&totalNumberOfErrors, quant8AllowedError](uint8_t expected, uint8_t actual) {
                          if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
                              EXPECT_NEAR(expected, actual, quant8AllowedError);
                          }
                          if (std::abs(expected - actual) > quant8AllowedError) {
                              totalNumberOfErrors++;
                          }
                      });
    compare_<int16_t>(golden.quant16SymmOperands, test.quant16SymmOperands,
                      [&totalNumberOfErrors](int16_t expected, int16_t actual) {
                          if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
                              EXPECT_NEAR(expected, actual, 1);
                          }
                          if (std::abs(expected - actual) > 1) {
                              totalNumberOfErrors++;
                          }
                      });
    compare_<_Float16>(golden.float16Operands, test.float16Operands,
                       [&totalNumberOfErrors, fpAtol, fpRtol](_Float16 expected, _Float16 actual) {
                           // Compute the range based on both absolute tolerance and relative
                           // tolerance
                           float fpRange = fpAtol + fpRtol * std::abs(static_cast<float>(expected));
                           if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
                               EXPECT_NEAR(expected, actual, fpRange);
                           }
                           if (std::abs(static_cast<float>(expected - actual)) > fpRange) {
                               totalNumberOfErrors++;
                           }
                       });
    compare_<bool8>(golden.bool8Operands, test.bool8Operands,
                    [&totalNumberOfErrors](bool expected, bool actual) {
                        if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
                            EXPECT_EQ(expected, actual);
                        }
                        if (expected != actual) {
                            totalNumberOfErrors++;
                        }
                    });
    compare_<int8_t>(golden.quant8ChannelOperands, test.quant8ChannelOperands,
                     [&totalNumberOfErrors, &quant8AllowedError](int8_t expected, int8_t actual) {
                         if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
                             EXPECT_NEAR(expected, actual, quant8AllowedError);
                         }
                         if (std::abs(static_cast<int>(expected) - static_cast<int>(actual)) >
                             quant8AllowedError) {
                             totalNumberOfErrors++;
                         }
                     });
    compare_<uint16_t>(golden.quant16AsymmOperands, test.quant16AsymmOperands,
                       [&totalNumberOfErrors](int16_t expected, int16_t actual) {
                           if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
                               EXPECT_NEAR(expected, actual, 1);
                           }
                           if (std::abs(expected - actual) > 1) {
                               totalNumberOfErrors++;
                           }
                       });
    compare_<int8_t>(golden.quant8SymmOperands, test.quant8SymmOperands,
                     [&totalNumberOfErrors, quant8AllowedError](int8_t expected, int8_t actual) {
                         if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
                             EXPECT_NEAR(expected, actual, quant8AllowedError);
                         }
                         if (std::abs(static_cast<int>(expected) - static_cast<int>(actual)) >
                             quant8AllowedError) {
                             totalNumberOfErrors++;
                         }
                     });

    static_assert(9 == MixedTyped::kNumTypes,
                  "Number of types in MixedTyped changed, but compare function wasn't updated");
    EXPECT_EQ(size_t{0}, totalNumberOfErrors);
}

// Calculates the expected probability from the unnormalized log-probability of
// each class in the input and compares it to the actual ocurrence of that class
// in the output.
inline void expectMultinomialDistributionWithinTolerance(const MixedTyped& test,
                                                         const MixedTypedExample& example) {
    // TODO: These should be parameters but aren't currently preserved in the example.
    const int kBatchSize = 1;
    const int kNumClasses = 1024;
    const int kNumSamples = 128;

    std::vector<int32_t> output = test.int32Operands.at(0);
    std::vector<int> class_counts;
    class_counts.resize(kNumClasses);
    for (int index : output) {
        class_counts[index]++;
    }
    std::vector<float> input;
    Float32Operands float32Operands = example.operands.first.float32Operands;
    if (!float32Operands.empty()) {
        input = example.operands.first.float32Operands.at(0);
    } else {
        std::vector<_Float16> inputFloat16 = example.operands.first.float16Operands.at(0);
        input.resize(inputFloat16.size());
        convertFloat16ToFloat32(inputFloat16.data(), &input);
    }
    for (int b = 0; b < kBatchSize; ++b) {
        float probability_sum = 0;
        const int batch_index = kBatchSize * b;
        for (int i = 0; i < kNumClasses; ++i) {
            probability_sum += expf(input[batch_index + i]);
        }
        for (int i = 0; i < kNumClasses; ++i) {
            float probability =
                    static_cast<float>(class_counts[i]) / static_cast<float>(kNumSamples);
            float probability_expected = expf(input[batch_index + i]) / probability_sum;
            EXPECT_THAT(probability,
                        ::testing::FloatNear(probability_expected,
                                             example.expectedMultinomialDistributionTolerance));
        }
    }
}

};  // namespace test_helper

#endif  // ANDROID_FRAMEWORKS_ML_NN_TOOLS_TEST_GENERATOR_TEST_HARNESS_H