Rename contrib as compiler (#4342)

* Rename contrib as compiler

This commit renames the top-level contrib directory as "compiler".

Signed-off-by: Jonghyun Park <jh1302.park@samsung.com>

* Fix CMakeLists.txt

8 files changed, 444 insertions, 0 deletions

diff --git a/compiler/tf2tflite/.FORMATCHECKED b/compiler/tf2tflite/.FORMATCHECKED
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/compiler/tf2tflite/.FORMATCHECKED
diff --git a/compiler/tf2tflite/.gitignore b/compiler/tf2tflite/.gitignore
new file mode 100644
index 000000000..8dbfa9012
--- /dev/null
+++ b/compiler/tf2tflite/.gitignore
@@ -0,0 +1 @@
+/test.local.lst
diff --git a/compiler/tf2tflite/CMakeLists.txt b/compiler/tf2tflite/CMakeLists.txt
new file mode 100644
index 000000000..5ba5b11d9
--- /dev/null
+++ b/compiler/tf2tflite/CMakeLists.txt
@@ -0,0 +1,161 @@
+nncc_include(TargetRequire)
+
+# TODO Allow users to force tf2tflite build
+if(NOT TARGET moco_tf_frontend)
+  return()
+endif(NOT TARGET moco_tf_frontend)
+
+if(NOT TARGET nnkit_support_tftestinfo)
+  return()
+endif(NOT TARGET nnkit_support_tftestinfo)
+
+if(NOT TARGET exo_tflite)
+  return()
+endif(NOT TARGET exo_tflite)
+
+file(GLOB_RECURSE SOURCES "src/*.cpp")
+
+add_executable(tf2tflite ${SOURCES})
+target_link_libraries(tf2tflite PRIVATE moco_tf_frontend)
+target_link_libraries(tf2tflite PRIVATE nnkit_support_tftestinfo)
+target_link_libraries(tf2tflite PRIVATE exo_tflite)
+target_link_libraries(tf2tflite PRIVATE locop)
+target_link_libraries(tf2tflite PRIVATE hermes_std)
+target_link_libraries(tf2tflite PRIVATE stdex)
+
+nncc_find_resource(TensorFlowTests)
+
+unset(REQUIRED_TARGETS)
+list(APPEND REQUIRED_TARGETS tfkit)
+list(APPEND REQUIRED_TARGETS nnkit-run)
+list(APPEND REQUIRED_TARGETS nnkit_tf_backend)
+list(APPEND REQUIRED_TARGETS nnkit_tflite_backend)
+list(APPEND REQUIRED_TARGETS nnkit_randomize_action)
+list(APPEND REQUIRED_TARGETS nnkit_HDF5_export_action)
+list(APPEND REQUIRED_TARGETS nnkit_HDF5_import_action)
+list(APPEND REQUIRED_TARGETS i5diff)
+TargetRequire_Return(${REQUIRED_TARGETS})
+
+message(STATUS "tf2tflite: run tests")
+
+#
+# Copy [PREFIX]/test.pbtxt to PREFIX.pbtxt in binary folder
+# Copy [PREFIX]/test.info to PREFIX.info in binary folder
+# Encode PREFIX.pbtxt to PREFIX.pb
+#
+set(TEST_REPO "${TensorFlowTests_DIR}")
+set(TEST_PBTXT_FILENAME "test.pbtxt")
+set(TEST_INFO_FILENAME "test.info")
+
+unset(TESTCASES)
+
+macro(add NAME)
+  list(APPEND TESTCASES ${NAME})
+endmacro(add)
+
+# Read "test.lst"
+include("test.lst")
+# Read "test.local.lst" if exists
+include("test.local.lst" OPTIONAL)
+
+unset(TEST_DEPS)
+unset(TEST_NAMES)
+
+foreach(PREFIX IN ITEMS ${TESTCASES})
+  if(NOT IS_DIRECTORY "${TEST_REPO}/${PREFIX}")
+    message(FATAL_ERROR "Missing '${PREFIX}' test")
+  endif()
+
+  set(PBTXT_SOURCE_PATH "${TEST_REPO}/${PREFIX}/${TEST_PBTXT_FILENAME}")
+  set(INFO_SOURCE_PATH "${TEST_REPO}/${PREFIX}/${TEST_INFO_FILENAME}")
+
+  set(PBTXT_FILE "${PREFIX}.pbtxt")
+  set(PBTXT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${PBTXT_FILE}")
+
+  set(INFO_FILE "${PREFIX}.info")
+  set(INFO_PATH "${CMAKE_CURRENT_BINARY_DIR}/${INFO_FILE}")
+
+  set(PB_FILE "${PREFIX}.pb")
+  set(PB_PATH "${CMAKE_CURRENT_BINARY_DIR}/${PB_FILE}")
+
+  # Copy .pbtxt
+  add_custom_command(OUTPUT ${PBTXT_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy "${PBTXT_SOURCE_PATH}" "${PBTXT_PATH}"
+    DEPENDS ${PBTXT_SOURCE_PATH}
+    COMMENT "Generate ${PBTXT_FILE}"
+  )
+
+  # Copy .info
+  add_custom_command(OUTPUT ${INFO_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy "${INFO_SOURCE_PATH}" "${INFO_PATH}"
+    DEPENDS ${INFO_SOURCE_PATH}
+    COMMENT "Generate ${INFO_FILE}"
+  )
+
+  # Generate .pb from .pbtxt
+  add_custom_command(OUTPUT ${PB_PATH}
+    COMMAND $<TARGET_FILE:tfkit> encode ${PBTXT_PATH} ${PB_PATH}
+    DEPENDS ${PBTXT_PATH}
+    COMMENT "Generate ${PB_FILE}"
+  )
+
+  list(APPEND TEST_DEPS ${INFO_PATH} ${PB_PATH})
+  list(APPEND TEST_NAMES ${PREFIX})
+endforeach(PREFIX)
+
+##
+## Copy testall
+##
+set(TEST_RUNNER "${CMAKE_CURRENT_BINARY_DIR}/testall")
+set(TEST_RUNNER_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/testall")
+
+add_custom_command(
+  OUTPUT ${TEST_RUNNER}
+  COMMAND ${CMAKE_COMMAND} -E copy "${TEST_RUNNER_SOURCE}" "${TEST_RUNNER}"
+  DEPENDS ${TEST_RUNNER_SOURCE}
+  COMMENT "Generate test runner"
+)
+
+list(APPEND TEST_DEPS "${TEST_RUNNER}")
+
+###
+### Generate test.config
+###
+set(TEST_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/test.config")
+
+add_custom_command(
+  OUTPUT ${TEST_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E remove -f ${TEST_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'NNKIT_RUN_PATH=\"$<TARGET_FILE:nnkit-run>\"' >> ${TEST_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'TF_BACKEND_PATH=\"$<TARGET_FILE:nnkit_tf_backend>\"' >> ${TEST_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'TFLITE_BACKEND_PATH=\"$<TARGET_FILE:nnkit_tflite_backend>\"' >> ${TEST_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'TF2TFLITE_PATH=\"$<TARGET_FILE:tf2tflite>\"' >> ${TEST_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'RANDOMIZE_ACTION_PATH=\"$<TARGET_FILE:nnkit_randomize_action>\"' >> ${TEST_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'HDF5_EXPORT_ACTION_PATH=\"$<TARGET_FILE:nnkit_HDF5_export_action>\"' >> ${TEST_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'HDF5_IMPORT_ACTION_PATH=\"$<TARGET_FILE:nnkit_HDF5_import_action>\"' >> ${TEST_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'I5DIFF_PATH=\"$<TARGET_FILE:i5diff>\"' >> ${TEST_CONFIG}
+  DEPENDS
+    nnkit-run
+    nnkit_tf_backend
+    nnkit_tflite_backend
+    tf2tflite
+    nnkit_randomize_action
+    nnkit_HDF5_export_action
+    nnkit_HDF5_import_action
+    i5diff
+  COMMENT "Generate test configuration"
+)
+
+list(APPEND TEST_DEPS "${TEST_CONFIG}")
+
+# This "tf2tflite_deps" target enforces CMake to generate all the dependencies during "build" phase
+add_custom_target(tf2tflite_deps ALL DEPENDS ${TEST_DEPS})
+
+# Run tests
+add_test(
+  NAME tf2tflite_test
+  COMMAND "${TEST_RUNNER}"
+          "${TEST_CONFIG}"
+          "${CMAKE_CURRENT_BINARY_DIR}"
+          ${TEST_NAMES}
+)
diff --git a/compiler/tf2tflite/README.md b/compiler/tf2tflite/README.md
new file mode 100644
index 000000000..64f4fb7c6
--- /dev/null
+++ b/compiler/tf2tflite/README.md
@@ -0,0 +1,3 @@
+# tf2tflite
+
+_tf2tflite_ is a TensorFlow-to-TensorFlow Lite model converter.
diff --git a/compiler/tf2tflite/requires.cmake b/compiler/tf2tflite/requires.cmake
new file mode 100644
index 000000000..ff9a2cb6d
--- /dev/null
+++ b/compiler/tf2tflite/requires.cmake
@@ -0,0 +1,9 @@
+require("stdex")
+require("hermes-std")
+require("moco-tf")
+require("nnkit")
+require("exo-tflite")
+require("locop")
+require("tfkit")
+require("nnkit")
+require("i5diff")
diff --git a/compiler/tf2tflite/src/Driver.cpp b/compiler/tf2tflite/src/Driver.cpp
new file mode 100644
index 000000000..d390af42c
--- /dev/null
+++ b/compiler/tf2tflite/src/Driver.cpp
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <moco/tf/Frontend.h>
+#include <exo/TFLExporter.h>
+
+#include <nnkit/support/tftestinfo/TensorInfoParser.h>
+
+#include <locop/FormattedGraph.h>
+
+#include <hermes/ConsoleReporter.h>
+#include <hermes/EnvConfig.h>
+
+#include <stdex/Memory.h>
+
+#include <cassert>
+
+#include <iostream>
+#include <stdexcept>
+#include <string>
+
+namespace
+{
+
+std::unique_ptr<loco::Graph> import(const moco::tf::ModelSignature &sig, const std::string &path)
+{
+  moco::tf::Frontend frontend;
+  return frontend.load(sig, path.c_str(), moco::tf::Frontend::FileType::Binary);
+}
+
+} // namespace
+
+//
+// Logging Support
+//
+namespace
+{
+
+struct Logger final : public hermes::Source
+{
+  Logger(hermes::Context *ctx) { activate(ctx->sources(), ctx->bus()); }
+  ~Logger() { deactivate(); }
+};
+
+struct LoggingContext
+{
+  static hermes::Context *get(void)
+  {
+    using EnvConfig = hermes::EnvConfig<hermes::EnvFormat::BooleanNumber>;
+
+    static hermes::Context *ctx = nullptr;
+
+    if (ctx == nullptr)
+    {
+      ctx = new hermes::Context;
+      ctx->sinks()->append(stdex::make_unique<hermes::ConsoleReporter>());
+      ctx->config(stdex::make_unique<EnvConfig>("TF2TFLITE_Log"));
+    }
+
+    return ctx;
+  }
+};
+
+} // namespace
+
+#define LOGGER(name) \
+  ::Logger name { ::LoggingContext::get() }
+
+#define INFO(name) HERMES_INFO(name)
+
+int main(int argc, char **argv)
+{
+  LOGGER(l);
+
+  if (argc != 4)
+  {
+    std::cerr << "ERROR: tf2tflite <path/to/info> <path/to/pb> <path/to/tflite/model>" << std::endl;
+    return 255;
+  }
+
+  std::string info_path{argv[1]};
+  std::string tf_path{argv[2]}; // .pb file
+  std::string tflite_path{argv[3]};
+
+  std::cout << "Read '" << info_path << "'" << std::endl;
+
+  moco::tf::ModelSignature sig;
+  {
+    for (const auto &info : nnkit::support::tftestinfo::parse(info_path.c_str()))
+    {
+      switch (info->kind())
+      {
+      case nnkit::support::tftestinfo::ParsedTensor::Kind::Input:
+        sig.add_input(moco::tf::TensorName{info->name()});
+        break;
+
+      case nnkit::support::tftestinfo::ParsedTensor::Kind::Output:
+        sig.add_output(moco::tf::TensorName{info->name()});
+        break;
+
+      default:
+        throw std::runtime_error{"Unknown kind"};
+      }
+    }
+  }
+
+  std::cout << "Read '" << info_path << "' - Done" << std::endl;
+
+  std::cout << "Import from '" << tf_path << "'" << std::endl;
+  auto g = import(sig, tf_path);
+  std::cout << "Import from '" << tf_path << "' - Done" << std::endl;
+
+  INFO(l) << "Import Graph" << std::endl;
+  INFO(l) << locop::fmt<locop::Formatter::LinearV1>(g) << std::endl;
+
+  std::cout << "Export into '" << tflite_path << "'" << std::endl;
+  exo::TFLExporter(g.get()).dumpToFile(tflite_path.c_str());
+  std::cout << "Export into '" << tflite_path << "' - Done" << std::endl;
+
+  return 0;
+}
diff --git a/compiler/tf2tflite/test.lst b/compiler/tf2tflite/test.lst
new file mode 100644
index 000000000..67dde1702
--- /dev/null
+++ b/compiler/tf2tflite/test.lst
@@ -0,0 +1,30 @@
+# TODO Enable skipped tests
+
+#add(Add_000)
+add(AvgPool_000)
+add(AvgPool_001)
+#add(BiasAdd_000)
+#add(BiasAdd_001)
+add(BiasAdd_002)
+#add(Concat_000)
+#add(Concat_001)
+add(Concat_002)
+#add(Const_000)
+#add(Const_001)
+add(Conv2D_000)
+add(MaxPool_000)
+add(MaxPool_001)
+#add(Mul_000)
+#add(Multiple_IO_000)
+#add(Multiple_IO_001)
+add(Multiple_IO_002)
+add(Placeholder_000)
+add(Relu_000)
+#add(Reshape_000)
+add(TF_IV3_AvgPool_GlobalPooling)
+add(TF_IV3_AvgPool_Module)
+add(TF_IV3_Conv2D_000)
+add(TF_IV3_MaxPool)
+add(TF_SMALL_NET_0000)
+add(TF_SMALL_NET_0001)
+add(TF_SMALL_NET_0002)
diff --git a/compiler/tf2tflite/testall b/compiler/tf2tflite/testall
new file mode 100755
index 000000000..f92c9ede0
--- /dev/null
+++ b/compiler/tf2tflite/testall
@@ -0,0 +1,106 @@
+#!/bin/bash
+
+# Need at least 2 arguments
+if [[ $# -lt 2 ]]; then
+  echo "USAGE: $0 ..."
+  echo
+  echo "ARGUMENTS:"
+  echo "  [test.config path]"
+  echo "  [WORKDIR]"
+  echo "  [Prefix1]"
+  echo "  [Prefix2]"
+  echo "  ..."
+  exit 255
+fi
+
+CONFIG_PATH="$1"; shift
+WORKDIR="$1"; shift
+
+source "${CONFIG_PATH}"
+
+echo "-- Found nnkit-run: ${NNKIT_RUN_PATH}"
+echo "-- Found TF backend: ${TF_BACKEND_PATH}"
+echo "-- Found TFLITE backend: ${TFLITE_BACKEND_PATH}"
+echo "-- Found TF2TFLITE: ${TF2TFLITE_PATH}"
+echo "-- Found randomize action: ${RANDOMIZE_ACTION_PATH}"
+echo "-- Found HDF5 export action: ${HDF5_EXPORT_ACTION_PATH}"
+echo "-- Found HDF5 import action: ${HDF5_IMPORT_ACTION_PATH}"
+echo "-- Found i5diff: ${I5DIFF_PATH}"
+echo "-- Found workdir: ${WORKDIR}"
+
+TESTED=()
+PASSED=()
+FAILED=()
+
+pushd "${WORKDIR}"
+while [[ $# -ne 0 ]]; do
+  PREFIX="$1"; shift
+
+  TESTED+=("${PREFIX}")
+
+  PASSED_TAG="${PREFIX}.passed"
+
+  rm -f "${PASSED_TAG}"
+
+  cat > "${PREFIX}.log" <(
+    exec 2>&1
+
+    echo "-- Found pb: ${PREFIX}.pb"
+
+    # Exit immediately if any command fails
+    set -e
+    # Show commands
+    set -x
+
+    # Generate tflite
+    "${TF2TFLITE_PATH}" \
+      "${WORKDIR}/${PREFIX}.info" \
+      "${WORKDIR}/${PREFIX}.pb" \
+      "${WORKDIR}/${PREFIX}.tflite"
+
+    # Run TensorFlow
+    "${NNKIT_RUN_PATH}" \
+      --backend "${TF_BACKEND_PATH}" \
+      --backend-arg "${WORKDIR}/${PREFIX}.pb" \
+      --backend-arg "${WORKDIR}/${PREFIX}.info" \
+      --pre "${RANDOMIZE_ACTION_PATH}" \
+      --pre "${HDF5_EXPORT_ACTION_PATH}" \
+      --pre-arg "${PREFIX}.input.h5" \
+      --post "${HDF5_EXPORT_ACTION_PATH}" \
+      --post-arg "${PREFIX}.expected.h5"
+
+    # Run TensorFlow Lite
+    "${NNKIT_RUN_PATH}" \
+      --backend "${TFLITE_BACKEND_PATH}" \
+      --backend-arg "${WORKDIR}/${PREFIX}.tflite" \
+      --pre "${HDF5_IMPORT_ACTION_PATH}" \
+      --pre-arg "${PREFIX}.input.h5" \
+      --post "${HDF5_EXPORT_ACTION_PATH}" \
+      --post-arg "${PREFIX}.obtained.h5"
+
+    "${I5DIFF_PATH}" -d 0.001 "${PREFIX}.expected.h5" "${PREFIX}.obtained.h5"
+
+    if [[ $? -eq 0 ]]; then
+      touch "${PASSED_TAG}"
+    fi
+  )
+
+  if [[ -f "${PASSED_TAG}" ]]; then
+    PASSED+=("$PREFIX")
+  else
+    FAILED+=("$PREFIX")
+  fi
+done
+popd
+
+if [[ ${#TESTED[@]} -ne ${#PASSED[@]} ]]; then
+  echo "FAILED"
+  for TEST in "${FAILED[@]}"
+  do
+    echo "- ${TEST}"
+  done
+  exit 255
+fi
+
+echo "PASSED"
+exit 0