diff options
Diffstat (limited to 'runtime/onert/backend/xnnpack')
22 files changed, 1568 insertions, 0 deletions
diff --git a/runtime/onert/backend/xnnpack/Backend.h b/runtime/onert/backend/xnnpack/Backend.h new file mode 100644 index 000000000..67494a534 --- /dev/null +++ b/runtime/onert/backend/xnnpack/Backend.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ONERT_BACKEND_XNNPACK_BACKEND_H__ +#define __ONERT_BACKEND_XNNPACK_BACKEND_H__ + +#include "BackendContext.h" +#include "Config.h" +#include "KernelGenerator.h" + +#include <backend/Backend.h> + +#include <memory> + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ + +class Backend : public ::onert::backend::Backend +{ +public: + Backend() : _config{std::make_shared<Config>()} {} + + std::shared_ptr<IConfig> config() const override { return _config; } + + std::unique_ptr<onert::backend::BackendContext> newContext(ContextData &&data) const override + { + auto custom_kernel_builder = data.custom_kernel_builder; + auto &graph = *data.graph; + auto context = std::make_unique<BackendContext>(this, std::move(data)); + auto tr = std::make_shared<basic::TensorRegistry>(); + auto tb = std::make_shared<TensorBuilder>(tr); + context->tensor_registry = tr; + context->tensor_builder = tb; + context->kernel_gen = std::make_shared<KernelGenerator>(graph, tb, tr, custom_kernel_builder, + context->external_context()); + return context; + } + +private: + std::shared_ptr<IConfig> _config; +}; + +} // namespace xnnpack +} // namespace backend +} // namespace onert + +#endif // __ONERT_BACKEND_XNNPACK_BACKEND_H__ diff --git a/runtime/onert/backend/xnnpack/BackendContext.cc b/runtime/onert/backend/xnnpack/BackendContext.cc new file mode 100644 index 000000000..b555a4ac6 --- /dev/null +++ b/runtime/onert/backend/xnnpack/BackendContext.cc @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "BackendContext.h" + +#include "TensorBuilder.h" +#include "KernelGenerator.h" +#include "util/logging.h" +#include "ir/Index.h" +#include "ir/OperandIndexMap.h" +#include "ir/OperandIndexSequence.h" +#include "backend/basic/BackendContextHelpers.h" + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ + +ITensorRegistry *BackendContext::genTensors() { return basic::genTensors(*this); } + +FunctionMap BackendContext::genKernels() +{ + FunctionMap ret; + + for (auto &&op_ind : _data.op_order) + { + auto fn_seq = kernel_gen->generate(op_ind); + ret.emplace_back(op_ind, std::move(fn_seq)); + } + + basic::initConsts(*this); + + // NOTE For memory optimization, we want to free some operand data + const_cast<ir::Graph &>(*_data.graph) + .operands() + .iterate([&](const ir::OperandIndex &, ir::Operand &obj) { obj.releaseData(); }); + + for (auto &&it : ret) + { + auto &fn_seq = it.second; + fn_seq->iterate([&](exec::IFunction &ifunc) { ifunc.prepare(); }); + } + + return ret; +} + +} // namespace xnnpack +} // namespace backend +} // namespace onert diff --git a/runtime/onert/backend/xnnpack/BackendContext.h b/runtime/onert/backend/xnnpack/BackendContext.h new file mode 100644 index 000000000..e3b66eef3 --- /dev/null +++ b/runtime/onert/backend/xnnpack/BackendContext.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ONERT_BACKEND_XNNPACK_BACKEND_CONTEXT_H__ +#define __ONERT_BACKEND_XNNPACK_BACKEND_CONTEXT_H__ + +#include <backend/BackendContext.h> +#include <util/ConfigSource.h> +#include "TensorBuilder.h" +#include "KernelGenerator.h" +#include "ExternalContext.h" + +const int kDefaultNumThreadpoolThreads = 1; + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ + +class BackendContext : public onert::backend::BackendContext +{ +public: + BackendContext(const Backend *backend, ContextData &&data, + std::shared_ptr<ITensorRegistry> tensor_registry = nullptr, + std::shared_ptr<TensorBuilder> tensor_builder = nullptr, + std::shared_ptr<KernelGenerator> kernel_gen = nullptr) + : onert::backend::BackendContext(backend, std::move(data), tensor_registry), + tensor_builder{tensor_builder}, kernel_gen{kernel_gen}, _external_context(nullptr) + { + int num_threads = util::getConfigInt(util::config::XNNPACK_THREADS); + if (num_threads < 1) + num_threads = kDefaultNumThreadpoolThreads; // default num of threads + _external_context.reset(new ExternalContext(static_cast<size_t>(num_threads))); + } + + ITensorRegistry *genTensors() override; + FunctionMap genKernels() override; + + std::shared_ptr<ExternalContext> external_context() { return _external_context; } + +public: + // TODO Make it private + std::shared_ptr<TensorBuilder> tensor_builder; + std::shared_ptr<KernelGenerator> kernel_gen; + +private: + std::shared_ptr<ExternalContext> _external_context; +}; + +} // namespace xnnpack +} // namespace backend +} // namespace onert + +#endif // __ONERT_BACKEND_XNNPACK_BACKEND_CONTEXT_H__ diff --git a/runtime/onert/backend/xnnpack/CMakeLists.txt b/runtime/onert/backend/xnnpack/CMakeLists.txt new file mode 100644 index 000000000..e3de31e6f --- /dev/null +++ b/runtime/onert/backend/xnnpack/CMakeLists.txt @@ -0,0 +1,26 @@ +set(LIB_ONERT_BACKEND_XNNPACK onert_backend_xnnpack) + +# Unsupported architecture +nnfw_find_package(Xnnpack QUIET) +if(NOT Xnnpack_FOUND) + return() +endif(NOT Xnnpack_FOUND) + +file(GLOB_RECURSE SOURCES "*.cc") + +add_library(${LIB_ONERT_BACKEND_XNNPACK} SHARED ${SOURCES}) + +target_link_libraries(${LIB_ONERT_BACKEND_XNNPACK} PRIVATE onert_core) +target_link_libraries(${LIB_ONERT_BACKEND_XNNPACK} PRIVATE nnfw_common) +target_link_libraries(${LIB_ONERT_BACKEND_XNNPACK} PRIVATE nnfw_coverage) +target_link_libraries(${LIB_ONERT_BACKEND_XNNPACK} PRIVATE pthreadpool) +target_link_libraries(${LIB_ONERT_BACKEND_XNNPACK} PRIVATE XNNPACK) + +set_target_properties(${LIB_ONERT_BACKEND_XNNPACK} PROPERTIES OUTPUT_NAME backend_xnnpack) + +if(CMAKE_BUILD_TYPE_LC STREQUAL "release") + add_custom_command(TARGET ${LIB_ONERT_BACKEND_XNNPACK} POST_BUILD + COMMAND ${CMAKE_STRIP} "--strip-unneeded" $<TARGET_FILE_NAME:${LIB_ONERT_BACKEND_XNNPACK}>) +endif() + +install(TARGETS ${LIB_ONERT_BACKEND_XNNPACK} DESTINATION lib) diff --git a/runtime/onert/backend/xnnpack/Config.cc b/runtime/onert/backend/xnnpack/Config.cc new file mode 100644 index 000000000..cc27f717f --- /dev/null +++ b/runtime/onert/backend/xnnpack/Config.cc @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Config.h" + +#include <xnnpack.h> + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ + +Config::~Config() { xnn_deinitialize(); } + +bool Config::initialize() +{ + xnn_status status = xnn_initialize(nullptr /* allocator */); + if (status != xnn_status_success) + { + throw std::runtime_error{"failed to initialize XNNPACK"}; + } + return true; +} + +ir::Layout Config::supportLayout(const ir::IOperation &, ir::Layout) { return ir::Layout::NHWC; } + +} // namespace xnnpack +} // namespace backend +} // namespace onert diff --git a/runtime/onert/backend/xnnpack/Config.h b/runtime/onert/backend/xnnpack/Config.h new file mode 100644 index 000000000..4c5fba587 --- /dev/null +++ b/runtime/onert/backend/xnnpack/Config.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ONERT_BACKEND_XNNPACK_CONFIG_H__ +#define __ONERT_BACKEND_XNNPACK_CONFIG_H__ + +#include <backend/IConfig.h> +#include <memory> +#include <util/ITimer.h> + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ + +class Config : public IConfig +{ +public: + virtual ~Config(); + +public: + std::string id() override { return "xnnpack"; } + bool initialize() override; + ir::Layout supportLayout(const ir::IOperation &node, ir::Layout frontend_layout) override; + bool supportPermutation() override { return true; } + bool supportDynamicTensor() override { return true; } + bool supportFP16() override { return false; } + + std::unique_ptr<util::ITimer> timer() override { return std::make_unique<util::CPUTimer>(); } +}; + +} // namespace xnnpack +} // namespace backend +} // namespace onert + +#endif // __ONERT_BACKEND_XNNPACK_CONFIG_H__ diff --git a/runtime/onert/backend/xnnpack/ExternalContext.cc b/runtime/onert/backend/xnnpack/ExternalContext.cc new file mode 100644 index 000000000..1fbcd4f02 --- /dev/null +++ b/runtime/onert/backend/xnnpack/ExternalContext.cc @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ExternalContext.h" + +#include <cassert> + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ + +ExternalContext::ExternalContext(size_t num_threads) + : _threadpool(pthreadpool_create(num_threads), pthreadpool_destroy) +{ + assert(_threadpool); +} + +} // namespace xnnpack +} // namespace backend +} // namespace onert diff --git a/runtime/onert/backend/xnnpack/ExternalContext.h b/runtime/onert/backend/xnnpack/ExternalContext.h new file mode 100644 index 000000000..682fd2e4e --- /dev/null +++ b/runtime/onert/backend/xnnpack/ExternalContext.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ONERT_BACKEND_XNNPACK_EXTERNAL_CONTEXT_H__ +#define __ONERT_BACKEND_XNNPACK_EXTERNAL_CONTEXT_H__ + +#include <memory> +#include <xnnpack.h> + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ + +class ExternalContext +{ +public: + ExternalContext(size_t num_threads); + +public: + pthreadpool *getThreadPool() { return _threadpool.get(); } + +private: + std::unique_ptr<pthreadpool, decltype(&pthreadpool_destroy)> _threadpool; +}; + +} // namespace xnnpack +} // namespace backend +} // namespace onert + +#endif // __ONERT_BACKEND_XNNPACK_EXTERNAL_CONTEXT_H__ diff --git a/runtime/onert/backend/xnnpack/KernelGenerator.cc b/runtime/onert/backend/xnnpack/KernelGenerator.cc new file mode 100644 index 000000000..b72149131 --- /dev/null +++ b/runtime/onert/backend/xnnpack/KernelGenerator.cc @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "KernelGenerator.h" + +#include "ops/ConvolutionLayer.h" +#include "ops/DepthwiseConvolutionLayer.h" +#include "ops/FullyConnectedLayer.h" + +#include <backend/Backend.h> +#include <backend/IConfig.h> +#include <memory> +#include <util/Utils.h> +#include <util/logging.h> +#include <exec/DynamicShapeInferer.h> + +#include <stdexcept> + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ + +KernelGenerator::KernelGenerator( + const ir::Graph &graph, const std::shared_ptr<TensorBuilder> &tensor_builder, + const std::shared_ptr<basic::TensorRegistry> &tensor_reg, + const std::shared_ptr<backend::custom::IKernelBuilder> &kernel_builder, + const std::shared_ptr<ExternalContext> &external_context) + : basic::KernelGeneratorBase{graph}, + _ctx(graph.operands()), _operations_ctx{graph.operations()}, _current_layout{graph.layout()}, + _tensor_builder(tensor_builder), _tensor_reg{tensor_reg}, _kernel_builder(kernel_builder), + _external_context(external_context) +{ + // DO NOTHING +} + +std::unique_ptr<exec::FunctionSequence> KernelGenerator::generate(ir::OperationIndex ind) +{ + auto ret = std::make_unique<exec::FunctionSequence>(); + + assert(_tensor_builder->dynamicTensorManager()); + assert(_tensor_reg); + + // Prepare to handle dynamic tensors later + auto dyn_ctx = std::make_shared<exec::FunctionSequence::DynamicTensorCtx>(); + { + dyn_ctx->op = &_operations_ctx.at(ind); + dyn_ctx->dynamic_shape_inferer = std::make_shared<exec::DynamicShapeInferer>(_ctx, _tensor_reg); + } + ret->dynamic_tensor_ctx(dyn_ctx); + + auto &op = _graph.operations().at(ind); + op.accept(*this); + assert(_return_fn); // _return_fn must have been generated + ret->append(std::move(_return_fn)); + + for (auto &&ind : (op.getInputs() | ir::Remove::UNDEFINED) + op.getOutputs()) + { + auto portable_tensor = _tensor_reg->getPortableTensor(ind); + if (portable_tensor) + { + assert(portable_tensor->layout() == ir::Layout::NHWC); + } + + auto tensor = _tensor_reg->getNativeTensor(ind); + if (tensor) + { + tensor->increase_ref(); + } + } + return ret; +} + +void KernelGenerator::visit(const ir::operation::Conv2D &node) +{ + using ir::operation::Conv2D; + + const auto ofm_index{node.getOutputs().at(0)}; + const auto ifm_index{node.getInputs().at(Conv2D::Input::INPUT)}; + const auto ker_index{node.getInputs().at(Conv2D::Input::KERNEL)}; + const auto bias_index{node.getInputs().at(Conv2D::Input::BIAS)}; + + auto ofm_tensor = _tensor_reg->getPortableTensor(ofm_index); + auto ifm_tensor = _tensor_reg->getPortableTensor(ifm_index); + auto ker_tensor = _tensor_reg->getPortableTensor(ker_index); + auto bias_tensor = _tensor_reg->getPortableTensor(bias_index); + + const auto stride = node.param().stride; + const auto activation = node.param().activation; + const auto ¶m_padding = node.param().padding; + const auto dilation = node.param().dilation; + auto fn = std::make_unique<ops::ConvolutionLayer>(_external_context); + + const auto ifm_shape = _ctx.at(ifm_index).shape().asFeature(_current_layout); + const auto ofm_shape = _ctx.at(ofm_index).shape().asFeature(_current_layout); + // Kernel format is [depth_out, kernel_height, kernel_width, depth_in]. + const auto &ker_shape = _ctx.at(ker_index).shape(); + const auto ker_height = ker_shape.dim(1); + const auto ker_width = ker_shape.dim(2); + + const auto padding = + ir::calculatePadding(param_padding, ifm_shape, ofm_shape, stride, ker_width, ker_height, + dilation.width_factor, dilation.height_factor); + + fn->configure(ifm_tensor, ker_tensor, bias_tensor, param_padding.type, padding.left, + padding.right, padding.top, padding.bottom, stride.horizontal, stride.vertical, + dilation.width_factor, dilation.height_factor, activation, ofm_tensor); + + _return_fn = std::move(fn); +} + +void KernelGenerator::visit(const ir::operation::DepthwiseConv2D &node) +{ + using ir::operation::DepthwiseConv2D; + + const auto ofm_index{node.getOutputs().at(0)}; + const auto ifm_index{node.getInputs().at(DepthwiseConv2D::Input::INPUT)}; + const auto ker_index{node.getInputs().at(DepthwiseConv2D::Input::KERNEL)}; + const auto bias_index{node.getInputs().at(DepthwiseConv2D::Input::BIAS)}; + + const auto stride = node.param().stride; + const auto ifm_shape = _ctx.at(ifm_index).shape().asFeature(_current_layout); + const auto ofm_shape = _ctx.at(ofm_index).shape().asFeature(_current_layout); + // Kernel format is [1, kernel_height, kernel_width, depth_out]. + const auto &ker_shape = _ctx.at(ker_index).shape(); + const auto ker_height = ker_shape.dim(1); + const auto ker_width = ker_shape.dim(2); + const auto dilation_width = node.param().dilation.width_factor; + const auto dilation_height = node.param().dilation.height_factor; + const auto ¶m_padding = node.param().padding; + const auto padding = ir::calculatePadding(param_padding, ifm_shape, ofm_shape, stride, ker_width, + ker_height, dilation_width, dilation_height); + const auto multiplier = node.param().multiplier; + const auto activation = node.param().activation; + + auto ofm_tensor = _tensor_reg->getPortableTensor(ofm_index); + auto ifm_tensor = _tensor_reg->getPortableTensor(ifm_index); + auto ker_tensor = _tensor_reg->getPortableTensor(ker_index); + auto bias_tensor = _tensor_reg->getPortableTensor(bias_index); + + auto fn = std::make_unique<ops::DepthwiseConvolutionLayer>(_external_context); + + fn->configure(ifm_tensor, ker_tensor, bias_tensor, param_padding.type, padding.left, + padding.right, padding.top, padding.bottom, stride.horizontal, stride.vertical, + multiplier, dilation_width, dilation_height, activation, ofm_tensor); + + _return_fn = std::move(fn); +} + +void KernelGenerator::visit(const ir::operation::FullyConnected &node) +{ + using ir::operation::FullyConnected; + + const auto output_index{node.getOutputs().at(0)}; + const auto input_index{node.getInputs().at(FullyConnected::Input::INPUT)}; + const auto weight_index{node.getInputs().at(FullyConnected::Input::WEIGHT)}; + const auto bias_index{node.getInputs().at(FullyConnected::Input::BIAS)}; + const auto activation = node.param().activation; + + auto output_tensor = _tensor_reg->getPortableTensor(output_index); + auto input_tensor = _tensor_reg->getPortableTensor(input_index); + auto weight_tensor = _tensor_reg->getPortableTensor(weight_index); + auto bias_tensor = bias_index.undefined() ? nullptr : _tensor_reg->getPortableTensor(bias_index); + + auto fn = std::make_unique<ops::FullyConnectedLayer>(_external_context); + + fn->configure(input_tensor, weight_tensor, bias_tensor, activation, output_tensor); + + _return_fn = std::move(fn); +} + +} // namespace xnnpack +} // namespace backend +} // namespace onert diff --git a/runtime/onert/backend/xnnpack/KernelGenerator.h b/runtime/onert/backend/xnnpack/KernelGenerator.h new file mode 100644 index 000000000..271a60653 --- /dev/null +++ b/runtime/onert/backend/xnnpack/KernelGenerator.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ONERT_BACKEND_XNNPACK_KERNEL_GENERATOR_H__ +#define __ONERT_BACKEND_XNNPACK_KERNEL_GENERATOR_H__ + +#include "ExternalContext.h" +#include "TensorBuilder.h" +#include "backend/basic/TensorRegistry.h" +#include "Tensor.h" + +#include <backend/CustomKernelBuilder.h> +#include <backend/basic/KernelGeneratorBase.h> +#include <ir/Operands.h> +#include <ir/Operations.h> + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ + +class KernelGenerator : public basic::KernelGeneratorBase +{ +public: + KernelGenerator(const ir::Graph &graph, const std::shared_ptr<TensorBuilder> &tensor_builder, + const std::shared_ptr<basic::TensorRegistry> &tensor_reg, + const std::shared_ptr<custom::IKernelBuilder> &kernel_builder, + const std::shared_ptr<ExternalContext> &external_context); + + std::unique_ptr<exec::FunctionSequence> generate(ir::OperationIndex ind) override; + +private: + void visit(const ir::operation::Conv2D &) override; + void visit(const ir::operation::DepthwiseConv2D &) override; + void visit(const ir::operation::FullyConnected &) override; + +private: + const ir::Operands &_ctx; + const ir::Operations &_operations_ctx; + ir::Layout _current_layout; + std::shared_ptr<TensorBuilder> _tensor_builder; + std::shared_ptr<basic::TensorRegistry> _tensor_reg; + std::shared_ptr<backend::custom::IKernelBuilder> _kernel_builder; + const std::shared_ptr<ExternalContext> _external_context; +}; + +} // namespace xnnpack +} // namespace backend +} // namespace onert + +#endif // __ONERT_BACKEND_XNNPACK_KERNEL_GENERATOR_H__ diff --git a/runtime/onert/backend/xnnpack/StaticTensorManager.h b/runtime/onert/backend/xnnpack/StaticTensorManager.h new file mode 100644 index 000000000..adaa3623d --- /dev/null +++ b/runtime/onert/backend/xnnpack/StaticTensorManager.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ONERT_BACKEND_XNNPACK_STATICTENSOR_MANAGER_H__ +#define __ONERT_BACKEND_XNNPACK_STATICTENSOR_MANAGER_H__ + +#include "backend/basic/StaticTensorManager.h" + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ + +using StaticTensorManager = basic::StaticTensorManager; + +} // namespace xnnpack +} // namespace backend +} // namespace onert + +#endif // __ONERT_BACKEND_XNNPACK_STATICTENSOR_MANAGER_H__ diff --git a/runtime/onert/backend/xnnpack/Tensor.h b/runtime/onert/backend/xnnpack/Tensor.h new file mode 100644 index 000000000..147361109 --- /dev/null +++ b/runtime/onert/backend/xnnpack/Tensor.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ONERT_BACKEND_XNNPACK_TENSOR_H__ +#define __ONERT_BACKEND_XNNPACK_TENSOR_H__ + +#include <backend/basic/Tensor.h> +#include <ir/Data.h> + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ + +using Tensor = basic::Tensor; +using ExternalTensor = basic::ExternalTensor; + +} // namespace xnnpack +} // namespace backend +} // namespace onert + +#endif // __ONERT_BACKEND_XNNPACK_TENSOR_H__ diff --git a/runtime/onert/backend/xnnpack/TensorBuilder.h b/runtime/onert/backend/xnnpack/TensorBuilder.h new file mode 100644 index 000000000..cbb7c9e18 --- /dev/null +++ b/runtime/onert/backend/xnnpack/TensorBuilder.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ONERT_BACKEND_XNNPACK_TENSOR_BUILDER_H__ +#define __ONERT_BACKEND_XNNPACK_TENSOR_BUILDER_H__ + +#include <backend/basic/TensorBuilder.h> + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ + +using TensorBuilder = basic::TensorBuilder; + +} // namespace xnnpack +} // namespace backend +} // namespace onert + +#endif // __ONERT_BACKEND_XNNPACK_TENSOR_BUILDER_H__ diff --git a/runtime/onert/backend/xnnpack/ops/ConvolutionLayer.cc b/runtime/onert/backend/xnnpack/ops/ConvolutionLayer.cc new file mode 100644 index 000000000..32ca99460 --- /dev/null +++ b/runtime/onert/backend/xnnpack/ops/ConvolutionLayer.cc @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ConvolutionLayer.h" + +#include "ir/Padding.h" + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ +namespace ops +{ +ConvolutionLayer::ConvolutionLayer(const std::shared_ptr<ExternalContext> external_context) + : Layer(external_context), _input(nullptr), _kernel(nullptr), _bias(nullptr), _output(nullptr), + _padding_type(ir::PaddingType::EXPLICIT), _padding_left(0), _padding_top(0), _padding_right(0), + _padding_bottom(0), _stride_width(0), _stride_height(0), _dilation_width_factor(1), + _dilation_height_factor(1), _activation(ir::Activation::NONE) +{ + // DO NOTHING +} + +void ConvolutionLayer::configure(const IPortableTensor *input, const IPortableTensor *kernel, + const IPortableTensor *bias, ir::PaddingType padding_type, + const uint32_t padding_left, const uint32_t padding_right, + const uint32_t padding_top, const uint32_t padding_bottom, + const uint32_t stride_width, const uint32_t stride_height, + const uint32_t dilation_width_factor, + const uint32_t dilation_height_factor, + const ir::Activation activation, IPortableTensor *output) +{ + _input = input; + _kernel = kernel; + _bias = bias; + _padding_type = padding_type; + _padding_left = padding_left; + _padding_right = padding_right; + _padding_top = padding_top; + _padding_bottom = padding_bottom; + _stride_width = stride_width; + _stride_height = stride_height; + _dilation_width_factor = dilation_width_factor; + _dilation_height_factor = dilation_height_factor; + _activation = activation; + _output = output; + + // TODO Support not nhwc layer + assert(_input->layout() == ir::Layout::NHWC); + + assert(_activation == ir::Activation::NONE || _activation == ir::Activation::RELU || + _activation == ir::Activation::RELU1 || _activation == ir::Activation::RELU6); +} + +void ConvolutionLayer::run() +{ + assert(_external_context && _external_context->getThreadPool()); + if (!_setup) + { + _setup = setup(); + assert(_setup); + } + + if (_input->data_type() == OperandType::FLOAT32) + { + enum xnn_status status = xnn_run_operator(_kernel_op, _external_context->getThreadPool()); + if (status != xnn_status_success) + { + throw std::runtime_error{"failed to run FP32 Convolution operator"}; + } + } + else + { + throw std::runtime_error{"XNNPACK Conv: unsupported data type"}; + } +} + +bool ConvolutionLayer::create() +{ + float output_activation_min = 0.f, output_activation_max = 0.f; + CalculateActivationRange<float>(_activation, &output_activation_min, &output_activation_max); + + // NHWC + // Kernel format is [depth_out, kernel_height, kernel_width, depth_in]. + const auto &kernel_shape = _kernel->getShape(); + uint32_t kernel_height = kernel_shape.dim(1); + uint32_t kernel_width = kernel_shape.dim(2); + uint32_t output_channels = kernel_shape.dim(0); + uint32_t input_channels = kernel_shape.dim(3); + assert(static_cast<uint32_t>(_input->getShape().dim(3)) == input_channels); + assert(static_cast<uint32_t>(_output->getShape().dim(3)) == output_channels); + + enum xnn_status status = xnn_create_convolution2d_nhwc_f32( + _padding_top, _padding_right, _padding_bottom, _padding_left, kernel_height, kernel_width, + _stride_height, _stride_width, _dilation_height_factor, _dilation_width_factor, 1 /* groups */, + input_channels /* group_input_channels */, output_channels /* group_output_channels */, + input_channels /* input_channel_stride */, output_channels /* output_channel_stride */, + reinterpret_cast<const float *>(_kernel->buffer()), + reinterpret_cast<const float *>(_bias->buffer()), output_activation_min, output_activation_max, + 0, &_kernel_op); + if (status != xnn_status_success) + { + throw std::runtime_error{"failed to create FP32 Convolution operator"}; + } + assert(_kernel_op != nullptr); + return true; +} + +bool ConvolutionLayer::setup() +{ + if (_input->buffer() == nullptr || _output->buffer() == nullptr) + { + // it could be models's input or output + return false; + } + + uint32_t input_width = _input->getShape().dim(2); + uint32_t input_height = _input->getShape().dim(1); + uint32_t batch_size = _input->getShape().dim(0); + enum xnn_status status = xnn_setup_convolution2d_nhwc_f32( + _kernel_op, batch_size, input_height, input_width, + reinterpret_cast<const float *>(_input->buffer()), reinterpret_cast<float *>(_output->buffer()), + _external_context->getThreadPool()); + if (status != xnn_status_success) + { + throw std::runtime_error{"failed to create FP32 Convolution operator"}; + } + return true; +} + +} // namespace ops +} // namespace xnnpack +} // namespace backend +} // namespace onert diff --git a/runtime/onert/backend/xnnpack/ops/ConvolutionLayer.h b/runtime/onert/backend/xnnpack/ops/ConvolutionLayer.h new file mode 100644 index 000000000..6cbaa9f3a --- /dev/null +++ b/runtime/onert/backend/xnnpack/ops/ConvolutionLayer.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ONERT_BACKEND_XNNPACK_OPS_CONVOLUTION_LAYER_H__ +#define __ONERT_BACKEND_XNNPACK_OPS_CONVOLUTION_LAYER_H__ + +#include "Layer.h" + +#include <xnnpack.h> + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ +namespace ops +{ + +class ConvolutionLayer : public Layer +{ +public: + ConvolutionLayer(const std::shared_ptr<ExternalContext> external_context); + +public: + void configure(const IPortableTensor *input, const IPortableTensor *kernel, + const IPortableTensor *bias, ir::PaddingType padding_type, + const uint32_t padding_left, const uint32_t padding_right, + const uint32_t padding_top, const uint32_t padding_bottom, + const uint32_t stride_width, const uint32_t stride_height, + const uint32_t dilation_width_factor, const uint32_t dilation_height_factor, + const ir::Activation activation, IPortableTensor *output); + + void run() override; + + bool create() override; + bool setup() override; + +private: + const IPortableTensor *_input; + const IPortableTensor *_kernel; + const IPortableTensor *_bias; + IPortableTensor *_output; + + ir::PaddingType _padding_type; + uint32_t _padding_left; + uint32_t _padding_top; + uint32_t _padding_right; + uint32_t _padding_bottom; + + uint32_t _stride_width; + uint32_t _stride_height; + uint32_t _dilation_width_factor; + uint32_t _dilation_height_factor; + + ir::Activation _activation; +}; + +} // namespace ops +} // namespace xnnpack +} // namespace backend +} // namespace onert + +#endif // __ONERT_BACKEND_XNNPACK_OPS_CONVOLUTION_LAYER_H__ diff --git a/runtime/onert/backend/xnnpack/ops/DepthwiseConvolutionLayer.cc b/runtime/onert/backend/xnnpack/ops/DepthwiseConvolutionLayer.cc new file mode 100644 index 000000000..9a671d487 --- /dev/null +++ b/runtime/onert/backend/xnnpack/ops/DepthwiseConvolutionLayer.cc @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "DepthwiseConvolutionLayer.h" + +#include "ir/Padding.h" + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ +namespace ops +{ + +DepthwiseConvolutionLayer::DepthwiseConvolutionLayer( + const std::shared_ptr<ExternalContext> external_context) + : Layer(external_context), _input(nullptr), _kernel(nullptr), _bias(nullptr), _output(nullptr), + _padding_type(ir::PaddingType::EXPLICIT), _padding_left(0), _padding_top(0), _padding_right(0), + _padding_bottom(0), _stride_width(0), _stride_height(0), _multiplier(1), + _dilation_width_factor(1), _dilation_height_factor(1), _activation(ir::Activation::NONE) +{ + // DO NOTHING +} + +void DepthwiseConvolutionLayer::configure( + const IPortableTensor *input, const IPortableTensor *kernel, const IPortableTensor *bias, + ir::PaddingType padding_type, const uint32_t padding_left, const uint32_t padding_right, + const uint32_t padding_top, const uint32_t padding_bottom, const uint32_t stride_width, + const uint32_t stride_height, const uint32_t multiplier, const uint32_t dilation_width_factor, + const uint32_t dilation_height_factor, const ir::Activation activation, IPortableTensor *output) +{ + _input = input; + _kernel = kernel; + _bias = bias; + _padding_type = padding_type; + _padding_left = padding_left; + _padding_right = padding_right; + _padding_top = padding_top; + _padding_bottom = padding_bottom; + _stride_width = stride_width; + _stride_height = stride_height; + _multiplier = multiplier; + _dilation_width_factor = dilation_width_factor; + _dilation_height_factor = dilation_height_factor; + _activation = activation; + _output = output; + + // TODO Support not nhwc layer + assert(_input->layout() == ir::Layout::NHWC); + + assert(_activation == ir::Activation::NONE || _activation == ir::Activation::RELU || + _activation == ir::Activation::RELU1 || _activation == ir::Activation::RELU6); +} + +void DepthwiseConvolutionLayer::run() +{ + assert(_external_context && _external_context->getThreadPool()); + if (!_setup) + { + _setup = setup(); + assert(_setup); + } + + if (_input->data_type() == OperandType::FLOAT32) + { + enum xnn_status status = xnn_run_operator(_kernel_op, _external_context->getThreadPool()); + if (status != xnn_status_success) + { + throw std::runtime_error{"failed to run FP32 DepthwiseConvolution operator"}; + } + } + else + { + throw std::runtime_error{"XNNPACK DepthwiseConv: unsupported data type"}; + } +} + +bool DepthwiseConvolutionLayer::create() +{ + float output_activation_min = 0.f, output_activation_max = 0.f; + CalculateActivationRange<float>(_activation, &output_activation_min, &output_activation_max); + + // NHWC + // Kernel format is [1, kernel_height, kernel_width, depth_out]. + const auto &kernel_shape = _kernel->getShape(); + uint32_t kernel_height = kernel_shape.dim(1); + uint32_t kernel_width = kernel_shape.dim(2); + uint32_t output_channels = kernel_shape.dim(3); + uint32_t input_channels = _input->getShape().dim(3); + assert(static_cast<uint32_t>(_output->getShape().dim(3)) == output_channels); + assert(output_channels == input_channels * _multiplier); + + enum xnn_status status = xnn_create_convolution2d_nhwc_f32( + _padding_top, _padding_right, _padding_bottom, _padding_left, kernel_height, kernel_width, + _stride_height, _stride_width, _dilation_height_factor, _dilation_width_factor, + input_channels /* groups */, 1 /* group_input_channels */, + _multiplier /* group_output_channels */, input_channels /* input_channel_stride */, + output_channels /* output_channel_stride */, reinterpret_cast<const float *>(_kernel->buffer()), + reinterpret_cast<const float *>(_bias->buffer()), output_activation_min, output_activation_max, + XNN_FLAG_DEPTHWISE_CONVOLUTION, &_kernel_op); + if (status != xnn_status_success) + { + throw std::runtime_error{"failed to create FP32 DepthwiseConvolution operator"}; + } + assert(_kernel_op != nullptr); + return true; +} + +bool DepthwiseConvolutionLayer::setup() +{ + if (_input->buffer() == nullptr || _output->buffer() == nullptr) + { + // it could be models's input or output + return false; + } + + uint32_t input_width = _input->getShape().dim(2); + uint32_t input_height = _input->getShape().dim(1); + uint32_t batch_size = _input->getShape().dim(0); + enum xnn_status status = xnn_setup_convolution2d_nhwc_f32( + _kernel_op, batch_size, input_height, input_width, + reinterpret_cast<const float *>(_input->buffer()), reinterpret_cast<float *>(_output->buffer()), + _external_context->getThreadPool()); + if (status != xnn_status_success) + { + throw std::runtime_error{"failed to create FP32 DepthwiseConvolution operator"}; + } + return true; +} + +} // namespace ops +} // namespace xnnpack +} // namespace backend +} // namespace onert diff --git a/runtime/onert/backend/xnnpack/ops/DepthwiseConvolutionLayer.h b/runtime/onert/backend/xnnpack/ops/DepthwiseConvolutionLayer.h new file mode 100644 index 000000000..10f840ae7 --- /dev/null +++ b/runtime/onert/backend/xnnpack/ops/DepthwiseConvolutionLayer.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ONERT_BACKEND_XNNPACK_OPS_DEPTHWISE_CONVOLUTION_LAYER_H__ +#define __ONERT_BACKEND_XNNPACK_OPS_DEPTHWISE_CONVOLUTION_LAYER_H__ + +#include "Layer.h" + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ +namespace ops +{ + +class DepthwiseConvolutionLayer : public Layer +{ +public: + DepthwiseConvolutionLayer(const std::shared_ptr<ExternalContext> external_context); + +public: + void configure(const IPortableTensor *input, const IPortableTensor *kernel, + const IPortableTensor *bias, ir::PaddingType padding_type, + const uint32_t padding_left, const uint32_t padding_right, + const uint32_t padding_top, const uint32_t padding_bottom, + const uint32_t stride_width, const uint32_t stride_height, + const uint32_t multiplier, const uint32_t dilation_width_factor, + const uint32_t dilation_height_factor, const ir::Activation activation, + IPortableTensor *output); + + void run() override; + + bool create() override; + bool setup() override; + +private: + const IPortableTensor *_input; + const IPortableTensor *_kernel; + const IPortableTensor *_bias; + IPortableTensor *_output; + + ir::PaddingType _padding_type; + uint32_t _padding_left; + uint32_t _padding_top; + uint32_t _padding_right; + uint32_t _padding_bottom; + + uint32_t _stride_width; + uint32_t _stride_height; + uint32_t _multiplier; + uint32_t _dilation_width_factor; + uint32_t _dilation_height_factor; + + ir::Activation _activation; +}; + +} // namespace ops +} // namespace xnnpack +} // namespace backend +} // namespace onert + +#endif // __ONERT_BACKEND_XNNPACK_OPS_DEPTHWISE_CONVOLUTION_LAYER_H__ diff --git a/runtime/onert/backend/xnnpack/ops/FullyConnectedLayer.cc b/runtime/onert/backend/xnnpack/ops/FullyConnectedLayer.cc new file mode 100644 index 000000000..66171ad42 --- /dev/null +++ b/runtime/onert/backend/xnnpack/ops/FullyConnectedLayer.cc @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "FullyConnectedLayer.h" + +#include "ir/Padding.h" + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ +namespace ops +{ + +FullyConnectedLayer::FullyConnectedLayer(const std::shared_ptr<ExternalContext> external_context) + : Layer(external_context), _input(nullptr), _kernel(nullptr), _bias(nullptr), _output(nullptr), + _activation(ir::Activation::NONE) +{ + // DO NOTHING +} + +void FullyConnectedLayer::configure(const IPortableTensor *input, const IPortableTensor *weights, + const IPortableTensor *bias, ir::Activation activation, + IPortableTensor *output) +{ + _input = input; + _kernel = weights; + _bias = bias; + _activation = activation; + _output = output; + + // TODO Support not nhwc layer + assert(_input->layout() == ir::Layout::NHWC); + + assert(_activation == ir::Activation::NONE || _activation == ir::Activation::RELU || + _activation == ir::Activation::RELU1 || _activation == ir::Activation::RELU6); +} + +void FullyConnectedLayer::run() +{ + assert(_external_context && _external_context->getThreadPool()); + if (!_setup) + { + _setup = setup(); + assert(_setup); + } + + if (_input->data_type() == OperandType::FLOAT32) + { + enum xnn_status status = xnn_run_operator(_kernel_op, _external_context->getThreadPool()); + if (status != xnn_status_success) + { + throw std::runtime_error{"failed to run FP32 FullyConnected operator"}; + } + } + else + { + throw std::runtime_error{"XNNPACK FC: unsupported data type"}; + } +} + +bool FullyConnectedLayer::create() +{ + float output_activation_min = 0.f, output_activation_max = 0.f; + CalculateActivationRange<float>(_activation, &output_activation_min, &output_activation_max); + + const auto &kernel_shape = _kernel->getShape(); + assert(kernel_shape.rank() == 2); + uint32_t output_channels = kernel_shape.dim(0); + uint32_t input_channels = kernel_shape.dim(1); + + const auto &input_shape = _input->getShape(); + const auto &output_shape = _output->getShape(); + uint32_t flag = 0; + if (input_shape.rank() != output_shape.rank()) + { + flag |= XNN_FLAG_TENSORFLOW_RESHAPE_2D; + assert(input_shape.num_elements() % input_channels == 0); + } + else + { + assert(static_cast<uint32_t>(input_shape.dim(input_shape.rank() - 1)) == input_channels); + } + + assert(_kernel && _kernel->buffer()); + const float *kernel_buffer = reinterpret_cast<const float *>(_kernel->buffer()); + const float *bias_buffer = (_bias) ? reinterpret_cast<const float *>(_bias->buffer()) : nullptr; + + enum xnn_status status = xnn_create_fully_connected_nc_f32( + input_channels, output_channels, input_channels /* input stride */, + output_channels /* output stride */, kernel_buffer, bias_buffer, output_activation_min, + output_activation_max, flag, &_kernel_op); + if (status != xnn_status_success) + { + throw std::runtime_error{"failed to create FP32 FullyConnected operator"}; + } + assert(_kernel_op != nullptr); + return true; +} + +bool FullyConnectedLayer::setup() +{ + if (_input->buffer() == nullptr || _output->buffer() == nullptr) + { + // it could be models's input or output + return false; + } + + uint32_t batch_size = _input->getShape().num_elements() / _kernel->getShape().dim(1); + enum xnn_status status = xnn_setup_fully_connected_nc_f32( + _kernel_op, batch_size, reinterpret_cast<const float *>(_input->buffer()), + reinterpret_cast<float *>(_output->buffer()), _external_context->getThreadPool()); + if (status != xnn_status_success) + { + throw std::runtime_error{"failed to create FP32 FullyConnected operator"}; + } + return true; +} + +} // namespace ops +} // namespace xnnpack +} // namespace backend +} // namespace onert diff --git a/runtime/onert/backend/xnnpack/ops/FullyConnectedLayer.h b/runtime/onert/backend/xnnpack/ops/FullyConnectedLayer.h new file mode 100644 index 000000000..883607ef9 --- /dev/null +++ b/runtime/onert/backend/xnnpack/ops/FullyConnectedLayer.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ONERT_BACKEND_XNNPACK_OPS_FULLY_CONNECTED_LAYER_H__ +#define __ONERT_BACKEND_XNNPACK_OPS_FULLY_CONNECTED_LAYER_H__ + +#include "Layer.h" + +#include <xnnpack.h> + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ +namespace ops +{ + +class FullyConnectedLayer : public Layer +{ +public: + FullyConnectedLayer(const std::shared_ptr<ExternalContext> external_context); + +public: + void configure(const IPortableTensor *input, const IPortableTensor *_kernel, + const IPortableTensor *bias, ir::Activation activation, IPortableTensor *output); + + void run() override; + + bool create() override; + bool setup() override; + +private: + const IPortableTensor *_input; + const IPortableTensor *_kernel; + const IPortableTensor *_bias; + IPortableTensor *_output; + + ir::Activation _activation; +}; + +} // namespace ops +} // namespace xnnpack +} // namespace backend +} // namespace onert + +#endif // __ONERT_BACKEND_XNNPACK_OPS_FULLY_CONNECTED_LAYER_H__ diff --git a/runtime/onert/backend/xnnpack/ops/Layer.h b/runtime/onert/backend/xnnpack/ops/Layer.h new file mode 100644 index 000000000..ec07e874f --- /dev/null +++ b/runtime/onert/backend/xnnpack/ops/Layer.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ONERT_BACKEND_XNNPACK_OPS_LAYER_H__ +#define __ONERT_BACKEND_XNNPACK_OPS_LAYER_H__ + +#include <exec/IFunction.h> +#include <backend/IPortableTensor.h> +#include "OperationUtils.h" +#include "../ExternalContext.h" +#include "../Tensor.h" + +#include <cassert> +#include <memory> + +#include <xnnpack.h> + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ +namespace ops +{ + +class Layer : public ::onert::exec::IFunction +{ +public: + Layer(const std::shared_ptr<ExternalContext> external_context) + : _kernel_op{nullptr}, _create{false}, _setup{false}, _external_context{external_context} + { + // DO NOTHING + } + + ~Layer() + { + if (_kernel_op) + xnn_delete_operator(_kernel_op); + } + +public: + void prepare() override + { + if (_create) + return; + + _create = create(); + assert(_create); + + _setup = setup(); + } + virtual bool create() = 0; + virtual bool setup() = 0; + +protected: + xnn_operator_t _kernel_op; + bool _create; + bool _setup; + const std::shared_ptr<ExternalContext> _external_context; +}; + +} // namespace ops +} // namespace xnnpack +} // namespace backend +} // namespace onert + +#endif // __ONERT_BACKEND_XNNPACK_OPS_LAYER_H__ diff --git a/runtime/onert/backend/xnnpack/ops/OperationUtils.h b/runtime/onert/backend/xnnpack/ops/OperationUtils.h new file mode 100644 index 000000000..fe93fccc0 --- /dev/null +++ b/runtime/onert/backend/xnnpack/ops/OperationUtils.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ONERT_BACKEND_XNNPACK_OPS_OPERATION_UTILS_H__ +#define __ONERT_BACKEND_XNNPACK_OPS_OPERATION_UTILS_H__ + +#include <ir/DataType.h> +#include <ir/InternalType.h> +#include <ir/Padding.h> +#include <util/CalculateActivationRange.h> + +namespace onert +{ +namespace backend +{ +namespace xnnpack +{ +namespace ops +{ + +using OperandType = ir::DataType; +using namespace onert::util; // CalculateActivationRange + +} // namespace ops +} // namespace xnnpack +} // namespace backend +} // namespace onert + +#endif // __ONERT_BACKEND_XNNPACK_OPS_OPERATION_UTILS_H__ diff --git a/runtime/onert/backend/xnnpack/xnnpack.cc b/runtime/onert/backend/xnnpack/xnnpack.cc new file mode 100644 index 000000000..38a6c5572 --- /dev/null +++ b/runtime/onert/backend/xnnpack/xnnpack.cc @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Backend.h" + +#include <util/logging.h> + +extern "C" { +onert::backend::Backend *onert_backend_create() +{ + VERBOSE(onert_backend_create) << "'xnnpack' loaded\n"; + return new onert::backend::xnnpack::Backend; +} + +void onert_backend_destroy(onert::backend::Backend *backend) +{ + VERBOSE(onert_backend_create) << "'xnnpack' unloaded\n"; + delete backend; +} +} |