From c55f8a6db48cda9d3a78048338b7f18c4cca62b8 Mon Sep 17 00:00:00 2001 From: Chunseok Lee Date: Wed, 28 Oct 2020 12:16:55 +0900 Subject: Imported Upstream version 1.10.0 --- .../frontend/nnapi/ANeuralNetworksModel.test.cc | 25 ---- runtime/onert/frontend/nnapi/execution.cc | 24 ++++ .../nnapi/wrapper/ANeuralNetworksExecution.cc | 50 +++++++ .../nnapi/wrapper/ANeuralNetworksExecution.h | 3 + .../nnapi/wrapper/ANeuralNetworksModel.test.cc | 25 ++++ .../frontend/nnapi/wrapper/OperationFactory.cc | 155 +++++++++++++++++---- 6 files changed, 227 insertions(+), 55 deletions(-) delete mode 100644 runtime/onert/frontend/nnapi/ANeuralNetworksModel.test.cc create mode 100644 runtime/onert/frontend/nnapi/wrapper/ANeuralNetworksModel.test.cc (limited to 'runtime/onert/frontend/nnapi') diff --git a/runtime/onert/frontend/nnapi/ANeuralNetworksModel.test.cc b/runtime/onert/frontend/nnapi/ANeuralNetworksModel.test.cc deleted file mode 100644 index 15a279a7e..000000000 --- a/runtime/onert/frontend/nnapi/ANeuralNetworksModel.test.cc +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "wrapper/ANeuralNetworksModel.h" - -TEST(MODEL, model_build) -{ - ANeuralNetworksModel model; - ASSERT_EQ(model.isFinished(), false); -} diff --git a/runtime/onert/frontend/nnapi/execution.cc b/runtime/onert/frontend/nnapi/execution.cc index ce7da579e..56ca5ef00 100644 --- a/runtime/onert/frontend/nnapi/execution.cc +++ b/runtime/onert/frontend/nnapi/execution.cc @@ -94,12 +94,36 @@ int ANeuralNetworksExecution_setInput(ANeuralNetworksExecution *execution, int32 // Omitted optional input // LSTM operation's some inputs can be optional input + // Transpose operation's permutation input can be optional input if ((buffer == nullptr) && (length == 0)) { + uint32_t dims[1] = {0}; + ANeuralNetworksOperandType compared_shape; + compared_shape.dimensionCount = 1; + compared_shape.dimensions = dims; if (execution->hasUnspecifiedDims(operand_index)) { return ANEURALNETWORKS_NO_ERROR; } + else if (type == nullptr && execution->IsOptionalInput(operand_index)) + { + if (!execution->setOptionalInput(index, type, buffer, length)) + { + VERBOSE(NNAPI::Execution) << "setInput: Fail to set optional input" << std::endl; + return ANEURALNETWORKS_BAD_DATA; + } + return ANEURALNETWORKS_NO_ERROR; + } + // TODO Changes the condition to check zero sized + else if (execution->compareShape(&compared_shape, operand_index)) + { + if (!execution->setInput(index, type, buffer, length)) + { + VERBOSE(NNAPI::Execution) << "setInput: Fail to set input" << std::endl; + return ANEURALNETWORKS_BAD_DATA; + } + return ANEURALNETWORKS_NO_ERROR; + } else { VERBOSE(NNAPI::Execution) << "setInput: Cannot handle fully-specified shape on model build " diff --git a/runtime/onert/frontend/nnapi/wrapper/ANeuralNetworksExecution.cc b/runtime/onert/frontend/nnapi/wrapper/ANeuralNetworksExecution.cc index eb12d7e76..6114b74b0 100644 --- a/runtime/onert/frontend/nnapi/wrapper/ANeuralNetworksExecution.cc +++ b/runtime/onert/frontend/nnapi/wrapper/ANeuralNetworksExecution.cc @@ -98,6 +98,17 @@ bool ANeuralNetworksExecution::compareShape(const ANeuralNetworksOperandType *ty return operand_shape == shape_from_type; } +bool ANeuralNetworksExecution::IsOptionalInput(const onert::ir::OperandIndex index) noexcept +{ + const auto &operand_shape = _execution->primary_subgraph().operands().at(index).shape(); + for (int32_t i = 0; i < operand_shape.rank(); ++i) + { + if (operand_shape.dim(i) != 0) + return false; + } + return true; +} + bool ANeuralNetworksExecution::hasUnspecifiedDims(const onert::ir::OperandIndex index) noexcept { const auto operand_shape = _execution->primary_subgraph().operands().at(index).shape(); @@ -148,6 +159,45 @@ bool ANeuralNetworksExecution::setInput(uint32_t index, const ANeuralNetworksOpe return true; } +bool ANeuralNetworksExecution::setOptionalInput(uint32_t index, + const ANeuralNetworksOperandType *type, + const void *buffer, size_t length) noexcept +{ + assert(type == nullptr); + assert(buffer == nullptr); + assert(length == 0); + try + { + onert::ir::IOIndex input_index{index}; + const auto operand_index = getInputOperandIndex(index); + + const auto type_info = _execution->primary_subgraph().operands().at(operand_index).typeInfo(); + const auto shape = (type != nullptr) + ? NNAPIConvert::getShape(type) + : _execution->primary_subgraph().operands().at(operand_index).shape(); + + // ANeuralNetworksExecution::setInput() uses only shape information + ANeuralNetworksOperandType optional_input_type; + optional_input_type.dimensionCount = shape.rank(); + std::vector dims(optional_input_type.dimensionCount); + for (uint32_t i = 0; i < optional_input_type.dimensionCount; ++i) + { + dims.at(i) = shape.dim(i); + } + optional_input_type.dimensions = dims.data(); + + return setInput(index, &optional_input_type, buffer, length); + } + catch (const std::exception &e) + { + VERBOSE(EXCEPTION) << e.what() << std::endl; + + return false; + } + + return true; +} + bool ANeuralNetworksExecution::setOutput(uint32_t index, const ANeuralNetworksOperandType *type, void *buffer, size_t length) noexcept { diff --git a/runtime/onert/frontend/nnapi/wrapper/ANeuralNetworksExecution.h b/runtime/onert/frontend/nnapi/wrapper/ANeuralNetworksExecution.h index 848ae743f..1f4b868f6 100644 --- a/runtime/onert/frontend/nnapi/wrapper/ANeuralNetworksExecution.h +++ b/runtime/onert/frontend/nnapi/wrapper/ANeuralNetworksExecution.h @@ -35,6 +35,8 @@ public: public: bool setInput(uint32_t index, const ANeuralNetworksOperandType *type, const void *buffer, size_t length) noexcept; + bool setOptionalInput(uint32_t index, const ANeuralNetworksOperandType *type, const void *buffer, + size_t length) noexcept; bool setOutput(uint32_t index, const ANeuralNetworksOperandType *type, void *buffer, size_t length) noexcept; bool startExecute(void) noexcept; @@ -46,6 +48,7 @@ public: const onert::ir::OperandIndex index) noexcept; bool compareShape(const ANeuralNetworksOperandType *type, const onert::ir::OperandIndex index) noexcept; + bool IsOptionalInput(const onert::ir::OperandIndex index) noexcept; bool hasUnspecifiedDims(const onert::ir::OperandIndex index) noexcept; size_t getOperandSize(const onert::ir::OperandIndex index) noexcept; const std::shared_ptr instance(void) noexcept; diff --git a/runtime/onert/frontend/nnapi/wrapper/ANeuralNetworksModel.test.cc b/runtime/onert/frontend/nnapi/wrapper/ANeuralNetworksModel.test.cc new file mode 100644 index 000000000..bb42f2b08 --- /dev/null +++ b/runtime/onert/frontend/nnapi/wrapper/ANeuralNetworksModel.test.cc @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "ANeuralNetworksModel.h" + +TEST(MODEL, neg_model_build) +{ + ANeuralNetworksModel model; + ASSERT_FALSE(model.isFinished()); +} diff --git a/runtime/onert/frontend/nnapi/wrapper/OperationFactory.cc b/runtime/onert/frontend/nnapi/wrapper/OperationFactory.cc index 8e3d83db4..e6c38f5f8 100644 --- a/runtime/onert/frontend/nnapi/wrapper/OperationFactory.cc +++ b/runtime/onert/frontend/nnapi/wrapper/OperationFactory.cc @@ -708,31 +708,7 @@ OperationFactory::OperationFactory() return new operation::StridedSlice{inputs, outputs, param}; }; - _map[ANEURALNETWORKS_TRANSPOSE] = [](const OperationFactory::Param &init_param, - Operands &operands) { - // TODO make this work with init_param.input_count == 1 (when permutation vector is optional) - - // Inputs - // 0: An n-D tensor, specifying the tensor to be transposed. - // 1: An optional 1-D Tensor of {@link ANEURALNETWORKS_TENSOR_INT32}, - // the permutation of the dimensions of the input tensor. - // The returned tensor's dimension i corresponds to the input dimension - // perm[i]. If perm is not given, it is set to (n-1...0), where n is the - // rank of the input tensor. Hence by default, this operation performs a - // regular matrix transpose on 2-D input Tensors. - assert(init_param.input_count == 2); - assert(init_param.output_count == 1); - - OperandIndexSequence inputs{init_param.inputs[0]}; - OperandIndexSequence outputs{init_param.outputs[0]}; - std::vector perm = - operands.at(OperandIndex{init_param.inputs[1]}).asVector(); - - operation::Transpose::Param param; - param.perm.assign(perm.cbegin(), perm.cend()); - - return new operation::Transpose{inputs, outputs, param}; - }; + _map[ANEURALNETWORKS_TRANSPOSE] = createSimpleBinaryOp; _map[ANEURALNETWORKS_MUL] = getBinaryArithmeticGenerator(onert::ir::operation::BinaryArithmetic::ArithmeticType::MUL); @@ -982,6 +958,28 @@ OperationFactory::OperationFactory() return new operation::ResizeBilinear{inputs, outputs, param}; }; + _map[ANEURALNETWORKS_RESIZE_NEAREST_NEIGHBOR] = [](const OperationFactory::Param &init_param, + Operands &operands) { + assert((init_param.input_count == 3 || init_param.input_count == 4) && + init_param.output_count == 1); + + OperandIndexSequence outputs{init_param.outputs[0]}; + + // Each input should be interpreted as follows: + // + // 0 -> IFM Index + // 1 -> Height Index + // 2 -> Width Index + OperandIndexSequence inputs{init_param.inputs[0]}; + + operation::ResizeNearestNeighbor::Param param; + param.height_out = operands.at(OperandIndex{init_param.inputs[1]}).asScalar(); + param.width_out = operands.at(OperandIndex{init_param.inputs[2]}).asScalar(); + param.align_corners = false; + // The layout input is not supported yet + return new operation::ResizeNearestNeighbor{inputs, outputs, param}; + }; + _map[ANEURALNETWORKS_RELU1] = getElementwiseActivationGenerator( onert::ir::operation::ElementwiseActivation::Type::RELU, 1.f, -1.f); @@ -1304,6 +1302,105 @@ OperationFactory::OperationFactory() } param.cell_threshold = operands.at(OperandIndex{init_param.inputs[21]}).asScalar(); param.projection_threshold = operands.at(OperandIndex{init_param.inputs[22]}).asScalar(); + // This is initialization to prevent warning or error by static code analyzer. LSTM operation + // does not need time_major + param.time_major = false; + + return new operation::LSTM{inputs, outputs, param}; + }; + + _map[ANEURALNETWORKS_UNIDIRECTIONAL_SEQUENCE_LSTM] = [](const OperationFactory::Param &init_param, + Operands &operands) { + assert((init_param.input_count >= 24 || init_param.input_count <= 28) && + (init_param.output_count >= 1 && init_param.output_count <= 3)); + + // Each input should be interpreted as follows: + // + // 0 -> Input Tensor Index + // 1 -> Input to Input Tensor Index + // 2 -> Input to Forget Tensor Index + // 3 -> Input to Cell Tensor Index + // 4 -> Input to Output Tensor Index + // 5 -> Recurrent to Input Weights Tensor Index + // 6 -> Recurrent to Forget Weights Tensor Index + // 7 -> Recurrent to Cell Weights Tensor Index + // 8 -> Recurrent to Output Weights Tensor Index + // 9 -> Cell to Input Weights Tensor Index + // 10 -> Cell to Forget Weights Tensor Index + // 11 -> Cell to Output Weights Tensor Index + // 12 -> Input Gate Bias Tensor Index + // 13 -> Forget Gate Bias Tensor Index + // 14 -> Cell Bias Tensor Index + // 15 -> Output Gate Bias Tensor Index + // 16 -> Projection Weights Tensor Index + // 17 -> Projection Bias Tensor Index + // 18 -> Output State In Tensor Index + // 19 -> Cell State In Tensor Index + assert(init_param.input_count - 3 > 20); + OperandIndexSequence inputs; + for (uint32_t n = 0; n < 20; ++n) + { + inputs.append(OperandIndex{init_param.inputs[n]}); + } + + // 24 -> Input Layer Normalization Weights Tensor Index + // 25 -> Forget Layer Normalization Weights Tensor Index + // 26 -> Cell Layer Normalization Weights Tensor Index + // 27 -> Output Layer Normalization Weights Tensor Index + if (init_param.input_count > 24) + { + for (uint32_t n = 24; n < 28; ++n) + { + if (init_param.input_count > n) + { + inputs.append(OperandIndex{init_param.inputs[n]}); + } + } + } + + // Each output should be interpreted as follows: + // + // 0 -> Output Tensor Index -> 3 + // 1 -> Output State Out Tensor Index + // 2 -> Cell State Out Tensor Index + const OperandIndex scratch_buffer_index; + OperandIndex output_state_index = + init_param.output_count >= 2 ? OperandIndex{init_param.outputs[1]} : OperandIndex(); + OperandIndex cell_state_index = + init_param.output_count >= 3 ? OperandIndex{init_param.outputs[2]} : OperandIndex(); + const OperandIndex output_index = OperandIndex{init_param.outputs[0]}; + OperandIndexSequence outputs{scratch_buffer_index, output_state_index, cell_state_index, + output_index}; + + operation::LSTM::Param param; + const auto activation_index = OperandIndex{init_param.inputs[20]}; + switch (operands.at(activation_index).asScalar()) + { + case 0: + param.activation = Activation::NONE; + break; + case 1: + param.activation = Activation::RELU; + break; + case 2: + param.activation = Activation::RELU1; + break; + case 3: + param.activation = Activation::RELU6; + break; + case 4: + param.activation = Activation::TANH; + break; + case 6: + param.activation = Activation::SIGMOID; + break; + default: + throw std::runtime_error("Unsupported activation type"); + break; + } + param.cell_threshold = operands.at(OperandIndex{init_param.inputs[21]}).asScalar(); + param.projection_threshold = operands.at(OperandIndex{init_param.inputs[22]}).asScalar(); + param.time_major = operands.at(OperandIndex{init_param.inputs[23]}).asScalar(); return new operation::LSTM{inputs, outputs, param}; }; @@ -1406,7 +1503,7 @@ OperationFactory::OperationFactory() // TODO Remove ANEURALNETWORKS_ABS_EX _map[ANEURALNETWORKS_ABS_EX] = _map[ANEURALNETWORKS_ABS]; - _map[ANEURALNETWORKS_ARGMAX] = [](const OperationFactory::Param &init_param, Operands &operands) { + _map[ANEURALNETWORKS_ARGMAX] = [](const OperationFactory::Param &init_param, Operands &) { assert(init_param.input_count == 2 && init_param.output_count == 1); OperandIndexSequence outputs{init_param.outputs[0]}; @@ -1415,10 +1512,9 @@ OperationFactory::OperationFactory() // // 0 -> Input Tensor Index // 1 -> Axis Tensor Index - OperandIndexSequence inputs{init_param.inputs[0]}; + OperandIndexSequence inputs{init_param.inputs[0], init_param.inputs[1]}; operation::ArgMax::Param param; - param.axis = operands.at(OperandIndex{init_param.inputs[1]}).asScalar(); // NNAPI ARGMAX output type is always int32 param.output_type = DataType::INT32; @@ -1517,7 +1613,7 @@ OperationFactory::OperationFactory() assert(init_param.input_count == 3); assert(init_param.output_count >= 1); // At least one output tensor and axis - OperandIndexSequence inputs{init_param.inputs[0]}; + OperandIndexSequence inputs{init_param.inputs[1], init_param.inputs[0]}; OperandIndexSequence outputs; for (uint32_t n = 0; n < init_param.output_count; ++n) { @@ -1525,7 +1621,6 @@ OperationFactory::OperationFactory() } operation::Split::Param param; - param.axis = operands.at(OperandIndex{init_param.inputs[1]}).asScalar(); param.num_splits = operands.at(OperandIndex{init_param.inputs[2]}).asScalar(); return new operation::Split{inputs, outputs, param}; -- cgit v1.2.3