diff options
Diffstat (limited to 'runtimes/neurun/frontend/tflite/loader.cc')
-rw-r--r-- | runtimes/neurun/frontend/tflite/loader.cc | 700 |
1 files changed, 700 insertions, 0 deletions
diff --git a/runtimes/neurun/frontend/tflite/loader.cc b/runtimes/neurun/frontend/tflite/loader.cc new file mode 100644 index 000000000..14acbfc6d --- /dev/null +++ b/runtimes/neurun/frontend/tflite/loader.cc @@ -0,0 +1,700 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "loader.h" + +#include "model/Operations.Include.h" + +#include "cpp14/memory.h" + +namespace +{ + +using namespace tflite; +using namespace neurun; + +model::Activation convertActivation(const ActivationFunctionType type) +{ + return static_cast<model::Activation>(static_cast<uint32_t>(type)); +} + +} // namespace anonymous + +namespace tflite_loader +{ +using namespace tflite; +using namespace neurun; + +void Loader::loadFromFile(const char *file_path) +{ + std::ifstream stream(file_path, std::fstream::in | std::fstream::binary); + + stream.seekg(0, stream.end); + auto size = stream.tellg(); + stream.seekg(0, stream.beg); + + _buffer.resize(size); + stream.read(_buffer.data(), size); + + stream.close(); + loadModel(); +} + +void Loader::loadFromBuffer(std::pair<const char *, const char *> ref) +{ + _buffer.reserve(ref.second - ref.first); + std::copy(ref.first, ref.second, _buffer.begin()); + loadModel(); +} + +model::DataType tensorTypeToDataType(const TensorType &type) +{ + switch (type) + { + case TensorType::TensorType_FLOAT32: + return model::DataType::FLOAT32; + case TensorType::TensorType_INT32: + return model::DataType::INT32; + case TensorType::TensorType_BOOL: + return model::DataType::BOOL8; + case TensorType::TensorType_INT8: + return model::DataType::QUANT8_ASYMM; + default: + throw std::runtime_error("NYI"); + } +} + +void Loader::loadOperand(const tflite::Tensor *tensor) +{ + model::Shape shape; + std::unique_ptr<model::Data> data_ptr; + // Shape + const auto *tensor_shape = tensor->shape(); + for (const auto &dim : *tensor_shape) + { + shape.append(dim); + } + // Type + model::DataType data_type = tensorTypeToDataType(tensor->type()); + // Create TypeInfo + model::TypeInfo type_info(data_type); + // Create operand + const auto &operand_index = _graph.addOperand(shape, type_info); + // Buffer index + if (tensor->buffer() != 0) + _tensor_to_operand[tensor->buffer()] = operand_index; + // Name unused + // auto name = tensor->name(); + // Quantization + auto quantization = tensor->quantization(); + if (quantization != nullptr) + { + auto scale = quantization->scale(); + auto zero_point = quantization->zero_point(); + if (scale != nullptr || zero_point != nullptr) + throw std::runtime_error("Quantization is not supported!"); + + auto details = quantization->details_as_CustomQuantization(); + if (details != nullptr) + throw std::runtime_error("Custom Quantization is not supported"); + } + // Variablie + if (tensor->is_variable()) + throw std::runtime_error("Variable tensor not supported!"); +} + +void loadOperationIO(const tflite::Operator *op, model::OperandIndexSequence &inputs, + model::OperandIndexSequence &outputs) +{ + for (const auto &idx : *op->inputs()) + { + inputs.append(model::OperandIndex(idx)); + } + + for (const auto &idx : *op->outputs()) + { + outputs.append(model::OperandIndex(idx)); + } +} + +template <typename T> +neurun::model::OperandIndex Loader::createOperand(const uint8_t *ptr, + const neurun::model::Shape &shape, + const neurun::model::TypeInfo &type_info) +{ + const auto &operand_index = _graph.addOperand(shape, type_info); + if (ptr != nullptr) + { + size_t size = shape.num_elements(); + auto data = nnfw::cpp14::make_unique<model::CachedData>(ptr, size * sizeof(T)); + auto &created_operand = _graph.operands().at(operand_index); + created_operand.data(std::move(data)); + } + return operand_index; +} + +template <typename Param, typename OptionsType> +void Loader::loadStridesAndPaddings(Param ¶m, const OptionsType *options) +{ + model::Shape shape; + model::TypeInfo type_info(neurun::model::DataType::INT32); + // Strides + param.stride.vertical = options->stride_w(); + param.stride.horizontal = options->stride_h(); + // Paddings + if (options->padding() == Padding::Padding_SAME) + param.padding.type = neurun::model::PaddingType::SAME; + if (options->padding() == Padding::Padding_VALID) + param.padding.type = neurun::model::PaddingType::VALID; + // param paddings indexes unused +} + +template <typename Param> void Loader::loadPool2D(Param ¶m, const Pool2DOptions *options) +{ + // Strides and Paddings + loadStridesAndPaddings(param, options); + // Filter width and height + model::Shape shape; + model::TypeInfo type_info(neurun::model::DataType::INT32); + // Strides + param.kw = options->filter_width(); + param.kh = options->filter_height(); + // Activation + param.activation = convertActivation(options->fused_activation_function()); +} + +void Loader::loadConv2D(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + model::operation::Conv2DNode::Param param; + const auto *options = op->builtin_options_as_Conv2DOptions(); + param.activation = convertActivation(options->fused_activation_function()); + loadStridesAndPaddings(param, options); + // Dilation h/w factor unused + std::unique_ptr<model::Operation> new_op( + new model::operation::Conv2DNode(inputs, outputs, param)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadDepthwiseConv2D(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + model::operation::DepthwiseConv2DNode::Param param; + const auto *options = op->builtin_options_as_DepthwiseConv2DOptions(); + param.activation = convertActivation(options->fused_activation_function()); + loadStridesAndPaddings(param, options); + // Multiplier + model::Shape shape; + model::TypeInfo type_info(neurun::model::DataType::INT32); + param.multiplier = options->depth_multiplier(); + // Dilation h/w factor unused + std::unique_ptr<model::Operation> new_op( + new model::operation::DepthwiseConv2DNode(inputs, outputs, param)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadTransposeConv(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + model::operation::TransposeConvNode::Param param; + const auto *options = op->builtin_options_as_TransposeConvOptions(); + loadStridesAndPaddings(param, options); + std::unique_ptr<model::Operation> new_op( + new model::operation::TransposeConvNode(inputs, outputs, param)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadAvgPool2D(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + neurun::model::operation::AvgPool2DNode::Param param; + const auto *options = op->builtin_options_as_Pool2DOptions(); + + loadPool2D(param, options); + + std::unique_ptr<model::Operation> new_op( + new model::operation::AvgPool2DNode(inputs, outputs, param)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadReshape(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + // const auto *options = op->builtin_options_as_ReshapeOptions(); + // No params + + std::unique_ptr<model::Operation> new_op(new model::operation::ReshapeNode(inputs, outputs)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadSoftmax(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + model::operation::SoftmaxNode::Param param; + const auto *options = op->builtin_options_as_SoftmaxOptions(); + // Beta + param.beta = options->beta(); + + std::unique_ptr<model::Operation> new_op( + new model::operation::SoftmaxNode(inputs, outputs, param)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadMaxPool2D(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + model::operation::MaxPool2DNode::Param param; + const auto *options = op->builtin_options_as_Pool2DOptions(); + + loadPool2D(param, options); + + std::unique_ptr<model::Operation> new_op( + new model::operation::MaxPool2DNode(inputs, outputs, param)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadConcatenation(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + model::operation::ConcatNode::Param param; + const auto *options = op->builtin_options_as_ConcatenationOptions(); + // Axis + model::Shape shape; + model::TypeInfo type_info(neurun::model::DataType::INT32); + param.axis = options->axis(); + // activation unused + + std::unique_ptr<model::Operation> new_op( + new model::operation::ConcatNode(inputs, outputs, param)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadFC(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + model::operation::FullyConnectedNode::Param param; + const auto *options = op->builtin_options_as_FullyConnectedOptions(); + + param.activation = convertActivation(options->fused_activation_function()); + // weights_format unused + + std::unique_ptr<model::Operation> new_op( + new model::operation::FullyConnectedNode(inputs, outputs, param)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadAdd(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + model::operation::AddNode::Param param; + const auto *options = op->builtin_options_as_AddOptions(); + + param.activation = convertActivation(options->fused_activation_function()); + + std::unique_ptr<model::Operation> new_op(new model::operation::AddNode(inputs, outputs, param)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadSub(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + model::operation::SubNode::Param param; + const auto *options = op->builtin_options_as_SubOptions(); + + param.activation = convertActivation(options->fused_activation_function()); + + std::unique_ptr<model::Operation> new_op(new model::operation::SubNode(inputs, outputs, param)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadMul(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + model::operation::SubNode::Param param; + const auto *options = op->builtin_options_as_SubOptions(); + + param.activation = convertActivation(options->fused_activation_function()); + + std::unique_ptr<model::Operation> new_op(new model::operation::SubNode(inputs, outputs, param)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadDiv(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + model::operation::DivNode::Param param; + const auto *options = op->builtin_options_as_DivOptions(); + + param.activation = convertActivation(options->fused_activation_function()); + + std::unique_ptr<model::Operation> new_op(new model::operation::DivNode(inputs, outputs, param)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadRelu(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + std::unique_ptr<model::Operation> new_op(new model::operation::ReLUNode(inputs, outputs)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadRelu6(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + std::unique_ptr<model::Operation> new_op(new model::operation::ReLU6Node(inputs, outputs)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadRsqrt(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + std::unique_ptr<model::Operation> new_op(new model::operation::RSQRTNode(inputs, outputs)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadSqrt(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + std::unique_ptr<model::Operation> new_op(new model::operation::SQRTNode(inputs, outputs)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadSquaredDifference(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + std::unique_ptr<model::Operation> new_op( + new model::operation::SquaredDifferenceNode(inputs, outputs)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadTanh(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + std::unique_ptr<model::Operation> new_op(new model::operation::TanhNode(inputs, outputs)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadTranspose(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + const auto input_index = (*op->inputs())[0]; + inputs.append(model::OperandIndex(input_index)); + const auto output_index = (*op->outputs())[0]; + outputs.append(model::OperandIndex(output_index)); + + model::operation::TransposeNode::Param param; + if (op->inputs()->size() == 2) + { + const auto perm_index = (*op->inputs())[1]; + param.perm = model::OperandIndex(perm_index); + } + + std::unique_ptr<model::Operation> new_op( + new model::operation::TransposeNode(inputs, outputs, param)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadMean(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + const auto input_index = (*op->inputs())[0]; + inputs.append(model::OperandIndex(input_index)); + const auto output_index = (*op->outputs())[0]; + outputs.append(model::OperandIndex(output_index)); + + model::operation::MeanNode::Param param; + param.axis_index = model::OperandIndex((*op->inputs())[1]); + param.keep_dims = op->builtin_options_as_ReducerOptions()->keep_dims(); + + std::unique_ptr<model::Operation> new_op(new model::operation::MeanNode(inputs, outputs, param)); + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadCustom(const tflite::Operator *op) +{ + model::OperandIndexSequence inputs; + model::OperandIndexSequence outputs; + + for (auto in_idx : *op->inputs()) + { + inputs.append(model::OperandIndex(in_idx)); + } + + for (auto out_idx : *op->outputs()) + { + outputs.append(model::OperandIndex(out_idx)); + } + + auto custom_op_id = _opcode_index_to_custom_opcode.at(op->opcode_index()); + + auto constraint = model::operation::OperandConstraint::createExact(inputs.size()); + + assert(op->custom_options_format() == CustomOptionsFormat_FLEXBUFFERS && + "Unsupported custom operation options format"); + + size_t custom_op_data_size = op->custom_options()->size(); + auto custom_op_data = new char[custom_op_data_size]; + std::copy(op->custom_options()->begin(), op->custom_options()->end(), custom_op_data); + + model::operation::CustomNode::Userdata userdata{}; + userdata.data = custom_op_data; + userdata.size = custom_op_data_size; + + auto new_op = nnfw::cpp14::make_unique<model::operation::CustomNode>(constraint, inputs, outputs, + custom_op_id, userdata); + + _graph.addOperation(std::move(new_op)); +} + +void Loader::loadOperation(const tflite::Operator *op) +{ + switch (_op_code_to_builtin_op[op->opcode_index()]) + { + case BuiltinOperator_CONV_2D: + loadConv2D(op); + return; + case BuiltinOperator_AVERAGE_POOL_2D: + loadAvgPool2D(op); + return; + case BuiltinOperator_DEPTHWISE_CONV_2D: + loadDepthwiseConv2D(op); + return; + case BuiltinOperator_TRANSPOSE_CONV: + loadTransposeConv(op); + return; + case BuiltinOperator_RESHAPE: + loadReshape(op); + return; + case BuiltinOperator_SOFTMAX: + loadSoftmax(op); + return; + case BuiltinOperator_MAX_POOL_2D: + loadMaxPool2D(op); + return; + case BuiltinOperator_CONCATENATION: + loadConcatenation(op); + return; + case BuiltinOperator_FULLY_CONNECTED: + loadFC(op); + return; + case BuiltinOperator_ADD: + loadAdd(op); + return; + case BuiltinOperator_SUB: + loadSub(op); + return; + case BuiltinOperator_MUL: + loadMul(op); + return; + case BuiltinOperator_DIV: + loadDiv(op); + return; + case BuiltinOperator_RELU: + loadRelu(op); + return; + case BuiltinOperator_RELU6: + loadRelu6(op); + return; + case BuiltinOperator_RSQRT: + loadRsqrt(op); + return; + case BuiltinOperator_SQRT: + loadSqrt(op); + return; + case BuiltinOperator_SQUARED_DIFFERENCE: + loadSquaredDifference(op); + return; + case BuiltinOperator_TANH: + loadTanh(op); + return; + case BuiltinOperator_TRANSPOSE: + loadTranspose(op); + return; + case BuiltinOperator_MEAN: + loadMean(op); + return; + case BuiltinOperator_CUSTOM: + loadCustom(op); + return; + default: + auto *names = EnumNamesBuiltinOperator(); + int enum_value = static_cast<int>(_op_code_to_builtin_op[op->opcode_index()]); + throw std::runtime_error(std::string("Unsupported operation: ").append(names[enum_value])); + } +} + +void Loader::loadSubgraph(const SubGraph *subgraph) +{ + // Load tensors + for (const auto *tensor : *subgraph->tensors()) + { + loadOperand(tensor); + } + // Set inputs + for (const auto &input_ind : *subgraph->inputs()) + { + _graph.addInput(model::OperandIndex(input_ind)); + } + // Set outputs + for (const auto &output_ind : *subgraph->outputs()) + { + _graph.addOutput(model::OperandIndex(output_ind)); + } + // Create operations + for (const auto *op : *subgraph->operators()) + { + loadOperation(op); + } + // Name unused +} + +void Loader::loadConstantTensor(const Buffer *buffer, const uint32_t &index) +{ + const auto *data = buffer->data(); + if (data != nullptr) + { + auto ptr = nnfw::cpp14::make_unique<model::CachedData>(data->data(), data->size()); + const auto &operand_index = _tensor_to_operand[index]; + auto &operand = _graph.operands().at(operand_index); + operand.data(std::move(ptr)); + } +} + +void Loader::loadModel() +{ + flatbuffers::Verifier verifier(reinterpret_cast<const std::uint8_t *>(_buffer.data()), + _buffer.size()); + if (!tflite::VerifyModelBuffer(verifier)) + throw std::runtime_error{"Invalid tflite model"}; + + const auto *model = GetModel(_buffer.data()); + // Version unused + // const auto version = model->version(); + const auto *op_codes = model->operator_codes(); + const auto *subgraphs = model->subgraphs(); + // Description unused + // const auto *description = model->description(); + const auto *buffers = model->buffers(); + // Metabuffer unsued + // const auto *metadata_buffer = model->metadata_buffer(); + // Use operator codes + for (const auto *op_code : *op_codes) + { + _op_code_to_builtin_op.push_back(op_code->builtin_code()); + + if (op_code->builtin_code() == BuiltinOperator_CUSTOM) + { + auto id = op_code->custom_code()->str(); + _opcode_index_to_custom_opcode[_op_code_to_builtin_op.size() - 1] = id; + } + // Custom code unsued + // Version unused + } + // Load subgraphs + for (const auto *subgraph : *subgraphs) + { + loadSubgraph(subgraph); + } + // Load buffers with constant tensors + for (uint32_t ind = 0; ind < buffers->size(); ind++) + { + loadConstantTensor(buffers->Get(ind), ind); + } + + _graph.finishBuilding(); +} + +} // namespace tflite_loader |