path: root/mv_machine_learning/object_detection/src/object_detection.cpp

/**
 * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <string.h>
#include <fstream>
#include <map>
#include <memory>
#include <algorithm>

#include "machine_learning_exception.h"
#include "mv_machine_learning_common.h"
#include "mv_object_detection_config.h"
#include "object_detection.h"

using namespace std;
using namespace mediavision::inference;
using namespace MediaVision::Common;
using namespace mediavision::common;
using namespace mediavision::machine_learning::exception;

namespace mediavision
{
namespace machine_learning
{
ObjectDetection::ObjectDetection(ObjectDetectionTaskType task_type)
		: _task_type(task_type), _backendType(), _targetDeviceType()
{
	_inference = make_unique<Inference>();
	_parser = make_unique<ObjectDetectionParser>();
}

void ObjectDetection::preDestroy()
{
	if (_thread_handle) {
		_exit_thread = true;
		_thread_handle->join();
	}
}

bool ObjectDetection::exitThread()
{
	return _exit_thread;
}

ObjectDetectionTaskType ObjectDetection::getTaskType()
{
	return _task_type;
}

void ObjectDetection::getEngineList()
{
	for (auto idx = MV_INFERENCE_BACKEND_NONE + 1; idx < MV_INFERENCE_BACKEND_MAX; ++idx) {
		auto backend = _inference->getSupportedInferenceBackend(idx);
		// TODO. we need to describe what inference engines are supported by each Task API,
		//       and based on it, below inference engine types should be checked
		//       if a given type is supported by this Task API later. As of now, tflite only.
		if (backend.second == true && backend.first.compare("tflite") == 0)
			_valid_backends.push_back(backend.first);
	}
}

void ObjectDetection::getDeviceList(const char *engine_type)
{
	// TODO. add device types available for a given engine type later.
	//       In default, cpu and gpu only.
	_valid_devices.push_back("cpu");
	_valid_devices.push_back("gpu");
}

void ObjectDetection::setEngineInfo(std::string engine_type, std::string device_type)
{
	if (engine_type.empty() || device_type.empty())
		throw InvalidParameter("Invalid engine info.");

	transform(engine_type.begin(), engine_type.end(), engine_type.begin(), ::toupper);
	transform(device_type.begin(), device_type.end(), device_type.begin(), ::toupper);

	_backendType = GetBackendType(engine_type);
	_targetDeviceType = GetDeviceType(device_type);

	LOGI("Engine type : %s => %d, Device type : %s => %d", engine_type.c_str(), GetBackendType(engine_type),
		 device_type.c_str(), GetDeviceType(device_type));

	if (_backendType == MEDIA_VISION_ERROR_INVALID_PARAMETER ||
		_targetDeviceType == MEDIA_VISION_ERROR_INVALID_PARAMETER)
		throw InvalidParameter("backend or target device type not found.");
}

void ObjectDetection::getNumberOfEngines(unsigned int *number_of_engines)
{
	if (!_valid_backends.empty()) {
		*number_of_engines = _valid_backends.size();
		return;
	}

	getEngineList();
	*number_of_engines = _valid_backends.size();
}

void ObjectDetection::getEngineType(unsigned int engine_index, char **engine_type)
{
	if (!_valid_backends.empty()) {
		if (_valid_backends.size() <= engine_index)
			throw InvalidParameter("Invalid engine index.");

		*engine_type = const_cast<char *>(_valid_backends[engine_index].data());
		return;
	}

	getEngineList();

	if (_valid_backends.size() <= engine_index)
		throw InvalidParameter("Invalid engine index.");

	*engine_type = const_cast<char *>(_valid_backends[engine_index].data());
}

void ObjectDetection::getNumberOfDevices(const char *engine_type, unsigned int *number_of_devices)
{
	if (!_valid_devices.empty()) {
		*number_of_devices = _valid_devices.size();
		return;
	}

	getDeviceList(engine_type);
	*number_of_devices = _valid_devices.size();
}

void ObjectDetection::getDeviceType(const char *engine_type, const unsigned int device_index, char **device_type)
{
	if (!_valid_devices.empty()) {
		if (_valid_devices.size() <= device_index)
			throw InvalidParameter("Invalid device index.");

		*device_type = const_cast<char *>(_valid_devices[device_index].data());
		return;
	}

	getDeviceList(engine_type);

	if (_valid_devices.size() <= device_index)
		throw InvalidParameter("Invalid device index.");

	*device_type = const_cast<char *>(_valid_devices[device_index].data());
}

void ObjectDetection::setUserModel(string model_file, string meta_file, string label_file)
{
	_modelFilePath = model_file;
	_modelMetaFilePath = meta_file;
	_modelLabelFilePath = label_file;
}

static bool IsJsonFile(const string &fileName)
{
	return (!fileName.substr(fileName.find_last_of(".") + 1).compare("json"));
}

void ObjectDetection::loadLabel()
{
	ifstream readFile;

	_labels.clear();
	readFile.open(_modelLabelFilePath.c_str());

	if (readFile.fail())
		throw InvalidOperation("Fail to open " + _modelLabelFilePath + " file.");

	string line;

	while (getline(readFile, line))
		_labels.push_back(line);

	readFile.close();
}

void ObjectDetection::parseMetaFile(string meta_file_name)
{
	_config = make_unique<EngineConfig>(string(MV_CONFIG_PATH) + meta_file_name);

	int ret = _config->getIntegerAttribute(string(MV_OBJECT_DETECTION_BACKEND_TYPE), &_backendType);
	if (ret != MEDIA_VISION_ERROR_NONE)
		throw InvalidOperation("Fail to get backend engine type.");

	ret = _config->getIntegerAttribute(string(MV_OBJECT_DETECTION_TARGET_DEVICE_TYPE), &_targetDeviceType);
	if (ret != MEDIA_VISION_ERROR_NONE)
		throw InvalidOperation("Fail to get target device type.");

	ret = _config->getStringAttribute(MV_OBJECT_DETECTION_MODEL_DEFAULT_PATH, &_modelDefaultPath);
	if (ret != MEDIA_VISION_ERROR_NONE)
		throw InvalidOperation("Fail to get model default path");

	if (_modelFilePath.empty()) {
		ret = _config->getStringAttribute(MV_OBJECT_DETECTION_MODEL_FILE_PATH, &_modelFilePath);
		if (ret != MEDIA_VISION_ERROR_NONE)
			throw InvalidOperation("Fail to get model file path");
	}

	_modelFilePath = _modelDefaultPath + _modelFilePath;
	LOGI("model file path = %s", _modelFilePath.c_str());

	if (_modelMetaFilePath.empty()) {
		ret = _config->getStringAttribute(MV_OBJECT_DETECTION_MODEL_META_FILE_PATH, &_modelMetaFilePath);
		if (ret != MEDIA_VISION_ERROR_NONE)
			throw InvalidOperation("Fail to get model meta file path");

		if (_modelMetaFilePath.empty())
			throw InvalidOperation("Model meta file doesn't exist.");

		if (!IsJsonFile(_modelMetaFilePath))
			throw InvalidOperation("Model meta file should be json");
	}

	_modelMetaFilePath = _modelDefaultPath + _modelMetaFilePath;
	LOGI("meta file path = %s", _modelMetaFilePath.c_str());

	_parser->setTaskType(static_cast<int>(_task_type));
	_parser->load(_modelMetaFilePath);

	if (_modelLabelFilePath.empty()) {
		ret = _config->getStringAttribute(MV_OBJECT_DETECTION_LABEL_FILE_NAME, &_modelLabelFilePath);
		if (ret != MEDIA_VISION_ERROR_NONE)
			throw InvalidOperation("Fail to get label file path");

		if (_modelLabelFilePath.empty())
			throw InvalidOperation("Model label file doesn't exist.");
	}

	_modelLabelFilePath = _modelDefaultPath + _modelLabelFilePath;
	LOGI("label file path = %s", _modelLabelFilePath.c_str());

	loadLabel();
}

void ObjectDetection::configure(string configFile)
{
	parseMetaFile(configFile);

	int ret = _inference->bind(_backendType, _targetDeviceType);
	if (ret != MEDIA_VISION_ERROR_NONE)
		throw InvalidOperation("Fail to bind a backend engine.");
}

void ObjectDetection::prepare()
{
	int ret = _inference->configureInputMetaInfo(_parser->getInputMetaMap());
	if (ret != MEDIA_VISION_ERROR_NONE)
		throw InvalidOperation("Fail to configure input tensor info from meta file.");

	ret = _inference->configureOutputMetaInfo(_parser->getOutputMetaMap());
	if (ret != MEDIA_VISION_ERROR_NONE)
		throw InvalidOperation("Fail to configure output tensor info from meta file.");

	_inference->configureModelFiles("", _modelFilePath, "");

	// Request to load model files to a backend engine.
	ret = _inference->load();
	if (ret != MEDIA_VISION_ERROR_NONE)
		throw InvalidOperation("Fail to load model files.");
}

shared_ptr<MetaInfo> ObjectDetection::getInputMetaInfo()
{
	TensorBuffer &tensor_buffer = _inference->getInputTensorBuffer();
	IETensorBuffer &tensor_info_map = tensor_buffer.getIETensorBuffer();

	// TODO. consider using multiple tensors later.
	if (tensor_info_map.size() != 1)
		throw InvalidOperation("Input tensor count not invalid.");

	auto tensor_buffer_iter = tensor_info_map.begin();

	// Get the meta information corresponding to a given input tensor name.
	return _parser->getInputMetaMap()[tensor_buffer_iter->first];
}

template<typename T>
void ObjectDetection::preprocess(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo, vector<T> &inputVector)
{
	LOGI("ENTER");

	PreprocessConfig config = { false,
								metaInfo->colorSpace,
								metaInfo->dataType,
								metaInfo->getChannel(),
								metaInfo->getWidth(),
								metaInfo->getHeight() };

	auto normalization = static_pointer_cast<DecodingNormal>(metaInfo->decodingTypeMap.at(DecodingType::NORMAL));
	if (normalization) {
		config.normalize = normalization->use;
		config.mean = normalization->mean;
		config.std = normalization->std;
	}

	auto quantization =
			static_pointer_cast<DecodingQuantization>(metaInfo->decodingTypeMap.at(DecodingType::QUANTIZATION));
	if (quantization) {
		config.quantize = quantization->use;
		config.scale = quantization->scale;
		config.zeropoint = quantization->zeropoint;
	}

	_preprocess.setConfig(config);
	_preprocess.run<T>(mv_src, inputVector);

	LOGI("LEAVE");
}

template<typename T> void ObjectDetection::inference(vector<vector<T> > &inputVectors)
{
	LOGI("ENTER");

	int ret = _inference->run<T>(inputVectors);
	if (ret != MEDIA_VISION_ERROR_NONE)
		throw InvalidOperation("Fail to run inference");

	LOGI("LEAVE");
}

template<typename T> void ObjectDetection::perform(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo)
{
	vector<T> inputVector;

	preprocess<T>(mv_src, metaInfo, inputVector);

	vector<vector<T> > inputVectors = { inputVector };

	inference<T>(inputVectors);
}

void ObjectDetection::perform(mv_source_h &mv_src)
{
	shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();
	if (metaInfo->dataType == MV_INFERENCE_DATA_UINT8)
		perform<unsigned char>(mv_src, metaInfo);
	else if (metaInfo->dataType == MV_INFERENCE_DATA_FLOAT32)
		perform<float>(mv_src, metaInfo);
	else
		throw InvalidOperation("Invalid model data type.");
}

template<typename T> void inferenceThreadLoop(ObjectDetection *object)
{
	// If user called destroy API then this thread loop will be terminated.
	while (!object->exitThread() || !object->isInputQueueEmpty<T>()) {
		// If input queue is empty then skip inference request.
		if (object->isInputQueueEmpty<T>())
			continue;

		ObjectDetectionQueue<T> input = object->popFromInput<T>();

		LOGD("Popped : input index = %lu", input.index);

		object->inference<T>(input.inputs);

		ObjectDetectionResult &result = object->result();

		object->pushToOutput(result);

		input.completion_cb(input.handle, input.user_data);
	}
}

template<typename T> void ObjectDetection::performAsync(ObjectDetectionInput &input, shared_ptr<MetaInfo> metaInfo)
{
	_input_index++;

	if (!isInputQueueEmpty<T>())
		return;

	vector<T> inputVector;

	preprocess<T>(input.inference_src, metaInfo, inputVector);

	vector<vector<T> > inputVectors = { inputVector };
	ObjectDetectionQueue<T> in_queue = { _input_index,		  input.handle, input.inference_src,
										 input.completion_cb, inputVectors, input.user_data };

	pushToInput<T>(in_queue);
	LOGD("Pushed : input index = %lu", in_queue.index);

	if (!_thread_handle)
		_thread_handle = make_unique<thread>(&inferenceThreadLoop<T>, this);
}

void ObjectDetection::performAsync(ObjectDetectionInput &input)
{
	shared_ptr<MetaInfo> metaInfo = getInputMetaInfo();

	if (metaInfo->dataType == MV_INFERENCE_DATA_UINT8) {
		performAsync<unsigned char>(input, metaInfo);
	} else if (metaInfo->dataType == MV_INFERENCE_DATA_FLOAT32) {
		performAsync<float>(input, metaInfo);
		// TODO
	} else {
		throw InvalidOperation("Invalid model data type.");
	}
}

ObjectDetectionResult &ObjectDetection::getOutput()
{
	if (_thread_handle) {
		if (isOutputQueueEmpty())
			throw InvalidOperation("No inference result.");

		_current_result = popFromOutput();
	} else {
		_current_result = result();
	}

	return _current_result;
}

ObjectDetectionResult &ObjectDetection::getOutputCache()
{
	return _current_result;
}

void ObjectDetection::getOutputNames(vector<string> &names)
{
	TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();
	IETensorBuffer &ie_tensor_buffer = tensor_buffer_obj.getIETensorBuffer();

	for (IETensorBuffer::iterator it = ie_tensor_buffer.begin(); it != ie_tensor_buffer.end(); it++)
		names.push_back(it->first);
}

void ObjectDetection::getOutputTensor(string target_name, vector<float> &tensor)
{
	TensorBuffer &tensor_buffer_obj = _inference->getOutputTensorBuffer();

	inference_engine_tensor_buffer *tensor_buffer = tensor_buffer_obj.getTensorBuffer(target_name);
	if (!tensor_buffer)
		throw InvalidOperation("Fail to get tensor buffer.");

	auto raw_buffer = static_cast<float *>(tensor_buffer->buffer);

	copy(&raw_buffer[0], &raw_buffer[tensor_buffer->size / sizeof(float)], back_inserter(tensor));
}

template<typename T> void ObjectDetection::pushToInput(ObjectDetectionQueue<T> &input)
{
	lock_guard<mutex> lock(_incoming_queue_mutex);
	_incoming_queue<T>.push(input);
}

template<typename T> ObjectDetectionQueue<T> ObjectDetection::popFromInput()
{
	lock_guard<mutex> lock(_incoming_queue_mutex);
	ObjectDetectionQueue<T> input = _incoming_queue<T>.front();
	_incoming_queue<T>.pop();

	return input;
}

template<typename T> bool ObjectDetection::isInputQueueEmpty()
{
	lock_guard<mutex> lock(_incoming_queue_mutex);
	return _incoming_queue<T>.empty();
}

void ObjectDetection::pushToOutput(ObjectDetectionResult &output)
{
	lock_guard<mutex> lock(_outgoing_queue_mutex);
	_outgoing_queue.push(output);
}

ObjectDetectionResult ObjectDetection::popFromOutput()
{
	lock_guard<mutex> lock(_outgoing_queue_mutex);
	ObjectDetectionResult output = _outgoing_queue.front();

	_outgoing_queue.pop();

	return output;
}

bool ObjectDetection::isOutputQueueEmpty()
{
	lock_guard<mutex> lock(_outgoing_queue_mutex);
	return _outgoing_queue.empty();
}

template<typename T> queue<ObjectDetectionQueue<T> > ObjectDetection::_incoming_queue;

template void ObjectDetection::preprocess<float>(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo,
												 vector<float> &inputVector);
template void ObjectDetection::inference<float>(vector<vector<float> > &inputVectors);
template void ObjectDetection::perform<float>(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo);
template void ObjectDetection::pushToInput<float>(ObjectDetectionQueue<float> &input);
template ObjectDetectionQueue<float> ObjectDetection::popFromInput();
template bool ObjectDetection::isInputQueueEmpty<float>();
template void ObjectDetection::performAsync<float>(ObjectDetectionInput &input, shared_ptr<MetaInfo> metaInfo);

template void ObjectDetection::preprocess<unsigned char>(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo,
														 vector<unsigned char> &inputVector);
template void ObjectDetection::inference<unsigned char>(vector<vector<unsigned char> > &inputVectors);
template void ObjectDetection::perform<unsigned char>(mv_source_h &mv_src, shared_ptr<MetaInfo> metaInfo);
template void ObjectDetection::pushToInput<unsigned char>(ObjectDetectionQueue<unsigned char> &input);
template ObjectDetectionQueue<unsigned char> ObjectDetection::popFromInput();
template bool ObjectDetection::isInputQueueEmpty<unsigned char>();

template void ObjectDetection::performAsync<unsigned char>(ObjectDetectionInput &input, shared_ptr<MetaInfo> metaInfo);

}
}