path: root/mv_inference/inference/src/mv_inference_open.cpp

/**
 * Copyright (c) 2018 Samsung Electronics Co., Ltd All Rights Reserved
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "mv_private.h"
#include "mv_inference_open.h"

#include "Inference.h"

#include <new>
#include <unistd.h>
#include <string>

using namespace mediavision::inference;

mv_engine_config_h mv_inference_get_engine_config(mv_inference_h infer)
{
	Inference *pInfer = static_cast<Inference *>(infer);
	return pInfer->GetEngineConfig();
}

int mv_inference_create_open(mv_inference_h *infer)
{
	if (infer == NULL ) {
		LOGE("Handle can't be created because handle pointer is NULL");
		return MEDIA_VISION_ERROR_INVALID_PARAMETER;
	}

	(*infer) = static_cast<mv_inference_h>(new (std::nothrow)Inference());

	if (*infer == NULL) {
		LOGE("Failed to create inference handle");
		return MEDIA_VISION_ERROR_OUT_OF_MEMORY;
	}

	LOGD("Inference handle [%p] has been created", *infer);

	return MEDIA_VISION_ERROR_NONE;
}

int mv_inference_destroy_open(mv_inference_h infer)
{
	if (!infer) {
		LOGE("Handle can't be destroyed because handle is NULL");
		return MEDIA_VISION_ERROR_INVALID_PARAMETER;
	}

	LOGD("Destroying inference handle [%p]", infer);
	delete static_cast<Inference*>(infer);
	LOGD("Inference handle has been destroyed");

	return MEDIA_VISION_ERROR_NONE;
}


int mv_inference_configure_model_open(mv_inference_h infer, mv_engine_config_h engine_config)
{
	LOGI("ENTER");

    Inference *pInfer = static_cast<Inference *>(infer);

    int ret = MEDIA_VISION_ERROR_NONE;

	char *modelConfigFilePath = NULL;
	char *modelWeightFilePath = NULL;
	char *modelUserFilePath = NULL;
	double modelMeanValue = 0.0;
	int backendType= 0;
	int targetTypes = 0;
	size_t userFileLength = 0;

    ret = mv_engine_config_get_string_attribute(engine_config,
											MV_INFERENCE_MODEL_CONFIGURATION_FILE_PATH,
											&modelConfigFilePath);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get model configuration file path");
		goto _ERROR_;
	}

	ret = mv_engine_config_get_string_attribute(engine_config,
											MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
											&modelWeightFilePath);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get model weight file path");
		goto _ERROR_;
	}

	ret = mv_engine_config_get_string_attribute(engine_config,
											MV_INFERENCE_MODEL_USER_FILE_PATH,
											&modelUserFilePath);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get model user file path");
		goto _ERROR_;
	}

	ret = mv_engine_config_get_double_attribute(engine_config,
												MV_INFERENCE_MODEL_MEAN_VALUE,
												&modelMeanValue);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get model mean value");
		goto _ERROR_;
	}

	ret = mv_engine_config_get_int_attribute(engine_config,
												MV_INFERENCE_BACKEND_TYPE,
												&backendType);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get inference backend type");
		goto _ERROR_;
	}

	ret = mv_engine_config_get_int_attribute(engine_config,
											MV_INFERENCE_TARGET_TYPE,
											&targetTypes);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get inference target type");
		goto _ERROR_;
	}

	if ( access(modelWeightFilePath, F_OK)) {
		LOGE("weightFilePath in [%s] ", modelWeightFilePath);
		ret = MEDIA_VISION_ERROR_INVALID_PATH;
		goto _ERROR_;
	}

	if ( (backendType > MV_INFERENCE_BACKEND_NONE  && backendType < MV_INFERENCE_BACKEND_MAX)
		&& (backendType != MV_INFERENCE_BACKEND_TFLITE) && (backendType != MV_INFERENCE_BACKEND_ARMNN)) {
		if ( access(modelConfigFilePath, F_OK)) {
			LOGE("modelConfigFilePath in [%s] ", modelConfigFilePath);
			ret = MEDIA_VISION_ERROR_INVALID_PATH;
			goto _ERROR_;
		}
	}

	userFileLength = strlen(modelUserFilePath);
	if (userFileLength > 0 && access(modelUserFilePath, F_OK)) {
		LOGE("categoryFilePath in [%s] ", modelUserFilePath);
		ret = MEDIA_VISION_ERROR_INVALID_PATH;
		goto _ERROR_;
	}

	ret = pInfer->ConfigureTargetTypes(targetTypes);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Tried to configure invalid target types.");
		goto _ERROR_;
	}

	pInfer->ConfigureModelFiles(std::string(modelConfigFilePath),
					std::string(modelWeightFilePath),
					std::string(modelUserFilePath));

_ERROR_:
	if (modelConfigFilePath)
		free(modelConfigFilePath);

	if (modelWeightFilePath)
		free(modelWeightFilePath);

	if (modelUserFilePath)
		free(modelUserFilePath);

	LOGI("LEAVE");

    return ret;
}

int mv_inference_configure_tensor_info_open(mv_inference_h infer, mv_engine_config_h engine_config)
{
    Inference *pInfer = static_cast<Inference *>(infer);

    int ret = MEDIA_VISION_ERROR_NONE;

	int tensorWidth, tensorHeight, tensorDim, tensorCh;
	double meanValue, stdValue;

	// This should be one. only one batch is supported
	tensorDim = 1;
    ret = mv_engine_config_get_int_attribute(engine_config,
											MV_INFERENCE_INPUT_TENSOR_WIDTH,
											&tensorWidth);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get tensor width");
		goto _ERROR_;
	}

    ret = mv_engine_config_get_int_attribute(engine_config,
											MV_INFERENCE_INPUT_TENSOR_HEIGHT,
											&tensorHeight);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get tensor height");
		goto _ERROR_;
	}

    ret = mv_engine_config_get_int_attribute(engine_config,
											MV_INFERENCE_INPUT_TENSOR_CHANNELS,
											&tensorCh);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get tensor channels");
		goto _ERROR_;
	}

	ret = mv_engine_config_get_double_attribute(engine_config,
											MV_INFERENCE_MODEL_MEAN_VALUE,
											&meanValue);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get meanValue");
		goto _ERROR_;
	}

	ret = mv_engine_config_get_double_attribute(engine_config,
											MV_INFERENCE_MODEL_STD_VALUE,
											&stdValue);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get stdValue");
		goto _ERROR_;
	}

    pInfer->ConfigureTensorInfo(tensorWidth, tensorHeight, tensorDim, tensorCh, stdValue, meanValue);

_ERROR_ :

    return ret;
}

int mv_inference_configure_engine_open(mv_inference_h infer, mv_engine_config_h engine_config)
{
    Inference *pInfer = static_cast<Inference *>(infer);
	int backendType = 0;
    int ret = MEDIA_VISION_ERROR_NONE;

	pInfer->SetEngineConfig(engine_config);

	ret = mv_engine_config_get_int_attribute(engine_config,
											MV_INFERENCE_BACKEND_TYPE,
											&backendType);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get inference backend type");
		goto _ERROR_;
	}

	ret = pInfer->ConfigureBackendType((mv_inference_backend_type_e)backendType);
    if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to configure a backend type.");
		goto _ERROR_;
    }

	// Create a inference-engine-common class object and load its corresponding library.
	ret = pInfer->Bind();
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to bind a backend engine.");
	}

_ERROR_:
	return ret;
}

int mv_inference_configure_output_open(mv_inference_h infer, mv_engine_config_h engine_config)
{
	Inference *pInfer = static_cast<Inference *>(infer);

	int maxOutput = 0;
	int ret = MEDIA_VISION_ERROR_NONE;

	ret = mv_engine_config_get_int_attribute(engine_config,
											MV_INFERENCE_OUTPUT_MAX_NUMBER,
											&maxOutput);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get inference output maximum numbers");
		goto _ERROR_;
	}

	pInfer->ConfigureOutput(maxOutput);

_ERROR_:
	return ret;
}

int mv_inference_configure_confidence_threshold_open(mv_inference_h infer, mv_engine_config_h engine_config)
{
	Inference *pInfer = static_cast<Inference *>(infer);

	double threshold = 0;
	int ret = MEDIA_VISION_ERROR_NONE;

	ret = mv_engine_config_get_double_attribute(engine_config,
											  MV_INFERENCE_CONFIDENCE_THRESHOLD,
											  &threshold);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get inference confidence threshold value");
		goto _ERROR_;
	}

	pInfer->ConfigureThreshold(threshold);

_ERROR_:
	return ret;
}

int mv_inference_configure_input_node_name_open(mv_inference_h infer, mv_engine_config_h engine_config)
{
    Inference *pInfer = static_cast<Inference *>(infer);

    int ret = MEDIA_VISION_ERROR_NONE;

	char *node_name = NULL;

    ret = mv_engine_config_get_string_attribute(engine_config,
											  MV_INFERENCE_INPUT_NODE_NAME,
											  &node_name);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get tensor width");
		goto _ERROR_;
	}

	pInfer->ConfigureInputNodeName(std::string(node_name));

_ERROR_:

	if (node_name) {
		free(node_name);
		node_name = NULL;
	}

	return ret;
}

int mv_inference_configure_output_node_names_open(mv_inference_h infer, mv_engine_config_h engine_config)
{
    Inference *pInfer = static_cast<Inference *>(infer);

    int ret = MEDIA_VISION_ERROR_NONE;
	int idx = 0;
	char **node_names = NULL;
	int size = 0;
	std::vector<std::string> names;
    ret = mv_engine_config_get_array_string_attribute(engine_config,
											  MV_INFERENCE_OUTPUT_NODE_NAMES,
											  &node_names,
											  &size);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get _output_node_names");
		goto _ERROR_;
	}

	for (idx = 0 ; idx < size; ++idx)
		names.push_back(std::string(node_names[idx]));

	pInfer->ConfigureOutputNodeNames(names);

_ERROR_:

	if (node_names) {
		for (idx = 0; idx < size; ++idx) {
			free(node_names[idx]);
		}
		free(node_names);
		node_names = NULL;
	}

	return ret;
}

int mv_inference_prepare_open(mv_inference_h infer)
{
	Inference *pInfer = static_cast<Inference *>(infer);

	int ret = MEDIA_VISION_ERROR_NONE;

    // Pass parameters needed to load model files to a backend engine.
	ret = pInfer->Prepare();
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to prepare inference");
        return ret;
    }

    // Request to load model files to a backend engine.
    ret = pInfer->Load();
    if (ret != MEDIA_VISION_ERROR_NONE)
        LOGE("Fail to load model files.");

	return ret;
}

int mv_inference_foreach_supported_engine_open(
	mv_inference_h infer,
	mv_inference_supported_engine_cb callback,
	void *user_data)
{
	Inference *pInfer = static_cast<Inference *>(infer);

	int ret = MEDIA_VISION_ERROR_NONE;

	//bool isSupported = false;
	//char str[1024] = {'\0'};
	std::pair<std::string, bool> backend;
	for (int i = 0; i < MV_INFERENCE_BACKEND_MAX; ++i) {
		backend = pInfer->GetSupportedInferenceBackend(i);
		callback((backend.first).c_str(), backend.second, user_data);
	}

	return ret;
}

int mv_inference_image_classify_open(
		mv_source_h source,
		mv_inference_h infer,
		mv_rectangle_s *roi,
		mv_inference_image_classified_cb classified_cb,
		void *user_data)
{
	Inference *pInfer = static_cast<Inference *>(infer);

	int ret = MEDIA_VISION_ERROR_NONE;
	int numberOfOutputs = 0;
	std::vector<mv_source_h> sources;
	std::vector<mv_rectangle_s> rects;

	sources.push_back(source);

	if (roi != NULL)
		rects.push_back(*roi);

	ret = pInfer->Run(sources, rects);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to run inference");
		return ret;
	}

	ImageClassificationResults classificationResults;

	ret = pInfer->GetClassficationResults(&classificationResults);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get inference results");
		return ret;
	}

	numberOfOutputs = classificationResults.number_of_classes;


	int *indices = classificationResults.indices.data();
	float *confidences = classificationResults.confidences.data();
	static const int START_CLASS_NUMBER = 10;
	static std::vector<const char*> names(START_CLASS_NUMBER);

	if (numberOfOutputs > START_CLASS_NUMBER)
		names.resize(numberOfOutputs);

	LOGE("mv_inference_open: number_of_classes: %d\n", classificationResults.number_of_classes);

	for (int n = 0; n < numberOfOutputs; ++n) {
		LOGE("names: %s", classificationResults.names[n].c_str());
		names[n] = classificationResults.names[n].c_str();
	}

	classified_cb(source, numberOfOutputs, indices, names.data(), confidences, user_data);

	return ret;
}


int mv_inference_object_detect_open(
	mv_source_h source,
	mv_inference_h infer,
	mv_inference_object_detected_cb detected_cb,
	void *user_data)
{
	Inference *pInfer = static_cast<Inference *>(infer);

	int ret = MEDIA_VISION_ERROR_NONE;
	int numberOfOutputs = 0;
	std::vector<mv_source_h> sources;
	std::vector<mv_rectangle_s> rects;

	sources.push_back(source);

	ret = pInfer->Run(sources, rects);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to run inference");
		return ret;
	}

	ObjectDetectionResults objectDetectionResults;
	ret = pInfer->GetObjectDetectionResults(&objectDetectionResults);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get inference results");
		return ret;
	}

	numberOfOutputs = objectDetectionResults.number_of_objects;

	int *indices = objectDetectionResults.indices.data();
	float *confidences = objectDetectionResults.confidences.data();
	static const int START_OBJECT_NUMBER = 20;
	static std::vector<const char*> names(START_OBJECT_NUMBER);
	static std::vector<mv_rectangle_s> locations(START_OBJECT_NUMBER);

	if (numberOfOutputs > START_OBJECT_NUMBER) {
		names.resize(numberOfOutputs);
		locations.resize(numberOfOutputs);
	}

	for (int n = 0; n < numberOfOutputs; ++n) {
		LOGE("names: %s", objectDetectionResults.names[n].c_str());
		names[n] = objectDetectionResults.names[n].c_str();

		locations[n].point.x = objectDetectionResults.locations[n].x;
		locations[n].point.y = objectDetectionResults.locations[n].y;
		locations[n].width = objectDetectionResults.locations[n].width;
		locations[n].height = objectDetectionResults.locations[n].height;
	}

	detected_cb(source, numberOfOutputs, indices, names.data(), confidences, locations.data(), user_data);

	return ret;
}

int mv_inference_face_detect_open(
	mv_source_h source,
	mv_inference_h infer,
	mv_inference_face_detected_cb detected_cb,
	void *user_data)
{
	Inference *pInfer = static_cast<Inference *>(infer);

	int ret = MEDIA_VISION_ERROR_NONE;
	int numberOfOutputs = 0;
	std::vector<mv_source_h> sources;
	std::vector<mv_rectangle_s> rects;

	sources.push_back(source);

	ret = pInfer->Run(sources, rects);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to run inference");
		return ret;
	}

	FaceDetectionResults faceDetectionResults;
	ret = pInfer->GetFaceDetectionResults(&faceDetectionResults);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get inference results");
		return ret;
	}

	numberOfOutputs = faceDetectionResults.number_of_faces;

	float *confidences = faceDetectionResults.confidences.data();
	std::vector<mv_rectangle_s> locations(numberOfOutputs);

	for (int n = 0; n < numberOfOutputs; ++n) {
		locations[n].point.x = faceDetectionResults.locations[n].x;
		locations[n].point.y = faceDetectionResults.locations[n].y;
		locations[n].width = faceDetectionResults.locations[n].width;
		locations[n].height = faceDetectionResults.locations[n].height;
	}

	detected_cb(source, numberOfOutputs, confidences, locations.data(), user_data);

	return ret;
}

int mv_inference_facial_landmark_detect_open(
	mv_source_h source,
	mv_inference_h infer,
	mv_rectangle_s *roi,
	mv_inference_facial_landmark_detected_cb detected_cb,
	void *user_data)
{
	Inference *pInfer = static_cast<Inference *>(infer);

	int ret = MEDIA_VISION_ERROR_NONE;
	int numberOfLandmarks = 0;
	std::vector<mv_source_h> sources;
	std::vector<mv_rectangle_s> rects;

	sources.push_back(source);

	if (roi != NULL)
		rects.push_back(*roi);

	ret = pInfer->Run(sources, rects);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to run inference");
		return ret;
	}

	FacialLandMarkDetectionResults facialLandMarkDetectionResults;
	ret = pInfer->GetFacialLandMarkDetectionResults(&facialLandMarkDetectionResults);
	if (ret != MEDIA_VISION_ERROR_NONE) {
		LOGE("Fail to get inference results");
		return ret;
	}

	numberOfLandmarks = facialLandMarkDetectionResults.number_of_landmarks;

	std::vector<mv_point_s> locations(numberOfLandmarks);

	for (int n = 0; n < numberOfLandmarks; ++n) {

		locations[n].x = facialLandMarkDetectionResults.locations[n].x;
		locations[n].y = facialLandMarkDetectionResults.locations[n].y;
	}

	detected_cb(source, numberOfLandmarks, locations.data(), user_data);

	return ret;
}