path: root/mv_inference/inference/include/Inference.h

/**
 * Copyright (c) 2019 Samsung Electronics Co., Ltd All Rights Reserved
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef __MEDIA_VISION_INFERENCE_H__
#define __MEDIA_VISION_INFERENCE_H__

#include <string>
#include <map>

#include "mv_common.h"
#include "inference_engine_error.h"
#include "inference_engine_common_impl.h"
#include <mv_inference_type.h>
#include <opencv2/core.hpp>
#include <opencv2/imgproc.hpp>

/**
 * @file Inference.h
 * @brief This file contains the inference class definition which
 *        provides inference interface.
 */
using namespace InferenceEngineInterface::Common;

typedef struct _ImageClassficationResults {
    int number_of_classes;
    std::vector<int> indices;
    std::vector<std::string> names;
    std::vector<float> confidences;
} ImageClassificationResults; /**< structure ImageClassificationResults */

typedef struct _ObjectDetectionResults {
    int number_of_objects;
    std::vector<int> indices;
    std::vector<std::string> names;
    std::vector<float> confidences;
    std::vector<cv::Rect> locations;
} ObjectDetectionResults;  /**< structure ObjectDetectionResults */

typedef struct _FaceDetectionResults {
    int number_of_faces;
    std::vector<float> confidences;
    std::vector<cv::Rect> locations;
} FaceDetectionResults;  /**< structure FaceDetectionResults */

typedef struct _FacialLandMarkDetectionResults {
    int number_of_landmarks;
    std::vector<cv::Point> locations;
} FacialLandMarkDetectionResults;  /**< structure FacialLandMarkDetectionResults */

typedef struct _PoseEstimationResults {
    int number_of_pose_estimation;
    std::vector<cv::Point> locations;
} PoseEstimationResults;  /**< structure PoseEstimationResults */

namespace mediavision {
namespace inference {

struct TensorInfo {
    int width;
    int height;
    int dim;
    int ch;
};

struct InferenceConfig {
	/**
	 * @brief Default constructor for the @ref InferenceConfig
	 *
	 * @since_tizen 5.0
	 */
	InferenceConfig();

	std::string mConfigFilePath; /**< Path of a model configuration file */

	std::string mWeightFilePath; /**< Path of a model weight file */

	std::string mUserFilePath; /**< Path of model user file */

	TensorInfo mTensorInfo; /**< Tensor information */

	mv_inference_backend_type_e mBackedType; /**< Backed type of model files */

	int mTargetTypes; /**< Target type to run inference */

	double mConfidenceThresHold; /**< Confidence threshold value */

	double mMeanValue; /**< The mean value for normalization */

	double mStdValue; /**< The scale factor value for normalization */

	int mMaxOutputNumbers;

	std::vector<std::string> mInputLayerNames; /**< The input layer names */
	std::vector<std::string> mOutputLayerNames; /**< The output layer names */
};


class Inference {
public:
	/**
	 * @brief   Creates an Inference class instance.
	 *
	 * @since_tizen 5.5
	 */
	Inference();

	/**
	 * @brief   Destroys an Inference class instance including
	 *           its all resources.
	 *
	 * @since_tizen 5.5
	 */
	~Inference();

	/**
	 * @brief   Configure modelfiles
	 *
	 * @since_tizen 5.5
	 */
    void ConfigureModelFiles(
                const std::string modelConfigFilePath,
		        const std::string modelWeightFilePath,
		        const std::string modelUserFilePath);

	/**
	 * @brief   Configure input tensor information
	 *
	 * @since_tizen 5.5
	 * @remarks deprecated Replayced by ConfigureInputInfo
	 */
    void ConfigureTensorInfo(int width,
                int height,
                int dim,
                int ch,
				double stdValue,
				double meanValue);

	/**
	 * @brief Configure input infomation
	 *
	 * @since_tizen 6.0
	 */
	void ConfigureInputInfo(int width,
		int height,
		int dim,
		int ch,
		double stdValue,
		double meanValue,
		const std::vector<std::string> names);

	void ConfigureOutputInfo(std::vector<std::string> names);

	/**
	 * @brief   Configure inference backend type.
	 *
	 * @since_tizen 6.0
	 */
	int ConfigureBackendType(const mv_inference_backend_type_e backendType);

	/**
	 * @brief   Configure inference target types such as CPU, GPU or NPU. (one more types can be combined)
	 *
	 * @since_tizen 6.0
	 */
	int ConfigureTargetTypes(const int targetTypes);

	/**
	 * @brief   Configure the maximum number of inference results
	 *
	 * @since_tizen 5.5
	 */
	void ConfigureOutput(const int maxOutputNumbers);

	/**
	 * @brief   Configure the confidence threshold
	 *
	 * @since_tizen 5.5
	 */
	void ConfigureThreshold(const double threshold);

	/**
	 * @brief   Configure the input node name
	 *
	 * @since_tizen 5.5
	 * @remarks deprecated Replayced by ConfigureInputInfo
	 */
	void ConfigureInputNodeName(const std::string nodeName);

	/**
	 * @brief   Configure the output node names
	 *
	 * @since_tizen 5.5
	 * @remarks deprecated Replaced by ConfigureOutputInfo
	 */
	void ConfigureOutputNodeNames(const std::vector<std::string> nodeNames);

	/**
	 * @brief   Bind a backend engine
	 * @details Use this function to bind a backend engine for the inference.
	 * 			This creates a inference engine common class object, and loads a backend
	 *			library which inferfaces with a Neural Network runtime such as TF Lite,
	 *			OpenCV, ARMNN and so on.
	 *
	 *			Ps. The created inference engine common object will be released and its
	 *				corresponding backend library will be unbound when deconstructor
	 *				of Inference class will be called.
	 *
	 * @since_tizen 6.0
	 *
	 * @return @c 0 on success, otherwise a negative error value
	 * @retval #MEDIA_VISION_ERROR_NONE Successful
	 * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter
	 */
	int Bind();

	/**
	 * @brief   Set default configuration for the inference
	 * @details Use this function to set default configuration given in json file by user.
	 *
	 * 			Ps. this callback should be called after Bind callback.
	 *
	 * @since_tizen 6.0
	 *
	 * @return @c 0 on success, otherwise a negative error value
	 * @retval #MEDIA_VISION_ERROR_NONE Successful
	 * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter
	 */
	int Prepare();

	/**
	 * @brief   Load model files
	 * @details Use this function to load given model files for the inference.
	 *
	 * 			Ps. this callback should be called after Prepare callback.
	 *
	 * @since_tizen 6.0
	 *
	 * @return @c 0 on success, otherwise a negative error value
	 * @retval #MEDIA_VISION_ERROR_NONE Successful
	 * @retval #MEDIA_VISION_ERROR_INVALID_PARAMETER Invalid parameter
	 */
	int Load();

	/**
	 * @brief	Runs inference with a region of a given image
	 * @details Use this function to run forward pass with the given image.
	 *          The given image is preprocessed and the region of the image is
	 *          thrown to neural network. Then, the output tensor is returned.
	 *          If roi is NULL, then full source will be analyzed.
	 *
	 * @since_tizen 5.5
	 * @return @c true on success, otherwise a negative error value
	 */
	int Run(std::vector<mv_source_h> &mvSources, std::vector<mv_rectangle_s> &rects);

	/**
	 * @brief	Gets that given engine is supported or not
	 *
	 * @since_tizen 5.5
	 * @return @c true on success, otherwise a negative error value
	 */
	std::pair<std::string, bool> GetSupportedInferenceBackend(int backend);

	/**
	 * @brief	Gets the ImageClassificationResults
	 *
	 * @since_tizen 5.5
	 * @return @c true on success, otherwise a negative error value
	 */
	int GetClassficationResults(ImageClassificationResults *classificationResults);

	/**
	 * @brief	Gets the ObjectDetectioResults
	 *
	 * @since_tizen 5.5
	 * @return @c true on success, otherwise a negative error value
	 */
	int GetObjectDetectionResults(ObjectDetectionResults *detectionResults);

	/**
	 * @brief	Gets the FaceDetectioResults
	 *
	 * @since_tizen 5.5
	 * @return @c true on success, otherwise a negative error value
	 */
	int GetFaceDetectionResults(FaceDetectionResults *detectionResults);

	/**
	 * @brief	Gets the FacialLandmarkDetectionResults
	 *
	 * @since_tizen 5.5
	 * @return @c true on success, otherwise a negative error value
	 */
	int GetFacialLandMarkDetectionResults(FacialLandMarkDetectionResults* results);

	/**
	 * @brief	Gets the PoseEstimationDetectionResults
	 *
	 * @since_tizen 6.0
	 * @return @c true on success, otherwise a negative error value
	 */
	int GetPoseEstimationDetectionResults(PoseEstimationResults* results);

	int GetResults(std::vector<std::vector<int>>* dimInfo, std::vector<float*> *results);

	mv_engine_config_h GetEngineConfig(void) { return engine_config; }

	void SetEngineConfig(mv_engine_config_h config) { engine_config = config; }

private:
	bool mCanRun; /**< The flag indicating ready to run Inference */
    InferenceConfig mConfig;
	inference_engine_capacity mBackendCapacity;
	std::map<int, std::pair<std::string, bool>> mSupportedInferenceBackend;
	cv::Size mInputSize;
	int mCh;
	int mDim;
	double mDeviation;
	double mMean;
	double mThreshold;
	int mOutputNumbers;
	cv::Size mSourceSize;
	cv::Mat mInputBuffer;

	mv_engine_config_h engine_config;

	InferenceEngineCommon * mBackend;

	std::map<std::string, int> mModelFormats;
    std::vector<std::string> mUserListName;

	std::vector<inference_engine_tensor_buffer> mInputTensorBuffers;
	inference_engine_layer_property mInputLayerProperty;
	std::vector<inference_engine_tensor_buffer> mOutputTensorBuffers;
	inference_engine_layer_property mOutputLayerProperty;

private:
	void CheckSupportedInferenceBackend();
	int ConvertEngineErrorToVisionError(int error);
	int ConvertTargetTypes(int given_types);
	int ConvertDataTypes(int given_type);
	int Preprocess(cv::Mat cvImg, cv::Mat cvDst, int data_type);
	int PrepareTenosrBuffers(void);
	void CleanupTensorBuffers(void);
	int SetUserFile(std::string filename);
	int FillOutputResult(tensor_t &outputData);
};

} /* Inference */
} /* MediaVision */

#endif /* __MEDIA_VISION_INFERENCE_H__ */