path: root/mv_face/face/src/FaceRecognitionModel.cpp

/**
 * Copyright (c) 2015 Samsung Electronics Co., Ltd All Rights Reserved
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "FaceRecognitionModel.h"

#include "mv_private.h"
#include "mv_common.h"

#include <map>

#include <stdio.h>
#include <unistd.h>
#include <fstream>

namespace MediaVision {
namespace Face {
namespace {

unsigned int DefaultUnisizeWidth = 200;
unsigned int DefaultUnisizeHeight = 200;

bool isEmptyAlgorithmParam(const std::string& path)
{
	char valid[256] = "";
	std::ifstream ifs;

	ifs.open(path.c_str());

	if (!ifs.is_open()) {
		LOGE("[%s] Can't open file.", path.c_str());
		return true;
	}

	ifs.getline(valid, 256);
	ifs.close();

	LOGD("Validation string: %s", valid);
	if (strlen(valid) <= 0) {
		LOGE("algorithm params is empty.");
		return true;
	}

	return false;
}

int CopyOpenCVAlgorithmParameters(const cv::Ptr<cv::face::FaceRecognizer>& srcAlg,
		cv::Ptr<cv::face::FaceRecognizer>& dstAlg)
{
	char tempPath[1024] = "";

	snprintf(tempPath, 1024, "/tmp/alg_copy_%p_%p", srcAlg.get(), dstAlg.get());

	srcAlg->write(tempPath);

	if (!isEmptyAlgorithmParam(tempPath))
		dstAlg->read(tempPath);

	if (0 != remove(tempPath))
		LOGW("Error removing serialized FaceRecognizer in %s", tempPath);

	/* todo: consider to uncomment this lines if OpenCV will support deep
			copy of AlgorithmInfo objects: */

	/*std::vector<std::string> paramNames;
	srcAlg->getParams(paramNames);
	size_t paramSize = paramNames.size();
	for (size_t i = 0; i < paramSize; ++i) {
		int pType = srcAlg->paramType(paramNames[i]);

		switch(pType) {
		case cv::Param::INT:
		case cv::Param::UNSIGNED_INT:
		case cv::Param::UINT64:
		case cv::Param::SHORT:
		case cv::Param::UCHAR:
			dstAlg->set(paramNames[i], srcAlg->getInt(paramNames[i]));
			break;
		case cv::Param::BOOLEAN:
			dstAlg->set(paramNames[i], srcAlg->getBool(paramNames[i]));
			break;
		case cv::Param::REAL:
		case cv::Param::FLOAT:
			dstAlg->set(paramNames[i], srcAlg->getDouble(paramNames[i]));
			break;
		case cv::Param::STRING:
			dstAlg->set(paramNames[i], srcAlg->getString(paramNames[i]));
			break;
		case cv::Param::MAT:
			dstAlg->set(paramNames[i], srcAlg->getMat(paramNames[i]));
			break;
		case cv::Param::MAT_VECTOR:
		{
			//std::vector<cv::Mat> value = srcAlg->getMatVector(paramNames[i]);
			//dstAlg->info()->addParam(*(dstAlg.obj), paramNames[i].c_str(), value);
			dstAlg->set(paramNames[i], srcAlg->getMatVector(paramNames[i]));
			break;
		}
		case cv::Param::ALGORITHM:
			dstAlg->set(paramNames[i], srcAlg->getAlgorithm(paramNames[i]));
			break;
		default:
			LOGE("While copying algorothm parameters unsupported parameter "
				"%s was found.", paramNames[i].c_str());

			return MEDIA_VISION_ERROR_NOT_SUPPORTED;
		}
	}*/

	return MEDIA_VISION_ERROR_NONE;
}

void ParseOpenCVLabels(
		const cv::Ptr<cv::face::FaceRecognizer>& recognizer,
		std::set<int>& outLabels)
{
	if (!recognizer.empty()) {
		cv::Mat labels = (dynamic_cast<cv::face::EigenFaceRecognizer*>(recognizer.get()))->getLabels();

		for (int i = 0; i < labels.rows; ++i)
			outLabels.insert(labels.at<int>(i, 0));
	}
}

} /* anonymous namespace */

FaceRecognitionModelConfig::FaceRecognitionModelConfig() :
		mModelType(MEDIA_VISION_FACE_MODEL_TYPE_UNKNOWN),
		mNumComponents(0),
		mThreshold(DBL_MAX),
		mRadius(1),
		mNeighbors(8),
		mGridX(8),
		mGridY(8),
		mImgWidth(DefaultUnisizeWidth),
		mImgHeight(DefaultUnisizeHeight)
{
	; /* NULL */
}

FaceRecognitionResults::FaceRecognitionResults() :
		mIsRecognized(false),
		mFaceLabel(-1),
		mConfidence(0.0)
{
	; /* NULL */
}

bool FaceRecognitionModelConfig::operator!=(
		const FaceRecognitionModelConfig& other) const
{
	return mModelType      != other.mModelType     ||
			mNumComponents != other.mNumComponents ||
			mThreshold     != other.mThreshold     ||
			mRadius        != other.mRadius        ||
			mNeighbors     != other.mNeighbors     ||
			mGridX         != other.mGridX         ||
			mGridY         != other.mGridY         ||
			mImgWidth      != other.mImgWidth      ||
			mImgHeight     != other.mImgHeight;
}

FaceRecognitionModel::FaceRecognitionModel() :
		m_canRecognize(false),
		m_recognizer() // The default constructor creates a null Ptr
{
	; /* NULL */
}

FaceRecognitionModel::FaceRecognitionModel(const FaceRecognitionModel& origin) :
		m_canRecognize(origin.m_canRecognize),
		m_faceSamples(origin.m_faceSamples),
		m_learnAlgorithmConfig(origin.m_learnAlgorithmConfig),
		m_recognizer(CreateRecognitionAlgorithm(origin.m_learnAlgorithmConfig)),
		m_learnedLabels(origin.m_learnedLabels)
{
	if (!m_recognizer.empty())
		CopyOpenCVAlgorithmParameters(origin.m_recognizer, m_recognizer);
}

FaceRecognitionModel& FaceRecognitionModel::operator=(
		const FaceRecognitionModel& copy)
{
	if (this != &copy) {
		m_canRecognize = copy.m_canRecognize;
		m_faceSamples = copy.m_faceSamples;
		m_learnAlgorithmConfig = copy.m_learnAlgorithmConfig;
		m_recognizer = CreateRecognitionAlgorithm(m_learnAlgorithmConfig);
		m_learnedLabels = copy.m_learnedLabels;

		if (!m_recognizer.empty())
			CopyOpenCVAlgorithmParameters(copy.m_recognizer, m_recognizer);
	}

	return *this;
}

FaceRecognitionModel::~FaceRecognitionModel()
{
	; /* NULL */
}

int FaceRecognitionModel::save(const std::string& fileName)
{
	if (!m_recognizer.empty()) {
		std::string filePath;

		filePath = fileName;

		std::string prefixPath = filePath.substr(0, filePath.find_last_of('/'));
		LOGD("prefixPath: %s", prefixPath.c_str());

		/* check the directory is available */
		if (access(prefixPath.c_str(), F_OK)) {
			LOGE("Can't save recognition model. Path[%s] doesn't existed.", prefixPath.c_str());

			return MEDIA_VISION_ERROR_INVALID_PATH;
		}

		cv::FileStorage storage(filePath, cv::FileStorage::WRITE);
		if (!storage.isOpened()) {
			LOGE("Can't save recognition model. Write to file permission denied.");
			return MEDIA_VISION_ERROR_PERMISSION_DENIED;
		}

		switch (m_learnAlgorithmConfig.mModelType) {
		case MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES:
			storage << "algorithm" << "Eigenfaces";
			storage << "resizeW" << m_learnAlgorithmConfig.mImgWidth;
			storage << "resizeH" << m_learnAlgorithmConfig.mImgHeight;
			break;
		case MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES:
			storage << "algorithm" << "Fisherfaces";
			storage << "resizeW" << m_learnAlgorithmConfig.mImgWidth;
			storage << "resizeH" << m_learnAlgorithmConfig.mImgHeight;
			break;
		case MEDIA_VISION_FACE_MODEL_TYPE_LBPH:
			storage << "algorithm" << "LBPH";
			break;
		default:
			storage.release();
			return MEDIA_VISION_ERROR_NOT_SUPPORTED_FORMAT;
		}

		storage << "can_recognize" << m_canRecognize;
		m_recognizer->write(storage);

		storage.release();
	} else {
		LOGE("Attempt to save recognition model before learn");
		return MEDIA_VISION_ERROR_INVALID_OPERATION;
	}

	return MEDIA_VISION_ERROR_NONE;
}

int FaceRecognitionModel::load(const std::string& fileName)
{
	std::string filePath;

	filePath = fileName;

	if (access(filePath.c_str(), F_OK)) {
		LOGE("Can't load face recognition model. File[%s] doesn't existed.", filePath.c_str());

		return MEDIA_VISION_ERROR_INVALID_PATH;
	}

	cv::FileStorage storage(filePath, cv::FileStorage::READ);
	if (!storage.isOpened()) {
		LOGE("Can't load recognition model. Read from file permission denied.");

		return MEDIA_VISION_ERROR_PERMISSION_DENIED;
	}

	LOGD("Loading recognition model from file.");

	std::string algName;
	int canRecognize = 0;
	storage["algorithm"] >> algName;
	storage["can_recognize"] >> canRecognize;

	cv::Ptr<cv::face::FaceRecognizer> tempRecognizer;
	FaceRecognitionModelConfig tempConfig;
	std::set<int> tempLearnedLabels;

	if (algName == "Eigenfaces") {
		tempRecognizer = cv::face::EigenFaceRecognizer::create();
		storage["resizeW"] >> tempConfig.mImgWidth;
		storage["resizeH"] >> tempConfig.mImgHeight;
		tempRecognizer->read(storage.root());
		tempConfig.mModelType =
				MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES;
		tempConfig.mNumComponents =
				(dynamic_cast<cv::face::EigenFaceRecognizer*>(tempRecognizer.get()))->getNumComponents();
		ParseOpenCVLabels(tempRecognizer, tempLearnedLabels);
	} else if (algName == "Fisherfaces") {
		tempRecognizer = cv::face::FisherFaceRecognizer::create();
		storage["resizeW"] >> tempConfig.mImgWidth;
		storage["resizeH"] >> tempConfig.mImgHeight;
		tempRecognizer->read(storage.root());
		tempConfig.mModelType =
				MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES;
		tempConfig.mNumComponents =
				(dynamic_cast<cv::face::FisherFaceRecognizer*>(tempRecognizer.get()))->getNumComponents();
		ParseOpenCVLabels(tempRecognizer, tempLearnedLabels);
	} else if (algName == "LBPH") {
		tempRecognizer = cv::face::LBPHFaceRecognizer::create();
		tempRecognizer->read(storage.root());
		tempConfig.mModelType =
				MEDIA_VISION_FACE_MODEL_TYPE_LBPH;
		tempConfig.mGridX = (dynamic_cast<cv::face::LBPHFaceRecognizer*>(tempRecognizer.get()))->getGridX();
		tempConfig.mGridY = (dynamic_cast<cv::face::LBPHFaceRecognizer*>(tempRecognizer.get()))->getGridY();
		tempConfig.mNeighbors = (dynamic_cast<cv::face::LBPHFaceRecognizer*>(tempRecognizer.get()))->getNeighbors();
		tempConfig.mRadius = (dynamic_cast<cv::face::LBPHFaceRecognizer*>(tempRecognizer.get()))->getRadius();
		ParseOpenCVLabels(tempRecognizer, tempLearnedLabels);
	} else {
		tempConfig = FaceRecognitionModelConfig();
		LOGE("Failed to load face recognition model from file. File is in "
				"unsupported format");

		storage.release();

		return MEDIA_VISION_ERROR_NOT_SUPPORTED_FORMAT;
	}

	tempConfig.mThreshold = tempRecognizer->getThreshold();

	LOGD("Recognition model of [%s] type has been loaded from file",
			algName.c_str());

	storage.release();

	m_recognizer = tempRecognizer;
	m_learnAlgorithmConfig = tempConfig;
	m_canRecognize = (canRecognize == 1);
	m_learnedLabels.clear();
	m_learnedLabels = tempLearnedLabels;

	return MEDIA_VISION_ERROR_NONE;
}

int FaceRecognitionModel::addFaceExample(
		const cv::Mat& faceImage,
		int faceLabel)
{
	m_faceSamples[faceLabel].push_back(faceImage);

	LOGD("Added face image example for label %i for recognition model",
			faceLabel);

	return MEDIA_VISION_ERROR_NONE;
}

int FaceRecognitionModel::resetFaceExamples(void)
{
	m_faceSamples.clear();

	LOGD("All face image examples have been removed from recognition model");

	return MEDIA_VISION_ERROR_NONE;
}

int FaceRecognitionModel::resetFaceExamples(int faceLabel)
{
	if (1 > m_faceSamples.erase(faceLabel)) {
		LOGD("Failed to remove face image examples for label %i. "
				"No such examples", faceLabel);

		return MEDIA_VISION_ERROR_KEY_NOT_AVAILABLE;
	}

	LOGD("Face image examples for label %i have been removed from "
			"recognition model", faceLabel);

	return MEDIA_VISION_ERROR_NONE;
}

const std::set<int>& FaceRecognitionModel::getFaceLabels(void) const
{
	return m_learnedLabels;
}

int FaceRecognitionModel::learn(const FaceRecognitionModelConfig& config)
{
	/* Check number of classes collected for learning, some algorithms
	 * require specific class number constraints. For example, Fisherfaces
	 * requires more that 1 class in training set */
	if (MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES == config.mModelType &&
			m_faceSamples.size() < 2) {
		LOGE("Can't apply Fisherfaces learning algorithm. It requires at "
				"least two classes (face labes) to learn on.");

		return MEDIA_VISION_ERROR_INVALID_OPERATION;
	}

	bool isIncremental = false;
	bool isUnisize = false;

	if (MEDIA_VISION_FACE_MODEL_TYPE_LBPH == config.mModelType)
		isIncremental = true;

	if (MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES == config.mModelType ||
			MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES == config.mModelType)
		isUnisize = true;

	std::vector<cv::Mat> samples;
	std::vector<int> labels;
	std::set<int> learnedLabels;

	if (isIncremental)
		learnedLabels.insert(m_learnedLabels.begin(), m_learnedLabels.end());

	std::map<int, std::vector<cv::Mat> >::const_iterator it = m_faceSamples.begin();

	for (; it != m_faceSamples.end(); ++it) {
		const size_t faceClassSamplesSize = it->second.size();
		labels.insert(labels.end(), faceClassSamplesSize, it->first);
		learnedLabels.insert(it->first);

		if (!isUnisize) {
			LOGD("%zu examples has been added with label %i",
					it->second.size(), it->first);
			samples.insert(samples.end(), it->second.begin(), it->second.end());
		} else {
			for (size_t sampleInd = 0; sampleInd < faceClassSamplesSize; ++sampleInd) {
				cv::Mat resizedSample;
				cv::resize(it->second[sampleInd],
						resizedSample,
						cv::Size(config.mImgWidth, config.mImgHeight),
						0.0, 0.0, cv::INTER_CUBIC);
				samples.push_back(resizedSample);
			}
		}
	}

	const size_t samplesSize = samples.size();
	const size_t labelsSize = labels.size();

	if (0 != samplesSize && samplesSize == labelsSize) {
		LOGD("Start to learn the model for %zu samples and %zu labels",
				samplesSize, labelsSize);

		if (m_learnAlgorithmConfig != config || m_recognizer.empty())
			m_recognizer = CreateRecognitionAlgorithm(config);

		if (m_recognizer.empty()) {
			LOGE("Can't create recognition algorithm for recognition model. "
					"Configuration is not supported by any of known algorithms.");

			return MEDIA_VISION_ERROR_NOT_SUPPORTED;
		}

		isIncremental ? m_recognizer->update(samples, labels) :
						m_recognizer->train(samples, labels);
		m_canRecognize = true;
		m_learnedLabels.clear();
		m_learnedLabels = learnedLabels;
	} else {
		LOGE("Can't create recognition algorithm for no examples. Try to add "
				"some face examples before learning");

		return MEDIA_VISION_ERROR_NO_DATA;
	}

	m_learnAlgorithmConfig = config;

	LOGD("Recognition model has been learned");

	return MEDIA_VISION_ERROR_NONE;
}

int FaceRecognitionModel::recognize(const cv::Mat& image, FaceRecognitionResults& results)
{
	if (!m_recognizer.empty() && m_canRecognize) {
		double absConf = 0.0;
		cv::Mat predictionImg(m_learnAlgorithmConfig.mImgWidth,
				m_learnAlgorithmConfig.mImgHeight, CV_8UC1);

		if ((MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES == m_learnAlgorithmConfig.mModelType ||
				MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES == m_learnAlgorithmConfig.mModelType) &&
				(image.cols != m_learnAlgorithmConfig.mImgWidth ||
				image.rows != m_learnAlgorithmConfig.mImgHeight))
			cv::resize(image, predictionImg, predictionImg.size());
		else
			predictionImg = image;

		m_recognizer->predict(predictionImg, results.mFaceLabel, absConf);

		if (-1 != results.mFaceLabel) {
			double normShift = 7.5;
			double normSmooth = 0.05;

			if (MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES == m_learnAlgorithmConfig.mModelType) {
				normShift = 5.0;
				normSmooth = 0.0015;
			} else if (MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES == m_learnAlgorithmConfig.mModelType) {
				normShift = 5.0;
				normSmooth = 0.01;
			}

			/* Normalize the absolute value of the confidence */
			absConf = exp(normShift - (normSmooth * absConf));
			results.mConfidence = absConf / (1 + absConf);
			results.mIsRecognized = true;
		} else {
			results.mConfidence = 0.0;
			results.mIsRecognized = false;
		}

		results.mFaceLocation = cv::Rect(0, 0, image.cols, image.rows);
	} else {
		LOGE("Attempt to recognize faces with untrained model");
		return MEDIA_VISION_ERROR_INVALID_OPERATION;
	}

	return MEDIA_VISION_ERROR_NONE;
}

cv::Ptr<cv::face::FaceRecognizer> FaceRecognitionModel::CreateRecognitionAlgorithm(
		const FaceRecognitionModelConfig& config)
{
	cv::Ptr<cv::face::FaceRecognizer> tempRecognizer;
	switch (config.mModelType) {
	case MEDIA_VISION_FACE_MODEL_TYPE_EIGENFACES:
		tempRecognizer = cv::face::EigenFaceRecognizer::create(
									config.mNumComponents,
									config.mThreshold);
		break;
	case MEDIA_VISION_FACE_MODEL_TYPE_FISHERFACES:
		tempRecognizer = cv::face::FisherFaceRecognizer::create(
									config.mNumComponents,
									config.mThreshold);
		break;
	case MEDIA_VISION_FACE_MODEL_TYPE_LBPH:
		tempRecognizer = cv::face::LBPHFaceRecognizer::create(
									config.mRadius,
									config.mNeighbors,
									config.mGridX,
									config.mGridY,
									config.mThreshold);
		break;
	default:
		LOGE("Unknown FaceRecognition model");
	}

	return tempRecognizer;
}

} /* Face */
} /* MediaVision */