diff options
author | Felix Abecassis <fabecassis@nvidia.com> | 2015-05-29 22:32:25 -0700 |
---|---|---|
committer | Felix Abecassis <fabecassis@nvidia.com> | 2015-05-29 22:32:25 -0700 |
commit | 4874c01487f552dd4b16313d0a6723634f1912e5 (patch) | |
tree | 5fc430737a533aca3e48f216fa4d0dc323bf88bb /examples | |
parent | 8df472af7f0d475ce866055d0665086e093aaf0e (diff) | |
download | caffeonacl-4874c01487f552dd4b16313d0a6723634f1912e5.tar.gz caffeonacl-4874c01487f552dd4b16313d0a6723634f1912e5.tar.bz2 caffeonacl-4874c01487f552dd4b16313d0a6723634f1912e5.zip |
Add a simple C++ classification example
Closes #2487
Example usage:
./build/examples/cpp_classification/classification.bin \
models/bvlc_reference_caffenet/deploy.prototxt \
models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel \
data/ilsvrc12/imagenet_mean.binaryproto \
data/ilsvrc12/synset_words.txt \
examples/images/cat.jpg
Diffstat (limited to 'examples')
-rw-r--r-- | examples/cpp_classification/classification.cpp | 255 | ||||
-rw-r--r-- | examples/cpp_classification/readme.md | 77 |
2 files changed, 332 insertions, 0 deletions
diff --git a/examples/cpp_classification/classification.cpp b/examples/cpp_classification/classification.cpp new file mode 100644 index 00000000..1c6371e3 --- /dev/null +++ b/examples/cpp_classification/classification.cpp @@ -0,0 +1,255 @@ +#include <caffe/caffe.hpp> +#include <opencv2/core/core.hpp> +#include <opencv2/highgui/highgui.hpp> +#include <opencv2/imgproc/imgproc.hpp> +#include <iosfwd> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +using namespace caffe; // NOLINT(build/namespaces) +using std::string; + +/* Pair (label, confidence) representing a prediction. */ +typedef std::pair<string, float> Prediction; + +class Classifier { + public: + Classifier(const string& model_file, + const string& trained_file, + const string& mean_file, + const string& label_file); + + std::vector<Prediction> Classify(const cv::Mat& img, int N = 5); + + private: + void SetMean(const string& mean_file); + + std::vector<float> Predict(const cv::Mat& img); + + void WrapInputLayer(std::vector<cv::Mat>* input_channels); + + void Preprocess(const cv::Mat& img, + std::vector<cv::Mat>* input_channels); + + private: + shared_ptr<Net<float> > net_; + cv::Size input_geometry_; + int num_channels_; + cv::Mat mean_; + std::vector<string> labels_; +}; + +Classifier::Classifier(const string& model_file, + const string& trained_file, + const string& mean_file, + const string& label_file) { +#ifdef CPU_ONLY + Caffe::set_mode(Caffe::CPU); +#else + Caffe::set_mode(Caffe::GPU); +#endif + + /* Load the network. */ + net_.reset(new Net<float>(model_file, TEST)); + net_->CopyTrainedLayersFrom(trained_file); + + CHECK_EQ(net_->num_inputs(), 1) << "Network should have exactly one input."; + CHECK_EQ(net_->num_outputs(), 1) << "Network should have exactly one output."; + + Blob<float>* input_layer = net_->input_blobs()[0]; + num_channels_ = input_layer->channels(); + CHECK(num_channels_ == 3 || num_channels_ == 1) + << "Input layer should have 1 or 3 channels."; + input_geometry_ = cv::Size(input_layer->width(), input_layer->height()); + + /* Load the binaryproto mean file. */ + SetMean(mean_file); + + /* Load labels. */ + std::ifstream labels(label_file.c_str()); + CHECK(labels) << "Unable to open labels file " << label_file; + string line; + while (std::getline(labels, line)) + labels_.push_back(string(line)); + + Blob<float>* output_layer = net_->output_blobs()[0]; + CHECK_EQ(labels_.size(), output_layer->channels()) + << "Number of labels is different from the output layer dimension."; +} + +static bool PairCompare(const std::pair<float, int>& lhs, + const std::pair<float, int>& rhs) { + return lhs.first > rhs.first; +} + +/* Return the indices of the top N values of vector v. */ +static std::vector<int> Argmax(const std::vector<float>& v, int N) { + std::vector<std::pair<float, int> > pairs; + for (size_t i = 0; i < v.size(); ++i) + pairs.push_back(std::make_pair(v[i], i)); + std::partial_sort(pairs.begin(), pairs.begin() + N, pairs.end(), PairCompare); + + std::vector<int> result; + for (int i = 0; i < N; ++i) + result.push_back(pairs[i].second); + return result; +} + +/* Return the top N predictions. */ +std::vector<Prediction> Classifier::Classify(const cv::Mat& img, int N) { + std::vector<float> output = Predict(img); + + std::vector<int> maxN = Argmax(output, N); + std::vector<Prediction> predictions; + for (int i = 0; i < N; ++i) { + int idx = maxN[i]; + predictions.push_back(std::make_pair(labels_[idx], output[idx])); + } + + return predictions; +} + +/* Load the mean file in binaryproto format. */ +void Classifier::SetMean(const string& mean_file) { + BlobProto blob_proto; + ReadProtoFromBinaryFileOrDie(mean_file.c_str(), &blob_proto); + + /* Convert from BlobProto to Blob<float> */ + Blob<float> mean_blob; + mean_blob.FromProto(blob_proto); + CHECK_EQ(mean_blob.channels(), num_channels_) + << "Number of channels of mean file doesn't match input layer."; + + /* The format of the mean file is planar 32-bit float BGR or grayscale. */ + std::vector<cv::Mat> channels; + float* data = mean_blob.mutable_cpu_data(); + for (int i = 0; i < num_channels_; ++i) { + /* Extract an individual channel. */ + cv::Mat channel(mean_blob.height(), mean_blob.width(), CV_32FC1, data); + channels.push_back(channel); + data += mean_blob.height() * mean_blob.width(); + } + + /* Merge the separate channels into a single image. */ + cv::Mat mean; + cv::merge(channels, mean); + + /* Compute the global mean pixel value and create a mean image + * filled with this value. */ + cv::Scalar channel_mean = cv::mean(mean); + mean_ = cv::Mat(input_geometry_, mean.type(), channel_mean); +} + +std::vector<float> Classifier::Predict(const cv::Mat& img) { + Blob<float>* input_layer = net_->input_blobs()[0]; + input_layer->Reshape(1, num_channels_, + input_geometry_.height, input_geometry_.width); + /* Forward dimension change to all layers. */ + net_->Reshape(); + + std::vector<cv::Mat> input_channels; + WrapInputLayer(&input_channels); + + Preprocess(img, &input_channels); + + net_->ForwardPrefilled(); + + /* Copy the output layer to a std::vector */ + Blob<float>* output_layer = net_->output_blobs()[0]; + const float* begin = output_layer->cpu_data(); + const float* end = begin + output_layer->channels(); + return std::vector<float>(begin, end); +} + +/* Wrap the input layer of the network in separate cv::Mat objects + * (one per channel). This way we save one memcpy operation and we + * don't need to rely on cudaMemcpy2D. The last preprocessing + * operation will write the separate channels directly to the input + * layer. */ +void Classifier::WrapInputLayer(std::vector<cv::Mat>* input_channels) { + Blob<float>* input_layer = net_->input_blobs()[0]; + + int width = input_layer->width(); + int height = input_layer->height(); + float* input_data = input_layer->mutable_cpu_data(); + for (int i = 0; i < input_layer->channels(); ++i) { + cv::Mat channel(height, width, CV_32FC1, input_data); + input_channels->push_back(channel); + input_data += width * height; + } +} + +void Classifier::Preprocess(const cv::Mat& img, + std::vector<cv::Mat>* input_channels) { + /* Convert the input image to the input image format of the network. */ + cv::Mat sample; + if (img.channels() == 3 && num_channels_ == 1) + cv::cvtColor(img, sample, CV_BGR2GRAY); + else if (img.channels() == 4 && num_channels_ == 1) + cv::cvtColor(img, sample, CV_BGRA2GRAY); + else if (img.channels() == 4 && num_channels_ == 3) + cv::cvtColor(img, sample, CV_BGRA2BGR); + else if (img.channels() == 1 && num_channels_ == 3) + cv::cvtColor(img, sample, CV_GRAY2BGR); + else + sample = img; + + cv::Mat sample_resized; + if (sample.size() != input_geometry_) + cv::resize(sample, sample_resized, input_geometry_); + else + sample_resized = sample; + + cv::Mat sample_float; + if (num_channels_ == 3) + sample_resized.convertTo(sample_float, CV_32FC3); + else + sample_resized.convertTo(sample_float, CV_32FC1); + + cv::Mat sample_normalized; + cv::subtract(sample_float, mean_, sample_normalized); + + /* This operation will write the separate BGR planes directly to the + * input layer of the network because it is wrapped by the cv::Mat + * objects in input_channels. */ + cv::split(sample_normalized, *input_channels); + + CHECK(reinterpret_cast<float*>(input_channels->at(0).data) + == net_->input_blobs()[0]->cpu_data()) + << "Input channels are not wrapping the input layer of the network."; +} + +int main(int argc, char** argv) { + if (argc != 6) { + std::cerr << "Usage: " << argv[0] + << " deploy.prototxt network.caffemodel" + << " mean.binaryproto labels.txt img.jpg" << std::endl; + return 1; + } + + ::google::InitGoogleLogging(argv[0]); + + string model_file = argv[1]; + string trained_file = argv[2]; + string mean_file = argv[3]; + string label_file = argv[4]; + Classifier classifier(model_file, trained_file, mean_file, label_file); + + string file = argv[5]; + + std::cout << "---------- Prediction for " + << file << " ----------" << std::endl; + + cv::Mat img = cv::imread(file, -1); + CHECK(!img.empty()) << "Unable to decode image " << file; + std::vector<Prediction> predictions = classifier.Classify(img); + + /* Print the top N predictions. */ + for (size_t i = 0; i < predictions.size(); ++i) { + Prediction p = predictions[i]; + std::cout << std::fixed << std::setprecision(4) << p.second << " - \"" + << p.first << "\"" << std::endl; + } +} diff --git a/examples/cpp_classification/readme.md b/examples/cpp_classification/readme.md new file mode 100644 index 00000000..a086db1a --- /dev/null +++ b/examples/cpp_classification/readme.md @@ -0,0 +1,77 @@ +--- +title: CaffeNet C++ Classification example +description: A simple example performing image classification using the low-level C++ API. +category: example +include_in_docs: true +priority: 10 +--- + +# Classifying ImageNet: using the C++ API + +Caffe, at its core, is written in C++. It is possible to use the C++ +API of Caffe to implement an image classification application similar +to the Python code presented in one of the Notebook example. To look +at a more general-purpose example of the Caffe C++ API, you should +study the source code of the command line tool `caffe` in `tools/caffe.cpp`. + +## Presentation + +A simple C++ code is proposed in +`examples/cpp_classification/classification.cpp`. For the sake of +simplicity, this example does not support oversampling of a single +sample nor batching of multiple independant samples. This example is +not trying to reach the maximum possible classification throughput on +a system, but special care was given to avoid unnecessary +pessimization while keeping the code readable. + +## Compiling + +The C++ example is built automatically when compiling Caffe. To +compile Caffe you should follow the documented instructions. The +classification example will be built as `examples/classification.bin` +in your build directory. + +## Usage + +To use the pre-trained CaffeNet model with the classification example, +you need to download it from the "Model Zoo" using the following +script: +``` +./scripts/download_model_binary.py models/bvlc_reference_caffenet +``` +The ImageNet labels file (also called the *synset file*) is also +required in order to map a prediction to the name of the class: +``` +./data/ilsvrc12/get_ilsvrc_aux.sh. +``` +Using the files that were downloaded, we can classify the provided cat +image (`examples/images/cat.jpg`) using this command: +``` +./build/examples/cpp_classification/classification.bin \ + models/bvlc_reference_caffenet/deploy.prototxt \ + models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel \ + data/ilsvrc12/imagenet_mean.binaryproto \ + data/ilsvrc12/synset_words.txt \ + examples/images/cat.jpg +``` +The output should look like this: +``` +---------- Prediction for examples/images/cat.jpg ---------- +0.3134 - "n02123045 tabby, tabby cat" +0.2380 - "n02123159 tiger cat" +0.1235 - "n02124075 Egyptian cat" +0.1003 - "n02119022 red fox, Vulpes vulpes" +0.0715 - "n02127052 lynx, catamount" +``` + +## Improving Performance + +To further improve performance, you will need to leverage the GPU +more, here are some guidelines: + +* Move the data on the GPU early and perform all preprocessing +operations there. +* If you have many images to classify simultaneously, you should use +batching (independent images are classified in a single forward pass). +* Use multiple classification threads to ensure the GPU is always fully +utilized and not waiting for an I/O blocked CPU thread. |