[example] image classification web demo

6 files changed, 421 insertions, 1 deletions

diff --git a/data/ilsvrc12/get_ilsvrc_aux.sh b/data/ilsvrc12/get_ilsvrc_aux.sh
index 3fa58dc7..b9b85d21 100755
--- a/data/ilsvrc12/get_ilsvrc_aux.sh
+++ b/data/ilsvrc12/get_ilsvrc_aux.sh
@@ -4,6 +4,7 @@
 # This script downloads the imagenet example auxiliary files including:
 # - the ilsvrc12 image mean, binaryproto
 # - synset ids and words
+# - Python pickle-format data of ImageNet graph structure and relative infogain
 # - the training splits with labels
 
 DIR="$( cd "$(dirname "$0")" ; pwd -P )"
diff --git a/docs/getting_pretrained_models.md b/docs/getting_pretrained_models.md
index bbac5ac4..14e6ee91 100644
--- a/docs/getting_pretrained_models.md
+++ b/docs/getting_pretrained_models.md
@@ -8,7 +8,8 @@ layout: default
 Note that unlike Caffe itself, these models are licensed for **academic research / non-commercial use only**.
 If you have any questions, please get in touch with us.
 
-This page will be updated as more models become available.
+*UPDATE* July 2014: we are actively working on a service for hosting user-uploaded model definition and trained weight files.
+Soon, the community will be able to easily contribute different architectures!
 
 ### ImageNet
 
@@ -26,4 +27,6 @@ This page will be updated as more models become available.
   validation accuracy 57.258% and loss 1.83948.
 - This model obtains a top-1 accuracy 57.1% and a top-5 accuracy 80.2% on the validation set, using just the center crop. (Using the average of 10 crops, (4 + 1 center) * 2 mirror, should obtain a bit higher accuracy)
 
+### Auxiliary Data
+
 Additionally, you will probably eventually need some auxiliary data (mean image, synset list, etc.): run `data/ilsvrc12/get_ilsvrc_aux.sh` from the root directory to obtain it.
diff --git a/examples/web_demo/app.py b/examples/web_demo/app.py
new file mode 100644
index 00000000..9bc4ed5c
--- /dev/null
+++ b/examples/web_demo/app.py
@@ -0,0 +1,215 @@
+import os
+import time
+import cPickle
+import datetime
+import logging
+import flask
+import werkzeug
+import optparse
+import tornado.wsgi
+import tornado.httpserver
+import numpy as np
+import pandas as pd
+from PIL import Image as PILImage
+import cStringIO as StringIO
+import urllib
+import caffe
+import exifutil
+
+REPO_DIRNAME = os.path.abspath(os.path.dirname(__file__) + '/../..')
+UPLOAD_FOLDER = '/tmp/caffe_demos_uploads'
+ALLOWED_IMAGE_EXTENSIONS = set(['png', 'bmp', 'jpg', 'jpe', 'jpeg', 'gif'])
+
+# Obtain the flask app object
+app = flask.Flask(__name__)
+
+
+@app.route('/')
+def index():
+    return flask.render_template('index.html', has_result=False)
+
+
+@app.route('/classify_url', methods=['GET'])
+def classify_url():
+    imageurl = flask.request.args.get('imageurl', '')
+    try:
+        string_buffer = StringIO.StringIO(
+            urllib.urlopen(imageurl).read())
+        image = caffe.io.load_image(string_buffer)
+
+    except Exception as err:
+        # For any exception we encounter in reading the image, we will just
+        # not continue.
+        logging.info('URL Image open error: %s', err)
+        return flask.render_template(
+            'index.html', has_result=True,
+            result=(False, 'Cannot open image from URL.')
+        )
+
+    logging.info('Image: %s', imageurl)
+    result = app.clf.classify_image(image)
+    return flask.render_template(
+        'index.html', has_result=True, result=result, imagesrc=imageurl)
+
+
+@app.route('/classify_upload', methods=['POST'])
+def classify_upload():
+    try:
+        # We will save the file to disk for possible data collection.
+        imagefile = flask.request.files['imagefile']
+        filename_ = str(datetime.datetime.now()).replace(' ', '_') + \
+            werkzeug.secure_filename(imagefile.filename)
+        filename = os.path.join(UPLOAD_FOLDER, filename_)
+        imagefile.save(filename)
+        logging.info('Saving to %s.', filename)
+        image = exifutil.open_oriented_im(filename)
+
+    except Exception as err:
+        logging.info('Uploaded image open error: %s', err)
+        return flask.render_template(
+            'index.html', has_result=True,
+            result=(False, 'Cannot open uploaded image.')
+        )
+
+    result = app.clf.classify_image(image)
+    return flask.render_template(
+        'index.html', has_result=True, result=result,
+        imagesrc=embed_image_html(image)
+    )
+
+
+def embed_image_html(image):
+    """Creates an image embedded in HTML base64 format."""
+    image_pil = PILImage.fromarray((255 * image).astype('uint8'))
+    image_pil = image_pil.resize((256, 256))
+    string_buf = StringIO.StringIO()
+    image_pil.save(string_buf, format='png')
+    data = string_buf.getvalue().encode('base64').replace('\n', '')
+    return 'data:image/png;base64,' + data
+
+
+def allowed_file(filename):
+    return (
+        '.' in filename and
+        filename.rsplit('.', 1)[1] in ALLOWED_IMAGE_EXTENSIONS
+    )
+
+
+class ImagenetClassifier(object):
+    default_args = {
+        'model_def_file': (
+            '{}/examples/imagenet/imagenet_deploy.prototxt'.format(REPO_DIRNAME)),
+        'pretrained_model_file': (
+            '{}/examples/imagenet/caffe_reference_imagenet_model'.format(REPO_DIRNAME)),
+        'mean_file': (
+            '{}/python/caffe/imagenet/ilsvrc_2012_mean.npy'.format(REPO_DIRNAME)),
+        'class_labels_file': (
+            '{}/data/ilsvrc12/synset_words.txt'.format(REPO_DIRNAME)),
+        'bet_file': (
+            '{}/data/ilsvrc12/imagenet.bet.pickle'.format(REPO_DIRNAME)),
+    }
+    for key, val in default_args.iteritems():
+        if not os.path.exists(val):
+            raise Exception(
+                "File for {} is missing. Should be at: {}".format(key, val))
+    default_args['image_dim'] = 227
+    default_args['gpu_mode'] = True
+
+    def __init__(self, model_def_file, pretrained_model_file, mean_file,
+                 class_labels_file, bet_file, image_dim, gpu_mode=False):
+        logging.info('Loading net and associated files...')
+        self.net = caffe.Classifier(
+            model_def_file, pretrained_model_file, input_scale=255,
+            image_dims=(image_dim, image_dim), gpu=gpu_mode,
+            mean_file=mean_file, channel_swap=(2, 1, 0)
+        )
+
+        with open(class_labels_file) as f:
+            labels_df = pd.DataFrame([
+                {
+                    'synset_id': l.strip().split(' ')[0],
+                    'name': ' '.join(l.strip().split(' ')[1:]).split(',')[0]
+                }
+                for l in f.readlines()
+            ])
+        self.labels = labels_df.sort('synset_id')['name'].values
+
+        self.bet = cPickle.load(open(bet_file))
+        # A bias to prefer children nodes in single-chain paths
+        # I am setting the value to 0.1 as a quick, simple model.
+        # We could use better psychological models here...
+        self.bet['infogain'] -= np.array(self.bet['preferences']) * 0.1
+
+    def classify_image(self, image):
+        try:
+            starttime = time.time()
+            scores = self.net.predict([image], oversample=True).flatten()
+            endtime = time.time()
+
+            indices = (-scores).argsort()[:5]
+            predictions = self.labels[indices]
+
+            # In addition to the prediction text, we will also produce
+            # the length for the progress bar visualization.
+            meta = [
+                (p, '%.5f' % scores[i])
+                for i, p in zip(indices, predictions)
+            ]
+            logging.info('result: %s', str(meta))
+
+            # Compute expected information gain
+            expected_infogain = np.dot(
+                self.bet['probmat'], scores[self.bet['idmapping']])
+            expected_infogain *= self.bet['infogain']
+
+            # sort the scores
+            infogain_sort = expected_infogain.argsort()[::-1]
+            bet_result = [(self.bet['words'][v], '%.5f' % expected_infogain[v])
+                          for v in infogain_sort[:5]]
+            logging.info('bet result: %s', str(bet_result))
+
+            return (True, meta, bet_result, '%.3f' % (endtime - starttime))
+
+        except Exception as err:
+            logging.info('Classification error: %s', err)
+            return (False, 'Something went wrong when classifying the '
+                           'image. Maybe try another one?')
+
+
+def start_tornado(app, port=5000):
+    http_server = tornado.httpserver.HTTPServer(
+        tornado.wsgi.WSGIContainer(app))
+    http_server.listen(port)
+    print("Tornado server starting on port {}".format(port))
+    tornado.ioloop.IOLoop.instance().start()
+
+
+def start_from_terminal(app):
+    """
+    Parse command line options and start the server.
+    """
+    parser = optparse.OptionParser()
+    parser.add_option(
+        '-d', '--debug',
+        help="enable debug mode",
+        action="store_true", default=False)
+    parser.add_option(
+        '-p', '--port',
+        help="which port to serve content on",
+        type='int', default=5000)
+    opts, args = parser.parse_args()
+
+    # Initialize classifier
+    app.clf = ImagenetClassifier(**ImagenetClassifier.default_args)
+
+    if opts.debug:
+        app.run(debug=True, host='0.0.0.0', port=opts.port)
+    else:
+        start_tornado(app, opts.port)
+
+
+if __name__ == '__main__':
+    logging.getLogger().setLevel(logging.INFO)
+    if not os.path.exists(UPLOAD_FOLDER):
+        os.makedirs(UPLOAD_FOLDER)
+    start_from_terminal(app)
diff --git a/examples/web_demo/exifutil.py b/examples/web_demo/exifutil.py
new file mode 100644
index 00000000..8c07aa88
--- /dev/null
+++ b/examples/web_demo/exifutil.py
@@ -0,0 +1,33 @@
+"""
+This script handles the skimage exif problem.
+"""
+
+from PIL import Image
+import numpy as np
+
+ORIENTATIONS = {   # used in apply_orientation
+    2: (Image.FLIP_LEFT_RIGHT,),
+    3: (Image.ROTATE_180,),
+    4: (Image.FLIP_TOP_BOTTOM,),
+    5: (Image.FLIP_LEFT_RIGHT, Image.ROTATE_90),
+    6: (Image.ROTATE_270,),
+    7: (Image.FLIP_LEFT_RIGHT, Image.ROTATE_270),
+    8: (Image.ROTATE_90,)
+}
+
+
+def open_oriented_im(im_path):
+    im = Image.open(im_path)
+    if hasattr(im, '_getexif'):
+        exif = im._getexif()
+        if exif is not None and 274 in exif:
+            orientation = exif[274]
+            im = apply_orientation(im, orientation)
+    return np.asarray(im).astype(np.float32) / 255.
+
+
+def apply_orientation(im, orientation):
+    if orientation in ORIENTATIONS:
+        for method in ORIENTATIONS[orientation]:
+            im = im.transpose(method)
+    return im
diff --git a/examples/web_demo/readme.md b/examples/web_demo/readme.md
new file mode 100644
index 00000000..559c41e0
--- /dev/null
+++ b/examples/web_demo/readme.md
@@ -0,0 +1,30 @@
+---
+title: Web demo
+description: Image classification demo running as a Flask web server.
+category: example
+layout: default
+include_in_docs: true
+---
+
+# Web Demo
+
+## Requirements
+
+The demo server requires Python with some dependencies.
+To make sure you have the dependencies, please run `pip install -r examples/web_demo/requirements.txt`, and also make sure that you've compiled the Python Caffe interface and that it is on your `PYTHONPATH` (see [installation instructions](/installation.html)).
+
+Make sure that you have obtained the Caffe Reference ImageNet Model and the ImageNet Auxiliary Data ([instructions](/getting_pretrained_models.html)).
+NOTE: if you run into trouble, try re-downloading the auxiliary files.
+
+## Run
+
+Running `python examples/web_demo/app.py` will bring up the demo server, accessible at `http://0.0.0.0:5000`.
+You can enable debug mode of the web server, or switch to a different port:
+
+    % python examples/web_demo/app.py -h
+    Usage: app.py [options]
+
+    Options:
+      -h, --help            show this help message and exit
+      -d, --debug           enable debug mode
+      -p PORT, --port=PORT  which port to serve content on
diff --git a/examples/web_demo/templates/index.html b/examples/web_demo/templates/index.html
new file mode 100644
index 00000000..87893341
--- /dev/null
+++ b/examples/web_demo/templates/index.html
@@ -0,0 +1,138 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="description" content="Caffe demos">
+    <meta name="author" content="BVLC (http://bvlc.eecs.berkeley.edu/)">
+
+    <title>Caffe Demos</title>
+
+    <link href="//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
+
+    <script type="text/javascript" src="//code.jquery.com/jquery-2.1.1.js"></script>
+    <script src="//netdna.bootstrapcdn.com/bootstrap/3.1.1/js/bootstrap.min.js"></script>
+
+    <!-- Script to instantly classify an image once it is uploaded. -->
+    <script type="text/javascript">
+      $(document).ready(
+        function(){
+          $('#classifyfile').attr('disabled',true);
+          $('#imagefile').change(
+            function(){
+              if ($(this).val()){
+                $('#formupload').submit();
+              }
+            }
+          );
+        }
+      );
+    </script>
+
+    <style>
+    body {
+      font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+      line-height:1.5em;
+      color: #232323;
+      -webkit-font-smoothing: antialiased;
+    }
+
+    h1, h2, h3 {
+      font-family: Times, serif;
+      line-height:1.5em;
+      border-bottom: 1px solid #ccc;
+    }
+    </style>
+  </head>
+
+  <body>
+    <!-- Begin page content -->
+    <div class="container">
+      <div class="page-header">
+        <h1><a href="/">Caffe Demos</a></h1>
+        <p>
+          The <a href="http://caffe.berkeleyvision.org">Caffe</a> neural network library makes implementing state-of-the-art computer vision systems easy.
+        </p>
+      </div>
+
+      <div>
+        <h2>Classification</h2>
+        <a href="/classify_url?imageurl=http%3A%2F%2Fi.telegraph.co.uk%2Fmultimedia%2Farchive%2F02351%2Fcross-eyed-cat_2351472k.jpg">Click for a Quick Example</a>
+      </div>
+
+      {% if has_result %}
+      {% if not result[0] %}
+      <!-- we have error in the result. -->
+      <div class="alert alert-danger">{{ result[1] }} Did you provide a valid URL or a valid image file? </div>
+      {% else %}
+      <div class="media">
+        <a class="pull-left" href="#"><img class="media-object" width="192" height="192" src={{ imagesrc }}></a>
+        <div class="media-body">
+          <div class="bs-example bs-example-tabs">
+            <ul id="myTab" class="nav nav-tabs">
+              <li class="active"><a href="#infopred" data-toggle="tab">Maximally accurate</a></li>
+              <li><a href="#flatpred" data-toggle="tab">Maximally specific</a></li>
+            </ul>
+            <div id="myTabContent" class="tab-content">
+              <div class="tab-pane fade in active" id="infopred">
+                <ul class="list-group">
+                  {% for single_pred in result[2] %}
+                  <li class="list-group-item">
+                  <span class="badge">{{ single_pred[1] }}</span>
+                  <h4 class="list-group-item-heading">
+                    <a href="https://www.google.com/#q={{ single_pred[0] }}" target="_blank">{{ single_pred[0] }}</a>
+                  </h4>
+                  </li>
+                  {% endfor %}
+                </ul>
+              </div>
+              <div class="tab-pane fade" id="flatpred">
+                <ul class="list-group">
+                  {% for single_pred in result[1] %}
+                  <li class="list-group-item">
+                  <span class="badge">{{ single_pred[1] }}</span>
+                  <h4 class="list-group-item-heading">
+                    <a href="https://www.google.com/#q={{ single_pred[0] }}" target="_blank">{{ single_pred[0] }}</a>
+                  </h4>
+                  </li>
+                  {% endfor %}
+                </ul>
+              </div>
+            </div>
+          </div>
+
+        </div>
+      </div>
+      <p> CNN took {{ result[3] }} seconds. </p>
+      {% endif %}
+      <hr>
+      {% endif %}
+
+      <form role="form" action="classify_url" method="get">
+        <div class="form-group">
+          <div class="input-group">
+            <input type="text" class="form-control" name="imageurl" id="imageurl" placeholder="Provide an image URL">
+            <span class="input-group-btn">
+              <input class="btn btn-primary" value="Classify URL" type="submit" id="classifyurl"></input>
+            </span>
+          </div><!-- /input-group -->
+        </div>
+      </form>
+
+      <form id="formupload" class="form-inline" role="form" action="classify_upload" method="post" enctype="multipart/form-data">
+        <div class="form-group">
+          <label for="imagefile">Or upload an image:</label>
+          <input type="file" name="imagefile" id="imagefile">
+        </div>
+        <!--<input type="submit" class="btn btn-primary" value="Classify File" id="classifyfile"></input>-->
+      </form>
+    </div>
+
+    <hr>
+    <div id="footer">
+      <div class="container">
+        <p>&copy; BVLC 2014</p>
+      </div>
+   </div>
+ </body>
+</html>