CIFAR-10, CIFAR-100 inference code

Updated on April 28, 2017 by corochann · Leave a comment

The code structure of inference/predict stage is quite similar to MNIST inference code, please read this for precise explanation. Here, I will simply put the code and its results. CIFAR-10 inference code Code is uploaded on github as predict_cifar10.py.

"""Inference/predict code for CIFAR-10

model must be trained before inference, 
train_cifar10.py must be executed beforehand.
"""
from __future__ import print_function
import os
import argparse

import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import chainer
import chainer.functions as F
import chainer.links as L
from chainer import training, iterators, serializers, optimizers, Variable, cuda
from chainer.training import extensions

from CNNSmall import CNNSmall
from CNNMedium import CNNMedium

CIFAR10_LABELS_LIST = [
    'airplane',
    'automobile',
    'bird',
    'cat',
    'deer',
    'dog',
    'frog',
    'horse',
    'ship',
    'truck'
]


def main():
    archs = {
        'cnnsmall': CNNSmall,
        'cnnmedium': CNNMedium,
    }

    parser = argparse.ArgumentParser(description='Cifar-10 CNN predict code')
    parser.add_argument('--arch', '-a', choices=archs.keys(),
                        default='cnnsmall', help='Convnet architecture')
    #parser.add_argument('--batchsize', '-b', type=int, default=64,
    #                    help='Number of images in each mini-batch')
    parser.add_argument('--modelpath', '-m', default='result-cifar10-cnnsmall/cnnsmall-cifar10.model',
                        help='Model path to be loaded')
    parser.add_argument('--gpu', '-g', type=int, default=-1,
                        help='GPU ID (negative value indicates CPU)')
    args = parser.parse_args()

    print('GPU: {}'.format(args.gpu))
    #print('# Minibatch-size: {}'.format(args.batchsize))
    print('')

    # 1. Setup model
    class_num = 10
    model = archs[args.arch](n_out=class_num)
    classifier_model = L.Classifier(model)
    if args.gpu >= 0:
        chainer.cuda.get_device(args.gpu).use()  # Make a specified GPU current
        classifier_model.to_gpu()  # Copy the model to the GPU
    xp = np if args.gpu < 0 else cuda.cupy

    serializers.load_npz(args.modelpath, model)

    # 2. Load the CIFAR-10 dataset
    train, test = chainer.datasets.get_cifar10()

    basedir = 'images'
    plot_predict_cifar(os.path.join(basedir, 'cifar10_predict.png'), model,
                       train, 4, 5, scale=5., label_list=CIFAR10_LABELS_LIST)


def plot_predict_cifar(filepath, model, data, row, col,
                       scale=3., label_list=None):
    fig_width = data[0][0].shape[1] / 80 * row * scale
    fig_height = data[0][0].shape[2] / 80 * col * scale
    fig, axes = plt.subplots(row,
                             col,
                             figsize=(fig_height, fig_width))
    for i in range(row * col):
        # train[i][0] is i-th image data with size 32x32
        image, label_index = data[i]
        xp = cuda.cupy
        x = Variable(xp.asarray(image.reshape(1, 3, 32, 32)))    # test data
        #t = Variable(xp.asarray([test[i][1]]))  # labels
        y = model(x)                              # Inference result
        prediction = y.data.argmax(axis=1)
        image = image.transpose(1, 2, 0)
        print('Predicted {}-th image, prediction={}, actual={}'
              .format(i, prediction[0], label_index))
        r, c = divmod(i, col)
        axes[r][c].imshow(image)  # cmap='gray' is for black and white picture.
        if label_list is None:
            axes[r][c].set_title('Predict:{}, Answer: {}'
                                 .format(label_index, prediction[0]))
        else:
            pred = int(prediction[0])
            axes[r][c].set_title('Predict:{} {}\nAnswer:{} {}'
                                 .format(label_index, label_list[label_index],
                                         pred, label_list[pred]))
        axes[r][c].axis('off')  # do not show axis value
    plt.tight_layout(pad=0.01)   # automatic padding between subplots
    plt.savefig(filepath)
    print('Result saved to {}'.format(filepath))


if __name__ == '__main__':
    main()

100

101

102

103

104

105

106

107

108

109

110

111

112

"""Inference/predict code for CIFAR-10

model must be trained before inference,

train_cifar10.py must be executed beforehand.

"""

from __future__ import print_function

import os

import argparse

import numpy as np

import matplotlib

matplotlib.use('Agg')

import matplotlib.pyplot as plt

import chainer

import chainer.functions as F

import chainer.links as L

from chainer import training, iterators, serializers, optimizers, Variable, cuda

from chainer.training import extensions

from CNNSmall import CNNSmall

from CNNMedium import CNNMedium

CIFAR10_LABELS_LIST = [

'airplane',

'automobile',

'bird',

'cat',

'deer',

'dog',

'frog',

'horse',

'ship',

'truck'

]

def main():

archs = {

'cnnsmall': CNNSmall,

'cnnmedium': CNNMedium,

}

parser = argparse.ArgumentParser(description='Cifar-10 CNN predict code')

parser.add_argument('--arch', '-a', choices=archs.keys(),

default='cnnsmall', help='Convnet architecture')

#parser.add_argument('--batchsize', '-b', type=int, default=64,

# help='Number of images in each mini-batch')

parser.add_argument('--modelpath', '-m', default='result-cifar10-cnnsmall/cnnsmall-cifar10.model',

help='Model path to be loaded')

parser.add_argument('--gpu', '-g', type=int, default=-1,

help='GPU ID (negative value indicates CPU)')

args = parser.parse_args()

print('GPU: {}'.format(args.gpu))

#print('# Minibatch-size: {}'.format(args.batchsize))

print('')

# 1. Setup model

class_num = 10

model = archs[args.arch](n_out=class_num)

classifier_model = L.Classifier(model)

if args.gpu >= 0:

chainer.cuda.get_device(args.gpu).use() # Make a specified GPU current

classifier_model.to_gpu() # Copy the model to the GPU

xp = np if args.gpu < 0 else cuda.cupy

serializers.load_npz(args.modelpath, model)

# 2. Load the CIFAR-10 dataset

train, test = chainer.datasets.get_cifar10()

basedir = 'images'

plot_predict_cifar(os.path.join(basedir, 'cifar10_predict.png'), model,

train, 4, 5, scale=5., label_list=CIFAR10_LABELS_LIST)

def plot_predict_cifar(filepath, model, data, row, col,

scale=3., label_list=None):

fig_width = data[0][0].shape[1] / 80 * row * scale

fig_height = data[0][0].shape[2] / 80 * col * scale

fig, axes = plt.subplots(row,

col,

figsize=(fig_height, fig_width))

for i in range(row * col):

# train[i][0] is i-th image data with size 32x32

image, label_index = data[i]

xp = cuda.cupy

x = Variable(xp.asarray(image.reshape(1, 3, 32, 32))) # test data

#t = Variable(xp.asarray([test[i][1]])) # labels

y = model(x) # Inference result

prediction = y.data.argmax(axis=1)

image = image.transpose(1, 2, 0)

print('Predicted {}-th image, prediction={}, actual={}'

.format(i, prediction[0], label_index))

r, c = divmod(i, col)

axes[r][c].imshow(image) # cmap='gray' is for black and white picture.

if label_list is None:

axes[r][c].set_title('Predict:{}, Answer: {}'

.format(label_index, prediction[0]))

else:

pred = int(prediction[0])

axes[r][c].set_title('Predict:{} {}\nAnswer:{} {}'

.format(label_index, label_list[label_index],

pred, label_list[pred]))

axes[r][c].axis('off') # do not show axis value

plt.tight_layout(pad=0.01) # automatic padding between subplots

plt.savefig(filepath)

print('Result saved to {}'.format(filepath))

if __name__ == '__main__':

main()

This outputs the result as, You can see that even small CNN, it successfully classifies most of the images. Of course this is just a simple example and you can improve the model accuracy by tuning the deep neural network! CIFAR-100 inference code In the same way, code is uploaded on github as predict_cifar100.py. CIFAR-100 is more difficult than CIFAR-10 in general because there are more class to classify but exists […]

Continue reading →

CIFAR-10, CIFAR-100 training with Convolutional Neural Network

Updated on April 26, 2017 by corochann · Leave a comment

Writing your CNN model This is example of small Convolutional Neural Network definition, CNNSmall

import chainer
import chainer.functions as F
import chainer.links as L


class CNNSmall(chainer.Chain):
    def __init__(self, n_out):
        super(CNNSmall, self).__init__(
            conv1=L.Convolution2D(None, 16, 3, 2),
            conv2=L.Convolution2D(16, 32, 3, 2),
            conv3=L.Convolution2D(32, 32, 3, 2),
            fc4=L.Linear(None, 100),
            fc5=L.Linear(100, n_out)
        )

    def __call__(self, x):
        h = F.relu(self.conv1(x))
        h = F.relu(self.conv2(h))
        h = F.relu(self.conv3(h))
        h = F.relu(self.fc4(h))
        h = self.fc5(h)
        return h

import chainer

import chainer.functions as F

import chainer.links as L

class CNNSmall(chainer.Chain):

def __init__(self, n_out):

super(CNNSmall, self).__init__(

conv1=L.Convolution2D(None, 16, 3, 2),

conv2=L.Convolution2D(16, 32, 3, 2),

conv3=L.Convolution2D(32, 32, 3, 2),

fc4=L.Linear(None, 100),

fc5=L.Linear(100, n_out)

)

def __call__(self, x):

h = F.relu(self.conv1(x))

h = F.relu(self.conv2(h))

h = F.relu(self.conv3(h))

h = F.relu(self.fc4(h))

h = self.fc5(h)

return h

I also made a slightly bigger CNN, called CNNMedium,

import chainer
import chainer.functions as F
import chainer.links as L


class CNNMedium(chainer.Chain):
    def __init__(self, n_out):
        super(CNNMedium, self).__init__(
            conv1=L.Convolution2D(None, 16, 3, 1),
            conv2=L.Convolution2D(16, 32, 3, 2),
            conv3=L.Convolution2D(32, 32, 3, 1),
            conv4=L.Convolution2D(32, 64, 3, 2),
            conv5=L.Convolution2D(64, 64, 3, 1),
            conv6=L.Convolution2D(64, 128, 3, 2),
            fc7=L.Linear(None, 100),
            fc8=L.Linear(100, n_out)
        )

    def __call__(self, x):
        h = F.relu(self.conv1(x))
        h = F.relu(self.conv2(h))
        h = F.relu(self.conv3(h))
        h = F.relu(self.conv4(h))
        h = F.relu(self.conv5(h))
        h = F.relu(self.conv6(h))
        h = F.relu(self.fc7(h))
        h = self.fc8(h)
        return h

import chainer

import chainer.functions as F

import chainer.links as L

class CNNMedium(chainer.Chain):

def __init__(self, n_out):

super(CNNMedium, self).__init__(

conv1=L.Convolution2D(None, 16, 3, 1),

conv2=L.Convolution2D(16, 32, 3, 2),

conv3=L.Convolution2D(32, 32, 3, 1),

conv4=L.Convolution2D(32, 64, 3, 2),

conv5=L.Convolution2D(64, 64, 3, 1),

conv6=L.Convolution2D(64, 128, 3, 2),

fc7=L.Linear(None, 100),

fc8=L.Linear(100, n_out)

)

def __call__(self, x):

h = F.relu(self.conv1(x))

h = F.relu(self.conv2(h))

h = F.relu(self.conv3(h))

h = F.relu(self.conv4(h))

h = F.relu(self.conv5(h))

h = F.relu(self.conv6(h))

h = F.relu(self.fc7(h))

h = self.fc8(h)

return h

It is nice to know the computational cost for Convolution layer, which is approximated as, $$ H_I \times W_I \times CH_I \times CH_O \times kernel_size ^ 2 $$ $ CH_I $ : Input image channel $ CH_O $ : Output image channel $ H_I $ : Input image height $ W_I $ : Input image width $ k $ : kernal size (assuming same for width & height) In above CNN definitions, the size of the channel is bigger for […]

Continue reading →

CIFAR-10, CIFAR-100 dataset introduction

Updated on April 24, 2017 by corochann · Leave a comment

Source code is uploaded on github. CIFAR-10 and CIFAR-100 are the small image datasets with its classification labeled. It is widely used for easy image classification task/benchmark in research community. Official page: CIFAR-10 and CIFAR-100 datasets In Chainer, CIFAR-10 and CIFAR-100 dataset can be obtained with build-in function. Setup code:

from __future__ import print_function
import os
import matplotlib.pyplot as plt
%matplotlib inline

import numpy as np

import chainer

basedir = './src/cnn/images'

from __future__ import print_function

import os

import matplotlib.pyplot as plt

%matplotlib inline

import numpy as np

import chainer

basedir = './src/cnn/images'

CIFAR-10 chainer.datasets.get_cifar10 method is prepared in Chainer to get CIFAR-10 dataset. Dataset is automatically downloaded from https://www.cs.toronto.edu only for the first time, and its cache is used from second time.

CIFAR10_LABELS_LIST = [
    'airplane', 
    'automobile',
    'bird',
    'cat',
    'deer',
    'dog',
    'frog',
    'horse',
    'ship',
    'truck'
]

train, test = chainer.datasets.get_cifar10()

CIFAR10_LABELS_LIST = [

'airplane',

'automobile',

'bird',

'cat',

'deer',

'dog',

'frog',

'horse',

'ship',

'truck'

]

train, test = chainer.datasets.get_cifar10()

The dataset structure is quite same with MNIST dataset, it is TupleDataset.train[i] represents i-th data, there are 50000 training data.test data structure is same, with 10000 […]

Continue reading →

Understanding convolutional layer

Updated on April 24, 2017 by corochann · Leave a comment

Source code is uploaded on github.The sample image is obtained from PEXELS. What is the difference between convolutional layer and linear layer? What kind of intuition is in behind of using convolutional layer in deep neural network? This hands on shows some effects by convolutional layer to provide some intution about what convolutional layer do.

import os

import numpy as np
import matplotlib.pyplot as plt
import cv2

%matplotlib inline

basedir = './src/cnn/images'


def read_rgb_image(imagepath):
    image = cv2.imread(imagepath)  # Height, Width, Channel
    (major, minor, _) = cv2.__version__.split(".")
    if major == '3':
        # version 3 is used, need to convert
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    else:
        # Version 2 is used, not necessary to convert
        pass
    return image


def read_gray_image(imagepath):
    image = cv2.imread(imagepath)  # Height, Width, Channel
    (major, minor, _) = cv2.__version__.split(".")
    if major == '3':
        # version 3 is used, need to convert
        image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    else:
        # Version 2 is used, not necessary to convert
        image = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
    return image


def plot_channels(array, filepath='out.jpg'):
    """Plot each channel component separately

    Args:
        array (numpy.ndarray): 3-D array (width, height, channel)

    """
    ch_number = array.shape[2]

    fig, axes = plt.subplots(1, ch_number)
    for i in range(ch_number):
        # Save each image
        # cv2.imwrite(os.path.join(basedir, 'output_conv1_{}.jpg'.format(i)), array[:, :, i])
        axes[i].set_title('Channel {}'.format(i))
        axes[i].axis('off')
        axes[i].imshow(array[:, :, i], cmap='gray')

    plt.savefig(filepath)

import os

import numpy as np

import matplotlib.pyplot as plt

import cv2

%matplotlib inline

basedir = './src/cnn/images'

def read_rgb_image(imagepath):

image = cv2.imread(imagepath) # Height, Width, Channel

(major, minor, _) = cv2.__version__.split(".")

if major == '3':

# version 3 is used, need to convert

image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

else:

# Version 2 is used, not necessary to convert

pass

return image

def read_gray_image(imagepath):

image = cv2.imread(imagepath) # Height, Width, Channel

(major, minor, _) = cv2.__version__.split(".")

if major == '3':

# version 3 is used, need to convert

image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

else:

# Version 2 is used, not necessary to convert

image = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)

return image

def plot_channels(array, filepath='out.jpg'):

"""Plot each channel component separately

Args:

array (numpy.ndarray): 3-D array (width, height, channel)

"""

ch_number = array.shape[2]

fig, axes = plt.subplots(1, ch_number)

for i in range(ch_number):

# Save each image

# cv2.imwrite(os.path.join(basedir, 'output_conv1_{}.jpg'.format(i)), array[:, :, i])

axes[i].set_title('Channel {}'.format(i))

axes[i].axis('off')

axes[i].imshow(array[:, :, i], cmap='gray')

plt.savefig(filepath)

Above type of diagram often appears in Convolutional neural network field. Below figure explains its notation. Cuboid represents the “image” array where this image might not mean the meaningful picture. Horizontal axis represents channel number, vertical axis for image height and depth axis for image width respectively. Convolution layer – […]

Continue reading →

Basic image processing tutorial

Updated on April 24, 2017 by corochann · Leave a comment

Basic image processing for deep learning. Refer github for the source code. The sample image is obtained from PEXELS. If you are not familiar with image processing, you can read this article before going to convolutional neural network. OpenCV is image processing library which supports loading image in numpy.ndarray format, save image converting image color format (RGB, YUV, Gray scale etc) resize and other useful image processing functionality. To install opencv, execute $conda install -c https://conda.binstar.org/menpo -y opencv3

import os

import matplotlib.pyplot as plt
import cv2

%matplotlib inline

def readRGBImage(imagepath):
    image = cv2.imread(imagepath)  # Height, Width, Channel
    (major, minor, _) = cv2.__version__.split(".")
    if major == '3':
        # version 3 is used, need to convert
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    else:
        # Version 2 is used, not necessary to convert
        pass
    return image

import os

import matplotlib.pyplot as plt

import cv2

%matplotlib inline

def readRGBImage(imagepath):

image = cv2.imread(imagepath) # Height, Width, Channel

(major, minor, _) = cv2.__version__.split(".")

if major == '3':

# version 3 is used, need to convert

image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

else:

# Version 2 is used, not necessary to convert

pass

return image

Loading and save image cv2.imread for loading image. cv2.imwrite for save image. plt.imshow for plotting, and plt.savefig for save plot image. OpenCV image format is usually 3 dimension (or 2 dimension if […]

Continue reading →

MNIST inference code

Updated on March 12, 2017 by corochann · Leave a comment

We already learned how to write training code in chainer, the last task is to use this trained model to inference (predict) the test input MNIST image. Inference code structure usually becomes as follows, Prepare input data Instantiate the trained model Load the trained model Feed input data into loaded model to get inference result You have already learned the necessary stuff, and it is easy. See inference_mnist.py for the source code. Prepare input data For MNIST, it is easy in one line

    # Load the MNIST dataset
    train, test = chainer.datasets.get_mnist()

1 2	# Load the MNIST dataset train, test = chainer.datasets.get_mnist()

Instantiate the trained model and load the model

    # Load trained model
    model = mlp.MLP(args.unit, 10)
    if args.gpu >= 0:
        chainer.cuda.get_device(args.gpu).use()  # Make a specified GPU current
        model.to_gpu()  # Copy the model to the GPU
    xp = np if args.gpu < 0 else cuda.cupy

    serializers.load_npz(args.modelpath, model)

# Load trained model

model = mlp.MLP(args.unit, 10)

if args.gpu >= 0:

chainer.cuda.get_device(args.gpu).use() # Make a specified GPU current

model.to_gpu() # Copy the model to the GPU

xp = np if args.gpu < 0 else cuda.cupy

serializers.load_npz(args.modelpath, model)

Here, note that model can be loaded after instantiating the model. This model must have […]

Continue reading →

Chainer family

Updated on March 11, 2017 by corochann · Leave a comment

Recently several sub-libraries for Chainer are released, ChainerRL RL: Reinforcement Learning Deep Reinforcement Learning library. cite from http://chainer.org/general/2017/02/22/ChainerRL-Deep-Reinforcement-Learning-Library.html Recent state-of-the-art deep reinforcement algorithms are implemented, including A3C (Asynchronous Advantage Actor-Critic) ACER (Actor-Critic with Experience Replay) (only the discrete-action version for now) Asynchronous N-step Q-learning DQN (including Double DQN, Persistent Advantage Learning (PAL), Double PAL, Dynamic Policy Programming (DPP)) DDPG (Deep Deterministic Poilcy Gradients) (including SVG(0)) PGT (Policy Gradient Theorem) How to install

pip install chainerrl

1	pip install chainerrl

github repository ChainerRL – Deep Reinforcement Learning Library ChainerCV CV: Computer Vision Image processing library for deep learning training. Common data-augmentation are implemented. How to install

pip install chainercv

1	pip install chainercv

github repository document ChainerMN MN: Multi Node […]

Continue reading →

Chainer version 2 – updated part

Updated on March 11, 2017 by corochann · Leave a comment

Chainer version 2 is planned to be released in Apr 2017. Pre-release version is already available, install by this command

$ pip install chainer --pre
$ pip install cupy

1 2	$ pip install chainer --pre $ pip install cupy

The biggest change is that cupy (Roughly, it is GPU version of numpy) becomes independent, and provided separately. Reference Chainer v2 alpha from Seiya Tokui

Continue reading →

Writing organized, reusable, clean training code using Trainer module

Updated on April 2, 2017 by corochann · Leave a comment

Training code abstraction with Trainer Until now, I was implementing the training code in “primitive” way to explain what kind of operations are going on in deep learning training (※). However, the code can be written in much clean way using Trainer modules in Chainer. ※ Trainer modules are implemented from version 1.11, and some of the open source projects are implemented without Trainer. So it helps to understand these codes by knowing the training implementation without Trainer module as well. Motivation for using Trainer We can notice there are many “typical” operations widely used in machine learning, for example Iterating minibatch training, with minibatch sampled ramdomly Separate train […]

Continue reading →

Design patterns for defining model

Updated on March 10, 2017 by corochann · Leave a comment

Machine learning consists of training phase and predict/inference phase, and what model need to calculate is different Training phase: calculate loss (between on output and target) Predict/Inference phase: calculate output To manage this, I often see below 2 patterns to manage this. Predictor – Classifier framework See train_mnist_2_predictor_classifier.py (train_mnist_1_minimum.py and train_mnist_4_trainer.py are also implemented in Predictor – Classifier framework) 2 Chain classes, “Predictor” and “Classifier” are used for this framework. Training phase: Predictor’s output is fed into Classifier to calculate loss. Predict/Inference phase: Only predictor’s output is used. Predictor Predictor simply calculates output based on input.

# Network definition
class MLP(chainer.Chain):

    def __init__(self, n_units, n_out):
        super(MLP, self).__init__(
            # the size of the inputs to each layer will be inferred
            l1=L.Linear(None, n_units),  # n_in -> n_units
            l2=L.Linear(None, n_units),  # n_units -> n_units
            l3=L.Linear(None, n_out),  # n_units -> n_out
        )

    def __call__(self, x):
        h1 = F.relu(self.l1(x))
        h2 = F.relu(self.l2(h1))
        return self.l3(h2)

# Network definition

class MLP(chainer.Chain):

def __init__(self, n_units, n_out):

super(MLP, self).__init__(

# the size of the inputs to each layer will be inferred

l1=L.Linear(None, n_units), # n_in -> n_units

l2=L.Linear(None, n_units), # n_units -> n_units

l3=L.Linear(None, n_out), # n_units -> n_out

)

def __call__(self, x):

h1 = F.relu(self.l1(x))

h2 = F.relu(self.l2(h1))

return self.l3(h2)

model = mlp.MLP(args.unit, 10)

1	model = mlp.MLP(args.unit, 10)

Classifier Classifier “wraps” predictors output y to […]

Continue reading →

corochannNote

Author: corochann

CIFAR-10, CIFAR-100 inference code

CIFAR-10, CIFAR-100 training with Convolutional Neural Network

CIFAR-10, CIFAR-100 dataset introduction

Understanding convolutional layer

Basic image processing tutorial

MNIST inference code

Chainer family

Chainer version 2 – updated part

Writing organized, reusable, clean training code using Trainer module

Design patterns for defining model