3.14. CIFAR10 CNN

Train a simple deep CNN on the CIFAR10 small images dataset.

Some constants we’ll use:

In [1]:
batch_size = 32
num_classes = 10
epochs = 200
data_augmentation = True
num_predictions = 20
In [2]:
from conx import *
conx, version 3.4.3
Using Theano backend.
In [3]:
net = Network("CIRAR10")
net.add(ImageLayer("input", (32, 32), 3)) # depends on K.image_data_format(), right?
net.add(Conv2DLayer("conv1", 32, (3, 3), padding='same', activation='relu'))
net.add(Conv2DLayer("conv2", 32, (3, 3), activation='relu'))
net.add(MaxPool2DLayer("pool1", pool_size=(2, 2), dropout=0.25))
net.add(Conv2DLayer("conv3", 64, (3, 3), padding='same', activation='relu'))
net.add(Conv2DLayer("conv4", 64, (3, 3), activation='relu'))
net.add(MaxPool2DLayer("pool2", pool_size=(2, 2), dropout=0.25))
net.add(FlattenLayer("flatten"))
net.add(Layer("hidden1", 512, activation='relu', vshape=(16, 32), dropout=0.5))
net.add(Layer("output", num_classes, activation='softmax'))
net.connect()

# initiate RMSprop optimizer
opt = RMSprop(lr=0.0001, decay=1e-6)

net.compile(loss='categorical_crossentropy',
            optimizer=opt)

# Let's train the model using RMSprop
net.compile(loss='categorical_crossentropy',
              optimizer=opt,
              metrics=['accuracy'])
model = net.model
In [58]:
net.dataset.get("cifar10")
In [59]:
net.dataset.summary()
Input Summary:
   count  : 60000 (10000 for training, 50000 for testing)
   shape  : [(32, 32, 3)]
   range  : (0.0, 1.0)
Target Summary:
   count  : 60000 (10000 for training, 50000 for testing)
   shape  : [(10,)]
   range  : (0.0, 1.0)
In [60]:
net.dataset._inputs[0].shape
Out[60]:
(60000, 32, 32, 3)
In [61]:
net.dataset.split(50000)
net.dataset.summary()
Input Summary:
   count  : 60000 (10000 for training, 50000 for testing)
   shape  : [(32, 32, 3)]
   range  : (0.0, 1.0)
Target Summary:
   count  : 60000 (10000 for training, 50000 for testing)
   shape  : [(10,)]
   range  : (0.0, 1.0)

3.14.1. Examine Input as Image

In [62]:
net.dataset.inputs.shape
Out[62]:
[(32, 32, 3)]
In [63]:
net.dataset.targets.shape
Out[63]:
[(10,)]
In [64]:
image = array2image(net.dataset.inputs[0], scale=5.0)
image
Out[64]:
_images/CIFAR10_CNN_12_0.png
In [65]:
net.dashboard()
In [66]:
net.propagate(net.dataset.inputs[1])
Out[66]:
[0.010080890730023384,
 0.41848239302635193,
 0.0075269537046551704,
 0.004321125335991383,
 0.0021690053399652243,
 0.003586387727409601,
 0.005634275730699301,
 0.005277491174638271,
 0.11723261326551437,
 0.42568886280059814]

3.14.2. Let Keras Take over from here

In [67]:
import keras
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
In [68]:
import numpy as np
In [34]:
net.train(plot=True)
_images/CIFAR10_CNN_18_0.svg
========================================================================
       |  Training |  Training |  Validate |  Validate
Epochs |     Error |  Accuracy |     Error |  Accuracy
------ | --------- | --------- | --------- | ---------
#    1 |   2.14204 |   0.20052 |   1.97837 |   0.29421
In [69]:
shape(x_test), shape(y_test)
Out[69]:
((1, 50000, 32, 32, 3), (1, 50000, 10))
In [70]:
model = net.model
In [71]:
(x_train, y_train), (x_test, y_test) = net.dataset._split_data()
In [72]:
shape(x_train), shape(y_train), shape(x_test), shape(y_test)
Out[72]:
((1, 10000, 32, 32, 3), (1, 10000, 10), (1, 50000, 32, 32, 3), (1, 50000, 10))
In [57]:
if not data_augmentation:
    print('Not using data augmentation.')
    model.fit(x_train, y_train,
              batch_size=batch_size,
              epochs=epochs,
              validation_data=(x_test, y_test),
              shuffle=True)
else:
    print('Using real-time data augmentation.')
    # This will do preprocessing and realtime data augmentation:
    datagen = ImageDataGenerator(
        featurewise_center=False,  # set input mean to 0 over the dataset
        samplewise_center=False,  # set each sample mean to 0
        featurewise_std_normalization=False,  # divide inputs by std of the dataset
        samplewise_std_normalization=False,  # divide each input by its std
        zca_whitening=False,  # apply ZCA whitening
        rotation_range=0,  # randomly rotate images in the range (degrees, 0 to 180)
        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)
        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)
        horizontal_flip=True,  # randomly flip images
        vertical_flip=False)  # randomly flip images

    # Compute quantities required for feature-wise normalization
    # (std, mean, and principal components if ZCA whitening is applied).
    datagen.fit(x_train[0])

    # Fit the model on the batches generated by datagen.flow().
    model.fit_generator(datagen.flow(x_train[0], y_train[0],
                                     batch_size=batch_size),
                        steps_per_epoch=x_train[0].shape[0] // batch_size,
                        epochs=epochs,
                        validation_data=(x_test, y_test))
Using real-time data augmentation.
Epoch 1/200
313/312 [==============================] - 219s 700ms/step - loss: 1.6980 - acc: 0.3732 - val_loss: 1.5751 - val_acc: 0.4305
Epoch 2/200
313/312 [==============================] - 225s 718ms/step - loss: 1.6487 - acc: 0.3901 - val_loss: 1.5533 - val_acc: 0.4324
Epoch 3/200
311/312 [============================>.] - ETA: 0s - loss: 1.6029 - acc: 0.4136
---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-57-1ff22741f6bb> in <module>()
     30                         steps_per_epoch=x_train[0].shape[0] // batch_size,
     31                         epochs=epochs,
---> 32                         validation_data=(x_test, y_test))

/usr/local/lib/python3.6/dist-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
     85                 warnings.warn('Update your `' + object_name +
     86                               '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 87             return func(*args, **kwargs)
     88         wrapper._original_function = func
     89         return wrapper

/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
   2142                                 batch_size=batch_size,
   2143                                 sample_weight=val_sample_weights,
-> 2144                                 verbose=0)
   2145                         if not isinstance(val_outs, list):
   2146                             val_outs = [val_outs]

/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in evaluate(self, x, y, batch_size, verbose, sample_weight, steps)
   1725                                batch_size=batch_size,
   1726                                verbose=verbose,
-> 1727                                steps=steps)
   1728
   1729     def predict(self, x,

/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in _test_loop(self, f, ins, batch_size, verbose, steps)
   1368                     ins_batch = _slice_arrays(ins, batch_ids)
   1369
-> 1370                 batch_outs = f(ins_batch)
   1371                 if isinstance(batch_outs, list):
   1372                     if batch_index == 0:

/usr/local/lib/python3.6/dist-packages/keras/backend/theano_backend.py in __call__(self, inputs)
   1221     def __call__(self, inputs):
   1222         assert isinstance(inputs, (list, tuple))
-> 1223         return self.function(*inputs)
   1224
   1225

/usr/local/lib/python3.6/dist-packages/theano/compile/function_module.py in __call__(self, *args, **kwargs)
    882         try:
    883             outputs =\
--> 884                 self.fn() if output_subset is None else\
    885                 self.fn(output_subset=output_subset)
    886         except Exception:

/usr/local/lib/python3.6/dist-packages/theano/ifelse.py in thunk()
    244         outputs = node.outputs
    245
--> 246         def thunk():
    247             if not compute_map[cond][0]:
    248                 return [0]

KeyboardInterrupt:
In [76]:
# Evaluate model with test data set and share sample prediction results
evaluation = model.evaluate_generator(datagen.flow(x_test[0], y_test[0],
                                      batch_size=batch_size),
                                      steps=x_test[0].shape[0] // batch_size)

print('Model Accuracy = %.2f' % (evaluation[1]))

predict_gen = model.predict_generator(datagen.flow(x_test[0], y_test[0],
                                      batch_size=batch_size),
                                      steps=x_test[0].shape[0] // batch_size)

for predict_index, predicted_y in enumerate(predict_gen):
    #actual_label = labels['label_names'][np.argmax(y_test[predict_index])]
    #predicted_label = labels['label_names'][np.argmax(predicted_y)]
    #print('Actual Label = %s vs. Predicted Label = %s' % (actual_label,
    #                                                      predicted_label))
    print('Actual Label = %s vs. Predicted Label = %s' % (y_test[0][predict_index],
                                                          predicted_y))
    if predict_index == num_predictions:
        break
Model Accuracy = 0.43
Actual Label = [ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.] vs. Predicted Label = [ 0.07781513  0.11049026  0.0632093   0.02424265  0.10382453  0.04574762
  0.03441789  0.42630658  0.02449505  0.08945103]
Actual Label = [ 0.  0.  0.  0.  0.  0.  1.  0.  0.  0.] vs. Predicted Label = [ 0.02727769  0.19264257  0.02373797  0.01587937  0.0090649   0.00988456
  0.06715904  0.01502883  0.06517704  0.57414806]
Actual Label = [ 0.  0.  0.  0.  0.  0.  1.  0.  0.  0.] vs. Predicted Label = [ 0.05388747  0.72588551  0.00831101  0.00158087  0.00303652  0.00182245
  0.00224504  0.00250599  0.10607026  0.0946549 ]
Actual Label = [ 0.  0.  0.  0.  0.  0.  0.  0.  1.  0.] vs. Predicted Label = [ 0.01729291  0.19587232  0.07844362  0.10607366  0.05843127  0.09728301
  0.20552364  0.0685864   0.01614707  0.15634608]
Actual Label = [ 0.  0.  0.  0.  0.  0.  0.  0.  1.  0.] vs. Predicted Label = [ 0.04557527  0.47399527  0.04536494  0.04711241  0.02142892  0.03853729
  0.02306257  0.03526938  0.04544082  0.22421315]
Actual Label = [ 0.  0.  0.  1.  0.  0.  0.  0.  0.  0.] vs. Predicted Label = [ 0.01227825  0.0398778   0.04749325  0.05364216  0.03648441  0.07180805
  0.13807741  0.3137106   0.00615273  0.28047535]
Actual Label = [ 0.  0.  0.  0.  1.  0.  0.  0.  0.  0.] vs. Predicted Label = [ 0.00148747  0.00222605  0.03411757  0.24753405  0.05446577  0.16512007
  0.44495872  0.04444287  0.00107558  0.00457185]
Actual Label = [ 0.  0.  0.  0.  0.  0.  1.  0.  0.  0.] vs. Predicted Label = [ 0.00741599  0.02212191  0.05528059  0.25858176  0.04116985  0.24418072
  0.209802    0.12378619  0.00374054  0.03392046]
Actual Label = [ 1.  0.  0.  0.  0.  0.  0.  0.  0.  0.] vs. Predicted Label = [ 0.00456545  0.00249086  0.09983079  0.03194337  0.23732299  0.03509691
  0.14823847  0.42950818  0.00175047  0.00925251]
Actual Label = [ 0.  0.  0.  0.  0.  0.  1.  0.  0.  0.] vs. Predicted Label = [ 0.01474431  0.00438144  0.08705295  0.17581116  0.1401801   0.10807402
  0.40283573  0.04501324  0.00754052  0.01436656]
Actual Label = [ 1.  0.  0.  0.  0.  0.  0.  0.  0.  0.] vs. Predicted Label = [ 0.20857221  0.03858071  0.11660673  0.02005922  0.05146967  0.01053343
  0.02061583  0.02458823  0.28485668  0.22411728]
Actual Label = [ 0.  0.  0.  1.  0.  0.  0.  0.  0.  0.] vs. Predicted Label = [ 0.04369825  0.00953077  0.28183886  0.03723752  0.39792955  0.04845857
  0.05786296  0.08070038  0.03028672  0.01245641]
Actual Label = [ 0.  0.  0.  0.  0.  0.  1.  0.  0.  0.] vs. Predicted Label = [ 0.03174659  0.14564584  0.01506849  0.00770176  0.00636712  0.00667131
  0.01012393  0.01783629  0.16324712  0.59559155]
Actual Label = [ 0.  0.  0.  0.  0.  0.  1.  0.  0.  0.] vs. Predicted Label = [ 0.01913743  0.42621526  0.00817715  0.00716725  0.0026362   0.00949341
  0.01235665  0.01647761  0.10527103  0.39306799]
Actual Label = [ 0.  0.  0.  0.  0.  1.  0.  0.  0.  0.] vs. Predicted Label = [ 0.28236881  0.04135361  0.1647729   0.03696084  0.15157856  0.03136211
  0.02248296  0.04669645  0.17347305  0.04895073]
Actual Label = [ 0.  0.  0.  0.  1.  0.  0.  0.  0.  0.] vs. Predicted Label = [ 0.05246869  0.45963278  0.00994109  0.00715726  0.0059837   0.0033786
  0.01398534  0.00770203  0.06684003  0.37291047]
Actual Label = [ 0.  0.  0.  0.  0.  0.  0.  0.  1.  0.] vs. Predicted Label = [ 0.04716396  0.04682264  0.11449952  0.13412473  0.13372231  0.10707654
  0.17844412  0.09313755  0.0549487   0.09005994]
Actual Label = [ 0.  0.  0.  1.  0.  0.  0.  0.  0.  0.] vs. Predicted Label = [ 0.03275985  0.08420437  0.18667556  0.0925509   0.07995293  0.05029331
  0.30199629  0.04829691  0.01449488  0.108775  ]
Actual Label = [ 0.  0.  1.  0.  0.  0.  0.  0.  0.  0.] vs. Predicted Label = [ 0.00613854  0.00442893  0.12136693  0.05807019  0.19370349  0.09075771
  0.36420116  0.14638041  0.00331027  0.0116424 ]
Actual Label = [ 0.  0.  0.  0.  0.  0.  1.  0.  0.  0.] vs. Predicted Label = [ 0.01641119  0.00571625  0.2858237   0.06443242  0.29150918  0.04093195
  0.22316463  0.05377265  0.00830103  0.00993697]
Actual Label = [ 1.  0.  0.  0.  0.  0.  0.  0.  0.  0.] vs. Predicted Label = [ 0.08868346  0.36420235  0.01048361  0.00159288  0.00550383  0.00113587
  0.00297236  0.00778512  0.1336575   0.38398302]