First, we shall start with a very simple deep neural network, one containing only a single FC hidden layer (with ReLU activation) and a softmax FC layer, with no convolutional layer. The next screenshot shows the network upside down. The input is a flattened image containing 28 x 28 nodes and 1,024 nodes in the hidden layer and 10 output nodes, corresponding to each of the digits to be classified:

Now let's implement the deep learning image classification with TF. First, we need to load the mnist dataset and divide the training images into two parts, the first one being the larger (we use 50k images) for training, and the second one (10k images) to be used for validation. Let's reformat the labels to represent the image classes with one-hot encoded binary vectors. Then the tensorflow graph needs to be initialized along with the variable, constant, and placeholder tensors. A mini-batch stochastic gradient descent (SGD) optimizer is to be used as the learning algorithm with a batch size of 256, to minimize the softmax cross-entropy logit loss function with L2 regularizers on the couple of weights layers (with hyperparameter values of λ1=λ2=1). Finally, the TensorFlow session object will be run for 6k steps (mini-batches) and the forward/backpropagation will be run to update the model (weights) learned, with subsequent evaluation of the model on the validation dataset. As can be seen, the accuracy obtained after the final batch completes is 96.5%:

%matplotlib inline
import numpy as np
# import data
from keras.datasets import mnist
import tensorflow as tf

# load data
(X_train, y_train), (X_test, y_test) = mnist.load_data()
np.random.seed(0)
train_indices = np.random.choice(60000, 50000, replace=False)
valid_indices = [i for i in range(60000) if i not in train_indices]
X_valid, y_valid = X_train[valid_indices,:,:], y_train[valid_indices]
X_train, y_train = X_train[train_indices,:,:], y_train[train_indices]
print(X_train.shape, X_valid.shape, X_test.shape)
# (50000, 28, 28) (10000, 28, 28) (10000, 28, 28)
image_size = 28
num_labels = 10

def reformat(dataset, labels):
 dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32)
 # one hot encoding: Map 1 to [0.0, 1.0, 0.0 ...], 2 to [0.0, 0.0, 1.0 ...] 
 labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)
 return dataset, labels

X_train, y_train = reformat(X_train, y_train)
X_valid, y_valid = reformat(X_valid, y_valid)
X_test, y_test = reformat(X_test, y_test)
print('Training set', X_train.shape, X_train.shape)
print('Validation set', X_valid.shape, X_valid.shape)
print('Test set', X_test.shape, X_test.shape)
# Training set (50000, 784) (50000, 784) # Validation set (10000, 784) (10000, 784) # Test set (10000, 784) (10000, 784)

def accuracy(predictions, labels):
 return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1)) / predictions.shape[0])

batch_size = 256 
num_hidden_units = 1024
lambda1 = 0.1 
lambda2 = 0.1 
 
graph = tf.Graph()
with graph.as_default():

 # Input data. For the training data, we use a placeholder that will be fed
 # at run time with a training minibatch.
 tf_train_dataset = tf.placeholder(tf.float32,
 shape=(batch_size, image_size * image_size))
 tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
 tf_valid_dataset = tf.constant(X_valid)
 tf_test_dataset = tf.constant(X_test)
 
 # Variables.
 weights1 = tf.Variable(tf.truncated_normal([image_size * image_size, num_hidden_units]))
 biases1 = tf.Variable(tf.zeros([num_hidden_units]))

 # connect inputs to every hidden unit. Add bias
 layer_1_outputs = tf.nn.relu(tf.matmul(tf_train_dataset, weights1) + biases1)

 weights2 = tf.Variable(tf.truncated_normal([num_hidden_units, num_labels]))
 biases2 = tf.Variable(tf.zeros([num_labels])) 

 # Training computation.
 logits = tf.matmul(layer_1_outputs, weights2) + biases2
 loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits) + 
 lambda1*tf.nn.l2_loss(weights1) + lambda2*tf.nn.l2_loss(weights2)) 
 
 # Optimizer.
 optimizer = tf.train.GradientDescentOptimizer(0.008).minimize(loss)
 
 # Predictions for the training, validation, and test data.
 train_prediction = tf.nn.softmax(logits)
 layer_1_outputs = tf.nn.relu(tf.matmul(tf_valid_dataset, weights1) + biases1)
 valid_prediction = tf.nn.softmax(tf.matmul(layer_1_outputs, weights2) + biases2)
 layer_1_outputs = tf.nn.relu(tf.matmul(tf_test_dataset, weights1) + biases1)
 test_prediction = tf.nn.softmax(tf.matmul(layer_1_outputs, weights2) + biases2) 

num_steps = 6001

ll = []
atr = []
av = []

import matplotlib.pylab as pylab

with tf.Session(graph=graph) as session:
 #tf.global_variables_initializer().run()
 session.run(tf.initialize_all_variables())
 print("Initialized")
 for step in range(num_steps):
 # Pick an offset within the training data, which has been randomized.
 # Note: we could use better randomization across epochs.
 offset = (step * batch_size) % (y_train.shape[0] - batch_size)
 # Generate a minibatch.
 batch_data = X_train[offset:(offset + batch_size), :]
 batch_labels = y_train[offset:(offset + batch_size), :]
 # Prepare a dictionary telling the session where to feed the minibatch.
 # The key of the dictionary is the placeholder node of the graph to be fed,
 # and the value is the numpy array to feed to it.
 feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
 _, l, predictions = session.run([optimizer, loss, train_prediction], feed_dict=feed_dict)
 if (step % 500 == 0):
 ll.append(l)
 a = accuracy(predictions, batch_labels)
 atr.append(a)
 print("Minibatch loss at step %d: %f" % (step, l))
 print("Minibatch accuracy: %.1f%%" % a)
 a = accuracy(valid_prediction.eval(), y_valid)
 av.append(a)
 print("Validation accuracy: %.1f%%" % a) 
 print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), y_test))

# Initialized
# Minibatch loss at step 0: 92091.781250
# Minibatch accuracy: 9.0%
# Validation accuracy: 21.6%
# 
# Minibatch loss at step 500: 35599.835938
# Minibatch accuracy: 50.4%
# Validation accuracy: 47.4%
# 
# Minibatch loss at step 1000: 15989.455078
# Minibatch accuracy: 46.5%
# Validation accuracy: 47.5%
# 
# Minibatch loss at step 1500: 7182.631836
# Minibatch accuracy: 59.0%
# Validation accuracy: 54.7%
# 
# Minibatch loss at step 2000: 3226.800781
# Minibatch accuracy: 68.4%
# Validation accuracy: 66.0%
# 
# Minibatch loss at step 2500: 1449.654785
# Minibatch accuracy: 79.3%
# Validation accuracy: 77.7%
# 
# Minibatch loss at step 3000: 651.267456
# Minibatch accuracy: 89.8%
# Validation accuracy: 87.7%
# 
# Minibatch loss at step 3500: 292.560272
# Minibatch accuracy: 94.5%
# Validation accuracy: 91.3%
# 
# Minibatch loss at step 4000: 131.462219
# Minibatch accuracy: 95.3%
# Validation accuracy: 93.7%
# 
# Minibatch loss at step 4500: 59.149700
# Minibatch accuracy: 95.3%
# Validation accuracy: 94.3%
# 
# Minibatch loss at step 5000: 26.656094
# Minibatch accuracy: 94.9%
# Validation accuracy: 95.5%
# 
# Minibatch loss at step 5500: 12.033947
# Minibatch accuracy: 97.3%
# Validation accuracy: 97.0%
# 
# Minibatch loss at step 6000: 5.521026
# Minibatch accuracy: 97.3%
# Validation accuracy: 96.6%
# 
# Test accuracy: 96.5%

Let's visualize the layer 1 weights at each step with the following code block:

images = weights1.eval()
pylab.figure(figsize=(18,18))
indices = np.random.choice(num_hidden_units, 225)
for j in range(225):
   pylab.subplot(15,15,j+1)
   pylab.imshow(np.reshape(images[:,indices[j]], (image_size,image_size)), cmap='gray')
   pylab.xticks([],[]), pylab.yticks([],[])
   pylab.subtitle('SGD after Step ' + str(step) + ' with lambda1=lambda2=' + str(lambda1))
pylab.show()

The preceding visualizes the weights learned for 225 (randomly chosen) hidden nodes in the FC layer 1 of the network after 4,000 steps. Observe that the weights are already learned some features from the input images the model was trained on:

The following code snippet plots the training accuracy and validation accuracy at different steps:

pylab.figure(figsize=(8,12))
pylab.subplot(211) 
pylab.plot(range(0,3001,500), atr, '.-', label='training accuracy')
pylab.plot(range(0,3001,500), av, '.-', label='validation accuracy')
pylab.xlabel('GD steps'), pylab.ylabel('Accuracy'), pylab.legend(loc='lower right')
pylab.subplot(212) 
pylab.plot(range(0,3001,500), ll, '.-')
pylab.xlabel('GD steps'), pylab.ylabel('Softmax Loss')
pylab.show()

The next screenshot shows the output of the previous code block; observe that the accuracies keep on increasing in general, but become almost constant in the end, which means learning no longer happens: