Visualizing Convet Filters

In this notebook, I will follow F. Chollet's technique to vizualize convnet filters on the Inception v3 pretrained convnet.

This technique aims at vizualizing the visual pattern that each filter responds to. To proceed, I will apply a gradient descent to the value of the input image to maximize the response of a specific filter.

%matplotlib inline
from keras.applications import VGG16
from keras import backend as K
import numpy as np
import matplotlib.pyplot as plt

model = VGG16(weights='imagenet',include_top = False)


model.summary()

_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_9 (InputLayer)         (None, None, None, 3)     0         
_________________________________________________________________
block1_conv1 (Conv2D)        (None, None, None, 64)    1792      
_________________________________________________________________
block1_conv2 (Conv2D)        (None, None, None, 64)    36928     
_________________________________________________________________
block1_pool (MaxPooling2D)   (None, None, None, 64)    0         
_________________________________________________________________
block2_conv1 (Conv2D)        (None, None, None, 128)   73856     
_________________________________________________________________
block2_conv2 (Conv2D)        (None, None, None, 128)   147584    
_________________________________________________________________
block2_pool (MaxPooling2D)   (None, None, None, 128)   0         
_________________________________________________________________
block3_conv1 (Conv2D)        (None, None, None, 256)   295168    
_________________________________________________________________
block3_conv2 (Conv2D)        (None, None, None, 256)   590080    
_________________________________________________________________
block3_conv3 (Conv2D)        (None, None, None, 256)   590080    
_________________________________________________________________
block3_pool (MaxPooling2D)   (None, None, None, 256)   0         
_________________________________________________________________
block4_conv1 (Conv2D)        (None, None, None, 512)   1180160   
_________________________________________________________________
block4_conv2 (Conv2D)        (None, None, None, 512)   2359808   
_________________________________________________________________
block4_conv3 (Conv2D)        (None, None, None, 512)   2359808   
_________________________________________________________________
block4_pool (MaxPooling2D)   (None, None, None, 512)   0         
_________________________________________________________________
block5_conv1 (Conv2D)        (None, None, None, 512)   2359808   
_________________________________________________________________
block5_conv2 (Conv2D)        (None, None, None, 512)   2359808   
_________________________________________________________________
block5_conv3 (Conv2D)        (None, None, None, 512)   2359808   
_________________________________________________________________
block5_pool (MaxPooling2D)   (None, None, None, 512)   0         
=================================================================
Total params: 14,714,688
Trainable params: 14,714,688
Non-trainable params: 0
_________________________________________________________________

Settings

Defining the Loss Tensor for Filter Vizualization

x = [l for l in model.layers]

layer_name = x[1].name
filter_index = 0

layer_output = model.get_layer(layer_name).output
loss = K.mean(layer_output[:,:,:,filter_index])

Obtaining the Gradient of the Loss

#K.gradients return a list of tensor
grads = K.gradients(loss,model.input)[0]

#Let us apply the "tensor trick"
grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)

Let us create a function which computes the value of the loss tensor and the gradient tensor for a given input image.

iterate = K.function([model.input],[loss,grads])

loss_value, grads_value = iterate([np.zeros((1,150,150,3))])

We can now create a loop to perform the stochastic gradient descent

input_img_data = np.random.random((1,150,150,3)) * 20 + 128.

step = 1
for i in range(40):
    loss_value, grads_value = iterate([(input_img_data)])
    input_img_data += grads_value * step

We can now deprocess the image

def deprocess_image(x):
    #Normalize image
    x -= x.mean()
    x /= (x.std() + 1e-5)
    x *= 0.1
    #Clip to 0/1
    x += 0.5
    x = np.clip(x,0,1)
    #Convert to RGB
    x *= 255
    x = np.clip(x,0,255).astype('uint8')
    return x
    

Let us arrange this altogether

def generate_pattern(layer_name, filter_index,size=150):
    layer_output = model.get_layer(layer_name).output
    loss = K.mean(layer_output[:,:,:,filter_index])
    grads = K.gradients(loss,model.input)[0]
    grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)
    
    iterate = K.function([model.input],[loss,grads])
    
    input_img_data = np.random.random((1,size,size,3)) * 20 + 128.
    
    step = 1.
    for i in range(40):
        loss_value, grads_value = iterate([input_img_data])
        input_img_data += grads_value * step
        
    img = input_img_data[0]
    return deprocess_image(img)

plt.imshow(generate_pattern(layer_name,0))

<matplotlib.image.AxesImage at 0x7fdf480b75f8>

Visualizing More Filters

Now, let us visualize more filters for more layers

size = 64
margin = 5

results = np.zeros((8 * size + 7 * margin, 8 * size + 7 * margin, 3))

for i in range(8):
    for j in range(8):
        filter_img = generate_pattern(layer_name, i + (j * 8), size = size)
        horizontal_start = i * size + i * margin
        horizontal_end = horizontal_start + size
        vertical_start = j * size + j * margin
        vertical_end = vertical_start + size
        results[horizontal_start : horizontal_end,
               vertical_start:vertical_end,:] = filter_img
        
plt.figure(figsize=(20,20))
plt.imshow(results)

<matplotlib.image.AxesImage at 0x7fdf42af7940>