{"cells":[{"metadata":{"_uuid":"b22afe81ac54e0ccd27469156bf4b3610baa8eec"},"cell_type":"markdown","source":"In this notebook, I extract and visualize features from VGG Net using two methods. VGG, Inception, ResNet have been trained on ImageNet which is a very large\ndatabase of images. We expect the extracted features from these nets  to encode some important image properties that may help in faster training. You should definitely go through this seminal paper by  [Zeiler et al.](https://arxiv.org/abs/1311.2901) to gain some insights.\n Also, you may want to experiment with different combination of features and different nets to find the best set of features.  Happy Kaggling :-)"},{"metadata":{"_uuid":"6b025c2a5c0b47a7c3cbe5b36caf5a8ef907aaec"},"cell_type":"markdown","source":"<h2> Load required libraries"},{"metadata":{"trusted":true,"_uuid":"00833d394e3069216af171fd979c814e7e1e430d","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"import os\nimport sys\nimport random\nimport warnings\n\nimport scipy\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport cv2\nfrom skimage import exposure\nimport pandas as pd\n\nfrom skimage.transform import resize\nfrom skimage.morphology import label\nfrom keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img\nfrom keras import backend as K\nfrom keras.applications.vgg16 import VGG16\nfrom keras.applications.xception import Xception\nfrom keras.preprocessing import image\nfrom keras.applications.vgg16 import preprocess_input\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2899128a79962671a7d948aead0dc0dd7ff64b09"},"cell_type":"markdown","source":"<h2> Some helper functions and values"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d20a26a4220017632d26c7541fd93c5d7690bc2a"},"cell_type":"code","source":"df = pd.read_csv('../input/train_ship_segmentations.csv')\ntrain_ids = df.ImageId.values\ndf = df.set_index('ImageId')\n# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\ndef rle_decode(mask_rle, shape=(768, 768)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T  # Needed to align to RLE direction","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c26cc4b1dc309eaf184298126a1e0d658f722fe0"},"cell_type":"markdown","source":"<h2> Download the Model Weights \n"},{"metadata":{"trusted":true,"_uuid":"5133f5b54b192a103c3ea3e164ff46eede0552e9"},"cell_type":"code","source":"model = VGG16(weights='imagenet', include_top=False)\nmodel.summary()\ninp = model.input                                           # input placeholder\noutputs = [layer.output for layer in model.layers]          # all layer outputs\nfunctors = [K.function([inp]+ [K.learning_phase()], [out]) for out in outputs]  # evaluation functions","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b313ad513bdcf74c267f0adfef596a26f51956c1"},"cell_type":"markdown","source":"<h2> Visualizing features: The Naive Way"},{"metadata":{"trusted":true,"_uuid":"97fbc430deaf14766f0a284e51cb8c8ce802bca6"},"cell_type":"code","source":"# vgg features that can be easily extracted and used for training deep networks\n# these features may be used along with original image\nids = os.listdir('../input/train')\nnp.random.seed(1)\n\nif df.index.name == 'ImageId':\n    df = df.reset_index()\nif df.index.name != 'ImageId':\n    df = df.set_index('ImageId')\n    \nidx = random.randint(0, len(ids))\nplt.figure(figsize=(30,15))\nplt.subplots_adjust(bottom=0.2, top=1.2)  #adjust this to change vertical and horiz. spacings..\nnImg = 5  #no. of images to process\ntemp = nImg\nj = 0\ncounter = 0\n# for j, img_name in enumerate(ids[idx:idx+temp]):\nwhile j < nImg:\n    q = j+1\n    img_name = ids[counter]\n    counter+=1\n    if str(df.loc[img_name,'EncodedPixels'])!=str(np.nan):   \n        j+=1\n        all_masks = np.zeros((768, 768))\n        try:\n            img_masks = df.loc[img_name,'EncodedPixels'].tolist()\n            for mask in img_masks:\n                all_masks += rle_decode(mask)\n            all_masks = np.expand_dims(all_masks,axis=2)\n            all_masks = np.repeat(all_masks,3,axis=2).astype('uint8')*255\n\n        except Exception as e:\n            all_masks = rle_decode(df.loc[img_name,'EncodedPixels'])\n            all_masks = np.expand_dims(all_masks,axis=2)*255\n            all_masks = np.repeat(all_masks,3,axis=2).astype('uint8')\n    else:\n        continue\n#         all_masks = np.zeros((224,224))\n\n    \n    img = image.load_img('../input/train/' + img_name, target_size=(224, 224))\n    x = image.img_to_array(img)\n    x = np.expand_dims(x, axis=0)\n    x = preprocess_input(x)\n\n    \n    layer_outs = [func([x, 0.]) for func in functors]\n    feat = np.reshape(layer_outs[4][0],(112,112,128))\n    layer4 = np.max(feat,axis=2)\n    \n    feat = np.reshape(layer_outs[6][0],(56,56,128))\n    layer6 = np.max(feat,axis=2)\n    \n    feat = np.reshape(layer_outs[10][0],(28,28,256))\n    layer10 = np.max(feat,axis=2)\n    \n    plt.subplot(nImg,6,q*6-5)\n    plt.imshow(img)\n    plt.title('Original Image')\n    \n    plt.subplot(nImg,6,q*6-4)\n    plt.imshow(all_masks)\n    plt.title('Image Mask')\n    \n    plt.subplot(nImg,6,q*6-3)    \n    plt.imshow(layer4)\n    plt.title('VGG Layer 4')\n    \n    plt.subplot(nImg,6,q*6-2)\n    plt.imshow(layer6)\n    plt.title('VGG Layer 6')\n    \n    plt.subplot(nImg,6,q*6-1)\n    plt.imshow(layer10)\n    plt.title('VGG Layer 10')\n\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea8e047689ad5b8b416c38c110532b796ec161be","_kg_hide-input":true,"collapsed":true},"cell_type":"code","source":"# please go through https://blog.keras.io/how-convolutional-neural-networks-see-the-world.html for more details\n\ndef preprocess_image(image_path):\n    # Util function to open, resize and format pictures\n    # into appropriate tensors.\n    img = load_img(image_path)\n    img = img_to_array(img)\n    img = np.expand_dims(img, axis=0)    \n    img = preprocess_input(img)\n    return img\n\n\nK.set_learning_phase(0)\n\n# Build the InceptionV3 network with our placeholder.\n# The model will be loaded with pre-trained ImageNet weights.\n# model = inception_v3.InceptionV3(weights='imagenet',\n#                                  include_top=False)\nmodel = VGG16(weights='imagenet',include_top=False)\ndream = model.input\nprint('Model loaded.')\n\n# Get the symbolic outputs of each \"key\" layer (we gave them unique names).\nlayer_dict = dict([(layer.name, layer) for layer in model.layers])\n\n# you can play with different settings...\nsettings = {\n    'features': {\n        'block1_pool': 0.5,\n        'block2_pool':0.4,\n        'block3_pool':0.1,\n    },\n}\n\n# Define the loss.\nloss = K.variable(0.)\nfor layer_name in settings['features']:\n    # Add the L2 norm of the features of a layer to the loss.\n    assert layer_name in layer_dict.keys(), 'Layer ' + layer_name + ' not found in model.'\n    coeff = settings['features'][layer_name]\n    x = layer_dict[layer_name].output\n    # We avoid border artifacts by only involving non-border pixels in the loss.\n    scaling = K.prod(K.cast(K.shape(x), 'float32'))\n    if K.image_data_format() == 'channels_first':\n        loss += coeff * K.sum(K.square(x[:, :, 2: -2, 2: -2])) / scaling\n    else:\n        loss += coeff * K.sum(K.square(x[:, 2: -2, 2: -2, :])) / scaling\n\n# Compute the gradients of the dream wrt the loss.\ngrads = K.gradients(loss, dream)[0]\n# Normalize gradients.\ngrads /= K.maximum(K.mean(K.abs(grads)), K.epsilon())\n\n# Set up function to retrieve the value\n# of the loss and gradients given an input image.\noutputs = [loss, grads]\nfetch_loss_and_grads = K.function([dream], outputs)\n\n\ndef eval_loss_and_grads(x):\n    outs = fetch_loss_and_grads([x])\n    loss_value = outs[0]\n    grad_values = outs[1]\n    return loss_value, grad_values\n\n\ndef resize_img(img, size):\n    img = np.copy(img)\n    if K.image_data_format() == 'channels_first':\n        factors = (1, 1,\n                   float(size[0]) / img.shape[2],\n                   float(size[1]) / img.shape[3])\n    else:\n        factors = (1,\n                   float(size[0]) / img.shape[1],\n                   float(size[1]) / img.shape[2],\n                   1)\n    return scipy.ndimage.zoom(img, factors, order=1)\n\n\ndef gradient_ascent(x, iterations, step, max_loss=None):\n    for i in range(iterations):\n        loss_value, grad_values = eval_loss_and_grads(x)\n        if max_loss is not None and loss_value > max_loss:\n            break\n#         print('..Loss value at', i, ':', loss_value)\n        x += step * grad_values\n    return x\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"59b94e1905a0273befede26be692447feedf2f60"},"cell_type":"markdown","source":"<h2> Visualizing features: The Deep Dream Way"},{"metadata":{"trusted":true,"_uuid":"28534da6f41c3dcad359d10943804ac39f54b634","collapsed":true},"cell_type":"code","source":"# Playing with these hyperparameters will also allow you to achieve new effects\nstep = 0.01  # Gradient ascent step size\nnum_octave = 3  # Number of scales at which to run gradient ascent\noctave_scale = 1.4  # Size ratio between scales\niterations = 100  # Number of ascent steps per scale\nmax_loss = 10.\n\nids = os.listdir('../input/train/')\nnp.random.seed(42)\nidx = np.random.randint(0, len(ids)//5)\nplt.figure(figsize=(30,15))\nplt.subplots_adjust(bottom=0.2, top=0.8, hspace=0)  #adjust this to change vertical and horiz. spacings..\nnImg = 5  #no. of images to process\nprocessed_img = []\norig_img = []\ncounter = 0\nj = 0\nwhile j < nImg:\n# for j, img_name in enumerate(ids[idx:idx+nImg]):\n    q = j+1\n    img_name = ids[idx+counter]\n    counter+=1\n    if str(df.loc[img_name,'EncodedPixels'])==str(np.nan):\n        continue\n    else:\n        j+=1\n#     img = load_img('../input/train/images/' + img_name, grayscale=True)\n    base_image_path = '../input/train/' + img_name\n    img = image.load_img(base_image_path)\n    img = image.img_to_array(img)\n    t = img.copy()\n    img = np.expand_dims(img, axis=0)\n    img = preprocess_input(img)\n\n#     img = preprocess_image(base_image_path)\n    original_shape = img.shape[1:3]\n\n    successive_shapes = [original_shape]\n    for i in range(1, num_octave):\n        shape = tuple([int(dim / (octave_scale ** i)) for dim in original_shape])\n        successive_shapes.append(shape)\n    successive_shapes = successive_shapes[::-1]\n    original_img = np.copy(img)\n    shrunk_original_img = resize_img(img, successive_shapes[0])\n\n    for shape in successive_shapes:\n        print('Processing image shape', shape)\n        img = resize_img(img, shape)\n        img = gradient_ascent(img,\n                              iterations=iterations,\n                              step=step,\n                              max_loss=max_loss)\n        upscaled_shrunk_original_img = resize_img(shrunk_original_img, shape)\n        same_size_original = resize_img(original_img, shape)\n        lost_detail = same_size_original - upscaled_shrunk_original_img\n\n        img += lost_detail\n        shrunk_original_img = resize_img(original_img, shape)    \n        processed_img.append(img)\n    orig_img.append(t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6cfaebd929b671cdf00010b763923e2385474a01","_kg_hide-input":true,"collapsed":true},"cell_type":"code","source":"plt.figure(figsize=(30,15))\nplt.subplots_adjust(bottom=0.2, top=1.2)  #adjust this to change vertical and horiz. spacings..\nfor i in range(nImg):\n    q  = i + 1\n    plt.subplot(nImg,5,q*5-4)\n    img = orig_img[i]\n    plt.imshow(img)\n    plt.title('Original Image')\n    \n    plt.subplot(nImg,5,q*5-3)\n    img = processed_img[i*3]\n    plt.imshow(exposure.rescale_intensity(img[0,:,:,:],in_range=(0,1)))\n    plt.title('Scale 1 (391, 391)')\n    \n    plt.subplot(nImg,5,q*5-2)\n    img = processed_img[i*3+1]\n    plt.imshow(exposure.rescale_intensity(img[0,:,:,:],in_range=(0,1)))\n    plt.title('Scale 2 (548, 548)')\n    \n    plt.subplot(nImg,5,q*5-1)\n    img = processed_img[i*3+2]\n    plt.imshow(exposure.rescale_intensity(img[0,:,:,:],in_range=(0,1)))\n    plt.title('Scale 3 (768, 768)')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"be25d8b4cb0bba338414b091e78f390d12432e05"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}