{"cells":[{"metadata":{"_uuid":"6db9c93d87fccc6ee46d456fbcc203f279f9b007"},"cell_type":"markdown","source":"**Initialization**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# coding: utf-8\n__author__ = 'ZFTurbo: https://kaggle.com/zfturbo'\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport random\nimport pickle\nfrom PIL import Image\nfrom os.path import isfile\nfrom IPython.display import Image, display\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"42cd6df028351057eb9ad175824d747854bfb681"},"cell_type":"markdown","source":"**Create model and load weights**"},{"metadata":{"trusted":true,"_uuid":"006a1eb5d039b00551b954d9c7e93aea93aa7b8a"},"cell_type":"code","source":"\nBOX_SIZE = 512\n\ndef preprocess_image(img):\n    from keras.applications.densenet import preprocess_input\n    return preprocess_input(img)\n\n\ndef get_branch_model(inp_shape):\n    from keras.applications.densenet import DenseNet121\n    model = DenseNet121(input_shape=inp_shape, include_top=False, weights=None, pooling='max')\n    return model\n\n\ndef build_model(img_shape, activation='sigmoid'):\n    from keras import backend as K\n    from keras.optimizers import Adam\n    from keras.engine.topology import Input\n    from keras.layers import Concatenate, Conv2D, Dense, Flatten, Lambda, Reshape\n    from keras.models import Model, load_model\n\n    optim = Adam(lr=0.0001)\n    branch_model = get_branch_model(img_shape)\n\n    mid = 32\n    xa_inp = Input(shape=branch_model.output_shape[1:], name='hm_inp_a')\n    xb_inp = Input(shape=branch_model.output_shape[1:], name='hm_inp_b')\n    x1 = Lambda(lambda x: x[0] * x[1], name='lambda_1')([xa_inp, xb_inp])\n    x2 = Lambda(lambda x: x[0] + x[1], name='lambda_2')([xa_inp, xb_inp])\n    x3 = Lambda(lambda x: K.abs(x[0] - x[1]), name='lambda_3')([xa_inp, xb_inp])\n    x4 = Lambda(lambda x: K.square(x), name='lambda_4')(x3)\n    x = Concatenate(name='concat_1')([x1, x2, x3, x4])\n    x = Reshape((4, branch_model.output_shape[1], 1), name='reshape1')(x)\n\n    # Per feature NN with shared weight is implemented using CONV2D with appropriate stride.\n    x = Conv2D(mid, (4, 1), activation='relu', padding='valid', name='hm_conv_2d_1')(x)\n    x = Reshape((branch_model.output_shape[1], mid, 1), name='hm_reshape_2')(x)\n    x = Conv2D(1, (1, mid), activation='linear', padding='valid', name='hm_conv_2d_2')(x)\n    x = Flatten(name='flatten')(x)\n\n    # Weighted sum implemented as a Dense layer.\n    x = Dense(1, use_bias=True, activation=activation, name='weighted-average')(x)\n    head_model = Model([xa_inp, xb_inp], x, name='head')\n\n    ########################\n    # SIAMESE NEURAL NETWORK\n    ########################\n    # Complete model is constructed by calling the branch model on each input image,\n    # and then the head model on the resulting 512-vectors.\n    img_a = Input(shape=img_shape)\n    img_b = Input(shape=img_shape)\n    xa = branch_model(img_a)\n    xb = branch_model(img_b)\n    x = head_model([xa, xb])\n    model = Model([img_a, img_b], x, name='full_model')\n    model.compile(optim, loss='binary_crossentropy', metrics=['binary_crossentropy', 'acc'])\n    return model, branch_model, head_model\n\n\ndef get_trained_model():\n    model, branch_model, head_model = build_model(img_shape=(BOX_SIZE, BOX_SIZE, 3))\n    model.load_weights('../input/whales-2019-models/ft_v5_512px_finetune_0_final_v2_0.969993.model')\n    return model, branch_model, head_model\n    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8615f45db90b4894d27de64ec5c6e2ecfe21cf2a"},"cell_type":"markdown","source":"**Load image routines and apply precomputed boxes**"},{"metadata":{"trusted":true,"_uuid":"9cb03159e1543498f4c938bba10e9ad2415c9f57"},"cell_type":"code","source":"def expand_path(p):\n    if isfile('../input/humpback-whale-identification/train/' + p):\n        return '../input/humpback-whale-identification/train/' + p\n    if isfile('../input/humpback-whale-identification/test/' + p):\n        return '../input/humpback-whale-identification/test/' + p\n    return p\n\n\ndef get_boxes():\n    temp_p2bb = pickle.load(open('../input/whales-2019-models/p2bb_averaged_v1.pkl', 'rb'))\n    p2bb = {}\n    for k in temp_p2bb:\n        p2bb[k + '.jpg'] = temp_p2bb[k]\n    return p2bb\n\n\ndef read_single_image(path):\n    try:\n        img = np.array(Image.open(path))\n    except:\n        try:\n            img = cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)\n        except:\n            print('Fail')\n            return None\n\n    if len(img.shape) == 2:\n        img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)\n\n    if img.shape[2] == 2:\n        img = img[:, :, :1]\n\n    if img.shape[2] == 1:\n        img = np.concatenate((img, img, img), axis=2)\n\n    if img.shape[2] > 3:\n        img = img[:, :, :3]\n\n    return img\n\n\ndef read_cropped_image(p, x0, y0, x1, y1, img_shape=(224, 224, 3)):\n    anisotropy = 2.15\n    crop_margin = 0.05\n\n    # Read the image\n    img = read_single_image(p)\n    size_x, size_y = img.shape[1], img.shape[0]\n\n    dx = x1 - x0\n    dy = y1 - y0\n    x0 -= dx * crop_margin\n    x1 += dx * crop_margin + 1\n    y0 -= dy * crop_margin\n    y1 += dy * crop_margin + 1\n    if x0 < 0: x0 = 0\n    if x1 > size_x: x1 = size_x\n    if y0 < 0: y0 = 0\n    if y1 > size_y: y1 = size_y\n    dx = x1 - x0\n    dy = y1 - y0\n    if dx > dy * anisotropy:\n        dy = 0.5 * (dx / anisotropy - dy)\n        y0 -= dy\n        y1 += dy\n    else:\n        dx = 0.5 * (dy * anisotropy - dx)\n        x0 -= dx\n        x1 += dx\n\n    if x0 < 0: x0 = 0\n    if x1 > size_x: x1 = size_x\n    if y0 < 0: y0 = 0\n    if y1 > size_y: y1 = size_y\n    x0, y0, x1, y1 = int(x0), int(y0), int(x1), int(y1)\n\n    if y0 != y1 and x0 != x1:\n        img = img[y0:y1, x0:x1, :]\n    img = cv2.resize(img, (img_shape[1], img_shape[0]), interpolation=cv2.INTER_LINEAR)\n    if len(img.shape) == 2:\n        img = np.concatenate((img, img, img), axis=2)\n\n    return img\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ef0c763a38b32d4c22fc83c8bfd6f1590dc447d"},"cell_type":"markdown","source":"**Create activation maps**\n\nIt's main functions. We got tensor from last BatchNormalization layer (also we can use last activation layer). It has shape like (10, 16, 16, 1024) or (16, 16, 1024) for single image in batch. On the next GlobalMaxPooling layer it converted to vector with 1024 elements in it. To generate N-th element of vector maximum value is taken from (16 x 16) matrix. So pixel located in position with maximum value define the vector value. This vector next goes to head model, which compare whale parameters. So we need to find places on (16x16) matrix where maximum value appear often. Also it's interesting to find places where values change often from one feature map to next feature map. We can use value of standard deviation to mark such places. \n\nSo colors on generated images:\n* Pink - places where value of max appears often\n* Green - places where Std is large\n* White - Pink + Green"},{"metadata":{"trusted":true,"_uuid":"84297af3f03f5821a95efffad29fc3c2512a635e"},"cell_type":"code","source":"def normalize_array(arr):\n    arr = 255.0 * (arr - arr.min()) / (arr.max() - arr.min())\n    return arr\n\n\ndef create_activation_map_for_images(model, images, preproc_image):\n    images_preproc = preproc_image(images.astype(np.float32))\n    preds = model.predict(images_preproc)\n    print('Shape of predictions: {}'.format(preds.shape))\n\n    hmaps = []\n    for z in range(images.shape[0]):\n        img_orig = images[z]\n        heatmap = preds[z]\n    \n        # Uncomment it to emulate RELU activation\n        # heatmap[heatmap < 0] = 0.\n\n        ch0 = np.zeros_like(heatmap[:, :, 0])\n        ch1 = np.zeros_like(heatmap[:, :, 0])\n        ch2 = np.zeros_like(heatmap[:, :, 0])\n\n        # Find how often maximum is in each pixel.\n        for k in range(heatmap.shape[2]):\n            p = heatmap[:, :, k]\n            mx = p.max()\n            if mx == 0:\n                continue\n            for i in range(heatmap.shape[0]):\n                for j in range(heatmap.shape[1]):\n                    if p[i, j] == mx:\n                        ch0[i, j] += 1\n                        ch2[i, j] += 1\n\n        for i in range(heatmap.shape[0]):\n            for j in range(heatmap.shape[1]):\n                mn = heatmap[i, j].min()\n                mx = heatmap[i, j].max()\n                mean = heatmap[i, j].mean()\n                std = heatmap[i, j].std()\n                # print(i, j, mn, mx, mean, std, mx - mn)\n                ch1[i, j] = std\n\n        ch0 = normalize_array(ch0)\n        ch1 = normalize_array(ch1)\n        ch2 = normalize_array(ch2)\n        ch = np.stack((ch0, ch1, ch2), axis=2)\n\n        ch = cv2.resize(ch.astype(np.uint8), (img_orig.shape[1], img_orig.shape[0]), interpolation=cv2.INTER_LANCZOS4)\n\n        ch = normalize_array(ch)\n\n        heat = (0.2 * img_orig + (0.1 * img_orig * ch) / 255 + 0.7 * ch).astype(np.uint8)\n        heat = heat.astype(np.uint8)\n        heat = normalize_array(heat)\n\n        img_line = np.concatenate(\n            (img_orig, ch, heat), axis=1\n        )\n        hmaps.append(img_line)\n\n    return np.array(hmaps)\n\n\ndef create_activation_maps():\n    from keras.models import Model\n    model, branch_model, head_model = get_trained_model()\n\n    bboxes = get_boxes()\n\n    x = branch_model.layers[-3].output\n    branch_model_modified = Model(inputs=branch_model.inputs, outputs=x)\n    # print(branch_model_modified.summary())\n\n    start = 10\n    end = 50\n    image_ids = pd.read_csv('../input/humpback-whale-identification/sample_submission.csv')['Image'].values[start:end]\n    images = []\n    for image_id in image_ids:\n        f = expand_path(image_id)\n        bb = bboxes[image_id]\n        img_orig = read_cropped_image(f, bb[0], bb[1], bb[2], bb[3], img_shape=(BOX_SIZE, BOX_SIZE, 3))\n        images.append(img_orig)\n\n    img_line = create_activation_map_for_images(branch_model_modified, np.array(images), preprocess_image)\n\n    for i, image_id in enumerate(image_ids):\n        img = cv2.cvtColor(img_line[i].astype(np.uint8), cv2.COLOR_RGB2BGR)\n        img_path = image_id\n        cv2.imwrite(img_path, img)\n        print('Heatmap for image: {}'.format(image_id))\n        # Image(img_line[i].astype(np.uint8))\n        display(Image(filename=img_path))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"74880982c1dd7dd03788fcab20dee1bee87381e5"},"cell_type":"markdown","source":"**Run script**"},{"metadata":{"trusted":true,"_uuid":"39f719930ff93621219a42811158afea05a5c9d9"},"cell_type":"code","source":"create_activation_maps()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"834ff6f4e3f333f21c7ee20cf852443a3eb99103"},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"00d40eba58c4f948eca9cb528c728b8b6263fe94"},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"cc21ba596db1c064478aac2d1c281d758b05954a"},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"a619d3ae2e0908c190299355ac1c3c3ae579347d"},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}