{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install lap\n# Read the dataset description\nimport gzip\n# Read or generate p2h, a dictionary of image name to image id (picture to hash)\nimport pickle\nimport platform\nimport random\n# Suppress annoying stderr output when importing keras.\nimport sys\nfrom lap import lapjv\nfrom math import sqrt\n# Determine the size of each image\nfrom os.path import isfile\n\nimport keras\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image as pil_image\nfrom imagehash import phash\nfrom keras import backend as K\nfrom keras import regularizers\nfrom keras.engine.topology import Input\nfrom keras.layers import Activation, Add, BatchNormalization, Concatenate, Conv2D, Dense, Flatten, GlobalMaxPooling2D, \\\n    Lambda, MaxPooling2D, Reshape\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom keras.preprocessing.image import img_to_array\nfrom keras.utils import Sequence\nfrom pandas import read_csv\nfrom scipy.ndimage import affine_transform\nfrom tqdm import tqdm_notebook as tqdm\nimport time","execution_count":2,"outputs":[{"output_type":"stream","text":"Requirement already satisfied: lap in /opt/conda/lib/python3.6/site-packages (0.4.0)\n\u001b[33mYou are using pip version 19.0.3, however version 19.1.1 is available.\nYou should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_DF = '../input/humpback-whale-identification/train.csv'\nSUB_Df = '../input/humpback-whale-identification/sample_submission.csv'\nTRAIN = '../input/humpback-whale-identification/train/'\nTEST = '../input/humpback-whale-identification/test/'\nP2H = '../input/metadata/p2h.pickle'\nP2SIZE = '../input/metadata/p2size.pickle'\nBB_DF = \"../input/metadata/bounding_boxes.csv\"","execution_count":13,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#putting what is in train.csv in tagged dictionary tagged\ntagged = dict([(p, w) for _, p, w in read_csv(TRAIN_DF).to_records()])\n#doing the same thing for submit\nsubmit = [p for _, p, _ in read_csv(SUB_Df).to_records()]\n#append submission photos to the dictionary\njoin = list(tagged.keys()) + submit","execution_count":14,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#creates the path for a picture by attaching name of picture to train or test path\ndef expand_path(p):\n    if isfile(TRAIN + p):\n        return TRAIN + p\n    if isfile(TEST + p):\n        return TEST + p\n    return p","execution_count":15,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#reading in file P2SIZE as f so it closes itself\n#P2SIZE contains sizes of all images in dataset\nwith open(P2SIZE, 'rb') as f:\n    p2size = pickle.load(f)","execution_count":16,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#P2H contains the pictures hashed values so we load that to p2h\nwith open(P2H, 'rb') as f:\n    p2h = pickle.load(f)\n        \n# For each image id, determine the list of pictures that hold that hash value\n# How many pictures are similar to each other\nh2ps = {}\nfor p, h in p2h.items():\n    if h not in h2ps: h2ps[h] = []\n    if p not in h2ps[h]: h2ps[h].append(p)","execution_count":17,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#prints out the images of the whales\ndef show_whale(imgs, per_row=2):\n    n = len(imgs)\n    rows = (n + per_row - 1) // per_row\n    cols = min(per_row, n)\n    fig, axes = plt.subplots(rows, cols, figsize=(24 // per_row * cols, 24 // per_row * rows))\n    for ax in axes.flatten(): ax.axis('off')\n    for i, (img, ax) in enumerate(zip(imgs, axes.flatten())): ax.imshow(img.convert('RGB'))\n\n#returns the image we need\ndef read_raw_image(p):\n    img = pil_image.open(expand_path(p))\n    return img","execution_count":18,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For each images id, select the prefered image based on size and resolution\ndef prefer(ps):\n    if len(ps) == 1: return ps[0]\n    best_p = ps[0]\n    best_s = p2size[best_p]\n    for i in range(1, len(ps)):\n        p = ps[i]\n        s = p2size[p]\n        if s[0] * s[1] > best_s[0] * best_s[1]:  # Select the image with highest resolution\n            best_p = p\n            best_s = s\n    return best_p\n\nh2p = {}\nfor h, ps in h2ps.items():\n    h2p[h] = prefer(ps)\nlen(h2p), list(h2p.items())[:5]","execution_count":19,"outputs":[{"output_type":"execute_result","execution_count":19,"data":{"text/plain":"(33317,\n [('d26698c3271c757c', '0000e88ab.jpg'),\n  ('ba8cc231ad489b77', '0001f9222.jpg'),\n  ('bbcad234a52d0f0b', '00029d126.jpg'),\n  ('c09ae7dc09f33a29', '00050a15a.jpg'),\n  ('d02f65ba9f74a08a', '0005c1ef8.jpg')])"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read the bounding box data from the bounding box kernel\n# Contains the cropping data for each of the images\np2bb = pd.read_csv(BB_DF).set_index(\"Image\")\n\nold_stderr = sys.stderr\nsys.stderr = open('/dev/null' if platform.system() != 'Windows' else 'nul', 'w')\n\nsys.stderr = old_stderr\n\n#The affine transformation maps a rectangular area of the original image to a square image with resolution 384x384x1  \nimg_shape = (384, 384, 1)\n# The rectangular area has a width over height aspect ratio of 2.15\nanisotropy = 2.15 \n#The rectangle is taken to be slightly larger than the computed bounding box, therefore we use a margin\ncrop_margin = 0.05  ","execution_count":20,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# During training, data augmentation by randomly zooming, shifting, rotating and shearing. \n# The random transform is skipped when testing.\ndef build_transform(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    \"\"\"\n    Build a transformation matrix with the specified characteristics.\n    \"\"\"\n    rotation = np.deg2rad(rotation)\n    shear = np.deg2rad(shear)\n    rotation_matrix = np.array(\n        [[np.cos(rotation), np.sin(rotation), 0], [-np.sin(rotation), np.cos(rotation), 0], [0, 0, 1]])\n    shift_matrix = np.array([[1, 0, height_shift], [0, 1, width_shift], [0, 0, 1]])\n    shear_matrix = np.array([[1, np.sin(shear), 0], [0, np.cos(shear), 0], [0, 0, 1]])\n    zoom_matrix = np.array([[1.0 / height_zoom, 0, 0], [0, 1.0 / width_zoom, 0], [0, 0, 1]])\n    shift_matrix = np.array([[1, 0, -height_shift], [0, 1, -width_shift], [0, 0, 1]])\n    return np.dot(np.dot(rotation_matrix, shear_matrix), np.dot(zoom_matrix, shift_matrix))","execution_count":21,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_cropped_image(p, augment):\n    \"\"\"\n    @param p : the name of the picture to read\n    @param augment: True/False if data augmentation should be performed\n    @return a numpy array with the transformed image\n    \"\"\"\n    # If an image id was given, convert to filename\n    if p in h2p:\n        p = h2p[p]\n    size_x, size_y = p2size[p]\n\n    # Determine the region of the original image we want to capture based on the bounding box.\n    row = p2bb.loc[p]\n    x0, y0, x1, y1 = row['x0'], row['y0'], row['x1'], row['y1']\n    dx = x1 - x0\n    dy = y1 - y0\n    x0 -= dx * crop_margin\n    x1 += dx * crop_margin + 1\n    y0 -= dy * crop_margin\n    y1 += dy * crop_margin + 1\n    if x0 < 0:\n        x0 = 0\n    if x1 > size_x:\n        x1 = size_x\n    if y0 < 0:\n        y0 = 0\n    if y1 > size_y:\n        y1 = size_y\n    dx = x1 - x0\n    dy = y1 - y0\n    if dx > dy * anisotropy:\n        dy = 0.5 * (dx / anisotropy - dy)\n        y0 -= dy\n        y1 += dy\n    else:\n        dx = 0.5 * (dy * anisotropy - dx)\n        x0 -= dx\n        x1 += dx\n\n    # Generate the transformation matrix randomly\n    trans = np.array([[1, 0, -0.5 * img_shape[0]], [0, 1, -0.5 * img_shape[1]], [0, 0, 1]])\n    trans = np.dot(np.array([[(y1 - y0) / img_shape[0], 0, 0], [0, (x1 - x0) / img_shape[1], 0], [0, 0, 1]]), trans)\n    if augment:\n        trans = np.dot(build_transform(\n            random.uniform(-5, 5),\n            random.uniform(-5, 5),\n            random.uniform(0.8, 1.0),\n            random.uniform(0.8, 1.0),\n            random.uniform(-0.05 * (y1 - y0), 0.05 * (y1 - y0)),\n            random.uniform(-0.05 * (x1 - x0), 0.05 * (x1 - x0))\n        ), trans)\n    trans = np.dot(np.array([[1, 0, 0.5 * (y1 + y0)], [0, 1, 0.5 * (x1 + x0)], [0, 0, 1]]), trans)\n\n    # Read the image, transform to black and white and convert to numpy array\n    img = read_raw_image(p).convert('L')\n    img = img_to_array(img)\n\n    # Apply affine transformation\n    matrix = trans[:2, :2]\n    offset = trans[:2, 2]\n    img = img.reshape(img.shape[:-1])\n    img = affine_transform(img, matrix, offset, output_shape=img_shape[:-1], order=1, mode='constant',\n                           cval=np.average(img))\n    img = img.reshape(img_shape)\n\n    # Normalize to zero mean and unit variance\n    img -= np.mean(img, keepdims=True)\n    img /= np.std(img, keepdims=True) + K.epsilon()\n    return img","execution_count":22,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_for_training(p):\n    \"\"\"\n    Read and preprocess an image with data augmentation (random transform).\n    \"\"\"\n    return read_cropped_image(p, True)\n\ndef read_for_validation(p):\n    \"\"\"\n    Read and preprocess an image without data augmentation (use for testing).\n    \"\"\"\n    return read_cropped_image(p, False)","execution_count":23,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#layers of the CNN that we are going to need\ndef subblock(x, filter, **kwargs):\n    x = BatchNormalization()(x)\n    y = x\n    y = Conv2D(filter, (1, 1), activation='relu', **kwargs)(y)  # Reduce the number of features to 'filter'\n    y = BatchNormalization()(y)\n    y = Conv2D(filter, (3, 3), activation='relu', **kwargs)(y)  # Extend the feature field\n    y = BatchNormalization()(y)\n    y = Conv2D(K.int_shape(x)[-1], (1, 1), **kwargs)(y)  # no activation # Restore the number of original features\n    y = Add()([x, y])  # Add the bypass connection\n    y = Activation('relu')(y)\n    return y\n\n#The branch model of the Siamese Nueral Network is a normal CNN\n#We perform the CNN to get the images into vectors so that they can be compared during the head branch\ndef build_model(lr, l2, activation='sigmoid'):\n    ##############\n    # BRANCH MODEL\n    ##############\n    regul = regularizers.l2(l2)\n    optim = Adam(lr=lr)\n    kwargs = {'padding': 'same', 'kernel_regularizer': regul}\n\n    inp = Input(shape=img_shape)  # 384x384x1\n    x = Conv2D(64, (9, 9), strides=2, activation='relu', **kwargs)(inp)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 96x96x64\n    for _ in range(2):\n        x = BatchNormalization()(x)\n        x = Conv2D(64, (3, 3), activation='relu', **kwargs)(x)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 48x48x64\n    x = BatchNormalization()(x)\n    x = Conv2D(128, (1, 1), activation='relu', **kwargs)(x)  # 48x48x128\n    for _ in range(4):\n        x = subblock(x, 64, **kwargs)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 24x24x128\n    x = BatchNormalization()(x)\n    x = Conv2D(256, (1, 1), activation='relu', **kwargs)(x)  # 24x24x256\n    for _ in range(4):\n        x = subblock(x, 64, **kwargs)\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 12x12x256\n    x = BatchNormalization()(x)\n    x = Conv2D(384, (1, 1), activation='relu', **kwargs)(x)  # 12x12x384\n    for _ in range(4):\n        x = subblock(x, 96, **kwargs)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 6x6x384\n    x = BatchNormalization()(x)\n    x = Conv2D(512, (1, 1), activation='relu', **kwargs)(x)  # 6x6x512\n    for _ in range(4):\n        x = subblock(x, 128, **kwargs)\n\n    x = GlobalMaxPooling2D()(x)  # 512\n    branch_model = Model(inp, x)\n\n    #Head tests image from test set against the images from the training set after they have been run through the CNN\n    ############\n    # HEAD MODEL\n    ############\n    mid = 32\n    xa_inp = Input(shape=branch_model.output_shape[1:])\n    xb_inp = Input(shape=branch_model.output_shape[1:])\n    x1 = Lambda(lambda x: x[0] * x[1])([xa_inp, xb_inp])\n    x2 = Lambda(lambda x: x[0] + x[1])([xa_inp, xb_inp])\n    x3 = Lambda(lambda x: K.abs(x[0] - x[1]))([xa_inp, xb_inp])\n    x4 = Lambda(lambda x: K.square(x))(x3)\n    x = Concatenate()([x1, x2, x3, x4])\n    x = Reshape((4, branch_model.output_shape[1], 1), name='reshape1')(x)\n\n    # Per feature NN with shared weight is implemented using CONV2D with appropriate stride.\n    x = Conv2D(mid, (4, 1), activation='relu', padding='valid')(x)\n    x = Reshape((branch_model.output_shape[1], mid, 1))(x)\n    x = Conv2D(1, (1, mid), activation='linear', padding='valid')(x)\n    x = Flatten(name='flatten')(x)\n\n    # Weighted sum implemented as a Dense layer.\n    x = Dense(1, use_bias=True, activation=activation, name='weighted-average')(x)\n    head_model = Model([xa_inp, xb_inp], x, name='head')\n    \n    ########################\n    # SIAMESE NEURAL NETWORK\n    ########################\n    # Complete model is constructed by calling the branch model on each input image,\n    # and then the head model on the resulting 512-vectors.\n    img_a = Input(shape=img_shape)\n    img_b = Input(shape=img_shape)\n    xa = branch_model(img_a) #we run CNN on image A\n    xb = branch_model(img_b) #run CNN on image B\n    x = head_model([xa, xb]) #compare using the head model\n    model = Model([img_a, img_b], x)\n    model.compile(optim, loss='binary_crossentropy', metrics=['binary_crossentropy', 'acc'])\n    return model, branch_model, head_model\n\n\nmodel, branch_model, head_model = build_model(64e-5, 0)\nhead_model.summary()","execution_count":24,"outputs":[{"output_type":"stream","text":"WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nColocations handled automatically by placer.\n__________________________________________________________________________________________________\nLayer (type)                    Output Shape         Param #     Connected to                     \n==================================================================================================\ninput_2 (InputLayer)            (None, 512)          0                                            \n__________________________________________________________________________________________________\ninput_3 (InputLayer)            (None, 512)          0                                            \n__________________________________________________________________________________________________\nlambda_3 (Lambda)               (None, 512)          0           input_2[0][0]                    \n                                                                 input_3[0][0]                    \n__________________________________________________________________________________________________\nlambda_1 (Lambda)               (None, 512)          0           input_2[0][0]                    \n                                                                 input_3[0][0]                    \n__________________________________________________________________________________________________\nlambda_2 (Lambda)               (None, 512)          0           input_2[0][0]                    \n                                                                 input_3[0][0]                    \n__________________________________________________________________________________________________\nlambda_4 (Lambda)               (None, 512)          0           lambda_3[0][0]                   \n__________________________________________________________________________________________________\nconcatenate_1 (Concatenate)     (None, 2048)         0           lambda_1[0][0]                   \n                                                                 lambda_2[0][0]                   \n                                                                 lambda_3[0][0]                   \n                                                                 lambda_4[0][0]                   \n__________________________________________________________________________________________________\nreshape1 (Reshape)              (None, 4, 512, 1)    0           concatenate_1[0][0]              \n__________________________________________________________________________________________________\nconv2d_56 (Conv2D)              (None, 1, 512, 32)   160         reshape1[0][0]                   \n__________________________________________________________________________________________________\nreshape_1 (Reshape)             (None, 512, 32, 1)   0           conv2d_56[0][0]                  \n__________________________________________________________________________________________________\nconv2d_57 (Conv2D)              (None, 512, 1, 1)    33          reshape_1[0][0]                  \n__________________________________________________________________________________________________\nflatten (Flatten)               (None, 512)          0           conv2d_57[0][0]                  \n__________________________________________________________________________________________________\nweighted-average (Dense)        (None, 1)            513         flatten[0][0]                    \n==================================================================================================\nTotal params: 706\nTrainable params: 706\nNon-trainable params: 0\n__________________________________________________________________________________________________\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"#put in a dictionary all of the items that are not new whales\n#items are the photo as the key with the whale id as the value\nh2ws = {}\nnew_whale = 'new_whale'\nfor p, w in tagged.items():\n    if w != new_whale:  # Use only identified whales\n        h = p2h[p]\n        if h not in h2ws: h2ws[h] = []\n        if w not in h2ws[h]: h2ws[h].append(w)\nfor h, ws in h2ws.items():\n    if len(ws) > 1:\n        h2ws[h] = sorted(ws)\n\n# For each whale, find the unambiguous images ids.\nw2hs = {}\nfor h, ws in h2ws.items():\n    if len(ws) == 1:  # Use only unambiguous pictures\n        w = ws[0]\n        if w not in w2hs: w2hs[w] = []\n        if h not in w2hs[w]: w2hs[w].append(h)\nfor w, hs in w2hs.items():\n    if len(hs) > 1:\n        w2hs[w] = sorted(hs)","execution_count":25,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Only keeping whales that we have more than two images on\ntrain = []  # A list of training image ids\nfor hs in w2hs.values():\n    if len(hs) > 1:\n        train += hs\nrandom.shuffle(train)\ntrain_set = set(train)\n\nw2ts = {}  # Associate the image ids from train to each whale id.\nfor w, hs in w2hs.items():\n    for h in hs:\n        if h in train_set:\n            if w not in w2ts:\n                w2ts[w] = []\n            if h not in w2ts[w]:\n                w2ts[w].append(h)\nfor w, ts in w2ts.items():\n    w2ts[w] = np.array(ts)\n\nt2i = {}  # The position in train of each training image id\nfor i, t in enumerate(train):\n    t2i[t] = i","execution_count":26,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainingData(Sequence):\n    def __init__(self, score, steps=1000, batch_size=32):\n        \"\"\"\n        @param score the cost matrix for the picture matching\n        @param steps the number of epoch we are planning with this score matrix\n        \"\"\"\n        super(TrainingData, self).__init__()\n        self.score = -score  # Maximizing the score is the same as minimuzing -score.\n        self.steps = steps\n        self.batch_size = batch_size\n        for ts in w2ts.values():\n            idxs = [t2i[t] for t in ts]\n            for i in idxs:\n                for j in idxs:\n                    self.score[\n                        i, j] = 10000.0  # Set a large value for matching whales -- eliminates this potential pairing\n        self.on_epoch_end()\n\n    def __getitem__(self, index):\n        start = self.batch_size * index\n        end = min(start + self.batch_size, len(self.match) + len(self.unmatch))\n        size = end - start\n        assert size > 0\n        a = np.zeros((size,) + img_shape, dtype=K.floatx())\n        b = np.zeros((size,) + img_shape, dtype=K.floatx())\n        c = np.zeros((size, 1), dtype=K.floatx())\n        j = start // 2\n        for i in range(0, size, 2):\n            a[i, :, :, :] = read_for_training(self.match[j][0])\n            b[i, :, :, :] = read_for_training(self.match[j][1])\n            c[i, 0] = 1  # This is a match\n            a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])\n            b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])\n            c[i + 1, 0] = 0  # Different whales\n            j += 1\n        return [a, b], c\n\n    def on_epoch_end(self):\n        if self.steps <= 0: return  # Skip this on the last epoch.\n        self.steps -= 1\n        self.match = []\n        self.unmatch = []\n        _, _, x = lapjv(self.score)  # Solve the linear assignment problem\n        y = np.arange(len(x), dtype=np.int32)\n\n        # Compute a derangement for matching whales\n        for ts in w2ts.values():\n            d = ts.copy()\n            while True:\n                random.shuffle(d)\n                if not np.any(ts == d): break\n            for ab in zip(ts, d): self.match.append(ab)\n\n        # Construct unmatched whale pairs from the LAP solution.\n        for i, j in zip(x, y):\n            if i == j:\n                print(self.score)\n                print(x)\n                print(y)\n                print(i, j)\n            assert i != j\n            self.unmatch.append((train[i], train[j]))\n\n        # Force a different choice for an eventual next epoch.\n        self.score[x, y] = 10000.0\n        self.score[y, x] = 10000.0\n        random.shuffle(self.match)\n        random.shuffle(self.unmatch)\n        # print(len(self.match), len(train), len(self.unmatch), len(train))\n        assert len(self.match) == len(train) and len(self.unmatch) == len(train)\n\n    def __len__(self):\n        return (len(self.match) + len(self.unmatch) + self.batch_size - 1) // self.batch_size","execution_count":27,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test on a batch of 32 with random costs.\nscore = np.random.random_sample(size=(len(train), len(train)))\ndata = TrainingData(score)\n(a, b), c = data[0]","execution_count":28,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#These are used to calculate the scores of the CNN on the training set\n# A Keras generator to evaluate only the BRANCH MODEL\nclass FeatureGen(Sequence):\n    def __init__(self, data, batch_size=64, verbose=1):\n        super(FeatureGen, self).__init__()\n        self.data = data\n        self.batch_size = batch_size\n        self.verbose = verbose\n        if self.verbose > 0: self.progress = tqdm(total=len(self), desc='Features')\n\n    def __getitem__(self, index):\n        start = self.batch_size * index\n        size = min(len(self.data) - start, self.batch_size)\n        a = np.zeros((size,) + img_shape, dtype=K.floatx())\n        for i in range(size): a[i, :, :, :] = read_for_validation(self.data[start + i])\n        if self.verbose > 0:\n            self.progress.update()\n            if self.progress.n >= len(self): self.progress.close()\n        return a\n\n    def __len__(self):\n        return (len(self.data) + self.batch_size - 1) // self.batch_size\n    \n# A Keras generator to evaluate on the HEAD MODEL on features already pre-computed.\n# It computes only the upper triangular matrix of the cost matrix if y is None.\nclass ScoreGen(Sequence):\n    def __init__(self, x, y=None, batch_size=2048, verbose=1):\n        super(ScoreGen, self).__init__()\n        self.x = x\n        self.y = y\n        self.batch_size = batch_size\n        self.verbose = verbose\n        if y is None:\n            self.y = self.x\n            self.ix, self.iy = np.triu_indices(x.shape[0], 1)\n        else:\n            self.iy, self.ix = np.indices((y.shape[0], x.shape[0]))\n            self.ix = self.ix.reshape((self.ix.size,))\n            self.iy = self.iy.reshape((self.iy.size,))\n        self.subbatch = (len(self.x) + self.batch_size - 1) // self.batch_size\n        if self.verbose > 0:\n            self.progress = tqdm(total=len(self), desc='Scores')\n\n    def __getitem__(self, index):\n        start = index * self.batch_size\n        end = min(start + self.batch_size, len(self.ix))\n        a = self.y[self.iy[start:end], :]\n        b = self.x[self.ix[start:end], :]\n        if self.verbose > 0:\n            self.progress.update()\n            if self.progress.n >= len(self): self.progress.close()\n        return [a, b]\n\n    def __len__(self):\n        return (len(self.ix) + self.batch_size - 1) // self.batch_size","execution_count":29,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#getter and setter for our model learning rate\ndef set_lr(model, lr):\n    K.set_value(model.optimizer.lr, float(lr))\n\ndef get_lr(model):\n    return K.get_value(model.optimizer.lr)","execution_count":30,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#converting the upper triangular matrix into a square matrix\ndef score_reshape(score, x, y=None):\n    \"\"\"\n    Tranformed the packed matrix 'score' into a square matrix.\n    @param score the packed matrix\n    @param x the first image feature tensor\n    @param y the second image feature tensor if different from x\n    @result the square matrix\n    \"\"\"\n    if y is None:\n        # When y is None, score is a packed upper triangular matrix.\n        # Unpack, and transpose to form the symmetrical lower triangular matrix.\n        m = np.zeros((x.shape[0], x.shape[0]), dtype=K.floatx())\n        m[np.triu_indices(x.shape[0], 1)] = score.squeeze()\n        m += m.transpose()\n    else:\n        m = np.zeros((y.shape[0], x.shape[0]), dtype=K.floatx())\n        iy, ix = np.indices((y.shape[0], x.shape[0]))\n        ix = ix.reshape((ix.size,))\n        iy = iy.reshape((iy.size,))\n        m[iy, ix] = score.squeeze()\n    return m","execution_count":31,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#use the FeatureGen and the ScoreGen to calculate the score for our trained models\ndef compute_score(verbose=1):\n    \"\"\"\n    Compute the score matrix by scoring every pictures from the training set against every other picture O(n^2).\n    \"\"\"\n    features = branch_model.predict_generator(FeatureGen(train, verbose=verbose), max_queue_size=12, workers=6,\n                                              verbose=0)\n    score = head_model.predict_generator(ScoreGen(features, verbose=verbose), max_queue_size=12, workers=6, verbose=0)\n    score = score_reshape(score, features)\n    return features, score","execution_count":32,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#this acutally preforms the epochs to train the model \ndef make_steps(step, ampl):\n    \"\"\"\n    Perform training epochs\n    @param step Number of epochs to perform\n    @param ampl the K, the randomized component of the score matrix.\n    \"\"\"\n    global w2ts, t2i, steps, features, score, histories\n\n    # shuffle the training pictures\n    random.shuffle(train)\n\n    # Map whale id to the list of associated training picture hash value\n    w2ts = {}\n    for w, hs in w2hs.items():\n        for h in hs:\n            if h in train_set:\n                if w not in w2ts: w2ts[w] = []\n                if h not in w2ts[w]: w2ts[w].append(h)\n    for w, ts in w2ts.items(): w2ts[w] = np.array(ts)\n\n    # Map training picture hash value to index in 'train' array    \n    t2i = {}\n    for i, t in enumerate(train): t2i[t] = i\n\n    # Compute the match score for each picture pair\n    features, score = compute_score()\n\n    # Train the model for 'step' epochs\n    history = model.fit_generator(\n        TrainingData(score + ampl * np.random.random_sample(size=score.shape), steps=step, batch_size=32),\n        initial_epoch=steps, epochs=steps + step, max_queue_size=12, workers=6, verbose=1).history\n    steps += step\n\n    # Collect history data\n    history['epochs'] = steps\n    history['ms'] = np.mean(score)\n    history['lr'] = get_lr(model)\n    print(history['epochs'], history['lr'], history['ms'])\n    histories.append(history)","execution_count":33,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"histories = []\nsteps = 0\ntmp = keras.models.load_model('../input/piotte/mpiotte-standard.model')\nmodel.set_weights(tmp.get_weights())\nmodel.summary()","execution_count":35,"outputs":[{"output_type":"stream","text":"__________________________________________________________________________________________________\nLayer (type)                    Output Shape         Param #     Connected to                     \n==================================================================================================\ninput_4 (InputLayer)            (None, 384, 384, 1)  0                                            \n__________________________________________________________________________________________________\ninput_5 (InputLayer)            (None, 384, 384, 1)  0                                            \n__________________________________________________________________________________________________\nmodel_1 (Model)                 (None, 512)          2692096     input_4[0][0]                    \n                                                                 input_5[0][0]                    \n__________________________________________________________________________________________________\nhead (Model)                    (None, 1)            706         model_1[1][0]                    \n                                                                 model_1[2][0]                    \n==================================================================================================\nTotal params: 2,692,802\nTrainable params: 2,675,010\nNon-trainable params: 17,792\n__________________________________________________________________________________________________\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"#The basic strategy is this. For each picture from the test set:\n\n#If the image duplicates one or more training set images, add the whales from the training image as the top candidate. (free answers)\n#For each image not new_whale from the training set, compute the image score, which is the model score for the image pair.\n#For each whale from the training set, compute the whale score as the maximum image score for this whale.\n#Add new_whale with a fixed new whale score of 'threshold'.\n#Sort the whales in decreasing score.\ndef prepare_submission(threshold, filename):\n    \"\"\"\n    Generate a Kaggle submission file.\n    @param threshold the score given to 'new_whale'\n    @param filename the submission file name\n    \"\"\"\n    vtop = 0\n    vhigh = 0\n    pos = [0, 0, 0, 0, 0, 0]\n    with open(filename, 'wt', newline='\\n') as f:\n        f.write('Image,Id\\n')\n        for i, p in enumerate(tqdm(submit)):\n            t = []\n            s = set()\n            a = score[i, :]\n            for j in list(reversed(np.argsort(a))):\n                h = known[j]\n                if a[j] < threshold and new_whale not in s:\n                    pos[len(t)] += 1\n                    s.add(new_whale)\n                    t.append(new_whale)\n                    if len(t) == 5: break;\n                for w in h2ws[h]:\n                    assert w != new_whale\n                    if w not in s:\n                        if a[j] > 1.0:\n                            vtop += 1\n                        elif a[j] >= threshold:\n                            vhigh += 1\n                        s.add(w)\n                        t.append(w)\n                        if len(t) == 5: break;\n                if len(t) == 5: break;\n            if new_whale not in s: pos[5] += 1\n            assert len(t) == 5 and len(s) == 5\n            f.write(p + ',' + ' '.join(t[:5]) + '\\n')\n    return vtop, vhigh, pos","execution_count":36,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find elements from training sets not 'new_whale'\ntic = time.time()\nh2ws = {}\nfor p, w in tagged.items():\n    if w != new_whale:  # Use only identified whales\n        h = p2h[p]\n        if h not in h2ws: h2ws[h] = []\n        if w not in h2ws[h]: h2ws[h].append(w)\nknown = sorted(list(h2ws.keys()))\n\n# Dictionary of picture indices\nh2i = {}\nfor i, h in enumerate(known): h2i[h] = i\n\n# Evaluate the model.\nfknown = branch_model.predict_generator(FeatureGen(known), max_queue_size=20, workers=10, verbose=0)\nfsubmit = branch_model.predict_generator(FeatureGen(submit), max_queue_size=20, workers=10, verbose=0)\nscore = head_model.predict_generator(ScoreGen(fknown, fsubmit), max_queue_size=20, workers=10, verbose=0)\nscore = score_reshape(score, fknown, fsubmit)\n\n# Generate the subsmission file.\nprepare_submission(0.99, 'submission.csv')\ntoc = time.time()\nprint(\"Submission time: \", (toc - tic) / 60.)","execution_count":37,"outputs":[{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, description='Features', max=246, style=ProgressStyle(description_width='in…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9ed706c74e5848718bb4c6b06d55958d"}},"metadata":{}},{"output_type":"stream","text":"\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, description='Features', max=125, style=ProgressStyle(description_width='in…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7ffaac04c5a444ecba9c2eabdc9bf8ba"}},"metadata":{}},{"output_type":"stream","text":"\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, description='Scores', max=61006, style=ProgressStyle(description_width='in…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e6c4230fb1be4aaab1c811823945c588"}},"metadata":{}},{"output_type":"stream","text":"\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, max=7960), HTML(value='')))","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bde85cd1a1264ce1a253cec362369bc3"}},"metadata":{}},{"output_type":"stream","text":"\nSubmission time:  15.892371435960134\n","name":"stdout"}]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"}},"nbformat":4,"nbformat_minor":1}