{"cells":[{"metadata":{"_uuid":"03d0dd6baf411efbb62a6aef0b43dc846668d2ca"},"cell_type":"markdown","source":"## TL;DR\n\nI tried to refactor [@martinpiotte](https://kaggle.com/martinpiotte)'s original kernel [here](https://www.kaggle.com/martinpiotte/whale-recognition-model-with-score-0-78563).\n\nI changed almost nothing beside commenting out the latter 380 epochs since it can't fit into a kernel. I also generated the new bounding boxes in my kernel [here](https://www.kaggle.com/suicaokhoailang/generating-whale-bounding-boxes) and saved it as a **.csv** instead of **pickle** for readability. \n\nA few things to point out:\n\n- Training more will probably improve your score, maybe as many as 500 epochs. We only train for 20 epochs in this kernel.\n\n- You may try to improve your training time by applying this technique (thanks **Brian**): https://www.kaggle.com/c/humpback-whale-identification/discussion/74402#444476 .\n\n- Consider using a pretrained model(s), good for blending."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install lapjv\n# Read the dataset description\nimport gzip\n# Read or generate p2h, a dictionary of image name to image id (picture to hash)\nimport pickle\nimport platform\nimport random\n# Suppress annoying stderr output when importing keras.\nimport sys\nfrom lapjv import lapjv\nfrom math import sqrt\n# Determine the size of each image\nfrom os.path import isfile\n\nimport keras\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image as pil_image\nfrom imagehash import phash\nfrom keras import backend as K\nfrom keras import regularizers\nfrom keras.engine.topology import Input\nfrom keras.layers import Activation, Add, BatchNormalization, Concatenate, Conv2D, Dense, Flatten, GlobalMaxPooling2D, \\\n    Lambda, MaxPooling2D, Reshape\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom keras.preprocessing.image import img_to_array\nfrom keras.utils import Sequence\nfrom pandas import read_csv\nfrom scipy.ndimage import affine_transform\nfrom tqdm import tqdm_notebook as tqdm","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"TRAIN_DF = '../input/humpback-whale-identification/train.csv'\nSUB_Df = '../input/humpback-whale-identification/sample_submission.csv'\nTRAIN = '../input/humpback-whale-identification/train/'\nTEST = '../input/humpback-whale-identification/test/'\nP2H = '../input/metadata/p2h.pickle'\nP2SIZE = '../input/metadata/p2size.pickle'\nBB_DF = \"../input/metadata/bounding_boxes.csv\"\ntagged = dict([(p, w) for _, p, w in read_csv(TRAIN_DF).to_records()])\nsubmit = [p for _, p, _ in read_csv(SUB_Df).to_records()]\njoin = list(tagged.keys()) + submit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd0e004efca215f2d92242a17535f0b4c6f3e323"},"cell_type":"code","source":"def expand_path(p):\n    if isfile(TRAIN + p):\n        return TRAIN + p\n    if isfile(TEST + p):\n        return TEST + p\n    return p","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ee3c6a569bfe93b81bcbb97a40e3e78363c79d2"},"cell_type":"markdown","source":"## Duplicate image identification\n\nThis part was from the original kernel, seems like in the playground competition dulicated images was a real issue. I don't know the case about this one but I took one for the team and generated the results anyway. I'm such a nice chap."},{"metadata":{"trusted":true,"_uuid":"aa24819e1cc9ef98db60d218cc01e56f8d4e5046"},"cell_type":"code","source":"if isfile(P2SIZE):\n    print(\"P2SIZE exists.\")\n    with open(P2SIZE, 'rb') as f:\n        p2size = pickle.load(f)\nelse:\n    p2size = {}\n    for p in tqdm(join):\n        size = pil_image.open(expand_path(p)).size\n        p2size[p] = size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cdc36543403de9841a8af37f40c0044e17f6aa69"},"cell_type":"code","source":"def match(h1, h2):\n    for p1 in h2ps[h1]:\n        for p2 in h2ps[h2]:\n            i1 = pil_image.open(expand_path(p1))\n            i2 = pil_image.open(expand_path(p2))\n            if i1.mode != i2.mode or i1.size != i2.size: return False\n            a1 = np.array(i1)\n            a1 = a1 - a1.mean()\n            a1 = a1 / sqrt((a1 ** 2).mean())\n            a2 = np.array(i2)\n            a2 = a2 - a2.mean()\n            a2 = a2 / sqrt((a2 ** 2).mean())\n            a = ((a1 - a2) ** 2).mean()\n            if a > 0.1: return False\n    return True\n\n\nif isfile(P2H):\n    print(\"P2H exists.\")\n    with open(P2H, 'rb') as f:\n        p2h = pickle.load(f)\nelse:\n    # Compute phash for each image in the training and test set.\n    p2h = {}\n    for p in tqdm(join):\n        img = pil_image.open(expand_path(p))\n        h = phash(img)\n        p2h[p] = h\n\n    # Find all images associated with a given phash value.\n    h2ps = {}\n    for p, h in p2h.items():\n        if h not in h2ps: h2ps[h] = []\n        if p not in h2ps[h]: h2ps[h].append(p)\n\n    # Find all distinct phash values\n    hs = list(h2ps.keys())\n\n    # If the images are close enough, associate the two phash values (this is the slow part: n^2 algorithm)\n    h2h = {}\n    for i, h1 in enumerate(tqdm(hs)):\n        for h2 in hs[:i]:\n            if h1 - h2 <= 6 and match(h1, h2):\n                s1 = str(h1)\n                s2 = str(h2)\n                if s1 < s2: s1, s2 = s2, s1\n                h2h[s1] = s2\n\n    # Group together images with equivalent phash, and replace by string format of phash (faster and more readable)\n    for p, h in p2h.items():\n        h = str(h)\n        if h in h2h: h = h2h[h]\n        p2h[p] = h\n#     with open(P2H, 'wb') as f:\n#         pickle.dump(p2h, f)\n# For each image id, determine the list of pictures\nh2ps = {}\nfor p, h in p2h.items():\n    if h not in h2ps: h2ps[h] = []\n    if p not in h2ps[h]: h2ps[h].append(p)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62cc064dede5463c4f553da833020bee789f0de5"},"cell_type":"code","source":"def show_whale(imgs, per_row=2):\n    n = len(imgs)\n    rows = (n + per_row - 1) // per_row\n    cols = min(per_row, n)\n    fig, axes = plt.subplots(rows, cols, figsize=(24 // per_row * cols, 24 // per_row * rows))\n    for ax in axes.flatten(): ax.axis('off')\n    for i, (img, ax) in enumerate(zip(imgs, axes.flatten())): ax.imshow(img.convert('RGB'))\n        \n\ndef read_raw_image(p):\n    img = pil_image.open(expand_path(p))\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87e36806009c7252803a2f49d7a071dbacfafda9"},"cell_type":"code","source":"# For each images id, select the prefered image\ndef prefer(ps):\n    if len(ps) == 1: return ps[0]\n    best_p = ps[0]\n    best_s = p2size[best_p]\n    for i in range(1, len(ps)):\n        p = ps[i]\n        s = p2size[p]\n        if s[0] * s[1] > best_s[0] * best_s[1]:  # Select the image with highest resolution\n            best_p = p\n            best_s = s\n    return best_p\n\nh2p = {}\nfor h, ps in h2ps.items():\n    h2p[h] = prefer(ps)\nlen(h2p), list(h2p.items())[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"208f5d98270a8081f7024c52f4dad14791e48182"},"cell_type":"code","source":"# Read the bounding box data from the bounding box kernel (see reference above)\np2bb = pd.read_csv(BB_DF).set_index(\"Image\")\n\nold_stderr = sys.stderr\nsys.stderr = open('/dev/null' if platform.system() != 'Windows' else 'nul', 'w')\n\nsys.stderr = old_stderr\n\nimg_shape = (384, 384, 1)  # The image shape used by the model\nanisotropy = 2.15  # The horizontal compression ratio\ncrop_margin = 0.05  # The margin added around the bounding box to compensate for bounding box inaccuracy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b0629a511b86810a0106f23c5e25710242edf11"},"cell_type":"code","source":"def build_transform(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    \"\"\"\n    Build a transformation matrix with the specified characteristics.\n    \"\"\"\n    rotation = np.deg2rad(rotation)\n    shear = np.deg2rad(shear)\n    rotation_matrix = np.array(\n        [[np.cos(rotation), np.sin(rotation), 0], [-np.sin(rotation), np.cos(rotation), 0], [0, 0, 1]])\n    shift_matrix = np.array([[1, 0, height_shift], [0, 1, width_shift], [0, 0, 1]])\n    shear_matrix = np.array([[1, np.sin(shear), 0], [0, np.cos(shear), 0], [0, 0, 1]])\n    zoom_matrix = np.array([[1.0 / height_zoom, 0, 0], [0, 1.0 / width_zoom, 0], [0, 0, 1]])\n    shift_matrix = np.array([[1, 0, -height_shift], [0, 1, -width_shift], [0, 0, 1]])\n    return np.dot(np.dot(rotation_matrix, shear_matrix), np.dot(zoom_matrix, shift_matrix))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9bf0641d287366059c95fcbe0f5e2812f4bf236"},"cell_type":"code","source":"def read_cropped_image(p, augment):\n    \"\"\"\n    @param p : the name of the picture to read\n    @param augment: True/False if data augmentation should be performed\n    @return a numpy array with the transformed image\n    \"\"\"\n    # If an image id was given, convert to filename\n    if p in h2p:\n        p = h2p[p]\n    size_x, size_y = p2size[p]\n\n    # Determine the region of the original image we want to capture based on the bounding box.\n    row = p2bb.loc[p]\n    x0, y0, x1, y1 = row['x0'], row['y0'], row['x1'], row['y1']\n    dx = x1 - x0\n    dy = y1 - y0\n    x0 -= dx * crop_margin\n    x1 += dx * crop_margin + 1\n    y0 -= dy * crop_margin\n    y1 += dy * crop_margin + 1\n    if x0 < 0:\n        x0 = 0\n    if x1 > size_x:\n        x1 = size_x\n    if y0 < 0:\n        y0 = 0\n    if y1 > size_y:\n        y1 = size_y\n    dx = x1 - x0\n    dy = y1 - y0\n    if dx > dy * anisotropy:\n        dy = 0.5 * (dx / anisotropy - dy)\n        y0 -= dy\n        y1 += dy\n    else:\n        dx = 0.5 * (dy * anisotropy - dx)\n        x0 -= dx\n        x1 += dx\n\n    # Generate the transformation matrix\n    trans = np.array([[1, 0, -0.5 * img_shape[0]], [0, 1, -0.5 * img_shape[1]], [0, 0, 1]])\n    trans = np.dot(np.array([[(y1 - y0) / img_shape[0], 0, 0], [0, (x1 - x0) / img_shape[1], 0], [0, 0, 1]]), trans)\n    if augment:\n        trans = np.dot(build_transform(\n            random.uniform(-5, 5),\n            random.uniform(-5, 5),\n            random.uniform(0.8, 1.0),\n            random.uniform(0.8, 1.0),\n            random.uniform(-0.05 * (y1 - y0), 0.05 * (y1 - y0)),\n            random.uniform(-0.05 * (x1 - x0), 0.05 * (x1 - x0))\n        ), trans)\n    trans = np.dot(np.array([[1, 0, 0.5 * (y1 + y0)], [0, 1, 0.5 * (x1 + x0)], [0, 0, 1]]), trans)\n\n    # Read the image, transform to black and white and comvert to numpy array\n    img = read_raw_image(p).convert('L')\n    img = img_to_array(img)\n\n    # Apply affine transformation\n    matrix = trans[:2, :2]\n    offset = trans[:2, 2]\n    img = img.reshape(img.shape[:-1])\n    img = affine_transform(img, matrix, offset, output_shape=img_shape[:-1], order=1, mode='constant',\n                           cval=np.average(img))\n    img = img.reshape(img_shape)\n\n    # Normalize to zero mean and unit variance\n    img -= np.mean(img, keepdims=True)\n    img /= np.std(img, keepdims=True) + K.epsilon()\n    return img\n\ndef read_for_training(p):\n    \"\"\"\n    Read and preprocess an image with data augmentation (random transform).\n    \"\"\"\n    return read_cropped_image(p, True)\n\n\ndef read_for_validation(p):\n    \"\"\"\n    Read and preprocess an image without data augmentation (use for testing).\n    \"\"\"\n    return read_cropped_image(p, False)\n\n\np = list(tagged.keys())[312]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"725e7e521b6bf776e8a165614dba50cb597d3a1c"},"cell_type":"code","source":"def subblock(x, filter, **kwargs):\n    x = BatchNormalization()(x)\n    y = x\n    y = Conv2D(filter, (1, 1), activation='relu', **kwargs)(y)  # Reduce the number of features to 'filter'\n    y = BatchNormalization()(y)\n    y = Conv2D(filter, (3, 3), activation='relu', **kwargs)(y)  # Extend the feature field\n    y = BatchNormalization()(y)\n    y = Conv2D(K.int_shape(x)[-1], (1, 1), **kwargs)(y)  # no activation # Restore the number of original features\n    y = Add()([x, y])  # Add the bypass connection\n    y = Activation('relu')(y)\n    return y\n\n\ndef build_model(lr, l2, activation='sigmoid'):\n    ##############\n    # BRANCH MODEL\n    ##############\n    regul = regularizers.l2(l2)\n    optim = Adam(lr=lr)\n    kwargs = {'padding': 'same', 'kernel_regularizer': regul}\n\n    inp = Input(shape=img_shape)  # 384x384x1\n    x = Conv2D(64, (9, 9), strides=2, activation='relu', **kwargs)(inp)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 96x96x64\n    for _ in range(2):\n        x = BatchNormalization()(x)\n        x = Conv2D(64, (3, 3), activation='relu', **kwargs)(x)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 48x48x64\n    x = BatchNormalization()(x)\n    x = Conv2D(128, (1, 1), activation='relu', **kwargs)(x)  # 48x48x128\n    for _ in range(4):\n        x = subblock(x, 64, **kwargs)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 24x24x128\n    x = BatchNormalization()(x)\n    x = Conv2D(256, (1, 1), activation='relu', **kwargs)(x)  # 24x24x256\n    for _ in range(4):\n        x = subblock(x, 64, **kwargs)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 12x12x256\n    x = BatchNormalization()(x)\n    x = Conv2D(384, (1, 1), activation='relu', **kwargs)(x)  # 12x12x384\n    for _ in range(4):\n        x = subblock(x, 96, **kwargs)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 6x6x384\n    x = BatchNormalization()(x)\n    x = Conv2D(512, (1, 1), activation='relu', **kwargs)(x)  # 6x6x512\n    for _ in range(4):\n        x = subblock(x, 128, **kwargs)\n\n    x = GlobalMaxPooling2D()(x)  # 512\n    branch_model = Model(inp, x)\n\n    ############\n    # HEAD MODEL\n    ############\n    mid = 32\n    xa_inp = Input(shape=branch_model.output_shape[1:])\n    xb_inp = Input(shape=branch_model.output_shape[1:])\n    x1 = Lambda(lambda x: x[0] * x[1])([xa_inp, xb_inp])\n    x2 = Lambda(lambda x: x[0] + x[1])([xa_inp, xb_inp])\n    x3 = Lambda(lambda x: K.abs(x[0] - x[1]))([xa_inp, xb_inp])\n    x4 = Lambda(lambda x: K.square(x))(x3)\n    x = Concatenate()([x1, x2, x3, x4])\n    x = Reshape((4, branch_model.output_shape[1], 1), name='reshape1')(x)\n\n    # Per feature NN with shared weight is implemented using CONV2D with appropriate stride.\n    x = Conv2D(mid, (4, 1), activation='relu', padding='valid')(x)\n    x = Reshape((branch_model.output_shape[1], mid, 1))(x)\n    x = Conv2D(1, (1, mid), activation='linear', padding='valid')(x)\n    x = Flatten(name='flatten')(x)\n\n    # Weighted sum implemented as a Dense layer.\n    x = Dense(1, use_bias=True, activation=activation, name='weighted-average')(x)\n    head_model = Model([xa_inp, xb_inp], x, name='head')\n\n    ########################\n    # SIAMESE NEURAL NETWORK\n    ########################\n    # Complete model is constructed by calling the branch model on each input image,\n    # and then the head model on the resulting 512-vectors.\n    img_a = Input(shape=img_shape)\n    img_b = Input(shape=img_shape)\n    xa = branch_model(img_a)\n    xb = branch_model(img_b)\n    x = head_model([xa, xb])\n    model = Model([img_a, img_b], x)\n    model.compile(optim, loss='binary_crossentropy', metrics=['binary_crossentropy', 'acc'])\n    return model, branch_model, head_model\n\n\nmodel, branch_model, head_model = build_model(64e-5, 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8266e1cc94c9c56ff75352aef4194aa37574972"},"cell_type":"code","source":"h2ws = {}\nnew_whale = 'new_whale'\nfor p, w in tagged.items():\n    if w != new_whale:  # Use only identified whales\n        h = p2h[p]\n        if h not in h2ws: h2ws[h] = []\n        if w not in h2ws[h]: h2ws[h].append(w)\nfor h, ws in h2ws.items():\n    if len(ws) > 1:\n        h2ws[h] = sorted(ws)\n\n# For each whale, find the unambiguous images ids.\nw2hs = {}\nfor h, ws in h2ws.items():\n    if len(ws) == 1:  # Use only unambiguous pictures\n        w = ws[0]\n        if w not in w2hs: w2hs[w] = []\n        if h not in w2hs[w]: w2hs[w].append(h)\nfor w, hs in w2hs.items():\n    if len(hs) > 1:\n        w2hs[w] = sorted(hs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f122bcb1d87721a96a18c31aa2f46aaad4037039"},"cell_type":"code","source":"train = []  # A list of training image ids\nfor hs in w2hs.values():\n    if len(hs) > 1:\n        train += hs\nrandom.shuffle(train)\ntrain_set = set(train)\n\nw2ts = {}  # Associate the image ids from train to each whale id.\nfor w, hs in w2hs.items():\n    for h in hs:\n        if h in train_set:\n            if w not in w2ts:\n                w2ts[w] = []\n            if h not in w2ts[w]:\n                w2ts[w].append(h)\nfor w, ts in w2ts.items():\n    w2ts[w] = np.array(ts)\n\nt2i = {}  # The position in train of each training image id\nfor i, t in enumerate(train):\n    t2i[t] = i","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa16cfd51fdbce44271883869486c1cad32f1091"},"cell_type":"code","source":"\nclass TrainingData(Sequence):\n    def __init__(self, score, steps=1000, batch_size=32):\n        \"\"\"\n        @param score the cost matrix for the picture matching\n        @param steps the number of epoch we are planning with this score matrix\n        \"\"\"\n        super(TrainingData, self).__init__()\n        self.score = -score  # Maximizing the score is the same as minimuzing -score.\n        self.steps = steps\n        self.batch_size = batch_size\n        for ts in w2ts.values():\n            idxs = [t2i[t] for t in ts]\n            for i in idxs:\n                for j in idxs:\n                    self.score[\n                        i, j] = 10000.0  # Set a large value for matching whales -- eliminates this potential pairing\n        self.on_epoch_end()\n\n    def __getitem__(self, index):\n        start = self.batch_size * index\n        end = min(start + self.batch_size, len(self.match) + len(self.unmatch))\n        size = end - start\n        assert size > 0\n        a = np.zeros((size,) + img_shape, dtype=K.floatx())\n        b = np.zeros((size,) + img_shape, dtype=K.floatx())\n        c = np.zeros((size, 1), dtype=K.floatx())\n        j = start // 2\n        for i in range(0, size, 2):\n            a[i, :, :, :] = read_for_training(self.match[j][0])\n            b[i, :, :, :] = read_for_training(self.match[j][1])\n            c[i, 0] = 1  # This is a match\n            a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])\n            b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])\n            c[i + 1, 0] = 0  # Different whales\n            j += 1\n        return [a, b], c\n\n    def on_epoch_end(self):\n        if self.steps <= 0: return  # Skip this on the last epoch.\n        self.steps -= 1\n        self.match = []\n        self.unmatch = []\n        x, _, _  = lapjv(self.score)  # Solve the linear assignment problem\n        y = np.arange(len(x), dtype=np.int32)\n\n        # Compute a derangement for matching whales\n        for ts in w2ts.values():\n            d = ts.copy()\n            while True:\n                random.shuffle(d)\n                if not np.any(ts == d): break\n            for ab in zip(ts, d): self.match.append(ab)\n\n        # Construct unmatched whale pairs from the LAP solution.\n        for i, j in zip(x, y):\n            if i == j:\n                print(self.score)\n                print(x)\n                print(y)\n                print(i, j)\n            assert i != j\n            self.unmatch.append((train[i], train[j]))\n\n        # Force a different choice for an eventual next epoch.\n        self.score[x, y] = 10000.0\n        self.score[y, x] = 10000.0\n        random.shuffle(self.match)\n        random.shuffle(self.unmatch)\n        # print(len(self.match), len(train), len(self.unmatch), len(train))\n        assert len(self.match) == len(train) and len(self.unmatch) == len(train)\n\n    def __len__(self):\n        return (len(self.match) + len(self.unmatch) + self.batch_size - 1) // self.batch_size\n\n\n# Test on a batch of 32 with random costs.\nscore = np.random.random_sample(size=(len(train), len(train)))\ndata = TrainingData(score)\n(a, b), c = data[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"005d041475be5ec97ac8e1d937bea6e7253226af"},"cell_type":"code","source":"# A Keras generator to evaluate only the BRANCH MODEL\nclass FeatureGen(Sequence):\n    def __init__(self, data, batch_size=64, verbose=1):\n        super(FeatureGen, self).__init__()\n        self.data = data\n        self.batch_size = batch_size\n        self.verbose = verbose\n        if self.verbose > 0: self.progress = tqdm(total=len(self), desc='Features')\n\n    def __getitem__(self, index):\n        start = self.batch_size * index\n        size = min(len(self.data) - start, self.batch_size)\n        a = np.zeros((size,) + img_shape, dtype=K.floatx())\n        for i in range(size): a[i, :, :, :] = read_for_validation(self.data[start + i])\n        if self.verbose > 0:\n            self.progress.update()\n            if self.progress.n >= len(self): self.progress.close()\n        return a\n\n    def __len__(self):\n        return (len(self.data) + self.batch_size - 1) // self.batch_size\n\n\nclass ScoreGen(Sequence):\n    def __init__(self, x, y=None, batch_size=2048, verbose=1):\n        super(ScoreGen, self).__init__()\n        self.x = x\n        self.y = y\n        self.batch_size = batch_size\n        self.verbose = verbose\n        if y is None:\n            self.y = self.x\n            self.ix, self.iy = np.triu_indices(x.shape[0], 1)\n        else:\n            self.iy, self.ix = np.indices((y.shape[0], x.shape[0]))\n            self.ix = self.ix.reshape((self.ix.size,))\n            self.iy = self.iy.reshape((self.iy.size,))\n        self.subbatch = (len(self.x) + self.batch_size - 1) // self.batch_size\n        if self.verbose > 0:\n            self.progress = tqdm(total=len(self), desc='Scores')\n\n    def __getitem__(self, index):\n        start = index * self.batch_size\n        end = min(start + self.batch_size, len(self.ix))\n        a = self.y[self.iy[start:end], :]\n        b = self.x[self.ix[start:end], :]\n        if self.verbose > 0:\n            self.progress.update()\n            if self.progress.n >= len(self): self.progress.close()\n        return [a, b]\n\n    def __len__(self):\n        return (len(self.ix) + self.batch_size - 1) // self.batch_size\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ac5f82779a71cca2bc2cd1bac8bac385cb48a23"},"cell_type":"code","source":"def set_lr(model, lr):\n    K.set_value(model.optimizer.lr, float(lr))\n\n\ndef get_lr(model):\n    return K.get_value(model.optimizer.lr)\n\n\ndef score_reshape(score, x, y=None):\n    \"\"\"\n    Tranformed the packed matrix 'score' into a square matrix.\n    @param score the packed matrix\n    @param x the first image feature tensor\n    @param y the second image feature tensor if different from x\n    @result the square matrix\n    \"\"\"\n    if y is None:\n        # When y is None, score is a packed upper triangular matrix.\n        # Unpack, and transpose to form the symmetrical lower triangular matrix.\n        m = np.zeros((x.shape[0], x.shape[0]), dtype=K.floatx())\n        m[np.triu_indices(x.shape[0], 1)] = score.squeeze()\n        m += m.transpose()\n    else:\n        m = np.zeros((y.shape[0], x.shape[0]), dtype=K.floatx())\n        iy, ix = np.indices((y.shape[0], x.shape[0]))\n        ix = ix.reshape((ix.size,))\n        iy = iy.reshape((iy.size,))\n        m[iy, ix] = score.squeeze()\n    return m\n\n\ndef compute_score(verbose=1):\n    \"\"\"\n    Compute the score matrix by scoring every pictures from the training set against every other picture O(n^2).\n    \"\"\"\n    features = branch_model.predict_generator(FeatureGen(train, verbose=verbose), max_queue_size=12, workers=6,\n                                              verbose=0)\n    score = head_model.predict_generator(ScoreGen(features, verbose=verbose), max_queue_size=12, workers=6, verbose=0)\n    score = score_reshape(score, features)\n    return features, score\n\n\ndef make_steps(step, ampl):\n    \"\"\"\n    Perform training epochs\n    @param step Number of epochs to perform\n    @param ampl the K, the randomized component of the score matrix.\n    \"\"\"\n    global w2ts, t2i, steps, features, score, histories\n\n    # shuffle the training pictures\n    random.shuffle(train)\n\n    # Map whale id to the list of associated training picture hash value\n    w2ts = {}\n    for w, hs in w2hs.items():\n        for h in hs:\n            if h in train_set:\n                if w not in w2ts: w2ts[w] = []\n                if h not in w2ts[w]: w2ts[w].append(h)\n    for w, ts in w2ts.items(): w2ts[w] = np.array(ts)\n\n    # Map training picture hash value to index in 'train' array    \n    t2i = {}\n    for i, t in enumerate(train): t2i[t] = i\n\n    # Compute the match score for each picture pair\n    features, score = compute_score()\n\n    # Train the model for 'step' epochs\n    history = model.fit_generator(\n        TrainingData(score + ampl * np.random.random_sample(size=score.shape), steps=step, batch_size=32),\n        initial_epoch=steps, epochs=steps + step, max_queue_size=12, workers=6, verbose=1).history\n    steps += step\n\n    # Collect history data\n    history['epochs'] = steps\n    history['ms'] = np.mean(score)\n    history['lr'] = get_lr(model)\n    print(history['epochs'], history['lr'], history['ms'])\n    histories.append(history)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87091f9dcd5a0d11dc255d427abd42cb6ba7506e"},"cell_type":"code","source":"histories = []\nsteps = 0\n\nif isfile('./models/standard.model'):\n    tmp = keras.models.load_model('./models/standard.model')\n    model.set_weights(tmp.get_weights())\nelse:\n    # epoch -> 10\n    make_steps(10, 1000)\n    ampl = 100.0\n    for _ in range(2):\n        print('noise ampl.  = ', ampl)\n        make_steps(5, ampl)\n        ampl = max(1.0, 100 ** -0.1 * ampl)\n#     # epoch -> 150\n#     for _ in range(18): make_steps(5, 1.0)\n#     # epoch -> 200\n#     set_lr(model, 16e-5)\n#     for _ in range(10): make_steps(5, 0.5)\n#     # epoch -> 240\n#     set_lr(model, 4e-5)\n#     for _ in range(8): make_steps(5, 0.25)\n#     # epoch -> 250\n#     set_lr(model, 1e-5)\n#     for _ in range(2): make_steps(5, 0.25)\n#     # epoch -> 300\n#     weights = model.get_weights()\n#     model, branch_model, head_model = build_model(64e-5, 0.0002)\n#     model.set_weights(weights)\n#     for _ in range(10): make_steps(5, 1.0)\n#     # epoch -> 350\n#     set_lr(model, 16e-5)\n#     for _ in range(10): make_steps(5, 0.5)\n#     # epoch -> 390\n#     set_lr(model, 4e-5)\n#     for _ in range(8): make_steps(5, 0.25)\n#     # epoch -> 400\n#     set_lr(model, 1e-5)\n#     for _ in range(2): make_steps(5, 0.25)\n#     model.save('standard.model')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f194458bbb728a7cc35b75b40752e98021b6493b"},"cell_type":"code","source":"def prepare_submission(threshold, filename):\n    \"\"\"\n    Generate a Kaggle submission file.\n    @param threshold the score given to 'new_whale'\n    @param filename the submission file name\n    \"\"\"\n    vtop = 0\n    vhigh = 0\n    pos = [0, 0, 0, 0, 0, 0]\n    with open(filename, 'wt', newline='\\n') as f:\n        f.write('Image,Id\\n')\n        for i, p in enumerate(tqdm(submit)):\n            t = []\n            s = set()\n            a = score[i, :]\n            for j in list(reversed(np.argsort(a))):\n                h = known[j]\n                if a[j] < threshold and new_whale not in s:\n                    pos[len(t)] += 1\n                    s.add(new_whale)\n                    t.append(new_whale)\n                    if len(t) == 5: break;\n                for w in h2ws[h]:\n                    assert w != new_whale\n                    if w not in s:\n                        if a[j] > 1.0:\n                            vtop += 1\n                        elif a[j] >= threshold:\n                            vhigh += 1\n                        s.add(w)\n                        t.append(w)\n                        if len(t) == 5: break;\n                if len(t) == 5: break;\n            if new_whale not in s: pos[5] += 1\n            assert len(t) == 5 and len(s) == 5\n            f.write(p + ',' + ' '.join(t[:5]) + '\\n')\n    return vtop, vhigh, pos","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"624143d234c6cc5f7374a2451aab84b955cecb09"},"cell_type":"code","source":"# Find elements from training sets not 'new_whale'\nh2ws = {}\nfor p, w in tagged.items():\n    if w != new_whale:  # Use only identified whales\n        h = p2h[p]\n        if h not in h2ws: h2ws[h] = []\n        if w not in h2ws[h]: h2ws[h].append(w)\nknown = sorted(list(h2ws.keys()))\n\n# Dictionary of picture indices\nh2i = {}\nfor i, h in enumerate(known): h2i[h] = i\n\n# Evaluate the model.\nfknown = branch_model.predict_generator(FeatureGen(known), max_queue_size=20, workers=10, verbose=0)\nfsubmit = branch_model.predict_generator(FeatureGen(submit), max_queue_size=20, workers=10, verbose=0)\nscore = head_model.predict_generator(ScoreGen(fknown, fsubmit), max_queue_size=20, workers=10, verbose=0)\nscore = score_reshape(score, fknown, fsubmit)\n\n# Generate the subsmission file.\nprepare_submission(0.99, 'submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d438a9807bb20cd7b4c3a26abbc276f0c9306c9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}