{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":1,"outputs":[{"output_type":"stream","text":"['metadata', 'piotte', 'humpback-whale-identification']\n","name":"stdout"}]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"!pip install lap\n# Read the dataset description\nimport gzip\n# Read or generate p2h, a dictionary of image name to image id (picture to hash)\nimport pickle\nimport platform\nimport random\n# Suppress annoying stderr output when importing keras.\nimport sys\nfrom lap import lapjv\nfrom math import sqrt\n# Determine the size of each image\nfrom os.path import isfile\n\nimport keras\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image as pil_image\nfrom imagehash import phash\nfrom keras import backend as K\nfrom keras import regularizers\nfrom keras.engine.topology import Input\nfrom keras.layers import Activation, Add, BatchNormalization, Concatenate, Conv2D, Dense, Flatten, GlobalMaxPooling2D, \\\n    Lambda, MaxPooling2D, Reshape\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom keras.preprocessing.image import img_to_array\nfrom keras.utils import Sequence\nfrom pandas import read_csv\nfrom scipy.ndimage import affine_transform\nfrom tqdm import tqdm_notebook as tqdm\nimport time","execution_count":2,"outputs":[{"output_type":"stream","text":"Collecting lap\n\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bf/64/d9fb6a75b15e783952b2fec6970f033462e67db32dc43dfbb404c14e91c2/lap-0.4.0.tar.gz (1.5MB)\n\u001b[K    100% |████████████████████████████████| 1.5MB 16.0MB/s ta 0:00:01\n\u001b[?25hBuilding wheels for collected packages: lap\n  Building wheel for lap (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Stored in directory: /tmp/.cache/pip/wheels/da/3e/af/eddcd6ffaa27df8d0ddac573758f8953c4e57c64c4c8c8b7d0\nSuccessfully built lap\nInstalling collected packages: lap\nSuccessfully installed lap-0.4.0\n\u001b[33mYou are using pip version 19.0.3, however version 19.1.1 is available.\nYou should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n","name":"stdout"},{"output_type":"stream","text":"Using TensorFlow backend.\n","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_DF = '../input/humpback-whale-identification/train.csv'\nSUB_Df = '../input/humpback-whale-identification/sample_submission.csv'\nTRAIN = '../input/humpback-whale-identification/train/'\nTEST = '../input/humpback-whale-identification/test/'\nP2H = '../input/metadata/p2h.pickle'\nP2SIZE = '../input/metadata/p2size.pickle'\nBB_DF = \"../input/metadata/bounding_boxes.csv\"\ntagged = dict([(p, w) for _, p, w in read_csv(TRAIN_DF).to_records()])\nsubmit = [p for _, p, _ in read_csv(SUB_Df).to_records()]\njoin = list(tagged.keys()) + submit","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def expand_path(p):\n    if isfile(TRAIN + p):\n        return TRAIN + p\n    if isfile(TEST + p):\n        return TEST + p\n    return p","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if isfile(P2SIZE):\n    print(\"P2SIZE exists.\")\n    with open(P2SIZE, 'rb') as f:\n        p2size = pickle.load(f)\nelse:\n    p2size = {}\n    for p in tqdm(join):\n        size = pil_image.open(expand_path(p)).size\n        p2size[p] = size","execution_count":5,"outputs":[{"output_type":"stream","text":"P2SIZE exists.\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"def match(h1, h2):\n    for p1 in h2ps[h1]:\n        for p2 in h2ps[h2]:\n            i1 = pil_image.open(expand_path(p1))\n            i2 = pil_image.open(expand_path(p2))\n            if i1.mode != i2.mode or i1.size != i2.size: return False\n            a1 = np.array(i1)\n            a1 = a1 - a1.mean()\n            a1 = a1 / sqrt((a1 ** 2).mean())\n            a2 = np.array(i2)\n            a2 = a2 - a2.mean()\n            a2 = a2 / sqrt((a2 ** 2).mean())\n            a = ((a1 - a2) ** 2).mean()\n            if a > 0.1: return False\n    return True\n\n\nif isfile(P2H):\n    print(\"P2H exists.\")\n    with open(P2H, 'rb') as f:\n        p2h = pickle.load(f)\nelse:\n    # Compute phash for each image in the training and test set.\n    p2h = {}\n    for p in tqdm(join):\n        img = pil_image.open(expand_path(p))\n        h = phash(img)\n        p2h[p] = h\n\n    # Find all images associated with a given phash value.\n    h2ps = {}\n    for p, h in p2h.items():\n        if h not in h2ps: h2ps[h] = []\n        if p not in h2ps[h]: h2ps[h].append(p)\n\n    # Find all distinct phash values\n    hs = list(h2ps.keys())\n\n    # If the images are close enough, associate the two phash values (this is the slow part: n^2 algorithm)\n    h2h = {}\n    for i, h1 in enumerate(tqdm(hs)):\n        for h2 in hs[:i]:\n            if h1 - h2 <= 6 and match(h1, h2):\n                s1 = str(h1)\n                s2 = str(h2)\n                if s1 < s2: s1, s2 = s2, s1\n                h2h[s1] = s2\n\n    # Group together images with equivalent phash, and replace by string format of phash (faster and more readable)\n    for p, h in p2h.items():\n        h = str(h)\n        if h in h2h: h = h2h[h]\n        p2h[p] = h\n#     with open(P2H, 'wb') as f:\n#         pickle.dump(p2h, f)\n# For each image id, determine the list of pictures\nh2ps = {}\nfor p, h in p2h.items():\n    if h not in h2ps: h2ps[h] = []\n    if p not in h2ps[h]: h2ps[h].append(p)","execution_count":6,"outputs":[{"output_type":"stream","text":"P2H exists.\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_whale(imgs, per_row=2):\n    n = len(imgs)\n    rows = (n + per_row - 1) // per_row\n    cols = min(per_row, n)\n    fig, axes = plt.subplots(rows, cols, figsize=(24 // per_row * cols, 24 // per_row * rows))\n    for ax in axes.flatten(): ax.axis('off')\n    for i, (img, ax) in enumerate(zip(imgs, axes.flatten())): ax.imshow(img.convert('RGB'))\n        \n\ndef read_raw_image(p):\n    img = pil_image.open(expand_path(p))\n    return img","execution_count":7,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For each images id, select the prefered image\ndef prefer(ps):\n    if len(ps) == 1: return ps[0]\n    best_p = ps[0]\n    best_s = p2size[best_p]\n    for i in range(1, len(ps)):\n        p = ps[i]\n        s = p2size[p]\n        if s[0] * s[1] > best_s[0] * best_s[1]:  # Select the image with highest resolution\n            best_p = p\n            best_s = s\n    return best_p\n\nh2p = {}\nfor h, ps in h2ps.items():\n    h2p[h] = prefer(ps)\nlen(h2p), list(h2p.items())[:5]","execution_count":8,"outputs":[{"output_type":"execute_result","execution_count":8,"data":{"text/plain":"(33317,\n [('d26698c3271c757c', '0000e88ab.jpg'),\n  ('ba8cc231ad489b77', '0001f9222.jpg'),\n  ('bbcad234a52d0f0b', '00029d126.jpg'),\n  ('c09ae7dc09f33a29', '00050a15a.jpg'),\n  ('d02f65ba9f74a08a', '0005c1ef8.jpg')])"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read the bounding box data from the bounding box kernel (see reference above)\np2bb = pd.read_csv(BB_DF).set_index(\"Image\")\n\nold_stderr = sys.stderr\nsys.stderr = open('/dev/null' if platform.system() != 'Windows' else 'nul', 'w')\n\nsys.stderr = old_stderr\n\nimg_shape = (384, 384, 1)  # The image shape used by the model\nanisotropy = 2.15  # The horizontal compression ratio\ncrop_margin = 0.05  # The margin added around the bounding box to compensate for bounding box inaccuracy","execution_count":9,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_transform(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    \"\"\"\n    Build a transformation matrix with the specified characteristics.\n    \"\"\"\n    rotation = np.deg2rad(rotation)\n    shear = np.deg2rad(shear)\n    rotation_matrix = np.array(\n        [[np.cos(rotation), np.sin(rotation), 0], [-np.sin(rotation), np.cos(rotation), 0], [0, 0, 1]])\n    shift_matrix = np.array([[1, 0, height_shift], [0, 1, width_shift], [0, 0, 1]])\n    shear_matrix = np.array([[1, np.sin(shear), 0], [0, np.cos(shear), 0], [0, 0, 1]])\n    zoom_matrix = np.array([[1.0 / height_zoom, 0, 0], [0, 1.0 / width_zoom, 0], [0, 0, 1]])\n    shift_matrix = np.array([[1, 0, -height_shift], [0, 1, -width_shift], [0, 0, 1]])\n    return np.dot(np.dot(rotation_matrix, shear_matrix), np.dot(zoom_matrix, shift_matrix))","execution_count":10,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_cropped_image(p, augment):\n    \"\"\"\n    @param p : the name of the picture to read\n    @param augment: True/False if data augmentation should be performed\n    @return a numpy array with the transformed image\n    \"\"\"\n    # If an image id was given, convert to filename\n    if p in h2p:\n        p = h2p[p]\n    size_x, size_y = p2size[p]\n\n    # Determine the region of the original image we want to capture based on the bounding box.\n    row = p2bb.loc[p]\n    x0, y0, x1, y1 = row['x0'], row['y0'], row['x1'], row['y1']\n    dx = x1 - x0\n    dy = y1 - y0\n    x0 -= dx * crop_margin\n    x1 += dx * crop_margin + 1\n    y0 -= dy * crop_margin\n    y1 += dy * crop_margin + 1\n    if x0 < 0:\n        x0 = 0\n    if x1 > size_x:\n        x1 = size_x\n    if y0 < 0:\n        y0 = 0\n    if y1 > size_y:\n        y1 = size_y\n    dx = x1 - x0\n    dy = y1 - y0\n    if dx > dy * anisotropy:\n        dy = 0.5 * (dx / anisotropy - dy)\n        y0 -= dy\n        y1 += dy\n    else:\n        dx = 0.5 * (dy * anisotropy - dx)\n        x0 -= dx\n        x1 += dx\n\n    # Generate the transformation matrix\n    trans = np.array([[1, 0, -0.5 * img_shape[0]], [0, 1, -0.5 * img_shape[1]], [0, 0, 1]])\n    trans = np.dot(np.array([[(y1 - y0) / img_shape[0], 0, 0], [0, (x1 - x0) / img_shape[1], 0], [0, 0, 1]]), trans)\n    if augment:\n        trans = np.dot(build_transform(\n            random.uniform(-5, 5),\n            random.uniform(-5, 5),\n            random.uniform(0.8, 1.0),\n            random.uniform(0.8, 1.0),\n            random.uniform(-0.05 * (y1 - y0), 0.05 * (y1 - y0)),\n            random.uniform(-0.05 * (x1 - x0), 0.05 * (x1 - x0))\n        ), trans)\n    trans = np.dot(np.array([[1, 0, 0.5 * (y1 + y0)], [0, 1, 0.5 * (x1 + x0)], [0, 0, 1]]), trans)\n\n    # Read the image, transform to black and white and comvert to numpy array\n    img = read_raw_image(p).convert('L')\n    img = img_to_array(img)\n\n    # Apply affine transformation\n    matrix = trans[:2, :2]\n    offset = trans[:2, 2]\n    img = img.reshape(img.shape[:-1])\n    img = affine_transform(img, matrix, offset, output_shape=img_shape[:-1], order=1, mode='constant',\n                           cval=np.average(img))\n    img = img.reshape(img_shape)\n\n    # Normalize to zero mean and unit variance\n    img -= np.mean(img, keepdims=True)\n    img /= np.std(img, keepdims=True) + K.epsilon()\n    return img\n\ndef read_for_training(p):\n    \"\"\"\n    Read and preprocess an image with data augmentation (random transform).\n    \"\"\"\n    return read_cropped_image(p, True)\n\n\ndef read_for_validation(p):\n    \"\"\"\n    Read and preprocess an image without data augmentation (use for testing).\n    \"\"\"\n    return read_cropped_image(p, False)\n\n\np = list(tagged.keys())[312]","execution_count":11,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def subblock(x, filter, **kwargs):\n    x = BatchNormalization()(x)\n    y = x\n    y = Conv2D(filter, (1, 1), activation='relu', **kwargs)(y)  # Reduce the number of features to 'filter'\n    y = BatchNormalization()(y)\n    y = Conv2D(filter, (3, 3), activation='relu', **kwargs)(y)  # Extend the feature field\n    y = BatchNormalization()(y)\n    y = Conv2D(K.int_shape(x)[-1], (1, 1), **kwargs)(y)  # no activation # Restore the number of original features\n    y = Add()([x, y])  # Add the bypass connection\n    y = Activation('relu')(y)\n    return y\n\n\ndef build_model(lr, l2, activation='sigmoid'):\n    ##############\n    # BRANCH MODEL\n    ##############\n    regul = regularizers.l2(l2)\n    optim = Adam(lr=lr)\n    kwargs = {'padding': 'same', 'kernel_regularizer': regul}\n\n    inp = Input(shape=img_shape)  # 384x384x1\n    x = Conv2D(64, (9, 9), strides=2, activation='relu', **kwargs)(inp)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 96x96x64\n    for _ in range(2):\n        x = BatchNormalization()(x)\n        x = Conv2D(64, (3, 3), activation='relu', **kwargs)(x)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 48x48x64\n    x = BatchNormalization()(x)\n    x = Conv2D(128, (1, 1), activation='relu', **kwargs)(x)  # 48x48x128\n    for _ in range(4):\n        x = subblock(x, 64, **kwargs)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 24x24x128\n    x = BatchNormalization()(x)\n    x = Conv2D(256, (1, 1), activation='relu', **kwargs)(x)  # 24x24x256\n    for _ in range(4):\n        x = subblock(x, 64, **kwargs)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 12x12x256\n    x = BatchNormalization()(x)\n    x = Conv2D(384, (1, 1), activation='relu', **kwargs)(x)  # 12x12x384\n    for _ in range(4):\n        x = subblock(x, 96, **kwargs)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 6x6x384\n    x = BatchNormalization()(x)\n    x = Conv2D(512, (1, 1), activation='relu', **kwargs)(x)  # 6x6x512\n    for _ in range(4):\n        x = subblock(x, 128, **kwargs)\n\n    x = GlobalMaxPooling2D()(x)  # 512\n    branch_model = Model(inp, x)\n\n    ############\n    # HEAD MODEL\n    ############\n    mid = 32\n    xa_inp = Input(shape=branch_model.output_shape[1:])\n    xb_inp = Input(shape=branch_model.output_shape[1:])\n    x1 = Lambda(lambda x: x[0] * x[1])([xa_inp, xb_inp])\n    x2 = Lambda(lambda x: x[0] + x[1])([xa_inp, xb_inp])\n    x3 = Lambda(lambda x: K.abs(x[0] - x[1]))([xa_inp, xb_inp])\n    x4 = Lambda(lambda x: K.square(x))(x3)\n    x = Concatenate()([x1, x2, x3, x4])\n    x = Reshape((4, branch_model.output_shape[1], 1), name='reshape1')(x)\n\n    # Per feature NN with shared weight is implemented using CONV2D with appropriate stride.\n    x = Conv2D(mid, (4, 1), activation='relu', padding='valid')(x)\n    x = Reshape((branch_model.output_shape[1], mid, 1))(x)\n    x = Conv2D(1, (1, mid), activation='linear', padding='valid')(x)\n    x = Flatten(name='flatten')(x)\n\n    # Weighted sum implemented as a Dense layer.\n    x = Dense(1, use_bias=True, activation=activation, name='weighted-average')(x)\n    head_model = Model([xa_inp, xb_inp], x, name='head')\n\n    ########################\n    # SIAMESE NEURAL NETWORK\n    ########################\n    # Complete model is constructed by calling the branch model on each input image,\n    # and then the head model on the resulting 512-vectors.\n    img_a = Input(shape=img_shape)\n    img_b = Input(shape=img_shape)\n    xa = branch_model(img_a)\n    xb = branch_model(img_b)\n    x = head_model([xa, xb])\n    model = Model([img_a, img_b], x)\n    model.compile(optim, loss='binary_crossentropy', metrics=['binary_crossentropy', 'acc'])\n    return model, branch_model, head_model\n\n\nmodel, branch_model, head_model = build_model(64e-5, 0)","execution_count":12,"outputs":[{"output_type":"stream","text":"WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nColocations handled automatically by placer.\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"h2ws = {}\nnew_whale = 'new_whale'\nfor p, w in tagged.items():\n    if w != new_whale:  # Use only identified whales\n        h = p2h[p]\n        if h not in h2ws: h2ws[h] = []\n        if w not in h2ws[h]: h2ws[h].append(w)\nfor h, ws in h2ws.items():\n    if len(ws) > 1:\n        h2ws[h] = sorted(ws)\n\n# For each whale, find the unambiguous images ids.\nw2hs = {}\nfor h, ws in h2ws.items():\n    if len(ws) == 1:  # Use only unambiguous pictures\n        w = ws[0]\n        if w not in w2hs: w2hs[w] = []\n        if h not in w2hs[w]: w2hs[w].append(h)\nfor w, hs in w2hs.items():\n    if len(hs) > 1:\n        w2hs[w] = sorted(hs)","execution_count":13,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = []  # A list of training image ids\nfor hs in w2hs.values():\n    if len(hs) > 1:\n        train += hs\nrandom.shuffle(train)\ntrain_set = set(train)\n\nw2ts = {}  # Associate the image ids from train to each whale id.\nfor w, hs in w2hs.items():\n    for h in hs:\n        if h in train_set:\n            if w not in w2ts:\n                w2ts[w] = []\n            if h not in w2ts[w]:\n                w2ts[w].append(h)\nfor w, ts in w2ts.items():\n    w2ts[w] = np.array(ts)\n\nt2i = {}  # The position in train of each training image id\nfor i, t in enumerate(train):\n    t2i[t] = i","execution_count":14,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainingData(Sequence):\n    def __init__(self, score, steps=1000, batch_size=32):\n        \"\"\"\n        @param score the cost matrix for the picture matching\n        @param steps the number of epoch we are planning with this score matrix\n        \"\"\"\n        super(TrainingData, self).__init__()\n        self.score = -score  # Maximizing the score is the same as minimuzing -score.\n        self.steps = steps\n        self.batch_size = batch_size\n        for ts in w2ts.values():\n            idxs = [t2i[t] for t in ts]\n            for i in idxs:\n                for j in idxs:\n                    self.score[\n                        i, j] = 10000.0  # Set a large value for matching whales -- eliminates this potential pairing\n        self.on_epoch_end()\n\n    def __getitem__(self, index):\n        start = self.batch_size * index\n        end = min(start + self.batch_size, len(self.match) + len(self.unmatch))\n        size = end - start\n        assert size > 0\n        a = np.zeros((size,) + img_shape, dtype=K.floatx())\n        b = np.zeros((size,) + img_shape, dtype=K.floatx())\n        c = np.zeros((size, 1), dtype=K.floatx())\n        j = start // 2\n        for i in range(0, size, 2):\n            a[i, :, :, :] = read_for_training(self.match[j][0])\n            b[i, :, :, :] = read_for_training(self.match[j][1])\n            c[i, 0] = 1  # This is a match\n            a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])\n            b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])\n            c[i + 1, 0] = 0  # Different whales\n            j += 1\n        return [a, b], c\n\n    def on_epoch_end(self):\n        if self.steps <= 0: return  # Skip this on the last epoch.\n        self.steps -= 1\n        self.match = []\n        self.unmatch = []\n        _, _, x = lapjv(self.score)  # Solve the linear assignment problem\n        y = np.arange(len(x), dtype=np.int32)\n\n        # Compute a derangement for matching whales\n        for ts in w2ts.values():\n            d = ts.copy()\n            while True:\n                random.shuffle(d)\n                if not np.any(ts == d): break\n            for ab in zip(ts, d): self.match.append(ab)\n\n        # Construct unmatched whale pairs from the LAP solution.\n        for i, j in zip(x, y):\n            if i == j:\n                print(self.score)\n                print(x)\n                print(y)\n                print(i, j)\n            assert i != j\n            self.unmatch.append((train[i], train[j]))\n\n        # Force a different choice for an eventual next epoch.\n        self.score[x, y] = 10000.0\n        self.score[y, x] = 10000.0\n        random.shuffle(self.match)\n        random.shuffle(self.unmatch)\n        # print(len(self.match), len(train), len(self.unmatch), len(train))\n        assert len(self.match) == len(train) and len(self.unmatch) == len(train)\n\n    def __len__(self):\n        return (len(self.match) + len(self.unmatch) + self.batch_size - 1) // self.batch_size\n\n\n# Test on a batch of 32 with random costs.\nscore = np.random.random_sample(size=(len(train), len(train)))\ndata = TrainingData(score)\n(a, b), c = data[0]","execution_count":15,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# A Keras generator to evaluate only the BRANCH MODEL\nclass FeatureGen(Sequence):\n    def __init__(self, data, batch_size=64, verbose=1):\n        super(FeatureGen, self).__init__()\n        self.data = data\n        self.batch_size = batch_size\n        self.verbose = verbose\n        if self.verbose > 0: self.progress = tqdm(total=len(self), desc='Features')\n\n    def __getitem__(self, index):\n        start = self.batch_size * index\n        size = min(len(self.data) - start, self.batch_size)\n        a = np.zeros((size,) + img_shape, dtype=K.floatx())\n        for i in range(size): a[i, :, :, :] = read_for_validation(self.data[start + i])\n        if self.verbose > 0:\n            self.progress.update()\n            if self.progress.n >= len(self): self.progress.close()\n        return a\n\n    def __len__(self):\n        return (len(self.data) + self.batch_size - 1) // self.batch_size\n\n\nclass ScoreGen(Sequence):\n    def __init__(self, x, y=None, batch_size=2048, verbose=1):\n        super(ScoreGen, self).__init__()\n        self.x = x\n        self.y = y\n        self.batch_size = batch_size\n        self.verbose = verbose\n        if y is None:\n            self.y = self.x\n            self.ix, self.iy = np.triu_indices(x.shape[0], 1)\n        else:\n            self.iy, self.ix = np.indices((y.shape[0], x.shape[0]))\n            self.ix = self.ix.reshape((self.ix.size,))\n            self.iy = self.iy.reshape((self.iy.size,))\n        self.subbatch = (len(self.x) + self.batch_size - 1) // self.batch_size\n        if self.verbose > 0:\n            self.progress = tqdm(total=len(self), desc='Scores')\n\n    def __getitem__(self, index):\n        start = index * self.batch_size\n        end = min(start + self.batch_size, len(self.ix))\n        a = self.y[self.iy[start:end], :]\n        b = self.x[self.ix[start:end], :]\n        if self.verbose > 0:\n            self.progress.update()\n            if self.progress.n >= len(self): self.progress.close()\n        return [a, b]\n\n    def __len__(self):\n        return (len(self.ix) + self.batch_size - 1) // self.batch_size","execution_count":16,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_lr(model, lr):\n    K.set_value(model.optimizer.lr, float(lr))\n\n\ndef get_lr(model):\n    return K.get_value(model.optimizer.lr)\n\n\ndef score_reshape(score, x, y=None):\n    \"\"\"\n    Tranformed the packed matrix 'score' into a square matrix.\n    @param score the packed matrix\n    @param x the first image feature tensor\n    @param y the second image feature tensor if different from x\n    @result the square matrix\n    \"\"\"\n    if y is None:\n        # When y is None, score is a packed upper triangular matrix.\n        # Unpack, and transpose to form the symmetrical lower triangular matrix.\n        m = np.zeros((x.shape[0], x.shape[0]), dtype=K.floatx())\n        m[np.triu_indices(x.shape[0], 1)] = score.squeeze()\n        m += m.transpose()\n    else:\n        m = np.zeros((y.shape[0], x.shape[0]), dtype=K.floatx())\n        iy, ix = np.indices((y.shape[0], x.shape[0]))\n        ix = ix.reshape((ix.size,))\n        iy = iy.reshape((iy.size,))\n        m[iy, ix] = score.squeeze()\n    return m\n\n\ndef compute_score(verbose=1):\n    \"\"\"\n    Compute the score matrix by scoring every pictures from the training set against every other picture O(n^2).\n    \"\"\"\n    features = branch_model.predict_generator(FeatureGen(train, verbose=verbose), max_queue_size=12, workers=6,\n                                              verbose=0)\n    score = head_model.predict_generator(ScoreGen(features, verbose=verbose), max_queue_size=12, workers=6, verbose=0)\n    score = score_reshape(score, features)\n    return features, score\n\n\ndef make_steps(step, ampl):\n    \"\"\"\n    Perform training epochs\n    @param step Number of epochs to perform\n    @param ampl the K, the randomized component of the score matrix.\n    \"\"\"\n    global w2ts, t2i, steps, features, score, histories\n\n    # shuffle the training pictures\n    random.shuffle(train)\n\n    # Map whale id to the list of associated training picture hash value\n    w2ts = {}\n    for w, hs in w2hs.items():\n        for h in hs:\n            if h in train_set:\n                if w not in w2ts: w2ts[w] = []\n                if h not in w2ts[w]: w2ts[w].append(h)\n    for w, ts in w2ts.items(): w2ts[w] = np.array(ts)\n\n    # Map training picture hash value to index in 'train' array    \n    t2i = {}\n    for i, t in enumerate(train): t2i[t] = i\n\n    # Compute the match score for each picture pair\n    features, score = compute_score()\n\n    # Train the model for 'step' epochs\n    history = model.fit_generator(\n        TrainingData(score + ampl * np.random.random_sample(size=score.shape), steps=step, batch_size=32),\n        initial_epoch=steps, epochs=steps + step, max_queue_size=12, workers=6, verbose=1).history\n    steps += step\n\n    # Collect history data\n    history['epochs'] = steps\n    history['ms'] = np.mean(score)\n    history['lr'] = get_lr(model)\n    print(history['epochs'], history['lr'], history['ms'])\n    histories.append(history)","execution_count":17,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_submission(threshold, filename):\n    \"\"\"\n    Generate a Kaggle submission file.\n    @param threshold the score given to 'new_whale'\n    @param filename the submission file name\n    \"\"\"\n    vtop = 0\n    vhigh = 0\n    pos = [0, 0, 0, 0, 0, 0]\n    with open(filename, 'wt', newline='\\n') as f:\n        f.write('Image,Id\\n')\n        for i, p in enumerate(tqdm(submit)):\n            t = []\n            s = set()\n            a = score[i, :]\n            for j in list(reversed(np.argsort(a))):\n                h = known[j]\n                if a[j] < threshold and new_whale not in s:\n                    pos[len(t)] += 1\n                    s.add(new_whale)\n                    t.append(new_whale)\n                    if len(t) == 5: break;\n                for w in h2ws[h]:\n                    assert w != new_whale\n                    if w not in s:\n                        if a[j] > 1.0:\n                            vtop += 1\n                        elif a[j] >= threshold:\n                            vhigh += 1\n                        s.add(w)\n                        t.append(w)\n                        if len(t) == 5: break;\n                if len(t) == 5: break;\n            if new_whale not in s: pos[5] += 1\n            assert len(t) == 5 and len(s) == 5\n            f.write(p + ',' + ' '.join(t[:5]) + '\\n')\n    return vtop, vhigh, pos","execution_count":18,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"histories = []\nsteps = 0\n\nif isfile('../input/piotte/mpiotte-standard.model'):\n    #############################\n    # Load standard model and compute score \n    tmp = keras.models.load_model('../input/piotte/mpiotte-standard.model')\n    model.set_weights(tmp.get_weights())\n    \n    # Find elements from training sets not 'new_whale'\n    tic = time.time()\n    h2ws = {}\n    for p, w in tagged.items():\n        if w != new_whale:  # Use only identified whales\n            h = p2h[p]\n            if h not in h2ws: h2ws[h] = []\n            if w not in h2ws[h]: h2ws[h].append(w)\n    known = sorted(list(h2ws.keys()))\n\n    # Dictionary of picture indices\n    h2i = {}\n    for i, h in enumerate(known): h2i[h] = i\n\n    # Evaluate the model.\n    fknown1 = branch_model.predict_generator(FeatureGen(known), max_queue_size=20, workers=10, verbose=1)\n    fsubmit1 = branch_model.predict_generator(FeatureGen(submit), max_queue_size=20, workers=10, verbose=1)\n    score1 = head_model.predict_generator(ScoreGen(fknown1, fsubmit1), max_queue_size=20, workers=10, verbose=1)\n    score1 = score_reshape(score1, fknown1, fsubmit1)\n    \n    ###########################\n    # Load bootstrap model and compute score \n    tmp = keras.models.load_model('../input/piotte/mpiotte-bootstrap.model')\n    model.set_weights(tmp.get_weights())\n    \n    # Find elements from training sets not 'new_whale'\n    tic = time.time()\n    h2ws = {}\n    for p, w in tagged.items():\n        if w != new_whale:  # Use only identified whales\n            h = p2h[p]\n            if h not in h2ws: h2ws[h] = []\n            if w not in h2ws[h]: h2ws[h].append(w)\n    known = sorted(list(h2ws.keys()))\n\n    # Dictionary of picture indices\n    h2i = {}\n    for i, h in enumerate(known): h2i[h] = i\n\n    # Evaluate the model.\n    fknown2 = branch_model.predict_generator(FeatureGen(known), max_queue_size=20, workers=10, verbose=1)\n    fsubmit2 = branch_model.predict_generator(FeatureGen(submit), max_queue_size=20, workers=10, verbose=1)\n    score2 = head_model.predict_generator(ScoreGen(fknown2, fsubmit2), max_queue_size=20, workers=10, verbose=1)\n    score2 = score_reshape(score2, fknown2, fsubmit2)\n\nelse:\n    # epoch -> 10\n    make_steps(10, 1000)\n    ampl = 100.0\n    for _ in range(2):\n        print('noise ampl.  = ', ampl)\n        make_steps(5, ampl)\n        ampl = max(1.0, 100 ** -0.1 * ampl)\n    # epoch -> 150\n    for _ in range(18): make_steps(5, 1.0)\n    # epoch -> 200\n    set_lr(model, 16e-5)\n    for _ in range(10): make_steps(5, 0.5)\n    # epoch -> 240\n    set_lr(model, 4e-5)\n    for _ in range(8): make_steps(5, 0.25)\n    # epoch -> 250\n    set_lr(model, 1e-5)\n    for _ in range(2): make_steps(5, 0.25)\n    # epoch -> 300\n    weights = model.get_weights()\n    model, branch_model, head_model = build_model(64e-5,0.0002)\n    model.set_weights(weights)\n    for _ in range(10): make_steps(5, 1.0)\n    # epoch -> 350\n    set_lr(model, 16e-5)\n    for _ in range(10): make_steps(5, 0.5)    \n    # epoch -> 390\n    set_lr(model, 4e-5)\n    for _ in range(8): make_steps(5, 0.25)\n    # epoch -> 400\n    set_lr(model, 1e-5)\n    for _ in range(2): make_steps(5, 0.25)","execution_count":27,"outputs":[{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, description='Features', max=246, style=ProgressStyle(description_width='in…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6380851430c34e2aa536dfe4f848ae55"}},"metadata":{}},{"output_type":"stream","text":"243/246 [============================>.] - ETA: 3s\n246/246 [==============================] - 254s 1s/step\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, description='Features', max=125, style=ProgressStyle(description_width='in…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"dba697525468402f8991d84c73ddf93b"}},"metadata":{}},{"output_type":"stream","text":"122/125 [============================>.] - ETA: 3s\n125/125 [==============================] - 127s 1s/step\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, description='Scores', max=61006, style=ProgressStyle(description_width='in…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"569f699f5267433f8e477199e32ab067"}},"metadata":{}},{"output_type":"stream","text":"60984/61006 [============================>.] - ETA: 0s\n61006/61006 [==============================] - 712s 12ms/step\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, description='Features', max=246, style=ProgressStyle(description_width='in…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d051a9ae8b7f443da16e12f2a8afda81"}},"metadata":{}},{"output_type":"stream","text":"244/246 [============================>.] - ETA: 2s\n246/246 [==============================] - 248s 1s/step\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, description='Features', max=125, style=ProgressStyle(description_width='in…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f3e74d1dcf3b4710b3ba6d13c7361f36"}},"metadata":{}},{"output_type":"stream","text":"122/125 [============================>.] - ETA: 3s\n125/125 [==============================] - 128s 1s/step\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, description='Scores', max=61006, style=ProgressStyle(description_width='in…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f01c68c68d77467586d49167668af3f6"}},"metadata":{}},{"output_type":"stream","text":"60983/61006 [============================>.] - ETA: 0s\n61006/61006 [==============================] - ETA:  - 751s 12ms/step\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Do the weighthing exaclty as Martin suggests\nscore = 0.45*score1 + 0.55*score2","execution_count":28,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate the subsmission file.\nprepare_submission(0.92, 'submission_0.45_standard_0.55_boostrap-400.csv')\ntoc = time.time()\nprint(\"Submission time: \", (toc - tic) / 60.)","execution_count":29,"outputs":[{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, max=7960), HTML(value='')))","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"60c9f906ab164acf88a932a4a4b11a5c"}},"metadata":{}},{"output_type":"stream","text":"\nSubmission time:  21.186413689454398\n","name":"stdout"}]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}