{"cells":[{"metadata":{"_uuid":"03d0dd6baf411efbb62a6aef0b43dc846668d2ca"},"cell_type":"markdown","source":"This kernel is forked from [here](https://www.kaggle.com/seesee/siamese-pretrained-0-822). I just used the ResNet model from keras applications instead."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install lap\n# Read the dataset description\nimport gzip\n# Read or generate p2h, a dictionary of image name to image id (picture to hash)\nimport pickle\nimport platform\nimport random\n# Suppress annoying stderr output when importing keras.\nimport sys\nfrom lap import lapjv\nfrom math import sqrt\n# Determine the size of each image\nfrom os.path import isfile\n\nimport keras\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image as pil_image\nfrom imagehash import phash\nfrom keras import backend as K\nfrom keras import regularizers\nfrom keras.engine.topology import Input\nfrom keras.layers import *\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom keras.preprocessing.image import img_to_array\nfrom keras.utils import Sequence\nfrom keras.applications.resnet50 import ResNet50\nfrom pandas import read_csv\nfrom scipy.ndimage import affine_transform\nfrom tqdm import tqdm_notebook as tqdm\nimport time","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"TRAIN_DF = '../input/humpback-whale-identification/train.csv'\nSUB_Df = '../input/humpback-whale-identification/sample_submission.csv'\nTRAIN = '../input/humpback-whale-identification/train/'\nTEST = '../input/humpback-whale-identification/test/'\nP2H = '../input/metadata/p2h.pickle'\nP2SIZE = '../input/metadata/p2size.pickle'\nBB_DF = \"../input/metadata/bounding_boxes.csv\"\ntagged = dict([(p, w) for _, p, w in read_csv(TRAIN_DF).to_records()])\nsubmit = [p for _, p, _ in read_csv(SUB_Df).to_records()]\njoin = list(tagged.keys()) + submit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd0e004efca215f2d92242a17535f0b4c6f3e323"},"cell_type":"code","source":"def expand_path(p):\n    if isfile(TRAIN + p):\n        return TRAIN + p\n    if isfile(TEST + p):\n        return TEST + p\n    return p","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ee3c6a569bfe93b81bcbb97a40e3e78363c79d2"},"cell_type":"markdown","source":"## Duplicate image identification\n\nThis part was from the original kernel, seems like in the playground competition dulicated images was a real issue. I don't know the case about this one but I took one for the team and generated the results anyway. I'm such a nice chap."},{"metadata":{"trusted":true,"_uuid":"aa24819e1cc9ef98db60d218cc01e56f8d4e5046"},"cell_type":"code","source":"if isfile(P2SIZE):\n    print(\"P2SIZE exists.\")\n    with open(P2SIZE, 'rb') as f:\n        p2size = pickle.load(f)\nelse:\n    p2size = {}\n    for p in tqdm(join):\n        size = pil_image.open(expand_path(p)).size\n        p2size[p] = size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cdc36543403de9841a8af37f40c0044e17f6aa69"},"cell_type":"code","source":"def match(h1, h2):\n    for p1 in h2ps[h1]:\n        for p2 in h2ps[h2]:\n            i1 = pil_image.open(expand_path(p1))\n            i2 = pil_image.open(expand_path(p2))\n            if i1.mode != i2.mode or i1.size != i2.size: return False\n            a1 = np.array(i1)\n            a1 = a1 - a1.mean()\n            a1 = a1 / sqrt((a1 ** 2).mean())\n            a2 = np.array(i2)\n            a2 = a2 - a2.mean()\n            a2 = a2 / sqrt((a2 ** 2).mean())\n            a = ((a1 - a2) ** 2).mean()\n            if a > 0.1: return False\n    return True\n\n\nif isfile(P2H):\n    print(\"P2H exists.\")\n    with open(P2H, 'rb') as f:\n        p2h = pickle.load(f)\nelse:\n    # Compute phash for each image in the training and test set.\n    p2h = {}\n    for p in tqdm(join):\n        img = pil_image.open(expand_path(p))\n        h = phash(img)\n        p2h[p] = h\n\n    # Find all images associated with a given phash value.\n    h2ps = {}\n    for p, h in p2h.items():\n        if h not in h2ps: h2ps[h] = []\n        if p not in h2ps[h]: h2ps[h].append(p)\n\n    # Find all distinct phash values\n    hs = list(h2ps.keys())\n\n    # If the images are close enough, associate the two phash values (this is the slow part: n^2 algorithm)\n    h2h = {}\n    for i, h1 in enumerate(tqdm(hs)):\n        for h2 in hs[:i]:\n            if h1 - h2 <= 6 and match(h1, h2):\n                s1 = str(h1)\n                s2 = str(h2)\n                if s1 < s2: s1, s2 = s2, s1\n                h2h[s1] = s2\n\n    # Group together images with equivalent phash, and replace by string format of phash (faster and more readable)\n    for p, h in p2h.items():\n        h = str(h)\n        if h in h2h: h = h2h[h]\n        p2h[p] = h\n#     with open(P2H, 'wb') as f:\n#         pickle.dump(p2h, f)\n# For each image id, determine the list of pictures\nh2ps = {}\nfor p, h in p2h.items():\n    if h not in h2ps: h2ps[h] = []\n    if p not in h2ps[h]: h2ps[h].append(p)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62cc064dede5463c4f553da833020bee789f0de5"},"cell_type":"code","source":"def show_whale(imgs, per_row=2):\n    n = len(imgs)\n    rows = (n + per_row - 1) // per_row\n    cols = min(per_row, n)\n    fig, axes = plt.subplots(rows, cols, figsize=(24 // per_row * cols, 24 // per_row * rows))\n    for ax in axes.flatten(): ax.axis('off')\n    for i, (img, ax) in enumerate(zip(imgs, axes.flatten())): ax.imshow(img.convert('RGB'))\n        \n\ndef read_raw_image(p):\n    img = pil_image.open(expand_path(p))\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87e36806009c7252803a2f49d7a071dbacfafda9"},"cell_type":"code","source":"# For each images id, select the prefered image\ndef prefer(ps):\n    if len(ps) == 1: return ps[0]\n    best_p = ps[0]\n    best_s = p2size[best_p]\n    for i in range(1, len(ps)):\n        p = ps[i]\n        s = p2size[p]\n        if s[0] * s[1] > best_s[0] * best_s[1]:  # Select the image with highest resolution\n            best_p = p\n            best_s = s\n    return best_p\n\nh2p = {}\nfor h, ps in h2ps.items():\n    h2p[h] = prefer(ps)\nlen(h2p), list(h2p.items())[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"208f5d98270a8081f7024c52f4dad14791e48182"},"cell_type":"code","source":"# Read the bounding box data from the bounding box kernel (see reference above)\np2bb = pd.read_csv(BB_DF).set_index(\"Image\")\n\nold_stderr = sys.stderr\nsys.stderr = open('/dev/null' if platform.system() != 'Windows' else 'nul', 'w')\n\nsys.stderr = old_stderr\n\nimg_shape = (384, 384, 1)  # The image shape used by the model\nanisotropy = 2.15  # The horizontal compression ratio\ncrop_margin = 0.05  # The margin added around the bounding box to compensate for bounding box inaccuracy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b0629a511b86810a0106f23c5e25710242edf11"},"cell_type":"code","source":"def build_transform(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    \"\"\"\n    Build a transformation matrix with the specified characteristics.\n    \"\"\"\n    rotation = np.deg2rad(rotation)\n    shear = np.deg2rad(shear)\n    rotation_matrix = np.array(\n        [[np.cos(rotation), np.sin(rotation), 0], [-np.sin(rotation), np.cos(rotation), 0], [0, 0, 1]])\n    shift_matrix = np.array([[1, 0, height_shift], [0, 1, width_shift], [0, 0, 1]])\n    shear_matrix = np.array([[1, np.sin(shear), 0], [0, np.cos(shear), 0], [0, 0, 1]])\n    zoom_matrix = np.array([[1.0 / height_zoom, 0, 0], [0, 1.0 / width_zoom, 0], [0, 0, 1]])\n    shift_matrix = np.array([[1, 0, -height_shift], [0, 1, -width_shift], [0, 0, 1]])\n    return np.dot(np.dot(rotation_matrix, shear_matrix), np.dot(zoom_matrix, shift_matrix))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9bf0641d287366059c95fcbe0f5e2812f4bf236"},"cell_type":"code","source":"def read_cropped_image(p, augment):\n    \"\"\"\n    @param p : the name of the picture to read\n    @param augment: True/False if data augmentation should be performed\n    @return a numpy array with the transformed image\n    \"\"\"\n    # If an image id was given, convert to filename\n    if p in h2p:\n        p = h2p[p]\n    size_x, size_y = p2size[p]\n\n    # Determine the region of the original image we want to capture based on the bounding box.\n    row = p2bb.loc[p]\n    x0, y0, x1, y1 = row['x0'], row['y0'], row['x1'], row['y1']\n    dx = x1 - x0\n    dy = y1 - y0\n    x0 -= dx * crop_margin\n    x1 += dx * crop_margin + 1\n    y0 -= dy * crop_margin\n    y1 += dy * crop_margin + 1\n    if x0 < 0:\n        x0 = 0\n    if x1 > size_x:\n        x1 = size_x\n    if y0 < 0:\n        y0 = 0\n    if y1 > size_y:\n        y1 = size_y\n    dx = x1 - x0\n    dy = y1 - y0\n    if dx > dy * anisotropy:\n        dy = 0.5 * (dx / anisotropy - dy)\n        y0 -= dy\n        y1 += dy\n    else:\n        dx = 0.5 * (dy * anisotropy - dx)\n        x0 -= dx\n        x1 += dx\n\n    # Generate the transformation matrix\n    trans = np.array([[1, 0, -0.5 * img_shape[0]], [0, 1, -0.5 * img_shape[1]], [0, 0, 1]])\n    trans = np.dot(np.array([[(y1 - y0) / img_shape[0], 0, 0], [0, (x1 - x0) / img_shape[1], 0], [0, 0, 1]]), trans)\n    if augment:\n        trans = np.dot(build_transform(\n            random.uniform(-5, 5),\n            random.uniform(-5, 5),\n            random.uniform(0.8, 1.0),\n            random.uniform(0.8, 1.0),\n            random.uniform(-0.05 * (y1 - y0), 0.05 * (y1 - y0)),\n            random.uniform(-0.05 * (x1 - x0), 0.05 * (x1 - x0))\n        ), trans)\n    trans = np.dot(np.array([[1, 0, 0.5 * (y1 + y0)], [0, 1, 0.5 * (x1 + x0)], [0, 0, 1]]), trans)\n\n    # Read the image, transform to black and white and comvert to numpy array\n    img = read_raw_image(p).convert('L')\n    img = img_to_array(img)\n\n    # Apply affine transformation\n    matrix = trans[:2, :2]\n    offset = trans[:2, 2]\n    img = img.reshape(img.shape[:-1])\n    img = affine_transform(img, matrix, offset, output_shape=img_shape[:-1], order=1, mode='constant',\n                           cval=np.average(img))\n    img = img.reshape(img_shape)\n\n    # Normalize to zero mean and unit variance\n    img -= np.mean(img, keepdims=True)\n    img /= np.std(img, keepdims=True) + K.epsilon()\n    return img\n\ndef read_for_training(p):\n    \"\"\"\n    Read and preprocess an image with data augmentation (random transform).\n    \"\"\"\n    return read_cropped_image(p, True)\n\n\ndef read_for_validation(p):\n    \"\"\"\n    Read and preprocess an image without data augmentation (use for testing).\n    \"\"\"\n    return read_cropped_image(p, False)\n\n\np = list(tagged.keys())[312]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b369c7652c546dfd15252b7258727ec8ec82a8e"},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"725e7e521b6bf776e8a165614dba50cb597d3a1c"},"cell_type":"code","source":"def build_model(lr, l2, activation='sigmoid'):\n    ##############\n    # BRANCH MODEL\n    ##############\n    inp = Input(shape=(384, 384, 1))\n    new_inp = concatenate([inp]*3, axis=3)\n    res_net = ResNet50(weights='imagenet', include_top=False)(new_inp)\n    conv_2d_1 = Conv2D(32, kernel_size=(5, 5),\n                       activation='relu')(res_net)\n    outp = Flatten()(conv_2d_1)\n    outp = Dropout(0.2)(outp)\n    branch_model = Model(inp, outp)\n\n    ############\n    # HEAD MODEL\n    ############\n    mid = 32\n    xa_inp = Input(shape=branch_model.output_shape[1:])\n    xb_inp = Input(shape=branch_model.output_shape[1:])\n    x1 = Lambda(lambda x: x[0] * x[1])([xa_inp, xb_inp])\n    x2 = Lambda(lambda x: x[0] + x[1])([xa_inp, xb_inp])\n    x3 = Lambda(lambda x: K.abs(x[0] - x[1]))([xa_inp, xb_inp])\n    x4 = Lambda(lambda x: K.square(x))(x3)\n    x = Concatenate()([x1, x2, x3, x4])\n    x = Reshape((4, branch_model.output_shape[1], 1), name='reshape1')(x)\n\n    # Per feature NN with shared weight is implemented using CONV2D with appropriate stride.\n    x = Conv2D(mid, (4, 1), activation='relu', padding='valid')(x)\n    x = Reshape((branch_model.output_shape[1], mid, 1))(x)\n    x = Conv2D(1, (1, mid), activation='linear', padding='valid')(x)\n    x = Flatten(name='flatten')(x)\n\n    # Weighted sum implemented as a Dense layer.\n    x = Dense(1, use_bias=True, activation=activation, name='weighted-average')(x)\n    head_model = Model([xa_inp, xb_inp], x, name='head')\n\n    ########################\n    # NAS Network\n    ########################\n    # Complete model is constructed by calling the branch model on each input image,\n    # and then the head model on the resulting 512-vectors.\n    img_a = Input(shape=img_shape)\n    img_b = Input(shape=img_shape)\n    xa = branch_model(img_a)\n    xb = branch_model(img_b)\n    x = head_model([xa, xb])\n    model = Model([img_a, img_b], x)\n    optim = Adam(lr=lr)\n    model.compile(optim, loss='binary_crossentropy', metrics=['binary_crossentropy', 'acc'])\n    return model, branch_model, head_model\n\n\nmodel, branch_model, head_model = build_model(64e-5, 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8266e1cc94c9c56ff75352aef4194aa37574972"},"cell_type":"code","source":"h2ws = {}\nnew_whale = 'new_whale'\nfor p, w in tagged.items():\n    if w != new_whale:  # Use only identified whales\n        h = p2h[p]\n        if h not in h2ws: h2ws[h] = []\n        if w not in h2ws[h]: h2ws[h].append(w)\nfor h, ws in h2ws.items():\n    if len(ws) > 1:\n        h2ws[h] = sorted(ws)\n\n# For each whale, find the unambiguous images ids.\nw2hs = {}\nfor h, ws in h2ws.items():\n    if len(ws) == 1:  # Use only unambiguous pictures\n        w = ws[0]\n        if w not in w2hs: w2hs[w] = []\n        if h not in w2hs[w]: w2hs[w].append(h)\nfor w, hs in w2hs.items():\n    if len(hs) > 1:\n        w2hs[w] = sorted(hs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f122bcb1d87721a96a18c31aa2f46aaad4037039"},"cell_type":"code","source":"train = []  # A list of training image ids\nfor hs in w2hs.values():\n    if len(hs) > 1:\n        train += hs\nrandom.shuffle(train)\ntrain_set = set(train)\n\nw2ts = {}  # Associate the image ids from train to each whale id.\nfor w, hs in w2hs.items():\n    for h in hs:\n        if h in train_set:\n            if w not in w2ts:\n                w2ts[w] = []\n            if h not in w2ts[w]:\n                w2ts[w].append(h)\nfor w, ts in w2ts.items():\n    w2ts[w] = np.array(ts)\n\nt2i = {}  # The position in train of each training image id\nfor i, t in enumerate(train):\n    t2i[t] = i","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa16cfd51fdbce44271883869486c1cad32f1091"},"cell_type":"code","source":"class TrainingData(Sequence):\n    def __init__(self, score, steps=1000, batch_size=32):\n        \"\"\"\n        @param score the cost matrix for the picture matching\n        @param steps the number of epoch we are planning with this score matrix\n        \"\"\"\n        super(TrainingData, self).__init__()\n        self.score = -score  # Maximizing the score is the same as minimuzing -score.\n        self.steps = steps\n        self.batch_size = batch_size\n        for ts in w2ts.values():\n            idxs = [t2i[t] for t in ts]\n            for i in idxs:\n                for j in idxs:\n                    self.score[\n                        i, j] = 10000.0  # Set a large value for matching whales -- eliminates this potential pairing\n        self.on_epoch_end()\n\n    def __getitem__(self, index):\n        start = self.batch_size * index\n        end = min(start + self.batch_size, len(self.match) + len(self.unmatch))\n        size = end - start\n        assert size > 0\n        a = np.zeros((size,) + img_shape, dtype=K.floatx())\n        b = np.zeros((size,) + img_shape, dtype=K.floatx())\n        c = np.zeros((size, 1), dtype=K.floatx())\n        j = start // 2\n        for i in range(0, size, 2):\n            a[i, :, :, :] = read_for_training(self.match[j][0])\n            b[i, :, :, :] = read_for_training(self.match[j][1])\n            c[i, 0] = 1  # This is a match\n            a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])\n            b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])\n            c[i + 1, 0] = 0  # Different whales\n            j += 1\n        return [a, b], c\n\n    def on_epoch_end(self):\n        if self.steps <= 0: return  # Skip this on the last epoch.\n        self.steps -= 1\n        self.match = []\n        self.unmatch = []\n        _, _, x = lapjv(self.score)  # Solve the linear assignment problem\n        y = np.arange(len(x), dtype=np.int32)\n\n        # Compute a derangement for matching whales\n        for ts in w2ts.values():\n            d = ts.copy()\n            while True:\n                random.shuffle(d)\n                if not np.any(ts == d): break\n            for ab in zip(ts, d): self.match.append(ab)\n\n        # Construct unmatched whale pairs from the LAP solution.\n        for i, j in zip(x, y):\n            if i == j:\n                print(self.score)\n                print(x)\n                print(y)\n                print(i, j)\n            assert i != j\n            self.unmatch.append((train[i], train[j]))\n\n        # Force a different choice for an eventual next epoch.\n        self.score[x, y] = 10000.0\n        self.score[y, x] = 10000.0\n        random.shuffle(self.match)\n        random.shuffle(self.unmatch)\n        # print(len(self.match), len(train), len(self.unmatch), len(train))\n        assert len(self.match) == len(train) and len(self.unmatch) == len(train)\n\n    def __len__(self):\n        return (len(self.match) + len(self.unmatch) + self.batch_size - 1) // self.batch_size\n\n\n# Test on a batch of 32 with random costs.\nscore = np.random.random_sample(size=(len(train), len(train)))\ndata = TrainingData(score)\n(a, b), c = data[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"005d041475be5ec97ac8e1d937bea6e7253226af"},"cell_type":"code","source":"# A Keras generator to evaluate only the BRANCH MODEL\nclass FeatureGen(Sequence):\n    def __init__(self, data, batch_size=64, verbose=1):\n        super(FeatureGen, self).__init__()\n        self.data = data\n        self.batch_size = batch_size\n        self.verbose = verbose\n        if self.verbose > 0: self.progress = tqdm(total=len(self), desc='Features')\n\n    def __getitem__(self, index):\n        start = self.batch_size * index\n        size = min(len(self.data) - start, self.batch_size)\n        a = np.zeros((size,) + img_shape, dtype=K.floatx())\n        for i in range(size): a[i, :, :, :] = read_for_validation(self.data[start + i])\n        if self.verbose > 0:\n            self.progress.update()\n            if self.progress.n >= len(self): self.progress.close()\n        return a\n\n    def __len__(self):\n        return (len(self.data) + self.batch_size - 1) // self.batch_size\n\n\nclass ScoreGen(Sequence):\n    def __init__(self, x, y=None, batch_size=2048, verbose=1):\n        super(ScoreGen, self).__init__()\n        self.x = x\n        self.y = y\n        self.batch_size = batch_size\n        self.verbose = verbose\n        if y is None:\n            self.y = self.x\n            self.ix, self.iy = np.triu_indices(x.shape[0], 1)\n        else:\n            self.iy, self.ix = np.indices((y.shape[0], x.shape[0]))\n            self.ix = self.ix.reshape((self.ix.size,))\n            self.iy = self.iy.reshape((self.iy.size,))\n        self.subbatch = (len(self.x) + self.batch_size - 1) // self.batch_size\n        if self.verbose > 0:\n            self.progress = tqdm(total=len(self), desc='Scores')\n\n    def __getitem__(self, index):\n        start = index * self.batch_size\n        end = min(start + self.batch_size, len(self.ix))\n        a = self.y[self.iy[start:end], :]\n        b = self.x[self.ix[start:end], :]\n        if self.verbose > 0:\n            self.progress.update()\n            if self.progress.n >= len(self): self.progress.close()\n        return [a, b]\n\n    def __len__(self):\n        return (len(self.ix) + self.batch_size - 1) // self.batch_size\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ac5f82779a71cca2bc2cd1bac8bac385cb48a23"},"cell_type":"code","source":"def set_lr(model, lr):\n    K.set_value(model.optimizer.lr, float(lr))\n\n\ndef get_lr(model):\n    return K.get_value(model.optimizer.lr)\n\n\ndef score_reshape(score, x, y=None):\n    \"\"\"\n    Tranformed the packed matrix 'score' into a square matrix.\n    @param score the packed matrix\n    @param x the first image feature tensor\n    @param y the second image feature tensor if different from x\n    @result the square matrix\n    \"\"\"\n    if y is None:\n        # When y is None, score is a packed upper triangular matrix.\n        # Unpack, and transpose to form the symmetrical lower triangular matrix.\n        m = np.zeros((x.shape[0], x.shape[0]), dtype=K.floatx())\n        m[np.triu_indices(x.shape[0], 1)] = score.squeeze()\n        m += m.transpose()\n    else:\n        m = np.zeros((y.shape[0], x.shape[0]), dtype=K.floatx())\n        iy, ix = np.indices((y.shape[0], x.shape[0]))\n        ix = ix.reshape((ix.size,))\n        iy = iy.reshape((iy.size,))\n        m[iy, ix] = score.squeeze()\n    return m\n\n\ndef compute_score(verbose=1):\n    \"\"\"\n    Compute the score matrix by scoring every pictures from the training set against every other picture O(n^2).\n    \"\"\"\n    features = branch_model.predict_generator(FeatureGen(train, verbose=verbose), max_queue_size=12, workers=6,\n                                              verbose=0)\n    score = head_model.predict_generator(ScoreGen(features, verbose=verbose), max_queue_size=12, workers=6, verbose=0)\n    score = score_reshape(score, features)\n    return features, score\n\n\ndef make_steps(step, ampl):\n    \"\"\"\n    Perform training epochs\n    @param step Number of epochs to perform\n    @param ampl the K, the randomized component of the score matrix.\n    \"\"\"\n    global w2ts, t2i, steps, features, score, histories\n\n    # shuffle the training pictures\n    random.shuffle(train)\n\n    # Map whale id to the list of associated training picture hash value\n    w2ts = {}\n    for w, hs in w2hs.items():\n        for h in hs:\n            if h in train_set:\n                if w not in w2ts: w2ts[w] = []\n                if h not in w2ts[w]: w2ts[w].append(h)\n    for w, ts in w2ts.items(): w2ts[w] = np.array(ts)\n\n    # Map training picture hash value to index in 'train' array    \n    t2i = {}\n    for i, t in enumerate(train): t2i[t] = i\n\n    # Compute the match score for each picture pair\n    features, score = compute_score()\n\n    # Train the model for 'step' epochs\n    history = model.fit_generator(\n        TrainingData(score + ampl * np.random.random_sample(size=score.shape), steps=step, batch_size=32),\n        initial_epoch=steps, epochs=steps + step, max_queue_size=12, workers=6, verbose=1).history\n    steps += step\n\n    # Collect history data\n    history['epochs'] = steps\n    history['ms'] = np.mean(score)\n    history['lr'] = get_lr(model)\n    print(history['epochs'], history['lr'], history['ms'])\n    histories.append(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87091f9dcd5a0d11dc255d427abd42cb6ba7506e"},"cell_type":"code","source":"histories = []\nsteps = 0\n\n# epoch -> 10\nmake_steps(10, 1000)\nampl = 100.0\nfor _ in range(2):\n    print('noise ampl.  = ', ampl)\n    make_steps(5, ampl)\n    ampl = max(1.0, 100 ** -0.1 * ampl)\n    \n# epoch -> 150\nfor _ in range(18): make_steps(5, 1.0)\n# epoch -> 200\nset_lr(model, 16e-5)\nfor _ in range(10): make_steps(5, 0.5)\n# epoch -> 240\nset_lr(model, 4e-5)\nfor _ in range(8): make_steps(5, 0.25)\n# epoch -> 250\nset_lr(model, 1e-5)\nfor _ in range(2): make_steps(5, 0.25)\n# epoch -> 300\nweights = model.get_weights()\nmodel, branch_model, head_model = build_model(64e-5, 0.0002)\nmodel.set_weights(weights)\nfor _ in range(10): make_steps(5, 1.0)\n# epoch -> 350\nset_lr(model, 16e-5)\nfor _ in range(10): make_steps(5, 0.5)\n# epoch -> 390\nset_lr(model, 4e-5)\nfor _ in range(8): make_steps(5, 0.25)\n# epoch -> 400\nset_lr(model, 1e-5)\nfor _ in range(2): make_steps(5, 0.25)\nmodel.save('standard.model')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fdeb2d88e3ea69c5452694fa7c764231c48c590a"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f194458bbb728a7cc35b75b40752e98021b6493b"},"cell_type":"code","source":"def prepare_submission(threshold, filename):\n    \"\"\"\n    Generate a Kaggle submission file.\n    @param threshold the score given to 'new_whale'\n    @param filename the submission file name\n    \"\"\"\n    vtop = 0\n    vhigh = 0\n    pos = [0, 0, 0, 0, 0, 0]\n    with open(filename, 'wt', newline='\\n') as f:\n        f.write('Image,Id\\n')\n        for i, p in enumerate(tqdm(submit)):\n            t = []\n            s = set()\n            a = score[i, :]\n            for j in list(reversed(np.argsort(a))):\n                h = known[j]\n                if a[j] < threshold and new_whale not in s:\n                    pos[len(t)] += 1\n                    s.add(new_whale)\n                    t.append(new_whale)\n                    if len(t) == 5: break;\n                for w in h2ws[h]:\n                    assert w != new_whale\n                    if w not in s:\n                        if a[j] > 1.0:\n                            vtop += 1\n                        elif a[j] >= threshold:\n                            vhigh += 1\n                        s.add(w)\n                        t.append(w)\n                        if len(t) == 5: break;\n                if len(t) == 5: break;\n            if new_whale not in s: pos[5] += 1\n            assert len(t) == 5 and len(s) == 5\n            f.write(p + ',' + ' '.join(t[:5]) + '\\n')\n    return vtop, vhigh, pos","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"624143d234c6cc5f7374a2451aab84b955cecb09"},"cell_type":"code","source":"# Find elements from training sets not 'new_whale'\ntic = time.time()\nh2ws = {}\nfor p, w in tagged.items():\n    if w != new_whale:  # Use only identified whales\n        h = p2h[p]\n        if h not in h2ws: h2ws[h] = []\n        if w not in h2ws[h]: h2ws[h].append(w)\nknown = sorted(list(h2ws.keys()))\n\n# Dictionary of picture indices\nh2i = {}\nfor i, h in enumerate(known): h2i[h] = i\n\n# Evaluate the model.\nfknown = branch_model.predict_generator(FeatureGen(known), max_queue_size=20, workers=10, verbose=0)\nfsubmit = branch_model.predict_generator(FeatureGen(submit), max_queue_size=20, workers=10, verbose=0)\nscore = head_model.predict_generator(ScoreGen(fknown, fsubmit), max_queue_size=20, workers=10, verbose=0)\nscore = score_reshape(score, fknown, fsubmit)\n\n# Generate the subsmission file.\nprepare_submission(0.99, 'submission.csv')\ntoc = time.time()\nprint(\"Submission time: \", (toc - tic) / 60.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d438a9807bb20cd7b4c3a26abbc276f0c9306c9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}