{"cells":[{"metadata":{"_uuid":"03d0dd6baf411efbb62a6aef0b43dc846668d2ca"},"cell_type":"markdown","source":"Classification, Metric learning and auto encoder are related tasks as they all need to extract high level features from original images.So i try to train a model with all these three task losses ensembled together since related tasks are supposed to boost each other.\n\nHowever, Keras seems to exist a bug with batchnormalization. when bn is shared used for more than three branch, its moving mean will increase to infinity and produce NAN. So i remove all bn layers and model converges very slowly which is the main reason i do not finish the training(i trained for about 80 epochs, the LBs are 0.47 and 0.49 respectively for siamese network head layer and softmax head layer).\n\nI think this method can produce a good result if bn layer can be used in the model(you should change some codes because this is a really rough one). Besides, since you can get two different resluts by classification layer and siamese network layer at the same time, it will be effectively for ensemble.\nI hope this kernel can help you guys get some idea.(please forgive me for such a bad kernel structure)\n\nThis kernel is based on [siamese 0.822 ](https://www.kaggle.com/seesee/siamese-pretrained-0-822) and [explanation of MAP5 scoring metric](https://www.kaggle.com/pestipeti/explanation-of-map5-scoring-metric)\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install lap\n# Read the dataset description\nimport gzip\n# Read or generate p2h, a dictionary of image name to image id (picture to hash)\nimport pickle\nimport platform\nimport random\n# Suppress annoying stderr output when importing keras.\nimport sys\nfrom lap import lapjv\nfrom math import sqrt\n# Determine the size of each image\nfrom os.path import isfile\n\nimport keras\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image as pil_image\nfrom imagehash import phash\nfrom keras import backend as K\nfrom keras import regularizers\nfrom keras.engine.topology import Input\nfrom keras.layers import Activation, Add, BatchNormalization, Concatenate, Conv2D, Dense, Flatten, GlobalMaxPooling2D, \\\n    Lambda, MaxPooling2D, Reshape\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom keras.preprocessing.image import img_to_array\nfrom keras.utils import Sequence\nfrom pandas import read_csv\nfrom scipy.ndimage import affine_transform\nfrom tqdm import tqdm_notebook as tqdm\nimport time","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"TRAIN_DF = '../input/humpback-whale-identification/train.csv'\nSUB_Df = '../input/humpback-whale-identification/sample_submission.csv'\nTRAIN = '../input/humpback-whale-identification/train/'\nTEST = '../input/humpback-whale-identification/test/'\nP2H = '../input/metadata/p2h.pickle'\nP2SIZE = '../input/metadata/p2size.pickle'\nBB_DF = \"../input/metadata/bounding_boxes.csv\"\nimg_shape = (224, 224, 1)  # The image shape used by the model\ncrop_margin = 0.05  # The margin added around the bounding box to compensate for bounding box inaccuracy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd0e004efca215f2d92242a17535f0b4c6f3e323"},"cell_type":"code","source":"def expand_path(p):\n    if isfile(TRAIN + p):\n        return TRAIN + p\n    if isfile(TEST + p):\n        return TEST + p\n    return p","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"016c0539a9098d2d21779a9efeaa451ce93ce7ec"},"cell_type":"code","source":"def get_alldata():\n    tagged = dict([(p, w) for _, p, w in read_csv(TRAIN_DF).to_records()])\n    submit = [p for _, p, _ in read_csv(SUB_Df).to_records()]\n    join = list(tagged.keys()) + submit\n    return tagged, submit, join\n\ndef get_p2size(join):\n    if isfile(P2SIZE):\n        print(\"P2SIZE exists.\")\n        with open(P2SIZE, 'rb') as f:\n            p2size = pickle.load(f)\n    else:\n        p2size = {}\n        for p in tqdm(join):\n            size = pil_image.open(expand_path(p)).size\n            p2size[p] = size\n    return p2size\n\ndef get_p2bb():\n    p2bb = pd.read_csv(BB_DF).set_index(\"Image\")\n    return p2bb\n\n# get data for simaese network and corresponding whale id, exclude new_whale\ndef get_p2ws(tagged):\n    new_whale = 'new_whale'\n    p2ws = {}\n    for p, w in tagged.items():\n        if w != new_whale:\n            if p not in p2ws:\n                p2ws[p] = []\n            if w not in p2ws[p]:\n                p2ws[p].append(w)\n    return p2ws\n\n# this is used for validation\ndef get_new_whale(tagged):\n    new_whales = []\n    for p, w in tagged.items():\n        if w == 'new_whale':\n            new_whales.append(p)\n    np.random.seed(44)\n    np.random.shuffle(new_whales)\n    np.random.seed(None)\n    return new_whales\n\n\ndef get_w2ps(p2ws):\n    w2ps = {}\n    for p, ws in p2ws.items():\n        for w in ws:\n            if w not in w2ps:\n                w2ps[w] = []\n            if p not in w2ps[w]:\n                w2ps[w].append(p)\n    return w2ps\n\n\ndef read_raw_image(p):\n    img = pil_image.open(expand_path(p))\n    return img\n\n\n# convert whale id to numbers for softmax loss\ndef get_w2idx(train_soft, w2ps):\n    train_soft_set = sorted(set(train_soft))\n    w2ts_soft = {}\n    for w, ps in w2ps.items():\n        for p in ps:\n            if p in train_soft_set:\n                if w not in w2ts_soft:\n                    w2ts_soft[w] = []\n                if p not in w2ts_soft[w]:\n                    w2ts_soft[w].append(p)\n    for w, ts in w2ts_soft.items():\n        w2ts_soft[w] = np.array(ts)\n\n    w2idx = {}\n    for idx, w in enumerate(w2ts_soft.keys()):\n        if w not in w2idx:\n            w2idx[w] = idx\n\n    idx2w = {}\n    for w, idx in w2idx.items():\n        idx2w[idx] = w\n    return w2ts_soft, w2idx, train_soft_set, idx2w\n\n\n# resize image with unchanged aspect ratio using padding\ndef letterbox_image(image, size):\n    iw, ih = image.size\n    w, h = size\n    scale = min(w/iw, h/ih)\n    nw = int(iw*scale)\n    nh = int(ih*scale)\n\n    image = image.resize((nw, nh), pil_image.BICUBIC)\n    new_image = pil_image.new('L', size)  # , (128, 128, 128))\n    new_image.paste(image, ((w-nw)//2, (h-nh)//2))\n    return new_image\n\ndef read_cropped_image(p, p2size, p2bb, augment):\n    size_x, size_y = p2size[p]\n\n    # Determine the region of the original image we want to capture based on the bounding box.\n    row = p2bb.loc[p]\n    x0, y0, x1, y1 = row['x0'], row['y0'], row['x1'], row['y1']\n    dx = x1 - x0\n    dy = y1 - y0\n    x0 -= dx * crop_margin\n    x1 += dx * crop_margin + 1\n    y0 -= dy * crop_margin\n    y1 += dy * crop_margin + 1\n    if x0 < 0:\n        x0 = 0\n    if x1 > size_x:\n        x1 = size_x\n    if y0 < 0:\n        y0 = 0\n    if y1 > size_y:\n        y1 = size_y\n\n    img = read_raw_image(p).convert('L')\n\n    bbox = (x0, y0, x1, y1)\n    img = img.crop(bbox)\n    img = letterbox_image(img, img_shape[:2])\n    img = np.array(img).reshape(img_shape)\n    img = img.astype(np.float32)\n    if augment:\n        theta = np.random.uniform(-10, 10)  # random rotation\n        h, w = img.shape[0], img.shape[1]\n        tx = np.random.uniform(-0.1, 0.1) * h\n        ty = np.random.uniform(-0.05, 0.05) * w  # random shift\n        zx, zy = np.random.uniform(0.9, 1.1, 2)  # random zoom\n        shear = np.random.uniform(-10, 10)  # random shear\n        img = apply_affine_transform(img, theta, tx, ty, shear, zx, zy)\n\n    img -= np.mean(img, keepdims=True)\n    img /= np.std(img, keepdims=True) + K.epsilon()\n    return img\n\n\ndef read_for_training(p, p2size, p2bb):\n    return read_cropped_image(p, p2size, p2bb, True)\n\n\ndef read_for_validation(p, p2size, p2bb):\n    return read_cropped_image(p, p2size, p2bb, False)\n\n\ndef split_train_test(w2ps):\n    np.random.seed(44)\n    train = []\n    test = []\n    train_soft = []\n    for ps in w2ps.values():\n        if len(ps) >= 8:\n            np.random.shuffle(ps)\n            test += ps[-3:]\n            train += ps[:-3]\n            train_soft += ps[:-3]\n            #train += ps\n            #train_soft += ps\n        elif len(ps) > 1:\n            train += ps\n            train_soft += ps\n        else:\n            train_soft += ps\n    np.random.seed(None)\n    train_set = sorted(set(train))\n    test_set = sorted(set(test))\n    np.random.shuffle(train)\n    np.random.shuffle(train_soft)\n\n    w2ts = {}  # Associate the image ids from train to each whale id.\n    for w, ps in w2ps.items():\n        for p in ps:\n            if p in train_set:\n                if w not in w2ts:\n                    w2ts[w] = []\n                if p not in w2ts[w]:\n                    w2ts[w].append(p)\n    for w, ts in w2ts.items():\n        w2ts[w] = np.array(ts)\n\n    w2vs = {}  # Associate the image ids from train to each whale id.\n    for w, ps in w2ps.items():\n        for p in ps:\n            if p in test_set:\n                if w not in w2vs:\n                    w2vs[w] = []\n                if p not in w2vs[w]:\n                    w2vs[w].append(p)\n    for w, vs in w2vs.items():\n        w2vs[w] = np.array(vs)\n\n    t2i = {}  # The position in train of each training image id\n    for i, t in enumerate(train):\n        t2i[t] = i\n\n    v2i = {}\n    for i, v in enumerate(test):\n        v2i[v] = i\n\n    return train, test, train_set, test_set, w2ts, w2vs, t2i, v2i, train_soft\n\n\ndef map_per_image(label, predictions):\n    try:\n        return 1.0 / (predictions[:5].index(label) + 1)\n    except ValueError:\n        return 0.0\n\n\ndef map_per_set(labels, predictions):\n    return np.mean([map_per_image(l, p) for l, p in zip(labels, predictions)])\n\n\ndef set_lr(model, lr):\n    K.set_value(model.optimizer.lr, float(lr))\n\n\ndef get_lr(model):\n    return K.get_value(model.optimizer.lr)\n\n\ndef score_reshape(score, x, y=None):\n    if y is None:\n        # When y is None, score is a packed upper triangular matrix.\n        # Unpack, and transpose to form the symmetrical lower triangular matrix.\n        m = np.zeros((x.shape[0], x.shape[0]), dtype=K.floatx())\n        m[np.triu_indices(x.shape[0], 1)] = score.squeeze()\n        m += m.transpose()\n    else:\n        m = np.zeros((y.shape[0], x.shape[0]), dtype=K.floatx())\n        iy, ix = np.indices((y.shape[0], x.shape[0]))\n        ix = ix.reshape((ix.size,))\n        iy = iy.reshape((iy.size,))\n        m[iy, ix] = score.squeeze()\n    return m\n\n# for cv validation\ndef val_score(test, threshold, known, p2ws, score_val):\n    new_whale = 'new_whale'\n    vtop = 0\n    vhigh = 0\n    pos = [0, 0, 0, 0, 0, 0]\n    predictions = []\n    for i, p_ in enumerate(tqdm(test)):\n        t = []\n        s = set()\n        a = score_val[i, :]\n        for j in list(reversed(np.argsort(a))):\n            p = known[j]\n            if a[j] < threshold and new_whale not in s:\n                pos[len(t)] += 1\n                s.add(new_whale)\n                t.append(new_whale)\n                if len(t) == 5:\n                    break\n            for w in p2ws[p]:\n                assert w != new_whale\n                if w not in s:\n                    if a[j] > 1.0:\n                        vtop += 1\n                    elif a[j] >= threshold:\n                        vhigh += 1\n                    s.add(w)\n                    t.append(w)\n                    if len(t) == 5:\n                        break\n            if len(t) == 5:\n                break\n        if new_whale not in s:\n            pos[5] += 1\n        assert len(t) == 5 and len(s) == 5\n        predictions.append(t[:5])\n    return predictions\n\n\ndef get_random_test_data(test, w2vs, v2i):\n    np.random.seed(10)\n    score = -1 * np.random.random_sample(size=(len(test), len(test)))\n    np.random.seed(None)\n    for vs in w2vs.values():\n        idxs = [v2i[v] for v in vs]\n        for i in idxs:\n            for j in idxs:\n                score[i, j] = 10000.0\n    match = []\n    unmatch = []\n    _, _, x = lapjv(score)  # Solve the linear assignment problem\n    y = np.arange(len(x), dtype=np.int32)\n\n    # Compute a derangement for matching whales\n    for vs in w2vs.values():\n        d = vs.copy()\n        while True:\n            random.shuffle(d)\n            if not np.any(vs == d):\n                break\n        for ab in zip(vs, d):\n            match.append(ab)\n\n    # Construct unmatched whale pairs from the LAP solution.\n    for i, j in zip(x, y):\n        if i == j:\n            print(score)\n            print(x)\n            print(y)\n            print(i, j)\n        assert i != j\n        unmatch.append((test[i], test[j]))\n\n    # print(len(self.match), len(train), len(self.unmatch), len(train))\n    assert len(match) == len(test) and len(unmatch) == len(test)\n    return match, unmatch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"208f5d98270a8081f7024c52f4dad14791e48182"},"cell_type":"code","source":"from keras import regularizers\nfrom keras.optimizers import Adam\nfrom keras.engine.topology import Input\nfrom keras.layers import Activation, Add, BatchNormalization, Concatenate, Conv2D, Dense, Flatten, GlobalMaxPooling2D, GlobalAveragePooling2D, Lambda, MaxPooling2D, Reshape,Multiply\nfrom keras.models import Model\nimport keras.backend as K\nfrom keras.layers import Dropout, UpSampling2D\nimport tensorflow as tf\n\n\ndef subblock(x, filter, block, num, **kwargs):\n    #y = BatchNormalization()(x)\n    y = Conv2D(filter, (1, 1), activation='relu', **kwargs)(x)  # Reduce the number of features to 'filter'\n    #y = BatchNormalization()(y)\n    y = Conv2D(filter, (3, 3), activation='relu', **kwargs)(y)  # Extend the feature field\n    #y = BatchNormalization()(y)\n    y = Conv2D(K.int_shape(x)[-1], (1, 1), **kwargs)(y)  # no activation # Restore the number of original features\n    #y = BatchNormalization()(y)\n\n    spatial_attention = Conv2D(K.int_shape(y)[-1] // 2, kernel_size=(1, 1), strides=(1, 1), activation='relu',\n                               name=block + '_' + str(num) + 'sa_conv1')(y)\n    spatial_attention = Conv2D(1, kernel_size=(1, 1), strides=(1, 1), activation='sigmoid', name=block + '_' + str(num) + 'sa_conv2')(spatial_attention)\n\n    channel_attention = GlobalAveragePooling2D(name=block + '_' + str(num) + 'ca_gmp')(y)\n    channel_attention = Reshape(target_shape=(-1, K.int_shape(channel_attention)[-1]), name=block + '_' + str(num) + 'ca_reshape1')(channel_attention)\n    channel_attention = Dense(K.int_shape(channel_attention)[-1], activation='sigmoid', name=block + '_' + str(num) + 'ca_dense1')(channel_attention)\n    channel_attention = Dense(K.int_shape(channel_attention)[-1], activation='sigmoid', name=block + '_' + str(num) + 'ca_dense2')(channel_attention)\n    channel_attention = Reshape(target_shape=(-1, 1, K.int_shape(channel_attention)[-1]), name=block + '_' + str(num) + 'ca_reshape2')(channel_attention)\n\n    y = Multiply(name=block + '_' + str(num) + 'ml1')([y, channel_attention])\n    y = Multiply(name=block + '_' + str(num) + 'ml2')([y, spatial_attention])\n\n    y = Add()([x, y])  # Add the bypass connection\n    y = Activation('relu')(y)\n    return y\n\n\ndef decoder_model(inp):\n    kwargs = {'padding': 'same'}\n    net = Dense(4096, activation=None, name='dec_den_1')(inp)\n    net = Reshape(target_shape=(4, 4, 256), name='dec_rs_1')(net)\n\n    net = UpSampling2D(name='dec_us1')(net)\n    net = Conv2D(128, (2, 2), padding='valid', activation='relu', name='dec_conv0')(net)\n    net = Conv2D(128, (3, 3), padding='same', activation='relu', name='dec_conv1')(net)\n    for i in range(3):\n        net = subblock(net, 128, 'dec_1', i, **kwargs)\n\n    net = UpSampling2D(name='dec_us2')(net)\n    net = Conv2D(64, (3, 3), padding='same', activation='relu', name='dec_conv2')(net)\n    for i in range(3):\n        net = subblock(net, 64, 'dec_2', i, **kwargs)\n\n    net = UpSampling2D(name='dec_us3')(net)\n    net = Conv2D(32, (3, 3), padding='same', activation='relu', name='dec_conv3')(net)\n    for i in range(3):\n        net = subblock(net, 32, 'dec_3', i, **kwargs)\n\n    net = UpSampling2D(name='dec_us4')(net)\n    net = Conv2D(16, (3, 3), padding='same', activation='relu', name='dec_conv4')(net)\n    for i in range(3):\n        net = subblock(net, 16, 'dec_4', i, **kwargs)\n\n    net = UpSampling2D(name='dec_us5')(net)\n    net = Conv2D(3, (3, 3), padding='same', activation='relu', name='dec_conv5')(net)\n    for i in range(3):\n        net = subblock(net, 3, 'dec_5', i, **kwargs)\n\n    net = UpSampling2D(name='dec_us6')(net)\n    net = Conv2D(1, (3, 3), padding='same', activation='relu', name='dec_conv6')(net)\n    net = Conv2D(1, (3, 3), padding='same', activation=None, name='dec_conv7')(net)\n\n    return net\n\n\ndef build_model(lr, l2, img_shape=(224, 224, 1), activation='sigmoid'):\n    ##############\n    # BRANCH MODEL\n    ##############\n    regul = regularizers.l2(l2)\n    optim = Adam(lr=lr)\n    kwargs = {'padding': 'same', 'kernel_regularizer': regul}\n\n    inp = Input(shape=img_shape)  \n    x = Conv2D(64, (9, 9), strides=2, activation='relu', **kwargs)(inp)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  \n    for _ in range(2):\n        #x = BatchNormalization()(x)\n        x = Conv2D(64, (3, 3), activation='relu', **kwargs)(x)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  \n    #x = BatchNormalization()(x)\n    x = Conv2D(128, (1, 1), activation='relu', **kwargs)(x)  \n    #x = BatchNormalization()(x)\n    for i in range(4):\n        x = subblock(x, 64, '1', i, **kwargs)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x) \n    #x = BatchNormalization()(x)\n    x = Conv2D(256, (1, 1), activation='relu', **kwargs)(x) \n    #x = BatchNormalization()(x)\n    for i in range(4):\n        x = subblock(x, 64, '2', i,  **kwargs)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)\n    #x = BatchNormalization()(x)\n    x = Conv2D(384, (1, 1), activation='relu', **kwargs)(x) \n    #x = BatchNormalization()(x)\n    for i in range(4):\n        x = subblock(x, 96, '3', i,  **kwargs)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  \n    #x = BatchNormalization()(x)\n    x = Conv2D(512, (1, 1), activation='relu', **kwargs)(x)\n    #x = BatchNormalization()(x)\n    for i in range(4):\n        x = subblock(x, 128, '4', i,  **kwargs)\n\n    x = GlobalMaxPooling2D()(x) \n    branch_model = Model(inp, x)\n\n    ############\n    # HEAD MODEL\n    ############\n    mid = 32\n    xa_inp = Input(shape=branch_model.output_shape[1:])\n    xb_inp = Input(shape=branch_model.output_shape[1:])\n\n    x1 = Lambda(lambda x: x[0] * x[1])([xa_inp, xb_inp])\n    x2 = Lambda(lambda x: x[0] + x[1])([xa_inp, xb_inp])\n    x3 = Lambda(lambda x: K.abs(x[0] - x[1]))([xa_inp, xb_inp])\n    x4 = Lambda(lambda x: K.square(x))(x3)\n    x = Concatenate()([x1, x2, x3, x4])\n    x = Reshape((4, branch_model.output_shape[1], 1), name='reshape1')(x)\n\n    # Per feature NN with shared weight is implemented using CONV2D with appropriate stride.\n    x = Conv2D(mid, (4, 1), activation='relu', padding='valid')(x)\n    x = Reshape((branch_model.output_shape[1], mid, 1))(x)\n    x = Conv2D(1, (1, mid), activation='linear', padding='valid')(x)\n    x = Flatten(name='flatten')(x)\n\n    # Weighted sum implemented as a Dense layer.\n    x = Dense(1, use_bias=True, activation=activation, name='weighted-average')(x)\n    head_model = Model([xa_inp, xb_inp], x, name='head')\n    \n    # for classification\n    x_inp_ = Input(shape=branch_model.output_shape[1:])\n    x_all = Dropout(0.5)(x_inp_)\n    x_all = Dense(512, activation='relu', kernel_regularizer=regul)(x_all)\n    x_all = Dropout(0.5)(x_all)\n    x_all = Dense(512, activation='relu', kernel_regularizer=regul)(x_all)\n    x_all = Dense(5004, activation='softmax')(x_all)\n    soft_model = Model(x_inp_, x_all, name='soft')\n\n    ########################\n    #  auto encoder\n    ########################\n    dec_inp = Input(shape=branch_model.output_shape[1:])\n    net = decoder_model(dec_inp)\n    dec_model = Model(dec_inp, net, name='decoder')\n\n    ########################\n    # SIAMESE NEURAL NETWORK\n    ########################\n    # Complete model is constructed by calling the branch model on each input image,\n    # and then the head model on the resulting 512-vectors.\n    img_a = Input(shape=img_shape)\n    img_b = Input(shape=img_shape)\n    img_c = Input(shape=img_shape)  # softmax\n    img_d = Input(shape=img_shape)  # decoder\n    xa = branch_model(img_a)\n    xb = branch_model(img_b)\n    xc = branch_model(img_c)  # softmax\n    xd = branch_model(img_d)  # decoder\n\n    y_decoder = dec_model(xd)  # decoder\n\n    x = head_model([xa, xb])\n    y_softmax = soft_model(xc)\n    model = Model([img_a, img_b, img_c, img_d], [x, y_softmax, y_decoder])\n    model.compile(optim, loss=['binary_crossentropy', 'categorical_crossentropy', decoder_loss], metrics=['acc'],\n                  loss_weights=[1, 0.5, 0.5])\n    return model, branch_model, head_model, dec_model, soft_model\n\n\ndef decoder_loss(y_true, y_pred):\n    return K.mean(K.abs(y_true - y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fdeb2d88e3ea69c5452694fa7c764231c48c590a"},"cell_type":"code","source":"import time\nimport os\nimport sys\nfrom keras.callbacks import Callback\nfrom keras.utils import Sequence","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d438a9807bb20cd7b4c3a26abbc276f0c9306c9"},"cell_type":"code","source":"tagged, submit, join = get_alldata()\n\np2size = get_p2size(join)\n\np2bb = get_p2bb()\n\np2ws = get_p2ws(tagged)\n\nnew_whales = get_new_whale(tagged)\n\nw2ps = get_w2ps(p2ws)\n\ntrain, test, train_set, test_set, w2ts, w2vs, t2i, v2i, train_soft = split_train_test(w2ps)\n\nmatch_test, unmatch_test = get_random_test_data(test, w2vs, v2i)\n\nw2ts_soft, w2idx, train_soft_set, idx2w = get_w2idx(train_soft, w2ps)\n\nmodel, branch_model, head_model, dec_model, soft_model = build_model(64e-5, 0.0002)\nnew_whale = 'new_whale'\n\np2wts = {}\nfor p, w in tagged.items():\n    if w != new_whale:  # Use only identified whales\n        if p in train_set:\n            if p not in p2wts:\n                p2wts[p] = []\n            if w not in p2wts[p]:\n                p2wts[p].append(w)\nknown = sorted(list(p2wts.keys()))\n\n# Dictionary of picture indices\nkt2i = {}\nfor i, p in enumerate(known): kt2i[p] = i\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bbd6d134c5ed5230cf8ba46187d6f0675330f6c8"},"cell_type":"code","source":"class TestingData(Sequence):\n    def __init__(self, batch_size=64):\n        super(TestingData, self).__init__()\n        self.batch_size = batch_size\n        self.match = match_test\n        self.unmatch = unmatch_test\n        # np.random.seed(10)\n        # self.score = -1 * np.random.random_sample(size=(len(test), len(test)))\n        # np.random.seed(None)\n        # self.batch_size = batch_size\n        # for vs in w2vs.values():\n        #     idxs = [v2i[v] for v in vs]\n        #     for i in idxs:\n        #         for j in idxs:\n        #             self.score[\n        #                 i, j] = 10000.0  # Set a large value for matching whales -- eliminates this potential pairing\n        # self.get_test_data()\n\n    def __getitem__(self, index):\n        start = self.batch_size * index\n        end = min(start + self.batch_size, len(self.match) + len(self.unmatch))\n        size = end - start\n        assert size > 0\n        a = np.zeros((size,) + img_shape, dtype=K.floatx())\n        b = np.zeros((size,) + img_shape, dtype=K.floatx())\n        c = np.zeros((size, 1), dtype=K.floatx())\n        d = np.zeros((size,) + img_shape, dtype=K.floatx())\n        e = np.zeros((size, 5004), dtype=K.floatx())\n        f = np.zeros((size,) + img_shape, dtype=K.floatx())\n        j = start // 2\n        for i in range(0, size, 2):\n            a[i, :, :, :] = read_for_validation(self.match[j][0], p2size, p2bb)\n            b[i, :, :, :] = read_for_validation(self.match[j][1], p2size, p2bb)\n            c[i, 0] = 1  # This is a match\n            a[i + 1, :, :, :] = read_for_validation(self.unmatch[j][0], p2size, p2bb)\n            b[i + 1, :, :, :] = read_for_validation(self.unmatch[j][1], p2size, p2bb)\n            c[i + 1, 0] = 0  # Different whales\n            j += 1\n        for i in range(size):\n            d[i, :, :, :] = read_for_validation(test[(start + i) % len(test)], p2size, p2bb)\n            e[i, w2idx[p2ws[test[(start + i) % len(test)]][0]]] = 1\n        return [a, b, d, f], [c, e, f]\n\n    # def get_test_data(self):\n    #     self.match = []\n    #     self.unmatch = []\n    #     _, _, x = lapjv(self.score)  # Solve the linear assignment problem\n    #     y = np.arange(len(x), dtype=np.int32)\n    #\n    #     # Compute a derangement for matching whales\n    #     for vs in w2vs.values():\n    #         d = vs.copy()\n    #         while True:\n    #             random.shuffle(d)\n    #             if not np.any(vs == d): break\n    #         for ab in zip(vs, d): self.match.append(ab)\n    #\n    #     # Construct unmatched whale pairs from the LAP solution.\n    #     for i, j in zip(x, y):\n    #         if i == j:\n    #             print(self.score)\n    #             print(x)\n    #             print(y)\n    #             print(i, j)\n    #         assert i != j\n    #         self.unmatch.append((test[i], test[j]))\n    #\n    #     # print(len(self.match), len(train), len(self.unmatch), len(train))\n    #     assert len(self.match) == len(test) and len(self.unmatch) == len(test)\n\n    def __len__(self):\n        return (len(self.match) + len(self.unmatch) + self.batch_size - 1) // self.batch_size\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"472b4a4b40e6665a05fc0dd869a595cfebb8666e"},"cell_type":"code","source":"class TrainingData(Sequence):\n    def __init__(self, score, train_soft, join, steps=1000, batch_size=64):\n        \"\"\"\n        @param score the cost matrix for the picture matching\n        @param steps the number of epoch we are planning with this score matrix\n        \"\"\"\n        super(TrainingData, self).__init__()\n        self.score = -score  # Maximizing the score is the same as minimuzing -score.\n        self.steps = steps\n        self.batch_size = batch_size\n        self.train_soft = train_soft\n        self.join = join\n        for ts in w2ts.values():\n            idxs = [t2i[t] for t in ts]\n            for i in idxs:\n                for j in idxs:\n                    self.score[\n                        i, j] = 10000.0  # Set a large value for matching whales -- eliminates this potential pairing\n        self.on_epoch_end()\n\n    def __getitem__(self, index):\n        start = self.batch_size * index\n        end = min(start + self.batch_size, len(self.match) + len(self.unmatch))\n        size = end - start\n        assert size > 0\n        a = np.zeros((size,) + img_shape, dtype=K.floatx())\n        b = np.zeros((size,) + img_shape, dtype=K.floatx())\n        c = np.zeros((size, 1), dtype=K.floatx())\n        d = np.zeros((size,) + img_shape, dtype=K.floatx())  # softmax loss x\n        e = np.zeros((size, 5004), dtype=K.floatx())         # softmax loss y\n        f = np.zeros((size,) + img_shape, dtype=K.floatx())  # decoder x, y\n        j = start // 2\n        for i in range(0, size, 2):\n            a[i, :, :, :] = read_for_training(self.match[j][0], p2size, p2bb)\n            b[i, :, :, :] = read_for_training(self.match[j][1], p2size, p2bb)\n            c[i, 0] = 1  # This is a match\n            a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0], p2size, p2bb)\n            b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1], p2size, p2bb)\n            c[i + 1, 0] = 0  # Different whales\n            j += 1\n        for i in range(size):\n            d[i, :, :, :] = read_for_training(self.train_soft[(start + i) % len(self.train_soft)], p2size, p2bb)\n            e[i, w2idx[p2ws[self.train_soft[(start + i) % len(self.train_soft)]][0]]] = 1\n        for i in range(size):\n            f[i, :, :, :] = read_for_training(self.join[(start + i) % len(self.join)], p2size, p2bb)\n        return [a, b, d, f], [c, e, f]\n    def on_epoch_end(self):\n        if self.steps <= 0:\n            return  # Skip this on the last epoch.\n        np.random.seed(None)\n        np.random.shuffle(self.train_soft)\n        np.random.shuffle(self.join)\n        self.steps -= 1\n        self.match = []\n        self.unmatch = []\n        _, _, x = lapjv(self.score)  # Solve the linear assignment problem\n        y = np.arange(len(x), dtype=np.int32)\n\n        # Compute a derangement for matching whales\n        for ts in w2ts.values():\n            d = ts.copy()\n            while True:\n                random.shuffle(d)\n                if not np.any(ts == d): break\n            for ab in zip(ts, d): self.match.append(ab)\n\n        # Construct unmatched whale pairs from the LAP solution.\n        for i, j in zip(x, y):\n            if i == j:\n                print(self.score)\n                print(x)\n                print(y)\n                print(i, j)\n            assert i != j\n            self.unmatch.append((train[i], train[j]))\n\n        # Force a different choice for an eventual next epoch.\n        self.score[x, y] = 10000.0\n        self.score[y, x] = 10000.0\n        random.shuffle(self.match)\n        random.shuffle(self.unmatch)\n        # print(len(self.match), len(train), len(self.unmatch), len(train))\n        assert len(self.match) == len(train) and len(self.unmatch) == len(train)\n\n    def __len__(self):\n        return (len(self.match) + len(self.unmatch) + self.batch_size - 1) // self.batch_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38877b08848da6180f89d1a9f0f3d6914e3418a8"},"cell_type":"code","source":"# A Keras generator to evaluate only the BRANCH MODEL\nclass FeatureGen(Sequence):\n    def __init__(self, data, batch_size=64, verbose=1):\n        super(FeatureGen, self).__init__()\n        self.data = data\n        self.batch_size = batch_size\n        self.verbose = verbose\n        if self.verbose > 0: self.progress = tqdm(total=len(self), desc='Features')\n\n    def __getitem__(self, index):\n        start = self.batch_size * index\n        size = min(len(self.data) - start, self.batch_size)\n        a = np.zeros((size,) + img_shape, dtype=K.floatx())\n        for i in range(size): a[i, :, :, :] = read_for_validation(self.data[start + i], p2size, p2bb)\n        if self.verbose > 0:\n            self.progress.update()\n            if self.progress.n >= len(self): self.progress.close()\n        return a\n\n    def __len__(self):\n        return (len(self.data) + self.batch_size - 1) // self.batch_size\n\n\nclass ScoreGen(Sequence):\n    def __init__(self, x, y=None, batch_size=2048, verbose=1):\n        super(ScoreGen, self).__init__()\n        self.x = x\n        self.y = y\n        self.batch_size = batch_size\n        self.verbose = verbose\n        if y is None:\n            self.y = self.x\n            self.ix, self.iy = np.triu_indices(x.shape[0], 1)\n        else:\n            self.iy, self.ix = np.indices((y.shape[0], x.shape[0]))\n            self.ix = self.ix.reshape((self.ix.size,))\n            self.iy = self.iy.reshape((self.iy.size,))\n        self.subbatch = (len(self.x) + self.batch_size - 1) // self.batch_size\n        if self.verbose > 0:\n            self.progress = tqdm(total=len(self), desc='Scores')\n\n    def __getitem__(self, index):\n        start = index * self.batch_size\n        end = min(start + self.batch_size, len(self.ix))\n        a = self.y[self.iy[start:end], :]\n        b = self.x[self.ix[start:end], :]\n        if self.verbose > 0:\n            self.progress.update()\n            if self.progress.n >= len(self): self.progress.close()\n        return [a, b]\n\n    def __len__(self):\n        return (len(self.ix) + self.batch_size - 1) // self.batch_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53d03b778650070dbce50c3d3e5f873f21ecf64b"},"cell_type":"code","source":"def compute_score(verbose=1):\n    \"\"\"\n    Compute the score matrix by scoring every pictures from the training set against every other picture O(n^2).\n    \"\"\"\n    features = branch_model.predict_generator(FeatureGen(train, verbose=verbose), max_queue_size=12, workers=6,\n                                              verbose=0)\n    score = head_model.predict_generator(ScoreGen(features, verbose=verbose), max_queue_size=12, workers=6, verbose=0)\n    score = score_reshape(score, features)\n    return features, score\n\n\nclass cv_callback(Callback):\n    def on_epoch_end(self, epoch, logs=None):\n        if epoch % 5 != 4:\n            return\n\n        cv_test = test + new_whales[:len(test)]\n        # Evaluate the model.\n        fknown = branch_model.predict_generator(FeatureGen(known), max_queue_size=20, workers=10, verbose=0)\n        fsubmit = branch_model.predict_generator(FeatureGen(cv_test), max_queue_size=20, workers=10, verbose=0)\n        score_val = head_model.predict_generator(ScoreGen(fknown, fsubmit), max_queue_size=20, workers=10, verbose=0)\n        score_val = score_reshape(score_val, fknown, fsubmit)\n        predictions = val_score(cv_test, args.threshold, known, p2wts, score_val)\n        labels = [tagged[p] for p in test]\n        labels_newwhale = ['new_whale' for p in new_whales[:len(test)]]\n        labels = labels + labels_newwhale\n\n        print('cv score: ' + str(map_per_set(labels, predictions)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cf18e99f683099ba75d9115dce287347bdc802b"},"cell_type":"code","source":"def make_steps(step, ampl):\n    \"\"\"\n    Perform training epochs\n    @param step Number of epochs to perform\n    @param ampl the K, the randomized component of the score matrix.\n    \"\"\"\n    global w2ts, t2i, steps, features, score, histories\n    np.random.seed(None)\n    np.random.shuffle(train)\n    # Compute the match score for each picture pair\n    features, score = compute_score()\n\n    # Train the model for 'step' epochs\n    history = model.fit_generator(\n        TrainingData(score + ampl * np.random.random_sample(size=score.shape), train_soft, join, steps=step, batch_size=32),\n        initial_epoch=steps, epochs=steps + step, max_queue_size=12, workers=6,\n        verbose=1, validation_data=TestingData(), callbacks=[cv_callback()]).history\n    steps += step\n\n    # Collect history data\n    history['epochs'] = steps\n    history['ms'] = np.mean(score)\n    history['lr'] = get_lr(model)\n    print(history['epochs'], history['lr'], history['ms'])\n    histories.append(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4bb32145190b295958f92ca1619da4c3126867df"},"cell_type":"code","source":"histories = []\nsteps = 0\n\n\n# epoch -> 10\nmake_steps(10, 1000)\nampl = 100.0\nfor _ in range(2):\n    print('noise ampl.  = ', ampl)\n    make_steps(5, ampl)\n    ampl = max(1.0, 100 ** -0.1 * ampl)\n# epoch -> 150\nfor _ in range(18): make_steps(5, 1.0)\n# epoch -> 200\nset_lr(model, 16e-5)\nfor _ in range(10): make_steps(5, 0.5)\n# epoch -> 240\nset_lr(model, 4e-5)\nfor _ in range(8): make_steps(5, 0.25)\n# epoch -> 250\nset_lr(model, 1e-5)\nfor _ in range(2): make_steps(5, 0.25)\n# epoch -> 300\nweights = model.get_weights()\nmodel, branch_model, head_model, dec_model, soft_model = build_model(64e-5, 0.0002)\nmodel.set_weights(weights)\nfor _ in range(10): make_steps(5, 1.0)\n# epoch -> 350\nset_lr(model, 16e-5)\nfor _ in range(10): make_steps(5, 0.5)\n# epoch -> 390\nset_lr(model, 4e-5)\nfor _ in range(8): make_steps(5, 0.25)\n# epoch -> 400\nset_lr(model, 1e-5)\nfor _ in range(2): make_steps(5, 0.25)\nmodel.save('standard.model')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"157af6a64a9d43b26434ec420dce034220c0efe2"},"cell_type":"code","source":"def prepare_submission(threshold, filename):\n    \"\"\"\n    Generate a Kaggle submission file.\n    @param threshold the score given to 'new_whale'\n    @param filename the submission file name\n    \"\"\"\n    vtop = 0\n    vhigh = 0\n    pos = [0, 0, 0, 0, 0, 0]\n    with open(filename, 'wt', newline='\\n') as f:\n        f.write('Image,Id\\n')\n        for i, p in enumerate(tqdm(submit)):\n            t = []\n            s = set()\n            a = score[i, :]\n            for j in list(reversed(np.argsort(a))):\n                h = known[j]\n                if a[j] < threshold and new_whale not in s:\n                    pos[len(t)] += 1\n                    s.add(new_whale)\n                    t.append(new_whale)\n                    if len(t) == 5: break;\n                for w in p2ws[h]:\n                    assert w != new_whale\n                    if w not in s:\n                        if a[j] > 1.0:\n                            vtop += 1\n                        elif a[j] >= threshold:\n                            vhigh += 1\n                        s.add(w)\n                        t.append(w)\n                        if len(t) == 5: break;\n                if len(t) == 5: break;\n            if new_whale not in s: pos[5] += 1\n            assert len(t) == 5 and len(s) == 5\n            f.write(p + ',' + ' '.join(t[:5]) + '\\n')\n    return vtop, vhigh, pos","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"968c635b204aa974c61151ad97d32bc4d3b4d1cb"},"cell_type":"code","source":"def prepare_submission_softmax(threshold, filename):\n    \"\"\"\n    Generate a Kaggle submission file.\n    @param threshold the score given to 'new_whale'\n    @param filename the submission file name\n    \"\"\"\n    vtop = 0\n    vhigh = 0\n    pos = [0, 0, 0, 0, 0, 0]\n    with open(filename, 'wt', newline='\\n') as f:\n        f.write('Image,Id\\n')\n        for i, p in enumerate(tqdm(submit)):\n            t = []\n            s = set()\n            a = sm_submit[i, :]\n            for j in list(reversed(np.argsort(a))):\n                if a[j] < threshold and new_whale not in s:\n                    pos[len(t)] += 1\n                    s.add(new_whale)\n                    t.append(new_whale)\n                    if len(t) == 5: break;\n                s.add(idx2w[j])\n                t.append(idx2w[j])\n                if len(t) == 5: break;\n            if new_whale not in s: pos[5] += 1\n            assert len(t) == 5 and len(s) == 5\n            f.write(p + ',' + ' '.join(t[:5]) + '\\n')\n    return vtop, vhigh, pos","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cbfe99168b3e0450f65a80ce84f4b67a700f4848"},"cell_type":"code","source":"tic = time.time()\n\nif True:\n    # Evaluate the model with siamese network.\n    fknown = branch_model.predict_generator(FeatureGen(known), max_queue_size=20, workers=10, verbose=0)\n    fsubmit = branch_model.predict_generator(FeatureGen(submit), max_queue_size=20, workers=10, verbose=0)\n    score = head_model.predict_generator(ScoreGen(fknown, fsubmit), max_queue_size=20, workers=10, verbose=0)\n    score = score_reshape(score, fknown, fsubmit)\n    prepare_submission(args.threshold, 'submission.csv')\nelse:\n    # Evaluate the model with classification model.\n    fsubmit = branch_model.predict_generator(FeatureGen(submit), max_queue_size=20, workers=10, verbose=0)\n    sm_submit = soft_model.predict(fsubmit, batch_size=128)\n    prepare_submission_softmax(args.threshold, 'submission.csv')\ntoc = time.time()\nprint(\"Submission time: \", (toc - tic) / 60.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c098f49525126d4d12041dc91d055cceed74ac3"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc3c1439f9a4c68943f583e04837a6209c60c72a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7e01f223f569288311d57260c886a6d6f294792"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"191d661ebd739086dbc5aaac7e9a01b29e506171"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d505b7d867c66e6aec8adf8235c5cfa4f3f7ede"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}