{"cells":[{"metadata":{"_uuid":"00f033e0e5995358433fe05bc3e064bf10e23eb0"},"cell_type":"markdown","source":"## 5.Required Packages"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from collections import defaultdict\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import normalize\nfrom scipy.stats import logistic\nfrom os.path import join\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom keras import backend as K\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom keras import regularizers\nfrom keras.layers import Input, Dense, Dropout, Lambda, Convolution2D, MaxPooling2D, Flatten, Conv2D, BatchNormalization\nfrom keras.layers import Activation, Add, BatchNormalization, Concatenate, Conv2D, Dense, Flatten, GlobalMaxPooling2D, \\\n    Lambda, MaxPooling2D, Reshape\nfrom keras.losses import categorical_crossentropy\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom keras.applications.resnet50 import ResNet50, preprocess_input\n# from keras.applications.inception_resnet_v2 import InceptionResNetV2, preprocess_input\n# from keras.applications.inception_v3 import InceptionV3, preprocess_input\nimport cv2\nimport os\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nplt.style.use('fivethirtyeight')\n%matplotlib inline\n\nimport warnings\nfor i in [DeprecationWarning,FutureWarning,UserWarning]:\n    warnings.filterwarnings(\"ignore\", category = i)\n\nprint(os.listdir(\"../input/\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"717414bdbd88e485356f5c411ca037969de5e59f"},"cell_type":"markdown","source":"## 6.Define Parameter"},{"metadata":{"trusted":true,"_uuid":"d7e0676f2fe134f34508b0f05d37845224aa7a41"},"cell_type":"code","source":"bbox = pd.read_csv(\"../input/generating-whale-bounding-boxes/bounding_boxes.csv\")\nprint(bbox.head())","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"batch_size = 24\nembedding_dim = 128\nimage_size = 192\npath_base = '../input/humpback-whale-identification/'\npath_train = join(path_base,'train')\npath_test = join(path_base,'test')\npath_model = join(path_base,'MyModel.hdf5')\npath_csv = '../input/humpback-whale-identification/train.csv'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c956c38aa755e547c17569ffe9ff4fa8d0e2e035"},"cell_type":"markdown","source":"## 7.Helping Function"},{"metadata":{"trusted":true,"_uuid":"721a989504b5d31f9c341e1c4db86a2098c72423"},"cell_type":"code","source":"class sample_gen(object):\n    def __init__(self, file_class_mapping):\n        self.file_class_mapping= file_class_mapping\n        self.class_to_list_files = defaultdict(list)\n        self.list_other_class = []\n        self.list_all_files = list(file_class_mapping.keys())\n        self.range_all_files = list(range(len(self.list_all_files)))\n\n        for file, class_ in file_class_mapping.items():\n            self.class_to_list_files[class_].append(file)\n        \n        self.list_classes = list(set(self.class_to_list_files.keys()))\n        self.range_list_classes= range(len(self.list_classes))\n        \n        print(\"class_to_list_files, list_classes\", len(self.class_to_list_files), len(self.list_classes))\n        \n        self.class_weight = np.array([len(self.class_to_list_files[class_]) for class_ in self.list_classes])\n        self.class_weight = self.class_weight/np.sum(self.class_weight)\n\n    def get_sample(self):\n        class_idx = np.random.choice(self.range_list_classes, 1)[0]\n        examples_class_idx = np.random.choice(range(len(self.class_to_list_files[self.list_classes[class_idx]])), 2)\n        positive_example_1 = self.class_to_list_files[self.list_classes[class_idx]][examples_class_idx[0]]\n        positive_example_2 = self.class_to_list_files[self.list_classes[class_idx]][examples_class_idx[1]]\n        pos_binary = np.random.randint(2, size=1)\n        \n        if pos_binary[0] == 1:\n            return positive_example_1, positive_example_2\n        \n        negative_example = None\n        while negative_example is None or self.file_class_mapping[negative_example] == \\\n                self.file_class_mapping[positive_example_1]:\n            negative_example_idx = np.random.choice(self.range_all_files, 1)[0]\n            negative_example = self.list_all_files[negative_example_idx]\n        return positive_example_1, negative_example\n    \ndef read_and_resize(filepath):\n    #print(filepath[-13:])\n    _, _, x0, y0, x1, y1 = list(bbox[bbox['Image'] == filepath[-13:]].reset_index().loc[0])\n    #print(x0, y0, x1, y1)\n    if x0 >= x1 or y0 >= y1:\n        x0 = 0\n        y0 = 0\n        x1 = image_size\n        y1 = image_size\n    im = Image.open((filepath)).convert('RGB')\n    im = np.array(im, dtype=\"float32\")\n    #print(im.shape, 'y')\n    imout = im[y0:y1, x0:x1, :]\n    #print(imout.shape, 'x')\n    imout = cv2.resize(imout, dsize=(image_size, image_size), interpolation=cv2.INTER_CUBIC)\n    #print(imout.shape)\n    return imout\n\n\ndef augment(im_array, fixed=False):\n    if fixed == True or np.random.uniform(0, 1) > 0.8:\n        im_array = np.fliplr(im_array)\n    return im_array\n\ndef gen(couplet_gen):\n    while True:\n        list_examples_1 = []\n        list_examples_2 = []\n        labels = []\n        print(\"ONE CALL\")\n\n        for i in range(batch_size):\n            example_1, example_2 = couplet_gen.get_sample()\n            path_pos1 = join(path_train, example_1)\n            path_pos2 = join(path_train, example_2)\n            \n            example_1_img = read_and_resize(path_pos1)\n            example_2_img = read_and_resize(path_pos2)\n            \n            example_1_img = augment(example_1_img)\n            if example_1 == example_2:\n                example_2_img = augment(example_2_img, fixed=True)\n                label = 1\n            else:\n                example_2_img = augment(example_2_img)\n                label = 0\n            \n            list_examples_1.append(example_1_img)\n            list_examples_2.append(example_2_img)\n            labels.append(label)\n        \n        list_examples_1 = preprocess_input(np.array(list_examples_1))\n        list_examples_2 = preprocess_input(np.array(list_examples_2))\n        labels = np.array(labels)\n        \n        yield ([list_examples_1, list_examples_2], labels)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8b219f7aa6680474d14061a20b929ea73f458c1d"},"cell_type":"markdown","source":"## 8.Introduction to Triplet Loss "},{"metadata":{"trusted":true,"_uuid":"679829a47db08ed4dc97b61a5d6781f70419fa2a"},"cell_type":"code","source":"def triplet_loss(inputs, dist='euclidean', margin='softplus'):\n    anchor, positive, negative = inputs\n    positive_distance = K.square(anchor - positive)\n    negative_distance = K.square(anchor - negative)\n    if dist == 'euclidean':\n        positive_distance = K.sqrt(K.sum(positive_distance, axis=-1, keepdims=True))\n        negative_distance = K.sqrt(K.sum(negative_distance, axis=-1, keepdims=True))\n    elif dist == 'sqeuclidean':\n        positive_distance = K.sum(positive_distance, axis=-1, keepdims=True)\n        negative_distance = K.sum(negative_distance, axis=-1, keepdims=True)\n    loss = 0.6 + positive_distance - negative_distance\n    if margin == 'maxplus':\n        loss = K.maximum(0.0, 1 + loss)\n    elif margin == 'softplus':\n        loss = K.log(1 + K.exp(loss))\n    return K.mean(loss)\n\ndef triplet_loss_np(inputs, dist='euclidean', margin='softplus'):\n    anchor, positive, negative = inputs\n    positive_distance = np.square(anchor - positive)\n    negative_distance = np.square(anchor - negative)\n    if dist == 'euclidean':\n        positive_distance = np.sqrt(np.sum(positive_distance, axis=-1, keepdims=True))\n        negative_distance = np.sqrt(np.sum(negative_distance, axis=-1, keepdims=True))\n    elif dist == 'sqeuclidean':\n        positive_distance = np.sum(positive_distance, axis=-1, keepdims=True)\n        negative_distance = np.sum(negative_distance, axis=-1, keepdims=True)\n    loss = .6 + positive_distance - negative_distance\n    if margin == 'maxplus':\n        loss = np.maximum(0.0, 1 + loss)\n    elif margin == 'softplus':\n        loss = np.log(1 + np.exp(loss))\n    return np.mean(loss)\n\ndef check_loss():\n    batch_size = 10\n    shape = (batch_size, 4096)\n\n    p1 = normalize(np.random.random(shape))\n    n = normalize(np.random.random(shape))\n    p2 = normalize(np.random.random(shape))\n    \n    input_tensor = [K.variable(p1), K.variable(n), K.variable(p2)]\n    out1 = K.eval(triplet_loss(input_tensor))\n    input_np = [p1, n, p2]\n    out2 = triplet_loss_np(input_np)\n\n    assert out1.shape == out2.shape\n    print(np.linalg.norm(out1))\n    print(np.linalg.norm(out2))\n    print(np.linalg.norm(out1-out2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90064c8daa36a84ca2f6b95ec540ebddb3dfdb0d"},"cell_type":"code","source":"check_loss()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dc8e689c18afdb010034fb8eda359ccba71b89f5"},"cell_type":"markdown","source":"## 9.Model Design"},{"metadata":{"trusted":true,"_uuid":"5d2dd749451e69ac7c7cddf46e879a3715ba2e61"},"cell_type":"code","source":"def subblock(x, filter, **kwargs):\n    x = BatchNormalization()(x)\n    y = x\n    y = Conv2D(filter, (1, 1), activation='relu', **kwargs)(y)  # Reduce the number of features to 'filter'\n    y = BatchNormalization()(y)\n    y = Conv2D(filter, (3, 3), activation='relu', **kwargs)(y)  # Extend the feature field\n    y = BatchNormalization()(y)\n    y = Conv2D(K.int_shape(x)[-1], (1, 1), **kwargs)(y)  # no activation # Restore the number of original features\n    y = Add()([x, y])  # Add the bypass connection\n    y = Activation('relu')(y)\n    return y\n\n\ndef build_model(lr, l2, activation='sigmoid'):\n    ##############\n    # BRANCH MODEL\n    ##############\n    regul = regularizers.l2(l2)\n    optim = Adam(lr=lr)\n    kwargs = {'padding': 'same', 'kernel_regularizer': regul}\n\n    inp = Input(shape=(image_size, image_size, 3))  # 384x384x1\n    x = Conv2D(64, (9, 9), strides=2, activation='relu', **kwargs)(inp)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 96x96x64\n    for _ in range(2):\n        x = BatchNormalization()(x)\n        x = Conv2D(64, (3, 3), activation='relu', **kwargs)(x)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 48x48x64\n    x = BatchNormalization()(x)\n    x = Conv2D(128, (1, 1), activation='relu', **kwargs)(x)  # 48x48x128\n    for _ in range(4):\n        x = subblock(x, 64, **kwargs)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 24x24x128\n    x = BatchNormalization()(x)\n    x = Conv2D(256, (1, 1), activation='relu', **kwargs)(x)  # 24x24x256\n    for _ in range(4):\n        x = subblock(x, 64, **kwargs)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 12x12x256\n    x = BatchNormalization()(x)\n    x = Conv2D(384, (1, 1), activation='relu', **kwargs)(x)  # 12x12x384\n    for _ in range(4):\n        x = subblock(x, 96, **kwargs)\n\n    x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 6x6x384\n    x = BatchNormalization()(x)\n    x = Conv2D(512, (1, 1), activation='relu', **kwargs)(x)  # 6x6x512\n    for _ in range(4):\n        x = subblock(x, 128, **kwargs)\n\n    x = GlobalMaxPooling2D()(x)  # 512\n    branch_model = Model(inp, x)\n\n    ############\n    # HEAD MODEL\n    ############\n    mid = 32\n    xa_inp = Input(shape=branch_model.output_shape[1:])\n    xb_inp = Input(shape=branch_model.output_shape[1:])\n    x1 = Lambda(lambda x: x[0] * x[1])([xa_inp, xb_inp])\n    x2 = Lambda(lambda x: x[0] + x[1])([xa_inp, xb_inp])\n    x3 = Lambda(lambda x: K.abs(x[0] - x[1]))([xa_inp, xb_inp])\n    x4 = Lambda(lambda x: K.square(x))(x3)\n    x = Concatenate()([x1, x2, x3, x4])\n    x = Reshape((4, branch_model.output_shape[1], 1), name='reshape1')(x)\n\n    # Per feature NN with shared weight is implemented using CONV2D with appropriate stride.\n    x = Conv2D(mid, (4, 1), activation='relu', padding='valid')(x)\n    x = Reshape((branch_model.output_shape[1], mid, 1))(x)\n    x = Conv2D(1, (1, mid), activation='linear', padding='valid')(x)\n    x = Flatten(name='flatten')(x)\n\n    # Weighted sum implemented as a Dense layer.\n    x = Dense(1, use_bias=True, activation=activation, name='weighted-average')(x)\n    head_model = Model([xa_inp, xb_inp], x, name='head')\n\n    ########################\n    # SIAMESE NEURAL NETWORK\n    ########################\n    # Complete model is constructed by calling the branch model on each input image,\n    # and then the head model on the resulting 512-vectors.\n    img_a = Input(shape=(image_size, image_size, 3))\n    img_b = Input(shape=(image_size, image_size, 3))\n    xa = branch_model(img_a)\n    xb = branch_model(img_b)\n    x = head_model([xa, xb])\n    model = Model([img_a, img_b], x)\n    model.compile(optim, loss='binary_crossentropy', metrics=['binary_crossentropy', 'acc'])\n    return model, branch_model, head_model\n\n\nmodel, branch_model, head_model = build_model(64e-5, 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e467fae795f6fe546e845b1df200020301d1ed33","scrolled":true},"cell_type":"code","source":"data = pd.read_csv(path_csv)\nprint(data.shape)\ndata = data[data['Id'] != 'new_whale']\nprint(data.shape)\ntrain, test = train_test_split(data, train_size=0.7, random_state=1337)\nfile_id_mapping_train = {k: v for k, v in zip(train.Image.values, train.Id.values)}\nfile_id_mapping_test = {k: v for k, v in zip(test.Image.values, test.Id.values)}\ngen_tr = gen(sample_gen(file_id_mapping_train))\ngen_te = gen(sample_gen(file_id_mapping_test))\n\ncheckpoint = ModelCheckpoint(path_model, monitor='loss', verbose=1, save_best_only=True, mode='min')\nreduce_lr = ReduceLROnPlateau(monitor='val_loss', mode = 'min',factor=0.5, patience=5, min_lr=0.00000001, verbose=1)\nearly = EarlyStopping(monitor=\"val_loss\", mode=\"min\", patience=20)\nmodel_checkpoint = ModelCheckpoint('siamese.model',monitor='val_loss', \n                                   mode = 'min', save_best_only=True, verbose=1)\ncallbacks_list = [model_checkpoint, reduce_lr]  # early","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0cff6195879e97b150606af7721bb06c3ff40488"},"cell_type":"code","source":"def ShowImg(img):\n    plt.figure(figsize=(15,8))\n    plt.imshow(img.astype('uint8'))\n    plt.show()\n    plt.close()\n\nbatch = next(gen_tr)\n\nimg = batch[0][0][0]\nprint(img.shape)\nmean = [103.939, 116.779, 123.68]\nimg[..., 0] += mean[0]\nimg[..., 1] += mean[1]\nimg[..., 2] += mean[2]\nimg = img[..., ::-1]\nShowImg(img)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"368b3aae41868bf9e1b5c44cf2906be90357e248"},"cell_type":"markdown","source":"# Installation of Resnet 50 Weight to keras"},{"metadata":{"trusted":true,"_uuid":"8ba76c4665ebb1d6d065c3f2902964cd0b24e73d","scrolled":false},"cell_type":"code","source":"history = model.fit_generator(gen_tr,\n                              validation_data=gen_te,\n                              epochs=60, \n                              verbose=1, \n                              workers=4,\n                              steps_per_epoch=200,\n                              validation_steps = 80,\n                              callbacks=callbacks_list,use_multiprocessing=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ea2186e92e56bac0ece43bf1e521d80e7c0fb62"},"cell_type":"code","source":"import pickle\nwith open('siamese.pickle', 'wb') as handle:\n    pickle.dump(triplet_model.get_weights(), handle, protocol=pickle.HIGHEST_PROTOCOL)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11d7567169a0c59d65ebc2d9531b9ad5f9b3c003"},"cell_type":"code","source":"def data_generator(fpaths, batch=16):\n    i = 0\n    imgs = []\n    fnames = []\n    for path in fpaths:\n        if i == 0:\n            imgs = []\n            fnames = []\n        i += 1\n        img = read_and_resize(path)\n        imgs.append(img)\n        fnames.append(os.path.basename(path))\n        if i == batch:\n            i = 0\n            imgs = np.array(imgs)\n            yield fnames, imgs\n            \n    if i != 0:\n        imgs = np.array(imgs)\n        yield fnames, imgs\n        \n    raise StopIteration()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93c43c75d69dd9e8f32ba4af15938d9e1d9e12da"},"cell_type":"code","source":"data = pd.read_csv(path_csv)\nfile_id_mapping = {k: v for k, v in zip(data.Image.values, data.Id.values)}\nimport glob\n\ntrain_files = glob.glob(join(path_train, '*.jpg'))\ntest_files = glob.glob(join(path_test, '*.jpg'))\nprint(len(train_files), len(test_files))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a617b87a15d2b121cea2e4c09fccaa582653845"},"cell_type":"code","source":"fdict = {}\ncounts = {}\n#print(path_train)\nfor file in train_files:\n    #print(str(file[15:]))\n    img = read_and_resize(file)\n    img = img.reshape((1, image_size, image_size, 3))\n    class_ = file_id_mapping[str(file[-13:])]              \n    features = embedding_model.predict(img)\n    if class_ not in fdict:\n        fdict[class_] = features\n        counts[class_] = 1\n    else:\n        fdict[class_] += features\n        counts[class_] += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e31eae8137e61eebf5265dce6ff41da2cd6e7253","scrolled":false},"cell_type":"code","source":"for class_ in fdict:\n    fdict[class_] /= counts[class_]\n\nfor file in train_files[:500]:\n    #print(str(file[15:]))\n    trueclass = file_id_mapping[str(file[-13:])]\n    if trueclass == 'new_whale':\n        continue\n    img = read_and_resize(file)\n    img = img.reshape((1, image_size, image_size, 3))\n    class_ = file_id_mapping[str(file[-13:])]              \n    features = embedding_model.predict(img)\n    dists = []\n    cl = []\n    for c in fdict:\n        dist = np.linalg.norm(features - fdict[c])\n        dists.append(dist)\n        cl.append(c)\n    res = [x for _,x in sorted(zip(dists, cl))]\n    d = [x for x,_ in sorted(zip(dists, cl))]\n    trueclass = file_id_mapping[str(file[-13:])]\n    print(res.index(trueclass), trueclass, counts[trueclass], d[:5])\n            ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}