{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip uninstall -y tensorflow\n# !pip install tensorflow==2.2.0 tensorflow_gcs_config==2.2.0\n# !pip install cloud-tpu-client","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:19:19.539875Z","iopub.execute_input":"2022-02-19T04:19:19.540647Z","iopub.status.idle":"2022-02-19T04:19:19.559075Z","shell.execute_reply.started":"2022-02-19T04:19:19.540517Z","shell.execute_reply":"2022-02-19T04:19:19.558381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import tensorflow as tf\n# print(tf.__version__)\n\n# from cloud_tpu_client import Client\n# Client().configure_tpu_version(tf.__version__, restart_type='ifNeeded')","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:19:19.560484Z","iopub.execute_input":"2022-02-19T04:19:19.560847Z","iopub.status.idle":"2022-02-19T04:19:19.563895Z","shell.execute_reply.started":"2022-02-19T04:19:19.560819Z","shell.execute_reply":"2022-02-19T04:19:19.563258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os, sys, cv2\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nprint(tf.__version__)\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping,LearningRateScheduler","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:19:19.565420Z","iopub.execute_input":"2022-02-19T04:19:19.565748Z","iopub.status.idle":"2022-02-19T04:19:25.635825Z","shell.execute_reply.started":"2022-02-19T04:19:19.565721Z","shell.execute_reply":"2022-02-19T04:19:25.635239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data.csv excluded id_freq>150\ndf= pd.read_csv('../input/dataframe-startnotebook/data.csv')\ndf_triplet= pd.read_csv('../input/dataframe-startnotebook/tripletData.csv')\nEncoder=LabelEncoder()\ndf['id_label']=Encoder.fit_transform(df.individual_id)\nnp.save('classes.npy', Encoder.classes_)\n# enc.classes_ = np.load('classes.npy', allow_pickle=True)\n# enc.inverse_transform([y1, y2])\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:19:25.636994Z","iopub.execute_input":"2022-02-19T04:19:25.637231Z","iopub.status.idle":"2022-02-19T04:19:25.959611Z","shell.execute_reply.started":"2022-02-19T04:19:25.637204Z","shell.execute_reply":"2022-02-19T04:19:25.958700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(df_triplet))\ndf_triplet.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:19:25.961706Z","iopub.execute_input":"2022-02-19T04:19:25.961948Z","iopub.status.idle":"2022-02-19T04:19:25.975103Z","shell.execute_reply.started":"2022-02-19T04:19:25.961922Z","shell.execute_reply":"2022-02-19T04:19:25.974314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_triplet2= df_triplet.dropna()\ndf_triplet= df_triplet[['anchor', 'positive', 'negative0']]\nnewDict= {'anchor':[], 'positive':[], 'negative0':[]}\nfor i, row in df_triplet2.iterrows():\n    newDict['anchor']+= [row.anchor, row.anchor]\n    newDict['positive']+= [row.positive, row.positive]\n    newDict['negative0']+= [row.negative1, row.negative2]\ndf_triplet2= pd.DataFrame(newDict)\ndf_triplet= pd.concat([df_triplet, df_triplet2])\ndf_triplet = df_triplet.sample(frac=1).reset_index(drop=True)\ndf_triplet.info()","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:19:25.976245Z","iopub.execute_input":"2022-02-19T04:19:25.976552Z","iopub.status.idle":"2022-02-19T04:19:27.040887Z","shell.execute_reply.started":"2022-02-19T04:19:25.976522Z","shell.execute_reply":"2022-02-19T04:19:27.039937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_size = 456\nn_epochs = 5\nlr= 0.0001\nval_split= 0.1\nseed= 20\nbatch_size=16","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:19:27.042044Z","iopub.execute_input":"2022-02-19T04:19:27.042340Z","iopub.status.idle":"2022-02-19T04:19:27.047285Z","shell.execute_reply.started":"2022-02-19T04:19:27.042293Z","shell.execute_reply":"2022-02-19T04:19:27.046297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n        \n        # set half precision policy\n        # Mixed precision is the use of both 16-bit and 32-bit floating-point types\n        # in a model during training to make it run faster and use less memory.\n        mixed_precision.set_policy('mixed_bfloat16' if TPU else 'float32')\n        print(f'Compute dtype: {mixed_precision.global_policy().compute_dtype}')\n        print(f'Variable dtype: {mixed_precision.global_policy().variable_dtype}')\n        \n        # enable XLA optmizations\n        tf.config.optimizer.set_jit(True)\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:19:27.048610Z","iopub.execute_input":"2022-02-19T04:19:27.048847Z","iopub.status.idle":"2022-02-19T04:19:27.061302Z","shell.execute_reply.started":"2022-02-19T04:19:27.048821Z","shell.execute_reply":"2022-02-19T04:19:27.060605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def readImg(target_size=(512, 512)):\n    def readOnly(path):\n        file_bytes = tf.io.read_file(path)\n        img = tf.image.decode_jpeg(file_bytes, channels=3)\n        img= tf.cast(img, tf.float32)/255.0\n        return tf.image.resize(img, target_size)\n    def read3(row):\n        path1, path2, path3= row['anchor'], row['positive'], row['negative0']\n        return readOnly(path1), readOnly(path2), readOnly(path3)\n    return read3\n\n\ndef build_dataset(df, bsize=20,\n                  decode_fn=None, repeat=True, buffer=200):\n    if decode_fn is None:\n        decode_fn = readImg()\n        \n    AUTO = tf.data.experimental.AUTOTUNE\n    dset = tf.data.Dataset.from_tensor_slices(dict(df))\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(buffer) if buffer else dset\n    dset = dset.batch(bsize).prefetch(AUTO) # overlaps data preprocessing and model execution while training\n    return dset","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:19:27.062218Z","iopub.execute_input":"2022-02-19T04:19:27.062470Z","iopub.status.idle":"2022-02-19T04:19:27.076768Z","shell.execute_reply.started":"2022-02-19T04:19:27.062434Z","shell.execute_reply":"2022-02-19T04:19:27.076081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TPU=True\nDATASET_NAME = \"happy-whale-and-dolphin\"\nstrategy = auto_select_accelerator()\nbatch_size = strategy.num_replicas_in_sync * batch_size\nprint('batch size', batch_size)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:19:27.077900Z","iopub.execute_input":"2022-02-19T04:19:27.078177Z","iopub.status.idle":"2022-02-19T04:19:32.779664Z","shell.execute_reply.started":"2022-02-19T04:19:27.078145Z","shell.execute_reply":"2022-02-19T04:19:32.779052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path(DATASET_NAME)\nGCS_DS_PATH #   ='gs://kds-129aef496145c7329bf35723a6d8c89a4a331d719ecacd84746ce6b8'","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:19:32.780587Z","iopub.execute_input":"2022-02-19T04:19:32.780802Z","iopub.status.idle":"2022-02-19T04:19:33.437198Z","shell.execute_reply.started":"2022-02-19T04:19:32.780777Z","shell.execute_reply":"2022-02-19T04:19:33.436174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_triplet = df_triplet.apply(lambda x: GCS_DS_PATH+ '/train_images/' + x)\ntrain_paths, val_paths = train_test_split(df_triplet, test_size=val_split, random_state=seed)\nprint(len(train_paths), len(val_paths))\ntrain_paths.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:19:33.438503Z","iopub.execute_input":"2022-02-19T04:19:33.438800Z","iopub.status.idle":"2022-02-19T04:19:33.526352Z","shell.execute_reply.started":"2022-02-19T04:19:33.438762Z","shell.execute_reply":"2022-02-19T04:19:33.525508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoder = readImg(target_size=(img_size, img_size))\n\n# Build the tensorflow datasets\ndtrain = build_dataset(\n    train_paths, bsize=batch_size, decode_fn=decoder)\n\ndvalid = build_dataset(\n    val_paths, bsize=batch_size*2, \n    repeat=False, buffer=False, decode_fn=decoder)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:19:33.527989Z","iopub.execute_input":"2022-02-19T04:19:33.528295Z","iopub.status.idle":"2022-02-19T04:19:33.829038Z","shell.execute_reply.started":"2022-02-19T04:19:33.528264Z","shell.execute_reply":"2022-02-19T04:19:33.828130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data0,data1= dtrain.take(2)\n\nimages=[]\nfor i in range(3):\n    images+=data0[i][:4].numpy().tolist()","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:19:33.832964Z","iopub.execute_input":"2022-02-19T04:19:33.834198Z","iopub.status.idle":"2022-02-19T04:20:41.057443Z","shell.execute_reply.started":"2022-02-19T04:19:33.834158Z","shell.execute_reply":"2022-02-19T04:20:41.056390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ANCHOR\n#POSITIVE \n#NEGATIVE\nfig, axes = plt.subplots(3, 4, figsize=(20,10))\naxes = axes.flatten()\nfor img, ax in zip(images, axes):\n    ax.imshow(img)\n    ax.axis('off')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:20:41.058678Z","iopub.execute_input":"2022-02-19T04:20:41.058934Z","iopub.status.idle":"2022-02-19T04:20:46.545754Z","shell.execute_reply.started":"2022-02-19T04:20:41.058905Z","shell.execute_reply":"2022-02-19T04:20:46.544647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class eluDistance(tf.keras.layers.Layer):\n    def __init__(self, **kwargs):\n        super().__init__(**kwargs)\n    def call(self, anchor, positive, negative):\n        ap_distance = tf.reduce_sum(tf.square(anchor - positive), -1)\n        an_distance = tf.reduce_sum(tf.square(anchor - negative), -1)\n        return (ap_distance, an_distance)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:20:46.547480Z","iopub.execute_input":"2022-02-19T04:20:46.547830Z","iopub.status.idle":"2022-02-19T04:20:46.554795Z","shell.execute_reply.started":"2022-02-19T04:20:46.547776Z","shell.execute_reply":"2022-02-19T04:20:46.554041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def buildModel():\n    anchor_input = layers.Input(name=\"anchor\", shape=(img_size, img_size, 3))\n    positive_input = layers.Input(name=\"positive\", shape=(img_size, img_size, 3))\n    negative_input = layers.Input(name=\"negative\", shape=(img_size, img_size, 3))\n    \n    base= tf.keras.applications.ResNet50V2(input_shape=(img_size, img_size, 3),\n                                           include_top=False, pooling='avg')\n    for layer in base.layers:\n        if isinstance(layer, layers.BatchNormalization):\n            layer.trainable = False\n        else:\n            layer.trainable = True\n    \n    dropout = layers.Dropout(0.25, name='dropout')\n    reduce = layers.Dense(512, activation='linear', name='reduce')\n    \n    distances = eluDistance()(\n        reduce(dropout(base(anchor_input))),\n        reduce(dropout(base(positive_input))),\n        reduce(dropout(base(negative_input))),\n    )\n    \n    return  tf.keras.Model(inputs=[anchor_input, positive_input, negative_input], outputs=distances)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:20:46.556463Z","iopub.execute_input":"2022-02-19T04:20:46.557044Z","iopub.status.idle":"2022-02-19T04:20:46.571691Z","shell.execute_reply.started":"2022-02-19T04:20:46.557005Z","shell.execute_reply":"2022-02-19T04:20:46.570950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/xhlulu/shopee-siamese-resnet-50-with-triplet-loss-on-tpu\n\nclass SiameseModel(tf.keras.Model):\n    def __init__(self, siamese_network, margin=0.5):\n        super(SiameseModel, self).__init__()\n        self.siamese_network = siamese_network\n        self.margin = margin\n        self.loss_tracker = tf.keras.metrics.Mean(name=\"loss\")\n        \n    def call(self, inputs):\n        return self.siamese_network(inputs)\n    \n    def _compute_loss(self, data):\n        ap_distance, an_distance= self.siamese_network(data)\n        loss = ap_distance - an_distance\n        loss = tf.maximum(loss + self.margin, 0.0)\n        return loss\n    \n    def train_step(self, data):\n        with tf.GradientTape() as tape:\n            loss = self._compute_loss(data)\n        gradients = tape.gradient(loss, self.siamese_network.trainable_weights)\n        self.optimizer.apply_gradients(zip(gradients, self.siamese_network.trainable_weights))\n        self.loss_tracker.update_state(loss)\n        return {\"loss\": self.loss_tracker.result()}\n    \n    def test_step(self, data):\n        loss = self._compute_loss(data)\n        self.loss_tracker.update_state(loss)\n        return {\"loss\": self.loss_tracker.result()}\n    @property\n    def metrics(self):\n        return [self.loss_tracker]","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:20:46.573162Z","iopub.execute_input":"2022-02-19T04:20:46.573624Z","iopub.status.idle":"2022-02-19T04:20:46.588475Z","shell.execute_reply.started":"2022-02-19T04:20:46.573593Z","shell.execute_reply":"2022-02-19T04:20:46.587778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def step_decay(epoch):\n    initial_lrate = 0.0001\n    drop = 0.5\n    epochs_drop = 5.0\n    lrate = initial_lrate* math.pow(drop, math.floor((1+epoch)/epochs_drop))\n    return lrate\n\nLR_START = 1e-5\nLR_MAX = 0.0002\nLR_RAMPUP_EPOCHS = 2\nLR_SUSTAIN_EPOCHS = 1\nLR_STEP_DECAY = 0.7\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = LR_MAX * LR_STEP_DECAY**((epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)//2)\n    return lr\n\nlrate = LearningRateScheduler(lrfn)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:20:46.589648Z","iopub.execute_input":"2022-02-19T04:20:46.590031Z","iopub.status.idle":"2022-02-19T04:20:46.605192Z","shell.execute_reply.started":"2022-02-19T04:20:46.589997Z","shell.execute_reply":"2022-02-19T04:20:46.604262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"name= 'SiameseResNet50V2.h5'\nckp = ModelCheckpoint(name,monitor = 'val_loss',\n                      verbose = 1, save_best_only = True, mode = 'min')\n        \nes = EarlyStopping(monitor = 'val_loss', min_delta = 1e-4, patience = 5, mode = 'min', \n                    restore_best_weights = True, verbose = 1)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:20:46.606338Z","iopub.execute_input":"2022-02-19T04:20:46.606566Z","iopub.status.idle":"2022-02-19T04:20:46.615688Z","shell.execute_reply.started":"2022-02-19T04:20:46.606540Z","shell.execute_reply":"2022-02-19T04:20:46.615074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model=buildModel()\n    siamese_model = SiameseModel(model)\n    siamese_model.compile(optimizer=Adam(lr))\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:20:46.616886Z","iopub.execute_input":"2022-02-19T04:20:46.618618Z","iopub.status.idle":"2022-02-19T04:21:01.089628Z","shell.execute_reply.started":"2022-02-19T04:20:46.618573Z","shell.execute_reply":"2022-02-19T04:21:01.088862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"steps_per_epoch = ((train_paths.shape[0] // batch_size)//100)*100-100\nsteps_per_epoch","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:21:01.090682Z","iopub.execute_input":"2022-02-19T04:21:01.091079Z","iopub.status.idle":"2022-02-19T04:21:01.097571Z","shell.execute_reply.started":"2022-02-19T04:21:01.091038Z","shell.execute_reply":"2022-02-19T04:21:01.096761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = siamese_model.fit(dtrain,                      \n                    validation_data=dvalid,                                       \n                    epochs=n_epochs,\n                    callbacks=[es,ckp,lrate],\n                    steps_per_epoch=steps_per_epoch,\n                    verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:21:01.099316Z","iopub.execute_input":"2022-02-19T04:21:01.099631Z","iopub.status.idle":"2022-02-19T04:21:06.674218Z","shell.execute_reply.started":"2022-02-19T04:21:01.099593Z","shell.execute_reply":"2022-02-19T04:21:06.672317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (12, 6))\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Loss\")\nplt.plot( history.history[\"loss\"], label = \"Training Loss\", marker='o')\nplt.plot( history.history[\"val_loss\"], label = \"Validation Loss\", marker='+')\nplt.grid(True)\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:21:06.675799Z","iopub.status.idle":"2022-02-19T04:21:06.676275Z","shell.execute_reply.started":"2022-02-19T04:21:06.676019Z","shell.execute_reply":"2022-02-19T04:21:06.676041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_= siamese_model(data0)\nsiamese_model.load_weights(name)","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:21:06.677562Z","iopub.status.idle":"2022-02-19T04:21:06.678039Z","shell.execute_reply.started":"2022-02-19T04:21:06.677778Z","shell.execute_reply":"2022-02-19T04:21:06.677802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    encoder = tf.keras.Sequential([\n        siamese_model.siamese_network.get_layer('resnet50v2'),\n        siamese_model.siamese_network.get_layer('dropout'),\n        siamese_model.siamese_network.get_layer('reduce'),\n    ])\n\n    encoder.save('encoder.h5')","metadata":{"execution":{"iopub.status.busy":"2022-02-19T04:21:06.679611Z","iopub.status.idle":"2022-02-19T04:21:06.680369Z","shell.execute_reply.started":"2022-02-19T04:21:06.680060Z","shell.execute_reply":"2022-02-19T04:21:06.680089Z"},"trusted":true},"execution_count":null,"outputs":[]}]}