{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Hello Friends 👋\n\nAssuming this as a **Multiclass classification** task, I'm trying-out end-to-end classification (*SparseCategoricalCrossentropy Loss*) (linking KAGGLE-data cloud bucket). \n\n\nNotebook is for-\n* getting started faster\n* beginners who want to try out TPU training\n\nSpecial thanks to-\n* https://www.kaggle.com/ks2019/happywhale-arcface-baseline-tpu\n* https://www.kaggle.com/docs/tpu\n* https://www.kaggle.com/product-feedback/129828","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os, sys, cv2, math\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping, LearningRateScheduler","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:40:16.474091Z","iopub.execute_input":"2022-02-07T07:40:16.474793Z","iopub.status.idle":"2022-02-07T07:40:22.645814Z","shell.execute_reply.started":"2022-02-07T07:40:16.474667Z","shell.execute_reply":"2022-02-07T07:40:22.644920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data.csv excluded id_freq>150\ndf= pd.read_csv('../input/dataframe-startnotebook/data.csv')\nEncoder=LabelEncoder()\ndf['id_label']=Encoder.fit_transform(df.individual_id)\nnp.save('classes.npy', Encoder.classes_)\n# enc.classes_ = np.load('classes.npy', allow_pickle=True)\n# enc.inverse_transform([y1, y2])\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:40:22.647460Z","iopub.execute_input":"2022-02-07T07:40:22.647734Z","iopub.status.idle":"2022-02-07T07:40:22.829701Z","shell.execute_reply.started":"2022-02-07T07:40:22.647705Z","shell.execute_reply":"2022-02-07T07:40:22.828724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_classes= df.id_label.max()+1\nimg_size = 600\nn_epochs = 40\nlr= 0.0001\nval_split= 0.2\nseed= 2001\nbatch_size=16\nn_classes","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:40:22.831156Z","iopub.execute_input":"2022-02-07T07:40:22.832031Z","iopub.status.idle":"2022-02-07T07:40:22.840523Z","shell.execute_reply.started":"2022-02-07T07:40:22.831986Z","shell.execute_reply":"2022-02-07T07:40:22.839774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TPU Input Pipeline\nUsefull links\n* https://www.tensorflow.org/guide/tpu\n* https://www.tensorflow.org/guide/data_performance","metadata":{}},{"cell_type":"code","source":"def auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:40:22.843025Z","iopub.execute_input":"2022-02-07T07:40:22.843296Z","iopub.status.idle":"2022-02-07T07:40:22.851254Z","shell.execute_reply.started":"2022-02-07T07:40:22.843256Z","shell.execute_reply":"2022-02-07T07:40:22.850522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def readImg(with_labels=True, target_size=(512, 512)):\n    def readOnly(path):\n        file_bytes = tf.io.read_file(path)\n        img = tf.image.decode_jpeg(file_bytes, channels=3)\n        img= tf.cast(img, tf.float32)/255.0\n        return tf.image.resize(img, target_size)\n    def readWithLabels(path, label):\n        return readOnly(path), label\n    return readWithLabels if with_labels else readOnly\n\ndef build_augmenter(with_labels=True):\n    def augment(img):\n        img = tf.image.random_flip_left_right(img)\n        #img = tf.image.random_flip_up_down(img)\n        img = tf.image.random_saturation(img, 0.8, 1.2)\n        img = tf.image.random_brightness(img, 0.1)\n        img = tf.image.random_contrast(img, 0.8, 1.2)\n        return img\n    def augment_with_labels(img, label):\n        return augment(img), label\n    return augment_with_labels if with_labels else augment\n\ndef build_dataset(paths, labels=None, bsize=20,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=2001,\n                  cache_dir=\"\", cache=True):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n        \n    if decode_fn is None:\n        decode_fn = readImg(labels is not None)\n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n        \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize).prefetch(AUTO) # overlaps data preprocessing and model execution while training\n    return dset","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:40:22.852412Z","iopub.execute_input":"2022-02-07T07:40:22.852806Z","iopub.status.idle":"2022-02-07T07:40:22.867128Z","shell.execute_reply.started":"2022-02-07T07:40:22.852762Z","shell.execute_reply":"2022-02-07T07:40:22.866369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATASET_NAME = \"happy-whale-and-dolphin\"\nstrategy = auto_select_accelerator()\nbatch_size = strategy.num_replicas_in_sync * batch_size\nprint('batch size', batch_size)","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:40:22.868390Z","iopub.execute_input":"2022-02-07T07:40:22.868607Z","iopub.status.idle":"2022-02-07T07:40:28.581515Z","shell.execute_reply.started":"2022-02-07T07:40:22.868582Z","shell.execute_reply":"2022-02-07T07:40:28.580703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path(DATASET_NAME)\npaths = GCS_DS_PATH + \"/train_images/\" + df['image']\nGCS_DS_PATH","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:40:28.582535Z","iopub.execute_input":"2022-02-07T07:40:28.582855Z","iopub.status.idle":"2022-02-07T07:40:29.020579Z","shell.execute_reply.started":"2022-02-07T07:40:28.582827Z","shell.execute_reply":"2022-02-07T07:40:29.019942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train test split\n(train_paths, valid_paths, \n  train_labels, valid_labels) = train_test_split(paths, df.id_label.values.reshape(-1,1).astype('float32'),\n                                                 test_size=val_split, random_state=seed)\n\nprint(train_paths.shape, valid_paths.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:40:29.021545Z","iopub.execute_input":"2022-02-07T07:40:29.022203Z","iopub.status.idle":"2022-02-07T07:40:29.039358Z","shell.execute_reply.started":"2022-02-07T07:40:29.022169Z","shell.execute_reply":"2022-02-07T07:40:29.038300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoder = readImg(with_labels=True, target_size=(img_size, img_size))\n\n# Build the tensorflow datasets\ndtrain = build_dataset(\n    train_paths, train_labels, bsize=batch_size, decode_fn=decoder)\n\ndvalid = build_dataset(\n    valid_paths, valid_labels, bsize=batch_size, \n    repeat=False, shuffle=False, augment=False, decode_fn=decoder)","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:40:29.041084Z","iopub.execute_input":"2022-02-07T07:40:29.041330Z","iopub.status.idle":"2022-02-07T07:40:29.327565Z","shell.execute_reply.started":"2022-02-07T07:40:29.041298Z","shell.execute_reply":"2022-02-07T07:40:29.326896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data, _ = dtrain.take(2)\nimages = data[0].numpy()","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:40:29.330277Z","iopub.execute_input":"2022-02-07T07:40:29.330936Z","iopub.status.idle":"2022-02-07T07:42:11.559579Z","shell.execute_reply.started":"2022-02-07T07:40:29.330892Z","shell.execute_reply":"2022-02-07T07:42:11.558573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(3, 4, figsize=(20,10))\naxes = axes.flatten()\nfor img, ax in zip(images, axes):\n    ax.imshow(img)\n    ax.axis('off')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:42:11.560871Z","iopub.execute_input":"2022-02-07T07:42:11.561106Z","iopub.status.idle":"2022-02-07T07:42:13.856309Z","shell.execute_reply.started":"2022-02-07T07:42:11.561079Z","shell.execute_reply":"2022-02-07T07:42:13.855611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CNN Model","metadata":{}},{"cell_type":"code","source":"# https://www.kaggle.com/ks2019/happywhale-arcface-baseline-tpu\n\nclass ArcMarginProduct(tf.keras.layers.Layer):\n    '''\n    Implements large margin arc distance.\n\n    Reference:\n        https://arxiv.org/pdf/1801.07698.pdf\n        https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n            blob/master/src/modeling/metric_learning.py\n    '''\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n\n        super(ArcMarginProduct, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n\n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(ArcMarginProduct, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n\n    def call(self, inputs):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n        cosine = tf.matmul(\n            tf.math.l2_normalize(X, axis=1),\n            tf.math.l2_normalize(self.W, axis=0)\n        )\n        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = tf.where(cosine > 0, phi, cosine)\n        else:\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n        one_hot = tf.cast(\n            tf.one_hot(y, depth=self.n_classes),\n            dtype=cosine.dtype\n        )\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:42:13.857428Z","iopub.execute_input":"2022-02-07T07:42:13.858115Z","iopub.status.idle":"2022-02-07T07:42:13.879948Z","shell.execute_reply.started":"2022-02-07T07:42:13.858072Z","shell.execute_reply":"2022-02-07T07:42:13.879070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model():\n    inp = layers.Input(shape = (img_size, img_size, 3))\n    base=tf.keras.applications.EfficientNetB6(input_tensor= inp, include_top=False,\n                                   classes= n_classes, weights='imagenet')\n    \n    x= base(inp)\n    x= layers.GlobalAveragePooling2D()(layers.Dropout(0.15)(x))\n    x= layers.Dropout(0.2)(x)\n    x= layers.Dense(n_classes, 'softmax')(x)\n    return tf.keras.Model(inp, x)","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:42:13.881490Z","iopub.execute_input":"2022-02-07T07:42:13.881793Z","iopub.status.idle":"2022-02-07T07:42:13.894570Z","shell.execute_reply.started":"2022-02-07T07:42:13.881766Z","shell.execute_reply":"2022-02-07T07:42:13.893958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def step_decay(epoch):\n    initial_lrate = 0.0001\n    drop = 0.5\n    epochs_drop = 5.0\n    lrate = initial_lrate* math.pow(drop, math.floor((1+epoch)/epochs_drop))\n    return lrate\n\nLR_START = 1e-5\nLR_MAX = 0.0001\nLR_RAMPUP_EPOCHS = 2\nLR_SUSTAIN_EPOCHS = 1\nLR_STEP_DECAY = 0.7\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = LR_MAX * LR_STEP_DECAY**((epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)//2)\n    return lr\n\nlrate = LearningRateScheduler(lrfn)","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:42:13.895916Z","iopub.execute_input":"2022-02-07T07:42:13.896366Z","iopub.status.idle":"2022-02-07T07:42:13.906685Z","shell.execute_reply.started":"2022-02-07T07:42:13.896338Z","shell.execute_reply":"2022-02-07T07:42:13.905916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model= build_model()\n    loss= tf.keras.losses.SparseCategoricalCrossentropy()\n    model.compile(Adam(lr=lr),loss=loss)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:42:13.907779Z","iopub.execute_input":"2022-02-07T07:42:13.908029Z","iopub.status.idle":"2022-02-07T07:42:49.660056Z","shell.execute_reply.started":"2022-02-07T07:42:13.908002Z","shell.execute_reply":"2022-02-07T07:42:49.659156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"name= 'EfficientNetB5v1.h5'\nckp = ModelCheckpoint(name,monitor = 'val_loss',\n                      verbose = 1, save_best_only = True, mode = 'min')\n        \nes = EarlyStopping(monitor = 'val_loss', min_delta = 1e-4, patience = 5, mode = 'min', \n                    restore_best_weights = True, verbose = 1)","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:42:49.661250Z","iopub.execute_input":"2022-02-07T07:42:49.661480Z","iopub.status.idle":"2022-02-07T07:42:49.667783Z","shell.execute_reply.started":"2022-02-07T07:42:49.661452Z","shell.execute_reply":"2022-02-07T07:42:49.666240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"steps_per_epoch = ((train_paths.shape[0] // batch_size)//100)*100 - 50\nsteps_per_epoch","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:42:49.668869Z","iopub.execute_input":"2022-02-07T07:42:49.669086Z","iopub.status.idle":"2022-02-07T07:42:49.857457Z","shell.execute_reply.started":"2022-02-07T07:42:49.669061Z","shell.execute_reply":"2022-02-07T07:42:49.856542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(dtrain,                      \n                    validation_data=dvalid,                                       \n                    epochs=n_epochs,\n                    callbacks=[es,ckp,lrate],\n                    steps_per_epoch=steps_per_epoch,\n                    verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-02-07T07:42:49.858886Z","iopub.execute_input":"2022-02-07T07:42:49.859349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (12, 6))\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Loss\")\nplt.plot( history.history[\"loss\"], label = \"Training Loss\", marker='o')\nplt.plot( history.history[\"val_loss\"], label = \"Validation Loss\", marker='+')\nplt.grid(True)\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}