{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This kernel is a training baseline using the dataset provided by @ayuraj. \n# I have made the dataset into TFRecords which I shall be using for this kernel.","metadata":{}},{"cell_type":"markdown","source":"## Dependencies","metadata":{}},{"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nimport tensorflow as tf\nfrom PIL import ImageFont\nfrom typing import List, Tuple\nfrom collections import Counter\nimport plotly.graph_objects as go\nfrom matplotlib import pyplot as plt\nfrom plotly.subplots import make_subplots\nfrom kaggle_datasets import KaggleDatasets\nimport seaborn as sns\nimport plotly.express as px\nimport tensorflow_addons as tfa\nfrom glob import glob","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\nfrom sklearn.model_selection import train_test_split\nfrom skmultilearn.model_selection import iterative_train_test_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We won't be needing the train_df here , as the labels we need are present in the TFRecords itself. ","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/hpasinglelabelcellcsv/singlelabelcellonly.csv\")\ntrain_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = KaggleDatasets().get_gcs_path('hpa-single-label-cell-level-tfrecords')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_DIR = os.path.join(BASE_DIR , 'tfrecords/')\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_DIR","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_TFRECORDS = tf.io.gfile.glob(os.path.join(IMG_DIR, '*.tfrec'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_TFRECORDS","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I have taken the following functions and some functions from this really neat [kernel](https://www.kaggle.com/soumikrakshit/hpa-baseline-on-tpu). Thanks @soumikrakshit.","metadata":{}},{"cell_type":"code","source":"class TFRecordLoader:\n\n    def __init__(self, image_size: List[int], n_classes: int):\n        self.image_size = image_size\n        self.n_classes = n_classes\n        \n\n    def _parse_image(self, image):\n        image = tf.image.decode_png(image, channels=3)\n        image = tf.cast(image, dtype=tf.float32) / 255.0\n        image = tf.image.resize(image, self.image_size)\n        \n        return image\n\n    def _parse_label(self, label):\n        indices = tf.strings.to_number(\n            label       \n        )\n        indices =tf.cast(indices ,dtype = tf.uint8)\n        return tf.one_hot(indices, depth=self.n_classes)\n        \n\n    def _make_example(self, example):\n        feature_format = {\n            'image': tf.io.FixedLenFeature([], dtype=tf.string),\n            'image_name': tf.io.FixedLenFeature([], dtype=tf.string),\n            'target': tf.io.FixedLenFeature([], dtype=tf.string)\n        }\n        features = tf.io.parse_single_example(example, features=feature_format)\n        image = self._parse_image(features['image'])\n        image_name = features['image_name']\n        label = self._parse_label(features['target'])\n        return image,  label\n\n   \n    \n\n    def get_dataset(self, train_tfrecord_files: List[str], ignore_order: bool = False):\n        options = tf.data.Options()\n        options.experimental_deterministic = False\n        dataset = tf.data.TFRecordDataset(\n            train_tfrecord_files, num_parallel_reads=tf.data.AUTOTUNE)\n        dataset = dataset.with_options(options) if ignore_order else dataset\n        dataset = dataset.map(\n            map_func=self._make_example, num_parallel_calls=tf.data.AUTOTUNE)\n        #dataset = self._preprocess(dataset)\n        return dataset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AugmentationFactory:\n\n    def __init__(self, include_flips: bool, include_rotation: bool, include_jitter: bool):\n        self.include_flips = include_flips\n        self.include_rotation = include_rotation\n        self.include_jitter = include_jitter\n\n    @staticmethod\n    def _flip_horizontal(image, seed):\n        image = tf.image.stateless_random_flip_left_right(image, seed)\n        return image\n\n    @staticmethod\n    def _flip_vertical(image, seed):\n        image = tf.image.stateless_random_flip_up_down(image, seed)\n        return image\n\n    @staticmethod\n    def _rotate(image):\n        rotation_k = tf.random.uniform((1,), minval=0, maxval=4, dtype=tf.int32)[0]\n        image = tf.image.rot90(image, k=rotation_k)\n        return image\n\n    @staticmethod\n    def _random_jitter(image, seed):\n        image = tf.image.stateless_random_saturation(image, 0.9, 1.1, seed)\n        image = tf.image.stateless_random_brightness(image, 0.075, seed)\n        image = tf.image.stateless_random_contrast(image, 0.9, 1.1, seed)\n        return image\n\n    def _map_augmentations(self, image, label):\n        seed = tf.random.uniform((2,), minval=0, maxval=100, dtype=tf.int32)\n        if self.include_flips:\n            image = self._flip_horizontal(image=image, seed=seed)\n            image = self._flip_vertical(image=image, seed=seed)\n        image = self._rotate(image=image) if self.include_rotation else image\n        image = self._random_jitter(image=image, seed=seed) if self.include_jitter else image\n        return image, label\n\n    def augment_dataset(self, dataset):\n        return dataset.map(\n            map_func=self._map_augmentations,\n            num_parallel_calls=tf.data.AUTOTUNE\n        )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loader = TFRecordLoader(\n    image_size=[224, 224], n_classes=19, \n)\ndataset = loader.get_dataset(TRAIN_TFRECORDS)\n\naugmentation_factory = AugmentationFactory(\n    include_flips=True, include_rotation=True, include_jitter=True\n)\ndataset = augmentation_factory.augment_dataset(dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for x in dataset.take(1):\n    plt.imshow(x[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_strategy():\n    try:  # detect TPUs\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    except ValueError:  # detect GPUs\n        strategy = tf.distribute.MirroredStrategy()  # for GPU or multi-GPU machines\n    print(\"Number of accelerators: \", strategy.num_replicas_in_sync)\n    return strategy","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_labels= tf.dtypes.cast(train_labels ,  dtype = tf.float32)\n# valid_labels = tf.dtypes.cast(valid_labels ,  dtype = tf.float32)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#IMSIZE = (224, 240, 260, 300, 380, 456, 528, 600)\nIMSIZE = 224\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def configure_train_dataset(augmented_dataset, shuffle_buffer: int = 128, batch_size: int = 16):\n    dataset = augmented_dataset.repeat()\n    dataset = augmented_dataset.shuffle(shuffle_buffer)\n    dataset = dataset.batch(batch_size)\n    dataset = dataset.prefetch(buffer_size=tf.data.AUTOTUNE)\n    return dataset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def configure_val_dataset(augmented_dataset, shuffle_buffer: int = 128, batch_size: int = 16):\n    dataset = augmented_dataset.repeat()\n    dataset = augmented_dataset.shuffle(shuffle_buffer)\n    dataset = dataset.batch(batch_size)\n    dataset = dataset.prefetch(buffer_size=tf.data.AUTOTUNE)\n    return dataset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Code for the model was taken from [this notebook](https://www.kaggle.com/dschettler8845/hpa-cellwise-classification-training/data).\nThanks @dschettler8845 for all the amazing work in this competition! :)","metadata":{}},{"cell_type":"code","source":"def get_backbone(efficientnet_name=\"efficientnet_b0\", input_shape=(224,224,3), include_top=False, weights=\"imagenet\", pooling=\"avg\"):\n    if \"b0\" in efficientnet_name:\n        eb = tf.keras.applications.EfficientNetB0(\n            include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n            )\n    elif \"b1\" in efficientnet_name:\n        eb = tf.keras.applications.EfficientNetB1(\n            include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n            )\n    elif \"b2\" in efficientnet_name:\n        eb = tf.keras.applications.EfficientNetB2(\n            include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n            )\n    elif \"b3\" in efficientnet_name:\n        eb = tf.keras.applications.EfficientNetB3(\n            include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n            )\n    elif \"b4\" in efficientnet_name:\n        eb = tf.keras.applications.EfficientNetB4(\n            include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n            )\n    elif \"b5\" in efficientnet_name:\n        eb = tf.keras.applications.EfficientNetB5(\n            include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n            )\n    elif \"b6\" in efficientnet_name:\n        eb = tf.keras.applications.EfficientNetB6(\n            include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n            )\n    elif \"b7\" in efficientnet_name:\n        eb = tf.keras.applications.EfficientNetB7(\n            include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n            )\n    else:\n        raise ValueError(\"Invalid EfficientNet Name!!!\")\n    return eb\n\n\ndef add_head_to_bb(bb, n_classes=19, dropout=0.05, head_layer_nodes=(512,)):\n    x = tf.keras.layers.BatchNormalization()(bb.output)\n    x = tf.keras.layers.Dropout(dropout)(x)\n    \n    for n_nodes in head_layer_nodes:\n        x = tf.keras.layers.Dense(n_nodes, activation=\"relu\")(x)\n        x = tf.keras.layers.BatchNormalization()(x)\n        x = tf.keras.layers.Dropout(dropout/2)(x)\n    \n    output = tf.keras.layers.Dense(n_classes, activation=\"sigmoid\")(x)\n    return tf.keras.Model(inputs=bb.inputs, outputs=output)\n\n\n#eb.compile(optimizer=OPTIMIZER, loss=LOSS_FN, metrics=[\"acc\", tf.keras.metrics.AUC(name=\"auc\", multi_label=True)])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since we do not have the length of our TFRecords dataset (it is a prefetch dataset) , I use the following functions to make a train/test split in our dataset. is_test returns 1 out of every 5 examples , is_train returns the remaining 4 out of 5. This results in a 80-20 train-test split of our dataset.","metadata":{}},{"cell_type":"code","source":"\ndef is_test(x, y):\n    return x % 5 == 0\n\ndef is_train(x, y):\n    return not is_test(x, y)\n\nrecover = lambda x,y: y\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy = get_strategy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loader = TFRecordLoader(\n    image_size=[IMSIZE, IMSIZE], n_classes=19\n    \n)\ndataset = loader.get_dataset(\n    TRAIN_TFRECORDS, ignore_order=True\n)\n\nval_dataset = dataset.enumerate() \\\n                    .filter(is_test) \\\n                    .map(recover)\n\ntrain_dataset = dataset.enumerate() \\\n                    .filter(is_train) \\\n                    .map(recover)\n        \n\naugmentation_factory = AugmentationFactory(\n    include_flips=True, include_rotation=False, include_jitter=True)\n\ntrain_dataset = augmentation_factory.augment_dataset(train_dataset)\nBATCH_SIZE = 64 * strategy.num_replicas_in_sync\ntrain_dataset = configure_train_dataset(\n    train_dataset, batch_size=BATCH_SIZE\n)\n\nval_dataset = configure_val_dataset(val_dataset, batch_size = BATCH_SIZE )\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    \nwith strategy.scope():\n    eff = get_backbone(\"b0\")\n    model = add_head_to_bb(eff, n_classes=19, dropout=0.5) \n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(),\n        loss=tf.keras.losses.BinaryCrossentropy(),\n        metrics=[tf.keras.metrics.AUC(multi_label=True)])\n        \n    model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#steps_per_epoch = train_paths.shape[0] // BATCH_SIZE\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(\n    'effb7model.h5', save_best_only=True, monitor='val_loss', mode='min')\nlr_reducer = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor=\"val_loss\", patience=3, min_lr=1e-6, mode='min')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training the Model","metadata":{}},{"cell_type":"markdown","source":"Steps per epoch is unknown to us as the length of the dataset is unknown , however after the first epoch it is calculated automatically by TF , so the first epoch will show x/unknown for the number of steps during the run.","metadata":{}},{"cell_type":"code","source":"history = model.fit(\n    train_dataset, \n    epochs=10,\n    verbose=1,\n    callbacks=[checkpoint, lr_reducer],\n    \n    validation_data=val_dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist_df = pd.DataFrame(history.history)\nhist_df.to_csv('history.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nsave_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\nmodel.save('./model', options=save_locally) # saving in Tensorflow's \"SavedModel\" format","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}