{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfrom tensorflow.keras.layers.experimental import preprocessing\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nfrom keras.layers import Dense, Flatten, Activation, Conv2D, MaxPooling2D, Dropout, Conv2D,MaxPooling2D,GlobalAveragePooling2D,BatchNormalization\nfrom tensorflow.keras import Model\nfrom keras.applications import ResNet50 \nfrom tensorflow.keras.applications import ResNet152\nfrom keras.applications.inception_resnet_v2 import InceptionResNetV2\nfrom keras.applications.inception_v3 import InceptionV3\nfrom keras.models import Sequential\nfrom keras import optimizers\nfrom keras.optimizers import Adam,RMSprop\nfrom keras.callbacks import EarlyStopping\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.callbacks import ModelCheckpoint\nimport tensorflow_hub as hub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"print(\"update TPU server tensorflow version...\")\n\n!pip install cloud-tpu-client\nimport tensorflow as tf \nfrom cloud_tpu_client import Client\nprint(tf.__version__)\nClient().configure_tpu_version(tf.__version__, restart_type='ifNeeded')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initializing Parameters.\nimg_size = 224\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy\n\n\ndef build_decoder(with_labels=True, target_size=(224, 224), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True):\n    def augment(img):\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n        img = tf.image.random_hue(img, 0.01)\n        img = tf.image.random_saturation(img, 0.70, 1.30)\n        img = tf.image.random_contrast(img, 0.80, 1.20)\n        img = tf.image.random_brightness(img, 0.10)\n        return img\n    \n    def augment_with_labels(img, label):\n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, bsize=32, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=2048, \n                  cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize).prefetch(AUTO)\n    \n    return dset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COMPETITION_NAME = \"ranzcr-clip-catheter-line-classification\"\nstrategy = auto_select_accelerator()\nBATCH_SIZE = strategy.num_replicas_in_sync * 4\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(COMPETITION_NAME)\nprint('batch size', BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Create Dataset.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '/kaggle/input/ranzcr-clip-catheter-line-classification/'\ntrain_df = pd.read_csv(path + 'train.csv')\ntrain_images = GCS_DS_PATH + \"/train/\" + train_df['StudyInstanceUID'] + '.jpg'\n\nsample_submissions_df = pd.read_csv(path + 'sample_submission.csv')\ntest_images = GCS_DS_PATH + \"/test/\" + sample_submissions_df['StudyInstanceUID'] + '.jpg'\n\n\n\n# Get the multi-labels.\nlabel_columns = sample_submissions_df.columns[1:]\nlabels = train_df[label_columns].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Split Training Data.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train Test Split.\ntrain_img, valid_img, train_labels, valid_labels = train_test_split(train_images, \n                                                                    labels, \n                                                                    test_size=0.10, \n                                                                    random_state=42,\n                                                                    shuffle=True\n                                                                   )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Build TensorFlow Dataset.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Build the Tensorflow Train and Validation datasets.\n\ndecoder = build_decoder(with_labels=True, \n                        target_size=(img_size, img_size)\n                       )\n\ntrain_data = build_dataset(train_img,\n                           train_labels, \n                           bsize=BATCH_SIZE, \n                           decode_fn=decoder \n                          )\n\nvalid_data = build_dataset(valid_img, \n                           valid_labels, \n                           bsize=BATCH_SIZE, \n                           repeat=False, \n                           shuffle=False, \n                           augment=False, \n                           decode_fn=decoder\n                          )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Visualizing Data.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualize training data with augmentation.\nimport matplotlib.pyplot as plt\n\ndata, _ = train_data.take(2)\nimages = data[0].numpy()\n\nfig, axes = plt.subplots(4, 4, figsize=(12,12))\naxes = axes.flatten()\nfor img, ax in zip(images, axes):\n    ax.imshow(img)\n    ax.axis('off')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Building Model :Transfer learning with BiT_m_r_152x4.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function for decaying the learning rate.\n# In BiT-HyperRule, we use a vanilla SGD optimiser with an initial learning rate of 0.003, momentum 0.9 and batch size 512. \n# We decay the learning rate by a factor of 10 at 30%, 60% and 90% of the training steps.\n\ndef decay(epoch):\n  if epoch < 6:\n    return 1e-3\n  elif epoch >= 6 and epoch < 12:\n    return 1e-4\n  else:\n    return 1e-5\n# Callback for printing the LR at the end of each epoch.\nclass PrintLR(tf.keras.callbacks.Callback):\n  def on_epoch_end(self, epoch, logs=None):\n    print('\\nLearning rate for epoch {} is {}'.format(epoch + 1,\n                                                      model.optimizer.lr.numpy()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODELPATH = KaggleDatasets().get_gcs_path('big-transfer-models-without-top')\n# module = hub.KerasLayer(f'{MODELPATH}/bit_m-r101x1_1/')\nmodule = hub.KerasLayer(f'{MODELPATH}/bit_m-r101x3_1/')\n# module = hub.KerasLayer(f'{MODELPATH}/bit_m-r152x4_1/')\n# module = hub.KerasLayer(f'{MODELPATH}/bit_m-r50x1_1/')\n# module = hub.KerasLayer(f'{MODELPATH}/bit_m-r50x3_1/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    inputs = tf.keras.layers.Input(shape=(224,224,3))\n    \n    MODELPATH = KaggleDatasets().get_gcs_path('big-transfer-models-without-top')\n    module = hub.KerasLayer(f'{MODELPATH}/bit_m-r101x3_1/')\n    \n    back_bone = module\n    back_bone.trainable = True\n    logits = back_bone(inputs)\n    outputs = tf.keras.layers.Dense(11, activation='sigmoid', dtype='float32')(logits)\n    model = tf.keras.Model(inputs=inputs, outputs=outputs, name='Dr_Kudzayi_bit_m-r101x3_1_ranzcr_clip_catheter_possition_model')\n    \n    # Compile full Model.\n    model.compile(optimizer = tf.keras.optimizers.SGD(learning_rate=0.03, momentum=0.9),\n                  loss='binary_crossentropy',\n                  metrics=[tf.keras.metrics.AUC(name='auc',multi_label=True)]\n                 )\n    \n    model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"steps_per_epoch = train_images.shape[0] // BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save best Model weights.\ncheck_point = ModelCheckpoint('BiT_m_r101x3_1_RANZCR_Model_Best_Weights_TPU.h5',\n                              monitor = 'val_loss',\n                              save_best_only = True, \n                              mode = 'min',\n                              verbose = 1\n                             )\n\n# Reduce learning rate when a metric has stopped improving.\nreduce_learning_rate = tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_auc\", \n                                                            patience=2, \n                                                            factor=0.1,\n                                                            min_delta = 1e-3, \n                                                            min_lr=1e-6, \n                                                            mode='max',\n                                                            verbose = 1\n                                                           )\n\nearly_stop = EarlyStopping(monitor = 'val_auc', \n                           min_delta = 1e-3, \n                           patience = 6, \n                           mode = 'max', \n                           restore_best_weights = True, \n                           verbose = 1)  \n\n# lrschedule = tf.keras.callbacks.LearningRateScheduler(decay)\n    \n\ncallbacks_list = [check_point,reduce_learning_rate, early_stop]\n\ninitial_epochs = 40\n\n# Train Model.\nhistory = model.fit(train_data, \n                    validation_data=valid_data,\n                    epochs= initial_epochs,\n                    steps_per_epoch=steps_per_epoch ,                  \n                    callbacks=callbacks_list\n                   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reduce learning rate when a metric has stopped improving.\nreduce_learning_rate = tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_auc\", \n                                                            patience=2, \n                                                            factor=0.1,\n                                                            min_delta = 1e-4, \n                                                            min_lr=1e-6, \n                                                            mode='max',\n                                                            verbose = 1\n                                                           )\n\n\n# Save best Model weights.\ncheck_point = ModelCheckpoint('./BiT_m_r_152x4_RANZCR_Model_Best_Weights_TPU.h5',\n                              monitor = 'val_auc',\n                              save_best_only = True, \n                              mode = 'max',\n                              verbose = 1\n                             )\n        \n\n\n\n\ncallbacks_list = [reduce_learning_rate, check_point]\n\ninitial_epochs = 50\n\n# Train Model......\n\n\nhistory = model.fit(train_data, \n                    validation_data=valid_data,\n                    epochs=initial_epochs,\n                    steps_per_epoch=steps_per_epoch ,                  \n                    callbacks=callbacks_list\n                    )","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}