{"cells":[{"metadata":{},"cell_type":"markdown","source":"<h1 style=\"border:2px solid Purple;text-align:center\">TensorFlow Implementation</h1>"},{"metadata":{},"cell_type":"markdown","source":"**This is nice and simple baseline implemented in Tensorflow. And good for those who want to get started with this competition with baseline Tensorflow implementation.**"},{"metadata":{},"cell_type":"markdown","source":"**Do remember to upvote if you liked the content :)**"},{"metadata":{},"cell_type":"markdown","source":"I took a lot of help from Harveen's Notebook [here](https://www.kaggle.com/harveenchadha/efficientnetb3-tf2-keras-baseline/notebook), please upvote his work as well."},{"metadata":{},"cell_type":"markdown","source":"<h1 style=\"border:2px solid Purple;text-align:center\">Basic Imports</h1>"},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten,GlobalAveragePooling2D,BatchNormalization, Activation\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.compat.v1 import ConfigProto\nfrom tensorflow.compat.v1 import InteractiveSession\nfrom tensorflow import keras\nfrom tensorflow.keras.applications import EfficientNetB3\n\n\nimport os\nimport glob\n\nconfig = ConfigProto()\nconfig.gpu_options.allow_growth = True\nsession = InteractiveSession(config=config)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h1 style=\"border:2px solid Purple;text-align:center\">Configurations</h1>"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n    \n    seed = 44\n    num_classes = 11\n    class_list = [0,1,2,3,4,5,6,7,8,9,10]\n    batch_size = 32\n    n_epochs = 15\n    drop_rate = 0.4\n   \n    #scheduler = 'CosineAnnealingLR'\n    \n    scheduler_params = {           \n                    'ReduceLROnPlateau': \n                        {\n                            'monitor':'val_loss',\n                            'mode':'max',\n                            'factor':0.5,\n                            'patience':2,\n                            'threshold':0.0001,\n                            'threshold_mode':'rel',\n                            'cooldown':0,\n                            'min_lr':1e-5,\n                            'eps':1e-08,\n                            'verbose':True\n                        },\n                \n                }\n    \n    \n    # optimizer\n    optimizer = tf.keras.optimizers.Adam(lr = 1e-4)\n\n    # criterion\n    loss = 'binary_crossentropy'\n    \n    target_size_dim = 300\n    \n    metrics = tf.keras.metrics.AUC(multi_label=True)\n    \n    reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=2, verbose=1, mode='auto', epsilon=0.0001, \n                                       cooldown=5, min_lr=0.00001)\n    \n    checkpoint = ModelCheckpoint('best_model.hdf5', \n                             monitor= 'val_loss', \n                             verbose=1, \n                             save_best_only=True, \n                             mode= 'min', \n                             save_weights_only = False)\n\n    checkpoint_last = ModelCheckpoint('last_model.hdf5', \n                             monitor= 'val_loss', \n                             verbose=1, \n                             save_best_only=False, \n                             mode= 'min', \n                             save_weights_only = False)\n\n\n    early = EarlyStopping(monitor= 'val_loss', \n                      mode= 'min', \n                      patience=5)\n    \n    labels = [\n                'ETT - Abnormal',\n                'ETT - Borderline',\n                'ETT - Normal',\n                'NGT - Abnormal',\n                'NGT - Borderline',\n                'NGT - Incompletely Imaged',\n                'NGT - Normal', \n                'CVC - Abnormal',\n                'CVC - Borderline',\n                'CVC - Normal',\n                'Swan Ganz Catheter Present'\n            ]\n    \n    paths = {\n                'train_path':'../input/ranzcr-clip-trainset-256x256',\n                'test_path': '../input/siim-isic-melanoma-classification/jpeg/test',\n                'csv_path': '../input/ranzcr-clip-catheter-line-classification/train.csv',\n                'model_weight_path_folder': '../input/tfkerasefficientnetimagenetnotop'\n            }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config = Config","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(config.reduceLROnPlat)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(config.seed)\ntf.random.set_seed(config.seed)\nos.environ['PYTHONHASHSEED'] = str(config.seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h1 style=\"border:2px solid Purple;text-align:center\">Prepare Data</h1>"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(config.paths['csv_path'])\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"    Columns of the Dataframe    \n    \n    StudyInstanceUID --> unique ID for each image\n    ETT - Abnormal --> endotracheal tube placement abnormal\n    ETT - Borderline --> endotracheal tube placement borderline abnormal\n    ETT - Normal --> endotracheal tube placement normal\n    NGT - Abnormal --> nasogastric tube placement abnormal\n    NGT - Borderline --> nasogastric tube placement borderline abnormal\n    NGT - Incompletely Imaged --> nasogastric tube placement inconclusive due to imaging\n    NGT - Normal --> nasogastric tube placement borderline normal\n    CVC - Abnormal --> central venous catheter placement abnormal\n    CVC - Borderline --> central venous catheter placement borderline abnormal\n    CVC - Normal --> central venous catheter placement normal\n    Swan Ganz Catheter Present\n    PatientID --> unique ID for each patient in the dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Number of Records: {} and Number of Patients: {}'.format(len(df), df['PatientID'].nunique()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Distribution of patient in all the records"},{"metadata":{"trusted":true},"cell_type":"code","source":"df['PatientID'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['path'] = '../input/ranzcr-clip-catheter-line-classification/train/' + df['StudyInstanceUID']+'.jpg'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h1 style=\"border:2px solid Purple;text-align:center\">Data Loader</h1>"},{"metadata":{},"cell_type":"markdown","source":"Splitting the df for training and Validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_valid = train_test_split(df, test_size = 0.1, random_state=config.seed, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"tf.data.Datasetfrom_tensor_slices --> Creates a Dataset whose elements are slices of the given tensors.\n\nThe given tensors are sliced along their first dimension. This operation preserves the structure of the input tensors, removing the first dimension of each tensor and using it as the dataset dimension. All input tensors must have the same size in their first dimensions."},{"metadata":{"trusted":true},"cell_type":"code","source":"Train_df = tf.data.Dataset.from_tensor_slices((X_train.path.values, X_train[config.labels].values))\n\nValid_df = tf.data.Dataset.from_tensor_slices((X_valid.path.values, X_valid[config.labels].values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for path, label in Train_df.take(5):\n    print ('Path: {}, Label: {}'.format(path, label))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h1 style=\"border:2px solid Purple;text-align:center\">Data Generator</h1>"},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_data_train(image_path, label):\n    \n    img = tf.io.read_file(image_path)\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.random_brightness(img, 0.3)\n    img = tf.image.random_flip_left_right(img)\n    img = tf.image.resize(img, [config.target_size_dim, config.target_size_dim])\n    #img = tf.image.per_image_standardization(img)\n    \n    return img, label\n\ndef process_data_valid(image_path, label):\n    \n    img = tf.io.read_file(image_path)\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img, [config.target_size_dim, config.target_size_dim])\n    \n    return img, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Train_df = Train_df.map(process_data_train, num_parallel_calls=tf.data.experimental.AUTOTUNE)\nValid_df = Valid_df.map(process_data_valid, num_parallel_calls=tf.data.experimental.AUTOTUNE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for image, label in Train_df.take(1):\n    \n    plt.imshow(image.numpy().astype('uint8'))\n    plt.show()\n    print(\"Image shape: \", image.numpy().shape)\n    print(\"Label: \", config.labels[np.argmax(label.numpy())])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def configure_for_performance(ds, batch_size = config.batch_size):\n    \n    ds = ds.cache('/kaggle/dump.tfcache') \n    ds = ds.repeat()\n    ds = ds.shuffle(buffer_size=1024)\n    ds = ds.batch(batch_size)\n    ds = ds.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n    \n    return ds\n\ntrain_ds_batch = configure_for_performance(Train_df)\nvalid_ds_batch = Valid_df.batch(config.batch_size*2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_batch, label_batch = next(iter(train_ds_batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nfor i in range(16):\n    \n    ax = plt.subplot(4, 4, i + 1)\n    plt.imshow(image_batch[i].numpy().astype(\"uint8\"))\n    label = config.labels[np.argmax(label_batch[i].numpy())]\n    plt.title(label)\n    plt.axis(\"off\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_augmentation = keras.Sequential(\n    [\n        tf.keras.layers.experimental.preprocessing.RandomRotation(0.05, interpolation='nearest'),\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 10))\n\nfor i in range(16):\n    \n    augmented_images = data_augmentation(image_batch)\n    ax = plt.subplot(4, 4, i + 1)\n    plt.imshow(augmented_images[i].numpy().astype(\"uint8\"))\n    label = config.labels[np.argmax(label_batch[i].numpy())]\n    plt.title(label)\n    plt.axis(\"off\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h1 style=\"border:2px solid Purple;text-align:center\">Model Creation</h1>"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_pretrained_model(weights_path, drop_connect, target_size_dim, layers_to_unfreeze=5):\n    \n    model = EfficientNetB3(\n            weights=None, \n            include_top=False, \n            drop_connect_rate=0.4\n        )\n    \n    model.load_weights(weights_path)\n    \n    model.trainable = True\n\n    return model\n\ndef build_my_model(base_model, optimizer, metrics, loss):\n    \n    inputs = tf.keras.layers.Input(shape=(config.target_size_dim, config.target_size_dim, 3))\n    x = data_augmentation(inputs)\n    outputs_eff = base_model(x)\n    global_avg_pooling = GlobalAveragePooling2D()(outputs_eff)\n    dense_1= Dense(256)(global_avg_pooling)\n    bn_1 = BatchNormalization()(dense_1)\n    activation = Activation('relu')(bn_1)\n    dropout = Dropout(0.3)(activation)\n    dense_2 = Dense(len(config.labels), activation='sigmoid')(dropout)\n\n    my_model = tf.keras.Model(inputs, dense_2)\n    \n    my_model.compile(\n        optimizer=config.optimizer,\n        loss=config.loss,\n        metrics=config.metrics\n    )\n    \n    return my_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_weights_path = '../input/tfkerasefficientnetimagenetnotop/efficientnetb3_notop.h5'\nmodel_weights_path\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_model = load_pretrained_model(model_weights_path, config.drop_rate, config.target_size_dim)\n\nmy_model = build_my_model(base_model, config.optimizer, metrics = [config.metrics], loss=config.loss)\n\nmy_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks_list = [config.checkpoint, config.checkpoint_last, config.early, config.reduceLROnPlat]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h1 style=\"border:2px solid Purple;text-align:center\">Model Training</h1>"},{"metadata":{"trusted":true},"cell_type":"code","source":"steps_per_epoch = len(X_train) // config.batch_size\n\nhistory = my_model.fit(\n                          train_ds_batch, \n                          validation_data = valid_ds_batch, \n                          epochs = config.n_epochs, \n                          callbacks = callbacks_list,\n                          steps_per_epoch = steps_per_epoch\n                      )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h1 style=\"border:2px solid Purple;text-align:center\">Model Evaluation</h1>"},{"metadata":{"trusted":true},"cell_type":"code","source":"my_model.load_weights('best_model.hdf5') ## load the best model or all your metrics would be on the last run not on the best one","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_valid_y = my_model.predict(valid_ds_batch, verbose = True, workers=4)\npred_valid_y_labels = np.argmax(pred_valid_y, axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_labels = np.concatenate([y.numpy() for x, y in valid_ds_batch], axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_labels","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h1 style=\"border:2px solid Purple;text-align:center\">Test Predictions</h1>"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_images = glob.glob('../input/ranzcr-clip-catheter-line-classification/test/*.jpg')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.DataFrame(np.array(test_images), columns=['Path'])\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = tf.data.Dataset.from_tensor_slices((df_test.Path.values))\n\n\ndef process_test(image_path):\n\n    img = tf.io.read_file(image_path)\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img, [config.target_size_dim, config.target_size_dim])\n    \n    return img\n    \ntest_ds = test_ds.map(process_test, num_parallel_calls=tf.data.experimental.AUTOTUNE).batch(config.batch_size*2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_y = my_model.predict(test_ds, workers=4, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_ss = pd.DataFrame(pred_y, columns = config.labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test['image_id'] = df_test.Path.str.split('/').str[-1].str[:-4]\ndf_ss['StudyInstanceUID'] = df_test['image_id']\ndf_ss.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols_reordered = ['StudyInstanceUID', 'ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal', 'NGT - Abnormal',\n       'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal',\n       'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',\n       'Swan Ganz Catheter Present']\n\ndf_order = df_ss[cols_reordered]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h1 style=\"border:2px solid Purple;text-align:center\">Final Submission</h1>"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_order.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Please upvote the Notebook if you liked the content**"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}