{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports and Setup","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport gc\nimport glob\nimport imageio\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport tensorflow as tf\nprint('TF version: ', tf.__version__)\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import *\n\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-05-09T18:11:45.112892Z","iopub.execute_input":"2022-05-09T18:11:45.113397Z","iopub.status.idle":"2022-05-09T18:11:50.280474Z","shell.execute_reply.started":"2022-05-09T18:11:45.113295Z","shell.execute_reply":"2022-05-09T18:11:50.279613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Set up Weights and Biases","metadata":{}},{"cell_type":"code","source":"import wandb\nprint('W&B version: ', wandb.__version__)\nfrom wandb.keras import WandbCallback\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"wandb_key\")\nwandb.login(key = secret_value_0)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T18:11:50.281818Z","iopub.execute_input":"2022-05-09T18:11:50.282150Z","iopub.status.idle":"2022-05-09T18:11:53.319632Z","shell.execute_reply.started":"2022-05-09T18:11:50.282115Z","shell.execute_reply":"2022-05-09T18:11:53.318833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpus = tf.config.list_physical_devices('GPU')\nif gpus:\n  try:\n    # Currently, memory growth needs to be the same across GPUs\n    for gpu in gpus:\n      tf.config.experimental.set_memory_growth(gpu, True)\n    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n  except RuntimeError as e:\n    # Memory growth must be set before GPUs have been initialized\n    print(e)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T18:11:53.321670Z","iopub.execute_input":"2022-05-09T18:11:53.322016Z","iopub.status.idle":"2022-05-09T18:11:55.062877Z","shell.execute_reply.started":"2022-05-09T18:11:53.321977Z","shell.execute_reply":"2022-05-09T18:11:55.062005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare Dataset\n\nThere are four sub-directories per patient corresponding to different MRI Image Sequencing methods. We are using FLAIR.","metadata":{}},{"cell_type":"code","source":"# Load training csv file\ndf = pd.read_csv('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv')\n\ndef get_patient_id(patient_id):\n    if patient_id < 10:\n        return '0000'+str(patient_id)\n    elif patient_id >= 10 and patient_id < 100:\n        return '000'+str(patient_id)\n    elif patient_id >= 100 and patient_id < 1000:\n        return '00'+str(patient_id)\n    else:\n        return '0'+str(patient_id)\n\ndef get_path(row):\n    patient_id = get_patient_id(row.BraTS21ID)\n    return f'../input/rsna-miccai-png/train/{patient_id}/FLAIR/'\n\ndf['path'] = df.apply(lambda row: get_path(row), axis=1)\n\n# Removing two patient ids from the dataframe since there are not FLAIR directories for these ids. \ndf = df.loc[df.BraTS21ID!=109]\ndf = df.loc[df.BraTS21ID!=709]\ndf = df.reset_index(drop=True)\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-09T18:11:55.064471Z","iopub.execute_input":"2022-05-09T18:11:55.064841Z","iopub.status.idle":"2022-05-09T18:11:55.128782Z","shell.execute_reply.started":"2022-05-09T18:11:55.064784Z","shell.execute_reply":"2022-05-09T18:11:55.128010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Prepare train-test split. Note that there are only 585 patients so if you are doing video classification, K-fold training might be beneficial. ","metadata":{}},{"cell_type":"code","source":"train_df, valid_df = train_test_split(df, test_size=0.1, stratify=df.MGMT_value.values)\nprint(f'Size of train_df: {len(train_df)}; valid_df: {len(valid_df)}')","metadata":{"execution":{"iopub.status.busy":"2022-05-09T18:11:55.129989Z","iopub.execute_input":"2022-05-09T18:11:55.130334Z","iopub.status.idle":"2022-05-09T18:11:55.140044Z","shell.execute_reply.started":"2022-05-09T18:11:55.130275Z","shell.execute_reply":"2022-05-09T18:11:55.138941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = dict(\n    NUM_FRAMES = 10,\n    BATCH_SIZE = 8,\n    EPOCHS = 100,\n    IMG_SIZE = 224,\n    LSTM_UNITS = 256,\n    competition = 'rsna-miccai-brain',\n    _wandb_kernel = 'ayut'\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T18:11:55.141959Z","iopub.execute_input":"2022-05-09T18:11:55.142587Z","iopub.status.idle":"2022-05-09T18:11:55.147606Z","shell.execute_reply.started":"2022-05-09T18:11:55.142547Z","shell.execute_reply":"2022-05-09T18:11:55.146658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef sorted_nicely(l): \n    \"\"\" Sort the given iterable in the way that humans expect.\"\"\" \n    convert = lambda text: int(text) if text.isdigit() else text \n    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ] \n    return sorted(l, key = alphanum_key)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T18:11:55.149216Z","iopub.execute_input":"2022-05-09T18:11:55.149994Z","iopub.status.idle":"2022-05-09T18:11:55.156921Z","shell.execute_reply.started":"2022-05-09T18:11:55.149955Z","shell.execute_reply":"2022-05-09T18:11:55.155999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image):\n    # convert the compressed string to a 3D uint8 tensor\n    image = tf.image.decode_png(image, channels=1)\n    # Normalize image\n    image = tf.image.convert_image_dtype(image, dtype=tf.float32)\n    \n    return image\n\ndef parse_frames(dirname):\n    # get MRI images file paths for given patient \n    paths = glob.glob(dirname.decode('utf8')+'/*.png')\n    # Sort the images to get sequential imaging\n    paths = sorted_nicely(paths)\n    \n    # randomly select a window of images to be used as sequence\n    start = tf.random.uniform((1,), maxval=len(paths)-CONFIG['NUM_FRAMES'], dtype=tf.int32)\n\n    paths = tf.slice(paths, start, [CONFIG['NUM_FRAMES']])\n    \n    def get_frames(path):\n        # Load image\n        image = tf.io.read_file(path)\n        image = decode_image(image)\n        # Resize image\n        image = tf.image.resize(image, (CONFIG['IMG_SIZE'], CONFIG['IMG_SIZE']))\n        \n        return image\n\n    mri_images = tf.nest.map_structure(tf.stop_gradient, tf.map_fn(fn=get_frames, elems=paths, fn_output_signature=tf.float32))\n    \n    return mri_images\n    \ndef load_frame(df_dict):\n    dirname = df_dict['path']\n    paths = tf.numpy_function(parse_frames, [dirname], tf.float32)\n    \n    # Parse label\n    label = df_dict['MGMT_value']\n    label = tf.cast(label, tf.float32)\n    \n    return paths, label","metadata":{"execution":{"iopub.status.busy":"2022-05-09T18:11:55.160253Z","iopub.execute_input":"2022-05-09T18:11:55.160663Z","iopub.status.idle":"2022-05-09T18:11:55.173179Z","shell.execute_reply.started":"2022-05-09T18:11:55.160625Z","shell.execute_reply":"2022-05-09T18:11:55.172225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\n\ntrainloader = tf.data.Dataset.from_tensor_slices(dict(train_df))\nvalidloader = tf.data.Dataset.from_tensor_slices(dict(valid_df))\n\n\ntrainloader = (\n    trainloader\n    .shuffle(1024)\n    .map(load_frame, num_parallel_calls=AUTOTUNE)\n    .batch(CONFIG['BATCH_SIZE'])\n    .prefetch(AUTOTUNE)\n)\n\nvalidloader = (\n    validloader\n    .map(load_frame, num_parallel_calls=AUTOTUNE)\n    .batch(CONFIG['BATCH_SIZE'])\n    .prefetch(AUTOTUNE)\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T18:11:55.175083Z","iopub.execute_input":"2022-05-09T18:11:55.175524Z","iopub.status.idle":"2022-05-09T18:11:55.299539Z","shell.execute_reply.started":"2022-05-09T18:11:55.175485Z","shell.execute_reply":"2022-05-09T18:11:55.298793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test out the trainloader\nframes, labels = next(iter(trainloader))","metadata":{"execution":{"iopub.status.busy":"2022-05-09T18:11:55.300662Z","iopub.execute_input":"2022-05-09T18:11:55.300984Z","iopub.status.idle":"2022-05-09T18:11:56.142980Z","shell.execute_reply.started":"2022-05-09T18:11:55.300951Z","shell.execute_reply":"2022-05-09T18:11:56.142109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In order to visualize the samples from our trainloader, I am using W&B. I find it easier to log everything onto W&B to visualize data than to write Matplotlib code. ","metadata":{}},{"cell_type":"code","source":"run = wandb.init(project='brain-tumor-video', job_type='dataloader-viz')\n\nos.makedirs('gifs/')\nfor i, frame in enumerate(frames):\n    imageio.mimsave(f'gifs/out_{i}.gif', (frame*255).numpy().astype('uint8'))    \n\nwandb.log({'examples': [wandb.Image(f'gifs/out_{i}.gif', caption=f'{label.numpy()}') for i, label in enumerate(labels)]})\n    \nrun.finish()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-09T18:11:56.144321Z","iopub.execute_input":"2022-05-09T18:11:56.144647Z","iopub.status.idle":"2022-05-09T18:12:13.691816Z","shell.execute_reply.started":"2022-05-09T18:11:56.144612Z","shell.execute_reply":"2022-05-09T18:12:13.690779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  Model\n\nIn order to model both spatial and temporal nature of videos, we can use a hybrid of CNN + LSTM model. \n\n* The `FeatureExtractor` model uses an EfficientNetB0 model as CNN backbone. It will be used to model the spatial aspect of videos. <br>\n* The `MRIModel` uses a `TimeDistributed` layer that runs the `FeatureExtractor` `NUM_FRAMES` times to get a vector of `(NUM_FRAMES, 1280)`. <br>\n* This is then fed to a single LSTM layer. You can use GRU and even Transformer in place of LSTM. I have used 256 units as it gave me the best results. ","metadata":{}},{"cell_type":"code","source":"def FeatureExtractor():\n    base_model = tf.keras.applications.EfficientNetB0(include_top=False, weights='imagenet')\n    base_model.trainabe = True\n\n    inputs = Input((CONFIG['IMG_SIZE'], CONFIG['IMG_SIZE'], 1))\n    x = Conv2D(3, kernel_size=(3, 3), padding='same', activation='relu')(inputs)\n    x = base_model(x, training=True)\n    flattened_output = GlobalAveragePooling2D()(x)\n    \n    return Model(inputs, flattened_output)\n\ntf.keras.backend.clear_session()\nmodel = FeatureExtractor()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-05-09T18:12:13.693258Z","iopub.execute_input":"2022-05-09T18:12:13.693642Z","iopub.status.idle":"2022-05-09T18:12:16.455451Z","shell.execute_reply.started":"2022-05-09T18:12:13.693601Z","shell.execute_reply":"2022-05-09T18:12:16.454557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def MRIModel():\n    inputs = Input((CONFIG['NUM_FRAMES'], CONFIG['IMG_SIZE'], CONFIG['IMG_SIZE'], 1))\n    feature_extractor = FeatureExtractor()\n    \n    time_wrapper = TimeDistributed(feature_extractor)(inputs)\n    \n    lstm_out = LSTM(CONFIG['LSTM_UNITS'], return_sequences=True, name=\"lstm\")(time_wrapper)\n    outputs = Dense(1, activation='sigmoid', name=\"lstm_sigmoid\")(lstm_out)\n    \n    return Model(inputs, outputs)\n\ntf.keras.backend.clear_session() \nmodel = MRIModel()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-05-09T18:12:16.456718Z","iopub.execute_input":"2022-05-09T18:12:16.457231Z","iopub.status.idle":"2022-05-09T18:12:20.071314Z","shell.execute_reply.started":"2022-05-09T18:12:16.457189Z","shell.execute_reply":"2022-05-09T18:12:20.070481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🚅 Train\n\nThis is a simple training pipeline that uses early stopping as regularizer and `WandbCallback` to log the metrics to Weights and Biases.","metadata":{}},{"cell_type":"code","source":"# Callbacks\nearlystopper = tf.keras.callbacks.EarlyStopping(\n    monitor='val_loss', patience=5, verbose=0, mode='min',\n    restore_best_weights=True\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T18:12:20.073158Z","iopub.execute_input":"2022-05-09T18:12:20.073752Z","iopub.status.idle":"2022-05-09T18:12:20.078319Z","shell.execute_reply.started":"2022-05-09T18:12:20.073710Z","shell.execute_reply":"2022-05-09T18:12:20.077309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session() \nmodel = MRIModel()\nmodel.compile('adam', 'binary_crossentropy', metrics=['acc'])\n\nrun = wandb.init(project='brain-tumor-video', \n                 group='EffnetB0-LSTM-256', \n                 job_type='train', \n                 config=CONFIG)\n\n# Train\n_ = model.fit(trainloader, \n              epochs=CONFIG['EPOCHS'],\n              validation_data=validloader,\n              callbacks=[WandbCallback(),\n                         earlystopper])\n\n# Evaluate\nloss, acc = model.evaluate(validloader)\nwandb.log({'Val Accuracy': round(acc, 3)})\n\nrun.finish()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-09T18:12:20.079620Z","iopub.execute_input":"2022-05-09T18:12:20.080026Z","iopub.status.idle":"2022-05-09T18:13:26.767018Z","shell.execute_reply.started":"2022-05-09T18:12:20.079989Z","shell.execute_reply":"2022-05-09T18:13:26.762994Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = dict(\n    NUM_FRAMES = 10,\n    BATCH_SIZE = 8,\n    EPOCHS = 100,\n    IMG_SIZE = 224,\n    LSTM_UNITS = 128,\n    competition = 'rsna-miccai-brain',\n    _wandb_kernel = 'ayut'\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-09T18:13:58.611948Z","iopub.execute_input":"2022-05-09T18:13:58.612390Z","iopub.status.idle":"2022-05-09T18:14:00.093488Z","shell.execute_reply.started":"2022-05-09T18:13:58.612346Z","shell.execute_reply":"2022-05-09T18:14:00.092540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def MRIModel():\n    inputs = Input((CONFIG['NUM_FRAMES'], CONFIG['IMG_SIZE'], CONFIG['IMG_SIZE'], 1))\n    feature_extractor = FeatureExtractor()\n    \n    time_wrapper = TimeDistributed(feature_extractor)(inputs)\n    \n    lstm_out = LSTM(CONFIG['LSTM_UNITS'], return_sequences=True, name=\"lstm\")(time_wrapper)\n    outputs = Dense(1, activation='sigmoid', name=\"lstm_sigmoid\")(lstm_out)\n    \n    return Model(inputs, outputs)\n\ntf.keras.backend.clear_session() \nmodel = MRIModel()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-05-09T18:14:00.094855Z","iopub.execute_input":"2022-05-09T18:14:00.095323Z","iopub.status.idle":"2022-05-09T18:14:05.778651Z","shell.execute_reply.started":"2022-05-09T18:14:00.095268Z","shell.execute_reply":"2022-05-09T18:14:05.777760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Callbacks\nearlystopper = tf.keras.callbacks.EarlyStopping(\n    monitor='val_loss', patience=5, verbose=0, mode='min',\n    restore_best_weights=True\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session() \nmodel = MRIModel()\nmodel.compile('adam', 'binary_crossentropy', metrics=['acc'])\n\nrun = wandb.init(project='brain-tumor-video', \n                 group='EffnetB0-LSTM-128', \n                 job_type='train', \n                 config=CONFIG)\n\n# Train\n_ = model.fit(trainloader, \n              epochs=CONFIG['EPOCHS'],\n              validation_data=validloader,\n              callbacks=[WandbCallback(),\n                         earlystopper])\n\n# Evaluate\nloss, acc = model.evaluate(validloader)\nwandb.log({'Val Accuracy': round(acc, 3)})\n\nrun.finish()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = dict(\n    NUM_FRAMES = 10,\n    BATCH_SIZE = 8,\n    EPOCHS = 100,\n    IMG_SIZE = 224,\n    LSTM_UNITS = 512,\n    competition = 'rsna-miccai-brain',\n    _wandb_kernel = 'ayut'\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def MRIModel():\n    inputs = Input((CONFIG['NUM_FRAMES'], CONFIG['IMG_SIZE'], CONFIG['IMG_SIZE'], 1))\n    feature_extractor = FeatureExtractor()\n    \n    time_wrapper = TimeDistributed(feature_extractor)(inputs)\n    \n    lstm_out = LSTM(CONFIG['LSTM_UNITS'], return_sequences=True, name=\"lstm\")(time_wrapper)\n    outputs = Dense(1, activation='sigmoid', name=\"lstm_sigmoid\")(lstm_out)\n    \n    return Model(inputs, outputs)\n\ntf.keras.backend.clear_session() \nmodel = MRIModel()\nmodel.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session() \nmodel = MRIModel()\nmodel.compile('adam', 'binary_crossentropy', metrics=['acc'])\n\nrun = wandb.init(project='brain-tumor-video', \n                 group='EffnetB0-LSTM-512', \n                 job_type='train', \n                 config=CONFIG)\n\n# Train\n_ = model.fit(trainloader, \n              epochs=CONFIG['EPOCHS'],\n              validation_data=validloader,\n              callbacks=[WandbCallback(),\n                         earlystopper])\n\n# Evaluate\nloss, acc = model.evaluate(validloader)\nwandb.log({'Val Accuracy': round(acc, 3)})\n\nrun.finish()","metadata":{},"execution_count":null,"outputs":[]}]}