{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         pass\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-31T09:59:16.096269Z","iopub.execute_input":"2022-01-31T09:59:16.096837Z","iopub.status.idle":"2022-01-31T09:59:16.117273Z","shell.execute_reply.started":"2022-01-31T09:59:16.096733Z","shell.execute_reply":"2022-01-31T09:59:16.116557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Importing required libraries**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tensorflow import keras\nfrom tensorflow.keras.models import Sequential, Model\nimport cv2\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.layers import Dense, Flatten, Dropout, Conv2D, MaxPooling2D, BatchNormalization\nfrom tensorflow.keras.metrics import AUC\nfrom tensorflow.keras.optimizers import Adam\nfrom sklearn.model_selection import train_test_split\nimport os\nfrom matplotlib import pyplot as plt\nimport pydicom\nimport csv\nimport gc\nfrom tqdm import tqdm\nimport tensorflow as tf\nimport albumentations\nimport time","metadata":{"execution":{"iopub.status.busy":"2022-01-31T09:59:16.118777Z","iopub.execute_input":"2022-01-31T09:59:16.119107Z","iopub.status.idle":"2022-01-31T09:59:22.259586Z","shell.execute_reply.started":"2022-01-31T09:59:16.119071Z","shell.execute_reply":"2022-01-31T09:59:22.258842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Added a custom sequential model ","metadata":{}},{"cell_type":"code","source":"class CustomSequentialModel(Model):\n    def __init__(self, units=30, activation='relu', **kwargs):\n        super().__init__(**kwargs)\n        self.layer1 = Conv2D(32, input_shape=(64, 64, 1), activation=activation, kernel_size=(3,3))\n        self.layer2 = BatchNormalization()\n        self.layer3 = Flatten()\n        self.layer4 = Dense(32, activation=activation, kernel_initializer='he_normal')\n        self.layer5 = Dropout(0.15)\n        self.model_output = Dense(2, activation='sigmoid', kernel_initializer='glorot_uniform')\n        \n    def call(self, model_input):\n        op_layer = self.layer1(tf.dtypes.cast(model_input, tf.float32))\n        op_layer = self.layer2(op_layer)\n        op_layer = self.layer3(op_layer)\n        op_layer = self.layer4(op_layer)\n        op_layer = self.layer5(op_layer)\n        model_output = self.model_output(op_layer)\n        return model_output","metadata":{"execution":{"iopub.status.busy":"2022-01-31T09:59:22.260872Z","iopub.execute_input":"2022-01-31T09:59:22.261145Z","iopub.status.idle":"2022-01-31T09:59:22.272756Z","shell.execute_reply.started":"2022-01-31T09:59:22.261103Z","shell.execute_reply":"2022-01-31T09:59:22.271655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Organising the folder paths and removing unneccessary contents","metadata":{}},{"cell_type":"code","source":"base_folder = \"../input/rsna-miccai-brain-tumor-radiogenomic-classification\"\ntrain_data = pd.read_csv(os.path.join(base_folder, \"train_labels.csv\"))\nexcluded_patients = [\"00109\", \"00123\", \"00709\"]\nimage_types = [\"T1wCE\", \"FLAIR\", \"T1w\", \"T2w\"]\nimage_types = [\"T1w\"]","metadata":{"execution":{"iopub.status.busy":"2022-01-31T09:59:22.275912Z","iopub.execute_input":"2022-01-31T09:59:22.276704Z","iopub.status.idle":"2022-01-31T09:59:22.298194Z","shell.execute_reply.started":"2022-01-31T09:59:22.276663Z","shell.execute_reply":"2022-01-31T09:59:22.297459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"### Constructing a image dataframe with file_path and cancer value","metadata":{}},{"cell_type":"code","source":"def construct_image_df(test_type):\n    image_list = []\n    for patient_id in os.listdir(os.path.join(base_folder, test_type)):\n        if patient_id not in excluded_patients:\n            patient_record_value = train_data[train_data[\"BraTS21ID\"] == int(patient_id)]['MGMT_value'].item()\n            for image_type in image_types:\n                folder_dir = os.path.join(base_folder, test_type, patient_id, image_type)\n                for file_name in os.listdir(folder_dir):\n                    image_list.append({\"file_path\": os.path.join(folder_dir, file_name), \"value\": str(patient_record_value), \"patient_id\":patient_id, \"image_type\": image_type})\n    return pd.DataFrame(image_list)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T09:59:22.299399Z","iopub.execute_input":"2022-01-31T09:59:22.299885Z","iopub.status.idle":"2022-01-31T09:59:22.307449Z","shell.execute_reply.started":"2022-01-31T09:59:22.299845Z","shell.execute_reply":"2022-01-31T09:59:22.306697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df = construct_image_df(\"train\")\ntrain_df, test_df = train_test_split(image_df, test_size=0.2, random_state=7)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T09:59:22.308973Z","iopub.execute_input":"2022-01-31T09:59:22.309447Z","iopub.status.idle":"2022-01-31T09:59:44.745524Z","shell.execute_reply.started":"2022-01-31T09:59:22.309401Z","shell.execute_reply":"2022-01-31T09:59:44.744691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Processed images by batch since it consumes the total RAM","metadata":{}},{"cell_type":"code","source":"class CustomDataGen(tf.keras.utils.Sequence):\n    def __init__(self, batch_size, df):\n        self.batch_size = batch_size\n        self.shuffle = True\n        self.df = df\n        self.n = len(self.df)\n    \n    def __len__(self):\n        l = int(len(self.df) / self.batch_size)\n        if l*self.batch_size < len(self.df):\n            l += 1\n        return l\n    \n    def __get_resized_image(self, image):\n        image_arr = cv2.resize(image.pixel_array, (64, 64))\n\n        return image_arr\n    \n    def __get_output(self, label, classes):\n        return to_categorical(label, num_classes=classes)\n    \n    def __get_cropped_image(self, image):\n        cropped_image = self.crop_pipeline(image=image.pixel_array)[\"image\"]\n        return cv2.resize(cropped_image, (64, 64))\n        \n    \n    def __get_data(self, batches):\n        X_batch, y_batch = [], []\n        for index, row in batches.iterrows():\n            image = pydicom.read_file(row['file_path'])\n            if (np.amax(image.pixel_array) != 0):\n                X_batch.append(self.__get_resized_image(image))\n                y_batch.append(self.__get_output(row['value'], 2))\n        return np.expand_dims(X_batch, axis=-1), np.array(y_batch)\n        \n    def __getitem__(self, index):\n        batches = self.df[index * self.batch_size: (index+1) * self.batch_size]\n        X,y = self.__get_data(batches)\n        return X,y","metadata":{"execution":{"iopub.status.busy":"2022-01-31T09:59:44.746904Z","iopub.execute_input":"2022-01-31T09:59:44.747377Z","iopub.status.idle":"2022-01-31T09:59:44.75965Z","shell.execute_reply.started":"2022-01-31T09:59:44.74734Z","shell.execute_reply":"2022-01-31T09:59:44.758933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datagen = CustomDataGen(batch_size=512, df=train_df)\nvalid_datagen = CustomDataGen(batch_size=512, df=test_df)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T09:59:44.761093Z","iopub.execute_input":"2022-01-31T09:59:44.76159Z","iopub.status.idle":"2022-01-31T09:59:44.773905Z","shell.execute_reply.started":"2022-01-31T09:59:44.761522Z","shell.execute_reply":"2022-01-31T09:59:44.773171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Building a Deep Learning Network","metadata":{}},{"cell_type":"code","source":"# model = CustomSequentialModel()\nmodel = Sequential()\n\nmodel.add(Conv2D(16, input_shape=(64, 64, 1), activation='relu', kernel_size=(4,4)))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(4,4))\n\nmodel.add(Conv2D(16, activation='relu', kernel_size=(4,4)))\nmodel.add(MaxPooling2D(2, 2))\n\nmodel.add(Conv2D(16, activation='relu', kernel_size=(4,4)))\nmodel.add(MaxPooling2D(2,2))\nmodel.add(Conv2D(16, activation='relu', kernel_size=(1, 1)))\nmodel.add(Dense(8, activation='relu'))\nmodel.add(Flatten())\nmodel.add(Dense(16, activation='relu', kernel_initializer='he_normal'))\nmodel.add(Dropout(0.25))\n\nmodel.add(Dense(2, activation='sigmoid'))\n\nmodel.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=[AUC()])\n# model.build(input_shape=(None, 64, 64, 1))\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T09:59:44.775212Z","iopub.execute_input":"2022-01-31T09:59:44.775829Z","iopub.status.idle":"2022-01-31T09:59:46.877325Z","shell.execute_reply.started":"2022-01-31T09:59:44.775793Z","shell.execute_reply":"2022-01-31T09:59:46.87664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_datagen,validation_data=valid_datagen, epochs=10, steps_per_epoch=len(train_df)/ 512)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T09:59:46.881197Z","iopub.execute_input":"2022-01-31T09:59:46.881393Z","iopub.status.idle":"2022-01-31T11:24:15.685859Z","shell.execute_reply.started":"2022-01-31T09:59:46.88137Z","shell.execute_reply":"2022-01-31T11:24:15.683847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Constructing a test dataframe","metadata":{}},{"cell_type":"code","source":"def construct_test_df(test_type=\"test\"):\n    image_list = []\n    for patient_id in os.listdir(os.path.join(base_folder, test_type)):\n        if patient_id not in excluded_patients:\n            for image_type in image_types:\n                folder_dir = os.path.join(base_folder, test_type, patient_id, image_type)\n                for file_name in os.listdir(folder_dir):\n                    image_list.append({\"file_path\": os.path.join(folder_dir, file_name), \"patient_id\":patient_id, \"image_type\": image_type})\n    return pd.DataFrame(image_list)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T11:24:15.690912Z","iopub.execute_input":"2022-01-31T11:24:15.691155Z","iopub.status.idle":"2022-01-31T11:24:15.698447Z","shell.execute_reply.started":"2022-01-31T11:24:15.691126Z","shell.execute_reply":"2022-01-31T11:24:15.697709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = construct_test_df()\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T11:24:15.700169Z","iopub.execute_input":"2022-01-31T11:24:15.700879Z","iopub.status.idle":"2022-01-31T11:24:19.83153Z","shell.execute_reply.started":"2022-01-31T11:24:15.700828Z","shell.execute_reply":"2022-01-31T11:24:19.830825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby(['image_type']).agg(['count'])","metadata":{"execution":{"iopub.status.busy":"2022-01-31T11:24:19.833595Z","iopub.execute_input":"2022-01-31T11:24:19.834401Z","iopub.status.idle":"2022-01-31T11:24:19.863354Z","shell.execute_reply.started":"2022-01-31T11:24:19.834335Z","shell.execute_reply":"2022-01-31T11:24:19.862684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Processing test images by batch","metadata":{}},{"cell_type":"code","source":"class TestDataGenerator(keras.utils.Sequence):\n    def __init__(self, df, batch_size):\n        self.df = df\n        self.batch_size = batch_size\n        \n    def __get_input_data(self, batches):\n        img_arr = []\n        for index, row in batches.iterrows():\n            image = pydicom.read_file(row['file_path'])\n            img_arr.append(cv2.resize(image.pixel_array, (64, 64)))\n        img_arr = np.expand_dims(img_arr, axis=-1)\n        return img_arr\n    \n    def __len__(self):\n        l = int(len(self.df) / self.batch_size)\n        if l*self.batch_size < len(self.df):\n            l += 1\n        return l\n    \n    def __getitem__(self, index):\n        batches = self.df[index * self.batch_size: (index+1) * self.batch_size]\n        X = self.__get_input_data(batches)\n        return X","metadata":{"execution":{"iopub.status.busy":"2022-01-31T11:24:19.864438Z","iopub.execute_input":"2022-01-31T11:24:19.865182Z","iopub.status.idle":"2022-01-31T11:24:19.873853Z","shell.execute_reply.started":"2022-01-31T11:24:19.865144Z","shell.execute_reply":"2022-01-31T11:24:19.873148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_datagen = TestDataGenerator(batch_size=256, df=test_df)\noutput = model.predict(test_datagen)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T11:24:19.875071Z","iopub.execute_input":"2022-01-31T11:24:19.87545Z","iopub.status.idle":"2022-01-31T11:26:30.576313Z","shell.execute_reply.started":"2022-01-31T11:24:19.875415Z","shell.execute_reply":"2022-01-31T11:26:30.575525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modified_output = np.amax(output, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T11:26:30.579902Z","iopub.execute_input":"2022-01-31T11:26:30.580119Z","iopub.status.idle":"2022-01-31T11:26:30.585561Z","shell.execute_reply.started":"2022-01-31T11:26:30.580095Z","shell.execute_reply":"2022-01-31T11:26:30.584664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['MGMT_value'] = modified_output","metadata":{"execution":{"iopub.status.busy":"2022-01-31T11:26:30.586668Z","iopub.execute_input":"2022-01-31T11:26:30.587343Z","iopub.status.idle":"2022-01-31T11:26:30.59507Z","shell.execute_reply.started":"2022-01-31T11:26:30.587302Z","shell.execute_reply":"2022-01-31T11:26:30.594366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2022-01-31T11:26:30.596797Z","iopub.execute_input":"2022-01-31T11:26:30.597519Z","iopub.status.idle":"2022-01-31T11:26:30.617899Z","shell.execute_reply.started":"2022-01-31T11:26:30.597482Z","shell.execute_reply":"2022-01-31T11:26:30.617105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df = test_df.groupby('patient_id', as_index=False).agg({\"MGMT_value\": ['mean']}).reset_index()\nresult_df.columns = ['id', 'BraTS21ID', 'MGMT_value']\nresult_df['BraTS21ID'] = result_df['BraTS21ID'].astype('string')","metadata":{"execution":{"iopub.status.busy":"2022-01-31T11:26:30.620986Z","iopub.execute_input":"2022-01-31T11:26:30.621879Z","iopub.status.idle":"2022-01-31T11:26:30.642752Z","shell.execute_reply.started":"2022-01-31T11:26:30.621837Z","shell.execute_reply":"2022-01-31T11:26:30.642064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df[[\"BraTS21ID\", \"MGMT_value\"]].head()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T11:26:30.644043Z","iopub.execute_input":"2022-01-31T11:26:30.644489Z","iopub.status.idle":"2022-01-31T11:26:30.658376Z","shell.execute_reply.started":"2022-01-31T11:26:30.644452Z","shell.execute_reply":"2022-01-31T11:26:30.657708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df[[\"BraTS21ID\", \"MGMT_value\"]].tail()\nmod_result_df = result_df.copy()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T11:26:30.6597Z","iopub.execute_input":"2022-01-31T11:26:30.66018Z","iopub.status.idle":"2022-01-31T11:26:30.665662Z","shell.execute_reply.started":"2022-01-31T11:26:30.660145Z","shell.execute_reply":"2022-01-31T11:26:30.664856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mod_result_df['MGMT_value'] = mod_result_df['MGMT_value'].round(1)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T11:26:30.666689Z","iopub.execute_input":"2022-01-31T11:26:30.668492Z","iopub.status.idle":"2022-01-31T11:26:30.674535Z","shell.execute_reply.started":"2022-01-31T11:26:30.668436Z","shell.execute_reply":"2022-01-31T11:26:30.673825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"### Submitting the result to kaggle for evaluation","metadata":{}},{"cell_type":"code","source":"mod_result_df[['BraTS21ID', 'MGMT_value']].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T11:26:30.675938Z","iopub.execute_input":"2022-01-31T11:26:30.676507Z","iopub.status.idle":"2022-01-31T11:26:30.815292Z","shell.execute_reply.started":"2022-01-31T11:26:30.67647Z","shell.execute_reply":"2022-01-31T11:26:30.814526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mod_result_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-31T11:26:30.816392Z","iopub.execute_input":"2022-01-31T11:26:30.817906Z","iopub.status.idle":"2022-01-31T11:26:30.823493Z","shell.execute_reply.started":"2022-01-31T11:26:30.817859Z","shell.execute_reply":"2022-01-31T11:26:30.822783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mod_result_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T11:26:30.825016Z","iopub.execute_input":"2022-01-31T11:26:30.825485Z","iopub.status.idle":"2022-01-31T11:26:30.839224Z","shell.execute_reply.started":"2022-01-31T11:26:30.825449Z","shell.execute_reply":"2022-01-31T11:26:30.838445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}