{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         pass\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-01T15:43:35.790824Z","iopub.execute_input":"2022-02-01T15:43:35.791157Z","iopub.status.idle":"2022-02-01T15:43:35.812906Z","shell.execute_reply.started":"2022-02-01T15:43:35.791072Z","shell.execute_reply":"2022-02-01T15:43:35.812255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Importing required libraries**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tensorflow import keras\nfrom tensorflow.keras.models import Sequential, Model\nimport cv2\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.layers import Dense, Flatten, Dropout, Conv2D, MaxPooling2D, BatchNormalization\nfrom tensorflow.keras.metrics import AUC\nfrom tensorflow.keras.optimizers import Adam\nfrom sklearn.model_selection import train_test_split\nimport os\nfrom matplotlib import pyplot as plt\nimport pydicom\nimport csv\nimport gc\nfrom tqdm import tqdm\nimport tensorflow as tf\nimport albumentations\nimport time","metadata":{"execution":{"iopub.status.busy":"2022-02-01T15:43:36.454398Z","iopub.execute_input":"2022-02-01T15:43:36.454955Z","iopub.status.idle":"2022-02-01T15:43:42.694132Z","shell.execute_reply.started":"2022-02-01T15:43:36.454919Z","shell.execute_reply":"2022-02-01T15:43:42.693392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Added a custom sequential model ","metadata":{}},{"cell_type":"code","source":"class CustomSequentialModel(Model):\n    def __init__(self, units=30, activation='relu', **kwargs):\n        super().__init__(**kwargs)\n        self.layer1 = Conv2D(32, input_shape=(64, 64, 1), activation=activation, kernel_size=(3,3))\n        self.layer2 = BatchNormalization()\n        self.layer3 = Flatten()\n        self.layer4 = Dense(32, activation=activation, kernel_initializer='he_normal')\n        self.layer5 = Dropout(0.15)\n        self.model_output = Dense(2, activation='sigmoid', kernel_initializer='glorot_uniform')\n        \n    def call(self, model_input):\n        op_layer = self.layer1(tf.dtypes.cast(model_input, tf.float32))\n        op_layer = self.layer2(op_layer)\n        op_layer = self.layer3(op_layer)\n        op_layer = self.layer4(op_layer)\n        op_layer = self.layer5(op_layer)\n        model_output = self.model_output(op_layer)\n        return model_output","metadata":{"execution":{"iopub.status.busy":"2022-02-01T15:43:42.695676Z","iopub.execute_input":"2022-02-01T15:43:42.695954Z","iopub.status.idle":"2022-02-01T15:43:42.705522Z","shell.execute_reply.started":"2022-02-01T15:43:42.69592Z","shell.execute_reply":"2022-02-01T15:43:42.704752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Organising the folder paths and removing unneccessary contents","metadata":{}},{"cell_type":"code","source":"base_folder = \"../input/rsna-miccai-brain-tumor-radiogenomic-classification\"\ntrain_data = pd.read_csv(os.path.join(base_folder, \"train_labels.csv\"))\nexcluded_patients = [\"00109\", \"00123\", \"00709\"]\nimage_types = [\"T1wCE\", \"FLAIR\", \"T1w\", \"T2w\"]\nimage_types = [\"T1w\"]","metadata":{"execution":{"iopub.status.busy":"2022-02-01T15:43:42.707046Z","iopub.execute_input":"2022-02-01T15:43:42.707333Z","iopub.status.idle":"2022-02-01T15:43:42.729114Z","shell.execute_reply.started":"2022-02-01T15:43:42.707301Z","shell.execute_reply":"2022-02-01T15:43:42.728509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"### Constructing a image dataframe with file_path and cancer value","metadata":{}},{"cell_type":"code","source":"def construct_image_df(test_type):\n    image_list = []\n    for patient_id in os.listdir(os.path.join(base_folder, test_type)):\n        if patient_id not in excluded_patients:\n            patient_record_value = train_data[train_data[\"BraTS21ID\"] == int(patient_id)]['MGMT_value'].item()\n            for image_type in image_types:\n                folder_dir = os.path.join(base_folder, test_type, patient_id, image_type)\n                for file_name in os.listdir(folder_dir):\n                    image_list.append({\"file_path\": os.path.join(folder_dir, file_name), \"value\": str(patient_record_value), \"patient_id\":patient_id, \"image_type\": image_type})\n    return pd.DataFrame(image_list)","metadata":{"execution":{"iopub.status.busy":"2022-02-01T15:43:42.730926Z","iopub.execute_input":"2022-02-01T15:43:42.731239Z","iopub.status.idle":"2022-02-01T15:43:42.738121Z","shell.execute_reply.started":"2022-02-01T15:43:42.731193Z","shell.execute_reply":"2022-02-01T15:43:42.737299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df = construct_image_df(\"train\")\ntrain_df, test_df = train_test_split(image_df, test_size=0.2, random_state=7)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-01T15:43:42.739606Z","iopub.execute_input":"2022-02-01T15:43:42.739874Z","iopub.status.idle":"2022-02-01T15:44:01.248674Z","shell.execute_reply.started":"2022-02-01T15:43:42.739841Z","shell.execute_reply":"2022-02-01T15:44:01.247991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Processed images by batch since it consumes the total RAM","metadata":{}},{"cell_type":"code","source":"class CustomDataGen(tf.keras.utils.Sequence):\n    def __init__(self, batch_size, df):\n        self.batch_size = batch_size\n        self.shuffle = True\n        self.df = df\n        self.n = len(self.df)\n    \n    def __len__(self):\n        l = int(len(self.df) / self.batch_size)\n        if l*self.batch_size < len(self.df):\n            l += 1\n        return l\n    \n    def __get_resized_image(self, image):\n        image_arr = cv2.resize(image.pixel_array, (64, 64))\n\n        return image_arr\n    \n    def __get_output(self, label, classes):\n        return to_categorical(label, num_classes=classes)\n    \n    def __get_cropped_image(self, image):\n        cropped_image = self.crop_pipeline(image=image.pixel_array)[\"image\"]\n        return cv2.resize(cropped_image, (64, 64))\n        \n    \n    def __get_data(self, batches):\n        X_batch, y_batch = [], []\n        for index, row in batches.iterrows():\n            image = pydicom.read_file(row['file_path'])\n            if (np.amax(image.pixel_array) != 0):\n                X_batch.append(self.__get_resized_image(image))\n                y_batch.append(self.__get_output(row['value'], 2))\n        return np.expand_dims(X_batch, axis=-1), np.array(y_batch)\n        \n    def __getitem__(self, index):\n        batches = self.df[index * self.batch_size: (index+1) * self.batch_size]\n        X,y = self.__get_data(batches)\n        return X,y","metadata":{"execution":{"iopub.status.busy":"2022-02-01T15:44:01.249769Z","iopub.execute_input":"2022-02-01T15:44:01.251355Z","iopub.status.idle":"2022-02-01T15:44:01.262612Z","shell.execute_reply.started":"2022-02-01T15:44:01.251315Z","shell.execute_reply":"2022-02-01T15:44:01.261861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datagen = CustomDataGen(batch_size=512, df=train_df)\nvalid_datagen = CustomDataGen(batch_size=512, df=test_df)","metadata":{"execution":{"iopub.status.busy":"2022-02-01T15:44:01.26403Z","iopub.execute_input":"2022-02-01T15:44:01.264328Z","iopub.status.idle":"2022-02-01T15:44:01.278778Z","shell.execute_reply.started":"2022-02-01T15:44:01.264289Z","shell.execute_reply":"2022-02-01T15:44:01.278054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Building a Deep Learning Network","metadata":{}},{"cell_type":"code","source":"# model = CustomSequentialModel()\nmodel = Sequential()\n\nmodel.add(Conv2D(16, input_shape=(64, 64, 1), activation='relu', kernel_size=(4,4)))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(4,4))\n\nmodel.add(Conv2D(16, activation='relu', kernel_size=(4,4)))\nmodel.add(MaxPooling2D(2, 2))\n\nmodel.add(Conv2D(16, activation='relu', kernel_size=(4,4)))\nmodel.add(MaxPooling2D(2,2))\nmodel.add(Conv2D(16, activation='relu', kernel_size=(1, 1)))\nmodel.add(Dense(8, activation='relu'))\nmodel.add(Flatten())\nmodel.add(Dense(16, activation='relu', kernel_initializer='he_normal'))\nmodel.add(Dropout(0.25))\n\nmodel.add(Dense(2, activation='sigmoid'))\n\nmodel.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=[AUC()])\n# model.build(input_shape=(None, 64, 64, 1))\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-02-01T15:44:01.280353Z","iopub.execute_input":"2022-02-01T15:44:01.280855Z","iopub.status.idle":"2022-02-01T15:44:03.359145Z","shell.execute_reply.started":"2022-02-01T15:44:01.280801Z","shell.execute_reply":"2022-02-01T15:44:03.358454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_datagen,validation_data=valid_datagen, epochs=10, steps_per_epoch=len(train_df)/ 512)","metadata":{"execution":{"iopub.status.busy":"2022-02-01T15:44:03.360542Z","iopub.execute_input":"2022-02-01T15:44:03.360778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Constructing a test dataframe","metadata":{}},{"cell_type":"code","source":"def construct_test_df(test_type=\"test\"):\n    image_list = []\n    for patient_id in os.listdir(os.path.join(base_folder, test_type)):\n        if patient_id not in excluded_patients:\n            for image_type in image_types:\n                folder_dir = os.path.join(base_folder, test_type, patient_id, image_type)\n                for file_name in os.listdir(folder_dir):\n                    image_list.append({\"file_path\": os.path.join(folder_dir, file_name), \"patient_id\":patient_id, \"image_type\": image_type})\n    return pd.DataFrame(image_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = construct_test_df()\ntest_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby(['image_type']).agg(['count'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Processing test images by batch","metadata":{}},{"cell_type":"code","source":"class TestDataGenerator(keras.utils.Sequence):\n    def __init__(self, df, batch_size):\n        self.df = df\n        self.batch_size = batch_size\n        \n    def __get_input_data(self, batches):\n        img_arr = []\n        for index, row in batches.iterrows():\n            image = pydicom.read_file(row['file_path'])\n            img_arr.append(cv2.resize(image.pixel_array, (64, 64)))\n        img_arr = np.expand_dims(img_arr, axis=-1)\n        return img_arr\n    \n    def __len__(self):\n        l = int(len(self.df) / self.batch_size)\n        if l*self.batch_size < len(self.df):\n            l += 1\n        return l\n    \n    def __getitem__(self, index):\n        batches = self.df[index * self.batch_size: (index+1) * self.batch_size]\n        X = self.__get_input_data(batches)\n        return X","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_datagen = TestDataGenerator(batch_size=256, df=test_df)\noutput = model.predict(test_datagen)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modified_output = np.amax(output, axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['MGMT_value'] = modified_output","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df = test_df.groupby('patient_id', as_index=False).agg({\"MGMT_value\": ['mean']}).reset_index()\nresult_df.columns = ['id', 'BraTS21ID', 'MGMT_value']\nresult_df['BraTS21ID'] = result_df['BraTS21ID'].astype('string')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df[[\"BraTS21ID\", \"MGMT_value\"]].head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df[[\"BraTS21ID\", \"MGMT_value\"]].tail()\nmod_result_df = result_df.copy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mod_result_df['MGMT_value'] = mod_result_df['MGMT_value'].round(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"### Submitting the result to kaggle for evaluation","metadata":{}},{"cell_type":"code","source":"mod_result_df[['BraTS21ID', 'MGMT_value']].to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mod_result_df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mod_result_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}