{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\npd.set_option('display.max_colwidth', -1) \nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib\nmatplotlib.rcParams['animation.html'] = 'jshtml'\nimport seaborn as sns\n\nimport tensorflow as tf\nimport tensorflow.keras as keras\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import *\nfrom kaggle_datasets import KaggleDatasets\n\nfrom sklearn.model_selection import train_test_split\nimport pydicom\nimport os, re, gc, glob, cv2\nimport imageio\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\n%matplotlib inline\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-09T10:28:52.904680Z","iopub.execute_input":"2021-08-09T10:28:52.905004Z","iopub.status.idle":"2021-08-09T10:28:52.917046Z","shell.execute_reply.started":"2021-08-09T10:28:52.904973Z","shell.execute_reply":"2021-08-09T10:28:52.916138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import wandb\n# print('W&B version: ', wandb.__version__)\n# from wandb.keras import WandbCallback\n\n# wandb.login()","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.905759Z","iopub.status.idle":"2021-08-09T10:27:50.906176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpus = tf.config.list_physical_devices('GPU')\nif gpus:\n try:\n # Currently, memory growth needs to be the same across GPUs\n for gpu in gpus:\n tf.config.experimental.set_memory_growth(gpu, True)\n logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n except RuntimeError as e:\n # Memory growth must be set before GPUs have been initialized\n print(e)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.907069Z","iopub.status.idle":"2021-08-09T10:27:50.907470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MRI(MAGNETIC RESONANCE IMAGING)\nA magnetic resonance imaging (MRI) scan is a common procedure around the world.
\nN.B. This excerpt is taken from https://m.blog.naver.com/PostView.naver?isHttpsRedirect=true&blogId=meryang63&logNo=221163428141\n\n\n#### MRI uses a strong magnetic field and radio waves to create detailed images of the organs and tissues within the body.\n\n\n","metadata":{}},{"cell_type":"markdown","source":"### Cross-sections and divisions
\n\n","metadata":{}},{"cell_type":"markdown","source":"## MRI IMAGING SEQUENCES\n### The most commons MRI imaging sequences are \n\n1. T1-weighted (T1w)\n2. T2-weighted (T2w)\n3. Fluid Attenuated Inversion Recovery (FLAIR)\n4. T1-weighted Gadolinium Post Contrast (T1wCE/T1Gd)","metadata":{}},{"cell_type":"markdown","source":"## Comparison of Different Sequences
\n","metadata":{}},{"cell_type":"markdown","source":"This table is from https://m.blog.naver.com/PostView.naver?isHttpsRedirect=true&blogId=meryang63&logNo=221163428141
\n\n
\n
\n
\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n  
\n
\n 		\n
\n T1
\n
\n 		\n
\n T2
\n
\n 		\n
\n FLAIR
\n
\n
\n
\n 		\n
- video before contrast. 
image T2which is for comparison with T2 image.
\n 		\n
- video after contrast, 
Brain lesions can be closely monitored
\n 		\n
-  T2video + dark CSF
-  lesions are brightly monitored  
- Unlike T2, CSF space can be observed    
-> Around ventricle, sulci  lesion easily can be observed
\n
\n
\n CSF
\n
\n 		\n
\n Dark
\n
\n 		\n
\n Bright
\n
\n 		\n
\n Dark
\n
\n
\n
\n White\n matter
\n
\n 		\n
\n Light
\n
\n 		\n
\n Darkgray
\n
\n 		\n
\n Darkgray
\n
\n
\n
\n Cortex
\n
\n 		\n
\n Gray
\n
\n 		\n
\n Light gray
\n
\n 		\n
\n Light gray
\n
\n
\n
\n Fat(B.M)
\n
\n 		\n
\n Bright
\n
\n 		\n
\n Light
\n
\n 		\n
\n Light
\n
\n
\n
\n Inflammation
\n (infection, demyelination)
\n
\n 		\n
\n Dark
\n
\n 		\n
\n Bright
\n
\n 		\n
\n Bright
\n
\n
\n
\n
\n
\n
\n
","metadata":{}},{"cell_type":"code","source":"root_dir = '../input/rsna-miccai-brain-tumor-radiogenomic-classification/'\ndf = pd.read_csv(root_dir+'train_labels.csv')\nsns.countplot(data=df, x='MGMT_value')","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.908371Z","iopub.status.idle":"2021-08-09T10:27:50.908748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## What is the 'MGMT_value'?
\n\nO[6]-methylguanine-DNA methyltransferase (MGMT) is a protein in cells, including tumour cells, that repairs damage to the cell’s DNA. For example, the damage caused by chemotherapy drugs to tumour cells. The more MGMT protein that the tumour produces, the less effective the chemotherapy drug is expected to be, as the protein will repair the damage to the tumour. Thus, determination of MGMT promoter methylation status in newly diagnosed GBM can influence treatment decision making.\n\nIn this dataset the MGMT promoter methylation status data is defined as a binary label (0: unmethylated, 1: methylated)","metadata":{}},{"cell_type":"code","source":"# Add the full paths for each id for different types of sequences to the csv \ndef full_ids(data):\n zeros = 5 - len(str(data))\n if zeros > 0:\n prefix = ''.join(['0' for i in range(zeros)])\n \n return prefix+str(data)\n \n\ndf['BraTS21ID_full'] = df['BraTS21ID'].apply(full_ids)\n\n# Add all the paths to the df for easy access\ndf['flair'] = df['BraTS21ID_full'].apply(lambda file_id : root_dir+'train/'+file_id+'/FLAIR/')\ndf['t1w'] = df['BraTS21ID_full'].apply(lambda file_id : root_dir+'train/'+file_id+'/T1w/')\ndf['t1wce'] = df['BraTS21ID_full'].apply(lambda file_id : root_dir+'train/'+file_id+'/T1wCE/')\ndf['t2w'] = df['BraTS21ID_full'].apply(lambda file_id : root_dir+'train/'+file_id+'/T2w/')\ndf","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.910054Z","iopub.status.idle":"2021-08-09T10:27:50.910476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DICOM\nDICOM is the international standard to communicate and manage medical images and data. Its mission is to ensure the interoperability of systems used to produce, store, share, display, send, query, process, retrieve and print medical images, as well as to manage related workflows.\n\n","metadata":{}},{"cell_type":"code","source":"data = pydicom.dcmread('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/00000/T1w/Image-24.dcm')\ndata","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.911398Z","iopub.status.idle":"2021-08-09T10:27:50.911765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_image(data):\n if np.max(data.pixel_array)==0:\n img = data.pixel_array\n else:\n img = data.pixel_array/np.max(data.pixel_array) \n return img\ndata = pydicom.dcmread('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/00000/T1w/Image-24.dcm')\nimg = get_image(data)\nplt.imshow(img, cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.912555Z","iopub.status.idle":"2021-08-09T10:27:50.912925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Display Sequence of MRIs as Animation
\nUsing Matplotlib to display the sequence of images as animation. This animations can be saved in video format.","metadata":{}},{"cell_type":"code","source":"def sorted_image_dirs(path: str):\n '''\n Sorts the list of image directories by image number in a path\n '''\n dirs = glob.glob(path+'*')\n dirs.sort(key=lambda x: int(x.split('/')[-1].split('-')[-1].split('.')[0]))\n \n return dirs\n\n\ndef get_all_images(path: str):\n '''\n Returns a list of images (non empty) from a given path\n '''\n image_dirs = sorted_image_dirs(path)\n images = []\n \n for dir in image_dirs:\n data = pydicom.dcmread(dir)\n img = get_image(data)\n \n # Exclude the blank images\n if np.max(img)!=0:\n images.append(img)\n else:\n pass\n \n return images\n \n \ndef show_animation(images: list):\n '''\n Displays an animation from the list of images.\n \n set: matplotlib.rcParams['animation.html'] = 'jshtml'\n \n '''\n fig = plt.figure(figsize=(6, 6))\n plt.axis('off')\n im = plt.imshow(images[0], cmap='gray')\n \n def animate_func(i):\n im.set_array(images[i])\n return [im]\n \n return matplotlib.animation.FuncAnimation(fig, animate_func, frames = len(images), interval = 20)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.913878Z","iopub.status.idle":"2021-08-09T10:27:50.914276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Show the FLAIR Sequence ","metadata":{}},{"cell_type":"code","source":"# flair_images = get_all_images(df['flair'][0])\n \n# print('No of images:', len(flair_images))\n# flair_animation = show_animation(flair_images)\n# flair_animation\n# # flair_animation.save('./a.mp4')","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.915218Z","iopub.status.idle":"2021-08-09T10:27:50.915621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Show the T2w Sequence ","metadata":{}},{"cell_type":"code","source":"# t2w_images = get_all_images(df['t2w'][0])\n \n# print('No of images:', len(t2w_images))\n# show_animation(t2w_images)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.916489Z","iopub.status.idle":"2021-08-09T10:27:50.916859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot of MRIs with Their respective Sequence, plane & MGMT_value","metadata":{}},{"cell_type":"code","source":"def get_image_plane(data):\n x1,y1,_,x2,y2,_ = [round(j) for j in data.ImageOrientationPatient]\n cords = [x1,y1,x2,y2]\n\n if cords == [1,0,0,0]:\n return 'coronal'\n if cords == [1,0,0,1]:\n return 'axial'\n if cords == [0,1,0,0]:\n return 'sagittal'","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.917640Z","iopub.status.idle":"2021-08-09T10:27:50.917985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(35,20))\n\nseq_types = ['flair', 't1w', 't1wce', 't2w']\n\nfor i in range(16):\n \n index = np.random.randint(low=0, high=len(df))\n seq_type = np.random.choice(seq_types)\n\n # path for the randomly selected image and sequence type\n seq_path = df[seq_type][index]\n\n # list of images in the seq_path\n tmp = sorted_image_dirs(seq_path)\n\n # Get the middle image dicom\n dicom = pydicom.dcmread(tmp[len(tmp)//2])\n \n patient_id, mgmt, plane = df['BraTS21ID_full'][index], df['MGMT_value'][index], get_image_plane(dicom)\n \n ax = fig.add_subplot(4,5,i+1)\n ax.imshow(get_image(dicom), cmap='gray')\n plt.title(f'ID: {patient_id}, MGMT_value: {mgmt}, Plane: {plane}, Seq_type: {seq_type}')","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.918809Z","iopub.status.idle":"2021-08-09T10:27:50.919209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Results 4 type of Sequences for the Same Patient are Not the Same!","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(35,20))\n\nindex = 0\n\nfor i in range(4):\n \n seq_type = seq_types[i]\n seq_path = df[seq_type][index]\n\n tmp = sorted_image_dirs(seq_path)\n\n # Get the middle image dicom\n dicom = pydicom.dcmread(tmp[len(tmp)//2])\n \n patient_id, mgmt, plane = df['BraTS21ID_full'][index], df['MGMT_value'][index], get_image_plane(dicom)\n \n ax = fig.add_subplot(1,4,i+1)\n ax.imshow(get_image(dicom), cmap='gray')\n plt.title(f'ID: {patient_id}, MGMT_value: {mgmt}, Plane: {plane}, Seq_type: {seq_type}')","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.920123Z","iopub.status.idle":"2021-08-09T10:27:50.920504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv\")\n# submission.to_csv(\"submission.csv\", index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.921257Z","iopub.status.idle":"2021-08-09T10:27:50.921599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load training csv file\ndf = pd.read_csv('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv')\n\ndef get_patient_id(patient_id):\n if patient_id < 10:\n return '0000'+str(patient_id)\n elif patient_id >= 10 and patient_id < 100:\n return '000'+str(patient_id)\n elif patient_id >= 100 and patient_id < 1000:\n return '00'+str(patient_id)\n else:\n return '0'+str(patient_id)\n\ndef get_path(row):\n patient_id = get_patient_id(row.BraTS21ID)\n return f'../input/rsna-miccai-png/train/{patient_id}/FLAIR/'\n\ndf['path'] = df.apply(lambda row: get_path(row), axis=1)\n\n# Removing two patient ids from the dataframe since there are not FLAIR directories for these ids. \ndf = df.loc[df.BraTS21ID!=109]\ndf = df.loc[df.BraTS21ID!=709]\ndf = df.reset_index(drop=True)\n\ndf.head()\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.922502Z","iopub.status.idle":"2021-08-09T10:27:50.922838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv\")\nsample","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.923588Z","iopub.status.idle":"2021-08-09T10:27:50.923979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"BraTS21ID is the patient's Id,\n\nClass 0 refers to people who do not have the methylation of the MGMT promoter.\n\nClass 1 seems to be someone who has the methylation of the MGMT promoter.","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv\")\ntrain","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.924717Z","iopub.status.idle":"2021-08-09T10:27:50.925100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Let's take a look at one dcm file.\nTo see what a dcm file looks like, firstly open one dcm file.","metadata":{}},{"cell_type":"code","source":"dataset = pydicom.filereader.dcmread('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/00000/FLAIR/Image-109.dcm')\nimg = dataset.pixel_array\n\nfig, ax = plt.subplots()\nax.imshow(img, cmap='gray')\nax.set_axis_off()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.925893Z","iopub.status.idle":"2021-08-09T10:27:50.926255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def makeimg(path):\n dataset = pydicom.filereader.dcmread(path)\n img = dataset.pixel_array\n return img","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.927068Z","iopub.status.idle":"2021-08-09T10:27:50.927447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Let's look at typical examples of images of FLAIR, T1w, T1wCE, and T2w.\n#### After issuing the path, use this function to see typical examples of FLAIR, T1w, T1wCE, and T2w images.\n\n#### Before that, I made sure whether the same number of dcm files in each folder or not.","metadata":{}},{"cell_type":"markdown","source":"## Create dcm folder path\nFirst, let's think about putting out the path of the dcm image.","metadata":{}},{"cell_type":"code","source":"train[\"imfolder\"] = ['{0:05d}'.format(s) for s in train[\"BraTS21ID\"]]\ntrain","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.928243Z","iopub.status.idle":"2021-08-09T10:27:50.928583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = \"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train\"\n\ntrain[\"path\"] = [os.path.join(train_path,s) for s in train[\"imfolder\"] ]\ntrain","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.929288Z","iopub.status.idle":"2021-08-09T10:27:50.929623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Scans = [\"FLAIR\",\"T1w\",\"T1wCE\",\"T2w\"]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.930349Z","iopub.status.idle":"2021-08-09T10:27:50.930737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Scans[1]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.931484Z","iopub.status.idle":"2021-08-09T10:27:50.931821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check the number of dcm files\n#### In the first place, check whether the number of dcm of these Scans is the same for each patient.","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.932565Z","iopub.status.idle":"2021-08-09T10:27:50.932903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.933831Z","iopub.status.idle":"2021-08-09T10:27:50.934304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### First of all, the FLAIR path on the 0th line of train can be expressed as follows.","metadata":{}},{"cell_type":"code","source":"os.path.join(train[\"path\"].iloc[0],\"FLAIR\")","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.935117Z","iopub.status.idle":"2021-08-09T10:27:50.935488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir(os.path.join(train[\"path\"].iloc[0],\"FLAIR\"))[:10]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.936268Z","iopub.status.idle":"2021-08-09T10:27:50.936647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sorted_nicely(l):\n \"\"\" Sort the given iterable in the way that humans expect.\"\"\"\n convert = lambda text: int(text) if text.isdigit() else text\n alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]\n print(alphanum_key)\n return sorted(l, key = alphanum_key)\npatient_ids = os.listdir('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train')\nIDX = np.random.choice(len(patient_ids))","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.937621Z","iopub.status.idle":"2021-08-09T10:27:50.937978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_id = patient_ids[IDX]\nflair_filenames = os.listdir(f'../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/{patient_id}/FLAIR/')\nflair_filenames = sorted_nicely(flair_filenames)\n\npatient_id = patient_ids[IDX]\nT1w_filenames = os.listdir(f'../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/{patient_id}/T1w/')\nT1w_filenames = sorted_nicely(T1w_filenames)\n\npatient_id = patient_ids[IDX]\nT1wCE_filenames = os.listdir(f'../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/{patient_id}/T1wCE/')\nT1wCE_filenames = sorted_nicely(T1w_filenames)\n\npatient_id = patient_ids[IDX]\nT2w_filenames = os.listdir(f'../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/{patient_id}/T2w/')\nT2w_filenames = sorted_nicely(T2w_filenames)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.938788Z","iopub.status.idle":"2021-08-09T10:27:50.939155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"flair_filenames[:5]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.940001Z","iopub.status.idle":"2021-08-09T10:27:50.940378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"T1w_filenames[:5]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.941163Z","iopub.status.idle":"2021-08-09T10:27:50.941523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir(os.path.join(train[\"path\"].iloc[0],\"FLAIR\")))","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.942359Z","iopub.status.idle":"2021-08-09T10:27:50.942719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for scan in Scans:\n train[scan +\"_count\"] = [ len(os.listdir(os.path.join(train[\"path\"].iloc[s],scan))) for s in tqdm(range(len(train))) ]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.943564Z","iopub.status.idle":"2021-08-09T10:27:50.943908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.944685Z","iopub.status.idle":"2021-08-09T10:27:50.945049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You can see that some IDs are the same and some are different.\n\nCheck if they are all the same","metadata":{}},{"cell_type":"code","source":"allsame = [train[\"FLAIR_count\"].iloc[s] == train[\"T1w_count\"].iloc[s] ==train[\"T1wCE_count\"].iloc[s] ==train[\"T2w_count\"].iloc[s] \n for s in range(len(train))]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.945841Z","iopub.status.idle":"2021-08-09T10:27:50.946216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"allsame\"] = allsame\ntrain","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.947038Z","iopub.status.idle":"2021-08-09T10:27:50.947392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"allsame\"].sum()","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.948429Z","iopub.status.idle":"2021-08-09T10:27:50.948827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"allsame\"].sum()/len(train) * 100","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.949645Z","iopub.status.idle":"2021-08-09T10:27:50.950043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Deepen your understanding by looking at the images of people who have the same number of images\n#### For an easy-to-understand image, let's try a person who has all the images.\n\n#### Moreover, let's look at the images of MGMT_value is 1 or 0 in each.","metadata":{}},{"cell_type":"code","source":"train[\"allsame\"]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.951057Z","iopub.status.idle":"2021-08-09T10:27:50.951415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train[\"allsame\"]]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.952285Z","iopub.status.idle":"2021-08-09T10:27:50.952645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"BraTS21ID\"]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.953503Z","iopub.status.idle":"2021-08-09T10:27:50.953885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The person with ID 100 on line 64 : MGMT_values 1(the person with the methylation of the MGMT promoter) and\n\nLet's take a look at all the images of the person with ID 102 : MGMT_values 0 (the person who does not have) on the 65th line.\n\nFirst, do the person on line 64, then make it a function to see the person on line 65.","metadata":{}},{"cell_type":"code","source":"row_id = 64","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.954998Z","iopub.status.idle":"2021-08-09T10:27:50.955409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"BraTS21ID\"].iloc[row_id]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.956409Z","iopub.status.idle":"2021-08-09T10:27:50.956784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_folder = train[\"path\"].iloc[row_id]\ntemp_folder","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.957726Z","iopub.status.idle":"2021-08-09T10:27:50.958137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_folder2 = os.path.join(temp_folder,\"FLAIR\")\ntemp_files = os.listdir(temp_folder2)\ntemp_files[:3]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.959010Z","iopub.status.idle":"2021-08-09T10:27:50.959415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imagenum = [s.split(\"-\")[1] for s in temp_files] # Image/4.dcm\nimagenum = [s.split(\".\")[0] for s in imagenum] # 4/dcm -> [0] index 니까 -> 4 \nimagenum[:3]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.960172Z","iopub.status.idle":"2021-08-09T10:27:50.960553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_path = [os.path.join(temp_folder2,s) for s in temp_files]\ntemp_path[:3]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.961500Z","iopub.status.idle":"2021-08-09T10:27:50.961903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tempdf = pd.DataFrame()\ntempdf[\"image_num\"] = imagenum\ntempdf[\"image_num\"] = tempdf[\"image_num\"].astype(\"int\")\n\ntempdf[\"temp_path\"] = temp_path\ntempdf.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.962818Z","iopub.status.idle":"2021-08-09T10:27:50.963237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tempdf = tempdf.sort_values(\"image_num\").reset_index(drop=True)\ntempdf.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.964091Z","iopub.status.idle":"2021-08-09T10:27:50.964452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"finpath = tempdf[\"temp_path\"]\nfinpath","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.965414Z","iopub.status.idle":"2021-08-09T10:27:50.965818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def makepath(row_ID,scan):\n \n temp_folder = train[\"path\"].iloc[row_ID]\n temp_folder2 = os.path.join(temp_folder,scan)\n temp_files = os.listdir(temp_folder2)\n imagenum = [s.split(\"-\")[1] for s in temp_files]\n imagenum = [s.split(\".\")[0] for s in imagenum]\n temp_path = [os.path.join(temp_folder2,s) for s in temp_files]\n tempdf = pd.DataFrame()\n tempdf[\"image_num\"] = imagenum\n tempdf[\"image_num\"] = tempdf[\"image_num\"].astype(\"int\")\n tempdf[\"temp_path\"] = temp_path\n tempdf = tempdf.sort_values(\"image_num\").reset_index(drop=True)\n finpath = tempdf[\"temp_path\"]\n return finpath","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.966807Z","iopub.status.idle":"2021-08-09T10:27:50.967178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Scans","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.967948Z","iopub.status.idle":"2021-08-09T10:27:50.968322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row_id=64\n\nsampledf = pd.DataFrame()\nfor scan in Scans:\n sampledf[scan + \"_path\"] = makepath(row_id,scan)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.969139Z","iopub.status.idle":"2021-08-09T10:27:50.969490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row_id=64\n\nsampledf2 = pd.DataFrame()\nfor scan in Scans:\n sampledf2[scan + \"_path\"] = makepath(row_id+1,scan)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.970334Z","iopub.status.idle":"2021-08-09T10:27:50.970709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampledf.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.971547Z","iopub.status.idle":"2021-08-09T10:27:50.971933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The one with MGMT_value = 1\n#### Visualize the person with gene methylation = people with favorable prognostic factors and strong predictors of responsiveness to chemotherapy","metadata":{}},{"cell_type":"code","source":"print(\"MGMT_value = \" + str(train[\"MGMT_value\"].iloc[row_id]))\n\n\nfor row in range(len(sampledf)):\n plt.figure(figsize=(80,10))\n for num,scan in enumerate(Scans):\n img = makeimg(sampledf[scan + \"_path\"].iloc[row])\n plt.subplot(4,25,num+1)\n #plt.axis(\"off\")\n plt.imshow(img)\n \n if row==0:\n plt.title(scan,fontsize=18)\n if num==0:\n plt.ylabel(\"row=\" + str(row),fontsize=18)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.972813Z","iopub.status.idle":"2021-08-09T10:27:50.973200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The one with MGMT_value = 0\n#### Visualize the person who does not have gene methylation = those who do not have favorable prognostic and strong predictors of responsiveness to chemotherapy","metadata":{}},{"cell_type":"code","source":"print(\"MGMT_value = \" + str(train[\"MGMT_value\"].iloc[row_id+1]))\n\n\nfor row in range(len(sampledf2)):\n plt.figure(figsize=(80,10))\n for num,scan in enumerate(Scans):\n img = makeimg(sampledf2[scan + \"_path\"].iloc[row])\n plt.subplot(4,25,num+1)\n #plt.axis(\"off\")\n plt.imshow(img)\n \n if row==0:\n plt.title(scan,fontsize=18)\n if num==0:\n plt.ylabel(\"row=\" + str(row),fontsize=18)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.974239Z","iopub.status.idle":"2021-08-09T10:27:50.974596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## When the number of files in each scan is not complete (this is about 90%)\n#### When the number of data is not the same for 4 types, are the numbers the same in chronological order, or are they just serial numbers? Let's look at the person with id = 0","metadata":{}},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.975344Z","iopub.status.idle":"2021-08-09T10:27:50.975750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row_id=0\n\nsampledf3 = pd.DataFrame()\nfor scan in Scans:\n sampledf3[scan + \"_path\"] = makepath(row_id,scan)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.976574Z","iopub.status.idle":"2021-08-09T10:27:50.976950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row_id2 = 0\n\nsampledf3 = pd.DataFrame()\nfor scan in Scans:\n sampledf3[scan + \"_path\"] = makepath(row_id2,scan)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.977806Z","iopub.status.idle":"2021-08-09T10:27:50.978227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampledf3","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.979189Z","iopub.status.idle":"2021-08-09T10:27:50.979563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampledf3[\"T1w_path\"].iloc[32]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.980405Z","iopub.status.idle":"2021-08-09T10:27:50.980796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I confirmed that the bottom is Nan. The numbers are sorted in ascending order, and the 32nd line is 33th, so the file names are serial numbers.","metadata":{}},{"cell_type":"code","source":"print(\"MGMT_value = \" + str(train[\"MGMT_value\"].iloc[row_id2]))\n\nfor row in range(33):\n plt.figure(figsize=(80,5))\n for num,scan in enumerate(Scans):\n img = makeimg(sampledf3[scan + \"_path\"].iloc[row])\n plt.subplot(4,33,num+1)\n #plt.axis(\"off\")\n plt.imshow(img)\n \n if row==0:\n plt.title(scan,fontsize=18)\n if num==0:\n plt.ylabel(\"row=\" + str(row),fontsize=18)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.981599Z","iopub.status.idle":"2021-08-09T10:27:50.981949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check how much the number of test data is the same.\n#### The code is the same as the basic train, so I will put it together.","metadata":{}},{"cell_type":"code","source":"sample[\"imfolder\"] = ['{0:05d}'.format(s) for s in sample[\"BraTS21ID\"]]\n\ntest_path = \"../input/rsna-miccai-brain-tumor-radiogenomic-classification/test\"\n\nsample[\"path\"] = [os.path.join(test_path,s) for s in sample[\"imfolder\"] ]\n\n\nallres = []\n\nfor scan in Scans:\n sample[scan +\"_count\"] = [ len(os.listdir(os.path.join(sample[\"path\"].iloc[s],scan))) for s in tqdm(range(len(sample))) ]","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.982772Z","iopub.status.idle":"2021-08-09T10:27:50.983162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.983986Z","iopub.status.idle":"2021-08-09T10:27:50.984382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"allsame = [sample[\"FLAIR_count\"].iloc[s] == sample[\"T1w_count\"].iloc[s] ==sample[\"T1wCE_count\"].iloc[s] ==sample[\"T2w_count\"].iloc[s] \n for s in range(len(sample))]\n\nsample[\"allsame\"] = allsame\n\nsample","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.985282Z","iopub.status.idle":"2021-08-09T10:27:50.985626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[\"allsame\"].sum()","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.986400Z","iopub.status.idle":"2021-08-09T10:27:50.986786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[\"allsame\"].sum()/len(sample) * 100","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:27:50.987625Z","iopub.status.idle":"2021-08-09T10:27:50.987976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_dir = '../input/rsna-miccai-brain-tumor-radiogenomic-classification/'\ndf = pd.read_csv(root_dir+'train_labels.csv')\nsns.countplot(data=df, x='MGMT_value')","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:33:16.668352Z","iopub.execute_input":"2021-08-09T10:33:16.668681Z","iopub.status.idle":"2021-08-09T10:33:16.801891Z","shell.execute_reply.started":"2021-08-09T10:33:16.668649Z","shell.execute_reply":"2021-08-09T10:33:16.800368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" pip install efficientnet_pytorch","metadata":{"execution":{"iopub.status.busy":"2021-08-09T10:38:10.398981Z","iopub.execute_input":"2021-08-09T10:38:10.399581Z","iopub.status.idle":"2021-08-09T10:38:21.324459Z","shell.execute_reply.started":"2021-08-09T10:38:10.399533Z","shell.execute_reply":"2021-08-09T10:38:21.323139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}