{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#import libraries\nimport os\nimport matplotlib.pyplot as plt\nimport cv2\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport pydicom\nimport numpy as np\nimport shutil\nfrom PIL import Image\nimport scipy\nimport torch \nimport torchvision\nimport torchvision.transforms as transforms\nfrom torchvision import models , datasets\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport time\nimport copy\nimport os\nfrom fastai.vision.all import *\nprint(\"All modules have been imported\")","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:26:10.405931Z","iopub.execute_input":"2021-08-26T15:26:10.406321Z","iopub.status.idle":"2021-08-26T15:26:10.945426Z","shell.execute_reply.started":"2021-08-26T15:26:10.406232Z","shell.execute_reply":"2021-08-26T15:26:10.944524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#making the folder\n!mkdir \"data\"\n!mkdir \"data/0\"\n!mkdir \"data/1\"\nlabels = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:26:19.438491Z","iopub.execute_input":"2021-08-26T15:26:19.442976Z","iopub.status.idle":"2021-08-26T15:26:21.589168Z","shell.execute_reply.started":"2021-08-26T15:26:19.442920Z","shell.execute_reply":"2021-08-26T15:26:21.588256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"main_folder_path = \"../input/converteddata/png_data/png_voxel_converted_ds\"\nmain_train_folder_path = os.path.join(main_folder_path  , \"train\")\nfor subject in tqdm(os.listdir(main_train_folder_path)):\n    subject_folder = os.path.join(main_train_folder_path , subject)\n    for mri_type in os.listdir(subject_folder):\n        mri_type_folder = os.path.join(subject_folder , mri_type)\n        for mri_image in os.listdir(mri_type_folder):\n            original_image_path = os.path.join(mri_type_folder , mri_image)\n            mri_image = subject +\"_\"+ mri_type +\"_\"+ mri_image\n            subject_num = int(subject)\n            idx = np.where(labels['BraTS21ID'] == subject_num)[0][0]\n            label = str(labels.loc[idx , 'MGMT_value'])\n            new_image_folder_path =os.path.join(\"data\" , label)\n            new_image_path = os.path.join(new_image_folder_path , mri_image)\n            shutil.copy(original_image_path , new_image_path)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:30:34.896481Z","iopub.execute_input":"2021-08-26T15:30:34.896839Z","iopub.status.idle":"2021-08-26T15:33:19.000558Z","shell.execute_reply.started":"2021-08-26T15:30:34.896796Z","shell.execute_reply":"2021-08-26T15:33:18.999686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Images with label 0 = \" , len(os.listdir(\"data/0\")) , \"Images with label 1 = \" , len(os.listdir(\"data/1\")))","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:33:55.237151Z","iopub.execute_input":"2021-08-26T15:33:55.237500Z","iopub.status.idle":"2021-08-26T15:33:55.291989Z","shell.execute_reply.started":"2021-08-26T15:33:55.237469Z","shell.execute_reply":"2021-08-26T15:33:55.291088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for folder in os.listdir(\"data\"):\n    folder_name = str(folder)\n    path = \"data/\"+folder_name\n    for file in tqdm(os.listdir(path)):\n        img = Image.open(path + '/' + file)\n        clrs = img.getcolors()\n        if len(clrs) == 1:\n            os.remove(path + '/' + file)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:34:01.373929Z","iopub.execute_input":"2021-08-26T15:34:01.374450Z","iopub.status.idle":"2021-08-26T15:34:53.237158Z","shell.execute_reply.started":"2021-08-26T15:34:01.374405Z","shell.execute_reply":"2021-08-26T15:34:53.236195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Model and training classes**","metadata":{}},{"cell_type":"code","source":"print(\"Images with label 0 = \" , len(os.listdir(\"data/0\")) , \"Images with label 1 = \" , len(os.listdir(\"data/1\")))","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:35:07.661320Z","iopub.execute_input":"2021-08-26T15:35:07.661636Z","iopub.status.idle":"2021-08-26T15:35:07.707332Z","shell.execute_reply.started":"2021-08-26T15:35:07.661607Z","shell.execute_reply":"2021-08-26T15:35:07.706513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir \"data/TRAIN\"\n!mkdir \"data/TRAIN/1\"\n!mkdir \"data/TRAIN/0\"\n!mkdir \"data/VAL\"\n!mkdir \"data/VAL/0\"\n!mkdir \"data/VAL/1\"\n!mkdir \"data/TEST\"\n!mkdir \"data/TEST/0\"\n!mkdir \"data/TEST/1\"","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:35:10.959362Z","iopub.execute_input":"2021-08-26T15:35:10.959682Z","iopub.status.idle":"2021-08-26T15:35:16.671939Z","shell.execute_reply.started":"2021-08-26T15:35:10.959651Z","shell.execute_reply":"2021-08-26T15:35:16.670920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_PATH = \"./data\"\n\n#split the data into train/test/val\nfor CLASS in tqdm([\"0\" , \"1\"]):\n    IMG_NUM = len(os.listdir(IMG_PATH +\"/\"+ CLASS))\n    for (n, FILE_NAME) in enumerate(os.listdir(IMG_PATH +\"/\"+ CLASS)):\n            img = IMG_PATH+ '/' +  CLASS + '/' + FILE_NAME\n            if n <4000 :\n                shutil.copy(img, 'data/TEST/' + str(CLASS) + '/' + FILE_NAME)\n            elif n < 0.9*IMG_NUM:\n                shutil.copy(img, 'data/TRAIN/'+ str(CLASS) + '/' + FILE_NAME)\n            else:\n                shutil.copy(img, 'data/VAL/'+ str(CLASS) + '/' + FILE_NAME)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:35:19.441837Z","iopub.execute_input":"2021-08-26T15:35:19.443948Z","iopub.status.idle":"2021-08-26T15:35:24.814542Z","shell.execute_reply.started":"2021-08-26T15:35:19.443898Z","shell.execute_reply":"2021-08-26T15:35:24.813710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:35:27.659692Z","iopub.execute_input":"2021-08-26T15:35:27.660052Z","iopub.status.idle":"2021-08-26T15:35:28.875152Z","shell.execute_reply.started":"2021-08-26T15:35:27.660021Z","shell.execute_reply":"2021-08-26T15:35:28.874273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path=Path('./data')","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:35:31.235030Z","iopub.execute_input":"2021-08-26T15:35:31.235355Z","iopub.status.idle":"2021-08-26T15:35:31.240853Z","shell.execute_reply.started":"2021-08-26T15:35:31.235326Z","shell.execute_reply":"2021-08-26T15:35:31.238242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path.ls()","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:35:34.142841Z","iopub.execute_input":"2021-08-26T15:35:34.143178Z","iopub.status.idle":"2021-08-26T15:35:34.150912Z","shell.execute_reply.started":"2021-08-26T15:35:34.143143Z","shell.execute_reply":"2021-08-26T15:35:34.150055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fields = DataBlock(blocks=(ImageBlock, CategoryBlock),\n   get_items=get_image_files,\n   get_y=parent_label,\n   splitter=RandomSplitter(valid_pct=0.2, seed=42),\n   item_tfms=RandomResizedCrop(224, min_scale=0.5),\n   batch_tfms=aug_transforms())","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:36:00.780917Z","iopub.execute_input":"2021-08-26T15:36:00.781256Z","iopub.status.idle":"2021-08-26T15:36:00.789563Z","shell.execute_reply.started":"2021-08-26T15:36:00.781227Z","shell.execute_reply":"2021-08-26T15:36:00.788597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls=fields.dataloaders(path)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:36:04.402483Z","iopub.execute_input":"2021-08-26T15:36:04.402858Z","iopub.status.idle":"2021-08-26T15:36:19.959466Z","shell.execute_reply.started":"2021-08-26T15:36:04.402827Z","shell.execute_reply":"2021-08-26T15:36:19.958593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.vocab","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:36:22.956070Z","iopub.execute_input":"2021-08-26T15:36:22.956398Z","iopub.status.idle":"2021-08-26T15:36:22.961547Z","shell.execute_reply.started":"2021-08-26T15:36:22.956368Z","shell.execute_reply":"2021-08-26T15:36:22.960696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.train.show_batch(max_n=8,nrows=2)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:36:25.726173Z","iopub.execute_input":"2021-08-26T15:36:25.726508Z","iopub.status.idle":"2021-08-26T15:36:26.582285Z","shell.execute_reply.started":"2021-08-26T15:36:25.726466Z","shell.execute_reply":"2021-08-26T15:36:26.579012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = cnn_learner(dls, resnet34, metrics=[error_rate, accuracy], model_dir=\"/tmp/model/\").to_fp16()","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:36:36.120297Z","iopub.execute_input":"2021-08-26T15:36:36.120627Z","iopub.status.idle":"2021-08-26T15:36:39.217235Z","shell.execute_reply.started":"2021-08-26T15:36:36.120597Z","shell.execute_reply":"2021-08-26T15:36:39.216379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.lr_find()","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:36:42.313664Z","iopub.execute_input":"2021-08-26T15:36:42.314048Z","iopub.status.idle":"2021-08-26T15:37:04.439291Z","shell.execute_reply.started":"2021-08-26T15:36:42.313997Z","shell.execute_reply":"2021-08-26T15:37:04.438215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.fine_tune(10, base_lr=1e-2)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:37:19.575657Z","iopub.execute_input":"2021-08-26T15:37:19.576037Z","iopub.status.idle":"2021-08-26T17:05:11.770555Z","shell.execute_reply.started":"2021-08-26T15:37:19.575999Z","shell.execute_reply":"2021-08-26T17:05:11.769394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.fit_one_cycle(10, slice(1e-2),cbs=[ShowGraphCallback()])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# interp = ClassificationInterpretation.from_learner(learn)\n# losses,idxs = interp.top_losses()\n# len(dls.valid_ds)==len(losses)==len(idxs)\n# interp.plot_confusion_matrix()","metadata":{"execution":{"iopub.status.busy":"2021-08-26T17:05:34.701489Z","iopub.execute_input":"2021-08-26T17:05:34.701837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #calculating TP,FP,TN,TP\n# upp, low = interp.confusion_matrix()\n# tn, fp = upp[0], upp[1]\n# fn, tp = low[0], low[1]\n# print(tn, fp, fn, tp)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T17:09:56.554864Z","iopub.execute_input":"2021-08-26T17:09:56.555349Z","iopub.status.idle":"2021-08-26T17:09:56.559443Z","shell.execute_reply.started":"2021-08-26T17:09:56.555245Z","shell.execute_reply":"2021-08-26T17:09:56.558593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# interp.plot_top_losses(6)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T17:10:02.857629Z","iopub.execute_input":"2021-08-26T17:10:02.858102Z","iopub.status.idle":"2021-08-26T17:10:02.863702Z","shell.execute_reply.started":"2021-08-26T17:10:02.858058Z","shell.execute_reply":"2021-08-26T17:10:02.862410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Model Training**","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}