{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from fastai.vision.all import *\nimport sys\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2021-09-25T06:28:25.775387Z","iopub.execute_input":"2021-09-25T06:28:25.776473Z","iopub.status.idle":"2021-09-25T06:28:28.374841Z","shell.execute_reply.started":"2021-09-25T06:28:25.776372Z","shell.execute_reply":"2021-09-25T06:28:28.374017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT = '../input/rsna-miccai-brain-tumor-radiogenomic-classification/'\n#IMAGE_OUTPUT = '../input/rsnamiccai-competition-2021-png-strips/'\nIMAGE_OUTPUT = '.'","metadata":{"execution":{"iopub.status.busy":"2021-09-25T06:28:39.46155Z","iopub.execute_input":"2021-09-25T06:28:39.462339Z","iopub.status.idle":"2021-09-25T06:28:39.465684Z","shell.execute_reply.started":"2021-09-25T06:28:39.4623Z","shell.execute_reply":"2021-09-25T06:28:39.465135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport random\ndf = pd.read_csv(os.path.join(ROOT, 'train_labels.csv'), header=0, names=['id','value'], dtype=object)\ndf = df[~df.id.isin([\"00109\", \"00123\", \"00709\"])]","metadata":{"execution":{"iopub.status.busy":"2021-09-25T06:28:58.490786Z","iopub.execute_input":"2021-09-25T06:28:58.491583Z","iopub.status.idle":"2021-09-25T06:28:58.523261Z","shell.execute_reply.started":"2021-09-25T06:28:58.491546Z","shell.execute_reply":"2021-09-25T06:28:58.522373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-09-25T06:29:09.290093Z","iopub.execute_input":"2021-09-25T06:29:09.290391Z","iopub.status.idle":"2021-09-25T06:29:09.308012Z","shell.execute_reply.started":"2021-09-25T06:29:09.290359Z","shell.execute_reply":"2021-09-25T06:29:09.307105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def natural_sort(l): \n    convert = lambda text: int(text) if text.isdigit() else text.lower()\n    alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]\n    return sorted(l, key=alphanum_key)","metadata":{"execution":{"iopub.status.busy":"2021-09-25T06:29:23.871724Z","iopub.execute_input":"2021-09-25T06:29:23.872037Z","iopub.status.idle":"2021-09-25T06:29:23.878504Z","shell.execute_reply.started":"2021-09-25T06:29:23.872008Z","shell.execute_reply":"2021-09-25T06:29:23.877367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.maximum((1,5),(3,4))","metadata":{"execution":{"iopub.status.busy":"2021-09-25T06:29:40.964132Z","iopub.execute_input":"2021-09-25T06:29:40.964417Z","iopub.status.idle":"2021-09-25T06:29:40.970873Z","shell.execute_reply.started":"2021-09-25T06:29:40.964391Z","shell.execute_reply":"2021-09-25T06:29:40.96988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.DataFrame(columns=['id', 'value'])\ndf_test.id = os.listdir(os.path.join(ROOT, 'test'))","metadata":{"execution":{"iopub.status.busy":"2021-09-25T06:29:55.123615Z","iopub.execute_input":"2021-09-25T06:29:55.123908Z","iopub.status.idle":"2021-09-25T06:29:55.144282Z","shell.execute_reply.started":"2021-09-25T06:29:55.123878Z","shell.execute_reply":"2021-09-25T06:29:55.1435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pydicom\nimport pandas as pd\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom tqdm import tqdm\nimport binascii\nfrom PIL import Image\nfrom multiprocessing import Pool\n\n\nif not os.path.exists('./train'):\n    os.makedirs('./train')\n    \n\nif not os.path.exists('./test'):\n    os.makedirs('./test')\n\n\ndef get_dicom_files(dataset='train'):\n    root = f\"{ROOT}/{dataset}\"\n    \n    ids = list(df.id) if dataset=='train' else list(df_test.id)\n    \n    with Pool(10) as p:\n        p.starmap(get_dicom_files_helper, [(dataset, root, cur_id) for cur_id in ids])\n    \n\ndef get_dicom_files_helper(dataset, root, cur_id):\n    final_image = np.array([])\n    for scan_type in ['FLAIR', 'T1w', 'T1wCE', 'T2w']:\n        cur_dir = os.path.join(root, cur_id, scan_type)\n        dicoms = natural_sort(os.listdir(cur_dir))\n        median = len(dicoms)//2\n        modulo = max(len(dicoms)//20, 1)\n        dicoms = [dicom for i, dicom in enumerate(dicoms) if i % modulo == 0]\n        for dicom in dicoms:\n            filepath = os.path.join(cur_dir, dicom)\n            data = process_dicom(filepath)\n            if len(final_image) != 0:\n                (data_rows, data_cols) = data.shape\n                (final_rows, final_cols) = final_image.shape\n                if final_cols != data_cols:\n                    if data_cols < final_cols:\n                        data = np.hstack((data, np.zeros((data_rows, final_cols - data_cols))))\n                    else:\n                        final_image = np.hstack((final_image, np.zeros((final_rows, data_cols - final_cols))))\n            final_image = np.concatenate([final_image,data]) if len(final_image) != 0 else data\n            data = None\n\n    outpath = os.path.join(f'./{dataset}',f'{cur_id}.png')\n    write_pixels(final_image, outpath)\n    final_image = None\n    print('Processed {}'.format(cur_id))\n        \n\ndef process_dicom(path):\n    dicom = pydicom.read_file(path)\n    data = apply_voi_lut(dicom.pixel_array, dicom)\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    \n    return data\n\n    \ndef write_pixels(data, outpath):\n    height = len(data)\n    width = len(data[0])\n    pixels_out = []\n    for row in data:\n        pixels_out.extend(row)\n    assert(len(pixels_out) == height * width)\n    \n    image_out = Image.new('L', (width, height))\n    image_out.putdata(pixels_out)\n    image_out.save(outpath)","metadata":{"execution":{"iopub.status.busy":"2021-09-25T06:30:24.163545Z","iopub.execute_input":"2021-09-25T06:30:24.163999Z","iopub.status.idle":"2021-09-25T06:30:24.294697Z","shell.execute_reply.started":"2021-09-25T06:30:24.163967Z","shell.execute_reply":"2021-09-25T06:30:24.293915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_dicom_files('train')\nget_dicom_files('test')","metadata":{"execution":{"iopub.status.busy":"2021-09-25T06:30:45.540707Z","iopub.execute_input":"2021-09-25T06:30:45.541005Z","iopub.status.idle":"2021-09-25T07:01:22.960946Z","shell.execute_reply.started":"2021-09-25T06:30:45.540976Z","shell.execute_reply":"2021-09-25T07:01:22.959433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for id_num in df.id:\n    full_path = os.path.join(IMAGE_OUTPUT, 'train/{}.png'.format(id_num))\n    df.loc[df.id == id_num, 'file'] = full_path","metadata":{"execution":{"iopub.status.busy":"2021-09-25T07:01:28.217681Z","iopub.execute_input":"2021-09-25T07:01:28.218632Z","iopub.status.idle":"2021-09-25T07:01:28.572801Z","shell.execute_reply.started":"2021-09-25T07:01:28.218592Z","shell.execute_reply":"2021-09-25T07:01:28.572064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2021-09-25T07:01:39.253263Z","iopub.execute_input":"2021-09-25T07:01:39.253792Z","iopub.status.idle":"2021-09-25T07:01:39.27618Z","shell.execute_reply.started":"2021-09-25T07:01:39.253759Z","shell.execute_reply":"2021-09-25T07:01:39.275126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = ImageDataLoaders.from_df(df, item_tfms=Resize((40000,1024),method='pad',pad_mode=PadMode.Zeros), bs=2, label_col =1, fn_col=2, valid_pct=0.05)","metadata":{"execution":{"iopub.status.busy":"2021-09-25T07:01:52.979098Z","iopub.execute_input":"2021-09-25T07:01:52.979382Z","iopub.status.idle":"2021-09-25T07:01:54.624539Z","shell.execute_reply.started":"2021-09-25T07:01:52.979355Z","shell.execute_reply":"2021-09-25T07:01:54.623506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch()","metadata":{"execution":{"iopub.status.busy":"2021-09-25T07:02:02.804968Z","iopub.execute_input":"2021-09-25T07:02:02.805286Z","iopub.status.idle":"2021-09-25T07:02:15.290963Z","shell.execute_reply.started":"2021-09-25T07:02:02.805253Z","shell.execute_reply":"2021-09-25T07:02:15.289955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch \nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass Net(nn.Module):\n    def __init__(self, pretrained=False):\n        super().__init__()\n        self.conv1 = nn.Conv2d(3, 6, 5)\n        self.pool = nn.MaxPool2d(2, 2)\n        #self.conv2 = nn.Conv2d(6, 6, 5)\n        self.innerConvLayers = nn.ModuleList()\n        for i in range(15):\n            self.innerConvLayers.append(nn.Conv2d(6, 6, 5))\n        self.fc1 = nn.Linear(6 * 5 * 5, 120)\n        self.fc2 = nn.Linear(120, 84)\n        self.fc3 = nn.Linear(84, 10)\n        self.dropout = nn.Dropout(0.2)\n\n    def forward(self, x):\n        x = self.pool(F.leaky_relu_(self.conv1(x)))\n        for layer in self.innerConvLayers:\n            x = self.dropout(self.pool(F.leaky_relu_(layer(x))))\n        x = torch.flatten(x, 1) # flatten all dimensions except batch\n        x = F.leaky_relu_(self.fc1(x))\n        x = F.leaky_relu_(self.fc2(x))\n        x = F.leaky_relu_(self.fc3(x))\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-09-25T07:02:24.398363Z","iopub.execute_input":"2021-09-25T07:02:24.398672Z","iopub.status.idle":"2021-09-25T07:02:24.411882Z","shell.execute_reply.started":"2021-09-25T07:02:24.398641Z","shell.execute_reply":"2021-09-25T07:02:24.410825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(Net())","metadata":{"execution":{"iopub.status.busy":"2021-09-25T07:02:32.975638Z","iopub.execute_input":"2021-09-25T07:02:32.976634Z","iopub.status.idle":"2021-09-25T07:02:32.996526Z","shell.execute_reply.started":"2021-09-25T07:02:32.976576Z","shell.execute_reply":"2021-09-25T07:02:32.995505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = cnn_learner(dls, Net, metrics=[error_rate, accuracy], model_dir=\"/tmp/model/\").to_fp16()","metadata":{"execution":{"iopub.status.busy":"2021-09-25T07:02:43.714087Z","iopub.execute_input":"2021-09-25T07:02:43.714393Z","iopub.status.idle":"2021-09-25T07:02:43.833414Z","shell.execute_reply.started":"2021-09-25T07:02:43.714362Z","shell.execute_reply":"2021-09-25T07:02:43.832699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.lr_find()","metadata":{"execution":{"iopub.status.busy":"2021-09-25T07:03:13.044906Z","iopub.execute_input":"2021-09-25T07:03:13.045545Z","iopub.status.idle":"2021-09-25T07:11:22.776469Z","shell.execute_reply.started":"2021-09-25T07:03:13.045505Z","shell.execute_reply":"2021-09-25T07:11:22.775296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.fit_one_cycle(3, lr_max=1e-2)","metadata":{"execution":{"iopub.status.busy":"2021-09-25T07:17:43.469367Z","iopub.execute_input":"2021-09-25T07:17:43.469718Z","iopub.status.idle":"2021-09-25T08:40:21.184344Z","shell.execute_reply.started":"2021-09-25T07:17:43.469683Z","shell.execute_reply":"2021-09-25T08:40:21.182937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.show_results()","metadata":{"execution":{"iopub.status.busy":"2021-09-25T08:45:53.527002Z","iopub.execute_input":"2021-09-25T08:45:53.527946Z","iopub.status.idle":"2021-09-25T08:46:12.627267Z","shell.execute_reply.started":"2021-09-25T08:45:53.527891Z","shell.execute_reply":"2021-09-25T08:46:12.622512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for id_num in df_test.id:\n    full_path = os.path.join(IMAGE_OUTPUT, 'test/{}.png'.format(id_num))\n    prediction = learn.predict(full_path)\n    print(prediction)\n    probability = prediction[2][1].item()\n    print(probability)\n    df_test.loc[df_test.id==id_num, 'value'] = probability","metadata":{"execution":{"iopub.status.busy":"2021-09-25T08:46:17.845811Z","iopub.execute_input":"2021-09-25T08:46:17.846338Z","iopub.status.idle":"2021-09-25T08:54:02.894683Z","shell.execute_reply.started":"2021-09-25T08:46:17.846305Z","shell.execute_reply":"2021-09-25T08:54:02.893425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-25T08:54:47.064586Z","iopub.execute_input":"2021-09-25T08:54:47.064959Z","iopub.status.idle":"2021-09-25T08:54:47.077879Z","shell.execute_reply.started":"2021-09-25T08:54:47.064901Z","shell.execute_reply":"2021-09-25T08:54:47.077202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.value.min()","metadata":{"execution":{"iopub.status.busy":"2021-09-25T08:54:58.94209Z","iopub.execute_input":"2021-09-25T08:54:58.942879Z","iopub.status.idle":"2021-09-25T08:54:58.951942Z","shell.execute_reply.started":"2021-09-25T08:54:58.942823Z","shell.execute_reply":"2021-09-25T08:54:58.950891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.value.max()","metadata":{"execution":{"iopub.status.busy":"2021-09-25T08:55:09.352709Z","iopub.execute_input":"2021-09-25T08:55:09.353042Z","iopub.status.idle":"2021-09-25T08:55:09.361051Z","shell.execute_reply.started":"2021-09-25T08:55:09.35301Z","shell.execute_reply":"2021-09-25T08:55:09.359997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.rename(columns={'id':'BraTS21ID','value':'MGMT_value'}).to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-25T08:55:22.537456Z","iopub.execute_input":"2021-09-25T08:55:22.537822Z","iopub.status.idle":"2021-09-25T08:55:22.554404Z","shell.execute_reply.started":"2021-09-25T08:55:22.537789Z","shell.execute_reply":"2021-09-25T08:55:22.553424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}