{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from fastai.vision.all import *\nimport sys\nimport numpy as np\n","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:45:21.368219Z","iopub.execute_input":"2021-08-22T16:45:21.368622Z","iopub.status.idle":"2021-08-22T16:45:22.330961Z","shell.execute_reply.started":"2021-08-22T16:45:21.368568Z","shell.execute_reply":"2021-08-22T16:45:22.330077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT = '../input/rsna-miccai-brain-tumor-radiogenomic-classification/'\n#IMAGE_OUTPUT = '../input/rsnamiccai-competition-2021-png-strips/'\nIMAGE_OUTPUT = '.'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport random\ndf = pd.read_csv(os.path.join(ROOT, 'train_labels.csv'), header=0, names=['id','value'], dtype=object)\ndf = df[~df.id.isin([\"00109\", \"00123\", \"00709\"])]","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:45:22.3379Z","iopub.execute_input":"2021-08-22T16:45:22.338249Z","iopub.status.idle":"2021-08-22T16:45:22.348788Z","shell.execute_reply.started":"2021-08-22T16:45:22.338211Z","shell.execute_reply":"2021-08-22T16:45:22.348008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:45:22.350498Z","iopub.execute_input":"2021-08-22T16:45:22.350889Z","iopub.status.idle":"2021-08-22T16:45:22.366382Z","shell.execute_reply.started":"2021-08-22T16:45:22.350853Z","shell.execute_reply":"2021-08-22T16:45:22.365461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://stackoverflow.com/a/4836734/8245487\ndef natural_sort(l): \n    convert = lambda text: int(text) if text.isdigit() else text.lower()\n    alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]\n    return sorted(l, key=alphanum_key)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:45:22.36769Z","iopub.execute_input":"2021-08-22T16:45:22.368053Z","iopub.status.idle":"2021-08-22T16:45:22.373771Z","shell.execute_reply.started":"2021-08-22T16:45:22.368018Z","shell.execute_reply":"2021-08-22T16:45:22.372365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.maximum((1,5),(3,4))","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:45:22.375466Z","iopub.execute_input":"2021-08-22T16:45:22.375823Z","iopub.status.idle":"2021-08-22T16:45:22.388214Z","shell.execute_reply.started":"2021-08-22T16:45:22.375788Z","shell.execute_reply":"2021-08-22T16:45:22.387194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.DataFrame(columns=['id', 'value'])\ndf_test.id = os.listdir(os.path.join(ROOT, 'test'))","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:45:22.389664Z","iopub.execute_input":"2021-08-22T16:45:22.390039Z","iopub.status.idle":"2021-08-22T16:45:22.399397Z","shell.execute_reply.started":"2021-08-22T16:45:22.39Z","shell.execute_reply":"2021-08-22T16:45:22.398658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Use the cell below to process the DICOMs into PNG animation strips","metadata":{}},{"cell_type":"code","source":"import os\nimport pydicom\nimport pandas as pd\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom tqdm import tqdm\nimport binascii\nfrom PIL import Image\nfrom multiprocessing import Pool\n\n\nif not os.path.exists('./train'):\n    os.makedirs('./train')\n    \n\nif not os.path.exists('./test'):\n    os.makedirs('./test')\n\n\ndef get_dicom_files(dataset='train'):\n    root = f\"{ROOT}/{dataset}\"\n    \n    ids = list(df.id) if dataset=='train' else list(df_test.id)\n    \n    with Pool(10) as p:\n        p.starmap(get_dicom_files_helper, [(dataset, root, cur_id) for cur_id in ids])\n    \n\ndef get_dicom_files_helper(dataset, root, cur_id):\n    final_image = np.array([])\n    for scan_type in ['FLAIR', 'T1w', 'T1wCE', 'T2w']:\n        cur_dir = os.path.join(root, cur_id, scan_type)\n        dicoms = natural_sort(os.listdir(cur_dir))\n        median = len(dicoms)//2\n        modulo = max(len(dicoms)//20, 1)\n        dicoms = [dicom for i, dicom in enumerate(dicoms) if i % modulo == 0]\n        for dicom in dicoms:\n            filepath = os.path.join(cur_dir, dicom)\n            data = process_dicom(filepath)\n            if len(final_image) != 0:\n                (data_rows, data_cols) = data.shape\n                (final_rows, final_cols) = final_image.shape\n                if final_cols != data_cols:\n                    if data_cols < final_cols:\n                        data = np.hstack((data, np.zeros((data_rows, final_cols - data_cols))))\n                    else:\n                        final_image = np.hstack((final_image, np.zeros((final_rows, data_cols - final_cols))))\n            final_image = np.concatenate([final_image,data]) if len(final_image) != 0 else data\n            data = None\n\n    outpath = os.path.join(f'./{dataset}',f'{cur_id}.png')\n    write_pixels(final_image, outpath)\n    final_image = None\n    print('Processed {}'.format(cur_id))\n        \n\ndef process_dicom(path):\n    dicom = pydicom.read_file(path)\n    data = apply_voi_lut(dicom.pixel_array, dicom)\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    \n    return data\n\n    \ndef write_pixels(data, outpath):\n    height = len(data)\n    width = len(data[0])\n    pixels_out = []\n    for row in data:\n        pixels_out.extend(row)\n    assert(len(pixels_out) == height * width)\n    \n    image_out = Image.new('L', (width, height))\n    image_out.putdata(pixels_out)\n    image_out.save(outpath)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:45:22.400595Z","iopub.execute_input":"2021-08-22T16:45:22.400965Z","iopub.status.idle":"2021-08-22T16:45:22.407188Z","shell.execute_reply.started":"2021-08-22T16:45:22.400929Z","shell.execute_reply":"2021-08-22T16:45:22.406235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_dicom_files('train')\nget_dicom_files('test')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for id_num in df.id:\n    full_path = os.path.join(IMAGE_OUTPUT, 'train/{}.png'.format(id_num))\n    df.loc[df.id == id_num, 'file'] = full_path\n    ","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:45:22.410668Z","iopub.execute_input":"2021-08-22T16:45:22.411036Z","iopub.status.idle":"2021-08-22T16:45:22.822316Z","shell.execute_reply.started":"2021-08-22T16:45:22.411Z","shell.execute_reply":"2021-08-22T16:45:22.821384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:45:22.824663Z","iopub.execute_input":"2021-08-22T16:45:22.825155Z","iopub.status.idle":"2021-08-22T16:45:22.838973Z","shell.execute_reply.started":"2021-08-22T16:45:22.825115Z","shell.execute_reply":"2021-08-22T16:45:22.837971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = ImageDataLoaders.from_df(df, item_tfms=Resize((40000,1024),method='pad',pad_mode=PadMode.Zeros), bs=2, label_col =1, fn_col=2, valid_pct=0.05)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:45:22.840677Z","iopub.execute_input":"2021-08-22T16:45:22.841075Z","iopub.status.idle":"2021-08-22T16:45:25.611464Z","shell.execute_reply.started":"2021-08-22T16:45:22.841009Z","shell.execute_reply":"2021-08-22T16:45:25.61056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:45:25.612811Z","iopub.execute_input":"2021-08-22T16:45:25.61326Z","iopub.status.idle":"2021-08-22T16:45:31.318207Z","shell.execute_reply.started":"2021-08-22T16:45:25.613201Z","shell.execute_reply":"2021-08-22T16:45:31.31672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch \nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass Net(nn.Module):\n    def __init__(self, pretrained=False):\n        super().__init__()\n        self.conv1 = nn.Conv2d(3, 6, 5)\n        self.pool = nn.MaxPool2d(2, 2)\n        #self.conv2 = nn.Conv2d(6, 6, 5)\n        self.innerConvLayers = nn.ModuleList()\n        for i in range(15):\n            self.innerConvLayers.append(nn.Conv2d(6, 6, 5))\n        self.fc1 = nn.Linear(6 * 5 * 5, 120)\n        self.fc2 = nn.Linear(120, 84)\n        self.fc3 = nn.Linear(84, 10)\n        self.dropout = nn.Dropout(0.2)\n\n    def forward(self, x):\n        x = self.pool(F.leaky_relu_(self.conv1(x)))\n        for layer in self.innerConvLayers:\n            x = self.dropout(self.pool(F.leaky_relu_(layer(x))))\n        x = torch.flatten(x, 1) # flatten all dimensions except batch\n        x = F.leaky_relu_(self.fc1(x))\n        x = F.leaky_relu_(self.fc2(x))\n        x = F.leaky_relu_(self.fc3(x))\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:45:31.319544Z","iopub.execute_input":"2021-08-22T16:45:31.3199Z","iopub.status.idle":"2021-08-22T16:45:31.329827Z","shell.execute_reply.started":"2021-08-22T16:45:31.319861Z","shell.execute_reply":"2021-08-22T16:45:31.328857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"print(Net())","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:45:31.331278Z","iopub.execute_input":"2021-08-22T16:45:31.331897Z","iopub.status.idle":"2021-08-22T16:45:31.349576Z","shell.execute_reply.started":"2021-08-22T16:45:31.33186Z","shell.execute_reply":"2021-08-22T16:45:31.348483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nlearn = cnn_learner(dls, Net, metrics=[error_rate, accuracy], model_dir=\"/tmp/model/\").to_fp16()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:45:31.351108Z","iopub.execute_input":"2021-08-22T16:45:31.351537Z","iopub.status.idle":"2021-08-22T16:45:31.366771Z","shell.execute_reply.started":"2021-08-22T16:45:31.351503Z","shell.execute_reply":"2021-08-22T16:45:31.366004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.lr_find()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:45:31.367865Z","iopub.execute_input":"2021-08-22T16:45:31.36822Z","iopub.status.idle":"2021-08-22T16:47:22.844066Z","shell.execute_reply.started":"2021-08-22T16:45:31.368183Z","shell.execute_reply":"2021-08-22T16:47:22.842243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.fit_one_cycle(3, lr_max=1e-2)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T16:47:22.845559Z","iopub.execute_input":"2021-08-22T16:47:22.845913Z","iopub.status.idle":"2021-08-22T17:53:32.344648Z","shell.execute_reply.started":"2021-08-22T16:47:22.845872Z","shell.execute_reply":"2021-08-22T17:53:32.343606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.show_results()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T17:53:32.346384Z","iopub.execute_input":"2021-08-22T17:53:32.346786Z","iopub.status.idle":"2021-08-22T17:53:40.38816Z","shell.execute_reply.started":"2021-08-22T17:53:32.34674Z","shell.execute_reply":"2021-08-22T17:53:40.387154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for id_num in df_test.id:\n    full_path = os.path.join(IMAGE_OUTPUT, 'test/{}.png'.format(id_num))\n    prediction = learn.predict(full_path)\n    print(prediction)\n    probability = prediction[2][1].item()\n    print(probability)\n    df_test.loc[df_test.id==id_num, 'value'] = probability","metadata":{"execution":{"iopub.status.busy":"2021-08-22T17:53:40.389612Z","iopub.execute_input":"2021-08-22T17:53:40.390009Z","iopub.status.idle":"2021-08-22T17:58:03.525519Z","shell.execute_reply.started":"2021-08-22T17:53:40.389968Z","shell.execute_reply":"2021-08-22T17:58:03.524458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T17:58:03.527018Z","iopub.execute_input":"2021-08-22T17:58:03.527386Z","iopub.status.idle":"2021-08-22T17:58:03.537514Z","shell.execute_reply.started":"2021-08-22T17:58:03.527335Z","shell.execute_reply":"2021-08-22T17:58:03.536588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.value.min()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T17:58:03.538805Z","iopub.execute_input":"2021-08-22T17:58:03.539419Z","iopub.status.idle":"2021-08-22T17:58:03.551175Z","shell.execute_reply.started":"2021-08-22T17:58:03.539374Z","shell.execute_reply":"2021-08-22T17:58:03.55008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.value.max()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T17:58:03.552896Z","iopub.execute_input":"2021-08-22T17:58:03.553322Z","iopub.status.idle":"2021-08-22T17:58:03.563463Z","shell.execute_reply.started":"2021-08-22T17:58:03.553285Z","shell.execute_reply":"2021-08-22T17:58:03.562433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.rename(columns={'id':'BraTS21ID','value':'MGMT_value'}).to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T17:58:03.564972Z","iopub.execute_input":"2021-08-22T17:58:03.565544Z","iopub.status.idle":"2021-08-22T17:58:03.576303Z","shell.execute_reply.started":"2021-08-22T17:58:03.565501Z","shell.execute_reply":"2021-08-22T17:58:03.575424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}