{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#basic\nimport numpy as np # linear algebra\nimport pandas as pd # data processing\nimport os\nfrom os.path import join\nimport sys\nfrom tqdm import tqdm\nimport threading\n\nimport torch\nfrom torch.utils.data import Dataset\n\n# image processing\n\nfrom scipy.ndimage import zoom # image resizing 3D\nfrom skimage.transform import resize\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-02T14:19:18.711178Z","iopub.execute_input":"2021-10-02T14:19:18.711617Z","iopub.status.idle":"2021-10-02T14:19:18.718785Z","shell.execute_reply.started":"2021-10-02T14:19:18.711563Z","shell.execute_reply":"2021-10-02T14:19:18.717719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_FOLDER = '../input/rsna-miccai-brain-tumor-radiogenomic-classification'\n\nlabels_df = pd.read_csv(join(INPUT_FOLDER, 'train_labels.csv'))\n\npatients_train = os.listdir(join(INPUT_FOLDER, 'train'))\npatients_test = os.listdir(join(INPUT_FOLDER, 'test'))\n# removing examples with errors mentioned in discussion\n# erronous examples\nerror_examples = ['00109', '00123', '00709']\n\n# remove from directory list\nfor error in error_examples: patients_train.remove(error)\nlabels_df = labels_df[[ x not in [int(y) for y in error_examples ] for x in labels_df.BraTS21ID ]]\npatients_train.sort()\n\nprint(f'Number of train data : {len(patients_train)}\\nNumber of test data : {len(patients_test)}')","metadata":{"execution":{"iopub.status.busy":"2021-10-02T14:18:10.858334Z","iopub.execute_input":"2021-10-02T14:18:10.85861Z","iopub.status.idle":"2021-10-02T14:18:10.876808Z","shell.execute_reply.started":"2021-10-02T14:18:10.858583Z","shell.execute_reply":"2021-10-02T14:18:10.875698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2021-10-02T14:18:12.66576Z","iopub.execute_input":"2021-10-02T14:18:12.666162Z","iopub.status.idle":"2021-10-02T14:18:12.676104Z","shell.execute_reply.started":"2021-10-02T14:18:12.666083Z","shell.execute_reply":"2021-10-02T14:18:12.674955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Constants","metadata":{}},{"cell_type":"code","source":"IMAGE_DEPTH = 64\nIMAGE_SIZE = [256, 256]\nIMAGE_DIMS = [IMAGE_DEPTH, *IMAGE_SIZE]\nBATCH_SIZE = 2\n\nclass MRITypes:\n    flair = 'FLAIR'\n    tw1ce = 'T1wCE'\n    t1w = 'T1w'\n    t2w = 'T2w'\n    \ndef get_mri_types():\n    return [ MRITypes.flair, MRITypes.tw1ce, MRITypes.t1w, MRITypes.t2w]\n    \ndef get_mri_type_index(mri_type : MRITypes):\n    return get_types().index(mri_type)","metadata":{"execution":{"iopub.status.busy":"2021-10-02T14:18:14.227271Z","iopub.execute_input":"2021-10-02T14:18:14.227862Z","iopub.status.idle":"2021-10-02T14:18:14.234624Z","shell.execute_reply.started":"2021-10-02T14:18:14.227825Z","shell.execute_reply":"2021-10-02T14:18:14.233777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loading dicoms","metadata":{}},{"cell_type":"code","source":"# load scans in a folder\ndef load_scan(path):\n    slices = [ pydicom.read_file(join(path, slice_file))\n                    for slice_file in os.listdir(path)]\n    \n    slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))\n    \n    return slices","metadata":{"execution":{"iopub.status.busy":"2021-10-02T14:18:15.694325Z","iopub.execute_input":"2021-10-02T14:18:15.695075Z","iopub.status.idle":"2021-10-02T14:18:15.699927Z","shell.execute_reply.started":"2021-10-02T14:18:15.695027Z","shell.execute_reply":"2021-10-02T14:18:15.699317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Since there is a lot of preprocessing done on the data, loading the data normally using the dataloader caused a bottleneck and each epoch took 3 hours to train.\n- So, I did my preprocessing and saved it to the disk, so that the dataloader would only load up the files and using num_workers in pytorch dataloader should speed that a lot more than the case of loading and preprocessing only.\n- However, since kaggle doesn't allow you to save your data through multiple notebooks, I had to download someone else's dataset and edit it slightly.","metadata":{}},{"cell_type":"code","source":"class DatasetPreprocessor:\n    def __init__(self, patients_train, patients_test):\n        self.patients_train = patients_train\n        self.patients_test = patients_test\n        self.my_train_path = 'my-train'\n        self.my_test_path = 'my-test'\n        \n    def preprocess_train(self):\n        os.makedirs(self.my_train_path)\n        \n        for patient_id in tqdm(self.patients_train):\n            patient_folder = join(self.my_train_path, patient_id) \n\n            patient_scans = np.zeros((4, 1, 64, 256, 256), dtype=np.uint8)\n            for i, scan_type in enumerate(get_mri_types()):\n                \n                # read the data for the scan and patient\n                patient_scan = load_scan(join(INPUT_FOLDER, 'train', patient_id, scan_type))\n                \n                # pre-process the data read above\n                \n                scan_3d = preprocess_image_train(patient_scan)\n                # add data to np array\n                \n                patient_scans[i] = scan_3d\n            threading.Thread(target=torch.save, args=(torch.from_numpy(patient_scans), patient_folder + '.pt')).start()\n            \n        \n        \ndef preprocess_image_train(image):\n    \n    trans_image = [ apply_voi_lut(x.pixel_array, x) for x in image]\n    \n    trans_image = [ np.amax(x) - x if dicom.PhotometricInterpretation == \"MONOCHROME1\" \n                               else x for x, dicom in zip(trans_image, image) ]\n    \n    trans_image = np.array([ resize(x, IMAGE_SIZE) for x in trans_image if  np.any(x != 0) ])\n    \n    \n    current_depth = trans_image.shape[0]\n    trans_image = zoom(trans_image, (IMAGE_DEPTH/current_depth, 1, 1))\n    \n    \n    \n    trans_image = [ x - np.min(x) for x in trans_image]\n    \n    trans_image = [ x / np.max(x) for x in trans_image]\n    \n    trans_image = [ (x * 255).astype(np.uint8) for x in trans_image ]\n    \n    \n    \n    return np.array(trans_image, dtype=np.uint8)","metadata":{"execution":{"iopub.status.busy":"2021-10-02T14:18:17.374444Z","iopub.execute_input":"2021-10-02T14:18:17.374873Z","iopub.status.idle":"2021-10-02T14:18:17.385594Z","shell.execute_reply.started":"2021-10-02T14:18:17.374831Z","shell.execute_reply":"2021-10-02T14:18:17.384677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! rm -rf my-train\ndataset_preprocessor = DatasetPreprocessor(patients_train, patients_test)\ndataset_preprocessor.preprocess_train()","metadata":{"execution":{"iopub.status.busy":"2021-10-02T14:19:29.268326Z","iopub.execute_input":"2021-10-02T14:19:29.26938Z","iopub.status.idle":"2021-10-02T14:20:10.093817Z","shell.execute_reply.started":"2021-10-02T14:19:29.269326Z","shell.execute_reply":"2021-10-02T14:20:10.092415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dataset to use after saving the files to disk","metadata":{}},{"cell_type":"code","source":"class DiskDataset(Dataset):\n    def __init__(self, train_labels):\n        self.filenames = train_labels.BraTS21ID.map( lambda x : join('my-train', x + '.pt').values\n        self.labels = train_labels.MGMT_value.values\n    \n    def __len__(self):\n        return len(self.file_names)\n    \n    def __getitem__(self, idx):\n        \n        return (\n            torch.load(self.filenames[idx]).to(torch.float32),\n            torch.tensor(self.labels[idx], dtype=torch.float32)\n            )","metadata":{"execution":{"iopub.status.busy":"2021-10-02T14:19:01.440518Z","iopub.status.idle":"2021-10-02T14:19:01.440873Z","shell.execute_reply.started":"2021-10-02T14:19:01.440685Z","shell.execute_reply":"2021-10-02T14:19:01.440708Z"},"trusted":true},"execution_count":null,"outputs":[]}]}