{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n#import json\nimport glob\n#import random\n#import collections\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport cv2\n#import matplotlib.pyplot as plt\n#import scipy\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-20T16:16:53.212788Z","iopub.execute_input":"2021-07-20T16:16:53.213173Z","iopub.status.idle":"2021-07-20T16:16:53.593206Z","shell.execute_reply.started":"2021-07-20T16:16:53.213129Z","shell.execute_reply":"2021-07-20T16:16:53.592389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-20T16:16:53.594761Z","iopub.execute_input":"2021-07-20T16:16:53.59525Z","iopub.status.idle":"2021-07-20T16:16:53.623722Z","shell.execute_reply.started":"2021-07-20T16:16:53.595212Z","shell.execute_reply":"2021-07-20T16:16:53.622818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"MGMT_value\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-07-20T16:16:53.625396Z","iopub.execute_input":"2021-07-20T16:16:53.625741Z","iopub.status.idle":"2021-07-20T16:16:53.635143Z","shell.execute_reply.started":"2021-07-20T16:16:53.6257Z","shell.execute_reply":"2021-07-20T16:16:53.634122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df[\"MGMT_value\"])","metadata":{"execution":{"iopub.status.busy":"2021-07-20T15:30:56.727835Z","iopub.execute_input":"2021-07-20T15:30:56.728716Z","iopub.status.idle":"2021-07-20T15:30:56.738557Z","shell.execute_reply.started":"2021-07-20T15:30:56.728684Z","shell.execute_reply":"2021-07-20T15:30:56.73743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/\"","metadata":{"execution":{"iopub.status.busy":"2021-07-20T16:16:53.636716Z","iopub.execute_input":"2021-07-20T16:16:53.637508Z","iopub.status.idle":"2021-07-20T16:16:53.643708Z","shell.execute_reply.started":"2021-07-20T16:16:53.637473Z","shell.execute_reply":"2021-07-20T16:16:53.642399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_slices(patient, folder, p):\n    path = p + patient + \"/\" + folder\n    slices = [pydicom.dcmread(path + \"/\" + s) for s in os.listdir(path)]\n    slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))\n    return slices","metadata":{"execution":{"iopub.status.busy":"2021-07-20T18:57:38.691322Z","iopub.execute_input":"2021-07-20T18:57:38.691668Z","iopub.status.idle":"2021-07-20T18:57:38.699951Z","shell.execute_reply.started":"2021-07-20T18:57:38.691636Z","shell.execute_reply":"2021-07-20T18:57:38.699076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize(scan, rm):\n    scan[scan<-1000] = -1000\n    scan[scan>400] = 400\n    while rm > 0:\n        dist = np.array([])\n        for i in range(len(scan)-1):\n            dist = np.append(dist,np.linalg.norm(scan[i]-scan[i+1]))\n        idx = np.argmin(dist)\n        if dist[idx] < 500:\n            scan[idx] = (scan[idx] + scan[idx+1])/2\n            scan = np.delete(scan,idx+1,axis=0)\n            rm -= 1\n        else:\n            break\n    while rm > 0:\n        i = 0\n        while i < len(scan)-1:\n            if i%2 == 1:\n                scan[i] = (scan[i] + scan[i+1])/2\n                scan = np.delete(scan,i,axis=0)\n                rm -= 1\n            if rm == 0:\n                break\n            i += 1\n    while rm < 0:\n        i = 0\n        while i < len(scan)-1:\n            if i%2 == 1:\n                img = (scan[i] + scan[i+1])/2\n                scan = np.insert(scan,i+1,img,axis=0)\n                rm += 1\n            if rm == 0:\n                break\n            i += 1\n    scan = scan - np.min(scan)\n    if np.max(scan) != 0:\n        scan = scan / np.max(scan)\n    return scan","metadata":{"execution":{"iopub.status.busy":"2021-07-20T16:16:54.769224Z","iopub.execute_input":"2021-07-20T16:16:54.770562Z","iopub.status.idle":"2021-07-20T16:16:54.788463Z","shell.execute_reply.started":"2021-07-20T16:16:54.770517Z","shell.execute_reply":"2021-07-20T16:16:54.787495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''total_data = {\"FLAIR\":[],\"T1w\":[],\"T1wCE\":[],\"T2w\":[]}\nfor _,patient in enumerate(tqdm(sorted(os.listdir(path)))):\n    for key in total_data.keys():\n        slices = load_slices(patient,key)\n        image = np.stack([cv2.resize(s.pixel_array,(92,92)) for s in slices])\n        #print(\"original:\",image.shape)\n        new_image = resize(image,len(slices)-40)\n        #print(\"new:\",new_image.shape)\n        total_data[key].append(new_image)\n#total_data[\"T2w\"] = np.array(total_data[\"T2w\"])'''","metadata":{"execution":{"iopub.status.busy":"2021-07-20T16:16:55.746861Z","iopub.execute_input":"2021-07-20T16:16:55.747204Z","iopub.status.idle":"2021-07-20T16:16:55.75372Z","shell.execute_reply.started":"2021-07-20T16:16:55.747174Z","shell.execute_reply":"2021-07-20T16:16:55.752612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''from matplotlib import animation, rc\nrc('animation', html='jshtml')\n\n\ndef create_animation(ims):\n    fig,ax = plt.subplots(1,1,figsize=(6, 6))\n    plt.axis('off')\n    im = ax.imshow(ims[0], cmap=\"gray\")\n    \n    def animate_func(i):\n        ax.imshow(ims[i], cmap=\"gray\")\n        return [ims]\n    return animation.FuncAnimation(fig, animate_func, frames = len(ims), interval = 1000//24)\n'''","metadata":{"execution":{"iopub.status.busy":"2021-07-20T15:09:23.518945Z","iopub.execute_input":"2021-07-20T15:09:23.519282Z","iopub.status.idle":"2021-07-20T15:09:23.525361Z","shell.execute_reply.started":"2021-07-20T15:09:23.519248Z","shell.execute_reply":"2021-07-20T15:09:23.52426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create_animation(total_data[\"FLAIR\"][-1])","metadata":{"execution":{"iopub.status.busy":"2021-07-18T16:11:01.389111Z","iopub.execute_input":"2021-07-18T16:11:01.389553Z","iopub.status.idle":"2021-07-18T16:11:25.820811Z","shell.execute_reply.started":"2021-07-18T16:11:01.389511Z","shell.execute_reply":"2021-07-18T16:11:25.820101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import Input, Conv3D, MaxPool3D, Flatten, Dense, Dropout, BatchNormalization\nfrom tensorflow.keras.models import Model\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom sklearn.model_selection import train_test_split\n\n#from keras.utils.np_utils import to_categorical","metadata":{"execution":{"iopub.status.busy":"2021-07-20T16:16:58.37588Z","iopub.execute_input":"2021-07-20T16:16:58.376215Z","iopub.status.idle":"2021-07-20T16:17:03.25168Z","shell.execute_reply.started":"2021-07-20T16:16:58.376185Z","shell.execute_reply":"2021-07-20T16:17:03.250759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keys = [\"FLAIR\",\"T1w\",\"T1wCE\",\"T2w\"]","metadata":{"execution":{"iopub.status.busy":"2021-07-20T16:17:03.253049Z","iopub.execute_input":"2021-07-20T16:17:03.253393Z","iopub.status.idle":"2021-07-20T16:17:03.2616Z","shell.execute_reply.started":"2021-07-20T16:17:03.253355Z","shell.execute_reply":"2021-07-20T16:17:03.26069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = {}","metadata":{"execution":{"iopub.status.busy":"2021-07-20T16:17:03.265508Z","iopub.execute_input":"2021-07-20T16:17:03.265762Z","iopub.status.idle":"2021-07-20T16:17:03.271361Z","shell.execute_reply.started":"2021-07-20T16:17:03.265739Z","shell.execute_reply":"2021-07-20T16:17:03.270606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for key in keys:\n    input_layer = Input(shape = (40, 80, 80,1))\n    x = Conv3D(filters = 64, kernel_size = 3, activation ='relu')(input_layer)\n    x = Conv3D(filters = 64, kernel_size = 3, activation ='relu')(x)\n    x = MaxPool3D(pool_size = 2)(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.2)(x)\n    \n    x = Conv3D(filters = 128, kernel_size = 4,padding = 'Same', activation ='relu')(x)\n    x = Conv3D(filters = 128, kernel_size = 4, activation ='relu')(x)\n    x = MaxPool3D(pool_size = 2)(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.3)(x)\n    \n    x = Conv3D(filters = 256, kernel_size = 5,padding = 'Same', activation ='relu')(x)\n    x = Conv3D(filters = 256, kernel_size = 5,padding = 'Same', activation ='relu')(x)\n    x = MaxPool3D(pool_size = 2)(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.4)(x)\n    \n    x = Flatten()(x)\n    x = Dense(units = 256, activation = 'relu')(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.5)(x)\n    output_layer = Dense(units = 1, activation = 'sigmoid')(x)\n\n    model = Model(input_layer, output_layer)\n    models[key] = model","metadata":{"execution":{"iopub.status.busy":"2021-07-20T16:17:20.565045Z","iopub.execute_input":"2021-07-20T16:17:20.565446Z","iopub.status.idle":"2021-07-20T16:17:23.192929Z","shell.execute_reply.started":"2021-07-20T16:17:20.565413Z","shell.execute_reply":"2021-07-20T16:17:23.19213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)\n\nearly_stopping_cb = EarlyStopping(monitor=\"val_acc\", patience=10)","metadata":{"execution":{"iopub.status.busy":"2021-07-20T16:17:25.494383Z","iopub.execute_input":"2021-07-20T16:17:25.494706Z","iopub.status.idle":"2021-07-20T16:17:25.501364Z","shell.execute_reply.started":"2021-07-20T16:17:25.494676Z","shell.execute_reply":"2021-07-20T16:17:25.500474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_pred = {}\n#val_pred = {}\nepochs = 30\nfor key in keys:\n    X = []\n    models[key].compile(\n        loss=\"binary_crossentropy\",\n        optimizer= \"adam\",\n        metrics=[\"acc\"],\n    )\n    for _,patient in enumerate(tqdm(sorted(os.listdir(path)))):\n        slices = load_slices(patient,key, path)\n        image = np.stack([cv2.resize(s.pixel_array,(80,80)) for s in slices])\n        #print(\"original:\",image.shape)\n        new_image = resize(image,len(slices)-40)\n        del image\n        #print(\"new:\",new_image.shape)\n        X.append(new_image)\n\n    X_train, X_val, y_train, y_val = train_test_split(X, train_df[\"MGMT_value\"], test_size = 0.15, random_state=42)\n    X_train = np.array(X_train)\n    X_val = np.array(X_val)\n    y_train = np.array(y_train)\n    y_val = np.array(y_val)\n    print(\"model for: \",key)\n    models[key].fit(X_train,y_train ,validation_data=(X_val,y_val), shuffle= True, epochs=epochs, batch_size=16,verbose=1, callbacks=[early_stopping_cb,learning_rate_reduction])\n    #train_pred[key] = models[key].predict(X_train)\n    #val_pred[key] = models[key].predict(X_val)\n    del X\n    del X_train\n    del X_val\n    del slices","metadata":{"execution":{"iopub.status.busy":"2021-07-20T16:17:26.621494Z","iopub.execute_input":"2021-07-20T16:17:26.621849Z","iopub.status.idle":"2021-07-20T17:56:20.84122Z","shell.execute_reply.started":"2021-07-20T16:17:26.621817Z","shell.execute_reply":"2021-07-20T17:56:20.84024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''for key in keys:\n    train_pred_c[key] = np.where(np.array(train_pred[key])>0.5,1,0)\n    val_pred_c[key] = np.where(np.array(val_pred[key])>0.5,1,0)\ntrain_pred_c = pd.DataFrame(train_pred_c)\nval_pred_c = pd.DataFrame(val_pred_c)'''","metadata":{"execution":{"iopub.status.busy":"2021-07-20T18:02:28.474611Z","iopub.execute_input":"2021-07-20T18:02:28.47497Z","iopub.status.idle":"2021-07-20T18:02:28.480448Z","shell.execute_reply.started":"2021-07-20T18:02:28.474941Z","shell.execute_reply":"2021-07-20T18:02:28.479394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score","metadata":{"execution":{"iopub.status.busy":"2021-07-20T18:02:56.531253Z","iopub.execute_input":"2021-07-20T18:02:56.531664Z","iopub.status.idle":"2021-07-20T18:02:56.536155Z","shell.execute_reply.started":"2021-07-20T18:02:56.531629Z","shell.execute_reply":"2021-07-20T18:02:56.534975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''for key in keys:\n    print(\"key:\",key,\" train:\",accuracy_score(y_train,train_pred_c[key]), \"val:\",accuracy_score(y_val,val_pred_c[key]))\nprint(\"average: \",\"train:\",accuracy_score(y_train, np.where(pd.DataFrame(train_pred_c).mean(axis=1).values>0.5,1,0)),\n      \"val:\",accuracy_score(y_val,np.where(pd.DataFrame(val_pred_c).mean(axis=1).values>0.5,1,0)))'''","metadata":{"execution":{"iopub.status.busy":"2021-07-20T18:11:42.677426Z","iopub.execute_input":"2021-07-20T18:11:42.677757Z","iopub.status.idle":"2021-07-20T18:11:42.706639Z","shell.execute_reply.started":"2021-07-20T18:11:42.677727Z","shell.execute_reply":"2021-07-20T18:11:42.705872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path= \"../input/rsna-miccai-brain-tumor-radiogenomic-classification/test/\"","metadata":{"execution":{"iopub.status.busy":"2021-07-20T18:58:38.847872Z","iopub.execute_input":"2021-07-20T18:58:38.84826Z","iopub.status.idle":"2021-07-20T18:58:38.852669Z","shell.execute_reply.started":"2021-07-20T18:58:38.848225Z","shell.execute_reply":"2021-07-20T18:58:38.851411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = []\ntest_pred = {}\nfor key in keys:\n    X = []\n    for _,patient in enumerate(tqdm(sorted(os.listdir(test_path)))):\n        slices = load_slices(patient,key,test_path)\n        image = np.stack([cv2.resize(s.pixel_array,(80,80)) for s in slices])\n        #print(\"original:\",image.shape)\n        new_image = resize(image,len(slices)-40)\n        del image\n        #print(\"new:\",new_image.shape)\n        X.append(new_image)\n    test_pred[key] = models[key].predict(np.array(X))\n    del X\n","metadata":{"execution":{"iopub.status.busy":"2021-07-20T19:14:57.189344Z","iopub.execute_input":"2021-07-20T19:14:57.189689Z","iopub.status.idle":"2021-07-20T19:22:14.265691Z","shell.execute_reply.started":"2021-07-20T19:14:57.189659Z","shell.execute_reply":"2021-07-20T19:22:14.264799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for key in keys:\n    test_pred[key] = [x[0] for x in test_pred[key]]\n","metadata":{"execution":{"iopub.status.busy":"2021-07-20T19:24:12.616358Z","iopub.execute_input":"2021-07-20T19:24:12.616735Z","iopub.status.idle":"2021-07-20T19:24:12.664472Z","shell.execute_reply.started":"2021-07-20T19:24:12.616701Z","shell.execute_reply":"2021-07-20T19:24:12.662619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = pd.DataFrame(test_pred).mean(axis=1)\npred.head(),len(pred)","metadata":{"execution":{"iopub.status.busy":"2021-07-20T19:24:51.285261Z","iopub.execute_input":"2021-07-20T19:24:51.285609Z","iopub.status.idle":"2021-07-20T19:24:51.293818Z","shell.execute_reply.started":"2021-07-20T19:24:51.28558Z","shell.execute_reply":"2021-07-20T19:24:51.292807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred = pd.DataFrame(test_pred)\nsubmission = pd.DataFrame({\"BraTS21ID\":[int(id) for id in sorted(os.listdir(test_path))],\n                           \"MGMT_value\":pred})","metadata":{"execution":{"iopub.status.busy":"2021-07-20T19:25:53.66614Z","iopub.execute_input":"2021-07-20T19:25:53.666482Z","iopub.status.idle":"2021-07-20T19:25:53.674731Z","shell.execute_reply.started":"2021-07-20T19:25:53.666452Z","shell.execute_reply":"2021-07-20T19:25:53.673736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2021-07-20T19:26:54.148747Z","iopub.execute_input":"2021-07-20T19:26:54.14912Z","iopub.status.idle":"2021-07-20T19:26:54.163704Z","shell.execute_reply.started":"2021-07-20T19:26:54.149077Z","shell.execute_reply":"2021-07-20T19:26:54.162446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]}]}