{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport json\nimport glob\nimport random\nimport collections\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random\nfrom tqdm.notebook import tqdm\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras import layers\n\n\nTYPES = [\"FLAIR\", \"T1w\", \"T2w\", \"T1wCE\"]\nWHITE_THRESHOLD = 10 # out of 255\nEXCLUDE = [109, 123, 709]\n\n\ntrain_df = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv\")\ntest_df = pd.read_csv('../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv')\ntrain_df = train_df[~train_df.BraTS21ID.isin(EXCLUDE)]","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-30T07:31:24.10444Z","iopub.execute_input":"2021-09-30T07:31:24.104829Z","iopub.status.idle":"2021-09-30T07:31:29.630825Z","shell.execute_reply.started":"2021-09-30T07:31:24.104742Z","shell.execute_reply":"2021-09-30T07:31:29.629964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom(path, size = 224):\n    ''' \n    Reads a DICOM image, standardizes so that the pixel values are between 0 and 1, then rescales to 0 and 255\n    \n    Note super sure if this kind of scaling is appropriate, but everyone seems to do it. \n    '''\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return cv2.resize(data, (size, size))\n\ndef get_all_image_paths(brats21id, image_type, folder='train'): \n    '''\n    Returns an arry of all the images of a particular type for a particular patient ID\n    '''\n    assert(image_type in TYPES)\n    \n    patient_path = os.path.join(\n        \"../input/rsna-miccai-brain-tumor-radiogenomic-classification/%s/\" % folder, \n        str(brats21id).zfill(5),\n    )\n\n    paths = sorted(\n        glob.glob(os.path.join(patient_path, image_type, \"*\")), \n        key=lambda x: int(x[:-4].split(\"-\")[-1]),\n    )\n    \n    num_images = len(paths)\n    \n    start = int(num_images * 0.25)\n    end = int(num_images * 0.75)\n\n    interval = 3\n    \n    if num_images < 10: \n        interval = 1\n    \n    return np.array(paths[start:end:interval])\n\ndef get_all_images(brats21id, image_type, folder='train', size=225):\n    return [load_dicom(path, size) for path in get_all_image_paths(brats21id, image_type, folder)]\n","metadata":{"execution":{"iopub.status.busy":"2021-09-30T07:31:29.632567Z","iopub.execute_input":"2021-09-30T07:31:29.633302Z","iopub.status.idle":"2021-09-30T07:31:29.64439Z","shell.execute_reply.started":"2021-09-30T07:31:29.633265Z","shell.execute_reply":"2021-09-30T07:31:29.643369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = 224\n\ndef get_all_data_for_train(image_type):\n    global train_df\n    \n    X = []\n    y = []\n    train_ids = []\n\n    for i in tqdm(train_df.index):\n        x = train_df.loc[i]\n        images = get_all_images(int(x['BraTS21ID']), image_type, 'train', IMAGE_SIZE)\n        label = x['MGMT_value']\n\n        X += images\n        y += [label] * len(images)\n        train_ids += [int(x['BraTS21ID'])] * len(images)\n        assert(len(X) == len(y))\n    return np.array(X), np.array(y), np.array(train_ids)\n\ndef get_all_data_for_test(image_type):\n    global test_df\n    \n    X = []\n    test_ids = []\n\n    for i in tqdm(test_df.index):\n        x = test_df.loc[i]\n        images = get_all_images(int(x['BraTS21ID']), image_type, 'test', IMAGE_SIZE)\n        X += images\n        test_ids += [int(x['BraTS21ID'])] * len(images)\n\n    return np.array(X), np.array(test_ids)","metadata":{"execution":{"iopub.status.busy":"2021-09-30T07:31:29.646958Z","iopub.execute_input":"2021-09-30T07:31:29.647649Z","iopub.status.idle":"2021-09-30T07:31:29.658311Z","shell.execute_reply.started":"2021-09-30T07:31:29.647609Z","shell.execute_reply":"2021-09-30T07:31:29.657343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X, y, trainidt = get_all_data_for_train('T1wCE')\nX_test, testidt = get_all_data_for_test('T1wCE')\nX.shape, y.shape, trainidt.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-30T07:31:29.659953Z","iopub.execute_input":"2021-09-30T07:31:29.660666Z","iopub.status.idle":"2021-09-30T07:33:47.38949Z","shell.execute_reply.started":"2021-09-30T07:31:29.660624Z","shell.execute_reply":"2021-09-30T07:33:47.388701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape, y.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-30T07:33:47.390668Z","iopub.execute_input":"2021-09-30T07:33:47.391006Z","iopub.status.idle":"2021-09-30T07:33:47.398993Z","shell.execute_reply.started":"2021-09-30T07:33:47.390954Z","shell.execute_reply":"2021-09-30T07:33:47.398005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid, trainidt_train, trainidt_valid = train_test_split(X, y, trainidt, test_size=0.2, random_state=40)\n\nsplit = int(X.shape[0] * 0.8)\n\n# X_train = X[:split]\n# X_valid = X[split:]\n\n# y_train = y[:split]\n# y_valid = y[split:]\n\n# trainidt_train = trainidt[:split]\n# trainidt_valid = trainidt[split:]\n\nX_train = tf.expand_dims(X_train, axis=-1)\nX_valid = tf.expand_dims(X_valid, axis=-1)\n\ny_train = to_categorical(y_train)\ny_valid = to_categorical(y_valid)\n\nX_train.shape, y_train.shape, X_valid.shape, y_valid.shape, trainidt_train.shape, trainidt_valid.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-30T07:33:47.400397Z","iopub.execute_input":"2021-09-30T07:33:47.400906Z","iopub.status.idle":"2021-09-30T07:33:50.140238Z","shell.execute_reply.started":"2021-09-30T07:33:47.400865Z","shell.execute_reply":"2021-09-30T07:33:50.139405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from __future__ import print_function\n\nimport numpy as np\nfrom keras import layers\n\nfrom keras.layers import Input\nfrom keras.layers import Dense,Conv2D,MaxPooling2D,ZeroPadding2D,AveragePooling2D\nfrom keras.layers import Activation,BatchNormalization,Flatten\nfrom keras.models import Model\n\nfrom keras.preprocessing import image\nimport keras.backend as K\nfrom keras.utils.data_utils import get_file\nfrom keras.applications.imagenet_utils import decode_predictions\nfrom keras.applications.imagenet_utils import preprocess_input\n\n\ndef identity_block(input_tensor, kernel_size, filters, stage, block):\n\n    filters1, filters2, filters3 = filters\n\n    conv_name_base = 'res' + str(stage) + block + '_branch'\n    bn_name_base = 'bn' + str(stage) + block + '_branch'\n\n    x = Conv2D(filters1, (1, 1), name=conv_name_base + '2a')(input_tensor)\n    x = BatchNormalization(name=bn_name_base + '2a')(x)\n    x = Activation('relu')(x)\n\n    x = Conv2D(filters2, kernel_size,padding='same', name=conv_name_base + '2b')(x)\n\n    x = BatchNormalization(name=bn_name_base + '2b')(x)\n    x = Activation('relu')(x)\n\n    x = Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x)\n    x = BatchNormalization(name=bn_name_base + '2c')(x)\n\n    x = layers.add([x, input_tensor])\n    x = Activation('relu')(x)\n    return x\n\n\ndef conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)):\n\n    filters1, filters2, filters3 = filters\n\n    conv_name_base = 'res' + str(stage) + block + '_branch'\n    bn_name_base = 'bn' + str(stage) + block + '_branch'\n\n    x = Conv2D(filters1, (1, 1), strides=strides,\n               name=conv_name_base + '2a')(input_tensor)\n    x = BatchNormalization(name=bn_name_base + '2a')(x)\n    x = Activation('relu')(x)\n\n    x = Conv2D(filters2, kernel_size, padding='same',\n               name=conv_name_base + '2b')(x)\n    x = BatchNormalization(name=bn_name_base + '2b')(x)\n    x = Activation('relu')(x)\n\n    x = Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x)\n    x = BatchNormalization(name=bn_name_base + '2c')(x)\n\n    shortcut = Conv2D(filters3, (1, 1), strides=strides,\n                      name=conv_name_base + '1')(input_tensor)\n    shortcut = BatchNormalization(name=bn_name_base + '1')(shortcut)\n\n    x = layers.add([x, shortcut])\n    x = Activation('relu')(x)\n    return x\n\n\ndef ResNet50(input_shape=X_train.get_shape().as_list()[1:],classes=2):\n\n    img_input = Input(shape=input_shape)\n    x = ZeroPadding2D((3, 3))(img_input)\n\n    x = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(x)\n    x = BatchNormalization(name='bn_conv1')(x)\n    x = Activation('relu')(x)\n    x = MaxPooling2D((3, 3), strides=(2, 2))(x)\n\n    x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))\n    x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')\n    x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')\n\n    x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')\n    x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')\n    x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')\n    x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')\n\n    x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')\n    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')\n    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')\n    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')\n    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')\n    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')\n\n    x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')\n    x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')\n    x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')\n\n    x = AveragePooling2D((7, 7), name='avg_pool')(x)\n\n    x = Flatten()(x)\n    x = Dense(256, activation = 'relu')(x)\n    x = Dense(128, activation = 'relu')(x)\n    x = Dense(classes, activation='softmax', name='fc1000')(x)\n\n    model = Model(img_input, x, name='resnet50')\n\n    #model.load_weights(\"../input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels.h5\")\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-09-30T07:37:46.531252Z","iopub.execute_input":"2021-09-30T07:37:46.531698Z","iopub.status.idle":"2021-09-30T07:37:46.560752Z","shell.execute_reply.started":"2021-09-30T07:37:46.531634Z","shell.execute_reply":"2021-09-30T07:37:46.559704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ResNet50()\n\ncheckpoint_filepath = 'best_model.h5'\nmodel_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\nfilepath=checkpoint_filepath,\nsave_weights_only=False,\nmonitor='val_auc',\nmode='max',\nsave_best_only=True,\nsave_freq='epoch')\n\nmodel.compile(loss='categorical_crossentropy',\n             optimizer='adam',\n             metrics=[tf.keras.metrics.AUC()])\n\nhistory = model.fit(x=X_train, y = y_train, epochs=50, callbacks=[model_checkpoint_callback], validation_data= (X_valid, y_valid))","metadata":{"execution":{"iopub.status.busy":"2021-09-30T07:40:12.076413Z","iopub.execute_input":"2021-09-30T07:40:12.076739Z","iopub.status.idle":"2021-09-30T07:41:48.144441Z","shell.execute_reply.started":"2021-09-30T07:40:12.076709Z","shell.execute_reply":"2021-09-30T07:41:48.141663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_best = tf.keras.models.load_model(filepath=checkpoint_filepath)","metadata":{"execution":{"iopub.status.busy":"2021-09-30T07:33:54.381304Z","iopub.status.idle":"2021-09-30T07:33:54.381986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model_best.predict(X_valid)\n\npred = np.argmax(y_pred, axis=1)\n\nresult=pd.DataFrame(trainidt_valid)\nresult[1]=pred\n\nresult.columns=['BraTS21ID','MGMT_value']\nresult2 = result.groupby('BraTS21ID',as_index=False).mean()\n\nresult2 = result2.merge(train_df, on='BraTS21ID')\nroc_auc_score(result2.MGMT_value_y, result2.MGMT_value_x,)","metadata":{"execution":{"iopub.status.busy":"2021-09-30T07:33:54.383246Z","iopub.status.idle":"2021-09-30T07:33:54.383879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(result2)","metadata":{"execution":{"iopub.status.busy":"2021-09-30T07:33:54.385Z","iopub.status.idle":"2021-09-30T07:33:54.38564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pd.read_csv('../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv')\n\ny_pred = model_best.predict(X_test)\n\npred = np.argmax(y_pred, axis=1)\n\nresult=pd.DataFrame(testidt)\nresult[1]=pred\n\nresult.columns=['BraTS21ID','MGMT_value']\nresult2 = result.groupby('BraTS21ID',as_index=False).mean()\nresult2['BraTS21ID'] = sample['BraTS21ID']\nresult2['MGMT_value'] = result2['MGMT_value'].apply(lambda x:round(x*10)/10)\nresult2.to_csv('submission.csv',index=False)\nresult2","metadata":{"execution":{"iopub.status.busy":"2021-09-30T07:33:54.386877Z","iopub.status.idle":"2021-09-30T07:33:54.387522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}