{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I'm enhancing this notebook bto use ensemble learning with random seeds.\n\nThe idea is,to generate different models by setting the random seed.  Many people use random=42, for example. Using different values will give slightly different results.\n\n5 produced better results\n10 produced worse results.\n\nI will try other variations after first implementing CV.\n\n***\nOriginal author:\n\nI'm reading through several existing notebooks and trying to distill down the information into a new notebook to help me understand the project.  All help appreciated!\n\n# References\n\n- [Advanced EDA - Brain Tumor Data](https://www.kaggle.com/smoschou55/advanced-eda-brain-tumor-data)\n- [Team 9 Second Week](https://www.kaggle.com/evanyao27/team-9-second-week)\n  - The only model that is working. get_model02()\n- [Dataset to Model with Tensorflow](https://www.kaggle.com/ohbewise/dataset-to-model-with-tensorflow)\n- [Brain Tumer Train Class Flair](https://www.kaggle.com/lucamtb/brain-tumer-train-class-flair)\n  - Uses TPU\n  - Generates a Tensorflow model: Brain_flair_model_effect_3e-05_0.0001.h5\n- [Brain Tumor very basic inference](https://www.kaggle.com/lucamtb/brain-tumor-very-basice-inference)\n  - Uses the above mentioned model: Brain_flair_model_effect_3e-05_0.0001.h5\n  - Add this Kaggle Dataset: https://www.kaggle.com/lucamtb/effect0-brain","metadata":{}},{"cell_type":"markdown","source":"# Versions \n- V6: Starting to add Cross Validation","metadata":{}},{"cell_type":"markdown","source":"# Load Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\n\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\n\nimport random\nfrom tqdm.notebook import tqdm\nimport pydicom # Handle MRI images\n\nimport cv2  # OpenCV - https://docs.opencv.org/master/d6/d00/tutorial_py_root.html\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import model_selection\nfrom sklearn.metrics import roc_auc_score\n\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras import layers\n","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:33:21.879012Z","iopub.execute_input":"2021-10-05T14:33:21.879318Z","iopub.status.idle":"2021-10-05T14:33:21.887575Z","shell.execute_reply.started":"2021-10-05T14:33:21.879286Z","shell.execute_reply":"2021-10-05T14:33:21.88628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration, Constants, Setup","metadata":{}},{"cell_type":"code","source":"data_dir = Path('../input/rsna-miccai-brain-tumor-radiogenomic-classification/')\n\nmri_types = [\"FLAIR\", \"T1w\", \"T2w\", \"T1wCE\"]\nexcluded_images = [109, 123, 709] # Bad images\n\nSEED_LENGTH = 10 # Number of seeds to use for the model ensemble","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:33:21.88956Z","iopub.execute_input":"2021-10-05T14:33:21.890454Z","iopub.status.idle":"2021-10-05T14:33:21.903676Z","shell.execute_reply.started":"2021-10-05T14:33:21.890409Z","shell.execute_reply":"2021-10-05T14:33:21.902514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Datasets","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(data_dir / \"train_labels.csv\",\n#                        index='id',\n#                       nrows=100000\n                      )\ntest_df = pd.read_csv(data_dir / \"sample_submission.csv\")\nsample_submission = pd.read_csv(data_dir / \"sample_submission.csv\")\n\ntrain_df = train_df[~train_df.BraTS21ID.isin(excluded_images)].reset_index()\n\nprint(f\"train data: Rows={train_df.shape[0]}, Columns={train_df.shape[1]}\")\n# print(f\"test data : Rows={test_df.shape[0]}, Columns={test_df.shape[1]}\")","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:33:21.906052Z","iopub.execute_input":"2021-10-05T14:33:21.906852Z","iopub.status.idle":"2021-10-05T14:33:21.936384Z","shell.execute_reply.started":"2021-10-05T14:33:21.906804Z","shell.execute_reply":"2021-10-05T14:33:21.935163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"K-Fold Sources:\n- https://www.kaggle.com/abhishek/same-old-creating-folds\n- https://www.kaggle.com/abhishek/30-days-create-folds","metadata":{}},{"cell_type":"code","source":"def create_folds(data, num_splits):\n    data[\"kfold\"] = -1\n    kf = model_selection.KFold(n_splits=num_splits, shuffle=True, random_state=42)\n    for f, (t, v) in enumerate(kf.split(X=data)):\n        data.loc[v, \"kfold\"] = f\n    return data\n","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:33:21.938212Z","iopub.execute_input":"2021-10-05T14:33:21.938665Z","iopub.status.idle":"2021-10-05T14:33:21.947061Z","shell.execute_reply.started":"2021-10-05T14:33:21.938619Z","shell.execute_reply":"2021-10-05T14:33:21.945636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_kf = create_folds(train_df, 5)","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:33:21.949544Z","iopub.execute_input":"2021-10-05T14:33:21.950842Z","iopub.status.idle":"2021-10-05T14:33:21.970179Z","shell.execute_reply.started":"2021-10-05T14:33:21.95079Z","shell.execute_reply":"2021-10-05T14:33:21.968764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_kf.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:33:21.972015Z","iopub.execute_input":"2021-10-05T14:33:21.972378Z","iopub.status.idle":"2021-10-05T14:33:21.995436Z","shell.execute_reply.started":"2021-10-05T14:33:21.972332Z","shell.execute_reply":"2021-10-05T14:33:21.994276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Functions","metadata":{}},{"cell_type":"markdown","source":"### There's a version that converts into grayscale: \n\n- https://www.kaggle.com/smoschou55/advanced-eda-brain-tumor-data\n","metadata":{}},{"cell_type":"code","source":"def load_dicom(path, size = 224):\n    ''' \n    Reads a DICOM image, standardizes so that the pixel values are between 0 and 1, then rescales to 0 and 255\n    \n    Not super sure if this kind of scaling is appropriate, but everyone seems to do it. \n    '''\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    # transform data into black and white scale / grayscale\n#     data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return cv2.resize(data, (size, size))","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:33:21.999419Z","iopub.execute_input":"2021-10-05T14:33:22.001151Z","iopub.status.idle":"2021-10-05T14:33:22.009401Z","shell.execute_reply.started":"2021-10-05T14:33:22.001098Z","shell.execute_reply":"2021-10-05T14:33:22.008232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_all_image_paths(brats21id, image_type, folder='train'): \n    '''\n    Returns an arry of all the images of a particular type for a particular patient ID\n    '''\n    assert(image_type in mri_types)\n    \n    patient_path = os.path.join(\n        \"../input/rsna-miccai-brain-tumor-radiogenomic-classification/%s/\" % folder, \n        str(brats21id).zfill(5),\n    )\n\n    paths = sorted(\n        glob.glob(os.path.join(patient_path, image_type, \"*\")), \n        key=lambda x: int(x[:-4].split(\"-\")[-1]),\n    )\n    \n    num_images = len(paths)\n    \n    start = int(num_images * 0.25)\n    end = int(num_images * 0.75)\n\n    interval = 3\n    \n    if num_images < 10: \n        interval = 1\n    \n    return np.array(paths[start:end:interval])\n\ndef get_all_images(brats21id, image_type, folder='train', size=225):\n    return [load_dicom(path, size) for path in get_all_image_paths(brats21id, image_type, folder)]","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:33:22.011352Z","iopub.execute_input":"2021-10-05T14:33:22.012649Z","iopub.status.idle":"2021-10-05T14:33:22.026285Z","shell.execute_reply.started":"2021-10-05T14:33:22.012603Z","shell.execute_reply":"2021-10-05T14:33:22.024935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Images We Will Need","metadata":{}},{"cell_type":"code","source":"def get_all_data_for_train(image_type, image_size=32):\n    global train_df\n    \n    X = []\n    y = []\n    train_ids = []\n\n    for i in tqdm(train_df.index):\n        x = train_df.loc[i]\n        images = get_all_images(int(x['BraTS21ID']), image_type, 'train', image_size)\n        label = x['MGMT_value']\n\n        X += images\n        y += [label] * len(images)\n        train_ids += [int(x['BraTS21ID'])] * len(images)\n        assert(len(X) == len(y))\n    return np.array(X), np.array(y), np.array(train_ids)","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:33:22.028042Z","iopub.execute_input":"2021-10-05T14:33:22.029022Z","iopub.status.idle":"2021-10-05T14:33:22.040821Z","shell.execute_reply.started":"2021-10-05T14:33:22.028976Z","shell.execute_reply":"2021-10-05T14:33:22.039331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_all_data_for_test(image_type, image_size=32):\n    global test_df\n    \n    X = []\n    test_ids = []\n\n    for i in tqdm(test_df.index):\n        x = test_df.loc[i]\n        images = get_all_images(int(x['BraTS21ID']), image_type, 'test', image_size)\n        X += images\n        test_ids += [int(x['BraTS21ID'])] * len(images)\n\n    return np.array(X), np.array(test_ids)","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:33:22.042926Z","iopub.execute_input":"2021-10-05T14:33:22.043648Z","iopub.status.idle":"2021-10-05T14:33:22.058561Z","shell.execute_reply.started":"2021-10-05T14:33:22.0436Z","shell.execute_reply":"2021-10-05T14:33:22.057276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X, y, trainidt = get_all_data_for_train('T1wCE', image_size=32)\nX_test, testidt = get_all_data_for_test('T1wCE', image_size=32)","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:33:22.062997Z","iopub.execute_input":"2021-10-05T14:33:22.063322Z","iopub.status.idle":"2021-10-05T14:34:36.661999Z","shell.execute_reply.started":"2021-10-05T14:33:22.063288Z","shell.execute_reply":"2021-10-05T14:34:36.6608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape, y.shape, trainidt.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:34:36.666484Z","iopub.execute_input":"2021-10-05T14:34:36.666776Z","iopub.status.idle":"2021-10-05T14:34:36.673772Z","shell.execute_reply.started":"2021-10-05T14:34:36.666728Z","shell.execute_reply":"2021-10-05T14:34:36.672569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train/Validation Split","metadata":{}},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid, trainidt_train, trainidt_valid = train_test_split(X, y, trainidt, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:34:36.67599Z","iopub.execute_input":"2021-10-05T14:34:36.676653Z","iopub.status.idle":"2021-10-05T14:34:36.69709Z","shell.execute_reply.started":"2021-10-05T14:34:36.676605Z","shell.execute_reply":"2021-10-05T14:34:36.696132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train_df","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:34:36.698586Z","iopub.execute_input":"2021-10-05T14:34:36.699014Z","iopub.status.idle":"2021-10-05T14:34:36.705046Z","shell.execute_reply.started":"2021-10-05T14:34:36.698969Z","shell.execute_reply":"2021-10-05T14:34:36.703793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in range(5):\n    xtrain =  df[df.kfold != fold].reset_index(drop=True)\n    xvalid = df[df.kfold == fold].reset_index(drop=True)\n#     print(xtrain)\n#     print(f\"xvalid {xvalid}\")\n#     xtest = df_test.copy()\n\n#     ytrain = xtrain.target\n#     yvalid = xvalid.target\n       \n  \n#     model = XGBRegressor(random_state=42, tree_method='gpu_hist', gpu_id=0, predictor=\"gpu_predictor\")\n#     model.fit(xtrain, ytrain)\n#     preds_valid = model.predict(xvalid)\n#     test_preds = model.predict(xtest)\n#     final_predictions.append(test_preds)\n#     rmse = mean_squared_error(yvalid, preds_valid, squared=False)\n#     print(fold, rmse)\n#     scores.append(rmse)\n\n# print(np.mean(scores), np.std(scores))","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:34:36.707304Z","iopub.execute_input":"2021-10-05T14:34:36.707671Z","iopub.status.idle":"2021-10-05T14:34:36.726236Z","shell.execute_reply.started":"2021-10-05T14:34:36.707624Z","shell.execute_reply":"2021-10-05T14:34:36.72524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Add dimension","metadata":{}},{"cell_type":"code","source":"X_train = tf.expand_dims(X_train, axis=-1)\nX_valid = tf.expand_dims(X_valid, axis=-1)","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:34:36.72824Z","iopub.execute_input":"2021-10-05T14:34:36.728606Z","iopub.status.idle":"2021-10-05T14:34:39.219595Z","shell.execute_reply.started":"2021-10-05T14:34:36.728561Z","shell.execute_reply":"2021-10-05T14:34:39.218514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## One-hot encode labels","metadata":{}},{"cell_type":"code","source":"y_train = to_categorical(y_train)\ny_valid = to_categorical(y_valid)","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:34:39.221962Z","iopub.execute_input":"2021-10-05T14:34:39.222625Z","iopub.status.idle":"2021-10-05T14:34:39.229026Z","shell.execute_reply.started":"2021-10-05T14:34:39.222578Z","shell.execute_reply":"2021-10-05T14:34:39.227978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tensorflow Models","metadata":{}},{"cell_type":"markdown","source":"## Model from:  https://www.kaggle.com/ohbewise/dataset-to-model-with-tensorflow","metadata":{}},{"cell_type":"code","source":"# Define, train, and evaluate model\n# source: https://keras.io/examples/vision/3D_image_classification/\ndef get_model01(width=128, height=128, depth=64, name='3dcnn'):\n    \"\"\"Build a 3D convolutional neural network model.\"\"\"\n\n    inputs = tf.keras.Input((width, height, depth, 1))\n\n    x = tf.keras.layers.Conv3D(filters=64, kernel_size=3, activation=\"relu\")(inputs)\n    x = tf.keras.layers.MaxPool3D(pool_size=2)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n\n    x = tf.keras.layers.Conv3D(filters=64, kernel_size=3, activation=\"relu\")(x)\n    x = tf.keras.layers.MaxPool3D(pool_size=2)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n\n    x = tf.keras.layers.Conv3D(filters=128, kernel_size=3, activation=\"relu\")(x)\n    x = tf.keras.layers.MaxPool3D(pool_size=2)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n\n    x = tf.keras.layers.Conv3D(filters=256, kernel_size=3, activation=\"relu\")(x)\n    x = tf.keras.layers.MaxPool3D(pool_size=2)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n\n    x = tf.keras.layers.GlobalAveragePooling3D()(x)\n    x = tf.keras.layers.Dense(units=512, activation=\"relu\")(x)\n    x = tf.keras.layers.Dropout(0.3)(x)\n\n    outputs = tf.keras.layers.Dense(units=1, activation=\"sigmoid\")(x)\n\n    # Define the model.\n    model = tf.keras.Model(inputs, outputs, name=name)\n    \n    # Compile model.\n    initial_learning_rate = 0.0001\n    lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(\n        initial_learning_rate, decay_steps=100000, decay_rate=0.96, staircase=True\n    )\n    model.compile(\n        loss=\"binary_crossentropy\",\n        optimizer=tf.keras.optimizers.Adam(learning_rate=lr_schedule),\n        metrics=[\"acc\"],\n    )\n    \n    return model\n\n","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:34:39.230925Z","iopub.execute_input":"2021-10-05T14:34:39.231476Z","iopub.status.idle":"2021-10-05T14:34:39.251031Z","shell.execute_reply.started":"2021-10-05T14:34:39.231426Z","shell.execute_reply":"2021-10-05T14:34:39.249615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model from: https://www.kaggle.com/evanyao27/team-9-second-week/notebook\n\n- Validation AUC=0.9148664856146349","metadata":{}},{"cell_type":"code","source":"def get_model02(seed=42, activation=\"relu\"):\n    np.random.seed(seed)\n    random.seed(seed)\n    tf.random.set_seed(seed)\n\n    inpt = keras.Input(shape=X_train.shape[1:])\n\n    h = keras.layers.experimental.preprocessing.Rescaling(1.0 / 255)(inpt)\n\n    h = keras.layers.Conv2D(64, kernel_size=(4, 4), activation=activation, name=\"Conv_1\")(h)\n    h = keras.layers.MaxPool2D(pool_size=(2, 2))(h)\n\n    h = keras.layers.Conv2D(32, kernel_size=(2, 2), activation=activation, name=\"Conv_2\")(h)\n    h = keras.layers.MaxPool2D(pool_size=(1, 1))(h)\n\n    h = keras.layers.Dropout(0.1)(h)\n\n    h = keras.layers.Flatten()(h)\n    h = keras.layers.Dense(32, activation=activation)(h)\n\n    output = keras.layers.Dense(2, activation=\"softmax\")(h)\n\n    model = keras.Model(inpt, output)\n    \n    initial_learning_rate = 0.0001\n    lr_schedule = keras.optimizers.schedules.ExponentialDecay(\n        initial_learning_rate, decay_steps=100000, decay_rate=0.96, staircase=True\n    )\n    model.compile(\n        loss=\"categorical_crossentropy\", optimizer=keras.optimizers.Adam(learning_rate=lr_schedule), metrics=[tf.keras.metrics.AUC()]\n    )\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:34:39.253206Z","iopub.execute_input":"2021-10-05T14:34:39.253645Z","iopub.status.idle":"2021-10-05T14:34:39.268043Z","shell.execute_reply.started":"2021-10-05T14:34:39.253599Z","shell.execute_reply":"2021-10-05T14:34:39.266906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set up Model Checkpoint","metadata":{}},{"cell_type":"markdown","source":"### Set a couple of seeds for ensemble","metadata":{}},{"cell_type":"code","source":"def get_seed_list(low=0, high=1000, length=5):\n    np.random.seed(42)\n    return np.random.randint(low=low, high=high, size=length)","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:34:39.270305Z","iopub.execute_input":"2021-10-05T14:34:39.27101Z","iopub.status.idle":"2021-10-05T14:34:39.28518Z","shell.execute_reply.started":"2021-10-05T14:34:39.270965Z","shell.execute_reply":"2021-10-05T14:34:39.284088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_list = get_seed_list(length=SEED_LENGTH)\nseed_list","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:34:39.287068Z","iopub.execute_input":"2021-10-05T14:34:39.287498Z","iopub.status.idle":"2021-10-05T14:34:39.300778Z","shell.execute_reply.started":"2021-10-05T14:34:39.287452Z","shell.execute_reply":"2021-10-05T14:34:39.299749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Note that rerunning the cell below will change val_acc to val_acc_N and the model will not be saved.","metadata":{}},{"cell_type":"code","source":"# Define early stopping callback.\nearly_stopping_cb = keras.callbacks.EarlyStopping(monitor=\"val_auc\", mode='max', verbose=1, patience=3)\nfor idx, seed in enumerate(seed_list):\n    tf.keras.backend.clear_session()\n    # Set up Model Checkpoint\n\n    checkpoint_filepath = f\"best_model_{seed}.h5\"\n\n    checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(\n        filepath=checkpoint_filepath,\n        save_weights_only=False,\n        monitor=\"val_auc\",\n        mode=\"max\",\n        save_best_only=True,\n        save_freq=\"epoch\",\n        verbose=1,\n    )\n\n    model = get_model02(seed=seed)\n\n    history = model.fit(x=X_train, y = y_train, epochs=200, callbacks=[checkpoint_cb, early_stopping_cb], validation_data= (X_valid, y_valid))","metadata":{"execution":{"iopub.status.busy":"2021-10-05T14:34:39.30302Z","iopub.execute_input":"2021-10-05T14:34:39.303788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Our Best Models","metadata":{}},{"cell_type":"code","source":"best_models = []\nfor seed in seed_list:\n    best_models.append(tf.keras.models.load_model(filepath=f\"best_model_{seed}.h5\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions on Validation Set","metadata":{}},{"cell_type":"code","source":"for idx, model in enumerate(best_models):\n    y_pred = model.predict(X_valid)\n\n    pred = np.argmax(y_pred, axis=1)\n\n    result = pd.DataFrame(trainidt_valid)\n    result[1] = pred\n\n    result.columns = [\"BraTS21ID\", \"MGMT_value\"]\n    result2 = result.groupby(\"BraTS21ID\", as_index=False).mean()\n\n    result2 = result2.merge(train_df, on=\"BraTS21ID\")\n    auc = roc_auc_score(\n        result2.MGMT_value_y,\n        result2.MGMT_value_x,\n    )\n    print(f\"Validation AUC of model {idx+1} = {auc}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions on the Test Set","metadata":{}},{"cell_type":"code","source":"y_pred = best_models[0].predict(X_test)\n\npred = np.argmax(y_pred, axis=1) #\nresult = pd.DataFrame(testidt)\n\nresult[1] = pred\npred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = pd.DataFrame(testidt)\nfor idx, model in enumerate(best_models):\n    y_pred = model.predict(X_test)\n\n    pred = np.argmax(y_pred, axis=1) #\n\n    result[idx+1] = pred\nresult","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission File","metadata":{}},{"cell_type":"code","source":"result.columns=['BraTS21ID'] + [f\"MGMT_value_{idx+1}\" for idx, _ in enumerate(best_models)]\n# for idx, _ in enumerate(best_models):\n\nresult2 = result.groupby('BraTS21ID',as_index=False).mean()\nresult2['BraTS21ID'] = sample_submission['BraTS21ID']\n\n# Rounding...\nsample_submission['MGMT_value'] = result2[[f\"MGMT_value_{idx+1}\" for idx, _ in enumerate(best_models)]].mean(axis=1)\n# sample_submission['MGMT_value'] = sample_submission['MGMT_value'].apply(lambda x:round(x*10)/10)\nsample_submission.to_csv('submission.csv',index=False)\nsample_submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}