{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Importar libreria**\n\nCargar todos los paquetes que se necesitan ","metadata":{}},{"cell_type":"code","source":"import math\nimport pydicom\nimport cv2\nimport torch\nimport cupy as cp\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom glob import glob\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport random\nfrom random import sample\nimport sklearn.model_selection as skl\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom keras_preprocessing.image.dataframe_iterator import DataFrameIterator\nfrom tensorflow.keras.applications import EfficientNetB3\nfrom tensorflow.keras.layers import InputLayer, GlobalAveragePooling2D, BatchNormalization, Dense, Dropout, Flatten, Conv2D, MaxPooling2D \nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.metrics import AUC\nfrom tensorflow.keras.optimizers import RMSprop\nimport tensorflow_hub as tfhub\nfrom tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nfrom keras_preprocessing.image.dataframe_iterator import DataFrameIterator\n\n# Confirmar si tenemos activa la GPU\nif tf.test.gpu_device_name():\n    print('GPU disponible')\nelse:\n    print('No se encontró GPU')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Cargar base de datos**","metadata":{}},{"cell_type":"code","source":"#Enlace de la base de datos\nbase_datos = '/kaggle/input/rsna-miccai-brain-tumor-radiogenomic-classification/'\nos.listdir(base_datos)\ntrain = base_datos+'train'\ntest  = base_datos+'test'\ntrain_csv = pd.read_csv(base_datos+'train_labels.csv')\ntest_csv  = pd.read_csv(base_datos+'sample_submission.csv')\nprint(train_csv.head(10)),print(test_csv.head(10))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Comprensión de los datos**","metadata":{}},{"cell_type":"markdown","source":"Análisis de valores NAN en el CSV","metadata":{}},{"cell_type":"code","source":"# obtener el número de puntos de datos faltantes por columna\nvalores_faltantes_train = train_csv.isnull().sum()\n\n# mira el # de puntos que faltan en las primeras diez columnas\nvalores_faltantes_train[0:2]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"La base de datos (train_labels.csv), no presenta valores NAN","metadata":{}},{"cell_type":"markdown","source":"# Comprensión de los datos (Imagenes)\n","metadata":{}},{"cell_type":"code","source":"'''\n#Imprime la cantidad de imágenes dentro de cada técnica de IRM\ndef contar_imagenes(ruta_principal, nombre_carpeta):\n    # Crear una lista para almacenar el recuento de imágenes por carpeta\n    conteos = []\n    # Inicializar el contador de imágenes\n    conteo_total = 0\n\n    # Recorrer cada carpeta en la ruta principal\n    for carpeta in os.listdir(ruta_principal):\n        ruta_carpeta = os.path.join(ruta_principal, carpeta)\n\n        # Verificar si es una carpeta\n        if os.path.isdir(ruta_carpeta):\n            # Verificar si tiene una carpeta con el nombre especificado\n            ruta_carpeta_especificada = os.path.join(ruta_carpeta, nombre_carpeta)\n            if os.path.isdir(ruta_carpeta_especificada):\n                # Contar el número de archivos DICOM en la carpeta especificada\n                conteo = len([nombre_archivo for nombre_archivo in os.listdir(ruta_carpeta_especificada) if nombre_archivo.endswith(\".dcm\")])\n                # Agregar el recuento a la lista\n                conteos.append((carpeta, conteo))\n                # Agregar el recuento al contador total\n                conteo_total += conteo\n    \n    # Imprimir los resultados\n    #for carpeta, conteo in conteos:\n        #print(f\"{carpeta}: {conteo} imágenes DICOM en {nombre_carpeta}\")\n        #print(f\"{conteo}\")\n        #print(f\"Total de imágenes en {nombre_carpeta}: {conteo_total}\")\n        # Devolver la lista de conteos y el recuento total\n    return conteos, conteo_total\nnombre_carpeta = \"T2w\"  #Tecnicas T1w, T2w, FLAIR y T1wCE\n#conteos, conteo_total = contar_imagenes(ruta_principal, nombre_carpeta)\nconteos, conteo_total = contar_imagenes(train, nombre_carpeta)\nprint(f\"El número total de imágenes DICOM en {nombre_carpeta} es {conteo_total}\")\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Entra a todo los pacientes a la técnica de IRM que se le indica y analiza las imágenes\n# DICOM si existen imágenes sin información y sacar un CSV con la ubicación de imagnes\n# sin información \n'''\ndef comprension_imagenes(carpeta_principal, tecnica):\n    pacientes = []\n    num_imagenes_negras = []\n    ubicaciones_imagenes_negras = []\n    contador = 0\n    for paciente in os.listdir(carpeta_principal):\n        carpeta_paciente = os.path.join(carpeta_principal, paciente, tecnica)\n        if os.path.isdir(carpeta_paciente):\n            num_imagenes = 0\n            ubicaciones_imagenes = []\n            for imagen in os.listdir(carpeta_paciente):\n                archivo_imagen = os.path.join(carpeta_paciente, imagen)\n                ds = pydicom.dcmread(archivo_imagen)\n                pixel_array_gpu = cp.array(ds.pixel_array)\n                if pixel_array_gpu.sum() == 0:\n                    num_imagenes += 1\n                    ubicaciones_imagenes.append(archivo_imagen)\n                    # print(f\"Procesando archivo: {archivo_imagen}\")\n            pacientes.append(paciente)\n            num_imagenes_negras.append(num_imagenes)\n            ubicaciones_imagenes_negras.append(', '.join(ubicaciones_imagenes))\n            contador += 1\n            print(f\"Contador: {contador} - Paciente: {paciente}\")\n    data = {'ID del paciente': pacientes,\n            'Técnica': [tecnica]*len(pacientes),\n            'Número de imágenes en negro': num_imagenes_negras,\n            'Ubicaciones de las imágenes en negro': ubicaciones_imagenes_negras}\n    df = pd.DataFrame(data)\n    df.to_csv(f'{tecnica}.csv', index=False)\n    \ncomprension_imagenes(train, 'FLAIR')\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Agregue las rutas completas para cada id para diferentes tipos de secuencias al csv\n\ndef full_ids(data):\n    zeros = 5 - len(str(data))\n    if zeros > 0:\n        prefix = ''.join(['0' for i in range(zeros)])\n        return prefix+str(data)\n     \ntrain_csv['BraTS21ID_full'] = train_csv['BraTS21ID'].apply(full_ids)\n# Add all the paths to the df for easy access\ntrain_csv['flair'] = train_csv['BraTS21ID_full'].apply(lambda file_id : base_datos+'train/'+file_id+'/FLAIR/')\ntrain_csv['t1w'] = train_csv['BraTS21ID_full'].apply(lambda file_id : base_datos+'train/'+file_id+'/T1w/')\ntrain_csv['t1wce'] = train_csv['BraTS21ID_full'].apply(lambda file_id : base_datos+'train/'+file_id+'/T1wCE/')\ntrain_csv['t2w'] = train_csv['BraTS21ID_full'].apply(lambda file_id : base_datos+'train/'+file_id+'/T2w/')\n#train_csv.to_csv('nuevo_train_labels.csv', index=False)\n#print(train_csv.head(585))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv['BraTS21ID_full'] = test_csv['BraTS21ID'].apply(full_ids)\n# Add all the paths to the df for easy access\ntest_csv['flair'] = test_csv['BraTS21ID_full'].apply(lambda file_id : base_datos+'test/'+file_id+'/FLAIR/')\ntest_csv['t1w'] = test_csv['BraTS21ID_full'].apply(lambda file_id : base_datos+'test/'+file_id+'/T1w/')\ntest_csv['t1wce'] = test_csv['BraTS21ID_full'].apply(lambda file_id : base_datos+'test/'+file_id+'/T1wCE/')\ntest_csv['t2w'] = test_csv['BraTS21ID_full'].apply(lambda file_id : base_datos+'test/'+file_id+'/T2w/')\n#print(test_csv.head(585))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#Divicion del 90% de los datos TRAIN para entrenar y dejar \n#el 10% para validar y organizar en un CSV\ndef get_train_val_dataframe(mri_type):  #mri_type es la tecnica: T1w ...\n    \n    all_img_files = []\n    all_img_labels = []\n    all_img_patient_ids = []\n    for row in train_csv.iterrows():\n        if row[1]['BraTS21ID_full'] == '00109' and mri_type == 'flair':\n            continue\n        if row[1]['BraTS21ID_full'] == '00123' and mri_type == 't1w':\n            continue\n        if row[1]['BraTS21ID_full'] == '00709' and mri_type == 'flair':\n            continue\n        img_dir = row[1][mri_type]\n        img_files = os.listdir(img_dir)\n        img_nums = sorted([int(ele.replace('Image-', '').replace('.dcm', '')) for ele in img_files])\n        mid_point = int(len(img_nums)/2)\n        start_point = mid_point - max(int(mid_point*0.2), 1)\n        end_point = mid_point + max(int(mid_point*0.2), 1)\n        img_names = [f'Image-{img_nums[i]}.dcm' for i in range(start_point, end_point+1)]\n        img_paths = [img_dir+ele for ele in img_names]\n        img_labels = [row[1]['MGMT_value']]*len(img_paths)\n        img_patient_ids = [row[1]['BraTS21ID']]*len(img_paths)\n        all_img_files.extend(img_paths)\n        all_img_labels.extend(img_labels)\n        all_img_patient_ids.extend(img_patient_ids)\n\n    train_val_df = pd.DataFrame({'patient_ids': all_img_patient_ids,\n                  'labels': all_img_labels,\n                  'file_paths': all_img_files})\n\n    train_val_df['labels'] = train_val_df['labels'].map({1: '1', 0: '0'})\n    \n    #stratifiied 90% split on patient_ids and labels  \n    class_prop= 0.90\n    \n    classes_splits  = {}\n    for i in range(2):\n        train_val_label_class = train_val_df[train_val_df['labels']==f'{i}']\n        train_val_list_ids =  list(train_val_label_class['patient_ids'].unique())\n        train_threshold = math.ceil(class_prop*len(train_val_list_ids))\n        train_ids = train_val_list_ids[:train_threshold]\n        val_ids = train_val_list_ids[train_threshold:]\n        classes_splits[f'train_{i}'] = train_val_label_class[train_val_label_class['patient_ids'].isin(train_ids)]\n        classes_splits[f'val_{i}'] = val_df = train_val_label_class[train_val_label_class['patient_ids'].isin(val_ids)]\n        \n    train_df = pd.concat([classes_splits['train_0'], classes_splits['train_1']], axis=0)\n    val_df = pd.concat([classes_splits['val_0'], classes_splits['val_1']], axis=0)\n  \n    return train_df, val_df\ndef get_test_dataframe(mri_type):\n    \n    all_test_img_files = []\n    all_test_img_labels = []\n    all_test_img_patient_ids = []\n    for row in test_csv.iterrows():\n        img_dir = row[1][mri_type]\n        img_files = os.listdir(img_dir)\n        img_nums = sorted([int(ele.replace('Image-', '').replace('.dcm', '')) for ele in img_files])\n        mid_point = int(len(img_nums)/2)\n        start_point = mid_point - max(int(mid_point*0.2), 1)\n        end_point = mid_point + max(int(mid_point*0.2), 1)\n        img_names = [f'Image-{img_nums[i]}.dcm' for i in range(start_point, end_point+1)]\n        img_paths = [img_dir+ele for ele in img_names]\n        img_labels = [row[1]['MGMT_value']]*len(img_paths)\n        img_patient_ids = [row[1]['BraTS21ID']]*len(img_paths)\n        all_test_img_files.extend(img_paths)\n        all_test_img_labels.extend(img_labels)\n        all_test_img_patient_ids.extend(img_patient_ids)\n\n    test_df = pd.DataFrame({'patient_ids': all_test_img_patient_ids,\n                  'labels': all_test_img_labels,\n                  'file_paths': all_test_img_files})\n    \n    test_df['labels'] = ['1']*(len(test_df)-1) + ['0'] # workaround for testing data gen\n    \n    return test_df\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DCMDataFrameIterator(DataFrameIterator):\n    def __init__(self, *arg, **kwargs):\n        self.white_list_formats = ('dcm')\n        super(DCMDataFrameIterator, self).__init__(*arg, **kwargs)\n        self.dataframe = kwargs['dataframe']\n        self.x = self.dataframe[kwargs['x_col']]\n        self.y = self.dataframe[kwargs['y_col']]\n        self.color_mode = kwargs['color_mode']\n        self.target_size = kwargs['target_size']\n\n    def _get_batches_of_transformed_samples(self, indices_array):\n        # get batch of images\n        batch_x = np.array([self.read_dcm_as_array(dcm_path, self.target_size, color_mode=self.color_mode)\n                            for dcm_path in self.x.iloc[indices_array]])\n\n        batch_y = np.array(self.y.iloc[indices_array].astype(np.uint8))  # astype because y was passed as str\n\n        # transform images\n        if self.image_data_generator is not None:\n            for i, (x, y) in enumerate(zip(batch_x, batch_y)):\n                transform_params = self.image_data_generator.get_random_transform(x.shape)\n                batch_x[i] = self.image_data_generator.apply_transform(x, transform_params)\n                # you can change y here as well, eg: in semantic segmentation you want to transform masks as well \n                # using the same image_data_generator transformations.\n\n        return batch_x, batch_y\n\n    @staticmethod\n    def read_dcm_as_array(dcm_path, target_size=(300, 300), color_mode='rgb'):\n        image_array = pydicom.dcmread(dcm_path).pixel_array\n        pixels = image_array - np.min(image_array)\n        pixels = pixels / np.max(pixels)\n        image_manual_norm = (pixels * 255).astype(np.uint8)\n        image_array = cv2.resize(image_manual_norm, target_size, interpolation=cv2.INTER_NEAREST)  #this returns a 2d array\n#         image_array = np.expand_dims(image_array, -1)\n        if color_mode == 'rgb':\n            image_array = np.dstack((image_array, np.zeros_like(image_array), np.zeros_like(image_array)))\n        return image_array\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model(weights_path):\n    \n    model = EfficientNetB3(include_top=False, weights=weights_path)\n    \n    model.trainable = False\n    \n    x = GlobalAveragePooling2D(name=\"avg_pool\")(model.output)\n    x = BatchNormalization()(x)\n\n    top_dropout_rate = 0.4\n    x = Dropout(top_dropout_rate)(x)\n    x = Dense(32, activation=\"relu\")(x)\n    x = BatchNormalization()(x)\n    x = Dropout(top_dropout_rate)(x)\n    outputs = Dense(1, activation=\"sigmoid\", name=\"pred\")(x)\n    \n\n    # Compile\n    model = Model(model.inputs, outputs, name=\"EfficientNet\")\n    \n\n    # Compile\n    optimizer =  tf.keras.optimizers.Adam(learning_rate=1e-3)\n    \n    model.compile(optimizer=optimizer, loss=\"binary_crossentropy\", metrics=[\"binary_accuracy\",AUC()])\n    return model\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 369\nBATCH_SIZE = 128\nCLASS_MODE = 'binary'\nCOLOR_MODE = 'rgb'\nTARGET_SIZE = (300, 300)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_data_generators(train_df,val_df, test_df):\n    train_augmentation_parameters = dict(\n        rescale=1.0/255,\n        zoom_range=0.2,\n        rotation_range=0.2,\n        fill_mode='nearest',\n        height_shift_range= 0.1,\n        width_shift_range=0.1,\n        horizontal_flip=True,\n        brightness_range = [0.8, 1.2]\n    )\n    \n    val_augmentation_parameters = dict(\n        rescale=1.0/255.0\n    )\n\n    test_augmentation_parameters = dict(\n        rescale=1.0/255.0\n    )\n\n    train_consts = {\n        'seed': SEED,\n        'batch_size': BATCH_SIZE,\n        'class_mode': CLASS_MODE,\n        'color_mode': COLOR_MODE,\n        'target_size': TARGET_SIZE,  \n    }\n    \n    val_consts = {\n    'batch_size': BATCH_SIZE,\n    'class_mode': CLASS_MODE,\n    'color_mode': COLOR_MODE,\n    'target_size': TARGET_SIZE,\n    'shuffle': False\n    }\n\n    test_consts = {\n        'batch_size': BATCH_SIZE,\n        'class_mode': CLASS_MODE,\n        'color_mode': COLOR_MODE,\n        'target_size': TARGET_SIZE,\n        'shuffle': False\n    }\n\n    train_augmenter = ImageDataGenerator(**train_augmentation_parameters)\n    val_augmenter = ImageDataGenerator(**val_augmentation_parameters)\n    test_augmenter = ImageDataGenerator(**test_augmentation_parameters)\n\n    train_generator = DCMDataFrameIterator(dataframe=train_df,\n                                 x_col='file_paths',\n                                 y_col='labels',\n                                 image_data_generator=train_augmenter,\n                                 **train_consts)\n    \n    val_generator = DCMDataFrameIterator(dataframe=val_df,\n                                 x_col='file_paths',\n                                 y_col='labels',\n                                 image_data_generator=val_augmenter,\n                                 **val_consts)\n    \n    test_generator = DCMDataFrameIterator(dataframe=test_df,\n                                 x_col='file_paths',\n                                 y_col='labels',\n                                 image_data_generator=test_augmenter,\n                                 **test_consts)\n    \n    return train_generator, val_generator, test_generator","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint_filepath = 'best_model.h5'\n\ndef train_model(model_name, train_generator, val_generator, epochs):\n    \n    print('training', model_name)\n    \n    model = build_model(\"../input/efficentnet-b0b5-tensorflow-24-notop/efficientnet-b3_tf24_imagenet_1000_notop.h5\")\n    \n    #weights_path = '/kaggle/input/efficientnetb3/efficientnetb3.h5'\n    #model = build_model(weights_path)\n    \n    #callbacks\n    \n    checkpoint_cb=ModelCheckpoint(\n        filepath=checkpoint_filepath,\n        save_weights_only=False,\n        monitor='val_loss',\n        mode='min',\n        save_best_only=True,\n        save_freq='epoch',\n        verbose=1)\n    \n    early_stopping_cb = tf.keras.callbacks.EarlyStopping(monitor='val_loss',\n                                                  patience=5,\n                                                  mode='min',\n                                                  verbose=1,\n                                                  restore_best_weights=True)\n\n    reduce_lr_cb=ReduceLROnPlateau(monitor='val_loss', factor=0.5,\n                                   patience=2, min_lr=0.00001,\n                                  verbose=1)\n\n    history = model.fit(\n                        train_generator,\n                        steps_per_epoch=len(train_generator),\n                        validation_data=val_generator,\n                        validation_steps=len(val_generator),\n                        epochs=epochs,\n                        workers=2,\n                        callbacks=[checkpoint_cb, reduce_lr_cb, early_stopping_cb]\n                        )\n\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n#Llamado de las 2 funciones anteriores\nall_test_preds = []\nfor mt in ['flair', 't1w', 't1wce', 't2w']:\n    train_df, val_df =    get_train_val_dataframe(mt)\n    test_df          =    get_test_dataframe(mt)\n    \n    train_g, val_g, test_g = get_data_generators(train_df, val_df, test_df)\n    best_model =  train_model(mt, train_g, val_g, epochs=5)\n    results = best_model.evaluate(test_g, steps=len(test_g))\n    print(f\"test loss, test acc, test AUC: {results}\")\n    test_pred = best_model.predict(test_g, steps=len(test_g))\n    test_df['pred_y'] = test_pred\n    #aggregate the predictions on all image for each person (take the most confident prediction out of all image predictions)\n    mean_pred = test_pred.mean()\n    test_pred_agg = test_df.groupby('patient_ids').apply(\n        lambda x: x['pred_y'].max()\n        if (x['pred_y'].max() - mean_pred) > (mean_pred - x['pred_y'].min()) \n        else x['pred_y'].min())\n    all_test_preds.append(test_pred_agg.values)\n    '''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n#Llamado de las 2 funciones anteriores\nall_test_preds = []\nfor mt in ['flair', 't1w', 't1wce', 't2w']:\n    train_df, val_df =    get_train_val_dataframe(mt)\n    test_df          =    get_test_dataframe(mt)\n    \n    train_g, val_g, test_g = get_data_generators(train_df, val_df, test_df)\n    best_model =  train_model(mt, train_g, val_g, epochs=5)\n    results = best_model.evaluate(test_g, steps=len(test_g))\n    print(f\"test loss, test acc, test AUC: {results}\")\n    test_pred = best_model.predict(test_g, steps=len(test_g))\n    test_df['pred_y'] = test_pred\n    #aggregate the predictions on all image for each person (take the most confident prediction out of all image predictions)\n    mean_pred = test_pred.mean()\n    test_pred_agg = test_df.groupby('patient_ids').apply(\n        lambda x: x['pred_y'].max()\n        if (x['pred_y'].max() - mean_pred) > (mean_pred - x['pred_y'].min()) \n        else x['pred_y'].min())\n    all_test_preds.append(test_pred_agg.values)\n    '''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sklearn.metrics\nimport numpy as np\n\nall_test_preds = []\nfor mt in ['flair', 't1w', 't1wce', 't2w']:\n    train_df, val_df = get_train_val_dataframe(mt)\n    test_df = get_test_dataframe(mt)\n\n    train_g, val_g, test_g = get_data_generators(train_df, val_df, test_df)\n    best_model = train_model(mt, train_g, val_g, epochs=10)\n\n    # Evaluate the model and calculate various metrics\n    results = best_model.evaluate(test_g, steps=len(test_g))\n    test_pred = best_model.predict(test_g, steps=len(test_g))\n    y_true = test_g.labels\n    y_pred = (test_pred > 0.5).astype(int)\n    \n    accuracy = sklearn.metrics.accuracy_score(y_true, y_pred)\n    precision = sklearn.metrics.precision_score(y_true, y_pred)\n    recall = sklearn.metrics.recall_score(y_true, y_pred)\n    f1_score = sklearn.metrics.f1_score(y_true, y_pred)\n    mcc = sklearn.metrics.matthews_corrcoef(y_true, y_pred)\n    auc_roc = sklearn.metrics.roc_auc_score(y_true, y_pred)\n    auc_pr = sklearn.metrics.average_precision_score(y_true, y_pred)\n    brier_score = np.mean((y_pred - y_true)**2)\n\n    print(f\"Modelo {mt}: test loss = ,{results[0]}, test acc = {accuracy}, test AUC-ROC = {auc_roc}, test AUC-PR = {auc_pr}, test precision = {precision}, test recall = {recall}, test F1-score = {f1_score}, test MCC = {mcc}, test Brier score = {brier_score}\")\n    print({results[0]}, {accuracy}, {auc_roc}, {auc_pr}, {precision}, {recall}, {f1_score}, {mcc}, {brier_score})\n    test_df['pred_y'] = test_pred\n    mean_pred = test_pred.mean()\n\n    test_pred_agg = test_df.groupby('patient_ids').apply(\n        lambda x: x['pred_y'].max() if (x['pred_y'].max() - mean_pred) > (mean_pred - x['pred_y'].min()) else x['pred_y'].min())\n\n    all_test_preds.append(test_pred_agg.values)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_test_preds = np.array(all_test_preds)\nplt.hist(all_test_preds.mean(0))\nsubm = pd.read_csv(base_datos+'sample_submission.csv')\nsubm['MGMT_value'] = all_test_preds.mean(0)\nsubm.to_csv(\"submission.csv\", index=False)\nsubm\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}