{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import math\nimport os\nfrom glob import glob\nimport random\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_hub as tfhub\nfrom kaggle_datasets import KaggleDatasets\nfrom keras_preprocessing.image.dataframe_iterator import DataFrameIterator\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.applications import EfficientNetB3\nfrom tensorflow.keras.callbacks import EarlyStopping, LearningRateScheduler, ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.keras.layers import BatchNormalization, Conv2D, Dense, Dropout, Flatten, GlobalAveragePooling2D, InputLayer, MaxPooling2D\nfrom tensorflow.keras.metrics import AUC\nfrom tensorflow.keras.optimizers import RMSprop\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nimport pydicom\nimport matplotlib.pyplot as plt\n\n# Confirmar si tenemos activa la GPU\nif tf.test.gpu_device_name():\n    print('GPU disponible')\nelse:\n    print('No se encontró GPU')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Cargar base de datos**","metadata":{}},{"cell_type":"code","source":"#Enlace de la base de datos\nbase_datos = '/kaggle/input/rsna-miccai-brain-tumor-radiogenomic-classification/'\nos.listdir(base_datos)\ntrain = base_datos+'train'\ntest  = base_datos+'test'\ntrain_csv = pd.read_csv(base_datos+'train_labels.csv')\ntest_csv  = pd.read_csv(base_datos+'sample_submission.csv')\nprint(train_csv.head(10)),print(test_csv.head(10))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Comprensión de los datos**","metadata":{}},{"cell_type":"code","source":"# obtener el número de puntos de datos faltantes por columna\n#Análisis de valores NAN en el CSV\nvalores_faltantes_train = train_csv.isnull().sum()\n\n# mira el # de puntos que faltan en las primeras diez columnas\nvalores_faltantes_train[0:2]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Comprensión de los datos (Imagenes)\n","metadata":{}},{"cell_type":"code","source":"# Agregar las rutas completas para cada ID para diferentes tipos de secuencias al dataframe\ndef full_ids(data):\n    return str(data).zfill(5)\n\ntrain_csv['BraTS21ID_full'] = train_csv['BraTS21ID'].apply(full_ids)\nbase_path = base_datos + 'train/'\ntrain_csv['flair'] = base_path + train_csv['BraTS21ID_full'] + '/FLAIR/'\ntrain_csv['t1w'] = base_path + train_csv['BraTS21ID_full'] + '/T1w/'\ntrain_csv['t1wce'] = base_path + train_csv['BraTS21ID_full'] + '/T1wCE/'\ntrain_csv['t2w'] = base_path + train_csv['BraTS21ID_full'] + '/T2w/'\n#train_csv.to_csv('nuevo_train_labels.csv', index=False)\n#print(train_csv.head(585))\n\n\n#test_csv\ntest_csv['BraTS21ID_full'] = test_csv['BraTS21ID'].apply(full_ids)\nbase_path = base_datos + 'test/'\ntest_csv['flair'] = base_path + test_csv['BraTS21ID_full'] + '/FLAIR/'\ntest_csv['t1w'] = base_path + test_csv['BraTS21ID_full'] + '/T1w/'\ntest_csv['t1wce'] = base_path + test_csv['BraTS21ID_full'] + '/T1wCE/'\ntest_csv['t2w'] = base_path + test_csv['BraTS21ID_full'] + '/T2w/'\n#print(test_csv.head(585))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Divicion del 90% de los datos TRAIN para entrenar y dejar \n#el 10% para validar y organizar en un CSV\ndef get_train_val_dataframe(mri_type):\n    \n    all_img_files = []\n    all_img_labels = []\n    all_img_patient_ids = []\n    for row in train_csv.iterrows():\n        if row[1]['BraTS21ID_full'] == '00109' and mri_type == 'flair':\n            continue\n        if row[1]['BraTS21ID_full'] == '00123' and mri_type == 't1w':\n            continue\n        if row[1]['BraTS21ID_full'] == '00709' and mri_type == 'flair':\n            continue\n        img_dir = row[1][mri_type]\n        img_files = os.listdir(img_dir)\n        img_nums = sorted([int(ele.replace('Image-', '').replace('.dcm', '')) for ele in img_files])\n        mid_point = int(len(img_nums)/2)\n        start_point = mid_point - max(int(mid_point*0.05), 1)\n        end_point = mid_point + max(int(mid_point*0.05), 1)\n        img_names = [f'Image-{img_nums[i]}.dcm' for i in range(start_point, end_point+1)]\n        img_paths = [img_dir+ele for ele in img_names]\n        img_labels = [row[1]['MGMT_value']]*len(img_paths)\n        img_patient_ids = [row[1]['BraTS21ID']]*len(img_paths)\n        all_img_files.extend(img_paths)\n        all_img_labels.extend(img_labels)\n        all_img_patient_ids.extend(img_patient_ids)\n\n    train_val_df = pd.DataFrame({'patient_ids': all_img_patient_ids,\n                  'labels': all_img_labels,\n                  'file_paths': all_img_files})\n\n    train_val_df['labels'] = train_val_df['labels'].map({1: '1', 0: '0'})\n    \n    #stratifiied 90% split on patient_ids and labels  \n    class_prop= 0.90\n    \n    classes_splits  = {}\n    for i in range(2):\n        train_val_label_class = train_val_df[train_val_df['labels']==f'{i}']\n        train_val_list_ids =  list(train_val_label_class['patient_ids'].unique())\n        train_threshold = math.ceil(class_prop*len(train_val_list_ids))\n        train_ids = train_val_list_ids[:train_threshold]\n        val_ids = train_val_list_ids[train_threshold:]\n        classes_splits[f'train_{i}'] = train_val_label_class[train_val_label_class['patient_ids'].isin(train_ids)]\n        classes_splits[f'val_{i}'] = val_df = train_val_label_class[train_val_label_class['patient_ids'].isin(val_ids)]\n        \n    train_df = pd.concat([classes_splits['train_0'], classes_splits['train_1']], axis=0)\n    val_df = pd.concat([classes_splits['val_0'], classes_splits['val_1']], axis=0)\n  \n    return train_df, val_df\ndef get_test_dataframe(mri_type):\n    \n    all_test_img_files = []\n    all_test_img_labels = []\n    all_test_img_patient_ids = []\n    for row in test_csv.iterrows():\n        img_dir = row[1][mri_type]\n        img_files = os.listdir(img_dir)\n        img_nums = sorted([int(ele.replace('Image-', '').replace('.dcm', '')) for ele in img_files])\n        mid_point = int(len(img_nums)/2)\n        start_point = mid_point - max(int(mid_point*0.05), 1)\n        end_point = mid_point + max(int(mid_point*0.05), 1)\n        img_names = [f'Image-{img_nums[i]}.dcm' for i in range(start_point, end_point+1)]\n        img_paths = [img_dir+ele for ele in img_names]\n        img_labels = [row[1]['MGMT_value']]*len(img_paths)\n        img_patient_ids = [row[1]['BraTS21ID']]*len(img_paths)\n        all_test_img_files.extend(img_paths)\n        all_test_img_labels.extend(img_labels)\n        all_test_img_patient_ids.extend(img_patient_ids)\n\n    test_df = pd.DataFrame({'patient_ids': all_test_img_patient_ids,\n                  'labels': all_test_img_labels,\n                  'file_paths': all_test_img_files})\n    \n    test_df['labels'] = ['1']*(len(test_df)-1) + ['0'] # workaround for testing data gen\n    \n    return test_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Falta entender","metadata":{}},{"cell_type":"code","source":"class DCMDataFrameIterator(DataFrameIterator):\n    def __init__(self, *arg, **kwargs):\n        self.white_list_formats = ('dcm')\n        super(DCMDataFrameIterator, self).__init__(*arg, **kwargs)\n        self.dataframe = kwargs['dataframe']\n        self.x = self.dataframe[kwargs['x_col']]\n        self.y = self.dataframe[kwargs['y_col']]\n        self.color_mode = kwargs['color_mode']\n        self.target_size = kwargs['target_size']\n\n    def _get_batches_of_transformed_samples(self, indices_array):\n        # get batch of images\n        batch_x = np.array([self.read_dcm_as_array(dcm_path, self.target_size, color_mode=self.color_mode)\n                            for dcm_path in self.x.iloc[indices_array]])\n\n        batch_y = np.array(self.y.iloc[indices_array].astype(np.uint8))  # astype because y was passed as str\n\n        # transform images\n        if self.image_data_generator is not None:\n            for i, (x, y) in enumerate(zip(batch_x, batch_y)):\n                transform_params = self.image_data_generator.get_random_transform(x.shape)\n                batch_x[i] = self.image_data_generator.apply_transform(x, transform_params)\n                # you can change y here as well, eg: in semantic segmentation you want to transform masks as well \n                # using the same image_data_generator transformations.\n\n        return batch_x, batch_y\n\n    @staticmethod\n    def read_dcm_as_array(dcm_path, target_size=(300, 300), color_mode='rgb'):\n        image_array = pydicom.dcmread(dcm_path).pixel_array\n        pixels = image_array - np.min(image_array)\n        pixels = pixels / np.max(pixels)\n        image_manual_norm = (pixels * 255).astype(np.uint8)\n        image_array = cv2.resize(image_manual_norm, target_size, interpolation=cv2.INTER_NEAREST)  #this returns a 2d array\n#         image_array = np.expand_dims(image_array, -1)\n        if color_mode == 'rgb':\n            image_array = np.dstack((image_array, np.zeros_like(image_array), np.zeros_like(image_array)))\n        return image_array\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model(weights_path):\n    \n    model = EfficientNetB3(include_top=False, weights=weights_path)\n    \n    model.trainable = False\n    \n    x = GlobalAveragePooling2D(name=\"avg_pool\")(model.output)\n    x = BatchNormalization()(x)\n\n    top_dropout_rate = 0.4\n    x = Dropout(top_dropout_rate)(x)\n    x = Dense(32, activation=\"relu\")(x)\n    x = BatchNormalization()(x)\n    x = Dropout(top_dropout_rate)(x)\n    outputs = Dense(1, activation=\"sigmoid\", name=\"pred\")(x)\n    \n\n    # Compile\n    model = Model(model.inputs, outputs, name=\"EfficientNet\")\n    \n\n    # Compile\n    optimizer =  tf.keras.optimizers.Adam(learning_rate=1e-3)\n    \n    # Compilar el modelo con la función de pérdida mixta y las métricas correspondientes\n    model.compile(optimizer='adam', loss=lambda y_true, y_pred: mixed_loss(y_true, y_pred, auc_metric, recall_metric, precision_metric), metrics=[auc_metric, recall_metric, precision_metric])\n    return model\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 369\nBATCH_SIZE = 128\nCLASS_MODE = 'binary'\nCOLOR_MODE = 'rgb'\nTARGET_SIZE = (300, 300)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_data_generators(train_df,val_df, test_df):\n    train_augmentation_parameters = dict(\n        rescale=1.0/255,\n        zoom_range=0.2,\n        rotation_range=0.2,\n        fill_mode='nearest',\n        height_shift_range= 0.1,\n        width_shift_range=0.1,\n        horizontal_flip=True,\n        brightness_range = [0.8, 1.2]\n    )\n    \n    val_augmentation_parameters = dict(\n        rescale=1.0/255.0\n    )\n\n    test_augmentation_parameters = dict(\n        rescale=1.0/255.0\n    )\n\n    train_consts = {\n        'seed': SEED,\n        'batch_size': BATCH_SIZE,\n        'class_mode': CLASS_MODE,\n        'color_mode': COLOR_MODE,\n        'target_size': TARGET_SIZE,  \n    }\n    \n    val_consts = {\n    'batch_size': BATCH_SIZE,\n    'class_mode': CLASS_MODE,\n    'color_mode': COLOR_MODE,\n    'target_size': TARGET_SIZE,\n    'shuffle': False\n    }\n\n    test_consts = {\n        'batch_size': BATCH_SIZE,\n        'class_mode': CLASS_MODE,\n        'color_mode': COLOR_MODE,\n        'target_size': TARGET_SIZE,\n        'shuffle': False\n    }\n\n    train_augmenter = ImageDataGenerator(**train_augmentation_parameters)\n    val_augmenter = ImageDataGenerator(**val_augmentation_parameters)\n    test_augmenter = ImageDataGenerator(**test_augmentation_parameters)\n\n    train_generator = DCMDataFrameIterator(dataframe=train_df,\n                                 x_col='file_paths',\n                                 y_col='labels',\n                                 image_data_generator=train_augmenter,\n                                 **train_consts)\n    \n    val_generator = DCMDataFrameIterator(dataframe=val_df,\n                                 x_col='file_paths',\n                                 y_col='labels',\n                                 image_data_generator=val_augmenter,\n                                 **val_consts)\n    \n    test_generator = DCMDataFrameIterator(dataframe=test_df,\n                                 x_col='file_paths',\n                                 y_col='labels',\n                                 image_data_generator=test_augmenter,\n                                 **test_consts)\n    \n    return train_generator, val_generator, test_generator","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\n\n# Definimos una función de pérdida que combine AUC-ROC, Recall y Precision\nimport signal\ncheckpoint_filepath = 'best_model.h5'\n\n# Definir los objetos de métricas\nauc_metric = tf.keras.metrics.AUC()\nrecall_metric = tf.keras.metrics.Recall()\nprecision_metric = tf.keras.metrics.Precision()\ndef mixed_loss(y_true, y_pred, auc, recall, precision):\n    bce_loss = tf.keras.losses.binary_crossentropy(y_true, y_pred)\n    auc.update_state(y_true, y_pred)\n    recall.update_state(y_true, y_pred)\n    precision.update_state(y_true, y_pred)\n    return bce_loss + (1 - auc.result()) + (1 - recall.result()) + (1 - precision.result())\n\ndef train_model(model_name, train_generator, val_generator, epochs, max_time=31200):\n    print('training', model_name)\n    model = build_model(\"../input/efficentnet-b0b5-tensorflow-24-notop/efficientnet-b3_tf24_imagenet_1000_notop.h5\")\n    #callbacks\n    checkpoint_cb=ModelCheckpoint(\n        filepath=checkpoint_filepath,\n        save_weights_only=False,\n        monitor='val_loss',\n        mode='min',\n        save_best_only=True,\n        save_freq='epoch',\n        verbose=1)\n    early_stopping_cb = tf.keras.callbacks.EarlyStopping(monitor='val_loss',\n                                                  patience=5,\n                                                  mode='min',\n                                                  verbose=1,\n                                                  restore_best_weights=True)\n    reduce_lr_cb=ReduceLROnPlateau(monitor='val_loss', factor=0.5,\n                                   patience=2, min_lr=0.00001,\n                                  verbose=1)\n\n    def timeout_handler(signum, frame):\n        raise TimeoutError(\"Training time exceeded the limit of {} seconds\".format(max_time))\n    signal.signal(signal.SIGALRM, timeout_handler)\n    signal.alarm(max_time)\n\n    try:\n        history = model.fit(\n                            train_generator,\n    steps_per_epoch=len(train_generator),\n    validation_data=val_generator,\n    validation_steps=len(val_generator),\n    epochs=epochs,\n    workers=2,\n    callbacks=[checkpoint_cb, reduce_lr_cb, early_stopping_cb]\n                            )\n    except TimeoutError:\n        print(\"Training time exceeded the limit of {} seconds\".format(max_time))\n    finally:\n        signal.alarm(0)  # Reset the alarm\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Inicializa la lista para guardar las predicciones de prueba de todas las imágenes\nall_test_preds = []\n\n# Itera solo sobre la técnica t2w\nmt = 't2w'\n\n# Obtiene los dataframes de entrenamiento, validación y prueba para la técnica t2w\ntrain_df, val_df = get_train_val_dataframe(mt)\ntest_df = get_test_dataframe(mt)\n\n# Obtiene los generadores de datos para los dataframes de entrenamiento, validación y prueba para la técnica t2w\ntrain_g, val_g, test_g = get_data_generators(train_df, val_df, test_df)\n\n# Entrena el modelo utilizando los generadores de datos para la técnica t2w\nbest_model = train_model(mt, train_g, val_g, epochs=10)\n\n# Evalúa el modelo entrenado en el conjunto de prueba y guarda los resultados\nresults = best_model.evaluate(test_g, steps=len(test_g))\n#print(f\"test loss, test acc, test AUC: {results}\")\nresults = best_model.evaluate(test_g, steps=len(test_g))\nprint(f\"test loss: {results[0]}\")\nprint(f\"test acc: {results[1]}\")\nprint(f\"test AUC-ROC: {results[2]}\")\nprint(f\"test AUC-PR: {results[3]}\")\n#print(f\"test precision: {results[4]}\")\n#print(f\"test recall: {results[5]}\")\n\n# Realiza predicciones en el conjunto de prueba utilizando el modelo entrenado y guarda las predicciones en el dataframe de prueba\ntest_pred = best_model.predict(test_g, steps=len(test_g))\ntest_df['pred_y'] = test_pred\n\n# Agrupa las predicciones por ID de paciente y toma la predicción más segura de todas las predicciones de imágenes para cada paciente\nmean_pred = test_pred.mean()\ntest_pred_agg = test_df.groupby('patient_ids').apply(\n    lambda x: x['pred_y'].max() if (x['pred_y'].max() - mean_pred) > (mean_pred - x['pred_y'].min()) else x['pred_y'].min())\n\n# Agrega las predicciones agrupadas a la lista de todas las predicciones de prueba\nall_test_preds.append(test_pred_agg.values)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_test_preds = np.array(all_test_preds)\nplt.hist(all_test_preds.mean(0))\nsubm = pd.read_csv(base_datos+'sample_submission.csv')\nsubm['MGMT_value'] = all_test_preds.mean(0)\nsubm.to_csv(\"submission.csv\", index=False)\nsubm\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}