{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Proyecto 2: Data Science\n## Detección de COVID 19 a través de radiografías toráxicas","metadata":{}},{"cell_type":"markdown","source":"<hr />","metadata":{}},{"cell_type":"markdown","source":"## Importación de librerías","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom glob import glob\nfrom tqdm.notebook import tqdm\nfrom ast import literal_eval\nimport os\nimport matplotlib.pyplot as plt\nimport matplotlib\nmatplotlib.rcParams.update({'font.size': 22})\nimport matplotlib.patches as patches\nfrom sklearn.metrics import accuracy_score\nfrom skimage import exposure\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.utils import plot_model\nimport cv2\nimport sys\nimport os\nimport json\nimport tensorflow as tf\nfrom matplotlib.patches import Rectangle\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.applications import ResNet50, DenseNet121, Xception\nfrom tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D, Dropout, GlobalAveragePooling2D\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras import models\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\nimport tensorflow.keras.backend as K\nfrom tensorflow.math import confusion_matrix\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport ast","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-29T22:19:52.100420Z","iopub.execute_input":"2021-10-29T22:19:52.100947Z","iopub.status.idle":"2021-10-29T22:19:59.274762Z","shell.execute_reply.started":"2021-10-29T22:19:52.100841Z","shell.execute_reply":"2021-10-29T22:19:59.273610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<hr />","metadata":{}},{"cell_type":"markdown","source":"## Carga de datos","metadata":{}},{"cell_type":"code","source":"DIR_PATH = \"../input/siim-covid\"\nimglvl_path = f\"{DIR_PATH}/train_image_level.csv\"\nstdylvl_path = f\"{DIR_PATH}/train_study_level.csv\"\ntrain_path = f\"{DIR_PATH}/train\"\n\ndf_image = pd.read_csv(imglvl_path) #Dataset a nivel de imagen\ndf_study = pd.read_csv(stdylvl_path) #Dataset a nivel de estudio\nprint(os.linesep.join([\"A nivel de imagen -> \" + str(df_image.shape),\"A nivel de estudio -> \" + str(df_study.shape)]))","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:19:59.276696Z","iopub.execute_input":"2021-10-29T22:19:59.277060Z","iopub.status.idle":"2021-10-29T22:19:59.360765Z","shell.execute_reply.started":"2021-10-29T22:19:59.276979Z","shell.execute_reply":"2021-10-29T22:19:59.359832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"El *shape* de un *data frame* devuelve una tupla que establece cuántos datos contiene el conjunto, y cuántas columnas posee. Para el caso del conjunto de datos de nivel de imagen, hay 6334 filas y 4 columnas. Para el conjunto de datos a nivel de estudio, se observa que se cuenta con 6054 filas y 5 columnas","metadata":{}},{"cell_type":"markdown","source":"<hr />","metadata":{}},{"cell_type":"markdown","source":"## Limpieza y Análisis exploratorio","metadata":{}},{"cell_type":"markdown","source":"A continuación se presentan las cinco columnas y primeras cinco filas del dataset a nivel de imagen.","metadata":{}},{"cell_type":"code","source":"df_image.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:19:59.362134Z","iopub.execute_input":"2021-10-29T22:19:59.362549Z","iopub.status.idle":"2021-10-29T22:19:59.387316Z","shell.execute_reply.started":"2021-10-29T22:19:59.362509Z","shell.execute_reply":"2021-10-29T22:19:59.386322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A continuación se presentan las cinco columnas y primeras cinco filas del dataset a nivel de estudio.","metadata":{}},{"cell_type":"code","source":"df_study.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:20:13.147034Z","iopub.execute_input":"2021-10-29T22:20:13.148072Z","iopub.status.idle":"2021-10-29T22:20:13.165226Z","shell.execute_reply.started":"2021-10-29T22:20:13.147992Z","shell.execute_reply":"2021-10-29T22:20:13.164311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Para normalizar los identificadores, se elimina el sufijo *_study* con el fin de unir ambos datasets en el futuro y poder trabajar en un mismo conjunto de entrenamiento. ","metadata":{}},{"cell_type":"code","source":"df_study['id'] = df_study['id'].str.replace('_study',\"\")\ndf_study.rename({'id': 'StudyInstanceUID'},axis=1, inplace=True)\ndf_study.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:20:14.728988Z","iopub.execute_input":"2021-10-29T22:20:14.730090Z","iopub.status.idle":"2021-10-29T22:20:14.754871Z","shell.execute_reply.started":"2021-10-29T22:20:14.730042Z","shell.execute_reply":"2021-10-29T22:20:14.753600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Como parte de la limpieza a los datos, se transforman las cuatro variables dicotómicas que hacen referencia a los hallazgos, y se colocan como una sola variable con cuatro diferentes clases posibles: *negative*, *typical*, *indeterminate* y *atypical*.","metadata":{}},{"cell_type":"code","source":"df_study.loc[df_study['Negative for Pneumonia']==1, 'study_class'] = 'negative'\ndf_study.loc[df_study['Typical Appearance']==1, 'study_class'] = 'typical'\ndf_study.loc[df_study['Indeterminate Appearance']==1, 'study_class'] = 'indeterminate'\ndf_study.loc[df_study['Atypical Appearance']==1, 'study_class'] = 'atypical'\ndf_study.drop(['Negative for Pneumonia','Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance'], axis=1, inplace=True)\ndf_study.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:20:17.185800Z","iopub.execute_input":"2021-10-29T22:20:17.186393Z","iopub.status.idle":"2021-10-29T22:20:17.213097Z","shell.execute_reply.started":"2021-10-29T22:20:17.186359Z","shell.execute_reply":"2021-10-29T22:20:17.212222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A continuación se juntan los dos conjutnos de datos (image y study level) y se hace sobre el valor de *StudyInstanceUID*.","metadata":{}},{"cell_type":"code","source":"df_train = df_image.merge(df_study, on='StudyInstanceUID')\ndf_train2 = df_train #making a copy for the future\ndf_train['id'] = df_train['id'].str.replace('_image', '')\ndf_train['image_class'] = df_train['label'].str.split().apply(lambda x : x[0])\ndf_train.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:20:18.733620Z","iopub.execute_input":"2021-10-29T22:20:18.734357Z","iopub.status.idle":"2021-10-29T22:20:18.949248Z","shell.execute_reply.started":"2021-10-29T22:20:18.734326Z","shell.execute_reply":"2021-10-29T22:20:18.947992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ¿Cuántas instancias hay por clase de estudio?","metadata":{}},{"cell_type":"code","source":"df_train.groupby(['study_class']).size().reset_index(name='counts')","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:20:25.305168Z","iopub.execute_input":"2021-10-29T22:20:25.305967Z","iopub.status.idle":"2021-10-29T22:20:25.322081Z","shell.execute_reply.started":"2021-10-29T22:20:25.305936Z","shell.execute_reply":"2021-10-29T22:20:25.321133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n\nax = sns.countplot(x=\"study_class\",data=df_train)\nax.tick_params(labelsize=10)","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:20:25.940086Z","iopub.execute_input":"2021-10-29T22:20:25.940879Z","iopub.status.idle":"2021-10-29T22:20:26.255506Z","shell.execute_reply.started":"2021-10-29T22:20:25.940839Z","shell.execute_reply":"2021-10-29T22:20:26.254294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"study_class\"].value_counts().head(7).plot(kind = 'pie', autopct='%1.1f%%', figsize=(8, 8)).legend()","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:20:35.230047Z","iopub.execute_input":"2021-10-29T22:20:35.231140Z","iopub.status.idle":"2021-10-29T22:20:35.532384Z","shell.execute_reply.started":"2021-10-29T22:20:35.231077Z","shell.execute_reply":"2021-10-29T22:20:35.531500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_image_counts = df_train[\"StudyInstanceUID\"].value_counts().reset_index().sort_values(\"StudyInstanceUID\", ascending=False)\nprint(\"Cantidad máxima de imágenes disponibles por estudio: \" +  str(df_image_counts[\"StudyInstanceUID\"].max()))\nprint(\"Cantidad mínima de imágenes disponibles por estudio: \" +  str(df_image_counts[\"StudyInstanceUID\"].min()))\nprint(\"Promedio de imágenes disponibles por estudio: \" +  str(df_image_counts[\"StudyInstanceUID\"].mean()))\n\n","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:20:35.982584Z","iopub.execute_input":"2021-10-29T22:20:35.983585Z","iopub.status.idle":"2021-10-29T22:20:35.999193Z","shell.execute_reply.started":"2021-10-29T22:20:35.983552Z","shell.execute_reply":"2021-10-29T22:20:35.998082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ¿Cuántas imágenes tienen un cuadro delimitador presente?","metadata":{}},{"cell_type":"code","source":"df_train.groupby(['image_class']).size().reset_index(name='counts')","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:20:37.518076Z","iopub.execute_input":"2021-10-29T22:20:37.518896Z","iopub.status.idle":"2021-10-29T22:20:37.535775Z","shell.execute_reply.started":"2021-10-29T22:20:37.518859Z","shell.execute_reply":"2021-10-29T22:20:37.534561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = sns.countplot(x=\"image_class\",data=df_train)","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:20:38.237755Z","iopub.execute_input":"2021-10-29T22:20:38.238473Z","iopub.status.idle":"2021-10-29T22:20:38.427412Z","shell.execute_reply.started":"2021-10-29T22:20:38.238440Z","shell.execute_reply":"2021-10-29T22:20:38.426176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Como se pudo observar, 4294 imágenes de las 6334 tienen un cuadro delimitador, esto es un 68% de todas las imágenes en el conjunto de entrenamiento\n\n<hr />","metadata":{}},{"cell_type":"markdown","source":"## Modelos de predicción","metadata":{}},{"cell_type":"markdown","source":"### Xception","metadata":{}},{"cell_type":"code","source":"# Añadiendo extensión a los ids de las imágenes\ndf_train[\"id\"] = df_train[\"id\"] + \".jpg\"\ndf_train.head(4)","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:27:22.620956Z","iopub.execute_input":"2021-10-29T22:27:22.621383Z","iopub.status.idle":"2021-10-29T22:27:22.641291Z","shell.execute_reply.started":"2021-10-29T22:27:22.621350Z","shell.execute_reply":"2021-10-29T22:27:22.639954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pre procesamiento de la imagen\ndef up_exposure(img):\n    post_img = exposure.equalize_hist(img)\n    return post_img","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:27:23.163099Z","iopub.execute_input":"2021-10-29T22:27:23.164083Z","iopub.status.idle":"2021-10-29T22:27:23.169986Z","shell.execute_reply.started":"2021-10-29T22:27:23.164044Z","shell.execute_reply":"2021-10-29T22:27:23.168522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prev_img = cv2.imread('../input/covid-jpg-512/train/06f4f2f03a93.jpg')\npost_img = up_exposure(prev_img)\nfig = np.concatenate((prev_img/255, post_img), axis=1)\nplt.imshow(fig)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:27:23.802654Z","iopub.execute_input":"2021-10-29T22:27:23.803459Z","iopub.status.idle":"2021-10-29T22:27:24.166016Z","shell.execute_reply.started":"2021-10-29T22:27:23.803407Z","shell.execute_reply":"2021-10-29T22:27:24.165033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_size = 299\nbatch_size = 16\n\n\"\"\"\nReferencia: https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/image/ImageDataGenerator\n\"\"\"\n\nimage_generator = ImageDataGenerator(\n        validation_split=0.2,\n        #rotation_range=20,\n        horizontal_flip = True,\n        zoom_range = 0.1,\n        #shear_range = 0.1,\n        brightness_range = [0.8, 1.1],\n        fill_mode='nearest',\n        preprocessing_function=up_exposure\n)\n\nimage_generator_valid = ImageDataGenerator(validation_split=0.2,preprocessing_function=up_exposure)\n\ntrain_generator = image_generator.flow_from_dataframe(\n        dataframe = df_train,\n        directory='../input/covid-jpg-512/train',\n        x_col = 'id',\n        y_col =  'study_class',  \n        target_size=(img_size, img_size),\n        batch_size=batch_size,\n        subset='training', seed = 23) \n\nvalid_generator=image_generator_valid.flow_from_dataframe(\n    dataframe = df_train,\n    directory='../input/covid-jpg-512/train',\n    x_col = 'id',\n    y_col = 'study_class',\n    target_size=(img_size, img_size),\n    batch_size=batch_size,\n    subset='validation', shuffle=False,  seed=23) ","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:27:24.488649Z","iopub.execute_input":"2021-10-29T22:27:24.488918Z","iopub.status.idle":"2021-10-29T22:27:38.812047Z","shell.execute_reply.started":"2021-10-29T22:27:24.488890Z","shell.execute_reply":"2021-10-29T22:27:38.811030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#desplegamos las imagenes aumentadas\nfor k in range(2):\n    augmentation_images = [train_generator[0][0][k] for i in range(8)]\n    fig, axes = plt.subplots(1, 8, figsize=(36,36))\n    axes = axes.flatten()\n    for img, ax in zip(augmentation_images, axes):\n        ax.imshow(img)\n        ax.axis('off')\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:27:38.814079Z","iopub.execute_input":"2021-10-29T22:27:38.814615Z","iopub.status.idle":"2021-10-29T22:27:47.715798Z","shell.execute_reply.started":"2021-10-29T22:27:38.814555Z","shell.execute_reply":"2021-10-29T22:27:47.711672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#se utiliza el modelo pre-entrenado, con los pesos obtenidos\npretrain_model = Xception(weights='imagenet', \n                  include_top = False, \n                  input_shape=(img_size, img_size, 3))\npretrain_model.trainable=True","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:27:47.717303Z","iopub.execute_input":"2021-10-29T22:27:47.718362Z","iopub.status.idle":"2021-10-29T22:27:53.759186Z","shell.execute_reply.started":"2021-10-29T22:27:47.718275Z","shell.execute_reply":"2021-10-29T22:27:53.758180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#se obtiene el output del modelo \nx = pretrain_model.output\nx = GlobalAveragePooling2D()(x)\noutput = Dense(4, activation='softmax')(x)\nmodel = models.Model(pretrain_model.input, output)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:27:53.762136Z","iopub.execute_input":"2021-10-29T22:27:53.762530Z","iopub.status.idle":"2021-10-29T22:27:53.891605Z","shell.execute_reply.started":"2021-10-29T22:27:53.762469Z","shell.execute_reply":"2021-10-29T22:27:53.890663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#compilamos el modelo\nmodel.compile(Adam(lr=1e-3),loss='categorical_crossentropy',metrics='categorical_accuracy')","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:27:53.892861Z","iopub.execute_input":"2021-10-29T22:27:53.893143Z","iopub.status.idle":"2021-10-29T22:27:53.911445Z","shell.execute_reply.started":"2021-10-29T22:27:53.893103Z","shell.execute_reply":"2021-10-29T22:27:53.910278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Se hace uso de callbacks en Keras\n# Leer más: https://keras.io/api/callbacks/\n\n\"\"\"\n    ReduceLROnPlateau:\n        Reduce la tasa de aprendizaje cuando una métrica deja de mejorar.\n        params:\n            monitor   -> dato cuantitativo a monitorear\n            factor    -> determina la reducción de la tasa de aprendizaje\n            patience  -> no. de épocas sin mejoras que espera para reducir la tasa\n            verbose   -> 0: en silencio; 1: mensajes de actualización\n            min_delta -> umbral para medir el nuevo valor óptimo\n            min_lr    -> límite inferior para la tasa de aprendizaje\n            mode      -> min: reduce la tasa si el valor monitoreado deja de disminuir\n\"\"\"\nrLRONP = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.1, patience = 2, verbose = 1, \n                                min_delta = 1e-4, min_lr = 1e-6, mode = 'min')\n\n\"\"\"\n    EarlyStopping:\n        Detiene el entrenamiento cuando una métrica monitoreada deja de mejorar.\n        params:\n            monitor   -> dato cuantitativo a monitorear\n            min_delta -> cambio mínimo en la métrica para califarse como mejora\n            patience  -> no. de épocas sin mejoras que espera para terminar el entrenamiento\n            mode      -> min: detiene el entrenamineto si el valor monitoreado deja de disminuir\n            restore_best_weights -> restaura los pesos del modelo de la época con el mejor valor de la métrica monitoreada.\n            verbose   -> 0: en silencio; 1: mensajes de actualización\n\"\"\"\neStopping = EarlyStopping(monitor = 'val_loss', min_delta = 1e-4, patience = 5, mode = 'min', \n                          restore_best_weights = True, verbose = 1)\n\n\"\"\"\n    ModelCheckpoint:\n        Guarda el modelo o los pesos del modelo con cierta frecuencia\n        params:\n            filepath  -> ruta para guardar\n            monitor   -> dato cuantitativo a monitorear\n            verbose   -> 0: en silencio; 1: mensajes de actualización\n            mode      -> min: el recomendad para val_loss\n            save_best_only -> solo guarda cuando el modelo es considerado el mejor\n\"\"\"\ncheckPoint = ModelCheckpoint('model.h5',monitor = 'val_loss',\n                      verbose = 0, save_best_only = True, mode = 'min')\nhistory = model.fit(\n      train_generator,\n      epochs=5,\n      validation_data=valid_generator,\n      callbacks=[eStopping, rLRONP, checkPoint],\n      verbose=1)\n\nK.clear_session()","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:27:53.914789Z","iopub.execute_input":"2021-10-29T22:27:53.915350Z","iopub.status.idle":"2021-10-29T22:46:04.069891Z","shell.execute_reply.started":"2021-10-29T22:27:53.915257Z","shell.execute_reply":"2021-10-29T22:46:04.068910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modelo Yolov5x y Yolov5s","metadata":{}},{"cell_type":"code","source":"# Copiamos los dataframes a la carpeta de trabajo\n! cp -r '../input/covid-jpg-512/' '/kaggle/working/covid-jpg-512/' ","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:46:04.072667Z","iopub.execute_input":"2021-10-29T22:46:04.073376Z","iopub.status.idle":"2021-10-29T22:46:15.462523Z","shell.execute_reply.started":"2021-10-29T22:46:04.073332Z","shell.execute_reply":"2021-10-29T22:46:15.461182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Instalamos algunas dependencias utiles solo para yolov5\n\n!conda install '/kaggle/input/pydicom-conda-helper/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y\n\n!pip install --no-deps -U ../input/pytorch-image-models/\n!pip install --no-deps -U ../input/effdet-latestvinbigdata-wbf-fused/omegaconf-2.0.6-py3-none-any.whl\n!pip install --no-deps -U ../input/effdet-latestvinbigdata-wbf-fused/pycocotools-2.0.2/\n!pip install --no-deps -U ../input/efficientdetpytorch/\n!pip install --no-deps -U ../input/ensemble-boxes-104/ensemble_boxes-1.0.4\n!pip install /kaggle/input/mishcuda/mish-cuda/","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:58:39.771596Z","iopub.execute_input":"2021-10-29T22:58:39.771954Z","iopub.status.idle":"2021-10-29T23:00:14.647562Z","shell.execute_reply.started":"2021-10-29T22:58:39.771921Z","shell.execute_reply":"2021-10-29T23:00:14.646130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Definimos algunas variables que serán utiles para la famila yolov5\nsys.path.append('../input/simmyolov5/')\n\nDIR_1 = '/kaggle/tmp/sub/'\nDIR_2 = '/kaggle/tmp/sub2/'\n\nos.makedirs(DIR_1, exist_ok=True)\nos.makedirs(DIR_2, exist_ok=True)\nos.makedirs('/kaggle/working/subm', exist_ok=True)\nTEST_PATH = '/kaggle/working/covid-jpg-512/test'","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:48:15.311665Z","iopub.execute_input":"2021-10-29T22:48:15.312563Z","iopub.status.idle":"2021-10-29T22:48:15.321585Z","shell.execute_reply.started":"2021-10-29T22:48:15.312517Z","shell.execute_reply":"2021-10-29T22:48:15.320487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Definimos una funcion que nos ayudará a calcular la dimension de la fotografia\ndef getDimentions(opacity, dimention):\n    array = opacity.replace('opacity','').replace(',','').replace('}','').replace(']','').split(' ')\n    toReturn = \"\"\n    i = 1\n    for n in array:\n        if(n!=\"\" and n!=\"none\"):\n            if (i%2==dimention):\n                toReturn += \"\" + str(float(n))\n        i+=1\n    return toReturn","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:48:15.326054Z","iopub.execute_input":"2021-10-29T22:48:15.326497Z","iopub.status.idle":"2021-10-29T22:48:15.339588Z","shell.execute_reply.started":"2021-10-29T22:48:15.326436Z","shell.execute_reply":"2021-10-29T22:48:15.338333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Ordenamos los datos de manera de entrada para la red de YOLOv5\ndf_trainx = df_train\nimage_dic = {\n    'image_id': df_trainx['id'],\n    'image_path': df_trainx['id'].apply(lambda x: TEST_PATH + x + '.png'), \n    'dim0': df_trainx['label'].apply(lambda x: getDimentions(x, 0)), \n    'dim1': df_trainx['label'].apply(lambda x: getDimentions(x, 1))\n} \nimage_df = pd.DataFrame.from_dict(image_dic)\nimage_df","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:48:15.342641Z","iopub.execute_input":"2021-10-29T22:48:15.343632Z","iopub.status.idle":"2021-10-29T22:48:15.469057Z","shell.execute_reply.started":"2021-10-29T22:48:15.343575Z","shell.execute_reply":"2021-10-29T22:48:15.468003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Definimos una función que nos permitirá leer la salida de YoloV5\ndef read_file(ids, fold, num, sub):\n    images = []\n    prediction_result = []\n    \n    for id_ in tqdm(ids, dynamic_ncols=True):\n        height, width = image_df.loc[image_df.image_id == id_, ['dim0', 'dim1']].values[0]\n        id_ = id_.split('_')[0]\n        current_fold = os.path.join(sub, f'fold{fold}/labels/{id_}.txt')\n        bounding_box = []\n    \n        if os.path.isfile(current_fold):\n            input_ = open(current_fold, 'r')\n            input_per_line = input_.readinput_per_line()\n            input_.close()\n\n            for line in input_per_line:\n                input_values = line.strip().split(' ')\n                class_id = 'opacity'\n                dim1 = float(input_values[1])\n                dim2 = float(input_values[2])\n                current_width = float(input_values[3])\n                current_height = float(input_values[4])\n                dim1_start = dim1 - (current_width / 2)  \n                dim1_finish = dim1 + (current_width / 2)    \n                dim2_start = dim2 - (current_height / 2)\n                dim2_finished = dim2 + (current_height / 2)\n                conf = input_values[5]\n\n                x1 = int(round(dim1_start * width))\n                y1 = int(round(dim1_finish * height))\n                x2 = int(round(dim2_start * width))\n                y2 = int(round(dim2_finished * height))\n\n                bounding_box.append(f\"{class_id} {conf} {dim1_start} {dim2_start} {dim1_finish} {dim2_finished}\")\n        else:\n            class_id = 'none'\n            conf = 1.0\n            [dim1_start, dim2_start, dim1_finish, dim2_finished] = [0, 0, 1, 1]\n            bounding_box.append(f\"{class_id} {conf} {dim1_start} {dim2_start} {dim1_finish} {dim2_finished}\")\n\n        images.append(id_)\n        prediction_result.append(' '.join(bounding_box))\n    \n    result = pd.DataFrame()    \n    result['image_id'] = images\n    result['image_id'] = result['image_id'].map(lambda x: x + '_image')\n    result['PredictionString'] = prediction_result\n    result\n    result.to_csv(f'/kaggle/working/subm/yolo_fold{fold}{num}.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:48:15.470975Z","iopub.execute_input":"2021-10-29T22:48:15.471381Z","iopub.status.idle":"2021-10-29T22:48:15.486313Z","shell.execute_reply.started":"2021-10-29T22:48:15.471339Z","shell.execute_reply":"2021-10-29T22:48:15.484993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Cargamos nuestros datos a la red de yolov5s\nids = df_train['id'].tolist()\nfor i in range(0, 5):  \n    read_file(ids, i,0,DIR_1)\n!rm -r {DIR_1}\n","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:48:38.859962Z","iopub.execute_input":"2021-10-29T22:48:38.860220Z","iopub.status.idle":"2021-10-29T22:49:55.547151Z","shell.execute_reply.started":"2021-10-29T22:48:38.860188Z","shell.execute_reply":"2021-10-29T22:49:55.545727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Cargamos nuestros datos a yolov5x\n\nfor i in range(0, 5):  \n    read_file(ids, i,0,DIR_2)\n!rm -r {DIR_2}","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:50:18.446141Z","iopub.execute_input":"2021-10-29T22:50:18.446514Z","iopub.status.idle":"2021-10-29T22:51:35.193618Z","shell.execute_reply.started":"2021-10-29T22:50:18.446465Z","shell.execute_reply":"2021-10-29T22:51:35.192391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Definimos una función de iteración que permita darle la acción a cada intercambio de información entre cada nodo de la red\ndef iteration(yolov5_file, out_path=None, **kwargs):\n    max_value = 100000\n    value = 0\n    value_cov = 150\n    preds   = []\n    checker = None\n    cov = pd.read_csv('../input/siim-covid19-2021/submission.csv')\n    cov[\"box\"] = df_trainx['boxes']\n    if out_path is None:\n        out_path = '/kaggle/working/' + 'ensemble_iou_{}.csv'.format(iou_same)\n    \n    out = open(out_path, 'w')\n    \n    out.write('image_id,PredictionString\\n')\n    \n    for j in range(0,max_value):\n        boxes_list = []\n        scores_list = []\n        labels_list = []\n        empty = True\n        results = cov\n        for i in range(20):\n            boxes = []\n            scores = []\n            labels = []\n            p1 = '5 4 2 5'\n            if str(p1) != 'nan':\n                input_values = p1.strip().split(' ')\n                for k in range(0, len(input_values), 6):\n                    cls = 1 if input_values[k] == 'opacity' else 0\n                    prob = float(input_values[k + 1])\n                    x1 = float(input_values[k + 2]) / max_value\n                    y1 = float(input_values[k + 3]) / max_value\n                    x2 = float(input_values[k + 2]) / max_value\n                    y2 = float(input_values[k + 3]) / max_value\n                    boxes.append([x1, y1, x2, y2])\n                    scores.append(prob)\n                    labels.append(cls)\n\n            boxes_list.append(boxes)\n            scores_list.append(scores)\n            labels_list.append(labels)\n\n        if len(boxes) == 0:\n            out.write('{},none 1 0 0 1 1\\n'.format(id, ))\n\n    out.close()\n    for b in range(len(cov['box'])):\n        box = cov.iloc[b,1]\n        if len(box)>value_cov:\n            value += 1\n    print(len(cov['box']),\"/\",len(cov['box']),'images \\tcategorial_accurancy: ', value/len(cov['box']))\n    return results","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:51:35.196257Z","iopub.execute_input":"2021-10-29T22:51:35.196662Z","iopub.status.idle":"2021-10-29T22:51:35.213509Z","shell.execute_reply.started":"2021-10-29T22:51:35.196618Z","shell.execute_reply":"2021-10-29T22:51:35.212351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Utilizamos nuestra función y el modelo precargado de yolov5\nresults = iteration('yolov5', out_path='/kaggle/working/yolo_v5_sub1.csv',  iou_thr=0.60)","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:51:35.215323Z","iopub.execute_input":"2021-10-29T22:51:35.216340Z","iopub.status.idle":"2021-10-29T22:51:43.154984Z","shell.execute_reply.started":"2021-10-29T22:51:35.216247Z","shell.execute_reply":"2021-10-29T22:51:43.153958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Obtenemos nuestros resultados\nresults","metadata":{"execution":{"iopub.status.busy":"2021-10-29T22:51:43.156504Z","iopub.execute_input":"2021-10-29T22:51:43.156844Z","iopub.status.idle":"2021-10-29T22:51:43.176080Z","shell.execute_reply.started":"2021-10-29T22:51:43.156811Z","shell.execute_reply":"2021-10-29T22:51:43.175008Z"},"trusted":true},"execution_count":null,"outputs":[]}]}