{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"> # Detección de neumonía en imágenes médicas.\n\nEl desafío de la competencia consta en crear un algoritmo para la localización de una señal visual de neumonía en imágenes médicas. Concretamente, el algoritmo debe detectar las opacidades pulmonares (en imágenes radiológicas torácicas) características de esta enfermedad.","metadata":{"papermill":{"duration":0.018171,"end_time":"2021-02-01T16:53:05.4241","exception":false,"start_time":"2021-02-01T16:53:05.405929","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Lectura de un archivo .csv, el que detalla la informacion de las clases.","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":0.017145,"end_time":"2021-02-01T16:53:05.458933","exception":false,"start_time":"2021-02-01T16:53:05.441788","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import csv\n\nfile = open('../input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv')\n\nprint(file.read()[0:500])","metadata":{"papermill":{"duration":0.109585,"end_time":"2021-02-01T16:53:05.586767","exception":false,"start_time":"2021-02-01T16:53:05.477182","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lo mismo, con pandas","metadata":{"papermill":{"duration":0.017853,"end_time":"2021-02-01T16:53:05.623473","exception":false,"start_time":"2021-02-01T16:53:05.60562","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import pandas as pd\n\ndataframe = pd.read_csv('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv')  \n\ndataframe.sample(6)","metadata":{"papermill":{"duration":0.127142,"end_time":"2021-02-01T16:53:05.768616","exception":false,"start_time":"2021-02-01T16:53:05.641474","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Características de las imágenes.\nimport pydicom\n\npatientId = dataframe['patientId'][0]\ndcm_file = '../input/rsna-pneumonia-detection-challenge/stage_2_train_images/%s.dcm' % patientId\ndcm_data = pydicom.read_file(dcm_file)\n\nim = dcm_data.pixel_array\nprint(type(im))\nprint(im.dtype)\nprint(im.shape)","metadata":{"papermill":{"duration":0.400284,"end_time":"2021-02-01T16:53:06.196665","exception":false,"start_time":"2021-02-01T16:53:05.796381","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" Clases y cantidad de imágenes por cada una:\n* No Lung Opacity / Not Normal: pulmones con alguna anomalía que no es neumonía.\n* Normal: no hay indicios de neumonía.\n* Lung Opacity: pulmones que potencialmente tienen neumonía.","metadata":{"papermill":{"duration":0.027192,"end_time":"2021-02-01T16:53:06.250531","exception":false,"start_time":"2021-02-01T16:53:06.223339","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_detailed = pd.read_csv('../input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv')\n\nsummary = {}\nfor n, row in df_detailed.iterrows():\n    if row['class'] not in summary:\n        summary[row['class']] = 0\n    summary[row['class']] += 1\n    \nprint(summary)","metadata":{"papermill":{"duration":4.489716,"end_time":"2021-02-01T16:53:10.760843","exception":false,"start_time":"2021-02-01T16:53:06.271127","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Histograma de las clases\nimport matplotlib.pyplot as plt\n\nclases = pd.read_csv('../input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv', index_col=0)\nplt.xlabel(\"Clases\")\nplt.ylabel(\"Cantidad de imágenes\")\nplt.hist(clases['class'], 5, color=\"brown\", ec='black')","metadata":{"papermill":{"duration":0.32161,"end_time":"2021-02-01T16:53:11.112089","exception":false,"start_time":"2021-02-01T16:53:10.790479","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Detección de posibles áreas con neumonía. ","metadata":{"papermill":{"duration":0.020762,"end_time":"2021-02-01T16:53:11.153616","exception":false,"start_time":"2021-02-01T16:53:11.132854","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def parse_data(dataframe, dataframe_detailed): \n # --- Define lambda to extract coords in list [y, x, height, width]\n    pids = []\n\n    extract_box = lambda row: [row['y'], row['x'], row['height'], row['width']]\n\n    parsed = {}\n    for n, row in dataframe.iterrows():\n        # --- Initialize patient entry into parsed \n        pid = row['patientId']\n        if pid not in parsed:\n            pids.append(pid)\n            parsed[pid] = {\n                'dicom': '../input/rsna-pneumonia-detection-challenge/stage_2_train_images/%s.dcm' % pid,\n                'label': row['Target'],\n                #'class': df_detailed.iloc[df_detailed.index[df_detailed.iloc[:]['patientId']==pid][0]]['class'],\n                'boxes': []}\n\n        # --- Add box if opacity is present\n        #if parsed[pid]['label'] == 1:\n        parsed[pid]['boxes'].append(extract_box(row))\n\n    return parsed, pids","metadata":{"papermill":{"duration":0.034085,"end_time":"2021-02-01T16:53:11.208011","exception":false,"start_time":"2021-02-01T16:53:11.173926","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parsed, pids = parse_data(dataframe, df_detailed)","metadata":{"papermill":{"duration":4.568164,"end_time":"2021-02-01T16:53:15.797733","exception":false,"start_time":"2021-02-01T16:53:11.229569","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pylab\n\ndef draw(data):\n    \"\"\"\n    Method to draw single patient with bounding box(es) if present \n    \"\"\"\n    # --- Open DICOM file\n    d = pydicom.read_file(data['dicom'])\n    im = d.pixel_array\n\n    # --- Convert from single-channel grayscale to 3-channel RGB\n    im = np.stack([im] * 3, axis=2)\n\n    # --- Add boxes with random color if present\n    for box in data['boxes']:\n        rgb = np.floor(np.random.rand(3) * 256).astype('int')\n        im = overlay_box(im=im, box=box, rgb=rgb, stroke=6)\n\n    pylab.imshow(im, cmap=pylab.cm.gist_gray)\n    pylab.axis('off')\n\ndef overlay_box(im, box, rgb, stroke=1):\n    \"\"\"\n    Method to overlay single box on image\n    \"\"\"\n    # --- Convert coordinates to integers\n    box = [int(b) for b in box]\n    \n    # --- Extract coordinates\n    y1, x1, height, width = box\n    y2 = y1 + height\n    x2 = x1 + width\n\n    im[y1:y1 + stroke, x1:x2] = rgb\n    im[y2:y2 + stroke, x1:x2] = rgb\n    im[y1:y2, x1:x1 + stroke] = rgb\n    im[y1:y2, x2:x2 + stroke] = rgb\n\n    return im\n\ndef crop_image(data):\n    \"\"\"\n    Method to overlay single box on image\n    \"\"\"\n    # --- Open DICOM file\n    d = pydicom.read_file(data['dicom'])\n    im = d.pixel_array\n\n    # --- Convert from single-channel grayscale to 3-channel RGB\n    im = np.stack([im] * 3, axis=2)\n    \n    for box in data['boxes']:\n\n        # --- Convert coordinates to integers\n        box = [int(b) for b in box]\n    \n        # --- Extract coordinates\n        y1, x1, height, width = box\n        y2 = y1 + height\n        x2 = x1 + width\n        \n        new_image = im[y1:y2, x1:x2]\n\n    return im, new_image","metadata":{"papermill":{"duration":0.058623,"end_time":"2021-02-01T16:53:15.877435","exception":false,"start_time":"2021-02-01T16:53:15.818812","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Imagen con neumonía.\nprint(parsed['00436515-870c-4b36-a041-de91049b9ab4'])","metadata":{"papermill":{"duration":0.03078,"end_time":"2021-02-01T16:53:15.928407","exception":false,"start_time":"2021-02-01T16:53:15.897627","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(parsed[pids[0]])","metadata":{"papermill":{"duration":0.031455,"end_time":"2021-02-01T16:53:15.980639","exception":false,"start_time":"2021-02-01T16:53:15.949184","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataframe.loc[dataframe['Target']==1, ['x', 'y', 'width', 'height']].isnull().any()","metadata":{"papermill":{"duration":0.070515,"end_time":"2021-02-01T16:53:16.072989","exception":false,"start_time":"2021-02-01T16:53:16.002474","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataframe.loc[dataframe['Target']==0, ['x', 'y', 'width', 'height']].isnull().all()","metadata":{"papermill":{"duration":0.043058,"end_time":"2021-02-01T16:53:16.138003","exception":false,"start_time":"2021-02-01T16:53:16.094945","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Información que contienen las imágenes .dcm :**","metadata":{}},{"cell_type":"code","source":"pId = dataframe['patientId'].sample(1).values[0]    \ndcmdata = pydicom.read_file('../input/rsna-pneumonia-detection-challenge/stage_2_train_images/'+pId+'.dcm')\nprint(dcmdata)","metadata":{"papermill":{"duration":0.052947,"end_time":"2021-02-01T16:53:16.213056","exception":false,"start_time":"2021-02-01T16:53:16.160109","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dcmimg = dcmdata.pixel_array\nplt.figure(figsize=(20,10))\nplt.imshow(dcmimg, cmap=pylab.cm.binary)\nplt.axis('off')","metadata":{"papermill":{"duration":0.261297,"end_time":"2021-02-01T16:53:16.49745","exception":false,"start_time":"2021-02-01T16:53:16.236153","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Nuevo dataset:","metadata":{"papermill":{"duration":0.026941,"end_time":"2021-02-01T16:53:16.553285","exception":false,"start_time":"2021-02-01T16:53:16.526344","status":"completed"},"tags":[]}},{"cell_type":"code","source":"dataframe.loc[dataframe['Target']==1]","metadata":{"papermill":{"duration":0.054103,"end_time":"2021-02-01T16:53:16.634846","exception":false,"start_time":"2021-02-01T16:53:16.580743","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, row in dataframe.iterrows():\n    if row['Target'] == 0:\n        dataframe.at[i,'x'] = 264\n        dataframe.at[i,'y'] = 152\n        dataframe.at[i,'width'] = 256\n        dataframe.at[i,'height'] = 379\ndataframe.loc[dataframe['Target']==0]\n\nparsed, pids = parse_data(dataframe, df_detailed)\n\n","metadata":{"papermill":{"duration":11.215636,"end_time":"2021-02-01T16:53:27.892135","exception":false,"start_time":"2021-02-01T16:53:16.676499","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Deteccion de áreas afectadas en la imagen.\nimage_original, image_cropped = crop_image(parsed['00436515-870c-4b36-a041-de91049b9ab4'])\nplt.figure(figsize=(20,10))\nplt.imshow(image_cropped, cmap=pylab.cm.binary)\nplt.axis('off')\nplt.figure(figsize=(20,10))\nplt.imshow(image_original, cmap=pylab.cm.binary)\nplt.axis('off')\n\nfrom skimage.transform import resize\nimage_resized = resize(image_cropped, (128,64))\nplt.figure(figsize=(20,10))\nplt.imshow(image_resized, cmap=pylab.cm.binary)\nplt.axis('off')\n","metadata":{"papermill":{"duration":2.10721,"end_time":"2021-02-01T16:53:30.041035","exception":false,"start_time":"2021-02-01T16:53:27.933825","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Redimensión de imágenes, del set de entrenamiento, en imágenes de 128x64:**","metadata":{}},{"cell_type":"code","source":"dataset_train_im = []\ndataset_train_labels = []\n\nfor i in pids[0:10000]:\n    image_original, image_cropped = crop_image(parsed[i])\n    image_resized = resize(image_cropped, (128,64))\n\n    dataset_train_im.append(image_resized)\n    dataset_train_labels.append(parsed[i]['label'])\n    ","metadata":{"papermill":{"duration":807.584367,"end_time":"2021-02-01T17:06:57.675105","exception":false,"start_time":"2021-02-01T16:53:30.090738","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt = 0\nfor i in dataset_train_im[0:10]:\n    print(dataset_train_labels[cnt])\n    cnt+=1\n    plt.figure(figsize=(20,10))\n    plt.imshow(i, cmap=pylab.cm.binary)\n    plt.axis('off')","metadata":{"papermill":{"duration":0.954494,"end_time":"2021-02-01T17:06:58.680513","exception":false,"start_time":"2021-02-01T17:06:57.726019","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Se divide set de datos: el 90% para entrenamiento, lo restante para el set de prueba, con sus respectivas etiquetas:","metadata":{}},{"cell_type":"code","source":"test_im = []\ntrain_im = []\n\nlong = (0.9*len(dataset_train_im))\n\nfor i in range(len(dataset_train_im)):\n  if i < long:\n    train_im.append(dataset_train_im[i])\n  else:\n    test_im.append(dataset_train_im[i])\n\nprint(len(train_im))\nprint(len(test_im))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = []\ntest_labels = []\n\nlong1 = (0.9*len(dataset_train_labels))\n\nfor i in range(len(dataset_train_labels)):\n  if i < long:\n    train_labels.append(dataset_train_labels[i])\n  else:\n    test_labels.append(dataset_train_labels[i])\n\nprint(len(train_labels))\nprint(len(test_labels))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Modelo:**","metadata":{}},{"cell_type":"code","source":"from __future__ import absolute_import, division, print_function, unicode_literals\ntry:\n  # %tensorflow_version only exists in Colab.\n  %tensorflow_version 2.x\nexcept Exception:\n  pass\n\n# TensorFlow y tf.keras\nimport tensorflow as tf\nfrom tensorflow import keras\n\nprint(tf.__version__)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.Sequential([\n    keras.layers.Flatten(input_shape=(128, 64)),\n    keras.layers.Dense(128, activation='relu'),\n    keras.layers.Dense(1, activation='sigmoid')\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam',\n              loss='binary_crossentropy',\n              metrics=['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Set de datos de entrenamiento:**","metadata":{}},{"cell_type":"code","source":"# Se transforman las imágenes de entrenamiento a escala de grises \ntrain_im = np.array(train_im)\ntrain_labels = np.array(train_labels)\n\nprint(train_im.shape)\ntrain_im_gray = []\nfor i in range(len(train_im)):\n    train_im_gray.append((np.dot(train_im[i], [0.2989, 0.5870, 0.1140])))\n\ntrain_im_gray = np.array(train_im_gray)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Entrenamiento del modelo\nmodel.fit(train_im_gray, train_labels, epochs=10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_im = np.array(test_im)\ntest_labels = np.array(test_labels)\n\n# Se transforman las imágenes de prueba a escala de grises \ntest_im_gray = []\nfor i in range(len(test_im)):\n    test_im_gray.append((np.dot(test_im[i], [0.2989, 0.5870, 0.1140])))\n\ntest_im_gray = np.array(test_im_gray)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Se evalúa la exactitud del modelo:\ntest_loss, test_acc = model.evaluate(test_im_gray,  test_labels, verbose=2)\n\nprint('\\nTest accuracy:', test_acc)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"El modelo reliza predicciones sobre el set de prueba:","metadata":{}},{"cell_type":"code","source":"print(test_im_gray.shape)\nprint(test_im_gray[0].shape)\npredicts = model.predict(test_im_gray)\nthreshold = 0.9\npredicts = (predicts >= threshold).astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ejemplo\npredicts[2]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Se corrobora la predicción con su etiqueta\ntest_labels[2]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nprint(os.listdir('../input/pruebas/'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../input/pruebas/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\n\nimg1 = cv2.imread('../input/pruebas/neumo1.png', cv2.IMREAD_GRAYSCALE)\nimg2 = cv2.imread('../input/pruebas/no_neumo.jpg', cv2.IMREAD_GRAYSCALE)\n#img = cv2.bitwise_not(img)\nimg1 = cv2.resize(img1,(64,128))\nimg2 = cv2.resize(img2,(64,128))\n\nplt.figure()\nplt.imshow(img1, cmap=pylab.cm.binary)\nplt.axis('off')\nplt.figure()\nplt.imshow(img2, cmap=pylab.cm.binary)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#img = (np.expand_dims(img,0))\n#print(img.shape)\nimg1 = np.expand_dims(img1, axis = 0)\nimg2 = np.expand_dims(img2, axis = 0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_single1 = model.predict(img1)\npredictions_single2 = model.predict(img2)\n\nprint(predictions_single1)\nprint(predictions_single2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('Etiqueta correcta')\n    plt.xlabel('Etiqueta predicha')\nfrom sklearn.metrics import confusion_matrix\nimport itertools\n\ncm = confusion_matrix(test_labels, predicts)\ntn, fp, fn, tp = confusion_matrix(test_labels, predicts).ravel()\nplot_confusion_matrix(cm,[\"0\",\"1\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}