{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n#import numpy as np # linear algebra\n#import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n#import os\n'''\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n'''\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-17T03:09:20.807696Z","iopub.execute_input":"2021-10-17T03:09:20.808112Z","iopub.status.idle":"2021-10-17T03:09:20.824353Z","shell.execute_reply.started":"2021-10-17T03:09:20.807994Z","shell.execute_reply":"2021-10-17T03:09:20.823302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Escolha framework\nIniciei tentando utilizar a biblioteca Pytorch, porém após não conseguir avançar na criação do dataset para treinamento, optei por utilizar o Tensorflow mais o Keras.\nTensorflow e Keras se mostram um pouco mais fáceis de compreender e utilizar, tanto na criação do modelo e , principalmente, na criação do dataset de treino.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom PIL import Image\nimport torch\nimport torchvision\nfrom torchvision.datasets import DatasetFolder\nfrom torchvision import transforms\nimport numpy as np\nimport collections\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport re\nfrom pydicom import dcmread\nfrom skimage.io import imsave\nimport os","metadata":{"execution":{"iopub.status.busy":"2021-10-17T10:24:43.89588Z","iopub.execute_input":"2021-10-17T10:24:43.896228Z","iopub.status.idle":"2021-10-17T10:24:50.162913Z","shell.execute_reply.started":"2021-10-17T10:24:43.896197Z","shell.execute_reply":"2021-10-17T10:24:50.162019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Alteração do diretório de trabalho para ter acesso aos arquivos do dataset da competição","metadata":{}},{"cell_type":"code","source":"INPUTDIR_PATH = '/kaggle/input/rsna-miccai-brain-tumor-radiogenomic-classification'\nos.chdir(INPUTDIR_PATH)\nsorted(os.listdir('train/00000'))","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-10-17T10:25:01.788575Z","iopub.execute_input":"2021-10-17T10:25:01.788946Z","iopub.status.idle":"2021-10-17T10:25:01.81336Z","shell.execute_reply.started":"2021-10-17T10:25:01.788915Z","shell.execute_reply":"2021-10-17T10:25:01.812595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Para facilitar a navegação entre diretórios, criei constantes para as principais pastas que utilizei.\nCriado também uma constante como se fosse um enumerador para cada tipo de ressonância.","metadata":{}},{"cell_type":"code","source":"# Declaração de constantes\nTRAIN_FOLDER = 'train'\nTEST_FOLDER = 'test'\nmri_types = collections.namedtuple('mri_types', ['FLAIR', 'T1W', 'T1WCE', 'T2W'])\nMRI_TYPES = mri_types('FLAIR', 'T1w', 'T1wCE', 'T2w')\nPNG_DATASET_DIR = '/kaggle/working/png_dataset'\nPNG_TEST_DIR = '/kaggle/working/png_test'\nWITH_MGMT_DIR = '/kaggle/working/png_dataset/with_mgmt'\nWITHOUT_MGMT_DIR = '/kaggle/working/png_dataset/without_mgmt'\n","metadata":{"execution":{"iopub.status.busy":"2021-10-17T10:25:10.812046Z","iopub.execute_input":"2021-10-17T10:25:10.812369Z","iopub.status.idle":"2021-10-17T10:25:10.817912Z","shell.execute_reply.started":"2021-10-17T10:25:10.81234Z","shell.execute_reply":"2021-10-17T10:25:10.816725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Criado os diretórios para salvar as imagens DICOM, que é o formato da ressonância magnética, convertidos em arquivos PNG.","metadata":{}},{"cell_type":"code","source":"os.mkdir(PNG_DATASET_DIR)\nos.mkdir(WITH_MGMT_DIR)\nos.mkdir(WITHOUT_MGMT_DIR)\nos.mkdir(PNG_TEST_DIR)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T10:25:13.93236Z","iopub.execute_input":"2021-10-17T10:25:13.932702Z","iopub.status.idle":"2021-10-17T10:25:13.940004Z","shell.execute_reply.started":"2021-10-17T10:25:13.93267Z","shell.execute_reply":"2021-10-17T10:25:13.93901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Funções auxiliares\nAqui foram criadas três funções auxiliares.\n\nA img_loader(path) é para carregar a imagem DICOM, verificar se é uma imagem vazia e retornar ela no formato de array. Caso o arquivo seja vazio, retorna 0.\n\nA função png_save(path) chama o método acima, verifica se é uma array ou 0 e caso seja um array, converte e salva a imagem DICOM no formato PNG.\n\nimgs_path_finder(folder, patient) percorre todos os pacientes e seus respectivos subdiretórios e retorna o caminho de cada arquivo DICOM de forma ordenada no formato de uma lista.","metadata":{}},{"cell_type":"code","source":"def img_loader(path):\n    ds = dcmread(os.path.join(INPUTDIR_PATH, path))\n    if (ds.pixel_array.sum() != 0):\n        arr = ds.pixel_array\n        # arr = tf.constant(arr)\n        return arr\n    else:\n        return 0\n    \ndef png_save(path):\n    image = img_loader(path)\n    if isinstance(image, np.ndarray): \n        fname = path.replace('/', '-')\n        fname = fname.replace('dcm', 'png')\n        imsave(fname, image)\n        \ndef imgs_path_finder(folder, patient):\n    images = []\n    image_path = []\n    for mri in MRI_TYPES:\n        images = sorted(os.listdir(os.path.join(folder, patient, mri)), \n                                                key=lambda file: int(re.sub('[^0-9]', '', file)))\n        for img in images:\n            path = os.path.join(folder, \n                                patient, \n                                mri,\n                                img)\n            image_path.append(path)\n            print(path)\n    return image_path","metadata":{"execution":{"iopub.status.busy":"2021-10-17T10:25:25.100294Z","iopub.execute_input":"2021-10-17T10:25:25.100618Z","iopub.status.idle":"2021-10-17T10:25:25.108729Z","shell.execute_reply.started":"2021-10-17T10:25:25.100587Z","shell.execute_reply":"2021-10-17T10:25:25.107908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Análise exploratória\n\nVerificamos alguns dados do arquivo .csv que contém a classificação de cada paciente para a presença do MGMT ou não da pasta TRAIN do dataset.","metadata":{}},{"cell_type":"code","source":"df_label = pd.read_csv(\"train_labels.csv\", dtype={'BraTS21ID':str, 'MGMT_value':int})\ndf_label.MGMT_value.unique()\ndf_label.describe()\ndf_label.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-17T10:25:41.031454Z","iopub.execute_input":"2021-10-17T10:25:41.031819Z","iopub.status.idle":"2021-10-17T10:25:41.05131Z","shell.execute_reply.started":"2021-10-17T10:25:41.031785Z","shell.execute_reply":"2021-10-17T10:25:41.050525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Gerado duas variáveis contendo a lista de pacientes do diretório de TRAIN e TEST.","metadata":{}},{"cell_type":"code","source":"# Lista os pacientes que fazem parte do diretório de treinamento\ntrain_patients = [subdirs for subdirs in os.listdir(TRAIN_FOLDER)]\nprint('Número de pacientes no diretório de treino', len(train_patients))\n\n# Lista os pacientes que fazem parte do diretório de teste\ntest_patients = [subdirs for subdirs in os.listdir(TEST_FOLDER)]\nprint('Número de pacientes no diretório de teste', len(test_patients))\n\nprint('Total de pacientes', len(test_patients)+len(train_patients))","metadata":{"execution":{"iopub.status.busy":"2021-10-17T10:25:57.657167Z","iopub.execute_input":"2021-10-17T10:25:57.657511Z","iopub.status.idle":"2021-10-17T10:25:57.780315Z","shell.execute_reply.started":"2021-10-17T10:25:57.657481Z","shell.execute_reply":"2021-10-17T10:25:57.779432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Função de exclusão de casos com falha no dataset\nNa descrição da competição, foi informado que os diretórios dos três pacientes 00109, 00123 e 00709 estavam gerando erro no processo de treinamento e orientavam realizar a exclusão deles.\nA função abaixo serve para eliminar as pastas dos três pacientes.","metadata":{}},{"cell_type":"code","source":"# Exclusão dos pacientes 00109, 00123, 00709 devido a falha do dataset\npatients_delete = ('00109', '00123', '00709')\ntry:\n    for patient in patients_delete:\n        df_label = df_label[df_label.BraTS21ID != patient]\n        train_patients.remove(patient)\nexcept Exception as err:\n    print('erro: ', err)\nprint('Número de pacientes no diretório de treino', len(train_patients))","metadata":{"execution":{"iopub.status.busy":"2021-10-17T10:26:07.630425Z","iopub.execute_input":"2021-10-17T10:26:07.631091Z","iopub.status.idle":"2021-10-17T10:26:07.643023Z","shell.execute_reply.started":"2021-10-17T10:26:07.631049Z","shell.execute_reply":"2021-10-17T10:26:07.642133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Após ordenação dos pacientes e dos arquivos de imagem, aparentemente a busca pelos caminhos da fotos ocorreu mais rápido, por isso eu faço a ordenação das listas de pacientes  abaixo.","metadata":{}},{"cell_type":"code","source":"test_patients = sorted(test_patients)\ntrain_patients = sorted(train_patients)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-10-17T10:26:12.587499Z","iopub.execute_input":"2021-10-17T10:26:12.587843Z","iopub.status.idle":"2021-10-17T10:26:12.607626Z","shell.execute_reply.started":"2021-10-17T10:26:12.587812Z","shell.execute_reply":"2021-10-17T10:26:12.60674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Criado a lista contendo uma tuple com o identificador do paciente, o caminho da imagem DICOM e a classificação de presença de MGMT para o diretório TRAIN.\n\nCriado também a lista com uma tuple com o identificador do paciente e o caminho da imagem DICOM para o diretório TEST. Não foi fornecido no dataset da competição a classificação de MGMT.","metadata":{}},{"cell_type":"code","source":"# retorna uma lista de duas dimensões\n# necessário a conversão de patient para list de forma que todos os elementos sejam do tipo list\n# acrescenta o label de presença de MGMT\nimages_path = []\nfor patient in train_patients[:25]:\n    images_path.append([\n        [patient],\n        imgs_path_finder(TRAIN_FOLDER, patient),\n        [str(int(df_label[df_label['BraTS21ID']==patient].MGMT_value))]\n    ])\n\ntest_images_path = []\nfor patient in test_patients[:10]:\n    test_images_path.append([\n        [patient],\n        imgs_path_finder(TEST_FOLDER, patient)\n    ])","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-10-17T10:26:36.600138Z","iopub.execute_input":"2021-10-17T10:26:36.600453Z","iopub.status.idle":"2021-10-17T10:26:44.259049Z","shell.execute_reply.started":"2021-10-17T10:26:36.600424Z","shell.execute_reply":"2021-10-17T10:26:44.249779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Diretório Imagens Treinamento\nAtravés do código abaixo, verificamos para cada paciente o Label de classificação para MGMT e chamamos as funções auxiliares para carregar e salvar os arquivos PNG na seguinte estrutura:\n\n![image.png](attachment:4e65bf0f-37d9-475e-8925-9bded6015d1f.png)\\\nEsse tipo de estrutura foi utilizado para utilizar as facilidades da biblioteca Keras, que a partir desse formato de diretórios automaticamente gera uma dataset para ser utilizado no treinamento dos modelos de rede neural.\nO Keras entende os subdiretórios WITH_MGMT_DIR e WITHOUT_MGMT_DIR como sendo a categoria e fazendo a associação dessa classe a cada imagem dentro do subdiretório.\n\n\n","metadata":{},"attachments":{"4e65bf0f-37d9-475e-8925-9bded6015d1f.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAWEAAAEPCAYAAACTAXX9AAAgAElEQVR4nO3d/08U56I/8Pun7Cb+E5Dwk8abe3JRY0DxRBTRtKGmUgOhDfIhjXpsTy9Q0lvtOUuIVctRey4GOMW7xH4kLtUGRTFs1ex6bj1wrUvnENguWTLJPsn7/kCeh5nZmdld2PVZ4L3JK2G/zT7PMzPvnXnmWZ5/CQR3gIiI9PgX3QUgItrOGMJERBoxhImINGIIExFpxBAmItIo7xCeiUZhvaXTK+jvv6yeD4fHYJom9uzdrx4LhfoAQN2vqKzCgwc/IpPJAABi8TgOHDyUd2HD4TH1+UII/GN2Fu+dfN/1tQMD12GaJioqq2yP+91CoT7bayORCbx+/dp1+Rc++RSLi0sAANM0EQ6PZdXbeTMMI6seXq/JpaW1zfa+xcUl2/rw+xxZVmc5nfUgotIrKITlDrpz1258/fUVCCHQ1d2DQHBthx8aHlHvcYbw5ORDLC4uorn5NGpq6xCLxZBIzOdd2HB4TIXU4fqjuP/gAYQQ+PCjdtfyAlDlc2MYRlbwOp8HgIZjx22Pt7S2QQiBGze/RU1tHXo+70U6vYLR0duu9fYTCvXlHbzOMgBAS2sbqqv34U9/CiGTybiGqPW1zs+W5ayorEL7mQ5bPYio9NYVwtKjqSnMRKMIBNdCOJ1eUUef1p18z979EEKgs/Nj9f49e/fnDEorawhLk5MPMTs7l/Va0zRhGAYikQnP5fmFcMOx4+rIdGDges5yDA2PwDTNrHrnUowQlo91dfdACIF33m3K+Vqvcg4MXEcqldK+YRJtFxsKYWsYhcNjWFhYwNJSUoWWdScPhfpcd+7/f3cc//mfF/Mqg1v4nT13Putotau7B6Zp5gw4vxAeHLyF2dk5hMNj6otGGh29jVQqZevq2LlrNxobT2TVO5dihnAguAOpVAqDg7fyeq1bOT/8qD3vshPRxm0ohCORCcTjLxEIrgXk0PAIlpaSCATtO7lbgBbKaxkAcPbceVu5ZqJRdaTt7E6Q/EI4Hn+JcHgMZ8+dV0e40oGDh5BOr+Cf//wn/vSnUFa/s84QnolGs47+Cz0SdtaXiEpn3SHc0dEJIQSuXL2GQHAtICsqq9RR6NsMYWuYWrsQZmfnsroTrK9zC+GKyioIIVSwm6ZpC/lAcAfqjzQgFo8DQFZfrNeFOWcIytcWO4SdR+75hnD9kQak0yu+XThEVFzrHh0hhLAFj7Nr4vXr174hLC96Acj7inw+R8LvvNtkO/qNRCYQi8Vcl+cVwr29X9iOBuPxl7g7Pu66jOrqfbg7Pm5rD1nvxsYTNs4jZvla3UfC1tvz5y9cy0lEpVFQCN+7F1GB4nzeGpDyItyjqSkVwhcvfWULtsP1R9HYeAILCwsbCmHZhylDd3DwVlawCCFcg8UrhCcnH2Ytw/q5h+uPorp6n+09N25+CyEEAkG93RELCwsF9wnLdWoYBoeoEb1lG+oTtnIGpDw6lGEkg7m39wvb+wrZ8fMZHRGPv8QP9+/bjkBN03QdgeEVwqlUCv/1X4Pq/Wc6/p8t6Gdn5/Boasr2HudFSB0hvNHREaFQX9ZYbyIqrZKF8IGDh2whLF+TTq+g/UwHqqv34fz5CzBNEze//WteZcg1TtjZl2stu1s/p1sIy8ByBpG1n/nipa8AQI0Tbj/TgURiHtPTTxEIbs5xwtZ68miY6O0pWQgHgmun9c7XyV/Muf3Ky0+uX8w5+3KlgYHrrkHnFsJDwyOu447liAt5/7PPutQv5jKZDB4/fpI1PtqvS0PaaAjL20Z+MecsjxCioF8yEtH6ldX/jnBe/JM359X+7cArQDc6woSIyktZhXBNbV3WiILGxhOoqa3TXra3rbp6n2tbHK4/qr1sRFQ8ZRXCRETbDUOYiEgjhjARkUYMYSIijRjCREQaMYSJiDRiCBMRaVRwCIdCfdvyxxNERKXAECYi0oghTESkEUOYiEgjhjARkUYMYSIijRjCREQaMYSJiDRiCBMRacQQJiLSiCFMRKQRQ5iISCOGMBGRRgxhIiKN+K8siYg0YggTEWnEECYi0oghTESkEUOYiEgjhjARkUZbaoja6OhtzM7OqXKapolAcAdaWtsAQD3udpN1molGXZ93viYcHrN9djg8BsMw8i4rAMTjL9X9zs6PAcC23PojDXjz5g0AIJPJ2J6TdYpEJtRjV65eAwCEQn3qea9bS2tb3uWUt3R6BUPDI1nbg19bOZchhEAsHseBg4e0by9E5WBLhXAo1KeCMBweAwA0HDuOru4epFKprNfLwPJaXjg85lrXYoWwEAIVlVUIBHfg7vi4LYQrKquQTq9gcvIhamrr0H6mA+n0CkZHbyMQXAth62fGYjHPOhUSvG5ttHPXbpw/fwHLy8ue69/vs+Xjh+uPIhaLIZGY1769EJWDLRXCH37UDiEEAsG1I9qu7h5bOFvpDmHTNNXnG4YB0zTVcq9cvYalpaTtPQMD17OO7k3TREtrGyoqqyCEsC3T+XkbCWF5/513myCEQFd3T97t6Xx8z979AICz585r32aIdNtSIXzq1AcAVrsdDMPA7OwcRkdve4ap7hCeiUYxOfkQLa1tME3TttyZaNTW1RAI7kDDseN48mQah35/WIXwTDSKwcFbCIX6MDs7B8MwShrCgeAOxOMvMTn5MO/2dHt8YWHBt+2JtostFcKB4FrYCCEwNDyC6emnmJx8WFBoSKUO4StXryGVSmFoeASPpqZsyzUMI+szrGQIX7l6DfH4SzyamsLQ8MhbCeFweMzWn52rPb2OhN2Opom2my0XwqlUCqFQHxYWFtDV3YNEYt41NAPBjYWw263QEG5pbUMqlUI6vaLatdAQ3rN3v+qGaGlte2shXEj3jvXxisoqzESjWFpKqv5wou1sy4VwIjGP6emnePbsueqe8Aum9YbwvXsRNDaeUO7dixQcwmfPncfk5EN1gc4rhJ0jHVpa22whLLsh5PvK8UjYelteXsZ7J9/Xvq0QlYMtF8Iz0SgMw1D9qUIIz1Nf3d0RoVAfenu/QCwWy1qu7EYJBFePHhsbT+CPf/yPrBBuaW3D0PAI7o6PIxB8OyH87NnzgvuEb9y4icbGE7g7Pl5QOxFtdVsuhCORCQDAwMB1BIKrR8ZeAVQOIey13CtXr8E0TdspuzV4rX9bl1HqEN7o6Ig9e/d7juAg2o62XAjL8cFy+JPsv3V7bTmHcEVlFRKJecRiMdTU1uFw/VH87W/fQQiBhmPH33oIF2uc8Hraimgr23IhLH/B1XDsOALB1R1ejq21Pu+8udVJZwgHgjtw4OAhxOJxAKs/7PjH7KzqS32bISxvG/nFnHN0hGma6myFaDsrWgjLI1DnTQaT389oZTh4jToo59B3k6stysVmKSfRVla0EK6u3mcbLSAdrj+KQHDt4pIb2e9ZU1vn+nxNbZ32hipErrYoF5ulnERb2ZbrjiAi2kz4ryyJiDRiCBMRacQQJiLSiCFMRKQRQ5iISCOGMBGRRlt2iNpmKScRbW8MYSIijRjCREQaMYSJiDRiCBMRacQQJiLSiCFMRKQRQ5iISCOGMBGRRgxhIiKNGMJERBoxhImINGIIExFpxBAmItKI/8qSiEgjhjARkUYMYSIijRjCREQaMYSJiDRiCBMRacQhapq1n+kAALSf6SjJ8i988imSySQAIJ1eQW/vF7bn3zv5PubnfwUAmKaJwcFbWcsIh8eQyWQAAG/evMGBg4cKWkZFZRUeP34CIQSEEIjF46iorMq7Di2tbfC6ydfMRKN5r+9UKoWh4ZGsx3N9Rjg8Zns8nV5Bf//lgtaHYRiun9HS2qZ9W9wIwzBgGEbW4zt37cbjx09gmuam2B91YAj7vN+6A25G77zbBCEEbtz8FjW1dfjzn/sghFA7fEVlFdLpFUQiE6iprcMf/vAJTNPExUtfqWUMDt5COr2C9jMdqKmtw6OpKSQS8+r5fJYxOfkQicQ8mppO4nD9UczOzmF6+mne9ZAh7PeafENYLmt2dq7gZYXDYypodu7ajZ7PeyGEsNU1X9ZlbXaDg7cAIKs+e/bux/LyMl69eoX2Mx3YuWu39rKWI4awz/s3ewiPjt7G69evbY/F4y8RiUwgENyBi5e+gmmatqPSSGQC8fhLdT+RmLcdNe7Zux8AcPbc+byXIYSwHYGfPXceALBn7/686lHMEB4cvKWORr0+P58Q9qprvrZKCB84eAhCCMxEo1n1uTs+DsMwCjrr2Y4Ywg5ep77ODUzuRP39l2GaJgDg8eMn6nlrN0Amk0E4PGZ7/0w06ns6Ggr1wTAMfHnxklp+oafxzjCpqKxCPP5ShYZbEPxw/z5SqZS6DwChUJ+6/97J92GaJgYGrue1DNme1vpd+ORTW5Dnu04KqauX2dk5FcTWeuWzLLe69vdfXleYeoWwYRh49eoVhBBIJpN48OBHAMDc3P+q1xw4eAixeFxtO7F4PKuLyLnd/Pzzz7Zt0LqMTCaDBw9+XNd+Nj39FJHIhGt9UqkUrly9tq7lWveBBw9+VF1ZznIahoGvv76i6mKaZlaXW0dHp9oX5+d/xZMn02WVYQxhn/f77fjh8Nhq/2YshsP1R1FTW4fq6n0IBO3dANXV+9QputsG6RZS8vOFEHj+/AVqauvQ3Hwa6fQK7o6P510Ha5gcOHgIv/zyCzKZjNpZrDtORWUVnj9/ofp+5TKsIXzhk09hmiaEEGqHzrUMZ/0GBq6rcPAKQa82yreuXuRRfEtrGyKRCUxOPixoWV5Hwn5dG37bj1cI3x0fR1PTSQgh1N/WL63Z2TnEYjHU1Nap7p1YLJbVXtZuKABqnVVUViGRmFddSO1nOpBOr2B09HZBdejq7kE6vYI9e/dn1afh2HEAwHffjSKZTK7rWoDcB2U5ZfeP9ZqDYRgwTRM9n/eiprYOd+58DyGEOsvZs3c/TNPM6i4rpwxjCOfYALyeD4fHsk7DpZ27dqOx8YTtsZlo1LUf1C+EAdiWf+fO9wXt8DJMOjo6kU6vYGh4xLazyL/rjzRgcXERk5MP0d9/2TWEh4ZHkE6voKOjE4ZhZIWw1zJk/T78qB2Tkw+xuLiI+iMN6wph583aZvmEcCjUp47Qu7p7bEf8bu3mts6tQdPR0QkhhDorKIRfCMt2sf5tba/GxhO2/lX5hS3vRyITWd1Qz549V+vsytVrWXUfGLiOhYWFvMtfUVmlzgTd6iPX2eLiItrPdKC5+bTaPjayDzq/9AzDwJ0739vKBQBd3T2qXs799M6d78sqwxjCBWwAVrn69D77rAtv3ryBEEKFhlt5coWw32c6uzSsRzvy+XR6BZlMBhc++TRrGfKLJJPJqCBxfi4ALC8vI5GYV6e8zhD2W4as3/LyMmKxmNoZrKHiFbLONmpsPGFj3bHyCeFHU1MqBORRcWfnx1mv8wth5815ZpLryyLX9pNPCNcfaUA0+pM6o3C210w0qvr9rY9Z15nXza+u1jYZHb1tC0OvEP7wo3b1WG/vF7Yvi/Xsg6FQH0zTtLWXs6vP2lbh8FhWn304PFZWGcYQLmADcK5IrxCWG2A4PKYC48WLF0UP4Zrauqxgkl0igeBaCL938n3bMqx9wkIIFdDyc519wtbwDARXN3xrn7DfMmT9nKEArJ1eV1RWZdXDeiZRrO4IZ2jJdZTvssLhMSwsLKCx8QSam0/DNM2so3m3eridLW0khGW/cXPzaTQ2nsCNGzcLDuF4/KVvm1dX78t6rqa2Tj1vPbhw3lpa21y3a/nYv/3u39e9DzKEg9snhLu6e9YdwqOjt7Oee/bsedFDOJfBwVtZ3RfWHbSru8e2QcvPsG608kKWvF9RWQUhhArQfJZhmqY6PbTW+W2Ojjh77jyEECq4GhtP4PZ/h127d/LtjtjICIf1hvCpUx9kbS/OLqS74+O+3RHyFN3Z/oX01zoD+t69iPqCkstxXiQrxpGwW3eEXwhfuXqN3RG6bLSc8sLCN9/8BdXV+9DcfNr2vN8OKANcXiy4cfNbNYzH+dpShrAcPiQv0MgLG9ZxwktLyawLNNaju4GB67ZxwpHIhK0M+SwjEpmwjROOxWIFrZt8Q/jFixeeZwZuR0ReXwb5hrC86JNv33Y+6zKfI2HTNPHD/fuqvZeXl23t4zY+3HrUL8d2v3r1Sq2T6emniEZ/Wvf+4laf0dHbSKdX0NR0Ek1NJ5FIzLv2CcvyTkz84LoPfPfdqG37tfbB5wphr3Hs5ZRhDGEfg4O31JX+xcUl22lUrkCUvzKTV4XvP3iAn356pp5368+17ijFCOFAMPsXc19evGR73vlrN7dfkjl/MVd/pKGgZbj9Yq6Qgfv5hrBfe75+/dr114CpVMoWdG43v/YPh8ewtJTM+ygy1y/m8glh6zpdXFzC0PAIfvvtNxz6/WH1Ob29X/gOUas/0qCGdcl14hzmVgivbdM6vOzx4yeu7ZQrhK3LcPbB5wrhQHAHPvyonUPUdNgs5SQqBWfYGYZR8BA03Yr1gylnW3iNVNKFIUy0xZw69QGSyaTqQhoaHoEQAu+826S9bIUoRgjv/tffIZlMqq5B+dN96zUK3RjCRFvQzW//inR6BQCQTCZto1c2i2IdCX958RIWF5dUd1mh/3Sp1PivLImINGIIExFpxBAmItKIIUxEpBFDmIhII4YwEZFGHKKmGeeYy41zzJU/5xxzcniZ2209P/PeyhjCPu8vxhhFnTjHnPeyOMdc8XjNMefU2fmx7R+u0yqGsM/7N3sIc465bJxjrrj85phzejQ1hUdTU9rLXG4Ywg6cY45zzHGOufz5zTFntWfvfgghXP+JvhfOMefTMOVUgVKVk3PMrS7DGsKcY45zzFn5zTHnNDQ8knVWlu8+yDnmXBqmnCpQqnJyjrnVZciw5BxznGPOKtccc05LS8mC24lzzPk0TDlVoFTl5Bxzq8sAOMectf2dN84xl3t/kDNqFHpBjtMb+TRMOVWgVOXkHHNrGzTnmFtrO84xt/p8rjnmrJ8di8XWdUGOIezTMOVUgVKVk3PMrd7nHHPe7c855vznmAsE/c84cuEccz4NU04VKFU5Ocfc6jI4x5x3+3OOudz1yeeCHOeYK/AN2yWEA0HOMeesC8A55jjHnPf27mwb+QXtNirIinPMFfiG7RTCRJsV55jzbgvOMfcWV+BmKCdRsXGOuTWcY07zCtwM5SQqBc4xt4ZzzBERkSeGMBGRRgxhIiKNGMJERBoxhImINGIIExFpxCFqREQaMYSJiDRiCBMRacQQJiLSiCFMRKQRQ5iISCOGMBGRRgxhIiKNGMJERBoxhH1MTj5EJpNBw7Hjrs97Tcsiyfmt/v73/9HeHkRUnhjCPhjCRFRqDGEiIo0YwkREGjGEiYg04r+yJCLSiCFMRKQRQ5iISCOGMBGRRgxhIiKNGMJERBpt2SFqRESbAUOYiEgjhjARkUYMYSIijRjCREQaMYSJiDRiCBMRacQQJiLSiCFMRKQRQ5iISCOGMBGRRgxhIiKNGMJERBoxhImINOK/siQi0oghTESkEUOYiEgjhjARkUYMYSIijRjCREQabakhaqOjtzE7O6fKaZomAsEdaGltAwD1uNtN1mkmGnV93vmacHjM9tnh8BgMw1D36480YH7+VwCAaZro779sez0AtLS22R4zDEMt1+8WCvXl1R6yLnv27lePLS0ls8r55s0bAEAmk8mqFwDE4y/V/c7OjwGgJOWUZYhGf8KBg4ds25zf+pBtZ729efMG7518X/s2SZTLlgrhUKhPBUw4PAYAaDh2HF3dPUilUlmvzxUU4fCYa11zhXBFZRWWlpKIRCZQU1uHns97IYRAV3eP7bP9Qtj5eL6B5iyntY4yQK3lTKdXMDn5EDW1dWg/04F0egWjo7dt5RRCoKKyCoHgDtwdH7eFcLHKKdu5ufk0nj9/gXR6xRbEudaZte1qausQiUwgnV6xfQERlaMtFcIfftQOIQQCwbUA6urusYWzValC+MrVa1haSmY9L4/S5We/jRA2TVPVIRweg2mavuUcGLiuziBkOU3TVJ9vGAZM0yxZCEuzs3OYnHyY9zpza7tUKoWBgevat0siP1sqhE+d+gDAareDYRiYnZ3D6OhtzzAtVQhPTz/NCpAPP2oHAHVE+bZCWAaxXM7k5ENVzploFJHIhO09DceO48mTaRz6/WFVzploFJOTD9HS2qZCvdQh7PwyyLXO3Nru2bPnruUkKidbKoQDwbVwE0JgaHhEBWIhR1XSekN4dnYu63nZLy2D922FcDg8hqWlJPr7L6u+aVlOr89zttGVq9eQSqUwNDyCR1NTbyWEZXs5uxMKPRIeHLylfZsk8rPlQjiVSiEU6sPCwgK6unuQSMx7hsZGQtjt5hduOkNY9o/ORKO2rpl8Q7iltQ2pVArp9Ipa/28rhJ1tlG8I37nzPYQQrv3KROVky4VwIjGP6emnePbsueqe8AqHjYTwvXsRNDaeUO7di5TlkXAkMoGu7h5VV68QluWTN2s5z547j8nJh+oCXbkeCVtvmUwGFz75VPv2SJTLlgvhmWgUhmGovk4hBADYRiZIb7NP+Oy583n1CVtHJlgf30i4yVEQe/but4WwtZwVlVVobDyBP/7xP7JCOBTqQ2/vF4jFYp71L0Y5rY/J7pN815lhGOqL8Ztv/gLTNDkygjaFLRfCkcgEAKir4onEvGvgBYKlHR3hHBLnHB3hvHJfUVmVNYxNKma4WUP4ytVrME1TfTEEgu5H7M7PfhshvNHREfl0tRCVgy0XwnJ88Nlz5xEIrvXfur22VCHsNU744qWv1OsHB28hnV5B+5kO1NTWIRaLIZGYdy1HqUK4orIKicQ8YrEYamrrcLj+KP72t+8ghEDDseOebVTKEC7GOGFZTx4N02aw5UJY/rpKhogcG+t83nlzq9N6QzgQzP2LOfmeTCYDAIjF454XkUoVwoHgDhw4eAixeBzA6o8y/jE7a/ul2dsKYWtf7np/Mecsk7VbiqhcFS2E5RGo8yZ3eOeFH+tNnvp6jToo59DXJZ/2LAebpZxEuhQthKur99lGC0iH648iEFy78ONG9knW1Na5Pl9TW6e9ocpNPu1ZDjZLOYl02XLdEUREmwn/lSURkUYMYSIijRjCREQaMYSJiDRiCBMRacQQJiLSaMsOUdss5SSi7Y0hTESkEUOYiEgjhjARkUYMYSIijRjCREQaMYSJiDRiCBMRacQQJiLSiCFMRKQRQ5iISCOGMBGRRgxhIiKNGMJERBrxX1kSEWnEECYi0oghTESkEUOYiEgjhjARkUYMYSIijThETbP2Mx0AgPYzHSVZ/oVPPkUymQQApNMr6O39wvb8eyffx/z8rwAA0zQxOHgraxnh8BgymQwA4M2bNzhw8JDn5xmGAcMwbOvB6xYK9eVVh5bWNs9lyNfMRKN5r+9UKoWh4ZGsx3N9Rjg8Zns8nV5Bf//lgtaHYRiun9HS2qZ9W9wI53qXdu7ajcePn8A0zU2xP+rAEPZ5v3UH3IzeebcJQgjcuPktamrr8Oc/90EIoXb4isoqpNMriEQmUFNbhz/84ROYpomLl75SyxgcvIV0egXtZzpQU1uHR1NTSCTmXT9vcPAWALjujFadnR9DCIE9e/fnVQ8Zwn6vyTeE5bJmZ+cKXlY4PKbqtnPXbvR83gshhK298mVd1mbntd737N2P5eVlvHr1Cu1nOrBz127tZS1HDGGf92/2EB4dvY3Xr1/bHovHXyISmUAguAMXL30F0zRRUVmlno9EJhCPv1T3E4l521Hjnr37AQBnz523LffAwUMQQmAmGs0ZLo+mpvBoairvehQzhAcHb6mjUa8vgXxC2Ku98rVVQthvvd8dH4dhGLbti7IxhB28Tn2dG5jcifr7L8M0TQDA48dP1PPWboBMJoNweMz2/plo1Pd0NBTqg2EY+PLiJbX8WDxe0AbtDJOKyirE4y9VaLgFwQ/37yOVSqn7gL3b4L2T78M0TQwMXLe9b3r6KSKRiZzhsmfvfggh0Nn5ccHrpJC6epmdnVNB7NUdUkgI9/dfXleYerWTYRh49eoVhBBIJpN48OBHAMDc3P+q1xw4eAixeFxtO7F4PKuLyLnd/Pzzz7Zt0LqMTCaDBw9+XNd+5rfeU6kUrly9tq7lWveBBw9+hBACQoischqGga+/vqLqYppmVpdbR0en2hfn53/FkyfTZZVhDGGf9/vt+OHwGIQQiMViOFx/FDW1daiu3odA0N4NUF29T53mu22QMmDcQlgIgefPX6Cmtg7NzaeRTq/g7vh43nWwhsmBg4fwyy+/IJPJqJ3FuuNUVFbh+fMXqu9XLsMawhc++RSmaUIIYduhu7p7kE6vYM/e/TlDeGh4JOvoPJdihbA8im9pbUMkMoHJyYcFLcvrSNiva8Nv+/EK4bvj42hqOgkhhPrbevYxOzuHWCyGmto6HK4/qu4728vaDQVArbOKyiokEvOqG6r9TAfS6RWMjt4uqA5+673h2HEAwHffjSKZTK7uKwUeRMh9UJZTdv9Yr1sYhgHTNNHzeS9qautw5873tq6uPXv3wzTNrC63csowhnCODcDr+XB4LOtUXtq5azcaG0/YHpuJRjE9/TTrtX4hDMC2/Dt3vi9oh5dh0tHRiXR6BUPDI7adRf5df6QBi4uLmJx8iP7+y64hPDQ8gnR6BR0dnTAMw7ZDyzMC6zK9yrS0lMw6is7F6+zE2mb5hHAo1KeO8ru6e2xH/G7t5rbOrXXr6OiEEKLg+vi1k/UI3fq39cuwsfGErX9VfmHL+5HIRNYX3bNnz9U6u3L1WlbdBwauY2FhIe/y51rvcp0tLi6i/UwHmptPq21sI/ug80vPMAzcufO9rVwA0NXdo+rl3E/v3Pm+rDKMIVzABmCVK2w++6wLb968gRBChYZbeXKFsN9nOrs0rEc78vl0egWZTAYXPvk0axnyiySTyaggcX4uACwvLyORmMo6PYEAAAW9SURBVFenvNYQHh29bdsp/Nqlt/cL1wtyXiHrfL6x8YSNdcfKJ4QfTU2pEJBHxW7dIn4h7Lw5z0xyfVnkaqd8Qrj+SAOi0Z9Ud4OzvWaiUdXvb31MrjO3ejiX4fYaa5vkWu9ynX34UXvW+t/IPhgK9cE0TVt7Obv6rG0VDo9l9dmHw2NllWEM4QI2AOeK9AobuQGGw2MqMF68eFH0EK6prcsKJtklEgiuhfB7J9+3LcPaJyyEUAEtP9fZJxyLxWyBZxiGCm3rl0yu8InFYq4X5Coqq7LqYT2TKFZ3hDO05DrKd1nh8BgWFhbQ2HgCzc2nYZpmVr+yWz3czpY2EsKy37i5+TQaG0/gxo2bBYdwPP7St82rq/dlPVdTW6eez7Xe3bZr+di//e7f170PMoSD2yeEu7p71h3Co6O3s5579ux50UM4l8HBW1ndF9YdtKu7x7ZBy8+wbrTyQpa8X1FZBSGE6p907qj37kVUUFnDx+/IM5dihPDZc+chhFDB1dh4Arf/O+zavZNvd8RGRjisN4RPnfoga3txdiHdHR/37Y6Qp+jOM5JC+mvzWe/Oi2TFOBJ2647wC+ErV6+xO0KXjZZTXlj45pu/oLp6H5qbT9ue99sBZYDLiwU3bn6rhvE4X1vKEJbDh+QFGnlhwzpOeGkpmXWBxnp0NzBw3TZOOBKZ8C2DVxnXc0HO2UZ+r5mJRvHixQvPMwO3IyK5XGcY5RvC8qJPvj86yaed8jkSNk0TP9y/r9bZ8vKyrX3cxodbj/rl+PBXr16hqekkDtcfxfT0U0SjP617f3Grz+jobaTTK2hqOommppNIJOZd+4RleScmfnDdB777btS2/Vr74HOFsNdY+HLKMIawj8HBW2q0wOLiku00Klcgyl+ZyavC9x88wE8/PVPPu/XnWneUYoRwIJj9i7kvL16yPe/8xZzbL8mcv5irP9LgW29nGWXYr3e4Ur4h7Neer1+/dv01YCqVsgWd282vbuHwGJaWknkfReb6xVw+IWxdp4uLSxgaHsFvv/2GQ78/rD6nt/cL3yFq9Uca1LAuuY36/RIyF69t0zq87PHjJ67tlCuErctw9sHnCuFAcAc+/KidQ9R02CzlJCoFZ9gZhlHwEDTdivWDKWdbeI1U0oUhTLTFnDr1AZLJpOpCGhoegRAC77zbpL1shShGCO/+198hmUyqrkH50305hK0cMISJtqCb3/4V6fQKACCZTNpGwGwWxToS/vLiJSwuLqkut0L/6VKp8V9ZEhFpxBAmItKIIUxEpBFDmIhII4YwEZFGDGEiIo04RE0zzjGXuw6cY678lWK9bxcMYZ/3W3fAzYhzzHkvi3PMFU+p1vt2wRD2ef9mD2HOMZeNc8wVVynX+3bBEHbgHHOcY45zzOWvlOudc8z5NEw5VaBU5eQcc6vLsIYw55jjHHNWpV7vnGPOp2HKqQKlKifnmFtdhgxhzjHHOeas3sZ65xxzPg1TThUoVTk5x9zqMgDOMWetm/PGOebWv97Xsw9yeqMgQ9i6IjnHHOeY4xxz61/v69kHGcLB7RPCnGNu9T7nmPNuf84xV9r1zjnmfBqmnCpQqnJyjrnVZXCOOe+6cY654qx3zjFX4Bu2SwgHgpxjzlkXgHPMcY654q93zjFX4Bu2UwgTbVacY867LTjH3FtcgZuhnETFxjnm1nCOOc0rcDOUk6gUOMfcGs4xR0REnhjCREQaMYSJiDRiCBMRacQQJiLSiCFMRKQRh6gREWnEECYi0oghTESkEUOYiEgjhjARkUYMYSIijRjCREQaMYSJiDRiCBMRacQQ9jE5+RCZTAYNx467Pu81LYsk57f6+9//R3t7EFF5Ygj7YAgTUakxhImINGIIExFpxBAmItKI/8qSiEgjhjARkUYMYSIijRjCREQaMYSJiDRiCBMRacQQJiLSiCFMRKQRQ5iISCOGMBGRRgxhIiKN/g/Slb+FIicp1wAAAABJRU5ErkJggg=="}}},{"cell_type":"code","source":"for patient in images_path:\n    if patient[2][0] == '1':\n        os.chdir(WITH_MGMT_DIR)\n        for image in patient[1]:\n            png_save(image)\n        os.chdir(INPUTDIR_PATH)\n    else:\n        os.chdir(WITHOUT_MGMT_DIR)\n        for image in patient[1]:\n            png_save(image)\n        os.chdir(INPUTDIR_PATH)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-10-17T10:34:45.942742Z","iopub.execute_input":"2021-10-17T10:34:45.943089Z","iopub.status.idle":"2021-10-17T10:43:55.537984Z","shell.execute_reply.started":"2021-10-17T10:34:45.94306Z","shell.execute_reply":"2021-10-17T10:43:55.537144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Aqui simplesmente buscamos as imagens de cada paciente no diretório TEST, convertemos em PNG e salvamos na pasta PNG_TEST_DIR.","metadata":{}},{"cell_type":"code","source":"os.chdir(PNG_TEST_DIR)\nfor patient in test_images_path:\n    for image in patient[1]:\n        png_save(image)\nos.chdir(INPUTDIR_PATH)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-10-17T10:46:30.43623Z","iopub.execute_input":"2021-10-17T10:46:30.436555Z","iopub.status.idle":"2021-10-17T10:50:17.32734Z","shell.execute_reply.started":"2021-10-17T10:46:30.436527Z","shell.execute_reply":"2021-10-17T10:50:17.326483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Keras e Tensorflow","metadata":{}},{"cell_type":"markdown","source":"Após especificar o tamanho das nossas imagens e do batch, utilizamos a API do Keras para automaticamente criarmos os datasets de treinamento e validação que iremos utilizar no modelo CNN. Configuramos através dos parâmetros \"validation_split\" e \"subset\" a porcentagem de 20% para validação e a divisão do diretório de imagens em dois para treino e validação.","metadata":{}},{"cell_type":"code","source":"image_size = (512, 512)\nbatch_size = 32\n\ntrain_ds = tf.keras.preprocessing.image_dataset_from_directory(\n    PNG_DATASET_DIR,\n    validation_split=0.2,\n    subset=\"training\",\n    seed=1337,\n    image_size=image_size,\n    batch_size=batch_size,\n)\nval_ds = tf.keras.preprocessing.image_dataset_from_directory(\n    PNG_DATASET_DIR,\n    validation_split=0.2,\n    subset=\"validation\",\n    seed=1337,\n    image_size=image_size,\n    batch_size=batch_size,\n)\n","metadata":{"execution":{"iopub.status.busy":"2021-10-17T10:51:36.833173Z","iopub.execute_input":"2021-10-17T10:51:36.833505Z","iopub.status.idle":"2021-10-17T10:51:43.667544Z","shell.execute_reply.started":"2021-10-17T10:51:36.833475Z","shell.execute_reply":"2021-10-17T10:51:43.666779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Apenas para exemplificação, pegamos uma amostra do train_ds e plotamos as imagens junto com as respectivas classificações.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nfor images, labels in train_ds.take(1):\n    for i in range(9):\n        ax = plt.subplot(3, 3, i + 1)\n        plt.imshow(images[i].numpy().astype(\"uint8\"))\n        plt.title(int(labels[i]))\n        plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2021-10-17T11:13:43.423857Z","iopub.execute_input":"2021-10-17T11:13:43.424189Z","iopub.status.idle":"2021-10-17T11:13:45.659894Z","shell.execute_reply.started":"2021-10-17T11:13:43.424161Z","shell.execute_reply":"2021-10-17T11:13:45.65902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Utilizado o método \"prefetch\" para agilizar o carregamento dos dados durante o treinamento da rede neural.","metadata":{}},{"cell_type":"code","source":"train_ds = train_ds.prefetch(buffer_size=32)\nval_ds = val_ds.prefetch(buffer_size=32)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T11:15:52.87261Z","iopub.execute_input":"2021-10-17T11:15:52.87298Z","iopub.status.idle":"2021-10-17T11:15:52.879082Z","shell.execute_reply.started":"2021-10-17T11:15:52.872949Z","shell.execute_reply":"2021-10-17T11:15:52.878062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CNN\nDe forma experimental, criamos uma rede neural convolucional de apenas oito camadas.","metadata":{}},{"cell_type":"code","source":"\ndef make_model(input_shape):\n    inputs = keras.Input(shape=input_shape)\n\n    x = keras.Sequential()\n    x = layers.experimental.preprocessing.Rescaling(1.0 / 255)(inputs)\n   \n    x = layers.Conv2D(32, 3, strides=2, padding=\"same\")(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation(\"relu\")(x)\n\n    x = layers.Conv2D(64, 3, padding=\"same\")(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation(\"relu\")(x)  \n     \n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.Dropout(0.5)(x)\n\n    outputs = layers.Dense(units=1, activation=\"sigmoid\")(x)\n    return keras.Model(inputs, outputs)\n\n\nmodel = make_model(input_shape=image_size + (3,))\n\nmodel.summary()\n","metadata":{"execution":{"iopub.status.busy":"2021-10-16T23:07:21.065955Z","iopub.execute_input":"2021-10-16T23:07:21.066289Z","iopub.status.idle":"2021-10-16T23:07:21.147481Z","shell.execute_reply.started":"2021-10-16T23:07:21.066257Z","shell.execute_reply":"2021-10-16T23:07:21.146684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"Definimos então o número de épocas e uma função para salvar o melhor modelo encontrado durante treinamento.\\\nAo compilar o modelo, escolhemos como otimizador o Adam, função de perda do tipo \"binary_crossentropy\", uma vez que nosso problema é do tipo tem ou não MGMT, e como métrica a acurácia.\\\nObservando a saída do método \"fit\", podemos observar que a nossa rede não performa de forma adequada, inclusive aparentemente ela não faz nenhum tipo de aprendizado significativo.","metadata":{}},{"cell_type":"code","source":"epochs = 20\n\ncallbacks = [\n    keras.callbacks.ModelCheckpoint(filepath=\"/kaggle/working/save_at_{epoch}.h5\",\n                                    save_best_only=True)\n]\n\nmodel.compile(\n    optimizer=keras.optimizers.Adam(1e-3),\n    loss=\"binary_crossentropy\",\n    metrics=[\"accuracy\"],\n)\nmodel.fit(\n    train_ds, epochs=epochs, callbacks=callbacks, validation_data=val_ds,\n)\n","metadata":{"execution":{"iopub.status.busy":"2021-10-16T23:07:24.117359Z","iopub.execute_input":"2021-10-16T23:07:24.117737Z","iopub.status.idle":"2021-10-16T23:09:59.145177Z","shell.execute_reply.started":"2021-10-16T23:07:24.117704Z","shell.execute_reply":"2021-10-16T23:09:59.14434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transferência de aprendizado\nPara melhorarmos nossa classificação, optamos por utilizar a transferência de aprendizado de um modelo já pré-treinado. No nosso caso, escolhemos o Xception.\\\nO próprio Keras fornece um método para obtermos esse modelo com os pesos pré-treinados no dataset do Imagenet.\\\nPara permitir que o modelo se adeque ao nosso problema, não incluímos a última camada de classificação do Xception ao criarmos nosso modelo base de rede neural.","metadata":{}},{"cell_type":"code","source":"base_model = keras.applications.Xception(\n    weights='imagenet',  # Load weights pre-trained on ImageNet.\n    input_shape=(512, 512, 3),\n    include_top=False)  # Do not include the ImageNet classifier at the top.\n","metadata":{"execution":{"iopub.status.busy":"2021-10-17T11:37:12.736119Z","iopub.execute_input":"2021-10-17T11:37:12.73644Z","iopub.status.idle":"2021-10-17T11:37:14.370833Z","shell.execute_reply.started":"2021-10-17T11:37:12.736411Z","shell.execute_reply":"2021-10-17T11:37:14.369889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Congelamos então os parâmetros do modelo base para não alterarmos os pesos já pré-treinados e criamos a partir dele um novo modelo, adicionando mais duas camdas, sendo a última a nossa camada de classificação.","metadata":{}},{"cell_type":"code","source":"base_model.trainable = False\n\ninputs_2 = keras.Input(shape=(512, 512, 3))\nx_2 = base_model(inputs_2, training=False)\n# Convert features of shape `base_model.output_shape[1:]` to vectors\nx_2 = keras.layers.GlobalAveragePooling2D()(x_2)\n# A Dense classifier with a single unit (binary classification)\noutputs_2 = keras.layers.Dense(1, activation=\"sigmoid\")(x_2)\nmodel_2 = keras.Model(inputs_2, outputs_2)\nmodel_2.summary()\n","metadata":{"execution":{"iopub.status.busy":"2021-10-17T11:37:17.66799Z","iopub.execute_input":"2021-10-17T11:37:17.668537Z","iopub.status.idle":"2021-10-17T11:37:18.022352Z","shell.execute_reply.started":"2021-10-17T11:37:17.668479Z","shell.execute_reply":"2021-10-17T11:37:18.020494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Com essa nova rede neural, obtemos uma acurácia próxima de 80%.","metadata":{}},{"cell_type":"code","source":"epochs = 20\ncallbacks = [\n    keras.callbacks.ModelCheckpoint(filepath=\"/kaggle/working/transfer_save_at_{epoch}.h5\",\n                                    save_best_only=True)\n]\nmodel_2.compile(optimizer=keras.optimizers.Adam(),\n              loss=keras.losses.BinaryCrossentropy(from_logits=True),\n              metrics=[keras.metrics.BinaryAccuracy()])\nmodel_2.fit(train_ds, epochs=epochs, callbacks=callbacks, validation_data=val_ds)\n","metadata":{"execution":{"iopub.status.busy":"2021-10-17T11:37:26.111065Z","iopub.execute_input":"2021-10-17T11:37:26.111376Z","iopub.status.idle":"2021-10-17T12:26:04.824075Z","shell.execute_reply.started":"2021-10-17T11:37:26.111348Z","shell.execute_reply":"2021-10-17T12:26:04.823299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utilizando modelo salvo\nDepois de termos salvo o melhor modelo durante trainamento, podemos restaurar o mesmo e fazer as nossas classificações.","metadata":{}},{"cell_type":"code","source":"#restored_model = keras.models.load_model(\"/kaggle/input/saved-models/transfer_save_at_17.h5\")\n#restored_model.summary()\npredictions = []\ni = 0\nfor file in os.listdir(PNG_TEST_DIR)[:100]:\n    image = tf.keras.preprocessing.image.load_img(os.path.join(PNG_TEST_DIR, file),\n                                                  target_size=image_size)\n    input_arr = keras.preprocessing.image.img_to_array(image)\n    input_arr = np.array([input_arr])  # Convert single image to a batch.\n    predictions.append([file, model_2.predict(input_arr)])\npredictions","metadata":{"execution":{"iopub.status.busy":"2021-10-17T11:16:18.619315Z","iopub.execute_input":"2021-10-17T11:16:18.619642Z","iopub.status.idle":"2021-10-17T11:16:21.988047Z","shell.execute_reply.started":"2021-10-17T11:16:18.619596Z","shell.execute_reply":"2021-10-17T11:16:21.987189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ajuste fino\nAtravés do código abaixo, poderíamos fazer um ajuste final do modelo ao descongelar os parâmetros do modelo base, porém devido as limitações de recursos do ambiente de desenvolvimento, o código abaixo gera erro de Out Of Memory.","metadata":{}},{"cell_type":"code","source":"'''\nbase_model.trainable = True\n\nmodel_2.compile(optimizer=keras.optimizers.Adam(1e-5),\n              loss=keras.losses.BinaryCrossentropy(from_logits=True),\n              metrics=[keras.metrics.BinaryAccuracy()])\n\n\nmodel_2.fit(train_ds, epochs=10, callbacks=callbacks, validation_data=val_ds)\n'''","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-10-17T12:38:23.197508Z","iopub.execute_input":"2021-10-17T12:38:23.197848Z","iopub.status.idle":"2021-10-17T12:38:29.438123Z","shell.execute_reply.started":"2021-10-17T12:38:23.197816Z","shell.execute_reply":"2021-10-17T12:38:29.437279Z"},"trusted":true},"execution_count":null,"outputs":[]}]}