{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":11848,"databundleVersionId":862157,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"colab":{"provenance":[],"gpuType":"T4"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"9668eed06863473daf78fbb757f7bf12":{"model_module":"@jupyter-widgets/controls","model_name":"VBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"VBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"VBoxView","box_style":"","children":["IPY_MODEL_ff743128abed491b9bb7a7304b612b2c"],"layout":"IPY_MODEL_a8ae7a0d8cc249fd9249dc56a500fdb3"}},"67fc2f9754a948db8ff8a31e48fd1389":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_f6cc2f2b9fd44dfb94d733d1271246ed","placeholder":"​","style":"IPY_MODEL_d05bd31f7d7047469bc254583480e695","value":"<center> <img\nsrc=https://www.kaggle.com/static/images/site-logo.png\nalt='Kaggle'> <br> Create an API token from <a\nhref=\"https://www.kaggle.com/settings/account\" target=\"_blank\">your Kaggle\nsettings page</a> and paste it below along with your Kaggle username. <br> </center>"}},"ca6f1744bc324e7c8781484af78c2ee7":{"model_module":"@jupyter-widgets/controls","model_name":"TextModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"TextModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"TextView","continuous_update":true,"description":"Username:","description_tooltip":null,"disabled":false,"layout":"IPY_MODEL_3d933931001e4a0e9627c3703558a2b0","placeholder":"​","style":"IPY_MODEL_e3475361ecb4481aab0d27d1519abead","value":"johnogilvie"}},"799e35ee675846fd9c5ef6c0c89e0568":{"model_module":"@jupyter-widgets/controls","model_name":"PasswordModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"PasswordModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"PasswordView","continuous_update":true,"description":"Token:","description_tooltip":null,"disabled":false,"layout":"IPY_MODEL_16e4d2a3b526476aa9d69913c08832ef","placeholder":"​","style":"IPY_MODEL_7d933373fc3f4423b23518b3bd137b37","value":""}},"4f1a1d499ee449f0872427a9bc695322":{"model_module":"@jupyter-widgets/controls","model_name":"ButtonModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ButtonModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ButtonView","button_style":"","description":"Login","disabled":false,"icon":"","layout":"IPY_MODEL_c2d88db2adbf43e1aab3c00394ae979d","style":"IPY_MODEL_1edfa78af6df444c8ca1c84e8382022f","tooltip":""}},"f9df7c904da844baba549d1533f990fd":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_61479902f30347f384781a5d59438733","placeholder":"​","style":"IPY_MODEL_0520c577e782446fa27207dc83f1a841","value":"\n<b>Thank You</b></center>"}},"a8ae7a0d8cc249fd9249dc56a500fdb3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":"center","align_self":null,"border":null,"bottom":null,"display":"flex","flex":null,"flex_flow":"column","grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"50%"}},"f6cc2f2b9fd44dfb94d733d1271246ed":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d05bd31f7d7047469bc254583480e695":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"3d933931001e4a0e9627c3703558a2b0":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e3475361ecb4481aab0d27d1519abead":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"16e4d2a3b526476aa9d69913c08832ef":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7d933373fc3f4423b23518b3bd137b37":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"c2d88db2adbf43e1aab3c00394ae979d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1edfa78af6df444c8ca1c84e8382022f":{"model_module":"@jupyter-widgets/controls","model_name":"ButtonStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ButtonStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","button_color":null,"font_weight":""}},"61479902f30347f384781a5d59438733":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0520c577e782446fa27207dc83f1a841":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"81dfbe56898144a0b0dea40757cd1fcf":{"model_module":"@jupyter-widgets/controls","model_name":"LabelModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"LabelModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"LabelView","description":"","description_tooltip":null,"layout":"IPY_MODEL_363a52e0fa1e4095a8578ced13da5b08","placeholder":"​","style":"IPY_MODEL_6bf3aea7f52240d2ad08c91c877b5797","value":"Connecting..."}},"363a52e0fa1e4095a8578ced13da5b08":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6bf3aea7f52240d2ad08c91c877b5797":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ff743128abed491b9bb7a7304b612b2c":{"model_module":"@jupyter-widgets/controls","model_name":"LabelModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"LabelModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"LabelView","description":"","description_tooltip":null,"layout":"IPY_MODEL_336f13b990ac46169576bbdef74afe5a","placeholder":"​","style":"IPY_MODEL_f131efeffd284d77ae9fac5f85d60126","value":"Kaggle credentials successfully validated."}},"336f13b990ac46169576bbdef74afe5a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f131efeffd284d77ae9fac5f85d60126":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## This notebook brings together in one place the best-performing model aspects from prior weeks of DSCI 598 Capstone\n* exploratory data analysis and preparation from Week3\n* https://www.kaggle.com/code/johnogilvie/jwo-histocancerdetect-eda-select-v1\n* 64x64 image size, Colab development from Week4\n* https://www.kaggle.com/code/johnogilvie/jwo-histocancerdetect-img-sizes-select-v1  \n* image augmentation, from Week5\n* https://www.kaggle.com/code/johnogilvie/jwo-histocancerdetect-data-augs-select-v1  \n* transfer learning with DenseNet121, from Week6\n* https://www.kaggle.com/code/johnogilvie/jwo-histocancerdetect-transfer-select-v1\n* use of multiple convolutional layers, from Week7 and Week6\n* https://www.kaggle.com/code/johnogilvie/jwo-histocancerdetect-layers-select-v1\n* https://www.kaggle.com/code/johnogilvie/jwo-histocancerdetect-transfer-select-v1","metadata":{"id":"toYTJKVtZkNu"}},{"cell_type":"code","source":"# IMPORTANT: SOME KAGGLE DATA SOURCES ARE PRIVATE\n# RUN THIS CELL IN ORDER TO IMPORT YOUR KAGGLE DATA SOURCES.\nimport kagglehub\nkagglehub.login()\n","metadata":{"id":"DzYaHq3nZkNt","outputId":"5e19779f-c44f-468f-d96e-8898a7c40c12"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# IMPORTANT: RUN THIS CELL IN ORDER TO IMPORT YOUR KAGGLE DATA SOURCES,\n# THEN FEEL FREE TO DELETE THIS CELL.\n# NOTE: THIS NOTEBOOK ENVIRONMENT DIFFERS FROM KAGGLE'S PYTHON\n# ENVIRONMENT SO THERE MAY BE MISSING LIBRARIES USED BY YOUR\n# NOTEBOOK.\n\nhistopathologic_cancer_detection_path = kagglehub.competition_download('histopathologic-cancer-detection')\n\nprint('Data source import complete.')\n","metadata":{"id":"xT-ENYMFZkNt","outputId":"58a86c7e-2a98-457e-8ecf-7862b9ec5d90"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Import packages","metadata":{"id":"AiPFxnThZkNv"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import shuffle\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras import models, layers, datasets\nfrom tensorflow.keras.activations import swish\nfrom tensorflow import keras","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:17:19.578822Z","iopub.execute_input":"2025-10-30T19:17:19.579496Z","iopub.status.idle":"2025-10-30T19:17:32.826462Z","shell.execute_reply.started":"2025-10-30T19:17:19.579461Z","shell.execute_reply":"2025-10-30T19:17:32.825931Z"},"id":"gARp1rP2ZkNv"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## View and prepare Data (Nulls check, Distributions, Image labels)","metadata":{"id":"ATTsiJutZkNw"}},{"cell_type":"code","source":"train = pd.read_csv(f'{histopathologic_cancer_detection_path}/train_labels.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:17:32.82716Z","iopub.execute_input":"2025-10-30T19:17:32.827616Z","iopub.status.idle":"2025-10-30T19:17:33.166373Z","shell.execute_reply.started":"2025-10-30T19:17:32.827597Z","shell.execute_reply":"2025-10-30T19:17:33.165816Z"},"id":"Y_EoOp6_ZkNw"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.isnull().sum().T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:17:33.187585Z","iopub.execute_input":"2025-10-30T19:17:33.187756Z","iopub.status.idle":"2025-10-30T19:17:33.203536Z","shell.execute_reply.started":"2025-10-30T19:17:33.187742Z","shell.execute_reply":"2025-10-30T19:17:33.202912Z"},"id":"cVefefM5ZkNw","outputId":"07c9b0a9-db29-4a27-cdbf-15619bd39d31"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(train.label.value_counts()/len(train.label)).to_frame().T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:17:33.204184Z","iopub.execute_input":"2025-10-30T19:17:33.204504Z","iopub.status.idle":"2025-10-30T19:17:33.221834Z","shell.execute_reply.started":"2025-10-30T19:17:33.204486Z","shell.execute_reply":"2025-10-30T19:17:33.221217Z"},"id":"cak2XOpyZkNw","outputId":"5571b18a-9d6b-49c7-8ab2-34c5ed80bdf0"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Challenge: The image ids are used as filenames for the images, but the ids are missing the \".tif\" extension.\n#You will need to add a copy to the DataFrame to store the complete filename rather than just the id\ntrain['filenames'] = train['id']+'.tif'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:17:33.222557Z","iopub.execute_input":"2025-10-30T19:17:33.222811Z","iopub.status.idle":"2025-10-30T19:17:33.260522Z","shell.execute_reply.started":"2025-10-30T19:17:33.222788Z","shell.execute_reply":"2025-10-30T19:17:33.259871Z"},"id":"itYi69MfZkNx"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:17:33.261192Z","iopub.execute_input":"2025-10-30T19:17:33.261422Z","iopub.status.idle":"2025-10-30T19:17:33.269709Z","shell.execute_reply.started":"2025-10-30T19:17:33.261405Z","shell.execute_reply":"2025-10-30T19:17:33.268942Z"},"id":"rW7i2FhtZkNx","outputId":"e32e13b7-dd6f-4fca-f34f-468feb3b7ab4"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Sample the Data to make training more efficient","metadata":{"id":"tdiEX3fJZkNx"}},{"cell_type":"code","source":"SS = 50000\nRS = 10\n\npositives = train[train['label']==1].sample(SS, random_state = RS)\nnegatives = train[train['label']==0].sample(SS, random_state = SS)\n\nnew_train = pd.concat([positives,negatives], axis = 0).reset_index(drop = True)\nnew_train = shuffle(new_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:17:34.277308Z","iopub.execute_input":"2025-10-30T19:17:34.277497Z","iopub.status.idle":"2025-10-30T19:17:34.351229Z","shell.execute_reply.started":"2025-10-30T19:17:34.277483Z","shell.execute_reply":"2025-10-30T19:17:34.350431Z"},"id":"SEboBRnOZkNx"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(new_train.label.value_counts()/len(new_train)).to_frame().T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:17:34.360326Z","iopub.execute_input":"2025-10-30T19:17:34.360807Z","iopub.status.idle":"2025-10-30T19:17:34.375092Z","shell.execute_reply.started":"2025-10-30T19:17:34.36079Z","shell.execute_reply":"2025-10-30T19:17:34.37441Z"},"id":"0rjES3gdZkNx","outputId":"10c31f54-6e01-4146-9a94-23ef90402626"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# new_train = new_train.sample(frac=0.05)  # use only a fraction of the dataset","metadata":{"id":"vmhZY99GasNm"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train_Test_Split","metadata":{"id":"mZqkDl17ZkNx"}},{"cell_type":"code","source":"train_df, val_df = train_test_split(new_train, test_size = .2, random_state = 10, stratify = new_train.label)\n\nprint(train_df.shape)\nprint(val_df.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:17:34.375866Z","iopub.execute_input":"2025-10-30T19:17:34.376167Z","iopub.status.idle":"2025-10-30T19:17:34.425624Z","shell.execute_reply.started":"2025-10-30T19:17:34.376149Z","shell.execute_reply":"2025-10-30T19:17:34.424839Z"},"id":"FWEDr0vWZkNx","outputId":"6f5a9ea6-41ad-45b9-bcb5-5378aac286fd"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Rescale Images with ImageDataGenerator","metadata":{"id":"XsDvp3v2ZkNx"}},{"cell_type":"code","source":"train_images_path = f'{histopathologic_cancer_detection_path}/train'","metadata":{"id":"xtmvce1WbCLz"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Challenge: You will need to use an image data generator to load the files from disk.\ntrain_datagen = ImageDataGenerator(rescale = 1/255)\nval_datagen = ImageDataGenerator(rescale = 1/255)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:17:34.426412Z","iopub.execute_input":"2025-10-30T19:17:34.426749Z","iopub.status.idle":"2025-10-30T19:17:34.430143Z","shell.execute_reply.started":"2025-10-30T19:17:34.426726Z","shell.execute_reply":"2025-10-30T19:17:34.42949Z"},"id":"FpL3E74HZkNy"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Make labels into strings","metadata":{"id":"mLWr4HkfZkNy"}},{"cell_type":"code","source":"train_df['label'] = train_df['label'].astype(str)\nval_df['label'] = val_df['label'].astype(str)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:17:34.430772Z","iopub.execute_input":"2025-10-30T19:17:34.431034Z","iopub.status.idle":"2025-10-30T19:17:34.463712Z","shell.execute_reply.started":"2025-10-30T19:17:34.431009Z","shell.execute_reply":"2025-10-30T19:17:34.463051Z"},"id":"XQFdW27rZkNy"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Create loaders using data augmentation\n* Images sized 64x64","metadata":{"id":"2WhFNRUTZkNy"}},{"cell_type":"code","source":"%%time\nbatch_size = 32\n\ntrain_loader = train_datagen.flow_from_dataframe(\n    dataframe = train_df,\n    directory = train_images_path,\n    x_col = 'filenames',\n    y_col = 'label',\n    batch_size = batch_size,\n    seed = 10,\n    shuffle = True,\n    class_mode = 'binary',\n    horizontal_flip = True,\n    vertical_flip = True,\n    height_shift_range = 0.15,\n    width_shift_range = 0.15,\n    rotation_range = 20,\n    target_size = (64,64)\n)\n\nval_loader = val_datagen.flow_from_dataframe(\n    dataframe = val_df,\n    directory = train_images_path,\n    x_col = 'filenames',\n    y_col = 'label',\n    batch_size = batch_size,\n    seed = 10,\n    shuffle = True,\n    class_mode = 'binary',\n    target_size = (64,64)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:17:34.464339Z","iopub.execute_input":"2025-10-30T19:17:34.464552Z","iopub.status.idle":"2025-10-30T19:20:25.492246Z","shell.execute_reply.started":"2025-10-30T19:17:34.464529Z","shell.execute_reply":"2025-10-30T19:20:25.491663Z"},"id":"J2k5r4xfZkNy","outputId":"d35b65ef-9555-4c6b-e1af-c0cfbd5bffd9"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TR_STEPS = len(train_loader)\nVAL_STEPS = len(val_loader)\n\nprint(TR_STEPS)\nprint(VAL_STEPS)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:20:25.493001Z","iopub.execute_input":"2025-10-30T19:20:25.493248Z","iopub.status.idle":"2025-10-30T19:20:25.497344Z","shell.execute_reply.started":"2025-10-30T19:20:25.493222Z","shell.execute_reply":"2025-10-30T19:20:25.496661Z"},"id":"xZm_zI0MZkNy","outputId":"ac05a369-2f4b-4b60-90be-863a8b2b9d82"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create model using best (of four tested) pretrained model as foundation","metadata":{"id":"OL2j-qRVfd24"}},{"cell_type":"markdown","source":"## Use callbacks for early stopping if the model doesn't improve\n","metadata":{"id":"MPYZO5yUZkNy"}},{"cell_type":"code","source":"early_stopping_callback = keras.callbacks.EarlyStopping(\n    monitor='val_AUC',\n    patience=10,\n    restore_best_weights=True,\n    mode='max',\n    verbose=1\n)\n\nlr_scheduler_callback = keras.callbacks.ReduceLROnPlateau(\n    monitor='val_AUC',\n    factor=0.5,\n    patience=5,\n    min_lr=1e-8,\n    mode='max',\n    verbose=1\n)\n\ncallbacks_list = [early_stopping_callback, lr_scheduler_callback]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:20:31.92824Z","iopub.execute_input":"2025-10-30T19:20:31.928465Z","iopub.status.idle":"2025-10-30T19:20:31.967154Z","shell.execute_reply.started":"2025-10-30T19:20:31.928441Z","shell.execute_reply":"2025-10-30T19:20:31.966468Z"},"id":"PKa1W_lXZkNy"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_epochs = 20","metadata":{"id":"2NUPXXIgf80u"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Function to display a model's performance\n","metadata":{"id":"mltWYdRhZkNy"}},{"cell_type":"code","source":"def display_perf(history, loss_title, auc_title):\n  epoch_range = range(1, len(history['loss'])+1)\n  plt.figure(figsize = [12,5])\n  plt.subplot(1,2,1)\n  plt.plot(epoch_range, history['loss'], label = 'Training')\n  plt.plot(epoch_range, history['val_loss'], label = 'Validation')\n  plt.xlabel('Epoch');plt.ylabel('Loss');plt.title(loss_title)\n\n  plt.subplot(1,2,2)\n  plt.plot(epoch_range, history['AUC'], label = 'Training')\n  plt.plot(epoch_range, history['val_AUC'], label = 'Validation')\n  plt.xlabel(\"Epoch\");plt.ylabel(\"AUC\");plt.title(auc_title)\n  plt.legend()\n  plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T20:37:56.041209Z","iopub.execute_input":"2025-10-30T20:37:56.041474Z","iopub.status.idle":"2025-10-30T20:37:56.343301Z","shell.execute_reply.started":"2025-10-30T20:37:56.041451Z","shell.execute_reply":"2025-10-30T20:37:56.342759Z"},"id":"OetSr53GZkNy"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get a sample batch from the validation loader\nsample_batch = next(iter(val_loader))\nsample_images, sample_labels = sample_batch\nsample = sample_images[0:1] # Take the first image in the batch","metadata":{"id":"c15bbzeDnyRR"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Create a model using the DenseNet121 model from keras as the foundation\n* Print the Architecture of the model\n* Input shape will be 64x64\n* Training only the last 30 layers of the model\n* Swish activation instead of the relu for the dense layers","metadata":{"id":"t1xVVrk_rchZ"}},{"cell_type":"code","source":"base_model_den=keras.applications.DenseNet121(\n    include_top=False,\n    weights=\"imagenet\",\n    input_tensor=None,\n    input_shape=(64,64,3),\n    pooling=None,\n    classes=2,\n    classifier_activation=\"Swish\",\n)\nbase_model_den.trainable = True\n\nfor layer in base_model_den.layers[:-30]:\n    layer.trainable = True\n\ncnn_model_den2 = models.Sequential([\n    base_model_den,\n    layers.GlobalAveragePooling2D(),\n    Dense(512, activation = 'swish'),\n    Dropout(.4),\n    Dense(256, activation = 'swish'),\n    Dropout(.3),\n    Dense(128, activation = 'swish'),\n    Dropout(.2),\n    Dense(1, activation = 'sigmoid')\n])\ncnn_model_den2.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:20:25.498055Z","iopub.execute_input":"2025-10-30T19:20:25.498382Z","iopub.status.idle":"2025-10-30T19:20:31.912624Z","shell.execute_reply.started":"2025-10-30T19:20:25.498364Z","shell.execute_reply":"2025-10-30T19:20:31.911873Z"},"outputId":"afb34dd1-d101-46e2-d88e-14c0ed96c216","id":"za0tg8MTrchZ"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"opt = tf.keras.optimizers.Adam(learning_rate = 1e-5)\ncnn_model_den2.compile(loss = 'binary_crossentropy', optimizer = opt, metrics = ['AUC'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:20:31.91346Z","iopub.execute_input":"2025-10-30T19:20:31.913818Z","iopub.status.idle":"2025-10-30T19:20:31.927595Z","shell.execute_reply.started":"2025-10-30T19:20:31.913793Z","shell.execute_reply":"2025-10-30T19:20:31.927084Z"},"id":"8qqaAiJ2rcha"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Run training iteration","metadata":{"id":"H_aohjjNrcha"}},{"cell_type":"code","source":"%%time\nhistory_den2 = cnn_model_den2.fit(\n    x = train_loader,\n    steps_per_epoch = TR_STEPS,\n    epochs = num_epochs,\n    validation_data = val_loader,\n    validation_steps = VAL_STEPS,\n    verbose = 1,\n    callbacks = callbacks_list\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T19:20:31.967843Z","iopub.execute_input":"2025-10-30T19:20:31.968073Z","iopub.status.idle":"2025-10-30T20:37:56.030652Z","shell.execute_reply.started":"2025-10-30T19:20:31.968058Z","shell.execute_reply":"2025-10-30T20:37:56.029849Z"},"outputId":"d2ad1715-9103-4baa-ce9e-4fee0f72636d","id":"11uya351rchb"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lossden2, aucden2 = cnn_model_den2.evaluate(val_loader, verbose=0)\nprint(f\"DenseNet121 with aug Test AUC: {aucden2:.4f}\")","metadata":{"outputId":"70061d91-6793-4901-babe-440ba0fec14f","id":"Q5tZinkFrchb"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Predict on new data\n\npredictionden2 = cnn_model_den2.predict(sample)\nprint(f\"DenseNet121 with aug predicted probability of class 1: {predictionden2[0][0]:.4f}\")","metadata":{"outputId":"b8870660-efd1-4209-fc93-f92cec589b10","id":"_hspe6YBrchb"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display_perf(history_den2.history, 'DenseNet121 with aug Training Loss', 'DenseNet121 with aug Training AUC')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T20:37:56.036734Z","iopub.execute_input":"2025-10-30T20:37:56.036951Z","iopub.status.idle":"2025-10-30T20:37:56.040312Z","shell.execute_reply.started":"2025-10-30T20:37:56.036936Z","shell.execute_reply":"2025-10-30T20:37:56.039602Z"},"id":"mK8n8YMjZkNy","outputId":"ba0a6ed7-414c-4de5-c746-066de48e09fa"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Save the model","metadata":{"id":"2bwZQls1ZkNz"}},{"cell_type":"code","source":"# save models in Google Drive\nfrom google.colab import drive\ndrive.mount('/content/drive')\nmodel_save_path = '/content/drive/MyDrive/Cancer_Detection_Models/'","metadata":{"id":"7f07Zs3z-A3B","outputId":"c8722fd0-470e-4802-b6b2-0176caa0878d"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pickle\nimport os\n\n# Create the directory if it doesn't exist\nos.makedirs(model_save_path, exist_ok=True)\n\ncnn_model_den2.save(f'{model_save_path}/Cancer_Detection_model_denbest2.keras')\npickle.dump(history_den2.history, open(f'{model_save_path}/Cancer_Detection_model_denbest2.pk1', 'wb'))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T20:37:56.345881Z","iopub.execute_input":"2025-10-30T20:37:56.346326Z","iopub.status.idle":"2025-10-30T20:37:57.325314Z","shell.execute_reply.started":"2025-10-30T20:37:56.346308Z","shell.execute_reply":"2025-10-30T20:37:57.324618Z"},"id":"C4MF9eVrL3va"},"outputs":[],"execution_count":null}]}