{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":11848,"databundleVersionId":862157},{"sourceType":"datasetVersion","sourceId":286125,"datasetId":118729,"databundleVersionId":298605}],"dockerImageVersionId":20477,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"[Reference]\n* https://www.kaggle.com/guntherthepenguin/fastai-v1-densenet169","metadata":{"_uuid":"2cc32ef9bba8300433885bf63496e728e48aabd1"}},{"cell_type":"markdown","source":" **Brief description of the problem and data**\n \n In this dataset, you are provided with a large number of small pathology images to classify. Files are named with an image id. The train_labels.csv file provides the ground truth for the images in the train folder. You are predicting the labels for the images in the test folder. A positive label indicates that the center 32x32px region of a patch contains at least one pixel of tumor tissue. Tumor tissue in the outer region of the patch does not influence the label. This outer region is provided to enable fully-convolutional models that do not use zero-padding, to ensure consistent behavior when applied to a whole-slide image.\n\nThe original PCam dataset contains duplicate images due to its probabilistic sampling, however, the version presented on Kaggle does not contain duplicates. We have otherwise maintained the same data and splits as the PCam benchmark.","metadata":{}},{"cell_type":"markdown","source":"**Exploratory Data Analysis (EDA) — Inspect, Visualize and Clean the Data**","metadata":{}},{"cell_type":"code","source":"# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n# import matplotlib.pyplot as plt\n# from PIL import Image\n# import tensorflow as tf\n# import os\n# import multiprocessing\n# from tqdm import tqdm\n# from PIL import ImageDraw\n# train_on_gpu = True\n\n# Path = '/kaggle/input/histopathologic-cancer-detection/'","metadata":{"execution":{"iopub.status.busy":"2024-08-19T03:15:26.704898Z","iopub.execute_input":"2024-08-19T03:15:26.705295Z","iopub.status.idle":"2024-08-19T03:15:29.492471Z","shell.execute_reply.started":"2024-08-19T03:15:26.705231Z","shell.execute_reply":"2024-08-19T03:15:29.491499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df = pd.read_csv(Path + 'train_labels.csv')\n# train_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-19T03:15:36.408018Z","iopub.execute_input":"2024-08-19T03:15:36.408343Z","iopub.status.idle":"2024-08-19T03:15:36.850126Z","shell.execute_reply.started":"2024-08-19T03:15:36.408294Z","shell.execute_reply":"2024-08-19T03:15:36.849303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_samples = pd.read_csv(Path + 'sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-08-19T03:15:46.567336Z","iopub.execute_input":"2024-08-19T03:15:46.567675Z","iopub.status.idle":"2024-08-19T03:15:46.671676Z","shell.execute_reply.started":"2024-08-19T03:15:46.567626Z","shell.execute_reply":"2024-08-19T03:15:46.670627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = \"/kaggle/input/histopathologic-cancer-detection/train/\"\n# test = \"/kaggle/input/histopathologic-cancer-detection/test\"\n\n# print(\"Number of training images: {}\".format(len(os.listdir(train))))\n# print(\"Number of test images: {}\".format(len(os.listdir(test))))","metadata":{"execution":{"iopub.status.busy":"2024-08-19T03:15:50.94373Z","iopub.execute_input":"2024-08-19T03:15:50.944064Z","iopub.status.idle":"2024-08-19T03:16:13.970532Z","shell.execute_reply.started":"2024-08-19T03:15:50.944017Z","shell.execute_reply":"2024-08-19T03:16:13.969671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig = plt.figure(figsize=(30, 6))\n# train_imgs = os.listdir(Path+\"train\")\n# for idx, img in enumerate(np.random.choice(train_imgs, 20)):\n#     ax = fig.add_subplot(2, 20//2, idx+1, xticks=[], yticks=[])\n#     im = Image.open(Path+\"train/\" + img)\n#     plt.imshow(im)\n#     label = train_df.loc[train_df['id'] == img.split('.')[0], 'label'].values[0]\n#     ax.set_title(f'Label: {label}')","metadata":{"execution":{"iopub.status.busy":"2024-08-19T03:16:24.366092Z","iopub.execute_input":"2024-08-19T03:16:24.366467Z","iopub.status.idle":"2024-08-19T03:16:27.370395Z","shell.execute_reply.started":"2024-08-19T03:16:24.366402Z","shell.execute_reply":"2024-08-19T03:16:27.369274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# missing_values = train_df.isnull().sum()\n# missing_values","metadata":{"execution":{"iopub.status.busy":"2024-08-19T03:19:14.563529Z","iopub.execute_input":"2024-08-19T03:19:14.563907Z","iopub.status.idle":"2024-08-19T03:19:14.611314Z","shell.execute_reply.started":"2024-08-19T03:19:14.563857Z","shell.execute_reply":"2024-08-19T03:19:14.610336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(train_df.shape)\n# print(train_df.columns)","metadata":{"execution":{"iopub.status.busy":"2024-08-19T03:19:18.650733Z","iopub.execute_input":"2024-08-19T03:19:18.651092Z","iopub.status.idle":"2024-08-19T03:19:18.657045Z","shell.execute_reply.started":"2024-08-19T03:19:18.651028Z","shell.execute_reply":"2024-08-19T03:19:18.655839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.array(im).shape","metadata":{"execution":{"iopub.status.busy":"2024-08-19T03:19:26.755967Z","iopub.execute_input":"2024-08-19T03:19:26.756357Z","iopub.status.idle":"2024-08-19T03:19:26.763158Z","shell.execute_reply.started":"2024-08-19T03:19:26.75629Z","shell.execute_reply":"2024-08-19T03:19:26.761987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-08-19T03:19:37.381104Z","iopub.execute_input":"2024-08-19T03:19:37.381442Z","iopub.status.idle":"2024-08-19T03:19:37.392337Z","shell.execute_reply.started":"2024-08-19T03:19:37.381393Z","shell.execute_reply":"2024-08-19T03:19:37.391511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df['label'].value_counts().plot(kind='pie', legend=True, autopct='%1.0f%%')","metadata":{"execution":{"iopub.status.busy":"2024-08-19T03:19:44.501737Z","iopub.execute_input":"2024-08-19T03:19:44.5021Z","iopub.status.idle":"2024-08-19T03:19:44.730197Z","shell.execute_reply.started":"2024-08-19T03:19:44.502048Z","shell.execute_reply":"2024-08-19T03:19:44.72917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"**DModel Architecture**\n\nResnet101 Architecture visualization: http://ethereon.github.io/netscope/#/gist/db945b393d40bfa26006\n\nxception","metadata":{}},{"cell_type":"markdown","source":"**Results and Analysis**","metadata":{}},{"cell_type":"code","source":"\n!ls ../input\n\ntorch.cuda.get_device_name(0)","metadata":{"_uuid":"eda84cac535b233620a5a521476896b92ab59b43","execution":{"iopub.status.busy":"2024-08-19T07:03:58.038161Z","iopub.execute_input":"2024-08-19T07:03:58.038513Z","iopub.status.idle":"2024-08-19T07:03:59.065338Z","shell.execute_reply.started":"2024-08-19T07:03:58.038436Z","shell.execute_reply":"2024-08-19T07:03:59.064618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import fastai\nfrom fastai.imports import *\nfrom fastai.vision import *\nfrom fastai.metrics import *\nfrom fastai.gen_doc.nbdoc import *\nprint('fast.ai version:{}'.format(fastai.__version__))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-19T06:59:19.149704Z","iopub.execute_input":"2024-08-19T06:59:19.15004Z","iopub.status.idle":"2024-08-19T06:59:19.155382Z","shell.execute_reply.started":"2024-08-19T06:59:19.14998Z","shell.execute_reply":"2024-08-19T06:59:19.15454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WORK_DIR = os.getcwd()","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2024-08-19T06:59:27.032511Z","iopub.execute_input":"2024-08-19T06:59:27.032851Z","iopub.status.idle":"2024-08-19T06:59:27.036489Z","shell.execute_reply.started":"2024-08-19T06:59:27.032787Z","shell.execute_reply":"2024-08-19T06:59:27.035487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_doc(get_transforms)","metadata":{"_uuid":"444540e3e26dc7d0ee2bf0bd282047073eb9b4da","execution":{"iopub.status.busy":"2024-08-19T06:59:30.139074Z","iopub.execute_input":"2024-08-19T06:59:30.139347Z","iopub.status.idle":"2024-08-19T06:59:30.152531Z","shell.execute_reply.started":"2024-08-19T06:59:30.139303Z","shell.execute_reply":"2024-08-19T06:59:30.151847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_doc(ImageDataBunch.from_csv)","metadata":{"_uuid":"bef8b66503a08a277e545a034a90a45b76f0750b","execution":{"iopub.status.busy":"2024-08-19T06:59:35.512094Z","iopub.execute_input":"2024-08-19T06:59:35.512632Z","iopub.status.idle":"2024-08-19T06:59:35.522233Z","shell.execute_reply.started":"2024-08-19T06:59:35.512326Z","shell.execute_reply":"2024-08-19T06:59:35.521499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_transforms = {\n    'flip_vert':True,\n    'max_rotate':90,\n    'max_zoom':1,\n    'max_warp':None,\n    'max_lighting':None,\n}\nparams = {'path': Path('../input/histopathologic-cancer-detection'),\n          'csv_labels':'train_labels.csv',\n          'ds_tfms':get_transforms(**param_transforms),\n          'suffix':'.tif',\n          'folder':'train',\n          'test':'test',\n          'size':32,\n          'bs':128,\n          'num_workers':0,\n         }\ndata = ImageDataBunch.from_csv(**params).normalize(imagenet_stats)\ndata","metadata":{"_uuid":"5b985c3c8183d1c2735a80737bdcd10ca3a1d5c8","execution":{"iopub.status.busy":"2024-08-19T06:59:37.785935Z","iopub.execute_input":"2024-08-19T06:59:37.786251Z","iopub.status.idle":"2024-08-19T07:01:26.383174Z","shell.execute_reply.started":"2024-08-19T06:59:37.786187Z","shell.execute_reply":"2024-08-19T07:01:26.382319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.show_batch(rows=3)","metadata":{"_uuid":"460dd818beee7fc33eae9ae654e150dd0d901fc6","execution":{"iopub.status.busy":"2024-08-19T06:53:28.687598Z","iopub.execute_input":"2024-08-19T06:53:28.687864Z","iopub.status.idle":"2024-08-19T06:53:37.167038Z","shell.execute_reply.started":"2024-08-19T06:53:28.687823Z","shell.execute_reply":"2024-08-19T06:53:37.166244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = create_cnn(data, models.resnet101, metrics=accuracy, path='.', model_dir='.')","metadata":{"_uuid":"ecd9b589132a49afec5db035d637d3fcc0ed98b0","execution":{"iopub.status.busy":"2024-08-19T11:30:16.233056Z","iopub.execute_input":"2024-08-19T11:30:16.233391Z","iopub.status.idle":"2024-08-19T11:30:17.965182Z","shell.execute_reply.started":"2024-08-19T11:30:16.233329Z","shell.execute_reply":"2024-08-19T11:30:17.964432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/input/","metadata":{"_uuid":"d84f3997339ab792eca3a3a34509aa816a527a23","execution":{"iopub.status.busy":"2024-08-19T07:03:43.493254Z","iopub.execute_input":"2024-08-19T07:03:43.493596Z","iopub.status.idle":"2024-08-19T07:03:44.535595Z","shell.execute_reply.started":"2024-08-19T07:03:43.493533Z","shell.execute_reply":"2024-08-19T07:03:44.534647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.load('histopathologic_cancer_detection')","metadata":{"_uuid":"622841b6c380f5b3ca1367ffad46aa026ffe3fa0","execution":{"iopub.status.busy":"2024-08-19T11:31:46.364817Z","iopub.execute_input":"2024-08-19T11:31:46.365119Z","iopub.status.idle":"2024-08-19T11:31:47.002325Z","shell.execute_reply.started":"2024-08-19T11:31:46.365076Z","shell.execute_reply":"2024-08-19T11:31:47.001644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.unfreeze()\n# learn.fit_one_cycle(16)\n#learn.fit_one_cycle(16,slice(1e-6,1e-4))","metadata":{"_uuid":"b4bddb0295b756555f09c01998e739a4e6b1cfd4","scrolled":true,"execution":{"iopub.status.busy":"2024-08-19T07:04:16.633268Z","iopub.execute_input":"2024-08-19T07:04:16.633825Z","iopub.status.idle":"2024-08-19T10:32:56.099465Z","shell.execute_reply.started":"2024-08-19T07:04:16.633605Z","shell.execute_reply":"2024-08-19T10:32:56.098726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#learn.save('histopathologic_cancer_detection')","metadata":{"_uuid":"abcb5a83fab1560a86cd879ed6c9981f94508548","execution":{"iopub.status.busy":"2024-08-19T10:33:19.761649Z","iopub.execute_input":"2024-08-19T10:33:19.761928Z","iopub.status.idle":"2024-08-19T10:33:20.388217Z","shell.execute_reply.started":"2024-08-19T10:33:19.761884Z","shell.execute_reply":"2024-08-19T10:33:20.387625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#learn.recorder.plot_losses()\n#plt.ylim([0,0.5])","metadata":{"_uuid":"9e25a5c6c1ecd16b7cf2b0a8cc0458e8188e8927","execution":{"iopub.status.busy":"2024-08-19T11:30:40.168644Z","iopub.execute_input":"2024-08-19T11:30:40.168958Z","iopub.status.idle":"2024-08-19T11:30:40.423955Z","shell.execute_reply.started":"2024-08-19T11:30:40.16891Z","shell.execute_reply":"2024-08-19T11:30:40.422518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check if DatasetType loaded\n# It's depend on difference between fast.ai version v1.0.39/v1.0.36.\ntry:\n    DatasetType\nexcept:\n    from fastai import DatasetType","metadata":{"_uuid":"4ade8fc06046c1e54a37d77239f166c1c8845c47","execution":{"iopub.status.busy":"2024-08-19T10:34:16.060678Z","iopub.execute_input":"2024-08-19T10:34:16.060954Z","iopub.status.idle":"2024-08-19T10:34:16.064953Z","shell.execute_reply.started":"2024-08-19T10:34:16.060911Z","shell.execute_reply":"2024-08-19T10:34:16.064169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    # TTA cause error @fast.ai v1.0.39\n    preds, _ = learn.TTA(ds_type=DatasetType.Test)\n    print('Predict w/ test time augument done.')\nexcept:\n    preds, _ = learn.get_preds(DatasetType.Test)","metadata":{"_uuid":"439616f809f25a1e68ed342b141b4ee677df4f53","execution":{"iopub.status.busy":"2024-08-19T10:34:36.610886Z","iopub.execute_input":"2024-08-19T10:34:36.611175Z","iopub.status.idle":"2024-08-19T10:58:10.808045Z","shell.execute_reply.started":"2024-08-19T10:34:36.611132Z","shell.execute_reply":"2024-08-19T10:58:10.807291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds","metadata":{"_uuid":"872974503d55f879beed2b59635a20616ed53e84","execution":{"iopub.status.busy":"2024-08-19T10:58:19.522246Z","iopub.execute_input":"2024-08-19T10:58:19.522571Z","iopub.status.idle":"2024-08-19T10:58:19.536807Z","shell.execute_reply.started":"2024-08-19T10:58:19.522522Z","shell.execute_reply":"2024-08-19T10:58:19.536147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = [file.stem for file in data.test_ds.x.items]\nids[:5]","metadata":{"_uuid":"2e105b3662c0db331deb25965d258794101578b9","execution":{"iopub.status.busy":"2024-08-19T10:58:25.988853Z","iopub.execute_input":"2024-08-19T10:58:25.989123Z","iopub.status.idle":"2024-08-19T10:58:26.09063Z","shell.execute_reply.started":"2024-08-19T10:58:25.98908Z","shell.execute_reply":"2024-08-19T10:58:26.089683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.classes","metadata":{"_uuid":"f045c2749386a30c3002f8996de52d3b7e24e9e4","execution":{"iopub.status.busy":"2024-08-19T10:58:31.124803Z","iopub.execute_input":"2024-08-19T10:58:31.125074Z","iopub.status.idle":"2024-08-19T10:58:31.129925Z","shell.execute_reply.started":"2024-08-19T10:58:31.12503Z","shell.execute_reply":"2024-08-19T10:58:31.129237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.read_csv('../input/histopathologic-cancer-detection/sample_submission.csv').set_index('id')\ndf.head()","metadata":{"_uuid":"863ad291fd10319fb7f1768be96d3b8efbcdbf6a","execution":{"iopub.status.busy":"2024-08-19T10:58:40.859775Z","iopub.execute_input":"2024-08-19T10:58:40.860056Z","iopub.status.idle":"2024-08-19T10:58:40.972764Z","shell.execute_reply.started":"2024-08-19T10:58:40.860012Z","shell.execute_reply":"2024-08-19T10:58:40.97191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.loc[ids,'label'] = preds[:,1].numpy()\ndf = df.reset_index()\ndf.head()","metadata":{"_uuid":"a6ba67643175d8d3f65840a42071f2f996690c04","execution":{"iopub.status.busy":"2024-08-19T10:58:47.395129Z","iopub.execute_input":"2024-08-19T10:58:47.395427Z","iopub.status.idle":"2024-08-19T10:58:47.493174Z","shell.execute_reply.started":"2024-08-19T10:58:47.39538Z","shell.execute_reply":"2024-08-19T10:58:47.492398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv('submission.csv', index=None)","metadata":{"_uuid":"c7762f5a74949c3fd3fb79f54d00a13b7b43f1ad","execution":{"iopub.status.busy":"2024-08-19T11:06:43.724245Z","iopub.execute_input":"2024-08-19T11:06:43.724566Z","iopub.status.idle":"2024-08-19T11:06:44.008395Z","shell.execute_reply.started":"2024-08-19T11:06:43.72452Z","shell.execute_reply":"2024-08-19T11:06:44.007633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conclusion**","metadata":{}},{"cell_type":"code","source":"!ls -la","metadata":{"_uuid":"2fe68e7b299795e44b4f2fa30137f863f9974f5e","execution":{"iopub.status.busy":"2024-08-19T11:09:38.673595Z","iopub.execute_input":"2024-08-19T11:09:38.673908Z","iopub.status.idle":"2024-08-19T11:09:39.707699Z","shell.execute_reply.started":"2024-08-19T11:09:38.67386Z","shell.execute_reply":"2024-08-19T11:09:39.706985Z"},"trusted":true},"execution_count":null,"outputs":[]}]}