{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":418031,"sourceType":"datasetVersion","datasetId":131128},{"sourceId":952401,"sourceType":"datasetVersion","datasetId":517172},{"sourceId":7987068,"sourceType":"datasetVersion","datasetId":4701600}],"dockerImageVersionId":30674,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport numpy as np\nimport pandas as pd\nimport random, os\nimport shutil\nimport matplotlib.pyplot as plt\nfrom matplotlib.image import imread\n#from keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.metrics import categorical_accuracy\nfrom sklearn.model_selection import train_test_split","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-02T03:24:25.077966Z","iopub.execute_input":"2024-04-02T03:24:25.078983Z","iopub.status.idle":"2024-04-02T03:24:25.085045Z","shell.execute_reply.started":"2024-04-02T03:24:25.078945Z","shell.execute_reply":"2024-04-02T03:24:25.083938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -Uqq fastbook","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:24:31.730531Z","iopub.execute_input":"2024-04-02T03:24:31.730917Z","iopub.status.idle":"2024-04-02T03:24:47.315569Z","shell.execute_reply.started":"2024-04-02T03:24:31.730873Z","shell.execute_reply":"2024-04-02T03:24:47.314353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastbook import *\nfrom fastai.vision.widgets import *","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:24:59.219635Z","iopub.execute_input":"2024-04-02T03:24:59.219996Z","iopub.status.idle":"2024-04-02T03:25:07.061744Z","shell.execute_reply.started":"2024-04-02T03:24:59.219968Z","shell.execute_reply":"2024-04-02T03:25:07.060923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(r'../input/diabetic-retinopathy-224x224-gaussian-filtered/train.csv')\n\ndiagnosis_dict_binary = {\n    0: 'No_DR',\n    1: 'DR',\n    2: 'DR',\n    3: 'DR',\n    4: 'DR'\n}\n\ndiagnosis_dict = {\n    0: 'No_DR',\n    1: 'Mild',\n    2: 'Moderate',\n    3: 'Severe',\n    4: 'Proliferate_DR',\n}\n\n\ndf['binary_type'] =  df['diagnosis'].map(diagnosis_dict_binary.get)\ndf['type'] = df['diagnosis'].map(diagnosis_dict.get)\ndf.head()\n","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:25:11.021591Z","iopub.execute_input":"2024-04-02T03:25:11.022359Z","iopub.status.idle":"2024-04-02T03:25:11.068247Z","shell.execute_reply.started":"2024-04-02T03:25:11.022324Z","shell.execute_reply":"2024-04-02T03:25:11.067348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['type'].value_counts().plot(kind='barh')","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:25:14.279788Z","iopub.execute_input":"2024-04-02T03:25:14.280492Z","iopub.status.idle":"2024-04-02T03:25:14.578347Z","shell.execute_reply.started":"2024-04-02T03:25:14.280459Z","shell.execute_reply":"2024-04-02T03:25:14.577385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train, test = train_test_split(df, test_size = 0.2, stratify = df['type'])\n#train, test = train_test_split(df, test_size = 0.2, stratify = train_a['type'])\n\nprint(\"For Training Dataset :\")\nprint(train['type'].value_counts(), '\\n')\nprint(\"For Testing Dataset :\")\nprint(test['type'].value_counts(), '\\n')","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:25:16.359808Z","iopub.execute_input":"2024-04-02T03:25:16.360229Z","iopub.status.idle":"2024-04-02T03:25:16.382898Z","shell.execute_reply.started":"2024-04-02T03:25:16.360189Z","shell.execute_reply":"2024-04-02T03:25:16.381947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_dir = ''\n\ntrain_dir = os.path.join(base_dir, 'train')\n#val_dir = os.path.join(base_dir, 'val')\ntest_dir = os.path.join(base_dir, 'test')\n\nif os.path.exists(train_dir):\n    shutil.rmtree(train_dir)\nos.makedirs(train_dir)\n\n# if os.path.exists(val_dir):\n#     shutil.rmtree(val_dir)\n# os.makedirs(val_dir)\n\nif os.path.exists(test_dir):\n    shutil.rmtree(test_dir)\nos.makedirs(test_dir)","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:25:22.827805Z","iopub.execute_input":"2024-04-02T03:25:22.828485Z","iopub.status.idle":"2024-04-02T03:25:22.834862Z","shell.execute_reply.started":"2024-04-02T03:25:22.828451Z","shell.execute_reply":"2024-04-02T03:25:22.833723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"src_dir = r'../input/diabetic-retinopathy-224x224-gaussian-filtered/gaussian_filtered_images/gaussian_filtered_images'\nfor index,row in train.iterrows():\n    diagnosis = row['type']\n    binary_diagnosis = row['binary_type']\n    id_code = row['id_code'] + \".png\"\n    srcfile = os.path.join(src_dir, diagnosis, id_code)\n    dstfile = os.path.join(train_dir, binary_diagnosis)\n    os.makedirs(dstfile, exist_ok = True)\n    shutil.copy(srcfile, dstfile)\n\n# for index, row in val.iterrows():\n#     diagnosis = row['type']\n#     binary_diagnosis = row['binary_type']\n#     id_code = row['id_code'] + \".png\"\n#     srcfile = os.path.join(src_dir, diagnosis, id_code)\n#     dstfile = os.path.join(val_dir, binary_diagnosis)\n#     os.makedirs(dstfile, exist_ok = True)\n#     shutil.copy(srcfile, dstfile)\n \nfor index, row in test.iterrows():\n    diagnosis = row['type']\n    binary_diagnosis = row['binary_type']\n    id_code = row['id_code'] + \".png\"\n    srcfile = os.path.join(src_dir, diagnosis, id_code)\n    dstfile = os.path.join(test_dir, binary_diagnosis)\n    os.makedirs(dstfile, exist_ok = True)\n    shutil.copy(srcfile, dstfile)","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:25:27.380288Z","iopub.execute_input":"2024-04-02T03:25:27.381089Z","iopub.status.idle":"2024-04-02T03:25:58.114700Z","shell.execute_reply.started":"2024-04-02T03:25:27.381055Z","shell.execute_reply":"2024-04-02T03:25:58.113618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**RESNET**","metadata":{}},{"cell_type":"code","source":"train_path = 'train'\ntest_path = 'test'\n\ntrain_batches = ImageDataGenerator(rescale = 1./255).flow_from_directory(train_path, target_size=(224,224), shuffle = True)\ntest_batches = ImageDataGenerator(rescale = 1./255).flow_from_directory(test_path, target_size=(224,224), shuffle = False)","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:25:58.116589Z","iopub.execute_input":"2024-04-02T03:25:58.117249Z","iopub.status.idle":"2024-04-02T03:25:58.312448Z","shell.execute_reply.started":"2024-04-02T03:25:58.117212Z","shell.execute_reply":"2024-04-02T03:25:58.311721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path=Path(\"/kaggle/working/train\")\ntest_path=Path(\"/kaggle/working/test\")","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:26:19.534640Z","iopub.execute_input":"2024-04-02T03:26:19.535006Z","iopub.status.idle":"2024-04-02T03:26:19.539735Z","shell.execute_reply.started":"2024-04-02T03:26:19.534978Z","shell.execute_reply":"2024-04-02T03:26:19.538721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dc = DataBlock(\n    blocks=(ImageBlock,CategoryBlock),\n    get_items=get_image_files,\n    splitter=RandomSplitter(valid_pct=0.2,seed=42),\n    get_y=parent_label,\n    item_tfms=Resize(128)\n)\ndls = dc.dataloaders(train_path)\ndls.valid.show_batch(max_n=4,nrows=1)","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:26:27.074437Z","iopub.execute_input":"2024-04-02T03:26:27.075111Z","iopub.status.idle":"2024-04-02T03:26:28.509744Z","shell.execute_reply.started":"2024-04-02T03:26:27.075077Z","shell.execute_reply":"2024-04-02T03:26:28.508825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = vision_learner(dls,resnet18,metrics=error_rate)\nlearn.fine_tune(8)","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:26:37.635072Z","iopub.execute_input":"2024-04-02T03:26:37.635439Z","iopub.status.idle":"2024-04-02T03:27:34.243609Z","shell.execute_reply.started":"2024-04-02T03:26:37.635408Z","shell.execute_reply":"2024-04-02T03:27:34.242474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interp = ClassificationInterpretation.from_learner(learn)\ninterp.plot_confusion_matrix()","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:28:08.470858Z","iopub.execute_input":"2024-04-02T03:28:08.471695Z","iopub.status.idle":"2024-04-02T03:28:11.481647Z","shell.execute_reply.started":"2024-04-02T03:28:08.471640Z","shell.execute_reply":"2024-04-02T03:28:11.479887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.export('/kaggle/working/stage-1_resnet.pkl')","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:28:24.229258Z","iopub.execute_input":"2024-04-02T03:28:24.230190Z","iopub.status.idle":"2024-04-02T03:28:24.352415Z","shell.execute_reply.started":"2024-04-02T03:28:24.230155Z","shell.execute_reply":"2024-04-02T03:28:24.351621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interp.plot_top_losses(5,nrows=1)","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:28:34.116336Z","iopub.execute_input":"2024-04-02T03:28:34.117080Z","iopub.status.idle":"2024-04-02T03:28:35.016742Z","shell.execute_reply.started":"2024-04-02T03:28:34.117049Z","shell.execute_reply":"2024-04-02T03:28:35.015594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cleaner = ImageClassifierCleaner(learn)\n# cleaner","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:29:09.966732Z","iopub.execute_input":"2024-04-02T03:29:09.967549Z","iopub.status.idle":"2024-04-02T03:29:09.971647Z","shell.execute_reply.started":"2024-04-02T03:29:09.967513Z","shell.execute_reply":"2024-04-02T03:29:09.970697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for idx in cleaner.delete(): cleaner.fns[idx].unlink()","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:29:10.617292Z","iopub.execute_input":"2024-04-02T03:29:10.617919Z","iopub.status.idle":"2024-04-02T03:29:10.622025Z","shell.execute_reply.started":"2024-04-02T03:29:10.617887Z","shell.execute_reply":"2024-04-02T03:29:10.621071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#learn = load_learner('path_to_model_directory')  # Load the pre-trained model\n\ntest_db = DataBlock(\n    blocks=(ImageBlock, CategoryBlock), \n    get_items=get_image_files,\n    #splitter=GrandparentSplitter(valid_name='test'),\n    \n    get_y=parent_label,\n    item_tfms=Resize(224)\n)\n\n\ntest_dl = test_db.dataloaders(test_path)\nlearn.test_dl = test_dl\nresults = learn.validate()\n#print(f\"loss:{results[0]}\")\nprint(f\"accuracy:{1-results[1]}\")","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:29:12.953262Z","iopub.execute_input":"2024-04-02T03:29:12.953663Z","iopub.status.idle":"2024-04-02T03:29:14.840756Z","shell.execute_reply.started":"2024-04-02T03:29:12.953629Z","shell.execute_reply":"2024-04-02T03:29:14.839506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apply_gaussian_filter(image, sigmaX=10):\n    # Applying Gaussian blur\n    gaussian = cv2.addWeighted(image, 4, cv2.GaussianBlur(image, (0,0), sigmaX), -4, 128)\n    return gaussian","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:29:29.203838Z","iopub.execute_input":"2024-04-02T03:29:29.204355Z","iopub.status.idle":"2024-04-02T03:29:29.209884Z","shell.execute_reply.started":"2024-04-02T03:29:29.204315Z","shell.execute_reply":"2024-04-02T03:29:29.208757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred,idx,probs = learn.predict(PILImage.create('/kaggle/input/diabetic-retinopathy-resized/resized_train_cropped/resized_train_cropped/3088_right.jpeg'))\nprint(pred)\nprint(f\"The probability of {pred} '{probs[idx]:.4f}'\")","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:29:31.709775Z","iopub.execute_input":"2024-04-02T03:29:31.710150Z","iopub.status.idle":"2024-04-02T03:29:31.845183Z","shell.execute_reply.started":"2024-04-02T03:29:31.710118Z","shell.execute_reply":"2024-04-02T03:29:31.844387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink \nFileLink(r'stage-1_resnet.pkl')","metadata":{"execution":{"iopub.status.busy":"2024-04-02T03:31:17.573612Z","iopub.execute_input":"2024-04-02T03:31:17.573990Z","iopub.status.idle":"2024-04-02T03:31:17.580616Z","shell.execute_reply.started":"2024-04-02T03:31:17.573962Z","shell.execute_reply":"2024-04-02T03:31:17.579697Z"},"trusted":true},"execution_count":null,"outputs":[]}]}