{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-07T02:57:32.264245Z","iopub.execute_input":"2023-12-07T02:57:32.264984Z","iopub.status.idle":"2023-12-07T02:57:37.558071Z","shell.execute_reply.started":"2023-12-07T02:57:32.264932Z","shell.execute_reply":"2023-12-07T02:57:37.557171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ref - https://stackoverflow.com/questions/14463277/how-to-disable-python-warnings\n\n# Basic Libs..\nimport multiprocessing\nfrom multiprocessing.pool import ThreadPool\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm,tqdm_notebook\nfrom prettytable import PrettyTable\nimport pickle\nimport os\nprint('CWD is ',os.getcwd())\n\n# Vis Libs..\nfrom sklearn.manifold import TSNE\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nplt.rcParams[\"axes.grid\"] = False\n\n# Image Libs.\nfrom PIL import Image\nimport cv2\n\n# sklearn libs..\nfrom sklearn.model_selection import train_test_split\n\n# DL Libs..\nimport keras\nfrom keras import applications\nfrom keras.preprocessing.image import ImageDataGenerator,img_to_array,array_to_img,load_img\nfrom keras import optimizers,Model,Sequential\nfrom keras.layers import Input,GlobalAveragePooling2D,Dropout,Dense,Activation\nfrom keras.callbacks import EarlyStopping,ReduceLROnPlateau","metadata":{"execution":{"iopub.status.busy":"2023-12-07T02:57:37.559842Z","iopub.execute_input":"2023-12-07T02:57:37.560744Z","iopub.status.idle":"2023-12-07T02:57:51.683841Z","shell.execute_reply.started":"2023-12-07T02:57:37.560711Z","shell.execute_reply":"2023-12-07T02:57:51.682867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nThis function reads data from the respective train and test directories\n'''\n\ndef load_data():\n    train = pd.read_csv('/kaggle/input/aptos2019-blindness-detection/train.csv')\n    test = pd.read_csv('/kaggle/input/aptos2019-blindness-detection/test.csv')\n    \n    train_dir = os.path.join('./','/kaggle/input/aptos2019-blindness-detection/train_images')\n    test_dir = os.path.join('./','/kaggle/input/aptos2019-blindness-detection/test_images')\n    \n    train['file_path'] = train['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))\n    test['file_path'] = test['id_code'].map(lambda x: os.path.join(test_dir,'{}.png'.format(x)))\n    \n    train['file_name'] = train[\"id_code\"].apply(lambda x: x + \".png\")\n    test['file_name'] = test[\"id_code\"].apply(lambda x: x + \".png\")\n    \n    train['diagnosis'] = train['diagnosis'].astype(str)\n    \n    return train,test","metadata":{"execution":{"iopub.status.busy":"2023-12-07T02:57:51.685003Z","iopub.execute_input":"2023-12-07T02:57:51.685634Z","iopub.status.idle":"2023-12-07T02:57:51.692982Z","shell.execute_reply.started":"2023-12-07T02:57:51.685604Z","shell.execute_reply":"2023-12-07T02:57:51.692081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train,df_test = load_data()\nprint(df_train.shape,df_test.shape,'\\n')\ndf_train.head(6)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T02:57:51.695645Z","iopub.execute_input":"2023-12-07T02:57:51.696015Z","iopub.status.idle":"2023-12-07T02:57:51.787069Z","shell.execute_reply.started":"2023-12-07T02:57:51.695983Z","shell.execute_reply":"2023-12-07T02:57:51.785846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_train,df_train_valid = train_test_split(df_train,test_size = 0.2)\nprint(df_train_train.shape,df_train_valid.shape)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T02:57:51.788664Z","iopub.execute_input":"2023-12-07T02:57:51.78942Z","iopub.status.idle":"2023-12-07T02:57:51.799553Z","shell.execute_reply.started":"2023-12-07T02:57:51.789379Z","shell.execute_reply":"2023-12-07T02:57:51.798561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''This Function Plots a Bar plot of output Classes Distribution'''\n\ndef plot_classes(df,title):\n    df_group = pd.DataFrame(df.groupby('diagnosis').agg('size').reset_index())\n    df_group.columns = ['diagnosis','count']\n\n    sns.set(rc={'figure.figsize':(10,5)}, style = 'whitegrid')\n    sns.barplot(x = 'diagnosis',y='count',data = df_group,palette = \"Blues_d\")\n    plt.title('Output Class Distribution ' + str(title))\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-07T02:57:51.801061Z","iopub.execute_input":"2023-12-07T02:57:51.801749Z","iopub.status.idle":"2023-12-07T02:57:51.809296Z","shell.execute_reply.started":"2023-12-07T02:57:51.801711Z","shell.execute_reply":"2023-12-07T02:57:51.808268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_classes(df_train_train,\"TRAIN DATA\")","metadata":{"execution":{"iopub.status.busy":"2023-12-07T02:57:51.810546Z","iopub.execute_input":"2023-12-07T02:57:51.810897Z","iopub.status.idle":"2023-12-07T02:57:52.167858Z","shell.execute_reply.started":"2023-12-07T02:57:51.810863Z","shell.execute_reply":"2023-12-07T02:57:52.166809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_classes(df_train_valid,'VALIDATION DATA')","metadata":{"execution":{"iopub.status.busy":"2023-12-07T02:57:52.169332Z","iopub.execute_input":"2023-12-07T02:57:52.169662Z","iopub.status.idle":"2023-12-07T02:57:52.467279Z","shell.execute_reply.started":"2023-12-07T02:57:52.169633Z","shell.execute_reply":"2023-12-07T02:57:52.466199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE  = 512","metadata":{"execution":{"iopub.status.busy":"2023-12-07T02:57:52.468771Z","iopub.execute_input":"2023-12-07T02:57:52.46967Z","iopub.status.idle":"2023-12-07T02:57:52.473507Z","shell.execute_reply.started":"2023-12-07T02:57:52.469637Z","shell.execute_reply":"2023-12-07T02:57:52.472779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Function loads an image from Folder , Resizes and saves in another directory '''\n\ndef image_resize_save(file):\n    input_filepath = os.path.join('./','/kaggle/input/aptos2019-blindness-detection/train_images','{}.png'.format(file))\n    output_filepath = os.path.join('./','valid_images_resized','{}.png'.format(file))\n    img = cv2.imread(input_filepath)\n    cv2.imwrite(output_filepath, cv2.resize(img, (IMG_SIZE,IMG_SIZE)))\nimage_resize_save(df_train.id_code.iloc[201])","metadata":{"execution":{"iopub.status.busy":"2023-12-07T02:57:52.476297Z","iopub.execute_input":"2023-12-07T02:57:52.476973Z","iopub.status.idle":"2023-12-07T02:57:52.569957Z","shell.execute_reply.started":"2023-12-07T02:57:52.476943Z","shell.execute_reply":"2023-12-07T02:57:52.569145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''This Function uses Multi processing for faster saving of images into folder'''\n\ndef multiprocess_image_downloader(process:int, imgs:list):\n    \"\"\"\n    Inputs:\n        process: (int) number of process to run\n        imgs:(list) list of images\n    \"\"\"\n    print(f'MESSAGE: Running {process} process')\n    results = ThreadPool(process).map(image_resize_save, imgs)\n    return results","metadata":{"execution":{"iopub.status.busy":"2023-12-07T02:57:52.571523Z","iopub.execute_input":"2023-12-07T02:57:52.572207Z","iopub.status.idle":"2023-12-07T02:57:52.578767Z","shell.execute_reply.started":"2023-12-07T02:57:52.572164Z","shell.execute_reply":"2023-12-07T02:57:52.577773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use 6 cores\nmultiprocess_image_downloader(6, list(df_train_valid.id_code.values))","metadata":{"execution":{"iopub.status.busy":"2023-12-07T02:57:52.580064Z","iopub.execute_input":"2023-12-07T02:57:52.580572Z","iopub.status.idle":"2023-12-07T02:58:25.202271Z","shell.execute_reply.started":"2023-12-07T02:57:52.580545Z","shell.execute_reply":"2023-12-07T02:58:25.200322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img\n\ndef circle_crop(img, sigmaX = 30):   \n    \"\"\"\n    Create circular crop around image centre    \n    \"\"\"    \n    img = crop_image_from_gray(img)    \n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    height, width, depth = img.shape    \n    \n    x = int(width/2)\n    y = int(height/2)\n    r = np.amin((x,y))\n    \n    circle_img = np.zeros((height, width), np.uint8)\n    cv2.circle(circle_img, (x,y), int(r), 1, thickness=-1)\n    img = cv2.bitwise_and(img, img, mask=circle_img)\n    img = crop_image_from_gray(img)\n    img=cv2.addWeighted(img,4, cv2.GaussianBlur( img , (0,0) , sigmaX) ,-4 ,128)\n    return img \n\ndef preprocess_image(file):\n    input_filepath = os.path.join('./','test_images_resized','{}.png'.format(file))\n    output_filepath = os.path.join('./','test_images_resized_preprocessed','{}.png'.format(file))\n    \n    img = cv2.imread(input_filepath)\n    img = circle_crop(img) \n    cv2.imwrite(output_filepath, cv2.resize(img, (IMG_SIZE,IMG_SIZE)))","metadata":{"execution":{"iopub.status.busy":"2023-12-07T02:58:25.203716Z","iopub.execute_input":"2023-12-07T02:58:25.204669Z","iopub.status.idle":"2023-12-07T02:58:25.217762Z","shell.execute_reply.started":"2023-12-07T02:58:25.204636Z","shell.execute_reply":"2023-12-07T02:58:25.216315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''This Function uses Multi processing for faster saving of images into folder'''\n\ndef multiprocess_image_processor(process:int, imgs:list):\n    \"\"\"\n    Inputs:\n        process: (int) number of process to run\n        imgs:(list) list of images\n    \"\"\"\n    print(f'MESSAGE: Running {process} process')\n    results = ThreadPool(process).map(preprocess_image, imgs)\n    return results","metadata":{"execution":{"iopub.status.busy":"2023-12-07T02:58:25.220095Z","iopub.execute_input":"2023-12-07T02:58:25.220839Z","iopub.status.idle":"2023-12-07T02:58:25.235519Z","shell.execute_reply.started":"2023-12-07T02:58:25.220797Z","shell.execute_reply":"2023-12-07T02:58:25.234662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use 6 cores\nmultiprocess_image_processor(6, list(df_train_valid.id_code.values))","metadata":{"execution":{"iopub.status.busy":"2023-12-07T02:58:25.236542Z","iopub.execute_input":"2023-12-07T02:58:25.237289Z","iopub.status.idle":"2023-12-07T02:58:26.122531Z","shell.execute_reply.started":"2023-12-07T02:58:25.237258Z","shell.execute_reply":"2023-12-07T02:58:26.121096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}