{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import datetime\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pathlib\nimport tensorflow as tf\n\n\nfrom tensorflow import keras\nfrom tensorflow.keras import datasets, layers, Sequential, optimizers\nfrom tensorflow.keras.preprocessing import image_dataset_from_directory\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Flatten, experimental, BatchNormalization","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:40:22.963145Z","iopub.execute_input":"2022-12-20T09:40:22.963476Z","iopub.status.idle":"2022-12-20T09:40:29.595099Z","shell.execute_reply.started":"2022-12-20T09:40:22.963399Z","shell.execute_reply":"2022-12-20T09:40:29.59404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip ../input/diabetic-retinopathy-detection/trainLabels.csv.zip","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:40:29.59711Z","iopub.execute_input":"2022-12-20T09:40:29.5978Z","iopub.status.idle":"2022-12-20T09:40:30.579021Z","shell.execute_reply.started":"2022-12-20T09:40:29.597761Z","shell.execute_reply":"2022-12-20T09:40:30.577853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!apt install p7zip-full -y\n!7z x ../input/diabetic-retinopathy-detection/train.zip.001 \"-i!train/11*.jpeg\" -y # restrict extracted file to about 100 for the disk restriction\n!mkdir data\n!mv train data/train_11","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:40:30.582018Z","iopub.execute_input":"2022-12-20T09:40:30.582381Z","iopub.status.idle":"2022-12-20T09:40:53.930307Z","shell.execute_reply.started":"2022-12-20T09:40:30.582348Z","shell.execute_reply":"2022-12-20T09:40:53.929021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\n\nimg = Image.open(\"./data/train_11/1116_right.jpeg\")\n\nimport matplotlib.pyplot as plt\n\nplt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:40:53.933059Z","iopub.execute_input":"2022-12-20T09:40:53.933802Z","iopub.status.idle":"2022-12-20T09:40:55.926041Z","shell.execute_reply.started":"2022-12-20T09:40:53.933749Z","shell.execute_reply":"2022-12-20T09:40:55.924779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nbase_image_dir = os.path.join('.', 'data/train_11')\ndf = pd.read_csv(os.path.join('./trainLabels.csv'))\ndf['path'] = df['image'].map(lambda x: os.path.join(base_image_dir,'{}.jpeg'.format(x)))\ndf['exists'] = df['path'].map(os.path.exists) #Most of the files do not exist because this is a sample of the original dataset\ndf = df[df['exists']]\ndf = df.drop(columns=['image','exists'])\ndf = df.sample(frac=1).reset_index(drop=True)#shuffle dataframe\ndf['level'] = df['level'].astype(str)\ndf.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:41:04.311687Z","iopub.execute_input":"2022-12-20T09:41:04.312074Z","iopub.status.idle":"2022-12-20T09:41:04.590891Z","shell.execute_reply.started":"2022-12-20T09:41:04.312043Z","shell.execute_reply":"2022-12-20T09:41:04.58997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"level\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:41:06.793699Z","iopub.execute_input":"2022-12-20T09:41:06.794157Z","iopub.status.idle":"2022-12-20T09:41:06.812457Z","shell.execute_reply.started":"2022-12-20T09:41:06.794114Z","shell.execute_reply":"2022-12-20T09:41:06.811549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def balance_data(class_size,df):\n    train_df = df.groupby(['level']).apply(lambda x: x.sample(class_size, replace = True)).reset_index(drop = True)\n    train_df = train_df.sample(frac=1).reset_index(drop=True)\n    print('New Data Size:', train_df.shape[0], 'Old Size:', df.shape[0])\n    train_df['level'].hist(figsize = (10, 5))\n    return train_df","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:41:10.797851Z","iopub.execute_input":"2022-12-20T09:41:10.798301Z","iopub.status.idle":"2022-12-20T09:41:10.808077Z","shell.execute_reply.started":"2022-12-20T09:41:10.798259Z","shell.execute_reply":"2022-12-20T09:41:10.807074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_df, val_df = train_test_split(df,test_size=0.2) # Here we will perform an 80%/20% split of the dataset, with stratification to keep similar distribution in validation set\ntrain_df['level'].hist(figsize = (10, 5))\nlen(val_df)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:41:12.859788Z","iopub.execute_input":"2022-12-20T09:41:12.860334Z","iopub.status.idle":"2022-12-20T09:41:13.701484Z","shell.execute_reply.started":"2022-12-20T09:41:12.860299Z","shell.execute_reply":"2022-12-20T09:41:13.700375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = balance_data(train_df.pivot_table(index='level', aggfunc=len).max().max(),train_df) # I will oversample such that all classes have the same number of images as the maximum\ntrain_df['level'].hist(figsize = (10, 5))","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:41:15.613507Z","iopub.execute_input":"2022-12-20T09:41:15.614013Z","iopub.status.idle":"2022-12-20T09:41:15.916764Z","shell.execute_reply.started":"2022-12-20T09:41:15.613971Z","shell.execute_reply":"2022-12-20T09:41:15.915773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\ntrain_datagen = ImageDataGenerator(\n    rescale=1.0/255,\n    horizontal_flip = True,\n    zoom_range=0.2\n)\n\ntest_datagen = ImageDataGenerator(\n    rescale=1.0/255,\n    validation_split = 0.2\n)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:41:17.939451Z","iopub.execute_input":"2022-12-20T09:41:17.94013Z","iopub.status.idle":"2022-12-20T09:41:17.945974Z","shell.execute_reply.started":"2022-12-20T09:41:17.940094Z","shell.execute_reply":"2022-12-20T09:41:17.944738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = train_datagen.flow_from_dataframe(\n        train_df,\n        directory=\".\",\n        x_col=\"path\",\n        y_col=\"level\",\n        target_size=(256, 256),\n        batch_size=32,\n        class_mode='categorical')\nx_test = test_datagen.flow_from_dataframe(\n        val_df,\n        x_col=\"path\",\n        y_col=\"level\",\n        directory=\".\",\n        target_size=(256, 256),\n        batch_size=32,\n        class_mode='categorical')","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:41:19.846621Z","iopub.execute_input":"2022-12-20T09:41:19.847307Z","iopub.status.idle":"2022-12-20T09:41:19.887348Z","shell.execute_reply.started":"2022-12-20T09:41:19.847273Z","shell.execute_reply":"2022-12-20T09:41:19.886433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#CNN\nmodel = Sequential()\n\n#1st Module\nmodel.add(Conv2D(32, (5, 5), (2,2), padding='same', activation='relu', input_shape=(264, 264, 3)))\nmodel.add(Conv2D(32, (5, 5), (2,2), padding='same', activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\n#2nd Module\nmodel.add(Conv2D(64, (3, 3), (2,2), padding='same', activation='relu'))\nmodel.add(Conv2D(64, (3, 3), (2,2), padding='same', activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\n#3rd Module\nmodel.add(Conv2D(128, (3, 3), (1,1), padding='same', activation='relu'))\nmodel.add(Conv2D(128, (3, 3), (1,1), padding='same', activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\n#4th Module\nmodel.add(Conv2D(256, (3, 3), (1,1), padding='same', activation='relu'))\nmodel.add(Conv2D(256, (3, 3), (1,1), padding='same', activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\n#Flatten the layers\nmodel.add(Flatten())\n\n#Fully connected layers\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dropout(0.25))\nmodel.add(Dense(10, activation='softmax'))\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:41:39.588299Z","iopub.execute_input":"2022-12-20T09:41:39.588676Z","iopub.status.idle":"2022-12-20T09:41:43.694097Z","shell.execute_reply.started":"2022-12-20T09:41:39.588645Z","shell.execute_reply":"2022-12-20T09:41:43.693224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(keras.optimizers.Adam(learning_rate=0.001), loss='categorical_crossentropy', metrics=['Accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:41:47.939314Z","iopub.execute_input":"2022-12-20T09:41:47.939692Z","iopub.status.idle":"2022-12-20T09:41:47.9559Z","shell.execute_reply.started":"2022-12-20T09:41:47.939659Z","shell.execute_reply":"2022-12-20T09:41:47.954719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(x_train, epochs=15, batch_size=32, validation_data=x_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:41:51.908781Z","iopub.execute_input":"2022-12-20T09:41:51.90914Z","iopub.status.idle":"2022-12-20T09:42:09.057041Z","shell.execute_reply.started":"2022-12-20T09:41:51.90911Z","shell.execute_reply":"2022-12-20T09:42:09.05562Z"},"trusted":true},"execution_count":null,"outputs":[]}]}