{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport tensorflow as tf\nimport keras\nimport glob\nimport cv2\n\nimport numpy as np # linear algebra\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd \nfrom keras.preprocessing.image import ImageDataGenerator, load_img\nfrom keras.utils import to_categorical\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nimport random\nimport os\nimport shutil\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/resized-2015-2019-blindness-detection-images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def empty_dir(path):\n    folder = path\n    for filename in os.listdir(folder):\n        file_path = os.path.join(folder, filename)\n        try:\n            if os.path.isfile(file_path) or os.path.islink(file_path):\n                os.unlink(file_path)\n            elif os.path.isdir(file_path):\n                shutil.rmtree(file_path)\n        except Exception as e:\n            print('Failed to delete %s. Reason: %s' % (file_path, e))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndf = pd.read_csv('../input/resized-2015-2019-blindness-detection-images/labels/trainLabels15.csv')\ndf.drop(df.loc[df['level']==0].index, inplace=True)\n\n\nit=df.head(5)\nDATADIR2 = '../input/resized-2015-2019-blindness-detection-images/resized train 15/'\nfor _,row in it.iterrows():\n    print (row['image'],' ',row['level'])   \n    path = os.path.join(DATADIR2,row['image']+\".jpg\")\nimage = load_img(path)\nprint(path)\nplt.imshow(image)\n    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"data = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\nprint(data.head())\n\nimage = load_img(\"../input/aptos2019-blindness-detection/train_images/44e951e45dca.png\")\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndfTrain2015 = pd.read_csv('../input/resized-2015-2019-blindness-detection-images/labels/trainLabels15.csv')\ndfTrain2015.drop(dfTrain2015.loc[dfTrain2015['level']==0].index, inplace=True)\n\ndfTest2015 = pd.read_csv('../input/resized-2015-2019-blindness-detection-images/labels/testLabels15.csv')\ndfTest2015.drop(dfTest2015.loc[dfTest2015['level']==0].index, inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.hist(bins=50,figsize=(10,5))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfTrain2015.hist(bins=50,figsize=(10,5))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Cropping functions**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport glob\nimport cv2\nimport numpy as np\n\ndef crop_image_from_gray(img,tol=7):\n    \"\"\"\n    Crop out black borders\n    https://www.kaggle.com/ratthachat/aptos-updated-preprocessing-ben-s-cropping\n    \"\"\"  \n    \n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n        mask = gray_img>tol        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0):\n            return img\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img = np.stack([img1,img2,img3],axis=-1)\n        return img\n\ndef circle_crop_v2(img):\n    \"\"\"\n    Create circular crop around image centre\n    \"\"\"\n    img = crop_image_from_gray(img)\n\n    height, width, depth = img.shape\n    largest_side = np.max((height, width))\n    img = cv2.resize(img, (largest_side, largest_side))\n\n    height, width, depth = img.shape\n\n    x = int(width / 2)\n    y = int(height / 2)\n    r = np.amin((x, y))\n\n    circle_img = np.zeros((height, width), np.uint8)\n    cv2.circle(circle_img, (x, y), int(r), 1, thickness=-1)\n    img = cv2.bitwise_and(img, img, mask=circle_img)\n    img = crop_image_from_gray(img)\n\n    return img\n\n#img = cv2.imread(\"../input/aptos2019-blindness-detection/train_images/0104b032c141.png\")\n#crop_image_from_gray(img)\n#plt.imshow(img)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Ben Graham's Preprocessing Function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def benGraham(img):\n    image = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n#         image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    IMG_SIZE = 512\n    image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n    image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n    image=cv2.addWeighted ( image,4, cv2.GaussianBlur( image , (0,0) , IMG_SIZE/10) ,-4 ,128) # the trick is to add this line\n    return image\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib import pyplot as plt\nfig, ax3 = plt.subplots(nrows=1, ncols=4, figsize=(20,4))\n\nspaceboy = cv2.imread('../input/aptos2019-blindness-detection/train_images/1df0a4c23c95.png')\nax3[0].imshow(cv2.cvtColor(spaceboy, cv2.COLOR_BGR2RGB))\nax3[0].axis('off')\n\n\ncropboy = cv2.imread('../input/aptos2019-blindness-detection/train_images/0a1076183736.png')\nax3[1].imshow(cv2.cvtColor(cropboy, cv2.COLOR_BGR2RGB))\nax3[1].axis('off')\n\nsquareboy = cv2.imread('../input/aptos2019-blindness-detection/train_images/0e3572b5884a.png')\nax3[2].imshow(cv2.cvtColor(squareboy, cv2.COLOR_BGR2RGB))\nax3[2].axis('off')\n\nsupercropboy = cv2.imread('../input/aptos2019-blindness-detection/train_images/698d6e422a80.png')\nax3[3].imshow(cv2.cvtColor(supercropboy, cv2.COLOR_BGR2RGB))\nax3[3].axis('off')\n###################################\nfig, ax0 = plt.subplots(nrows=1, ncols=4, figsize=(20,4))\n\nimg = cv2.imread(\"../input/aptos2019-blindness-detection/train_images/1df0a4c23c95.png\")\n\nspaceboy = crop_image_from_gray(img)\nax0[0].imshow(cv2.cvtColor(spaceboy, cv2.COLOR_BGR2RGB))\nax0[0].axis('off')\n\nimg = cv2.imread(\"../input/aptos2019-blindness-detection/train_images/0a1076183736.png\")\n\ncropboy = crop_image_from_gray(img)\nax0[1].imshow(cv2.cvtColor(cropboy, cv2.COLOR_BGR2RGB))\nax0[1].axis('off')\n\nimg = cv2.imread(\"../input/aptos2019-blindness-detection/train_images/0e3572b5884a.png\")\n\nsquareboy = crop_image_from_gray(img)\nax0[2].imshow(cv2.cvtColor(squareboy, cv2.COLOR_BGR2RGB))\nax0[2].axis('off')\n\nimg = cv2.imread(\"../input/aptos2019-blindness-detection/train_images/698d6e422a80.png\")\n\nsupercropboy = crop_image_from_gray(img)\nax0[3].imshow(cv2.cvtColor(supercropboy, cv2.COLOR_BGR2RGB))\nax0[3].axis('off')\n\n######################################################\nfig, ax2 = plt.subplots(nrows=1, ncols=4, figsize=(20,4))\nspaceboy = cv2.imread('../input/aptos2019-blindness-detection/train_images/1df0a4c23c95.png')\n\nspaceboy = circle_crop_v2(img)\nax2[0].imshow(cv2.cvtColor(spaceboy, cv2.COLOR_BGR2RGB))\nax2[0].axis('off')\nimg = cv2.imread(\"../input/aptos2019-blindness-detection/train_images/0a1076183736.png\")\n\ncropboy = circle_crop_v2(img)\nax2[1].imshow(cv2.cvtColor(cropboy, cv2.COLOR_BGR2RGB))\nax2[1].axis('off')\nimg = cv2.imread(\"../input/aptos2019-blindness-detection/train_images/0e3572b5884a.png\")\n\nsquareboy = circle_crop_v2(img)\nax2[2].imshow(cv2.cvtColor(squareboy, cv2.COLOR_BGR2RGB))\nax2[2].axis('off')\nimg = cv2.imread(\"../input/aptos2019-blindness-detection/train_images/698d6e422a80.png\")\n\nsupercropboy = circle_crop_v2(img)\nax2[3].imshow(cv2.cvtColor(supercropboy, cv2.COLOR_BGR2RGB))\nax2[3].axis('off')\n#####################################################\nfig, ax4 = plt.subplots(nrows=1, ncols=4, figsize=(20,4))\n\nspaceboy = benGraham(spaceboy)\nax4[0].imshow(cv2.cvtColor(spaceboy, cv2.COLOR_BGR2RGB))\nax4[0].axis('off')\n\ncropboy = benGraham(cropboy)\nax4[1].imshow(cv2.cvtColor(cropboy, cv2.COLOR_BGR2RGB))\nax4[1].axis('off')\n\nsquareboy = benGraham(squareboy)\nax4[2].imshow(cv2.cvtColor(squareboy, cv2.COLOR_BGR2RGB))\nax4[2].axis('off')\n\nsupercropboy = benGraham(supercropboy)\nax4[3].imshow(cv2.cvtColor(supercropboy, cv2.COLOR_BGR2RGB))\nax4[3].axis('off')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Combiation of Cropping and Ben Garham functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def processingWithCircleCrop(data,beforePath,afterPath,ext):\n    for i in data:\n        path=os.path.join(beforePath+i+ext)\n        img=cv2.imread(path)\n        img=circle_crop_v2(img)\n        img = benGraham(img)\n        outImg=Image.fromarray(img)\n        outImg.save(os.path.join(afterPath,i+'.png'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef process(data,beforePath,afterPath,ext):\n    processingWithCircleCrop(data,beforePath,afterPath,ext)\n    print(\"Process completed\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir '/kaggle/working/train.zip/'\n    \nAfterPath='/kaggle/working/trainData2015/'\nif(os.path.exists(AfterPath)== False):\n    !mkdir '/kaggle/working/trainData2015/'\n    \n    \nAfterPath='/kaggle/working/testData2019/'\n\nif(os.path.exists(AfterPath)== False):\n    !mkdir '/kaggle/working/testData2019/'\n    \nAfterPath='/kaggle/working/testData2015/'\n\nif(os.path.exists(AfterPath)== False):\n    !mkdir '/kaggle/working/testData2015/'\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainData2019BeforePath='../input/aptos2019-blindness-detection/train_images/'\ntrainData2015BeforePath='../input/resized-2015-2019-blindness-detection-images/resized train 15/'\n\ntestData2019BeforePath='../input/aptos2019-blindness-detection/test_images/'  \ntestData2015BeforePath='../input/resized-2015-2019-blindness-detection-images/resized test 15/'\n###\ntrainData2019AfterPath='/kaggle/working/trainData2019/'\ntrainData2015AfterPath='/kaggle/working/trainData2015/'\n\ntestData2019AfterPath='/kaggle/working/testData2019/'    \ntestData2015AfterPath='/kaggle/working/testData2015/'\n###\ndfTrain2015 = pd.read_csv('../input/resized-2015-2019-blindness-detection-images/labels/trainLabels15.csv')\ndfTrain2015.drop(dfTrain2015.loc[dfTrain2015['level']==0].index, inplace=True)\n\ndfTrain2019 = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n\n####\n\ntrainData2015=dfTrain2015['image']\n\n\ntrainData2019=dfTrain2019['id_code']\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Combining 2015 and 2019 Data for a more balanced Data destribution ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def setIndx(lvl):\n    lvl.set_index([\"image\"], inplace = True, \n                    append = True, drop = True) \n   \n    # resetting index \n\n    lvl.reset_index(inplace = True) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = dfTrain2015\n\nlvl1 =df.loc[df['level'] == 1]\nlvl2 =df.loc[df['level'] == 2]\nlvl3 =df.loc[df['level'] == 3]\nlvl4 =df.loc[df['level'] == 4]\nsetIndx(lvl1)\nsetIndx(lvl2)\nsetIndx(lvl3)\nsetIndx(lvl4)\n\nprint(\"##################\")\nprint(lvl1.head())\nprint(\"##################\")\nprint(lvl2.tail())\nprint(\"##################\")\nprint(lvl3.head())\nprint(\"##################\")\nprint(lvl4.head())\nprint(lvl1.shape,lvl2.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lvl1 = lvl1.drop(lvl1[lvl1.index > 1299].index) \nlvl2 = lvl2.drop(lvl2[lvl2.index > 699].index) \nprint(lvl1.shape,lvl2.shape,lvl3.shape,lvl4.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nframes = [lvl1, lvl2, lvl3,lvl4]\nfinal2015df = pd.concat(frames)\nfinal2015df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfrom sklearn.utils import shuffle\nfinal2015df=shuffle(final2015df)\nfinal2015df = final2015df.drop('level_0', 1)\nsetIndx(final2015df)\nfinal2015df = final2015df.drop('level_0', 1)\nfinal2015df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final2015df.rename({'image': 'id_code', 'level': 'diagnosis'}, axis=1, inplace=True)\nframes = [final2015df,dfTrain2019]\nfulldf = pd.concat(frames)\nfulldf.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Difference of Data destribution between the original dataset and the combined dataset:","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"old data:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data.hist(bins=50,figsize=(10,5))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"combined data:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fulldf.hist(bins=50,figsize=(10,5))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final2015df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing The data ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"trainData2015=final2015df['id_code']\ntrainData2019=dfTrain2019['id_code']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"process(trainData2019,trainData2019BeforePath,trainData2019AfterPath,'.png')\nprocess(trainData201,trainData2015BeforePath,trainData2015AfterPath,'.jpg')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import zipfile\n\ndef zipdir(path, ziph):\n    # ziph is zipfile handle\n    for root, dirs, files in os.walk(path):\n        for file in files:\n            ziph.write(os.path.join(root, file))\n\nzipf = zipfile.ZipFile('/kaggle/working/train.zip', 'w', zipfile.ZIP_DEFLATED)\nzipdir(trainData2015AfterPath, zipf)\nzipdir(trainData2019AfterPath, zipf)\n\nzipf.close()\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}