{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-19T16:00:27.957871Z","iopub.execute_input":"2021-08-19T16:00:27.958411Z","iopub.status.idle":"2021-08-19T16:00:27.963596Z","shell.execute_reply.started":"2021-08-19T16:00:27.958294Z","shell.execute_reply":"2021-08-19T16:00:27.962578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport itertools\nimport os\nimport sys\nfrom prettytable import PrettyTable\nimport pickle\nfrom random import shuffle\nfrom sklearn.model_selection import train_test_split\nimport multiprocessing\nfrom multiprocessing.pool import ThreadPool\nfrom tqdm import tqdm_notebook\nprint(multiprocessing.cpu_count(),\" CPU cores\")\n\nimport seaborn as sns\n%matplotlib inline\nimport matplotlib.pyplot as plt\nplt.rcParams[\"axes.grid\"] = False\n\nfrom sklearn.metrics import confusion_matrix, cohen_kappa_score,accuracy_score\nfrom sklearn.preprocessing import MultiLabelBinarizer\n\nfrom PIL import Image\nimport cv2\nimport tensorflow as tf\n\nimport tensorflow.keras as keras\nfrom keras import applications\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras import optimizers,Model,Sequential\nfrom keras.layers import Input,GlobalAveragePooling2D,Dropout,Dense,Activation,BatchNormalization,GlobalMaxPooling2D,concatenate,Flatten\nfrom keras.callbacks import EarlyStopping,ReduceLROnPlateau,Callback\nfrom keras.initializers import random_normal\nfrom keras.models import load_model\nfrom keras.losses import binary_crossentropy,categorical_crossentropy,mean_squared_error\nfrom keras import backend as K\n","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:00:27.972636Z","iopub.execute_input":"2021-08-19T16:00:27.973069Z","iopub.status.idle":"2021-08-19T16:00:35.506693Z","shell.execute_reply.started":"2021-08-19T16:00:27.973009Z","shell.execute_reply":"2021-08-19T16:00:35.505669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nos.chdir('../input/aptos2019-blindness-detection')\nprint(\"We are currently in the folder of \",os.getcwd())","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:00:35.508337Z","iopub.execute_input":"2021-08-19T16:00:35.508645Z","iopub.status.idle":"2021-08-19T16:00:35.515361Z","shell.execute_reply.started":"2021-08-19T16:00:35.508616Z","shell.execute_reply":"2021-08-19T16:00:35.514280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data():\n    train = pd.read_csv('train.csv')\n    test = pd.read_csv('test.csv')\n    \n    train_dir = os.path.join('./','train_images/')\n    test_dir = os.path.join('./','test_images/')\n    \n    train['file_path'] = train['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))\n    test['file_path'] = test['id_code'].map(lambda x: os.path.join(test_dir,'{}.png'.format(x)))\n    \n    train['file_name'] = train[\"id_code\"].apply(lambda x: x + \".png\")\n    test['file_name'] = test[\"id_code\"].apply(lambda x: x + \".png\")\n    \n    train['diagnosis'] = train['diagnosis'].astype(str)\n    \n    return train,test","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:00:35.517488Z","iopub.execute_input":"2021-08-19T16:00:35.517992Z","iopub.status.idle":"2021-08-19T16:00:35.527742Z","shell.execute_reply.started":"2021-08-19T16:00:35.517891Z","shell.execute_reply":"2021-08-19T16:00:35.526520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train,df_test = load_data()\nprint(df_train.shape,df_test.shape,'\\n')\ndf_train.head(6)","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:00:35.529759Z","iopub.execute_input":"2021-08-19T16:00:35.530150Z","iopub.status.idle":"2021-08-19T16:00:35.618091Z","shell.execute_reply.started":"2021-08-19T16:00:35.530116Z","shell.execute_reply":"2021-08-19T16:00:35.616906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_classes(df):\n    df_group = pd.DataFrame(df.groupby('diagnosis').agg('size').reset_index())\n    df_group.columns = ['diagnosis','count']\n\n    sns.set(rc={'figure.figsize':(10,5)}, style = 'whitegrid')\n    sns.barplot(x = 'diagnosis',y='count',data = df_group,palette = \"Blues_d\")\n    plt.title('Output Class Distribution')\n    plt.show()\nplot_classes(df_train)","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:00:35.619809Z","iopub.execute_input":"2021-08-19T16:00:35.620260Z","iopub.status.idle":"2021-08-19T16:00:35.845166Z","shell.execute_reply.started":"2021-08-19T16:00:35.620209Z","shell.execute_reply":"2021-08-19T16:00:35.844228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE = 200\n'''This Function converts a color image to gray scale image'''\n\ndef conv_gray(img):\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    img = cv2.resize(img, (IMG_SIZE,IMG_SIZE))\n    return img\n  \n    \n'''\nThis Function shows the visual Image photo of 'n x 5' points (5 of each class)\n'''\n\ndef visualize_imgs(df,pts_per_class,color_scale):\n    df = df.groupby('diagnosis',group_keys = False).apply(lambda df: df.sample(pts_per_class))\n    df = df.reset_index(drop = True)\n    \n    plt.rcParams[\"axes.grid\"] = False\n    for pt in range(pts_per_class):\n        f, axarr = plt.subplots(1,5,figsize = (15,15))\n        axarr[0].set_ylabel(\"Sample Data Points\")\n        \n        df_temp = df[df.index.isin([pt + (pts_per_class*0),pt + (pts_per_class*1), pt + (pts_per_class*2),pt + (pts_per_class*3),pt + (pts_per_class*4)])]\n        for i in range(5):\n            if color_scale == 'gray':\n                img = conv_gray(cv2.imread(df_temp.file_path.iloc[i]))\n                axarr[i].imshow(img,cmap = color_scale)\n            else:\n                axarr[i].imshow(Image.open(df_temp.file_path.iloc[i]).resize((IMG_SIZE,IMG_SIZE)))\n            axarr[i].set_xlabel('Class '+str(df_temp.diagnosis.iloc[i]))\n\n        plt.show()\nvisualize_imgs(df_train,3,color_scale = None)","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:00:35.846465Z","iopub.execute_input":"2021-08-19T16:00:35.846756Z","iopub.status.idle":"2021-08-19T16:00:42.304488Z","shell.execute_reply.started":"2021-08-19T16:00:35.846726Z","shell.execute_reply":"2021-08-19T16:00:42.303219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nThis section of code applies gaussian blur on top of image\n'''\n\nrn = np.random.randint(low = 0,high = len(df_train) - 1)\n\nimg = cv2.imread(df_train.file_path.iloc[rn])\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nimg = cv2.resize(img, (IMG_SIZE,IMG_SIZE))\n\nimg_t = cv2.addWeighted(img,4, cv2.GaussianBlur(img , (0,0) , 30) ,-4 ,128)\n\nf, axarr = plt.subplots(1,2,figsize = (11,11))\naxarr[0].imshow(img)\naxarr[1].imshow(img_t)\nplt.title('After applying Gaussian Blur')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:00:42.306321Z","iopub.execute_input":"2021-08-19T16:00:42.306739Z","iopub.status.idle":"2021-08-19T16:00:42.972684Z","shell.execute_reply.started":"2021-08-19T16:00:42.306693Z","shell.execute_reply":"2021-08-19T16:00:42.968688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nThis Function performs image processing on top of images by performing Gaussian Blur and Circle Crop\n'''\n\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img\n    \n    \ndef circle_crop(img, sigmaX):   \n    \"\"\"\n    Create circular crop around image centre    \n    \"\"\"    \n    img = crop_image_from_gray(img)    \n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    height, width, depth = img.shape    \n    \n    x = int(width/2)\n    y = int(height/2)\n    r = np.amin((x,y))\n    \n    circle_img = np.zeros((height, width), np.uint8)\n    cv2.circle(circle_img, (x,y), int(r), 1, thickness=-1)\n    img = cv2.bitwise_and(img, img, mask=circle_img)\n    img = crop_image_from_gray(img)\n    img=cv2.addWeighted(img,4, cv2.GaussianBlur( img , (0,0) , sigmaX) ,-4 ,128)\n    return img","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:00:42.974049Z","iopub.execute_input":"2021-08-19T16:00:42.974364Z","iopub.status.idle":"2021-08-19T16:00:42.989656Z","shell.execute_reply.started":"2021-08-19T16:00:42.974331Z","shell.execute_reply":"2021-08-19T16:00:42.988423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rn = np.random.randint(low = 0,high = len(df_train) - 1)\n\n#img = img_t\nimg = cv2.imread(df_train.file_path.iloc[rn])\nimg_t = circle_crop(img,sigmaX = 30)\n\nf, axarr = plt.subplots(1,2,figsize = (11,11))\naxarr[0].imshow(cv2.resize(cv2.cvtColor(img, cv2.COLOR_BGR2RGB),(IMG_SIZE,IMG_SIZE)))\naxarr[1].imshow(img_t)\nplt.title('After applying Circular Crop and Gaussian Blur')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:00:42.992543Z","iopub.execute_input":"2021-08-19T16:00:42.992858Z","iopub.status.idle":"2021-08-19T16:00:44.545890Z","shell.execute_reply.started":"2021-08-19T16:00:42.992828Z","shell.execute_reply":"2021-08-19T16:00:44.544823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_img_process(df,pts_per_class,sigmaX):\n    df = df.groupby('diagnosis',group_keys = False).apply(lambda df: df.sample(pts_per_class))\n    df = df.reset_index(drop = True)\n    \n    plt.rcParams[\"axes.grid\"] = False\n    for pt in range(pts_per_class):\n        f, axarr = plt.subplots(1,5,figsize = (15,15))\n        axarr[0].set_ylabel(\"Sample Data Points\")\n        \n        df_temp = df[df.index.isin([pt + (pts_per_class*0),pt + (pts_per_class*1), pt + (pts_per_class*2),pt + (pts_per_class*3),pt + (pts_per_class*4)])]\n        for i in range(5):\n            img = cv2.imread(df_temp.file_path.iloc[i])\n            img = circle_crop(img,sigmaX)\n            axarr[i].imshow(img)\n            axarr[i].set_xlabel('Class '+str(df_temp.diagnosis.iloc[i]))\n\n        plt.show()\nvisualize_img_process(df_train,3,sigmaX = 30)","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:00:44.548244Z","iopub.execute_input":"2021-08-19T16:00:44.548815Z","iopub.status.idle":"2021-08-19T16:01:02.189917Z","shell.execute_reply.started":"2021-08-19T16:00:44.548772Z","shell.execute_reply":"2021-08-19T16:01:02.188815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_train,df_train_valid = train_test_split(df_train,test_size = 0.2)\nprint(df_train_train.shape,df_train_valid.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:01:02.191317Z","iopub.execute_input":"2021-08-19T16:01:02.191908Z","iopub.status.idle":"2021-08-19T16:01:02.201621Z","shell.execute_reply.started":"2021-08-19T16:01:02.191863Z","shell.execute_reply":"2021-08-19T16:01:02.200443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_train.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:01:02.202984Z","iopub.execute_input":"2021-08-19T16:01:02.203328Z","iopub.status.idle":"2021-08-19T16:01:02.223940Z","shell.execute_reply.started":"2021-08-19T16:01:02.203296Z","shell.execute_reply":"2021-08-19T16:01:02.222465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_valid.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:01:02.225310Z","iopub.execute_input":"2021-08-19T16:01:02.225824Z","iopub.status.idle":"2021-08-19T16:01:02.242587Z","shell.execute_reply.started":"2021-08-19T16:01:02.225777Z","shell.execute_reply":"2021-08-19T16:01:02.241718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_test = df_train_valid\nprint(df_train_train.shape,df_train_valid.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:01:02.244164Z","iopub.execute_input":"2021-08-19T16:01:02.244464Z","iopub.status.idle":"2021-08-19T16:01:02.257213Z","shell.execute_reply.started":"2021-08-19T16:01:02.244434Z","shell.execute_reply":"2021-08-19T16:01:02.255988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train_test = df_train_valid\nfile = open('/kaggle/working/df_train_train', 'wb')\npickle.dump(df_train_train, file)\nfile.close()\n\nfile = open('/kaggle/working/df_train_test', 'wb')\npickle.dump(df_train_test, file)\nfile.close()","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:01:02.258576Z","iopub.execute_input":"2021-08-19T16:01:02.259109Z","iopub.status.idle":"2021-08-19T16:01:02.274634Z","shell.execute_reply.started":"2021-08-19T16:01:02.259054Z","shell.execute_reply":"2021-08-19T16:01:02.273442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = open('/kaggle/working/df_train_train', 'rb')\ndf_train_train = pickle.load(file)\nfile.close()\n\nfile = open('/kaggle/working/df_train_test', 'rb')\ndf_train_test = pickle.load(file)\nfile.close()","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:01:02.276343Z","iopub.execute_input":"2021-08-19T16:01:02.276710Z","iopub.status.idle":"2021-08-19T16:01:02.289430Z","shell.execute_reply.started":"2021-08-19T16:01:02.276675Z","shell.execute_reply":"2021-08-19T16:01:02.288157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train_train.shape,df_train_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:01:02.291085Z","iopub.execute_input":"2021-08-19T16:01:02.291536Z","iopub.status.idle":"2021-08-19T16:01:02.298201Z","shell.execute_reply.started":"2021-08-19T16:01:02.291491Z","shell.execute_reply":"2021-08-19T16:01:02.297229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE  = 512","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:01:02.299544Z","iopub.execute_input":"2021-08-19T16:01:02.299858Z","iopub.status.idle":"2021-08-19T16:01:02.311939Z","shell.execute_reply.started":"2021-08-19T16:01:02.299830Z","shell.execute_reply":"2021-08-19T16:01:02.310952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('/kaggle/working/train_images_resized_preprocessed')\nos.mkdir('/kaggle/working/test_images_resized_preprocessed')","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:01:02.313593Z","iopub.execute_input":"2021-08-19T16:01:02.314225Z","iopub.status.idle":"2021-08-19T16:01:02.325635Z","shell.execute_reply.started":"2021-08-19T16:01:02.314166Z","shell.execute_reply":"2021-08-19T16:01:02.324306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img\n\ndef circle_crop(img, sigmaX = 30):   \n    \"\"\"\n    Create circular crop around image centre    \n    \"\"\"    \n    img = crop_image_from_gray(img)    \n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    height, width, depth = img.shape    \n    \n    x = int(width/2)\n    y = int(height/2)\n    r = np.amin((x,y))\n    \n    circle_img = np.zeros((height, width), np.uint8)\n    cv2.circle(circle_img, (x,y), int(r), 1, thickness=-1)\n    img = cv2.bitwise_and(img, img, mask=circle_img)\n    img = crop_image_from_gray(img)\n    img=cv2.addWeighted(img,4, cv2.GaussianBlur( img , (0,0) , sigmaX) ,-4 ,128)\n    return img \n\ndef preprocess_image(file):\n    input_filepath = os.path.join('./','train_images/','{}.png'.format(file))\n    output_filepath = os.path.join('/kaggle/working/','train_images_resized_preprocessed/','{}.png'.format(file))\n    \n    img = cv2.imread(input_filepath)\n    img = circle_crop(img) \n    cv2.imwrite(output_filepath, cv2.resize(img, (IMG_SIZE,IMG_SIZE)))\n    \ndef preprocess_image1(file):\n    input_filepath = os.path.join('./','train_images/','{}.png'.format(file))\n    output_filepath = os.path.join('/kaggle/working/','test_images_resized_preprocessed/','{}.png'.format(file))\n    \n    img = cv2.imread(input_filepath)\n    img = circle_crop(img) \n    cv2.imwrite(output_filepath, cv2.resize(img, (IMG_SIZE,IMG_SIZE)))","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:01:02.327631Z","iopub.execute_input":"2021-08-19T16:01:02.328079Z","iopub.status.idle":"2021-08-19T16:01:02.349902Z","shell.execute_reply.started":"2021-08-19T16:01:02.327934Z","shell.execute_reply":"2021-08-19T16:01:02.348354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''This Function uses Multi processing for faster saving of images into folder'''\n\ndef multiprocess_image_processor(process:int, imgs:list):\n    \"\"\"\n    Inputs:\n        process: (int) number of process to run\n        imgs:(list) list of images\n    \"\"\"\n    print(f'MESSAGE: Running {process} process')\n    results = ThreadPool(process).map(preprocess_image, imgs)\n    return results\ndef multiprocess_image_processor1(process:int, imgs:list):\n    \"\"\"\n    Inputs:\n        process: (int) number of process to run\n        imgs:(list) list of images\n    \"\"\"\n    print(f'MESSAGE: Running {process} process')\n    results = ThreadPool(process).map(preprocess_image1, imgs)\n#     return results","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:01:02.351512Z","iopub.execute_input":"2021-08-19T16:01:02.351976Z","iopub.status.idle":"2021-08-19T16:01:02.372446Z","shell.execute_reply.started":"2021-08-19T16:01:02.351941Z","shell.execute_reply":"2021-08-19T16:01:02.370856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use 6 cores\nmultiprocess_image_processor(4, list(df_train_train.id_code.values))\nmultiprocess_image_processor1(4, list(df_train_test.id_code.values))","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:01:02.374217Z","iopub.execute_input":"2021-08-19T16:01:02.374707Z","iopub.status.idle":"2021-08-19T16:37:04.366375Z","shell.execute_reply.started":"2021-08-19T16:01:02.374654Z","shell.execute_reply":"2021-08-19T16:37:04.364240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FACTOR = 4\nBATCH_SIZE = 8 * FACTOR\nIMG_SIZE = 512\nEPOCHS = 20\nWARMUP_EPOCHS = 5\nLEARNING_RATE = 1e-4 * FACTOR\nWARMUP_LEARNING_RATE = 1e-3 * FACTOR\nHEIGHT = 320\nWIDTH = 320\nCANAL = 3\nN_CLASSES = df_train_train['diagnosis'].nunique()\nES_PATIENCE = 5\nRLROP_PATIENCE = 3\nDECAY_DROP = 0.5\n\nLR_WARMUP_EPOCHS_1st = 2\nLR_WARMUP_EPOCHS_2nd = 5\nSTEP_SIZE = len(df_train_train) // BATCH_SIZE\nTOTAL_STEPS_1st = WARMUP_EPOCHS * STEP_SIZE\nTOTAL_STEPS_2nd = EPOCHS * STEP_SIZE\nWARMUP_STEPS_1st = LR_WARMUP_EPOCHS_1st * STEP_SIZE\nWARMUP_STEPS_2nd = LR_WARMUP_EPOCHS_2nd * STEP_SIZE","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:37:04.369545Z","iopub.execute_input":"2021-08-19T16:37:04.370186Z","iopub.status.idle":"2021-08-19T16:37:04.385697Z","shell.execute_reply.started":"2021-08-19T16:37:04.370101Z","shell.execute_reply":"2021-08-19T16:37:04.384728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def multiple_outputs(generator,dataframe, image_dir, batch_size, height,width, subset):\n    gen = generator.flow_from_dataframe(\n        dataframe = dataframe,\n        x_col = \"file_name\",\n        y_col = \"diagnosis\",\n        directory = image_dir,\n        target_size=(height, width),\n        batch_size=batch_size,\n        class_mode='categorical',\n        subset=subset)\n    \n    mlb = MultiLabelBinarizer(classes = range(N_CLASSES))\n    \n    while True:\n        gnext = gen.next()\n        yield gnext[0], [np.argmax(gnext[1],axis = -1),gnext[1],mlb.fit_transform([list(range(x+1)) for x in np.argmax(gnext[1],axis = -1)])]","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:37:04.387118Z","iopub.execute_input":"2021-08-19T16:37:04.387695Z","iopub.status.idle":"2021-08-19T16:37:04.415703Z","shell.execute_reply.started":"2021-08-19T16:37:04.387638Z","shell.execute_reply":"2021-08-19T16:37:04.414355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datagen=ImageDataGenerator(rescale=1./255, rotation_range=360,brightness_range=[0.5, 1.5],\n                                     zoom_range=[1, 1.2],zca_whitening=True,horizontal_flip=True,\n                                     vertical_flip=True,fill_mode='constant',cval=0.,validation_split = 0.0)\n\ntrain_generator = multiple_outputs(generator = train_datagen,dataframe = df_train_train,\n                                   image_dir=\"/kaggle/working/train_images_resized_preprocessed/\",\n                                   batch_size=BATCH_SIZE,height = HEIGHT,width = WIDTH,\n                                   subset='training')\n     \nvalid_generator = multiple_outputs(generator = train_datagen,dataframe = df_train_test,\n                                   image_dir=\"/kaggle/working/test_images_resized_preprocessed/\",\n                                   batch_size=BATCH_SIZE,height = HEIGHT,width = WIDTH,\n                                   subset='validation')","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:37:04.417604Z","iopub.execute_input":"2021-08-19T16:37:04.418013Z","iopub.status.idle":"2021-08-19T16:37:04.444077Z","shell.execute_reply.started":"2021-08-19T16:37:04.417973Z","shell.execute_reply":"2021-08-19T16:37:04.441627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_tensor = Input(shape=(HEIGHT, WIDTH, CANAL))\nbase_model = applications.ResNet50(weights=None, include_top=False,input_tensor=input_tensor)\nbase_model.load_weights('/kaggle/input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')\n\n\nx1 = GlobalAveragePooling2D()(base_model.output)\nx1 = BatchNormalization()(x1)\n\nx2 = GlobalMaxPooling2D()(base_model.output)\nx2 = BatchNormalization()(x2)\n\nx = concatenate([x1,x2])\n\n# Regression Head\nxr = Dense(2048, activation='relu')(x)\nxr = Dropout(0.5)(xr)\nxr = Dense(1,activation = 'linear',name = 'regression_output')(xr)\n\n# Classification Head\nxc = Dense(2048, activation='relu')(x)\nxc = Dropout(0.5)(xc)\nxc = Dense(N_CLASSES,activation = 'softmax',name = 'classification_output')(xc)\n\n# Ordinal Regression Head\nxo = Dense(2048, activation='relu')(x)\nxo = Dropout(0.5)(xo)\nxo = Dense(N_CLASSES,activation = 'softmax',name = 'ordinal_regression_output')(xo)\n\nmodel = Model(inputs = [input_tensor], outputs = [xr,xc,xo])","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:37:04.445597Z","iopub.execute_input":"2021-08-19T16:37:04.446615Z","iopub.status.idle":"2021-08-19T16:37:10.367620Z","shell.execute_reply.started":"2021-08-19T16:37:04.446548Z","shell.execute_reply":"2021-08-19T16:37:10.365923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for layer in model.layers:\n    layer.trainable = True\n# model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:37:10.372996Z","iopub.execute_input":"2021-08-19T16:37:10.374122Z","iopub.status.idle":"2021-08-19T16:37:10.388367Z","shell.execute_reply.started":"2021-08-19T16:37:10.374062Z","shell.execute_reply":"2021-08-19T16:37:10.386336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"STEP_SIZE_TRAIN = len(df_train_train)//BATCH_SIZE\nSTEP_SIZE_VALID = len(df_train_test)//BATCH_SIZE\nprint(STEP_SIZE_TRAIN,STEP_SIZE_VALID)","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:37:10.390864Z","iopub.execute_input":"2021-08-19T16:37:10.391529Z","iopub.status.idle":"2021-08-19T16:37:10.406558Z","shell.execute_reply.started":"2021-08-19T16:37:10.391425Z","shell.execute_reply":"2021-08-19T16:37:10.404867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cosine_decay_with_warmup(global_step,\n                             learning_rate_base,\n                             total_steps,\n                             warmup_learning_rate=0.0,\n                             warmup_steps=0,\n                             hold_base_rate_steps=0):\n    \"\"\"\n    Cosine decay schedule with warm up period.\n    In this schedule, the learning rate grows linearly from warmup_learning_rate\n    to learning_rate_base for warmup_steps, then transitions to a cosine decay\n    schedule.\n    :param global_step {int}: global step.\n    :param learning_rate_base {float}: base learning rate.\n    :param total_steps {int}: total number of training steps.\n    :param warmup_learning_rate {float}: initial learning rate for warm up. (default: {0.0}).\n    :param warmup_steps {int}: number of warmup steps. (default: {0}).\n    :param hold_base_rate_steps {int}: Optional number of steps to hold base learning rate before decaying. (default: {0}).\n    :param global_step {int}: global step.\n    :Returns : a float representing learning rate.\n    :Raises ValueError: if warmup_learning_rate is larger than learning_rate_base, or if warmup_steps is larger than total_steps.\n    \"\"\"\n\n    if total_steps < warmup_steps:\n        raise ValueError('total_steps must be larger or equal to warmup_steps.')\n    learning_rate = 0.5 * learning_rate_base * (1 + np.cos(\n        np.pi *\n        (global_step - warmup_steps - hold_base_rate_steps\n         ) / float(total_steps - warmup_steps - hold_base_rate_steps)))\n    if hold_base_rate_steps > 0:\n        learning_rate = np.where(global_step > warmup_steps + hold_base_rate_steps,\n                                 learning_rate, learning_rate_base)\n    if warmup_steps > 0:\n        if learning_rate_base < warmup_learning_rate:\n            raise ValueError('learning_rate_base must be larger or equal to warmup_learning_rate.')\n        slope = (learning_rate_base - warmup_learning_rate) / warmup_steps\n        warmup_rate = slope * global_step + warmup_learning_rate\n        learning_rate = np.where(global_step < warmup_steps, warmup_rate,\n                                 learning_rate)\n    return np.where(global_step > total_steps, 0.0, learning_rate)\n\n\nclass WarmUpCosineDecayScheduler(Callback):\n    \"\"\"Cosine decay with warmup learning rate scheduler\"\"\"\n\n    def __init__(self,\n                 learning_rate_base,\n                 total_steps,\n                 global_step_init=0,\n                 warmup_learning_rate=0.0,\n                 warmup_steps=0,\n                 hold_base_rate_steps=0,\n                 verbose=0):\n        \"\"\"\n        Constructor for cosine decay with warmup learning rate scheduler.\n        :param learning_rate_base {float}: base learning rate.\n        :param total_steps {int}: total number of training steps.\n        :param global_step_init {int}: initial global step, e.g. from previous checkpoint.\n        :param warmup_learning_rate {float}: initial learning rate for warm up. (default: {0.0}).\n        :param warmup_steps {int}: number of warmup steps. (default: {0}).\n        :param hold_base_rate_steps {int}: Optional number of steps to hold base learning rate before decaying. (default: {0}).\n        :param verbose {int}: quiet, 1: update messages. (default: {0}).\n        \"\"\"\n\n        super(WarmUpCosineDecayScheduler, self).__init__()\n        self.learning_rate_base = learning_rate_base\n        self.total_steps = total_steps\n        self.global_step = global_step_init\n        self.warmup_learning_rate = warmup_learning_rate\n        self.warmup_steps = warmup_steps\n        self.hold_base_rate_steps = hold_base_rate_steps\n        self.verbose = verbose\n        self.learning_rates = []\n\n    def on_batch_end(self, batch, logs=None):\n        self.global_step = self.global_step + 1\n        lr = K.get_value(self.model.optimizer.lr)\n        self.learning_rates.append(lr)\n\n    def on_batch_begin(self, batch, logs=None):\n        lr = cosine_decay_with_warmup(global_step=self.global_step,\n                                      learning_rate_base=self.learning_rate_base,\n                                      total_steps=self.total_steps,\n                                      warmup_learning_rate=self.warmup_learning_rate,\n                                      warmup_steps=self.warmup_steps,\n                                      hold_base_rate_steps=self.hold_base_rate_steps)\n        K.set_value(self.model.optimizer.lr, lr)\n        if self.verbose > 0:\n            print('\\nBatch %02d: setting learning rate to %s.' % (self.global_step + 1, lr))","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:37:10.408882Z","iopub.execute_input":"2021-08-19T16:37:10.410255Z","iopub.status.idle":"2021-08-19T16:37:10.433868Z","shell.execute_reply.started":"2021-08-19T16:37:10.410186Z","shell.execute_reply":"2021-08-19T16:37:10.432005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cosine_lr = WarmUpCosineDecayScheduler(learning_rate_base = LEARNING_RATE,\n                                       total_steps=TOTAL_STEPS_1st,\n                                       warmup_learning_rate=0.0,\n                                       warmup_steps=TOTAL_STEPS_1st,\n                                       hold_base_rate_steps=(2 * STEP_SIZE))\n\ncallback_list = [cosine_lr]","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:37:10.436302Z","iopub.execute_input":"2021-08-19T16:37:10.437290Z","iopub.status.idle":"2021-08-19T16:37:10.456424Z","shell.execute_reply.started":"2021-08-19T16:37:10.437114Z","shell.execute_reply":"2021-08-19T16:37:10.453947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer = optimizers.SGD(lr=LEARNING_RATE),\n              loss={'regression_output': 'mean_absolute_error', \n                    'classification_output': 'categorical_crossentropy',\n                    'ordinal_regression_output' : 'binary_crossentropy'\n                    },\n              metrics = ['accuracy'])\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:37:10.458523Z","iopub.execute_input":"2021-08-19T16:37:10.459009Z","iopub.status.idle":"2021-08-19T16:37:10.504305Z","shell.execute_reply.started":"2021-08-19T16:37:10.458973Z","shell.execute_reply":"2021-08-19T16:37:10.503216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history = model.fit_generator(generator=train_generator,\n#                               steps_per_epoch=STEP_SIZE_TRAIN,\n#                               validation_data=valid_generator,\n#                               validation_steps=STEP_SIZE_VALID,\n#                               epochs=20,\n#                               callbacks = callback_list,\n#                               verbose=1)\n\n# model.save(\"model_pre_training.h5\")\n# f = open(\"history_pre_training\",\"wb\")\n# pickle.dump(history,f)\n# f.close()","metadata":{"execution":{"iopub.status.busy":"2021-08-19T16:37:10.505981Z","iopub.execute_input":"2021-08-19T16:37:10.506848Z","iopub.status.idle":"2021-08-19T16:37:10.513059Z","shell.execute_reply.started":"2021-08-19T16:37:10.506791Z","shell.execute_reply":"2021-08-19T16:37:10.512165Z"},"trusted":true},"execution_count":null,"outputs":[]}]}