{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import json\nimport math\nimport os\n\nimport cv2\nfrom PIL import Image\nimport numpy as np\nfrom keras import layers\nfrom keras.applications import DenseNet121\nfrom keras.callbacks import Callback, ModelCheckpoint\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.optimizers import Adam, SGD\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score, confusion_matrix\nimport scipy\nfrom tqdm import tqdm\nimport sys\n%matplotlib inline\n\nprint(os.listdir(\"../input\"))\n\n#sys.path.append(os.path.abspath('../input/efficientnet/'))\n#import efficientnet.keras as efn\n\nsys.path.append(os.path.abspath(\"../input/efficientnet/efficientnet/\")) #kaggle sheet\nimport efficientnet.keras as efn #kaggle sheet\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import seaborn as sns\n\n#IMSIZE = 224 #for Densenet121 / EfficientNetB0\n#IMSIZE = 240 #for EfficientNetB1\n#IMSIZE = 260 #for EfficientNetB2\nIMSIZE = 300 #for EfficientNetB3\n#IMSIZE = 380 #for EfficientNetB4\n\naptos2019_path = \"../input/aptos2019-blindness-detection/\"\naptos2019_train_path = \"../input/aptos2019-blindness-detection/train_images/\"\naptos2019_test_path = \"../input/aptos2019-blindness-detection/test_images/\"\naptos2019_extension = \".png\"\n\n#aptos2015_path = \"../input/aptos2015-blindness-detection/\"\n#aptos2015_train_path = \"../input/aptos2015-blindness-detection/train_images/\"\n#aptos2015_test_path = \"../input/aptos2015-blindness-detection/test_images/\"\naptos2015_path = \"../input/aptos2015blindnessdetection/\" #kaggles heet\naptos2015_train_path = \"../input/aptos2015blindnessdetection/train_images/\" #kaggles heet\naptos2015_test_path = \"../input/aptos2015blindnessdetection/test_images/\" #kaggles heet\naptos2015_extension = \".jpg\"\n\ndef plot_diag_hist(dataframe, title='NoTitle'):\n    f, ax = plt.subplots(figsize=(7, 4))\n    ax = sns.countplot(x=\"diagnosis\", data=dataframe, palette=\"GnBu_d\")\n    sns.despine()\n    plt.title(title)\n    plt.show()\n\ntrainbase_2019_df = pd.read_csv(aptos2019_path+\"train.csv\")\ntest_2019_df = pd.read_csv(aptos2019_path+\"test.csv\")\nprint(\"Shape of 2019 Training Data: {}\".format(trainbase_2019_df.shape))\nprint(\"Shape of 2019 Test Data: {}\\n\".format(test_2019_df.shape))\n#print(\"Appearance of 2019 Dataframe:\\n{}\\n\".format(trainbase_2019_df.head()))\n#print(\"Diagnosis Distribution:\\n{}\\n\".format(trainbase_2019_df['diagnosis'].value_counts()/trainbase_2019_df.shape[0]))\n\n\ntrainbase_2015_df = pd.read_csv(aptos2015_path+\"train.csv\")\ntest_2015_df = pd.read_csv(aptos2015_path+\"test.csv\")\nprint(\"Shape of 2015 Training Data: {}\".format(trainbase_2015_df.shape))\nprint(\"Shape of 2015 Test Data: {}\\n\".format(test_2015_df.shape))\ntrainbase_2015_df.rename(columns={\"image\": \"id_code\", \"level\": \"diagnosis\"}, inplace=True)\ntest_2015_df.rename(columns={\"image\": \"id_code\", \"level\": \"diagnosis\"}, inplace=True)\ntest_2015_df.drop(columns=[\"Usage\"], inplace=True)\n\nplot_diag_hist(trainbase_2019_df, title=\"Predictions of 2019 Training Data\")\nplot_diag_hist(trainbase_2015_df, title=\"Predictions of 2015 Training Data\")\n#plot_diag_hist(test_2015_df, title=\"Predictions of 2015 Test Data\")\n\nfull_path_train_2019 = lambda id_code : aptos2019_train_path+id_code+aptos2019_extension\nfull_path_test_2019 = lambda id_code : aptos2019_test_path+id_code+aptos2019_extension\nfull_path_train_2015 = lambda id_code : aptos2015_train_path+id_code+aptos2015_extension\nfull_path_test_2015 = lambda id_code : aptos2015_test_path+id_code+aptos2015_extension\n\ntrainbase_2019_df[\"path\"] = trainbase_2019_df[\"id_code\"].apply(full_path_train_2019)\ntest_2019_df[\"path\"] = test_2019_df[\"id_code\"].apply(full_path_test_2019)\ntrainbase_2015_df[\"path\"] = trainbase_2015_df[\"id_code\"].apply(full_path_train_2015)\ntest_2015_df[\"path\"] = test_2015_df[\"id_code\"].apply(full_path_test_2015)\n\n#trainbase_2019_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import imagehash\nimport psutil\nfrom joblib import Parallel, delayed\n\n\ndef getImageMetaData(file_path):\n    with Image.open(file_path) as img:\n        img_hash = imagehash.phash(img)\n        return img.size, img.mode, str(img_hash), file_path\n\n\ndef getContradictoryImages(datframe):   \n    #paralle processing (faster but fills memory)\n    #img_meta_l = Parallel(n_jobs=psutil.cpu_count(), verbose=1)(\n    #    (delayed(getImageMetaData)(fp) for fp in df_all[\"path\"]))\n\n    #serial processing\n    img_meta_l = []\n    for i in tqdm(range(datframe.shape[0])):\n        img_meta_l.append(getImageMetaData(datframe.iloc[i][\"path\"]))\n\n    img_meta_df = pd.DataFrame(np.array(img_meta_l))\n    img_meta_df.columns = ['Size', 'Mode', 'Hash', 'path']\n\n    datframe = datframe.merge(img_meta_df, on='path', how='left')\n    df_hashgroup = datframe.groupby('Hash').count().reset_index()\n    df_hashgroup_dup = df_hashgroup.query('path > 1')\n    dup_hash_list = df_hashgroup_dup['Hash'].values\n    df_dup = datframe.loc[datframe['Hash'].isin(dup_hash_list)].sort_values('Hash')\n    print(\"{} duplicates found\".format(df_dup.shape))\n    df_dup[\"diagnosis_mean\"] = np.zeros(df_dup.shape[0])\n    for everyhash in dup_hash_list:\n        meanvalue = np.mean(df_dup[df_dup[\"Hash\"] == everyhash][\"diagnosis\"].values)\n        df_dup.loc[df_dup[\"Hash\"] == everyhash, [\"diagnosis_mean\"]] = meanvalue\n    df_dup[\"diagnosis_mean\"] = df_dup[\"diagnosis_mean\"] - df_dup[\"diagnosis\"]\n    df_dup = df_dup[df_dup[\"diagnosis_mean\"] != 0]\n    print(\"{} contradictory duplicates found\".format(df_dup.shape))\n    contradictory_ids = df_dup[\"id_code\"]\n    return contradictory_ids\n\n\n#contradictory_2019 = getContradictoryImages(trainbase_2019_df)\ncontradictory_2019 = ['1632c4311fc9', 'a75bab2463d4', '8273fdb4405e', 'f0098e9d4aee',\n 'f066db7a2efe', '278aa860dffd', '8ef2eb8c51c4', '8446826853d0',\n '9a3c03a5ad0f', 'f03d3c4ce7fb', 'b91ef82e723a', '2df07eb5779f',\n '42a850acd2ac', '51131b48f9d4', '8cb6b0efaaac', '0cb14014117d',\n '4a44cc840ebe', 'f7edc074f06b', '9bf060db8376', '4fecf87184e6',\n 'e4151feb8443', '46cdc8b685bd', 'd0079cc188e9', '3dbfbc11e105',\n '16ce555748d8', '4d9fc85a8259', 'bacfb1029f6b', 'e12d41e7b221',\n '6165081b9021', '42985aa2e32f', '521d3e264d71', 'fe0fc67c7980',\n 'e8d1c6c07cf2', 'f23902998c21', 'b9127e38d9b9', 'e39b627cf648',\n '33778d136069', '4ccfa0b4e96c', '1c9c583c10bf', 'ea15a290eb96',\n '7525ebb3434d', '3cd801ffdbf0', 'c546670d9684', '30cab14951ac',\n 'fcc6aa6755e6', '772af553b8b7', '60f15dd68d30', '0243404e8a00',\n '3ddb86eb530e', '7005be54cab1', '3ee4841936ef', '2f284b6a1940',\n 'bb5083fae98f', '35aa7f5c2ec0', '1c4d87baaffc', '98f7136d2e7a',\n 'e740af6ac6ea', 'f0f89314e860', '14e3f84445f7', '2f7789c1e046',\n 'a8e88d4891c4', 'a9e984b57556', '5eb311bcb5f9', '9e3510963315',\n 'b187b3c93afb', 'a1b12fdce6c3', '5e7db41b3bee', 'ab50123abadb',\n '80964d8e0863', 'fda39982a810', '0ac436400db4', '8688f3d0fcaf',\n 'd85ea1220a03', 'bfefa7344e7d', '6253f23229b1', '76cfe8967f7d',\n 'd035c2bd9104', 'cd93a472e5cd']\n\n\n\n#contradictory_2015 = getContradictoryImages(trainbase_2015_df)\ncontradictory_2015 = ['39670_left', '20116_left', '21488_right', '1178_right',\n       '38749_right', '18706_right', '20050_left', '24369_left',\n       '6200_left', '11496_left', '17616_left', '14481_left',\n       '28732_left', '33706_left', '26434_left', '10646_left',\n       '22491_left', '2642_right', '16458_right', '30552_right',\n       '10734_right', '20187_right', '29023_right', '25064_left',\n       '11437_left', '24820_left', '27344_left', '19549_left',\n       '12525_left', '12003_right', '35788_right', '16523_right',\n       '43803_right', '19071_right', '3026_right', '29246_right',\n       '21760_right', '32216_right', '39105_right', '7638_right',\n       '997_right', '15254_left', '40674_left', '26111_left',\n       '11003_left', '13971_left', '31363_right', '16991_right',\n       '42433_right', '24754_right', '15453_right', '35726_right',\n       '7271_right', '18476_right', '29986_right', '38481_right',\n       '43711_right', '43047_right', '21571_right', '7646_right',\n       '16013_right', '24033_right', '20101_right', '16759_right',\n       '8187_right', '2379_right', '24318_right', '31729_right',\n       '37213_right', '4326_right', '25945_right', '43167_right',\n       '13012_right', '19339_right', '10104_right', '17757_right',\n       '29312_right', '7250_right', '42873_right', '31849_right',\n       '1267_right', '11980_right', '18688_right', '26312_right',\n       '21432_right', '1029_right', '23718_right', '12614_right',\n       '41134_right', '32112_left', '2229_left', '21077_right',\n       '17181_right', '37730_left', '9899_left', '13938_left',\n       '32493_left', '12596_left', '20635_left', '32388_left',\n       '34701_left', '33173_left', '18304_left', '40704_left',\n       '39778_left', '17230_left', '32633_left', '42621_left',\n       '42405_left', '15732_right', '18059_right', '41311_left',\n       '2500_left', '7179_left', '10808_left', '40198_left', '2010_left',\n       '16609_left', '39389_left', '16785_left', '17373_left',\n       '2205_left', '33148_left', '9798_left', '27498_left', '10551_left',\n       '22118_left', '13264_left', '29740_left', '4676_left', '3059_left',\n       '28992_left', '32206_left', '35090_left', '24089_left',\n       '40827_left', '2132_left', '18724_left', '34718_left',\n       '40058_left', '7315_left', '1607_right', '31709_right',\n       '30174_right', '35667_right', '30925_left', '32246_left',\n       '41957_left', '28351_left', '5010_left', '11378_left',\n       '23531_left', '32807_left', '26383_right', '8889_right',\n       '8866_right', '21518_right', '20540_right', '13971_right',\n       '27136_right', '8695_right', '7048_right', '20023_left',\n       '17629_left', '10003_left', '41514_left', '18950_left',\n       '19261_left', '19158_left', '18760_left', '10669_left',\n       '23146_left', '23787_left', '37532_left', '15520_left',\n       '8621_left', '11375_left', '40001_left', '13811_left',\n       '16807_left', '41131_left', '2094_left', '38215_left',\n       '26966_left', '28010_left', '13716_left', '34379_left',\n       '38324_left', '34780_left', '2812_left', '17739_left',\n       '15811_left', '41080_left', '18235_left', '39978_left',\n       '37372_left', '8675_left', '14621_left', '11930_left',\n       '41012_left', '36021_left', '32098_left', '1988_left', '3464_left',\n       '3040_left', '9384_left', '783_left', '18945_left', '8284_left',\n       '14987_left', '31000_left', '3754_left', '26051_left',\n       '36566_left', '41327_left', '31981_left', '2851_left',\n       '22994_left', '42922_left', '4285_left', '11811_left', '9216_left',\n       '39863_left', '12208_left', '14160_left', '11409_left',\n       '5267_left', '1639_left', '28629_left', '23369_right', '551_right',\n       '43440_left', '27440_left', '24828_left', '29894_left',\n       '37488_left', '26043_left', '37042_left']\n\n\n\nprint(\"Shape of 2019 Training Data: {}\".format(trainbase_2019_df.shape))\ntrainbase_2019_df = trainbase_2019_df[-trainbase_2019_df[\"id_code\"].isin(contradictory_2019)]\nprint(\"Shape of 2019 Training Data after removal of bad data: {}\".format(trainbase_2019_df.shape))\nprint(\"Shape of 2015 Training Data: {}\".format(trainbase_2015_df.shape))\ntrainbase_2015_df = trainbase_2015_df[-trainbase_2015_df[\"id_code\"].isin(contradictory_2015)]\nprint(\"Shape of 2015 Training Data after removal of bad data: {}\".format(trainbase_2015_df.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#keep all images from classes > 0 and 5000 images from class 0\ntrainbase_2015_df_reviewed = trainbase_2015_df[trainbase_2015_df[\"diagnosis\"]>0]\ntemp = trainbase_2015_df[trainbase_2015_df[\"diagnosis\"]==0].sample(5000, replace=True)\ntrainbase_2015_df_reviewed = pd.concat([trainbase_2015_df_reviewed, temp])\ntrainbase_2015_df = trainbase_2015_df_reviewed\nplot_diag_hist(trainbase_2015_df, title=\"Predictions of Reviewed 2015 Training Data\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_raw_df, validation_df = train_test_split(trainbase_2019_df, test_size=0.2, random_state=42, \n                                               stratify=trainbase_2019_df[['diagnosis']])\nprint(\"Shape of Training Data: {}\".format(train_raw_df.shape))\nprint(\"Diagnosis Distribution in Training Set:\\n{}\\n\".format(train_raw_df['diagnosis'].value_counts()/train_raw_df.shape[0]))\nprint(\"Shape of Validation Data: {}\".format(validation_df.shape))\nprint(\"Diagnosis Distribution in Validation Set:\\n{}\\n\".format(validation_df['diagnosis'].value_counts()/validation_df.shape[0]))\n\ntrain_raw_2015_df, validation_2015_df = train_test_split(trainbase_2015_df, test_size=0.2, random_state=42, \n                                               stratify=trainbase_2015_df[['diagnosis']])\nprint(\"Shape of additional 2015 Training Data: {}\".format(train_raw_2015_df.shape))\nprint(\"Diagnosis Distribution in 2015 Training Set:\\n{}\\n\".format(train_raw_2015_df['diagnosis'].value_counts()/train_raw_2015_df.shape[0]))\nprint(\"Shape of additional 2015 Validation Data: {}\".format(validation_2015_df.shape))\nprint(\"Diagnosis Distribution in 2015 Validation Set:\\n{}\\n\".format(validation_2015_df['diagnosis'].value_counts()/validation_2015_df.shape[0]))\n\n\n# add all images with classes > 0 from 2015 dataset\n#train_df = pd.concat([train_raw_df, train_raw_2015_df[train_raw_2015_df[\"diagnosis\"]>0]], axis=0) \n# align all class sizes to biggest class\n#max_size = train_df['diagnosis'].value_counts().max()\n\n#lst = [train_df]\n#class_size = [1.0, 1.0, 1.0, 1.0, 1.0]\n\n#for class_index, group in train_df.groupby('diagnosis'):\n#    print(\"Class_index: {}, Is: {}, Should be: {}\".format(class_index, group.shape[0], max_size*class_size[class_index]))\n#    if (class_index == 0):\n#        pass\n#        #add data from 2015 set\n#        lst.append(train_raw_2015_df[train_raw_2015_df[\"diagnosis\"]==0].sample((max_size*class_size[class_index]).astype(int)-len(group), replace=True))\n#    else:\n#        pass\n#        #duplicate data from mixed 2015 & 2019 set\n#        #lst.append(group.sample((max_size*class_size[class_index]).astype(int)-len(group), replace=True))\n#train_df = pd.concat(lst)\n#train_df = train_raw_df\n#train_2015_df = train_raw_2015_df # select only classes > 0\n#validation_df = pd.concat([validation_df, validation_2015_df], axis=0) \n\ntrain_df = train_raw_df\ntrain_2015_df = train_raw_2015_df\n\nprint(\"Shape of Oversampled Training Data: {}\".format(train_df.shape))\nprint(\"Shape of Merged Validation Data: {}\".format(validation_df.shape))\nprint(\"Appearance of Oversampled Training Data:\\n{}\\n\".format(train_df.head()))\nprint(\"Diagnosis Distribution in Oversampled Training Set:\\n{}\\n\".format(train_df['diagnosis'].value_counts()/train_df.shape[0]))\nplot_diag_hist(train_df, title=\"Predictions of Stratified and Oversampled Training Data\")\n\ntrain_df = train_df.sample(frac=1, random_state=42).reset_index(drop=True)\nvalidation_df = validation_df.sample(frac=1, random_state=42).reset_index(drop=True)\ntrain_2015_df = train_2015_df.sample(frac=1, random_state=42).reset_index(drop=True)\nvalidation_2015_df = validation_2015_df.sample(frac=1, random_state=42).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations\nimport time\nimport cv2\n\n\ndef show_image(image, figsize=None, title=None):\n    if figsize is not None:\n        fig = plt.figure(figsize=figsize)\n    if image.ndim == 2:\n        plt.imshow(image,cmap='gray')\n    else:\n        plt.imshow(image)\n    if title is not None:\n        plt.title(title)\n        \ndef show_Nimages(imgs,scale=1):\n    N=len(imgs)\n    fig = plt.figure(figsize=(25/scale, 16/scale))\n    for i, img in enumerate(imgs):\n        ax = fig.add_subplot(1, N, i + 1, xticks=[], yticks=[])\n        show_image(img)\n\n\ndef augment_aptos(aug, img):\n    return aug(image=img)['image']\n\n\nalbumentations\n\naug_rotate = albumentations.Rotate(p=1, limit=180, interpolation=cv2.INTER_LANCZOS4, \n                    border_mode=cv2.BORDER_CONSTANT,value=0) # value=black\naug_flip = albumentations.Flip(p=0.5)\naug_bright = albumentations.RandomBrightnessContrast(brightness_limit=0.45, contrast_limit=0.45,p=1)\nh_min=np.round(IMSIZE*0.8).astype(int)\nh_max= np.round(IMSIZE*1.0).astype(int)\naug_crop = albumentations.RandomSizedCrop((h_min, h_max),IMSIZE,IMSIZE, w2h_ratio=1,p=1)\nmax_hole_size = int(IMSIZE/10)\naug_hole = albumentations.Cutout(p=1, max_h_size=max_hole_size, max_w_size=max_hole_size, num_holes=8 )\naug_flare = albumentations.RandomSunFlare(src_radius=max_hole_size, num_flare_circles_lower=10, num_flare_circles_upper=20, p=1)\n\nfull_augmentation = albumentations.Compose([aug_rotate, aug_flip, aug_bright, aug_crop, aug_hole, aug_flare],p=1)\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image_preprocess(image_path, black_threshold=20, imsize=IMSIZE, augmentation=False, sub_gauss=True, sub_median=False, zoom_boxed=True):\n    img = cv2.imread(image_path)\n    #print(\"Path: {}\".format(image_path))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    crop_black = True #if false, image is just made quadratic\n    \n    #get mask from blurred image to find background  assuminig background is black below \"black_threshold\"\n    gray_img = cv2.medianBlur(cv2.cvtColor(img, cv2.COLOR_RGB2GRAY), 5)\n    mask = gray_img>black_threshold\n        \n    if (crop_black):\n        if (not(mask.any(axis=0).any())): #image to dark, nothing found\n            centerx = int(img.shape[0]//2)\n            centery = int(img.shape[1]//2)\n            maxdistance = max(img.shape)//2\n            border_u_d = int(np.round(maxdistance-centerx))\n            border_l_r = int(np.round(maxdistance-centery))\n            #print(\"border: ({}/{})\".format(border_u_d, border_l_r))\n            #print(\"Old shape: {}\".format(imnew.shape))\n            imnew = cv2.copyMakeBorder(img,border_u_d, border_u_d, border_l_r, border_l_r, borderType=cv2.BORDER_CONSTANT, value=0)\n            #print(\"New shape: {}\".format(imnew.shape))\n        else:\n            #get most left/right/upper/lower point that is not background\n            xvect = np.where(mask.any(axis=0))[0]\n            yvect = np.where(mask.any(axis=1))[0]\n            subx = (np.max(xvect) - np.min(xvect)) % 2 #assure even number of pixels\n            suby = (np.max(yvect) - np.min(yvect)) % 2 \n            #print(\"Modulos: {} / {}\".format(subx, suby))\n            #print(\"{} {} {} {}\".format(np.min(yvect), np.max(yvect), np.min(xvect), np.max(xvect)))\n            #crop to image content without background\n            imnew = img[np.min(yvect):(np.max(yvect)-suby),np.min(xvect):(np.max(xvect)-subx)]\n            maskknew = mask[np.min(yvect):(np.max(yvect)-suby),np.min(xvect):(np.max(xvect)-subx)]\n            \n            centerx = int(imnew.shape[0]/2)\n            centery = int(imnew.shape[1]/2)\n            #print(\"center: ({}/{})\".format(centerx, centery))\n            distance_border = 0\n            pointlist = []\n            distances = []\n\n            coords = np.where(maskknew[distance_border,:])\n            if (coords[0].shape[0] > 0):\n                pointlist.append((distance_border, np.min(coords)))\n                pointlist.append((distance_border, np.max(coords)))\n            coords = np.where(maskknew[imnew.shape[0]-distance_border-1,:])\n            if (coords[0].shape[0] > 0):\n                pointlist.append((imnew.shape[0]-distance_border-1, np.min(coords)))\n                pointlist.append((imnew.shape[0]-distance_border-1, np.max(coords)))\n            coords = np.where(maskknew[:,distance_border])\n            if (coords[0].shape[0] > 0):\n                pointlist.append((np.min(coords), distance_border))\n                pointlist.append((np.max(coords), distance_border))\n            coords = np.where(maskknew[:,imnew.shape[1]-distance_border-1])\n            if (coords[0].shape[0] > 0):\n                pointlist.append((np.min(coords), imnew.shape[1]-distance_border-1))\n                pointlist.append((np.max(coords), imnew.shape[1]-distance_border-1))\n            for xp, yp in pointlist:\n                distances.append(((xp-centerx)*(xp-centerx)+(yp-centery)*(yp-centery)))\n            maxdistance = int(np.round(np.sqrt(max(distances)))) # retina image radius\n            #print(\"Radius: {}, Diameter: {}\".format(maxdistance, maxdistance*2))\n            border_u_d = int(np.round(maxdistance-centerx))\n            border_l_r = int(np.round(maxdistance-centery))\n            #print(\"border: ({}/{})\".format(border_u_d, border_l_r))\n            #print(\"Old shape: {}\".format(imnew.shape))\n            imnew = cv2.copyMakeBorder(imnew,border_u_d, border_u_d, border_l_r, border_l_r, borderType=cv2.BORDER_CONSTANT, value=0)\n            #print(\"New shape: {}\".format(imnew.shape))\n    else:\n        centerx = int(img.shape[0]//2)\n        centery = int(img.shape[1]//2)\n        maxdistance = max(img.shape)//2\n        border_u_d = int(np.round(maxdistance-centerx))\n        border_l_r = int(np.round(maxdistance-centery))\n        #print(\"border: ({}/{})\".format(border_u_d, border_l_r))\n        #print(\"Old shape: {}\".format(imnew.shape))\n        imnew = cv2.copyMakeBorder(img,border_u_d, border_u_d, border_l_r, border_l_r, borderType=cv2.BORDER_CONSTANT, value=0)\n        #print(\"New shape: {}\".format(imnew.shape))\n\n    \n    \n    \n    scaling = imsize/imnew.shape[0]\n    rotation = 0\n    translate_x = 0\n    translate_y = 0\n    \n    if zoom_boxed:\n        scaling *= 1.4\n    M = cv2.getRotationMatrix2D((maxdistance,maxdistance), rotation, scaling)\n    M[0,2] -= (maxdistance - imsize/2 + translate_x)\n    M[1,2] -= (maxdistance - imsize/2 + translate_y)\n    imnew = cv2.warpAffine(imnew, M, (imsize,imsize))\n    \n    if sub_gauss:\n        k = (imsize//40)*2+1\n        bg = cv2.GaussianBlur(imnew ,(0,0) ,k)\n        imnew = cv2.addWeighted (imnew, 4, bg, -4, 128)\n\n    if sub_median:\n        k = (imsize//40)*2+1\n        bg = cv2.medianBlur(imnew, k)\n        imnew = cv2.addWeighted (imnew, 4, bg, -4, 128)\n    \n    if augmentation:\n        #Flip H/V\n        #if (np.random.rand() > 0.5): #flip h\n        #    imnew = cv2.flip(imnew, 0)\n        #if (np.random.rand() > 0.5): #flip v\n        #    imnew = cv2.flip(imnew, 1)\n        #brightness\n        #imnew = np.clip((imnew * float(0.75 + np.random.rand()*0.5)),0,255).astype(\"uint8\") #brightness\n        #add blur randomly\n        #do_blur = np.random.rand()\n        #if (do_blur>0.8):\n        #    imnew = cv2.GaussianBlur(imnew ,(0,0) ,1)\n        #    #print(\"blurring\")\n        imnew = augment_aptos(full_augmentation,imnew)\n     \n    #imnew = (255-imnew) #invert image\n    #clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\n    #imnew[:,:,1] = clahe.apply(imnew[:,:,1])\n    #imnew[:,:,2] = clahe.apply(imnew[:,:,2])\n    #imnew[:,:,0] = clahe.apply(imnew[:,:,0])\n    \n    \n\n        \n    return imnew\n\n\ndef display_samples_preprocess_steps(df, columns=3, rows=2):\n    fig=plt.figure(figsize=(5*columns, 4*rows))\n    for i in range(rows):\n        image_path = df.loc[i,\"path\"]\n        image_id = df.loc[i,'diagnosis']\n        \n        img_0 = load_image_preprocess(image_path,\n                                     imsize=IMSIZE,\n                                     augmentation=False,\n                                     sub_gauss=False,\n                                     sub_median=False)\n        fig.add_subplot(rows, columns, i*columns + 1)\n        plt.title(image_id)\n        plt.imshow(img_0)\n        \n        img_1 = load_image_preprocess(image_path,\n                                     imsize=IMSIZE,\n                                     augmentation=False,\n                                     sub_gauss=True,\n                                     sub_median=False)\n        fig.add_subplot(rows, columns, i*columns + 2)\n        plt.title(image_id)\n        plt.imshow(img_1)\n        \n        img_2 = load_image_preprocess(image_path,\n                                     imsize=IMSIZE,\n                                     augmentation=False,\n                                     sub_gauss=False,\n                                     sub_median=True)\n        fig.add_subplot(rows, columns, i*columns + 3)\n        plt.title(image_id)\n        plt.imshow(img_2)\n    plt.tight_layout()\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imagepath = train_df.loc[4,\"path\"]\nimg = load_image_preprocess(imagepath, augmentation=False)\n#image_path, black_threshold=20, imsize=224, augmentation=False, sub_gauss=False, sub_median=False, zoom_boxed=False\n#plt.imshow(seq.augment_image(img))\nplt.imshow(img)\nprint(img.shape)\nprint((img[np.newaxis]).shape)\n\ndisplay_samples_preprocess_steps(train_df)\n\n\n#aug_rotate, aug_flip, aug_bright, aug_crop, aug_hole, aug_flare\nimg1 = augment_aptos(aug_rotate,img)\nimg2 = augment_aptos(aug_rotate,img)\nshow_Nimages([img,img1,img2],scale=2)\n\nimg1 = augment_aptos(aug_flip,img)\nimg2 = augment_aptos(aug_flip,img)\nimg3 = augment_aptos(aug_flip,img)\nshow_Nimages([img3,img1,img2],scale=2)\n\nimg1 = augment_aptos(aug_bright,img)\nimg2 = augment_aptos(aug_bright,img)\nimg3 = augment_aptos(aug_bright,img)\nshow_Nimages([img3,img1,img2],scale=2)\n\nimg1 = augment_aptos(aug_crop,img)\nimg2 = augment_aptos(aug_crop,img)\nimg3 = augment_aptos(aug_crop,img)\nshow_Nimages([img3,img1,img2],scale=2)\n\nimg1 = augment_aptos(aug_hole,img)\nimg2 = augment_aptos(aug_hole,img)\nimg3 = augment_aptos(aug_hole,img)\nshow_Nimages([img3,img1,img2],scale=2)\n\nimg1 = augment_aptos(aug_flare,img)\nimg2 = augment_aptos(aug_flare,img)\nimg3 = augment_aptos(aug_flare,img)\nshow_Nimages([img3,img1,img2],scale=2)\n\nimg1 = augment_aptos(full_augmentation,img)\nimg2 = augment_aptos(full_augmentation,img)\nimg3 = augment_aptos(full_augmentation,img)\nshow_Nimages([img3,img1,img2],scale=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IS_REGRESSION = False\n\ndef extract_target_vector_multi(dataframe_X):\n    y_targets = pd.get_dummies(dataframe_X['diagnosis']).values\n    y_targets_multi = np.empty(y_targets.shape, dtype=y_targets.dtype)\n    y_targets_multi[:, 4] = y_targets[:, 4]\n    for i in range(3, -1, -1):\n        y_targets_multi[:, i] = np.logical_or(y_targets[:, i], y_targets_multi[:, i+1])\n    #use only 4 cols\n    #y_train_multi = y_train_multi[:,1:]\n    return y_targets_multi\n\ndef extract_target_vector_regression(dataframe_X):\n    y_targets = dataframe_X['diagnosis'].values\n    return y_targets\n\n\ndef result_from_prediction_regression(y_pred):\n    return (np.clip(np.round(y_pred),0,4)).astype(\"uint\").flatten()\n\ndef result_from_prediction_multi(y_pred):\n    return (y_pred > 0.5).astype(int).sum(axis=1) - 1\n\n\n#datasets are\n# trainbase_2019_df / y_trainbase_2019_multi\n# train_df / y_train_multi\n# validation_df / y_validate_multi\n# test_2019_df\n#\n# trainbase_2015_df / y_trainbase_2015_multi\n# train_2015_df\n# validation_2015_df\n# test_2015_df / y_test_2015_multi\n\nif (IS_REGRESSION):\n    y_train_multi = extract_target_vector_regression(train_df)\n    y_validate_multi = extract_target_vector_regression(validation_df)\n    y_trainbase_2019_multi = extract_target_vector_regression(trainbase_2019_df)\n    y_trainbase_2015_multi = extract_target_vector_regression(trainbase_2015_df)\n    y_test_2015_multi = extract_target_vector_regression(test_2015_df)\n    result_from_prediction = result_from_prediction_regression\nelse:\n    y_train_multi = extract_target_vector_multi(train_df)\n    y_validate_multi = extract_target_vector_multi(validation_df)\n    y_trainbase_2019_multi = extract_target_vector_multi(trainbase_2019_df)\n    y_trainbase_2015_multi = extract_target_vector_multi(trainbase_2015_df)\n    y_test_2015_multi = extract_target_vector_multi(test_2015_df)\n    result_from_prediction = result_from_prediction_multi\n\n\nprint(\"Training data:\\nXshape: {}\\nyshape: {}\\n\".format(train_df.shape,y_train_multi.shape))\nprint(\"Validation data:\\nXshape: {}\\nyshape: {}\\n\".format(validation_df.shape,y_validate_multi.shape))\nprint(\"Test data:\\nXshape: {}\\n\".format(test_2019_df.shape))\nprint(\"Training data multilabel diagnosis vector:\", y_train_multi.sum(axis=0))\nprint(\"Validation data multilabel diagnosis vector:\", y_validate_multi.sum(axis=0))\nprint(\"Training data 2015 multilabel diagnosis vector:\", y_trainbase_2015_multi.sum(axis=0))\nprint(\"Training data 2015 multilabel diagnosis vector:\", y_test_2015_multi.sum(axis=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.utils import Sequence\nfrom sklearn.utils import shuffle\n\nBATCH_SIZE = 32\n\nclass My_Image_Generator(Sequence):\n\n    def __init__(self, image_filenames, labels, batch_size, shuffle=True, augment=False, train=False):\n        self.image_filenames, self.labels = image_filenames, labels\n        self.is_train = train\n        self.batch_size = batch_size\n        self.is_shuffle = shuffle\n        self.is_augment = augment\n        if(self.is_shuffle):\n            self.on_epoch_end()\n            \n\n    def __len__(self):\n        return int(np.ceil(len(self.image_filenames) / float(self.batch_size)))\n\n    def __getitem__(self, idx):\n        batch_x = self.image_filenames[idx * self.batch_size:(idx + 1) * self.batch_size]\n        if (self.is_train):\n            batch_y = self.labels[idx * self.batch_size:(idx + 1) * self.batch_size]\n        else:\n            batch_y = np.zeros(self.batch_size)\n        return self.batch_generate(batch_x, batch_y)\n\n    def on_epoch_end(self):\n        if(self.is_shuffle):\n            self.image_filenames, self.labels = shuffle(self.image_filenames, self.labels)\n        else:\n            pass\n\n    def batch_generate(self, batch_x, batch_y):\n        batch_images = []\n        for (sample, label) in zip(batch_x, batch_y):\n            if(self.is_augment):\n                img = load_image_preprocess(sample, augmentation=True)\n            else:\n                img = load_image_preprocess(sample, augmentation=False)\n            batch_images.append(img)\n        batch_images = np.array(batch_images, np.float32) / 255\n        batch_y = np.array(batch_y, np.float32)\n        #print(batch_images.shape)\n        #print(batch_y.shape)\n        return batch_images, batch_y\n    \n\n#datasets are\n# train_df / y_train_multi / train_data_generator_aug / train_data_generator_no_aug\n# validation_df / y_validate_multi / valid_data_generator_aug / valid_data_generator_no_aug\n# trainbase_2019_df / y_trainbase_2019_multi / trainbase_2019_data_generator_aug / trainbase_2019_data_generator_no_aug\n# test_2019_df / test_data_generator_aug / test_data_generator_no_aug\n#\n# trainbase_2015_df / y_trainbase_2015_multi / trainbase_2015_data_generator_aug / trainbase_2015_data_generator_no_aug\n# train_2015_df\n# validation_2015_df\n# test_2015_df / y_test_2015_multi / test_data_2015_generator_no_aug\n    \n    \ntrain_data_generator_aug = My_Image_Generator(train_df[\"path\"], labels=y_train_multi, shuffle=True, train=True,\n                                              batch_size=BATCH_SIZE, augment=True)\ntrain_data_generator_no_aug = My_Image_Generator(train_df[\"path\"], labels=y_train_multi, shuffle=False, train=True,\n                                                 batch_size=BATCH_SIZE, augment=False)\nvalid_data_generator_aug = My_Image_Generator(validation_df[\"path\"], y_validate_multi, shuffle=False, train=True,\n                                              batch_size=BATCH_SIZE, augment=True)\nvalid_data_generator_no_aug = My_Image_Generator(validation_df[\"path\"], y_validate_multi, shuffle=False, train=True,\n                                                 batch_size=BATCH_SIZE, augment=False)\n\n\ntrainbase_2019_data_generator_aug = My_Image_Generator(trainbase_2019_df[\"path\"], labels=y_trainbase_2019_multi, \n                                                       shuffle=True, train=True, batch_size=BATCH_SIZE, augment=True)\ntrainbase_2019_data_generator_no_aug = My_Image_Generator(trainbase_2019_df[\"path\"], labels=y_trainbase_2019_multi, \n                                                          shuffle=False, train=True, batch_size=BATCH_SIZE, augment=False)\ntrainbase_2015_data_generator_aug = My_Image_Generator(trainbase_2015_df[\"path\"], labels=y_trainbase_2015_multi, \n                                                       shuffle=True, train=True, batch_size=BATCH_SIZE, augment=True)\ntrainbase_2015_data_generator_no_aug = My_Image_Generator(trainbase_2015_df[\"path\"], labels=y_trainbase_2015_multi, \n                                                          shuffle=False, train=True, batch_size=BATCH_SIZE, augment=False)\n\n\ntest_data_generator_aug = My_Image_Generator(test_2019_df[\"path\"], labels=None, shuffle=False, train=False,\n                                             batch_size=BATCH_SIZE, augment=True)\ntest_data_generator_no_aug = My_Image_Generator(test_2019_df[\"path\"], labels=None, shuffle=False, train=False,\n                                                batch_size=BATCH_SIZE, augment=False)\ntest_data_2015_generator_no_aug = My_Image_Generator(test_2015_df[\"path\"], labels=y_test_2015_multi, shuffle=False, train=True,\n                                                      batch_size=BATCH_SIZE, augment=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_image(image, figsize=None, title=None):\n    if figsize is not None:\n        fig = plt.figure(figsize=figsize)\n    if image.ndim == 2:\n        plt.imshow(image,cmap='gray')\n    else:\n        plt.imshow(image)\n    if title is not None:\n        plt.title(title)\n\ndef display_generator_images(generator):\n    for images, targets in generator:\n        N = images.shape[0]\n        fig = plt.figure(figsize=(25, 16))\n        for i, img in enumerate(images):\n            ax = fig.add_subplot(1, N, i + 1, xticks=[], yticks=[])\n            show_image(img)\n        break\n\ndisplay_generator_images(train_data_generator_aug)\ndisplay_generator_images(train_data_generator_no_aug)\ndisplay_generator_images(valid_data_generator_aug)\ndisplay_generator_images(valid_data_generator_no_aug)\ndisplay_generator_images(trainbase_2019_data_generator_aug)\ndisplay_generator_images(trainbase_2019_data_generator_no_aug)\ndisplay_generator_images(trainbase_2015_data_generator_aug)\ndisplay_generator_images(trainbase_2015_data_generator_no_aug)\ndisplay_generator_images(test_data_generator_aug)\ndisplay_generator_images(test_data_generator_no_aug)\ndisplay_generator_images(test_data_2015_generator_no_aug)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ReduceLROnPlateau, EarlyStopping\n\nclass QWK_Metrics(Callback):\n    def __init__(self, model, validation_generator_X, validation_y):\n        self.model_to_save = model\n        self.valid_generator = validation_generator_X\n        self.valid_y = validation_y\n    def on_train_begin(self, logs={}):\n        self.val_kappas = [] #init qwk values array\n    def on_epoch_end(self, epoch, logs={}):\n        y_val_pred = model.predict_generator(generator=self.valid_generator, \n                                             steps=-(self.valid_y.shape[0] // -BATCH_SIZE), \n                                             verbose=1)\n        y_val_pred = result_from_prediction(y_val_pred)\n        y_val_true = result_from_prediction(self.valid_y)\n        _val_kappa = cohen_kappa_score(y_val_pred, y_val_true, weights='quadratic')\n        self.val_kappas.append(_val_kappa)\n        print(f\"val_kappa: {_val_kappa:.4f}\")   \n        #if best value, overwrite and save as new best model\n        if _val_kappa == max(self.val_kappas):\n            print(\"Validation Kappa has improved. Saving model.\")\n            self.model_to_save.save('model_kappa.h5')\n        return\n  \n    \nreduceLR = ReduceLROnPlateau(\n    monitor='val_loss',\n    factor=0.1,\n    patience=5,\n    min_lr=1e-6,\n    verbose=1,\n    mode='min'\n)\n\nearlyStopping = EarlyStopping(\n    monitor='val_loss',\n    patience=10,\n    verbose=1,\n    mode='min',\n    restore_best_weights=True\n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.utils import multi_gpu_model\nfrom tensorflow.python.client import device_lib\nimport tensorflow as tf\n\ndef get_available_gpus():\n    local_device_protos = device_lib.list_local_devices()\n    return [x.name for x in local_device_protos if x.device_type == \"GPU\"]\nnum_gpu = len(get_available_gpus())\nprint(\"Number of available GPUs: {}\".format(num_gpu))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers.convolutional import Conv2D, MaxPooling2D\nfrom keras.constraints import maxnorm\nfrom keras.layers import Dropout\nfrom keras.layers import Flatten\nfrom keras.layers import Dense\nfrom keras import regularizers, optimizers\nimport tensorflow as tf\nfrom keras.utils import multi_gpu_model\nfrom keras.optimizers import Adam, SGD\n\n#used_init = \"he_uniform\"\n#used_init = \"lecun_normal\"\n#used_init = \"golrot_uniform\" #standard\n#used_init = \"golrot_normal\"\nused_init = \"lecun_uniform\"\n\nreg_value_kernel = 5e-5\nreg_value_bias = 5e-5\n\ndef build_model_plainCNN():\n    # create model\n    model = Sequential()\n    model.add(Conv2D(32, (5, 5), input_shape=[224,224,3], activation='relu', kernel_initializer=used_init, \n                     kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias))) # --> 220\n    #model.add(GaussianDropout(0.3))\n    model.add(MaxPooling2D(pool_size=(2, 2))) # --> 110\n    model.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer=used_init, \n                     kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias))) # --> 108\n    model.add(MaxPooling2D(pool_size=(2, 2))) # -->  54\n    model.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer=used_init, \n                     kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias))) # --> 52\n    model.add(MaxPooling2D(pool_size=(2, 2))) # --> 26\n    model.add(Conv2D(256, (3, 3), activation='relu', kernel_initializer=used_init, \n                     kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias))) # --> 24\n    model.add(MaxPooling2D(pool_size=(2, 2))) # --> 12\n    model.add(Conv2D(512, (3, 3), activation='relu', kernel_initializer=used_init, \n                     kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias))) #--> 10\n    model.add(MaxPooling2D(pool_size=(2, 2))) # --> 5\n    model.add(Dropout(0.5))\n    model.add(Flatten())\n    model.add(Dense(512, activation='relu', kernel_initializer=used_init, \n                    kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias)))\n    model.add(Dropout(0.5))\n    model.add(Dense(5, activation='sigmoid', kernel_initializer=used_init))\n    \n    #model.compile(loss='binary_crossentropy', optimizer=optimizers.Adam(lr=1e-3), metrics=['accuracy'])\n    \n    return model\n\n\ndef build_model_flat():\n    # create model\n    model = Sequential()\n    model.add(Conv2D(32, (9, 9), input_shape=[224,224,3], activation='relu', kernel_initializer=used_init)) # --> 216\n    #model.add(GaussianDropout(0.3))\n    model.add(MaxPooling2D(pool_size=(4, 4))) # --> 54\n    model.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer=used_init)) # --> 52\n    model.add(MaxPooling2D(pool_size=(4, 4))) # -->  13\n    model.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer=used_init)) # --> 11\n    model.add(MaxPooling2D(pool_size=(4, 4))) # -->  13\n    model.add(Dropout(0.2))\n    model.add(Flatten())\n    model.add(Dense(256, activation='relu', kernel_initializer=used_init))\n    model.add(Dropout(0.2))\n    model.add(Dense(5, activation='sigmoid', kernel_initializer=used_init))\n    \n    #model.compile(loss='binary_crossentropy', optimizer=optimizers.Adam(lr=1e-4), metrics=['accuracy'])\n    \n    return model\n\n\n\ndef build_model_densenet121():\n    densenet = DenseNet121(weights='../input/densenetpretrained/DenseNet-BC-121-32-no-top.h5', include_top=False, \n                           input_shape=(IMSIZE,IMSIZE,3))\n    ly_gap = layers.GlobalAveragePooling2D()\n    ly_dropout_1 = layers.Dropout(0.5)\n    ly_dense_1 = layers.Dense(256,activation=\"relu\")\n    ly_dropout_2 = layers.Dropout(0.5)\n    ly_dense_2 = layers.Dense(5, activation='sigmoid')\n    model = Sequential()\n    model.add(densenet)\n    model.add(ly_gap)\n    model.add(ly_dropout_1)\n    model.add(ly_dense_1)\n    model.add(ly_dropout_2)\n    model.add(ly_dense_2)\n    return model\n\n\ndef build_model_efficientnetB0():\n    effnet = efn.EfficientNetB0(weights=None,\n                           include_top=False,\n                           input_shape=(IMSIZE, IMSIZE, 3))\n    effnet.load_weights(\"../input/efficientnetpretrained/efficientnet-b0_imagenet_1000_notop.h5\")\n    ly_gap = layers.GlobalAveragePooling2D()\n    ly_dropout_1 = layers.Dropout(0.5)\n    ly_dense_1 = layers.Dense(256,activation=\"relu\")\n    ly_dropout_2 = layers.Dropout(0.5)\n    ly_dense_2 = layers.Dense(5, activation='sigmoid')\n    model = Sequential()\n    model.add(effnet)\n    model.add(ly_gap)\n    model.add(ly_dropout_1)\n    model.add(ly_dense_1)\n    model.add(ly_dropout_2)\n    model.add(ly_dense_2)\n    return model\n    \n\ndef build_model_efficientnetB1(): # Batch Size 16 on Quadro P2000\n    effnet = efn.EfficientNetB1(weights=None,\n                           include_top=False,\n                           input_shape=(IMSIZE, IMSIZE, 3))\n    effnet.load_weights(\"../input/efficientnetpretrained/efficientnet-b1_imagenet_1000_notop.h5\")\n    ly_gap = layers.GlobalAveragePooling2D()\n    ly_dropout_1 = layers.Dropout(0.5)\n    ly_dense_1 = layers.Dense(256,activation=\"relu\")\n    ly_dropout_2 = layers.Dropout(0.5)\n    ly_dense_2 = layers.Dense(5, activation='sigmoid')\n    model = Sequential()\n    model.add(effnet)\n    model.add(ly_gap)\n    model.add(ly_dropout_1)\n    model.add(ly_dense_1)\n    model.add(ly_dropout_2)\n    model.add(ly_dense_2)\n    return model\n     \ndef build_model_efficientnetB2(): #requires Batch Size 8 on Quadro P2000\n    effnet = efn.EfficientNetB2(weights=None,\n                           include_top=False,\n                           input_shape=(IMSIZE, IMSIZE, 3))\n    effnet.load_weights(\"../input/efficientnet/pretrainedweights/efficientnet-b2_imagenet_1000_notop.h5\")\n    ly_gap = layers.GlobalAveragePooling2D()\n    ly_dropout_1 = layers.Dropout(0.5)\n    ly_dense_1 = layers.Dense(256,activation=\"relu\")\n    ly_dropout_2 = layers.Dropout(0.5)\n    ly_dense_2 = layers.Dense(5, activation='sigmoid')\n    model = Sequential()\n    model.add(effnet)\n    model.add(ly_gap)\n    model.add(ly_dropout_1)\n    model.add(ly_dense_1)\n    model.add(ly_dropout_2)\n    model.add(ly_dense_2)\n    return model\n\ndef build_model_efficientnetB3(): #requires Batch Size 8 on Quadro P2000\n    effnet = efn.EfficientNetB3(weights=None,\n                           include_top=False,\n                           input_shape=(IMSIZE, IMSIZE, 3))\n    effnet.load_weights(\"../input/efficientnet/pretrainedweights/pretrainedweights/efficientnet-b3_imagenet_1000_notop.h5\")\n    ly_gap = layers.GlobalAveragePooling2D()\n    ly_dropout_1 = layers.Dropout(0.5)\n    ly_dense_1 = layers.Dense(256,activation=\"relu\")\n    ly_dropout_2 = layers.Dropout(0.5)\n    if (IS_REGRESSION):\n        ly_dense_2 = layers.Dense(1, activation='linear')\n    else:\n        ly_dense_2 = layers.Dense(5, activation='sigmoid')\n    model = Sequential()\n    model.add(effnet)\n    model.add(ly_gap)\n    model.add(ly_dropout_1)\n    model.add(ly_dense_1)\n    model.add(ly_dropout_2)\n    model.add(ly_dense_2)\n    return model\n\ndef build_model_efficientnetB4(): #requires Batch Size ? on Quadro P2000\n    effnet = efn.EfficientNetB4(weights=None,\n                           include_top=False,\n                           input_shape=(IMSIZE, IMSIZE, 3))\n    effnet.load_weights(\"../input/efficientnet/pretrainedweights/efficientnet-b4_imagenet_1000_notop.h5\")\n    ly_gap = layers.GlobalAveragePooling2D()\n    ly_dropout_1 = layers.Dropout(0.5)\n    ly_dense_1 = layers.Dense(256,activation=\"relu\")\n    ly_dropout_2 = layers.Dropout(0.5)\n    ly_dense_2 = layers.Dense(5, activation='sigmoid')\n    model = Sequential()\n    model.add(effnet)\n    model.add(ly_gap)\n    model.add(ly_dropout_1)\n    model.add(ly_dense_1)\n    model.add(ly_dropout_2)\n    model.add(ly_dense_2)\n    return model\n\n\nif (num_gpu == 0):\n    #base_model = build_model_flat()\n    #base_model = build_model_plainCNN()\n    #base_model = build_model_densenet121()\n    #base_model = build_model_efficientnetB0()\n    #base_model = build_model_efficientnetB2()\n    base_model = build_model_efficientnetB3()\n    #base_model = build_model_efficientnetB4()\n    model = base_model\n    print(\"Training on single CPU... no GPU avaliable\")\nelif (num_gpu == 1):\n    #base_model = build_model_flat()\n    #base_model = build_model_plainCNN()\n    #base_model = build_model_densenet121()\n    #base_model = build_model_efficientnetB0()\n    #base_model = build_model_efficientnetB2()\n    base_model = build_model_efficientnetB3()\n    #base_model = build_model_efficientnetB4()\n    model = base_model\n    print(\"Training on single GPU...\")\nelse:\n    with tf.device('/cpu:0'):\n        #base_model = build_model_flat()\n        #base_model = build_model_plainCNN()\n        #base_model = build_model_densenet121()\n        #base_model = build_model_efficientnetB0()\n        #base_model = build_model_efficientnetB2()\n        base_model = build_model_efficientnetB3()\n        #base_model = build_model_efficientnetB4()\n    try:\n        model = multi_gpu_model(base_model, gpus=4)\n        print(\"Training on multiple GPUs..\")\n    except ValueError:\n        model = base_model\n        print(\"Training on single GPU or CPU... error when compiling multi_gpu_model...\")\n\n\n#my_optimizer = Adam(lr=1e-4, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\n\nif (IS_REGRESSION):\n    model.compile(loss='mse', optimizer=Adam(lr=5e-5), metrics=['accuracy'])\nelse:\n    model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-4), metrics=['accuracy'])    \n    \nprint(\"Model Summary:\")\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#datasets are\n# train_df / y_train_multi / train_data_generator_aug / train_data_generator_no_aug\n# validation_df / y_validate_multi / valid_data_generator_aug / valid_data_generator_no_aug\n# trainbase_2019_df / y_trainbase_2019_multi / trainbase_2019_data_generator_aug / trainbase_2019_data_generator_no_aug\n# test_2019_df / test_data_generator_aug / test_data_generator_no_aug\n#\n# trainbase_2015_df / y_trainbase_2015_multi / trainbase_2015_data_generator_aug / trainbase_2015_data_generator_no_aug\n# train_2015_df\n# validation_2015_df\n# test_2015_df / y_test_2015_multi / test_data_2015_generator_no_aug\n\nfor layer in model.layers:\n    layer.trainable = True\nfor layer in model.layers[:-5]:\n    layer.trainable = False\n\nif (IS_REGRESSION):\n    model.compile(loss='mse', optimizer=Adam(lr=1e-4), metrics=['accuracy'])\nelse:\n    model.compile(loss='binary_crossentropy', optimizer=Adam(lr=5e-4), metrics=['accuracy']) \n\nprint(\"Model Summary:\")\nmodel.summary()\n\n\n#train on old 2015 dataset, validate on new dataset until early stopping\nkappa_metrics_1 = QWK_Metrics(base_model, trainbase_2019_data_generator_no_aug, y_trainbase_2019_multi) #sets the \"not-parallel\" for parameter saving in callback function\n#train from directory\nhistory_1 = model.fit_generator(\n    generator=trainbase_2015_data_generator_aug,\n    steps_per_epoch=-(y_trainbase_2015_multi.shape[0] // -BATCH_SIZE),\n    #steps_per_epoch=1,\n    epochs=5,\n    validation_data=trainbase_2019_data_generator_no_aug,\n    validation_steps=-(y_trainbase_2019_multi.shape[0] // -BATCH_SIZE),\n    #validation_steps=1,\n    callbacks=[kappa_metrics_1, reduceLR, earlyStopping]\n)\n\nfor layer in model.layers:\n    layer.trainable = True\n\nif (IS_REGRESSION):\n    model.compile(loss='mse', optimizer=Adam(lr=5e-5), metrics=['accuracy'])\nelse:\n    model.compile(loss='binary_crossentropy', optimizer=Adam(lr=5e-5), metrics=['accuracy']) \n\nprint(\"Model Summary:\")\nmodel.summary()\n\nkappa_metrics_2 = QWK_Metrics(base_model, trainbase_2019_data_generator_no_aug, y_trainbase_2019_multi) #sets the \"not-parallel\" for parameter saving in callback function\n#train from directory\nhistory_2 = model.fit_generator(\n    generator=trainbase_2015_data_generator_aug,\n    steps_per_epoch=-(y_trainbase_2015_multi.shape[0] // -BATCH_SIZE),\n    #steps_per_epoch=1,\n    epochs=5,\n    validation_data=trainbase_2019_data_generator_no_aug,\n    validation_steps=-(y_trainbase_2019_multi.shape[0] // -BATCH_SIZE),\n    #validation_steps=1,\n    callbacks=[kappa_metrics_2, reduceLR, earlyStopping]\n)\n\nbase_model.save('model_phase_1.h5')\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if (IS_REGRESSION):\n    model.compile(loss='mse', optimizer=Adam(lr=1e-5), metrics=['accuracy'])\nelse:\n    model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-5), metrics=['accuracy'])\nprint(\"Model Summary:\")\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train on new 2019 dataset, validate on new dataset\nkappa_metrics_3 = QWK_Metrics(base_model, valid_data_generator_no_aug, y_validate_multi) #sets the \"not-parallel\" for parameter saving in callback function\n#train from directory\nhistory_3 = model.fit_generator(\n    generator=train_data_generator_aug,\n    steps_per_epoch=-(y_train_multi.shape[0] // -BATCH_SIZE),\n    #steps_per_epoch=1,\n    epochs=5,\n    validation_data=valid_data_generator_no_aug,\n    validation_steps=-(y_validate_multi.shape[0] // -BATCH_SIZE),\n    #validation_steps=1,\n    callbacks=[kappa_metrics_3, reduceLR, earlyStopping]\n)\n\nbase_model.save('model_phase_2.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def print_learn_results(history, kappa_metrics):\n    history_df = pd.DataFrame(history.history)\n    f = plt.figure(figsize=(15,5))\n    ax = f.add_subplot(131)\n    ax.plot(history_df[\"loss\"], label=\"loss\")\n    ax.plot(history_df[\"val_loss\"], label = \"val_loss\")\n    ax.legend()\n    #history_df[['loss', 'val_loss']].plot()\n    ax = f.add_subplot(132)\n    ax.plot(history_df[\"acc\"], label=\"acc\")\n    ax.plot(history_df[\"val_acc\"], label=\"val_acc\")\n    ax.legend()\n    #history_df[['acc', 'val_acc']].plot()\n    ax = f.add_subplot(133)\n    kappa_df = pd.DataFrame(kappa_metrics.val_kappas, columns=[\"kappa\"])\n    ax.plot(kappa_df[\"kappa\"], label=\"kappa\")\n    ax.legend()\n    \n\nprint_learn_results(history_1, kappa_metrics_1)\nprint_learn_results(history_2, kappa_metrics_2)\nprint_learn_results(history_3, kappa_metrics_3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kappa_metrics = kappa_metrics_3\nhistory_df = pd.DataFrame(history_3.history)\n\nprint(\"Max Kappa: {}\".format(max(kappa_metrics.val_kappas)))\nbest_epoch = np.argmax(kappa_metrics.val_kappas)+1\nprint(\"Max Kappa at epoch {}.\".format(best_epoch))\nprint(\"Epoch {}: Acc: {:.4f}, ValAcc: {:.4f}, Loss: {:.4f}, ValLoss: {:.4f}\".format(best_epoch, \n                                                                    history_df[\"acc\"][best_epoch-1], \n                                                                    history_df[\"val_acc\"][best_epoch-1], \n                                                                    history_df[\"loss\"][best_epoch-1], \n                                                                    history_df[\"val_loss\"][best_epoch-1]\n                                                                   ))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_model.load_weights('model_kappa.h5')\n#base_model.load_weights('model_phase_2.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tta_cycles = 5\n\n#test from disk\ny_test_tta = model.predict_generator(generator=test_data_generator_no_aug, \n                                       steps=-(test_2019_df.shape[0] // -BATCH_SIZE), \n                                       verbose=1)\n\n# TTA\n#for i in tqdm(range(tta_cycles)):\n#    y_test_tta += model.predict_generator(generator=test_data_generator_aug,\n#                                            steps=-(test_df.shape[0] // -BATCH_SIZE),\n#                                            verbose=1)\n\n#print(\"Test prediction differences: {}\".format(np.max(y_test_tta-y_test_tta_2)))  \n    \n    \n#y_test_tta /= tta_cycles+1\ny_test_tta = result_from_prediction(y_test_tta)\ntest_2019_df['diagnosis'] = y_test_tta\ntest_2019_df.drop([\"path\"], inplace=True, axis=1)\ntest_2019_df.to_csv(\"submission.csv\",index=False)\nprint(\"Submission complete!\")\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}