{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q efficientnet","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:02:59.89474Z","iopub.execute_input":"2022-01-30T23:02:59.895499Z","iopub.status.idle":"2022-01-30T23:03:08.800947Z","shell.execute_reply.started":"2022-01-30T23:02:59.895401Z","shell.execute_reply":"2022-01-30T23:03:08.799959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, random, re, math, time\nrandom.seed(a=42)\nfrom sklearn.model_selection import train_test_split\nimport numpy as np\nimport pandas as pd\nimport PIL\n\nfrom kaggle_datasets import KaggleDatasets\n\nimport tensorflow as tf\nfrom tensorflow import keras\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\nfrom tensorflow.keras.callbacks import EarlyStopping\nimport tensorflow_addons as tfa","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:08.803601Z","iopub.execute_input":"2022-01-30T23:03:08.804163Z","iopub.status.idle":"2022-01-30T23:03:13.950455Z","shell.execute_reply.started":"2022-01-30T23:03:08.804102Z","shell.execute_reply":"2022-01-30T23:03:13.949696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = \"TPU\"\n\nCFG = dict(\n    epochs            =  100,\n    net_count         =  1,\n    batch_size        =  32,\n    \n    read_size         = 256,\n    crop_size         = 256,\n    net_size          = 256,\n    \n    LR_START          =   0.000005,\n    LR_MAX            =   0.000020,\n    LR_MIN            =   0.000001,\n#     LR_START          =   0.000005,\n#     LR_MAX            =   0.0001,\n#     LR_MIN            =   0.0000001,\n    \n    LR_RAMPUP_EPOCHS  =   5,\n    LR_SUSTAIN_EPOCHS =   0,\n    LR_EXP_DECAY      =   0.8,\n    \n    rot               = 180.0,\n    shr               =   2.0,\n    hzoom             =   8.0,\n    wzoom             =   8.0,\n    hshift            =   8.0,\n    wshift            =   8.0,\n\n    optimizer         = 'adam',\n    label_smooth_fac  =   0.05,\n    \n    tta_steps         =  25    \n)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:13.951926Z","iopub.execute_input":"2022-01-30T23:03:13.952314Z","iopub.status.idle":"2022-01-30T23:03:13.959822Z","shell.execute_reply.started":"2022-01-30T23:03:13.95228Z","shell.execute_reply":"2022-01-30T23:03:13.95873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASEPATH = \"../input/siim-isic-melanoma-classification\"\ndf_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))\ndf_test  = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))\ndf_sub   = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))\n\nGCS_PATH    = KaggleDatasets().get_gcs_path('melanoma256x256')\nGCS_PATH_AB    = KaggleDatasets().get_gcs_path('melanoma-256x256-aug1')\n# GCS_PATH_2019 = KaggleDatasets().get_gcs_path('isic2019-256x256-trainb')\n\nfiles_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')))\n\nfiles_traina = tf.io.gfile.glob(GCS_PATH_AB + '/traina*.tfrec')\nfiles_trainb = tf.io.gfile.glob(GCS_PATH_AB + '/trainb*.tfrec')\n# files_trainb_2019 = tf.io.gfile.glob(GCS_PATH_2019 + '/trainb*.tfrec')\nfiles_trainba = np.array(files_trainb + files_traina)\n\n# files_train = np.array([\n#         'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb00-600.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina00-1700.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb01-600.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina01-1700.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb02-600.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina02-1700.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb03-600.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina03-1700.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb04-600.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina04-1700.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb05-600.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina05-1700.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb06-600.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina06-1700.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb07-600.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina07-1700.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb08-600.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina08-1700.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb09-600.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina09-1700.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb10-600.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina10-1700.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb11-600.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina11-1700.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb12-600.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina12-1700.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb13-600.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina13-1700.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb14-600.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina14-1700.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/trainb15-340.tfrec',\n#        'gs://kds-7312da59b24e8b74c87f3e6be7cc48cd8151207bfa1292ae6aebf66a/traina15-533.tfrec'])\n\n# files_train = np.array([\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina00-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n    \n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina01-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n    \n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina02-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n    \n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina03-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n    \n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina04-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n    \n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina05-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n    \n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina06-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n    \n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina07-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n    \n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina08-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n    \n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina09-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n    \n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina10-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n    \n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina11-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n    \n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina12-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n    \n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina13-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n    \n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina14-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n    \n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina15-533.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec'])\n\n# files_train = np.array([\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina00-1700.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb00-30.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina01-1700.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb01-30.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina02-1700.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb02-30.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina03-1700.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb03-30.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina04-1700.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb04-30.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina05-1700.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb05-30.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina06-1700.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb06-30.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina07-1700.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb07-30.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina08-1700.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb08-30.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina09-1700.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb09-30.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina10-1700.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb10-30.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina11-1700.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb11-30.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina12-1700.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb12-30.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina13-1700.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb13-30.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina14-1700.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb14-30.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/traina15-533.tfrec',\n#     'gs://kds-fbf1c04197b4200bfff643c7f29d308b07fe39bf3c967ce8db284b0e/trainb15-17.tfrec'])\n\n# files_train = np.array([\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina00-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb00-30.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb00-300.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina01-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb01-30.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb01-300.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina02-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb02-30.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb02-300.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina03-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb03-30.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb03-300.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina04-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb04-30.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb04-300.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina05-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb05-30.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb05-300.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina06-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb06-30.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb06-300.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina07-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb07-30.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb07-300.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina08-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb08-30.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb08-300.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina09-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb09-30.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb09-300.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina10-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb10-30.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb10-300.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina11-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb11-30.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb11-300.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina12-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb12-30.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb12-300.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina13-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb13-30.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb13-300.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina14-1700.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb14-30.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb14-300.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/traina15-533.tfrec',\n#     'gs://kds-86a2a8c03410870f3858ac2f18fbc35a46463063090ecc747c392225/trainb15-17.tfrec',\n#     'gs://kds-25d02fc31a2cb81b5f97f5fff7f3dd48013697f1d73d2e175d2866e6/trainbb15-22.tfrec'])\n\nfiles_test  = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:04:08.245974Z","iopub.execute_input":"2022-01-30T23:04:08.246578Z","iopub.status.idle":"2022-01-30T23:04:09.836263Z","shell.execute_reply.started":"2022-01-30T23:04:08.246543Z","shell.execute_reply":"2022-01-30T23:04:09.835496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files_traina","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:15.905036Z","iopub.execute_input":"2022-01-30T23:03:15.90527Z","iopub.status.idle":"2022-01-30T23:03:15.912015Z","shell.execute_reply.started":"2022-01-30T23:03:15.905244Z","shell.execute_reply":"2022-01-30T23:03:15.91135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"neg, pos = np.bincount(df_train['target'])\ntotal = neg + pos\nprint('Examples:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n    total, pos, 100 * pos / total))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:15.91346Z","iopub.execute_input":"2022-01-30T23:03:15.913954Z","iopub.status.idle":"2022-01-30T23:03:15.929025Z","shell.execute_reply.started":"2022-01-30T23:03:15.913916Z","shell.execute_reply":"2022-01-30T23:03:15.928265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:15.930408Z","iopub.execute_input":"2022-01-30T23:03:15.930673Z","iopub.status.idle":"2022-01-30T23:03:15.937718Z","shell.execute_reply.started":"2022-01-30T23:03:15.930639Z","shell.execute_reply":"2022-01-30T23:03:15.936915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# neg = int(count_data_items(files_traina))\n# pos = int(count_data_items(files_trainb))\n# total = neg + pos\n# print('Examples:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n#     total, pos, 100 * pos / total))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:15.938715Z","iopub.execute_input":"2022-01-30T23:03:15.940274Z","iopub.status.idle":"2022-01-30T23:03:15.946425Z","shell.execute_reply.started":"2022-01-30T23:03:15.940238Z","shell.execute_reply":"2022-01-30T23:03:15.945666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with validation data\n# files_train, files_val = train_test_split(files_train,test_size=0.2, random_state=42)\nfiles_val = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/val*.tfrec')))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:15.947666Z","iopub.execute_input":"2022-01-30T23:03:15.948497Z","iopub.status.idle":"2022-01-30T23:03:16.091034Z","shell.execute_reply.started":"2022-01-30T23:03:15.948463Z","shell.execute_reply":"2022-01-30T23:03:16.090364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-01-30T23:03:16.09399Z","iopub.execute_input":"2022-01-30T23:03:16.094194Z","iopub.status.idle":"2022-01-30T23:03:16.247531Z","shell.execute_reply.started":"2022-01-30T23:03:16.094172Z","shell.execute_reply":"2022-01-30T23:03:16.246597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, cfg):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = cfg[\"read_size\"]\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = cfg['rot'] * tf.random.normal([1], dtype='float32')\n    shr = cfg['shr'] * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / cfg['hzoom']\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / cfg['wzoom']\n    h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32') \n    w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n    \n    return tf.reshape(d,[DIM, DIM,3])","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:16.249414Z","iopub.execute_input":"2022-01-30T23:03:16.250175Z","iopub.status.idle":"2022-01-30T23:03:16.270051Z","shell.execute_reply.started":"2022-01-30T23:03:16.250096Z","shell.execute_reply":"2022-01-30T23:03:16.269266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n#         'patient_id'                   : tf.io.FixedLenFeature([], tf.int64),\n#         'sex'                          : tf.io.FixedLenFeature([], tf.int64),\n#         'age_approx'                   : tf.io.FixedLenFeature([], tf.int64),\n#         'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),\n#         'diagnosis'                    : tf.io.FixedLenFeature([], tf.int64),\n        'target'                       : tf.io.FixedLenFeature([], tf.int64)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], example['target']\n\n\ndef read_unlabeled_tfrecord(example, return_image_name):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], example['image_name'] if return_image_name else 0\n\n \ndef prepare_image(img, cfg=None, augment=True):\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])\n    img = tf.cast(img, tf.float32) / 255.0\n    \n    if augment:\n        img = transform(img, cfg)\n        img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_hue(img, 0.01)\n        img = tf.image.random_saturation(img, 0.7, 1.3)\n        img = tf.image.random_contrast(img, 0.8, 1.2)\n        img = tf.image.random_brightness(img, 0.1)\n\n    else:\n        img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])\n\n    img = tf.image.resize(img, [cfg['net_size'], cfg['net_size']])\n    img = tf.reshape(img, [cfg['net_size'], cfg['net_size'], 3])\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:16.271618Z","iopub.execute_input":"2022-01-30T23:03:16.271984Z","iopub.status.idle":"2022-01-30T23:03:16.288051Z","shell.execute_reply.started":"2022-01-30T23:03:16.271938Z","shell.execute_reply":"2022-01-30T23:03:16.286991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(files, cfg, augment = False, shuffle = False, repeat = False, \n                labeled=True, return_image_names=True):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle:\n        ds = ds.shuffle(1024*8)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled:\n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), \n                    num_parallel_calls=AUTO)\n    \n    ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment, cfg=cfg), \n                                               imgname_or_label), \n                num_parallel_calls=AUTO)\n    \n    ds = ds.batch(cfg['batch_size'] * REPLICAS)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:16.291104Z","iopub.execute_input":"2022-01-30T23:03:16.291797Z","iopub.status.idle":"2022-01-30T23:03:16.301321Z","shell.execute_reply.started":"2022-01-30T23:03:16.291757Z","shell.execute_reply":"2022-01-30T23:03:16.300538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test the input pipeline\nBefore calling any neural net I always test the input pipeline.\nHere are images from the train data.","metadata":{}},{"cell_type":"code","source":"def show_dataset(thumb_size, cols, rows, ds):\n    mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols + (cols-1), \n                                             thumb_size*rows + (rows-1)))\n   \n    for idx, data in enumerate(iter(ds)):\n        img, target_or_imgid = data\n        ix  = idx % cols\n        iy  = idx // cols\n        img = np.clip(img.numpy() * 255, 0, 255).astype(np.uint8)\n        img = PIL.Image.fromarray(img)\n        img = img.resize((thumb_size, thumb_size), resample=PIL.Image.BILINEAR)\n        mosaic.paste(img, (ix*thumb_size + ix, \n                           iy*thumb_size + iy))\n\n    display(mosaic)\n    \nds = get_dataset(files_train, CFG).unbatch().take(12*5)   \nshow_dataset(64, 12, 5, ds)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:04:15.756381Z","iopub.execute_input":"2022-01-30T23:04:15.756927Z","iopub.status.idle":"2022-01-30T23:04:16.862563Z","shell.execute_reply.started":"2022-01-30T23:04:15.75689Z","shell.execute_reply":"2022-01-30T23:04:16.861193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test of image augmentation","metadata":{}},{"cell_type":"code","source":"ds = tf.data.TFRecordDataset(files_train, num_parallel_reads=AUTO)\nds = ds.take(1).cache().repeat()\nds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\nds = ds.map(lambda img, target: (prepare_image(img, cfg=CFG, augment=True), target), \n            num_parallel_calls=AUTO)\nds = ds.take(12*5)\nds = ds.prefetch(AUTO)\n\nshow_dataset(64, 12, 5, ds)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.146169Z","iopub.status.idle":"2022-01-30T23:03:23.14646Z","shell.execute_reply.started":"2022-01-30T23:03:23.146301Z","shell.execute_reply":"2022-01-30T23:03:23.146321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Images from the test data","metadata":{}},{"cell_type":"code","source":"ds = get_dataset(files_trainb, CFG, labeled=False).unbatch().take(12*5)   \nshow_dataset(64, 12, 5, ds)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.147572Z","iopub.status.idle":"2022-01-30T23:03:23.14802Z","shell.execute_reply.started":"2022-01-30T23:03:23.14778Z","shell.execute_reply":"2022-01-30T23:03:23.147804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MODELS","metadata":{}},{"cell_type":"code","source":"# MeanSquare Loss\ndef loss_clf_mse(gamma=2., alpha=.25):\n    \"\"\"\n    Binary form of focal loss.\n      FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)\n      where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def loss_clf_mse_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred:  A tensor resulting from a sigmoid\n        :return: Output tensor.\n        \"\"\"\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n        epsilon = K.epsilon()\n        # clip to prevent NaN's and Inf's\n        pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n        pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n        return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\\n               -K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n    return loss_clf_mse_fixed\n\n# FOCAL LOSS\ndef binary_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Binary form of focal loss.\n      FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)\n      where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def binary_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred:  A tensor resulting from a sigmoid\n        :return: Output tensor.\n        \"\"\"\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n        epsilon = K.epsilon()\n        # clip to prevent NaN's and Inf's\n        pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n        pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n        return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\\n               -K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n    return binary_focal_loss_fixed\n\ndef focal_loss(gamma=2., alpha=.25):\n\tdef focal_loss_fixed(y_true, y_pred):\n\t\tpt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n\t\tpt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\t\treturn -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) - K.mean((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n\treturn focal_loss_fixed","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.149831Z","iopub.status.idle":"2022-01-30T23:03:23.150608Z","shell.execute_reply.started":"2022-01-30T23:03:23.150363Z","shell.execute_reply":"2022-01-30T23:03:23.150387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_f(model_name, dim, output_bias=None):\n    if output_bias is not None:\n        output_bias = keras.initializers.Constant(output_bias)\n        \n    inp = tf.keras.layers.Input(shape=(dim,dim,3))\n    \n    if model_name == 'f6':\n        base = efn.EfficientNetB6(input_shape=(dim,dim,3),weights='imagenet',include_top=False)\n    elif model_name == 'f7':\n        base = efn.EfficientNetB7(input_shape=(dim,dim,3),weights='imagenet',include_top=False)\n    \n    x = base(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(1,activation='sigmoid',bias_initializer=output_bias)(x)\n    model = tf.keras.Model(inputs=inp,outputs=x)\n    # ---------------------------------------------------------------------------------\n    opt = tf.keras.optimizers.Adam(learning_rate=5e-06)\n#     loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)\n    loss = [binary_focal_loss(gamma = 2.0, alpha = 0.8)]\n#     loss = tfa.losses.SigmoidFocalCrossEntropy()\n#     loss = [focal_loss()]\n    # ---------------------------------------------------------------------------------\n    metrics = [\n        keras.metrics.TruePositives(name='tp'),\n        keras.metrics.FalsePositives(name='fp'),\n        keras.metrics.TrueNegatives(name='tn'),\n        keras.metrics.FalseNegatives(name='fn'),\n        keras.metrics.BinaryAccuracy(name='accuracy'),\n        keras.metrics.Precision(name='precision'),\n        keras.metrics.Recall(name='recall'),\n        keras.metrics.AUC(name='auc'),\n        keras.metrics.AUC(name='prc', curve='PR'), # precision-recall curve\n    ]\n    # ---------------------------------------------------------------------------------\n    model.compile(optimizer=opt,loss=loss,metrics=metrics)\n    # ---------------------------------------------------------------------------------\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.151653Z","iopub.status.idle":"2022-01-30T23:03:23.152434Z","shell.execute_reply.started":"2022-01-30T23:03:23.152196Z","shell.execute_reply":"2022-01-30T23:03:23.15222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with strategy.scope():\n#     model_f0 = model_f0(dim=256)\n#     model_f1 = model_f1(dim=256)\n#     model_f2 = model_f2(dim=256)\n#     model_f3 = model_f3(dim=256)\n#     model_f4 = model_f4(dim=256)\n#     model_f5 = model_f5(dim=256)\n#     model_f6 = model_f('f6',dim=256)\n#     model_f7 = model_f('f7',dim=256)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.153566Z","iopub.status.idle":"2022-01-30T23:03:23.154354Z","shell.execute_reply.started":"2022-01-30T23:03:23.154094Z","shell.execute_reply":"2022-01-30T23:03:23.154118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MODELS","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.layers.experimental import preprocessing\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models, optimizers","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.155494Z","iopub.status.idle":"2022-01-30T23:03:23.156282Z","shell.execute_reply.started":"2022-01-30T23:03:23.156023Z","shell.execute_reply":"2022-01-30T23:03:23.156049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Training_Model(model_name, IMG_SIZE, NUM_CHANNELS, Dropout_rate, output_bias=None):\n    if output_bias is not None:\n        output_bias = keras.initializers.Constant(output_bias)\n    \n    inputs = layers.Input(shape=(IMG_SIZE, IMG_SIZE, 3))\n    \n    if model_name == 'VGG19':\n        base_model = tf.keras.applications.vgg19.VGG19(input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n    if model_name == 'ResNet152V2':\n        base_model = tf.keras.applications.resnet_v2.ResNet152V2( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n    if model_name == 'InceptionV3':\n        base_model = tf.keras.applications.InceptionV3( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n    if model_name == 'Xception':\n        base_model=tf.keras.applications.Xception( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n    if model_name == 'EfficientNetB2':\n        base_model=tf.keras.applications.EfficientNetB2( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n    if model_name == 'EfficientNetB3':\n        base_model=tf.keras.applications.EfficientNetB3( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n    if model_name == 'EfficientNetB4':\n        base_model=tf.keras.applications.EfficientNetB4( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n    if model_name == 'EfficientNetB5':\n        base_model=tf.keras.applications.EfficientNetB5( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n    if model_name == 'EfficientNetB6':\n        base_model=tf.keras.applications.EfficientNetB6( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n    if model_name == 'EfficientNetB7':\n        base_model=tf.keras.applications.EfficientNetB7( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n    if model_name == 'InceptionResNetV2':\n        base_model=tf.keras.applications.InceptionResNetV2( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n    if model_name == 'DenseNet121':\n        base_model=tf.keras.applications.DenseNet121( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n    if model_name == 'DenseNet169':\n        base_model=tf.keras.applications.DenseNet169( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n    if model_name == 'DenseNet201':\n        base_model=tf.keras.applications.DenseNet201( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n    if model_name == 'MobileNetV2':\n        base_model=tf.keras.applications.MobileNetV2( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n    if model_name == 'ResNet101V2':\n        base_model=tf.keras.applications.ResNet101V2( input_shape=(IMG_SIZE, IMG_SIZE, NUM_CHANNELS), include_top=False, weights='imagenet')\n        \n    #base_model.trainable = False\n    x = base_model.output\n    x = layers.GlobalAveragePooling2D()(x)\n    #x = layers.Dense(1000, activation='relu')(x)\n    #x = layers.Dropout(Dropout_rate)(x)\n    x = layers.Dense(1, activation='sigmoid', bias_initializer=output_bias)(x)\n    \n    model = models.Model(inputs=base_model.input, outputs=x)\n    # ---------------------------------------------------------------------------------\n#     opt = keras.optimizers.Adam(learning_rate=0.000005)\n#     loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)\n    opt = keras.optimizers.Adam(learning_rate=0.00001)\n#     loss = keras.losses.BinaryCrossentropy()\n    loss = [binary_focal_loss(gamma = 2.0, alpha = 0.80)]\n    \n    METRICS = [\n        keras.metrics.TruePositives(name='tp'),\n        keras.metrics.FalsePositives(name='fp'),\n        keras.metrics.TrueNegatives(name='tn'),\n        keras.metrics.FalseNegatives(name='fn'),\n        keras.metrics.BinaryAccuracy(name='accuracy'),\n        keras.metrics.Precision(name='precision'),\n        keras.metrics.Recall(name='recall'),\n        keras.metrics.AUC(name='auc'),\n        keras.metrics.AUC(name='prc', curve='PR'), # precision-recall curve\n    ]\n    # ---------------------------------------------------------------------------------\n#     model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['AUC'])\n#     model.compile(optimizer='adam', loss=BinaryFocalLoss(gamma=2), metrics=['AUC'])\n    model.compile(optimizer=opt,loss=loss,metrics=METRICS)\n    return model","metadata":{"_kg_hide-input":true,"scrolled":true,"execution":{"iopub.status.busy":"2022-01-30T23:03:23.157519Z","iopub.status.idle":"2022-01-30T23:03:23.158212Z","shell.execute_reply.started":"2022-01-30T23:03:23.157924Z","shell.execute_reply":"2022-01-30T23:03:23.157949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_train     = get_dataset(files_train, CFG, augment=True, shuffle=False, repeat=False)\nds_train     = ds_train.map(lambda img, label: (img, tuple([label])))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.159494Z","iopub.status.idle":"2022-01-30T23:03:23.160273Z","shell.execute_reply.started":"2022-01-30T23:03:23.160018Z","shell.execute_reply":"2022-01-30T23:03:23.160042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model_f6 = Training_Model(model_name='DenseNet169', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.5)\n#     model_f6 = model_f('f6',dim=256)\n    \nresults = model_f6.evaluate(ds_train, batch_size=(CFG['batch_size'] * REPLICAS), verbose=0)\nprint(\"Loss: {:0.4f}\".format(results[0]))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.161412Z","iopub.status.idle":"2022-01-30T23:03:23.162197Z","shell.execute_reply.started":"2022-01-30T23:03:23.161929Z","shell.execute_reply":"2022-01-30T23:03:23.161953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"initial_bias = np.log([pos/neg])\ninitial_bias","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.163325Z","iopub.status.idle":"2022-01-30T23:03:23.164085Z","shell.execute_reply.started":"2022-01-30T23:03:23.16383Z","shell.execute_reply":"2022-01-30T23:03:23.163855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model_f6 = Training_Model(model_name='DenseNet169', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.5, output_bias=initial_bias)\n#     model_f6 = model_f('f6',dim=256,output_bias=initial_bias)\n    \nresults = model_f6.evaluate(ds_train, batch_size=(CFG['batch_size'] * REPLICAS), verbose=0)\nprint(\"Loss: {:0.4f}\".format(results[0]))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.165204Z","iopub.status.idle":"2022-01-30T23:03:23.165965Z","shell.execute_reply.started":"2022-01-30T23:03:23.165725Z","shell.execute_reply":"2022-01-30T23:03:23.165749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_f6.save_weights(\"./initial_weights.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.167264Z","iopub.status.idle":"2022-01-30T23:03:23.167678Z","shell.execute_reply.started":"2022-01-30T23:03:23.16745Z","shell.execute_reply":"2022-01-30T23:03:23.167478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#################################################################################################################################################","metadata":{}},{"cell_type":"code","source":"# with strategy.scope():\n#     model1 = Training_Model(model_name='VGG19', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.4)\n#     model2 = Training_Model(model_name='ResNet152V2', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.4)\n#     model3 = Training_Model(model_name='InceptionV3', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.4)\n#     model4 = Training_Model(model_name='Xception', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.4)\n#     model5 = Training_Model(model_name='EfficientNetB5', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.4)\n#     model6 = Training_Model(model_name='EfficientNetB6', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.4)\n#     model7 = Training_Model(model_name='EfficientNetB7', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.4)\n#     model8 = Training_Model(model_name='InceptionResNetV2', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.4)\n#     model9 = Training_Model(model_name='DenseNet201', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.5)\n#     model10 = Training_Model(model_name='MobileNetV2', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.4)\n#     model11 = Training_Model(model_name='ResNet101V2', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.4)\n#     model12 = Training_Model(model_name='DenseNet169', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.5)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.168965Z","iopub.status.idle":"2022-01-30T23:03:23.169594Z","shell.execute_reply.started":"2022-01-30T23:03:23.169348Z","shell.execute_reply":"2022-01-30T23:03:23.169371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fit the Model","metadata":{}},{"cell_type":"code","source":"num_training_images = int(count_data_items(files_train))\nnum_val_images = count_data_items(files_val)\n# num_test_images = count_data_items(files_test)\n# print('Dataset: {} training images, {} unlabeled test images'.format(num_training_images, num_test_images))\nprint('Dataset: {} training images, {} validation images'.format(num_training_images, num_val_images))\n\nnum_training_a_images = int(count_data_items(files_traina))\nnum_training_b_images = int(count_data_items(files_trainb))\nprint('Dataset: {} training a images, {} training b images'.format(num_training_a_images, num_training_b_images))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.170909Z","iopub.status.idle":"2022-01-30T23:03:23.171492Z","shell.execute_reply.started":"2022-01-30T23:03:23.171246Z","shell.execute_reply":"2022-01-30T23:03:23.171269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_train     = get_dataset(files_train, CFG, augment=True, shuffle=True, repeat=True)\n# ds_train     = ds_train.map(lambda img, label: (img, tuple([label])))\n# -------------------------------------------------------------------------------------\nds_val       = get_dataset(files_val, CFG, augment=True, shuffle=True, repeat=True)\nds_val       = ds_val.map(lambda img, label: (img, tuple([label] * CFG['net_count'])))\n# -------------------------------------------------------------------------------------\nsteps_train  = count_data_items(files_train) / (CFG['batch_size'] * REPLICAS)\nsteps_val    = count_data_items(files_val) / (CFG['batch_size'] * REPLICAS)\nprint(steps_train)\nprint(steps_val)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.172641Z","iopub.status.idle":"2022-01-30T23:03:23.173354Z","shell.execute_reply.started":"2022-01-30T23:03:23.173109Z","shell.execute_reply":"2022-01-30T23:03:23.173132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback(cfg):\n    lr_start   = cfg['LR_START']\n    lr_max     = cfg['LR_MAX'] * strategy.num_replicas_in_sync\n    lr_min     = cfg['LR_MIN']\n    lr_ramp_ep = cfg['LR_RAMPUP_EPOCHS']\n    lr_sus_ep  = cfg['LR_SUSTAIN_EPOCHS']\n    lr_decay   = cfg['LR_EXP_DECAY']\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)\n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.174724Z","iopub.status.idle":"2022-01-30T23:03:23.175133Z","shell.execute_reply.started":"2022-01-30T23:03:23.174914Z","shell.execute_reply":"2022-01-30T23:03:23.174934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callback_list = []\n\n# if the model does not improve for 10 epochs, stop the training\nstop_early = EarlyStopping(monitor='val_prc', mode='max', patience=10, restore_best_weights=True)\ncallback_list.append(stop_early)\n\nlr_callback = get_lr_callback(CFG)\ncallback_list.append(lr_callback)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.176433Z","iopub.status.idle":"2022-01-30T23:03:23.176855Z","shell.execute_reply.started":"2022-01-30T23:03:23.176632Z","shell.execute_reply":"2022-01-30T23:03:23.176653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(\"=========== Model_f0 =============\")\n# history0 = model_f0.fit(ds_train, verbose = 1, steps_per_epoch = steps_train, epochs = CFG['epochs'], callbacks = [get_lr_callback(CFG)],validation_data=ds_val)\n# model_f0.save('model_f0.hdf5')\n\n# print(\"=========== Model_f1 =============\")\n# history1 = model_f1.fit(ds_train, verbose = 1, steps_per_epoch = steps_train, epochs = CFG['epochs'], callbacks = [get_lr_callback(CFG)],validation_data=ds_val)\n# model_f1.save('model_f1.hdf5')\n\n# print(\"=========== Model_f2 =============\")\n# history2 = model_f2.fit(ds_train, verbose = 1, steps_per_epoch = steps_train, epochs = CFG['epochs'], callbacks = [get_lr_callback(CFG)],validation_data=ds_val)\n# model_f2.save('model_f2.hdf5')\n\n# print(\"=========== Model_f3 =============\")\n# history3 = model_f3.fit(ds_train, verbose = 1, steps_per_epoch = steps_train, epochs = CFG['epochs'], callbacks = [get_lr_callback(CFG)],validation_data=ds_val)\n# model_f3.save('model_f3.hdf5')\n\n# print(\"=========== Model_f4 =============\")\n# history4 = model_f4.fit(ds_train, verbose = 1, steps_per_epoch = steps_train, epochs = CFG['epochs'], callbacks = [get_lr_callback(CFG)],validation_data=ds_val)\n# model_f4.save('model_f4.hdf5')\n\n# print(\"=========== Model_f5 =============\")\n# history5 = model_f5.fit(ds_train, verbose = 1, steps_per_epoch = steps_train, epochs = CFG['epochs'], callbacks = [get_lr_callback(CFG)],validation_data=ds_val)\n# model_f5.save('model_f5.hdf5')\n\n# print(\"=========== Model_f6 =============\")\n# history6 = model_f6.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, validation_data=ds_val, validation_steps=steps_val)\n# model_f6.save('model_f6.hdf5')\n\n# print(\"=========== Model_f7 =============\")\n# history7 = model_f7.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, validation_data=ds_val, validation_steps=steps_val)\n# model_f7.save('model_f7.hdf5')\n\n# ###################################################################################################################################################\n# print(\"=========== Model1 =============\")\n# history1 = model1.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, \n#                       validation_data=ds_val, validation_steps=steps_val\n#                      )\n# model1.save('model1.hdf5')\n\n# print(\"=========== Model2 =============\")\n# history2 = model2.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, validation_data=ds_val, validation_steps=steps_val)\n# model2.save('model2.hdf5')\n\n# print(\"=========== Model3 =============\")\n# history3 = model3.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, validation_data=ds_val, validation_steps=steps_val)\n# model3.save('model3.hdf5')\n\n# print(\"=========== Model4 =============\")\n# history4 = model4.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, \n#                       validation_data=ds_val, validation_steps=steps_val\n#                      )\n# model4.save('model4.hdf5')\n\n# print(\"=========== Model5 =============\")\n# history5 = model5.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, validation_data=ds_val, validation_steps=steps_val)\n# model5.save('model5.hdf5')\n\n# print(\"=========== Model6 =============\")\n# history6 = model6.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, validation_data=ds_val, validation_steps=steps_val)\n# model6.save('model6.hdf5')\n\n# print(\"=========== Model7 =============\")\n# history7 = model7.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, validation_data=ds_val, validation_steps=steps_val)\n# model7.save('model7.hdf5')\n\n# print(\"=========== Model8 =============\")\n# history8 = model8.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, validation_data=ds_val, validation_steps=steps_val)\n# model8.save('model8.hdf5')\n\n# print(\"=========== Model9 =============\")\n# history9 = model9.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, validation_data=ds_val, validation_steps=steps_val)\n# model9.save('model9.hdf5')\n\n# print(\"=========== Model10 =============\")\n# history10 = model10.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, validation_data=ds_val, validation_steps=steps_val)\n# model10.save('model10.hdf5')\n\n# print(\"=========== Model11 =============\")\n# history11 = model11.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, validation_data=ds_val, validation_steps=steps_val)\n# model11.save('model11.hdf5')\n\n# print(\"=========== Model12 =============\")\n# history12 = model12.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, \n#                       validation_data=ds_val, validation_steps=steps_val\n#                      )\n# model12.save('model12.hdf5')","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.17837Z","iopub.status.idle":"2022-01-30T23:03:23.178838Z","shell.execute_reply.started":"2022-01-30T23:03:23.17859Z","shell.execute_reply":"2022-01-30T23:03:23.178613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Zero Bias\n# with strategy.scope():\n# #     model_f6 = Training_Model(model_name='DenseNet169', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.5)\n# #     model_f6.load_weights(\"./initial_weights.h5\")\n# #     model_f6.layers[-1].bias.assign([0.0])\n    \n#     model_f6 = model_f('f6',dim=256)\n#     model_f6.load_weights(\"./initial_weights.h5\")\n#     model_f6.layers[-1].bias.assign([0.0])\n\n# history_f6_zero = model_f6.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, \n#                       validation_data=ds_val, validation_steps=steps_val\n#                      )\n# model_f6.save('model_f6.hdf5')","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.180307Z","iopub.status.idle":"2022-01-30T23:03:23.18073Z","shell.execute_reply.started":"2022-01-30T23:03:23.180508Z","shell.execute_reply":"2022-01-30T23:03:23.180529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Careful Bias\n# with strategy.scope():\n# #     model_f6_bias = Training_Model(model_name='DenseNet169', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.5)\n# #     model_f6_bias.load_weights(\"./initial_weights.h5\")\n    \n#     model_f6_bias = model_f('f6',dim=256)\n#     model_f6_bias.load_weights(\"./initial_weights.h5\")\n\n# history_f6_bias = model_f6_bias.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, \n#                       validation_data=ds_val, validation_steps=steps_val\n#                      )\n# model_f6_bias.save('model_f6_bias.hdf5')","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.182765Z","iopub.status.idle":"2022-01-30T23:03:23.183443Z","shell.execute_reply.started":"2022-01-30T23:03:23.183209Z","shell.execute_reply":"2022-01-30T23:03:23.183231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Scaling by total/2 helps keep the loss to a similar magnitude.\n# The sum of the weights of all examples stays the same.\nweight_for_0 = (1 / neg) * (total / 2.0)\nweight_for_1 = (1 / pos) * (total / 2.0)\n\nclass_weight = {0: weight_for_0, 1: weight_for_1}\n\nprint('Weight for class 0: {:.2f}'.format(weight_for_0))\nprint('Weight for class 1: {:.2f}'.format(weight_for_1))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.18471Z","iopub.status.idle":"2022-01-30T23:03:23.185273Z","shell.execute_reply.started":"2022-01-30T23:03:23.185022Z","shell.execute_reply":"2022-01-30T23:03:23.185045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Weighted Model\nwith strategy.scope():\n    model_f6_weighted = Training_Model(model_name='DenseNet169', IMG_SIZE=256, NUM_CHANNELS=3, Dropout_rate=0.5)\n    model_f6_weighted.load_weights(\"./initial_weights.h5\")\n    \n#     model_f6_weighted = model_f('f6',dim=256)\n#     model_f6_weighted.load_weights(\"./initial_weights.h5\")\n\nhistory_f6_weighted = model_f6_weighted.fit(ds_train, verbose=1, steps_per_epoch=steps_train, epochs=CFG['epochs'], callbacks=callback_list, \n                               validation_data=ds_val, validation_steps=steps_val,\n                               # The class weights go here\n                               class_weight=class_weight\n                              )\n\nmodel_f6_weighted.save('model_f6_weighted.hdf5')","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.186448Z","iopub.status.idle":"2022-01-30T23:03:23.187179Z","shell.execute_reply.started":"2022-01-30T23:03:23.186921Z","shell.execute_reply":"2022-01-30T23:03:23.186944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluate metrics","metadata":{}},{"cell_type":"code","source":"import matplotlib as mpl\nimport matplotlib.pyplot as plt\n\nmpl.rcParams['figure.figsize'] = (12, 10)\ncolors = plt.rcParams['axes.prop_cycle'].by_key()['color']\n\ndef plot_loss(history, label, n):\n    # Use a log scale on y-axis to show the wide range of values.\n    plt.semilogy(history.epoch, history.history['loss'],\n               color=colors[n], label='Train ' + label)\n    plt.semilogy(history.epoch, history.history['val_loss'],\n               color=colors[n], label='Val ' + label,\n               linestyle=\"--\")\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.188599Z","iopub.status.idle":"2022-01-30T23:03:23.189036Z","shell.execute_reply.started":"2022-01-30T23:03:23.1888Z","shell.execute_reply":"2022-01-30T23:03:23.188822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot_loss(history_f6_zero, \"Zero Bias\", 0)\n# plot_loss(history_f6_bias, \"Careful Bias\", 1)\nplot_loss(history_f6_weighted, \"Weighted History\", 2)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.19035Z","iopub.status.idle":"2022-01-30T23:03:23.190918Z","shell.execute_reply.started":"2022-01-30T23:03:23.190678Z","shell.execute_reply":"2022-01-30T23:03:23.190701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_metrics(history):\n    metrics = ['loss', 'prc', 'precision', 'recall']\n    for n, metric in enumerate(metrics):\n        name = metric.replace(\"_\",\" \").capitalize()\n        plt.subplot(2,2,n+1)\n        plt.plot(history.epoch, history.history[metric], color=colors[0], label='Train')\n        plt.plot(history.epoch, history.history['val_'+metric],\n                 color=colors[0], linestyle=\"--\", label='Val')\n        plt.xlabel('Epoch')\n        plt.ylabel(name)\n        if metric == 'loss':\n            plt.ylim([0, plt.ylim()[1]])\n        elif metric == 'auc':\n            plt.ylim([0.8,1])\n        else:\n            plt.ylim([0,1])\n\n        plt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.192225Z","iopub.status.idle":"2022-01-30T23:03:23.192639Z","shell.execute_reply.started":"2022-01-30T23:03:23.192412Z","shell.execute_reply":"2022-01-30T23:03:23.192433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot_metrics(history_f6_zero)\n# plot_metrics(history_f6_bias)\nplot_metrics(history_f6_weighted)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.193908Z","iopub.status.idle":"2022-01-30T23:03:23.194483Z","shell.execute_reply.started":"2022-01-30T23:03:23.194236Z","shell.execute_reply":"2022-01-30T23:03:23.194259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_val     = get_dataset(files_val, CFG, augment=True, shuffle=False, repeat=False, labeled=True)\nds_val     = ds_val.map(lambda img, label: (img, tuple([label] * CFG['net_count'])))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.195813Z","iopub.status.idle":"2022-01-30T23:03:23.196247Z","shell.execute_reply.started":"2022-01-30T23:03:23.196009Z","shell.execute_reply":"2022-01-30T23:03:23.19603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# true labels\nval_labels = []\nfor idx, data in enumerate(iter(ds_val)):\n    _, target = data\n\n    val_labels.append(target[0].numpy())\n\nval_labels = np.concatenate(val_labels)\nprint(val_labels)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.197622Z","iopub.status.idle":"2022-01-30T23:03:23.198065Z","shell.execute_reply.started":"2022-01-30T23:03:23.197831Z","shell.execute_reply":"2022-01-30T23:03:23.197853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_predictions_zero = model12.predict(ds_train, steps=steps_train, verbose=1)\n# val_predictions_zero = model_f6.predict(ds_val, steps=steps_val, verbose=1)\n\n# train_predictions_bias = model12_bias.predict(ds_train, steps=steps_train, verbose=1)\n# val_predictions_bias = model_f6_bias.predict(ds_val, steps=steps_val, verbose=1)\n\n# train_predictions_weighted = model12_weighted.predict(ds_train, steps=steps_train, verbose=1)\nval_predictions_weighted = model_f6_weighted.predict(ds_val, steps=steps_val, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.199179Z","iopub.status.idle":"2022-01-30T23:03:23.199868Z","shell.execute_reply.started":"2022-01-30T23:03:23.199624Z","shell.execute_reply":"2022-01-30T23:03:23.199649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nfrom sklearn.metrics import roc_curve, auc, precision_recall_curve, plot_precision_recall_curve, confusion_matrix\n\ndef plot_cm(labels, predictions, p=0.5):\n    cm = confusion_matrix(labels, predictions > p)\n    plt.figure(figsize=(5,5))\n    sns.heatmap(cm, annot=True, fmt=\"d\")\n    plt.title('Confusion matrix @{:.2f}'.format(p))\n    plt.ylabel('Actual label')\n    plt.xlabel('Predicted label')\n    plt.savefig('cm.png')\n\n    print('Benign Detected (True Negatives): ', cm[0][0])\n    print('Benign Detected (False Positives): ', cm[0][1])\n    print('Malignant Missed (False Negatives): ', cm[1][0])\n    print('Malignant Detected (True Positives): ', cm[1][1])\n    print('Total Malignant: ', np.sum(cm[1]))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.201055Z","iopub.status.idle":"2022-01-30T23:03:23.201731Z","shell.execute_reply.started":"2022-01-30T23:03:23.201488Z","shell.execute_reply":"2022-01-30T23:03:23.201511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# zero_results = model_f6.evaluate(ds_val, steps=steps_val, verbose=0)\n# for name, value in zip(model_f6.metrics_names, zero_results):\n#     print(name, ': ', value)\n# print()\n\n# plot_cm(val_labels, val_predictions_zero)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.203162Z","iopub.status.idle":"2022-01-30T23:03:23.203584Z","shell.execute_reply.started":"2022-01-30T23:03:23.203351Z","shell.execute_reply":"2022-01-30T23:03:23.203372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bias_results = model_f6_bias.evaluate(ds_val, steps=steps_val, verbose=0)\n# for name, value in zip(model_f6_bias.metrics_names, bias_results):\n#     print(name, ': ', value)\n# print()\n\n# plot_cm(val_labels, val_predictions_bias)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.20499Z","iopub.status.idle":"2022-01-30T23:03:23.205502Z","shell.execute_reply.started":"2022-01-30T23:03:23.205234Z","shell.execute_reply":"2022-01-30T23:03:23.205258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weighted_results = model_f6_weighted.evaluate(ds_val, steps=steps_val, verbose=0)\nfor name, value in zip(model_f6_weighted.metrics_names, weighted_results):\n    print(name, ': ', value)\nprint()\n\nplot_cm(val_labels, val_predictions_weighted)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.206983Z","iopub.status.idle":"2022-01-30T23:03:23.20757Z","shell.execute_reply.started":"2022-01-30T23:03:23.207331Z","shell.execute_reply":"2022-01-30T23:03:23.207354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict","metadata":{}},{"cell_type":"code","source":"ds_val     = get_dataset(files_val, CFG, augment=True, shuffle=False, repeat=False, labeled=True)\nds_val     = ds_val.map(lambda img, label: (img, tuple([label] * CFG['net_count'])))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.209086Z","iopub.status.idle":"2022-01-30T23:03:23.209525Z","shell.execute_reply.started":"2022-01-30T23:03:23.209287Z","shell.execute_reply":"2022-01-30T23:03:23.209308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load models\nfrom tensorflow.keras.models import load_model\n\n# model_f6 = load_model('../input/efficientnetb6-tpu/model_f6.hdf5')\n# model_f7 = load_model('../input/efficientnetb6-tpu/model_f7.hdf5')\n# model1 = load_model('../input/efficientnetb6-tpu/model1.hdf5')\n# model2 = load_model('../input/efficientnetb6-tpu/model2.hdf5')","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.210949Z","iopub.status.idle":"2022-01-30T23:03:23.211566Z","shell.execute_reply.started":"2022-01-30T23:03:23.211315Z","shell.execute_reply":"2022-01-30T23:03:23.21134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Computing predictions...')\n# model_list = [model_f6,model_f6_bias,model_f6_weighted]\nmodel_list = [model_f6_weighted]\n\nens_probabilities = [model.predict(ds_val, verbose=1, steps=steps_val) for model in model_list]\n\naverage_prob = np.sum(ens_probabilities, axis=0)/len(model_list)\nens_probabilities.append(average_prob)\nprint(\"========================  Done  ============================\")","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.212916Z","iopub.status.idle":"2022-01-30T23:03:23.213337Z","shell.execute_reply.started":"2022-01-30T23:03:23.213101Z","shell.execute_reply":"2022-01-30T23:03:23.213122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluate","metadata":{}},{"cell_type":"markdown","source":"## AUC Curve, Precision, Recall","metadata":{}},{"cell_type":"code","source":"# true labels\ny_t = []\nfor idx, data in enumerate(iter(ds_val)):\n    _, target = data\n\n    y_t.append(target[0].numpy())\n\ny_t = np.concatenate(y_t)\nprint(len(y_t))\n\n# y predict\ny_ps = [np.stack(y, axis=1).reshape(-1) for y in ens_probabilities]\nprint(len(y_ps[0]))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.214508Z","iopub.status.idle":"2022-01-30T23:03:23.215156Z","shell.execute_reply.started":"2022-01-30T23:03:23.214894Z","shell.execute_reply":"2022-01-30T23:03:23.214916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\" Helper function to plot the auc curve\n\n    Parameters:\n        t_y (array): True binary labels\n        p_y (array): Target scores\n\n    Returns:\n        Null\n\"\"\"\ndef plot_auc(t_y, p_y):\n    fpr, tpr, thresholds = roc_curve(t_y, p_y, pos_label=1)\n    fig, c_ax = plt.subplots(1,1, figsize = (8, 8))\n    c_ax.plot(fpr, tpr, label = '%s (AUC:%0.2f)'  % ('Target', auc(fpr, tpr)))\n    c_ax.plot([0, 1], [0, 1], color='navy', lw=1, linestyle='--')\n    c_ax.legend()\n    c_ax.set_xlabel('False Positive Rate')\n    c_ax.set_ylabel('True Positive Rate')\n    plt.savefig('auc.png')\n\nfor y_p in y_ps:\n    plot_auc(y_t, y_p)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.216553Z","iopub.status.idle":"2022-01-30T23:03:23.216956Z","shell.execute_reply.started":"2022-01-30T23:03:23.216738Z","shell.execute_reply":"2022-01-30T23:03:23.216759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Confusion Matrix","metadata":{}},{"cell_type":"code","source":"\"\"\" Helper function to calculate the F1 Score\n    Parameters:\n        prec (int): precision\n        recall (int): recall\n\n    Returns:\n        f1 score (int)\n\"\"\"\ndef calc_f1(prec, recall):\n    return 2*(prec*recall)/(prec+recall) if recall and prec else 0\n\n\"\"\" Helper function turn the model predictions into a binary (0,1) format\n    Parameters:\n        pred (float): Model prediction\n\n    Returns:\n        binary prediction (int)\n\"\"\"\ndef pred_to_binary(pred):\n    if pred < threshold:\n        return 0\n    else:\n        return 1\n    \nimport itertools\n\n\"\"\" Helper function to plot a confusion matrix\n\n    Parameters:\n        cm (confusion matrix)\n\n    Returns:\n        Null\n\"\"\"\ndef plot_confusion_matrix(cm, labels):\n    plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)\n    plt.title('Confusion Matrix')\n    plt.colorbar()\n    tick_marks = np.arange(len(labels))\n    plt.xticks(tick_marks, labels, rotation=55)\n    plt.yticks(tick_marks, labels)\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], 'd'), horizontalalignment=\"center\", color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout()\n    plt.savefig('confusion_matrix.png')\n    plt.show()\n    \ncm_plot_label =['benign', 'malignant']\n\nfor y_p in y_ps:\n    # calculate the precision, recall and the thresholds\n    precision, recall, thresholds = precision_recall_curve(y_t, y_p)\n\n    # calculate the f1 score\n    f1score = [calc_f1(precision[i],recall[i]) for i in range(len(thresholds))]\n\n    # get the index from the highest f1 score\n    idx = np.argmax(f1score)\n\n    # get the precision, recall, threshold and the f1score\n    precision = round(precision[idx], 4)\n    recall = round(recall[idx], 4)\n#     threshold = round(thresholds[idx], 4)\n    threshold = 0.28\n    f1score = round(f1score[idx], 4)\n\n    print('Precision:', precision)\n    print('Recall:', recall)\n    print('Threshold:', threshold)\n    print('F1 Score:', f1score)\n    print('===========================')\n    \n    y_pred_binary = [pred_to_binary(x) for x in y_p]\n    # create a confusion matrix\n    cm =  confusion_matrix(y_t, y_pred_binary)\n    \n    plot_confusion_matrix(cm, cm_plot_label)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.219174Z","iopub.status.idle":"2022-01-30T23:03:23.219945Z","shell.execute_reply.started":"2022-01-30T23:03:23.2197Z","shell.execute_reply":"2022-01-30T23:03:23.219724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### predict the test set using augmented images","metadata":{}},{"cell_type":"code","source":"# test_ds = get_dataset(files_test, CFG, augment=True, repeat=False, labeled=False, return_image_names=False)\n\n# print('Computing predictions...')\n# test_images_ds = test_ds.map(lambda image, idnum: image)\n\n# # model_list = [model_f0,model_f1,model_f2,model_f3,model_f4,model_f5,mmodel_f6]\n# model_list = [model_f6]\n\n\n# ens_probabilities = [model.predict(test_images_ds) for model in model_list]\n\n# print(\"========================  Done  ============================\")","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.221053Z","iopub.status.idle":"2022-01-30T23:03:23.221935Z","shell.execute_reply.started":"2022-01-30T23:03:23.221686Z","shell.execute_reply":"2022-01-30T23:03:23.22171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Average the predictions of models\n# average_prob = np.sum(ens_probabilities, axis=0)/len(model_list)\n\n# weight the prediction of models\n\n# weights = [0.3, 0.3, 0.4]\n# #Use tensordot to sum the products of all elements over specified axes.\n# weighted_prob = np.tensordot(ens_probabilities, weights, axes=((0),(0)))\n","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.22302Z","iopub.status.idle":"2022-01-30T23:03:23.223879Z","shell.execute_reply.started":"2022-01-30T23:03:23.223644Z","shell.execute_reply":"2022-01-30T23:03:23.223666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ds = get_dataset(files_test, CFG, augment=False, repeat=False, \n#                  labeled=False, return_image_names=True)\n\n# image_names = np.array([img_name.numpy().decode(\"utf-8\") \n#                         for img, img_name in iter(ds.unbatch())])","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.224918Z","iopub.status.idle":"2022-01-30T23:03:23.22578Z","shell.execute_reply.started":"2022-01-30T23:03:23.225547Z","shell.execute_reply":"2022-01-30T23:03:23.22557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_average = pd.DataFrame({'image_name': image_names, 'target': np.concatenate(average_prob)})\n\n# ef0 = pd.DataFrame({'image_name': image_names, 'target': np.concatenate(ens_probabilities[0])})\n# ef1 = pd.DataFrame({'image_name': image_names, 'target': np.concatenate(ens_probabilities[1])})\n# ef2 = pd.DataFrame({'image_name': image_names, 'target': np.concatenate(ens_probabilities[2])})\n# ef3 = pd.DataFrame({'image_name': image_names, 'target': np.concatenate(ens_probabilities[3])})\n# ef4 = pd.DataFrame({'image_name': image_names, 'target': np.concatenate(ens_probabilities[4])})\n# ef5 = pd.DataFrame({'image_name': image_names, 'target': np.concatenate(ens_probabilities[5])})\n# ef6 = pd.DataFrame({'image_name': image_names, 'target': np.concatenate(ens_probabilities[6])})\n\n# ef6 = pd.DataFrame({'image_name': image_names, 'target': np.concatenate(ens_probabilities[0])})","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.22684Z","iopub.status.idle":"2022-01-30T23:03:23.227741Z","shell.execute_reply.started":"2022-01-30T23:03:23.227482Z","shell.execute_reply":"2022-01-30T23:03:23.227505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_average.to_csv('pred_average.csv', index=False)\n# ef0.to_csv('ef0.csv', index=False)\n# ef1.to_csv('ef1.csv', index=False)\n# ef2.to_csv('ef2.csv', index=False)\n# ef3.to_csv('ef3.csv', index=False)\n# ef4.to_csv('ef4.csv', index=False)\n# ef5.to_csv('ef5.csv', index=False)\n# ef6.to_csv('ef6.csv', index=False)\n\n# ef6.to_csv('ef6.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.228948Z","iopub.status.idle":"2022-01-30T23:03:23.229689Z","shell.execute_reply.started":"2022-01-30T23:03:23.22944Z","shell.execute_reply":"2022-01-30T23:03:23.229463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # load models\n# from tf.keras.models import load_model\n\n# m0 = load_model('./model_f0.hdf5')\n# m1 = load_model('./model_f1.hdf5')\n# m2 = load_model('./model_f3.hdf5')\n# m3 = load_model('./model_f3.hdf5')\n# m4 = load_model('./model_f4.hdf5')\n# m5 = load_model('./model_f5.hdf5')\n# m6 = load_model('./model_f6.hdf5')","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.230897Z","iopub.status.idle":"2022-01-30T23:03:23.231443Z","shell.execute_reply.started":"2022-01-30T23:03:23.231214Z","shell.execute_reply":"2022-01-30T23:03:23.231237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# models = [m1,m2,m3,m4,m5,m6]\n# test1 = test_ds.take(1)\n# test10 = test_ds.take(10)\n# test50 = test_ds.take(50)\n# test100 = test_ds.take(100)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.232502Z","iopub.status.idle":"2022-01-30T23:03:23.233053Z","shell.execute_reply.started":"2022-01-30T23:03:23.232818Z","shell.execute_reply":"2022-01-30T23:03:23.23284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import time\n\n# st = time.time()\n# pr = m6.predict(test10)\n# ed = time.time()\n# print(\"Single model time for 10 data: \", (ed - st))\n\n# start10 = time.time()\n# ens = [model.predict(test10) for model in models]\n# av = np.sum(ens, axis=0)/len(models)\n# end10 = time.time()\n# print(\"ensemble time for 10 data: \", (end10 - start10))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.2341Z","iopub.status.idle":"2022-01-30T23:03:23.234663Z","shell.execute_reply.started":"2022-01-30T23:03:23.234429Z","shell.execute_reply":"2022-01-30T23:03:23.234452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# st = time.time()\n# pr = m6.predict(test1)\n# ed = time.time()\n# print(\"Single model time for 1 data: \", (ed - st))\n\n# start1 = time.time()\n# ens = [model.predict(test1) for model in models]\n# av = np.sum(ens, axis=0)/len(models)\n# end1 = time.time()\n# print(\"ensemble time for 1 data: \", (end1 - start1))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.235704Z","iopub.status.idle":"2022-01-30T23:03:23.236272Z","shell.execute_reply.started":"2022-01-30T23:03:23.236015Z","shell.execute_reply":"2022-01-30T23:03:23.236039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# st = time.time()\n# pr = m6.predict(test50)\n# ed = time.time()\n# print(\"Single model time for 50 data: \", (ed - st))\n\n# start50 = time.time()\n# ens = [model.predict(test50) for model in models]\n# av = np.sum(ens, axis=0)/len(models)\n# end50 = time.time()\n# print(\"ensemble time for 50 data: \", (end50 - start50))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:03:23.237312Z","iopub.status.idle":"2022-01-30T23:03:23.237886Z","shell.execute_reply.started":"2022-01-30T23:03:23.237631Z","shell.execute_reply":"2022-01-30T23:03:23.237653Z"},"trusted":true},"execution_count":null,"outputs":[]}]}