{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-08-10T05:31:54.171808Z","iopub.status.busy":"2020-08-10T05:31:54.171011Z","iopub.status.idle":"2020-08-10T05:31:55.359087Z","shell.execute_reply":"2020-08-10T05:31:55.358383Z"},"papermill":{"duration":1.228622,"end_time":"2020-08-10T05:31:55.359241","exception":false,"start_time":"2020-08-10T05:31:54.130619","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nimport random\nimport re\nimport math\nimport time\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2020-08-10T05:31:55.436Z","iopub.status.busy":"2020-08-10T05:31:55.43505Z","iopub.status.idle":"2020-08-10T05:31:55.548207Z","shell.execute_reply":"2020-08-10T05:31:55.547361Z"},"papermill":{"duration":0.156822,"end_time":"2020-08-10T05:31:55.548339","exception":false,"start_time":"2020-08-10T05:31:55.391517","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/siim-isic-melanoma-classification/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/siim-isic-melanoma-classification/test.csv\")\nsample = pd.read_csv(\"/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:31:55.629131Z","iopub.status.busy":"2020-08-10T05:31:55.628274Z","iopub.status.idle":"2020-08-10T05:31:55.643696Z","shell.execute_reply":"2020-08-10T05:31:55.644357Z"},"papermill":{"duration":0.064414,"end_time":"2020-08-10T05:31:55.644547","exception":false,"start_time":"2020-08-10T05:31:55.580133","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:31:55.725127Z","iopub.status.busy":"2020-08-10T05:31:55.72416Z","iopub.status.idle":"2020-08-10T05:32:07.134627Z","shell.execute_reply":"2020-08-10T05:32:07.135217Z"},"papermill":{"duration":11.453381,"end_time":"2020-08-10T05:32:07.135392","exception":false,"start_time":"2020-08-10T05:31:55.682011","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:07.207979Z","iopub.status.busy":"2020-08-10T05:32:07.206891Z","iopub.status.idle":"2020-08-10T05:32:13.447157Z","shell.execute_reply":"2020-08-10T05:32:13.447729Z"},"papermill":{"duration":6.280007,"end_time":"2020-08-10T05:32:13.447944","exception":false,"start_time":"2020-08-10T05:32:07.167937","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\nfrom kaggle_datasets import KaggleDatasets","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:13.52444Z","iopub.status.busy":"2020-08-10T05:32:13.523688Z","iopub.status.idle":"2020-08-10T05:32:19.201177Z","shell.execute_reply":"2020-08-10T05:32:19.200329Z"},"papermill":{"duration":5.720363,"end_time":"2020-08-10T05:32:19.201314","exception":false,"start_time":"2020-08-10T05:32:13.480951","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-512x512')\nGCS_PATH2 = KaggleDatasets().get_gcs_path('malignant-v2-512x512')\nGCS_PATH3 = KaggleDatasets().get_gcs_path('isic2019-512x512')\nfilenames_train1 = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\nfilenames_train2 = tf.io.gfile.glob(GCS_PATH2 + '/train%.2i*.tfrec'%(2*x) for x in range(25))\nfilenames_train3 = tf.io.gfile.glob(GCS_PATH3 + '/train%.2i*.tfrec'%(2*x) for x in range(25))\nfilenames_test = np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec'))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:19.272483Z","iopub.status.busy":"2020-08-10T05:32:19.271406Z","iopub.status.idle":"2020-08-10T05:32:19.275456Z","shell.execute_reply":"2020-08-10T05:32:19.274792Z"},"papermill":{"duration":0.041788,"end_time":"2020-08-10T05:32:19.275592","exception":false,"start_time":"2020-08-10T05:32:19.233804","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"filenames_train = np.array(filenames_train1+filenames_train2+filenames_train3)\nnp.random.shuffle(filenames_train)\nnp.random.shuffle(filenames_train)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:19.446635Z","iopub.status.busy":"2020-08-10T05:32:19.349109Z","iopub.status.idle":"2020-08-10T05:32:23.635357Z","shell.execute_reply":"2020-08-10T05:32:23.634472Z"},"papermill":{"duration":4.327471,"end_time":"2020-08-10T05:32:23.635524","exception":false,"start_time":"2020-08-10T05:32:19.308053","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:23.708482Z","iopub.status.busy":"2020-08-10T05:32:23.707772Z","iopub.status.idle":"2020-08-10T05:32:23.71152Z","shell.execute_reply":"2020-08-10T05:32:23.710718Z"},"papermill":{"duration":0.042264,"end_time":"2020-08-10T05:32:23.711678","exception":false,"start_time":"2020-08-10T05:32:23.669414","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:23.79659Z","iopub.status.busy":"2020-08-10T05:32:23.795422Z","iopub.status.idle":"2020-08-10T05:32:23.798866Z","shell.execute_reply":"2020-08-10T05:32:23.798083Z"},"papermill":{"duration":0.049946,"end_time":"2020-08-10T05:32:23.799009","exception":false,"start_time":"2020-08-10T05:32:23.749063","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"cfg = dict(\n           batch_size=32,\n           img_size=512,\n    \n           lr_start=0.000005,\n           lr_max=0.00000125,\n           lr_min=0.000001,\n           lr_rampup=5,\n           lr_sustain=0,\n           lr_decay=0.8,\n           epochs=12,\n    \n           transform_prob=1.0,\n           rot=180.0,\n           shr=2.0,\n           hzoom=8.0,\n           wzoom=8.0,\n           hshift=8.0,\n           wshift=8.0,\n    \n           optimizer='adam',\n           label_smooth_fac=0.05,\n           tta_steps=20\n            \n        )","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:23.885422Z","iopub.status.busy":"2020-08-10T05:32:23.875984Z","iopub.status.idle":"2020-08-10T05:32:23.889308Z","shell.execute_reply":"2020-08-10T05:32:23.888562Z"},"papermill":{"duration":0.057032,"end_time":"2020-08-10T05:32:23.889454","exception":false,"start_time":"2020-08-10T05:32:23.832422","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n\n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1], dtype='float32')\n    zero = tf.constant([0], dtype='float32')\n    rotation_matrix = tf.reshape(\n        tf.concat([c1, s1, zero, -s1, c1, zero, zero, zero, one], axis=0),\n        [3, 3])\n\n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(\n        tf.concat([one, s2, zero, zero, c2, zero, zero, zero, one], axis=0),\n        [3, 3])\n\n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape(\n        tf.concat([\n            one / height_zoom, zero, zero, zero, one / width_zoom, zero, zero,\n            zero, one\n        ],axis=0), [3, 3])\n\n    # SHIFT MATRIX\n    shift_matrix = tf.reshape(\n        tf.concat(\n            [one, zero, height_shift, zero, one, width_shift, zero, zero, one],\n            axis=0), [3, 3])\n\n    return K.dot(K.dot(rotation_matrix, shear_matrix),\n                 K.dot(zoom_matrix, shift_matrix))\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:23.97797Z","iopub.status.busy":"2020-08-10T05:32:23.967627Z","iopub.status.idle":"2020-08-10T05:32:23.981957Z","shell.execute_reply":"2020-08-10T05:32:23.981271Z"},"papermill":{"duration":0.056512,"end_time":"2020-08-10T05:32:23.982101","exception":false,"start_time":"2020-08-10T05:32:23.925589","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def transform(image, cfg):\n    \n    DIM = cfg['img_size']\n    XDIM = DIM % 2  # fix for size 331\n\n    rot = cfg['rot'] * tf.random.normal([1], dtype='float32')\n    shr = cfg['shr'] * tf.random.normal([1], dtype='float32')\n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / cfg['hzoom']\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / cfg['wzoom']\n    h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')\n    w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')\n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot, shr, h_zoom, w_zoom, h_shift, w_shift)\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat(tf.range(DIM // 2, -DIM // 2, -1), DIM)\n    y = tf.tile(tf.range(-DIM // 2, DIM // 2), [DIM])\n    z = tf.ones([DIM * DIM], dtype='int32')\n    idx = tf.stack([x, y, z])\n\n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM // 2 + XDIM + 1, DIM // 2)\n\n    # FIND ORIGIN PIXEL VALUES\n    idx3 = tf.stack([DIM // 2 - idx2[0, ], DIM // 2 - 1 + idx2[1, ]])\n    d = tf.gather_nd(image, tf.transpose(idx3))\n\n    return tf.reshape(d, [DIM, DIM, 3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dropout(image, DIM=512, PROBABILITY = 0.75, CT = 8, SZ = 0.2):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(CT==0)|(SZ==0): return image\n    \n    for k in range(CT):\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        # COMPUTE SQUARE \n        WIDTH = tf.cast( SZ*DIM,tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # DROPOUT IMAGE\n        one = image[ya:yb,0:xa,:]\n        two = tf.zeros([yb-ya,xb-xa,3]) \n        three = image[ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM,DIM,3])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:24.061266Z","iopub.status.busy":"2020-08-10T05:32:24.059966Z","iopub.status.idle":"2020-08-10T05:32:24.062824Z","shell.execute_reply":"2020-08-10T05:32:24.063492Z"},"papermill":{"duration":0.04792,"end_time":"2020-08-10T05:32:24.063675","exception":false,"start_time":"2020-08-10T05:32:24.015755","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def prepare_image(img, cfg=None,droprate=0.5,dropct=8,dropsize=0.2):\n    \n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img, [cfg['img_size'], cfg['img_size']],\n                          antialias=True)\n    img = tf.cast(img, tf.float32) / 255.0\n\n    if cfg['transform_prob'] > tf.random.uniform([1], minval=0, maxval=1):\n        img = transform(img, cfg)\n    \n    if (tf.random.uniform([1], minval=0, maxval=1) > 0.5) & (droprate!=0)&(dropct!=0)&(dropsize!=0):\n        img = dropout(img, DIM=512, PROBABILITY=droprate, CT=dropct, SZ=dropsize)\n\n    img = tf.image.random_flip_left_right(img)\n    img = tf.image.random_saturation(img, 0.7, 1.3)\n    img = tf.image.random_contrast(img, 0.8, 1.2)\n    img = tf.image.random_brightness(img, 0.1)\n\n    return img","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:24.138461Z","iopub.status.busy":"2020-08-10T05:32:24.137535Z","iopub.status.idle":"2020-08-10T05:32:24.142069Z","shell.execute_reply":"2020-08-10T05:32:24.141419Z"},"papermill":{"duration":0.045293,"end_time":"2020-08-10T05:32:24.142206","exception":false,"start_time":"2020-08-10T05:32:24.096913","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'image_name': tf.io.FixedLenFeature([], tf.string),\n        'target': tf.io.FixedLenFeature([], tf.int64)\n    }\n\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    return example['image'], example['target']\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:24.220328Z","iopub.status.busy":"2020-08-10T05:32:24.219315Z","iopub.status.idle":"2020-08-10T05:32:24.221739Z","shell.execute_reply":"2020-08-10T05:32:24.222262Z"},"papermill":{"duration":0.043319,"end_time":"2020-08-10T05:32:24.22242","exception":false,"start_time":"2020-08-10T05:32:24.179101","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'image_name': tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    return example['image'], example['image_name']\n\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:24.296683Z","iopub.status.busy":"2020-08-10T05:32:24.295856Z","iopub.status.idle":"2020-08-10T05:32:24.299338Z","shell.execute_reply":"2020-08-10T05:32:24.298681Z"},"papermill":{"duration":0.043449,"end_time":"2020-08-10T05:32:24.299473","exception":false,"start_time":"2020-08-10T05:32:24.256024","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [\n        int(re.compile(r'-([0-9]*)\\.').search(filename).group(1))\n        for filename in filenames\n    ]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:24.376282Z","iopub.status.busy":"2020-08-10T05:32:24.374541Z","iopub.status.idle":"2020-08-10T05:32:24.379634Z","shell.execute_reply":"2020-08-10T05:32:24.379006Z"},"papermill":{"duration":0.046945,"end_time":"2020-08-10T05:32:24.379782","exception":false,"start_time":"2020-08-10T05:32:24.332837","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def getTrainDataset(files, cfg):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n\n    opt = tf.data.Options()\n    opt.experimental_deterministic = False\n    ds = ds.with_options(opt)\n\n    ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    ds = ds.repeat()\n    \n    ds = ds.shuffle(2048)\n    ds = ds.map(lambda img, label:\n                (prepare_image(img, cfg=cfg), label),\n                num_parallel_calls=AUTO)\n    ds = ds.batch(cfg['batch_size'] * strategy.num_replicas_in_sync)\n    ds = ds.prefetch(AUTO)\n    return ds\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:24.457242Z","iopub.status.busy":"2020-08-10T05:32:24.456128Z","iopub.status.idle":"2020-08-10T05:32:24.459511Z","shell.execute_reply":"2020-08-10T05:32:24.458876Z"},"papermill":{"duration":0.046505,"end_time":"2020-08-10T05:32:24.459692","exception":false,"start_time":"2020-08-10T05:32:24.413187","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"\ndef getTestDataset(files, cfg, augment=False, repeat=False):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    if repeat:\n        ds = ds.repeat()\n    ds = ds.map(read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    ds = ds.map(lambda img, idnum:\n                (prepare_image(img, cfg=cfg), idnum),\n                num_parallel_calls=AUTO)\n    ds = ds.batch(cfg['batch_size'] * strategy.num_replicas_in_sync)\n    ds = ds.prefetch(AUTO)\n    return ds","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:24.537977Z","iopub.status.busy":"2020-08-10T05:32:24.537111Z","iopub.status.idle":"2020-08-10T05:32:24.540574Z","shell.execute_reply":"2020-08-10T05:32:24.539953Z"},"papermill":{"duration":0.047325,"end_time":"2020-08-10T05:32:24.540727","exception":false,"start_time":"2020-08-10T05:32:24.493402","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def getLearnRateCallback(cfg):\n    \n    lr_start = cfg['lr_start']\n    lr_max = cfg['lr_max'] * strategy.num_replicas_in_sync * cfg['batch_size']\n    lr_min = cfg['lr_min']\n    lr_rampup = cfg['lr_rampup']\n    lr_sustain = cfg['lr_sustain']\n    lr_decay = cfg['lr_decay']\n\n    def lrfn(epoch):\n        if epoch < lr_rampup:\n            lr = (lr_max - lr_start) / lr_rampup * epoch + lr_start\n        elif epoch < lr_rampup + lr_sustain:\n            lr = lr_max\n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_rampup -\n                                                lr_sustain) + lr_min\n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:32:24.627322Z","iopub.status.busy":"2020-08-10T05:32:24.626506Z","iopub.status.idle":"2020-08-10T05:34:47.277666Z","shell.execute_reply":"2020-08-10T05:34:47.276959Z"},"papermill":{"duration":142.703751,"end_time":"2020-08-10T05:34:47.277813","exception":false,"start_time":"2020-08-10T05:32:24.574062","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"with strategy.scope():\n    model_input = tf.keras.Input(shape=(cfg['img_size'], cfg['img_size'], 3),\n                                 name='img_input')\n\n    dummy = tf.keras.layers.Lambda(lambda x: x)(model_input)\n\n    outputs = []\n\n    x = efn.EfficientNetB3(include_top=False,\n                           weights='imagenet',\n                           input_shape=(cfg['img_size'], cfg['img_size'], 3),\n                           pooling='avg')(dummy)\n    x = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n    outputs.append(x)\n\n    x = efn.EfficientNetB4(include_top=False,\n                           weights='imagenet',\n                           input_shape=(cfg['img_size'], cfg['img_size'], 3),\n                           pooling='avg')(dummy)\n    x = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n    outputs.append(x)\n\n    x = efn.EfficientNetB5(include_top=False,\n                           weights='imagenet',\n                           input_shape=(cfg['img_size'], cfg['img_size'], 3),\n                           pooling='avg')(dummy)\n    x = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n    outputs.append(x)\n    \n    model = tf.keras.Model(model_input, outputs, name='aNetwork')\n\n    model.compile(optimizer=cfg['optimizer'],\n                  loss=[\n                      tf.keras.losses.BinaryCrossentropy(\n                          label_smoothing=cfg['label_smooth_fac']),\n                      tf.keras.losses.BinaryCrossentropy(\n                          label_smoothing=cfg['label_smooth_fac']),\n                      tf.keras.losses.BinaryCrossentropy(\n                          label_smoothing=cfg['label_smooth_fac'])\n                  ],\n                  metrics=[tf.keras.metrics.AUC(name='auc')])\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:34:47.786504Z","iopub.status.busy":"2020-08-10T05:34:47.785556Z","iopub.status.idle":"2020-08-10T05:34:49.449427Z","shell.execute_reply":"2020-08-10T05:34:49.450015Z"},"papermill":{"duration":1.752218,"end_time":"2020-08-10T05:34:49.450195","exception":false,"start_time":"2020-08-10T05:34:47.697977","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"ds_train = getTrainDataset(filenames_train, cfg).map(lambda img, label: (img, (label, label, label)))\nstepsTrain = count_data_items(filenames_train) /(cfg['batch_size'] * strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_auc', mode = 'max', patience = 4, \n#                                                 verbose = 1, min_delta = 0.0001, restore_best_weights = True)\ncallbacks = [getLearnRateCallback(cfg)]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T05:34:49.626336Z","iopub.status.busy":"2020-08-10T05:34:49.621208Z","iopub.status.idle":"2020-08-10T06:25:44.046069Z","shell.execute_reply":"2020-08-10T06:25:44.045383Z"},"papermill":{"duration":3054.517163,"end_time":"2020-08-10T06:25:44.046222","exception":false,"start_time":"2020-08-10T05:34:49.529059","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"history = model.fit(ds_train,\n                    validation_data  = None,\n                    verbose=1,\n                    steps_per_epoch  = stepsTrain,\n                    validation_steps = 0,\n                    epochs=14,\n                    callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T06:25:45.703482Z","iopub.status.busy":"2020-08-10T06:25:45.702696Z","iopub.status.idle":"2020-08-10T06:31:38.096299Z","shell.execute_reply":"2020-08-10T06:31:38.095488Z"},"papermill":{"duration":353.229199,"end_time":"2020-08-10T06:31:38.096468","exception":false,"start_time":"2020-08-10T06:25:44.867269","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"steps = count_data_items(filenames_test) / (cfg['batch_size'] * strategy.num_replicas_in_sync)\nz = np.zeros((cfg['batch_size'] * strategy.num_replicas_in_sync))\n\nds_testAug = getTestDataset(filenames_test, cfg, augment=True,\n    repeat=True).map(lambda img, label: (img, (z, z, z)))\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T06:31:40.518115Z","iopub.status.busy":"2020-08-10T06:31:40.517208Z","iopub.status.idle":"2020-08-10T06:31:40.520559Z","shell.execute_reply":"2020-08-10T06:31:40.521141Z"},"papermill":{"duration":1.164019,"end_time":"2020-08-10T06:31:40.521307","exception":false,"start_time":"2020-08-10T06:31:39.357288","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"probs = model.predict(ds_testAug, verbose=1, steps=steps * cfg['tta_steps'])\nprobs = np.stack(probs)\nprobs = probs[:, :count_data_items(filenames_test) * cfg['tta_steps']]\nprobs = np.stack(np.split(probs, cfg['tta_steps'], axis=1), axis=1)\nprobs = np.mean(probs, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T06:31:47.676731Z","iopub.status.busy":"2020-08-10T06:31:47.675942Z","iopub.status.idle":"2020-08-10T06:32:20.449627Z","shell.execute_reply":"2020-08-10T06:32:20.448754Z"},"papermill":{"duration":33.99887,"end_time":"2020-08-10T06:32:20.449821","exception":false,"start_time":"2020-08-10T06:31:46.450951","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"y_test_sorted = np.zeros((3, probs.shape[1]))\ntest = test.reset_index()\ntest = test.set_index('image_name')\n\ni = 0\nds_test = getTestDataset(filenames_test, cfg)\nfor img, imgid in tqdm(iter(ds_test.unbatch())):\n    imgid = imgid.numpy().decode('utf-8')\n    y_test_sorted[:, test.loc[imgid]['index']] = probs[:, i, 0]\n    i += 1\n\nfor i in range(y_test_sorted.shape[0]):\n    submission = sample\n    submission['target'] = y_test_sorted[i]\n    submission.to_csv('submission_model_%s.csv' % i, index=False)\n\nsubmission = sample\nsubmission['target'] = np.mean(y_test_sorted, axis=0)\nsubmission.to_csv('blended_effnets.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-10T06:32:23.050687Z","iopub.status.busy":"2020-08-10T06:32:23.04991Z","iopub.status.idle":"2020-08-10T06:32:25.666141Z","shell.execute_reply":"2020-08-10T06:32:25.665444Z"},"papermill":{"duration":3.889188,"end_time":"2020-08-10T06:32:25.666285","exception":false,"start_time":"2020-08-10T06:32:21.777097","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"!gzip submission_model_0.csv\n!gzip submission_model_1.csv\n!gzip submission_model_2.csv\n!gzip blended_effnets.csv","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":1.272042,"end_time":"2020-08-10T06:32:39.328228","exception":false,"start_time":"2020-08-10T06:32:38.056186","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}