{"cells":[{"metadata":{},"cell_type":"markdown","source":"### References\n* Ensembling \n[MinMax highest public LB=.9619](https://www.kaggle.com/paklau9/minmax-highest-public-lb-9619/data)\n([Pak Lau](https://www.kaggle.com/paklau9))\n* High Performin base line model \n[Triple Stratified KFold with TFRecords](https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords)\n([Chris Deotte](https://www.kaggle.com/cdeotte))","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Import libraries","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport re, math\nimport datetime\n\nfrom IPython.core.display import display, HTML\nimport matplotlib.pyplot as plt\n\n!pip install -q efficientnet\nimport efficientnet.tfkeras as efn\n\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.layers as L\nfrom tensorflow.keras import Sequential, Model\nfrom tensorflow.keras.layers import Conv2D, Dense, Input, Flatten, AveragePooling2D, GlobalAveragePooling2D, BatchNormalization, Dropout, Activation\n\nfrom kaggle_datasets import KaggleDatasets\n\nfrom sklearn.model_selection import train_test_split\nimport pickle\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Detect hardware, return appropriate distribution strategy","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_strategy():\n    gpu = \"\"\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n        os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n        gpu = tf.config.list_physical_devices(\"GPU\")\n        if len(gpu) == 1:\n            print('Running on GPU ', gpu)\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        tf.config.experimental_connect_to_cluster(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        GCS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')\n    elif len(gpu) == 1:\n        strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n        tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\":True})\n        GCS_PATH = \"/kaggle/input/siim-isic-melanoma-classification/\"\n    else:\n        strategy = tf.distribute.get_strategy()\n        GCS_PATH = \"/kaggle/input/siim-isic-melanoma-classification/\"\n\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n    base_dir = \"/kaggle/input/siim-isic-melanoma-classification/\"\n    return strategy, GCS_PATH, base_dir\n\nstrategy,GCS_PATH, base_dir = get_strategy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Settings","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"VERSION = 24\nKEY_CHANGES = \"Code cleanup, \"\n\nAUTO = tf.data.experimental.AUTOTUNE\n\nROT_ = 45.0\nSHR_ = 2.0\nHZOOM_ = 10.0\nWZOOM_ = 10.0\nHSHIFT_ = 10.0\nWSHIFT_ = 10.0\n\nDIM=384\nTFREC_DIM=384\nSEED = 42\n\nTFREC_GCS_DIR = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(TFREC_DIM,TFREC_DIM))\nPRV_TFREC_GCS_DIR = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(TFREC_DIM,TFREC_DIM))\n\n\nBLN_TRAIN_MODEL=False\n\nREPLICAS = strategy.num_replicas_in_sync\nBATCH_SIZE = 16 * REPLICAS\nEPOCHS=12\nKFOLD_SPLITS=5\n\nTTA=25","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Write log\nImportant to document results of various experiments as we go along","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"log_sno = 1\nlog_start = datetime.datetime.now()\n\n    \ndef write_log(data_type, data):\n    global log_sno\n    filename = \"log_melanoma_classification.txt\"\n    prv_filename = \"../input/amazingly-fast-kernel/\" + filename\n    if (not os.path.exists(filename)) and os.path.exists(prv_filename):\n        os.popen('cp ' + prv_filename + ' ' + filename)\n        \n    f = open(filename,\"a\")\n   \n    log_cur = datetime.datetime.now()\n    delta = log_cur - log_start\n    f.write(\"{},{},{},{},{},{}\".format(VERSION, log_sno, log_start, data_type, data, delta.total_seconds()))\n    log_sno = log_sno + 1\n    \n    f.close()\nwrite_log(\"KEY_CHANGES\", KEY_CHANGES)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load and perform basic EDA on training data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = pd.read_csv(base_dir + \"train.csv\")\nprint(display(HTML(train_data.head(1).to_html())))\nprint(train_data[\"target\"].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data[\"anatom_site_general_challenge\"].fillna(\"Unknown\", inplace=True)\ngroup_data = train_data.groupby([\"anatom_site_general_challenge\"])[\"benign_malignant\"].value_counts().unstack(-1)\ngroup_data[\"perc_malignant\"] = np.round((group_data[\"malignant\"] * 100) /(group_data[\"benign\"] + group_data[\"malignant\"]),2)\ngroup_data.sort_values(\"perc_malignant\", inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nfix, ax = plt.subplots(1,4, figsize=(13,3))\nax[0].hist(\"age_approx\", data=train_data[train_data[\"target\"]==0], bins=100)\n\nax[0].hist(\"age_approx\", data=train_data[train_data[\"target\"]==0], bins=100)\nax[0].set_title(\"Age histogram-Benign\");\nax[1].hist(\"age_approx\", data=train_data[train_data[\"target\"]==1], bins=100);\nax[1].set_title(\"Age histogram-Malignant\");\n\nsex_data = train_data[[\"sex\",\"target\",\"image_name\"]].groupby([\"sex\",\"target\"]).count().reset_index()\nsns.barplot(\"sex\",\"image_name\", data=sex_data, hue=\"target\" , ax=ax[2]);\nax[2].set_title(\"Sex bar graph\");\n\ngroup_data = train_data[[\"sex\",\"target\",\"image_name\"]].groupby([\"sex\",\"target\"]).count().unstack(-1)\nsum_data = group_data.sum(axis=1).values\ngroup_data[\"total\"] = list(sum_data)\ngroup_data.iloc[:,1] = list(np.round(group_data.iloc[:,1].values * 100 / sum_data,2))\ngroup_data.columns = [\"benign_count\",\"malignant_perc\",\"total_count\"]\ngroup_data.reset_index(inplace=True)\nsns.barplot(\"sex\",\"malignant_perc\", data=group_data,  ax=ax[3]);\n\nplt.tight_layout();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fix, ax = plt.subplots(1,2, figsize=(13,4))\nsex_data = train_data[[\"anatom_site_general_challenge\",\"target\",\"image_name\"]].groupby([\"anatom_site_general_challenge\",\"target\"]).count().reset_index()\nsns.barplot(\"anatom_site_general_challenge\",\"image_name\", data=sex_data, hue=\"target\" , ax=ax[0]);\nax[0].set_title(\"Anatomy bar graph\");\n\ngroup_data = train_data[[\"anatom_site_general_challenge\",\"target\",\"image_name\"]].groupby([\"anatom_site_general_challenge\",\"target\"]).count().unstack(-1)\nsum_data = group_data.sum(axis=1).values\ngroup_data[\"total\"] = list(sum_data)\ngroup_data.iloc[:,1] = list(np.round(group_data.iloc[:,1].values * 100 / sum_data,2))\ngroup_data.columns = [\"benign_count\",\"malignant_perc\",\"total_count\"]\ngroup_data.reset_index(inplace=True)\nsns.barplot(\"anatom_site_general_challenge\",\"malignant_perc\", data=group_data,  ax=ax[1]);\n\nplt.setp(ax[0].xaxis.get_majorticklabels(), rotation=45)\nplt.setp(ax[1].xaxis.get_majorticklabels(), rotation=45)\nplt.tight_layout();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Augmentation\nThis plays a major role in Image classification. I think this is how we can make our algo think better than what humans do.\nLifting straight away from [Chris Deotte Notebook](https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords)\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef aug(image, DIM=DIM):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    img = tf.reshape(d,[DIM, DIM,3])\n    img = tf.image.random_flip_left_right(img)\n    #img = tf.image.random_hue(img, 0.01)\n    img = tf.image.random_saturation(img, 0.7, 1.3)\n    img = tf.image.random_contrast(img, 0.8, 1.2)\n    img = tf.image.random_brightness(img, 0.1)\n    return img\n\ndef aug_img_label(img, label):\n    img = aug(img)\n    return img, label\n\ndef aug_img(img):\n    img = aug(img)\n    return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Methods to parse tfrecords","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef parse_rec_train(data):           \n    feature_set = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'target': tf.io.FixedLenFeature([], tf.int64)\n    }\n    features = tf.io.parse_single_example(data, features= feature_set )\n    return features\n\ndef parse_rec_validate(data):           \n    feature_set = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'target': tf.io.FixedLenFeature([], tf.int64),\n        'image_name': tf.io.FixedLenFeature([], tf.string)\n    }\n    features = tf.io.parse_single_example(data, features= feature_set )\n    return features\n\ndef parse_rec_test(data):           \n    feature_set = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'image_name': tf.io.FixedLenFeature([], tf.string)\n    }\n    features = tf.io.parse_single_example(data, features= feature_set )\n    return features\n\n\ndef process_img(img):\n    img = tf.image.decode_image(img)\n    img = tf.ensure_shape(img, (TFREC_DIM,TFREC_DIM,3))\n    img = tf.image.resize(img, [DIM,DIM])\n    img = float(img)/255.00\n    return tf.cast(img, tf.float32)\n\ndef get_img_label(features):\n    target = features[\"target\"]\n    features.pop(\"target\")\n    img = process_img(features[\"image\"])\n    return img, target\n\n\ndef get_img(features, label=None):\n    img = process_img(features[\"image\"])\n    return img\n\ndef get_img_and_name(features, label=None):\n    img = process_img(features[\"image\"])\n    image_name = features[\"image_name\"]\n    return img, image_name\n\ndef get_img_name(features):\n    image_name = features[\"image_name\"]\n    return image_name","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Familiarise with TFRec","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tfrec_files_train_all = np.sort(np.array(tf.io.gfile.glob(TFREC_GCS_DIR + '/train*.tfrec')))\ntfrec_files_train_2019 = np.sort(np.array(tf.io.gfile.glob(PRV_TFREC_GCS_DIR + '/train%.2i*.tfrec'%x for x in range(1, 31, 2))))\ntfrec_files_train_2018 = np.sort(np.array(tf.io.gfile.glob(PRV_TFREC_GCS_DIR + '/train%.2i*.tfrec'%x for x in range(0, 30, 2))))\ntfrec_files_test  = np.sort(np.array(tf.io.gfile.glob(TFREC_GCS_DIR + '/test*.tfrec')))\ndataset_peek = tf.data.TFRecordDataset(tfrec_files_train_all)\nfor data in dataset_peek.take(1):\n    example = tf.train.Example()\n    example.ParseFromString(data.numpy())\n    print(str(example)[-600:])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Peek a few images in the TFRec dataset","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def get_img_list(dataset):\n    arr_img = []\n    for img, label in dataset.take(8):\n        arr_img.append(img)\n    return arr_img\n    \ndef show_img(img_list):\n    row=3; col=8;\n    plt.figure(figsize=(20,row*12/col))\n    x = 1\n    for k in range(3):\n        if k == 0:\n            for img in img_list:\n                plt.subplot(row,col,x)\n                plt.imshow(img)\n                x = x + 1\n        else:\n            for img in img_list:\n                img = aug(img)\n                plt.subplot(row,col,x)\n                plt.imshow(img)\n                x = x + 1\n\ndataset_train = dataset_peek.map(parse_rec_train).map(get_img_label)\nshow_img(get_img_list(dataset_train)) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define Model","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"EFNS = [efn.EfficientNetB0, efn.EfficientNetB4, efn.EfficientNetB6, efn.EfficientNetB6, efn.EfficientNetB6]\ndef build_model(fold):\n    if BLN_TRAIN_MODEL:\n        if 1==2:\n            base_model = efn.EfficientNetB0(weights='imagenet', include_top=False, input_shape=(DIM,DIM,3))\n            x = base_model.output\n            x = GlobalAveragePooling2D()(x)\n            x = Dense(1024)(x)\n            x = tf.keras.layers.BatchNormalization()(x)\n            x = tf.keras.layers.Activation('relu')(x)\n            x = tf.keras.layers.Dropout(0.02)(x)\n            predictions = Dense(1, activation='sigmoid')(x)\n            model = Model(inputs=base_model.input, outputs=predictions)\n        \n        inp = tf.keras.layers.Input(shape=(DIM,DIM,3))\n        base = EFNS[fold](input_shape=(DIM,DIM,3),weights='imagenet',include_top=False)\n        x = base(inp)\n        x = tf.keras.layers.GlobalAveragePooling2D()(x)\n        x = tf.keras.layers.Dense(1,activation='sigmoid')(x)\n        model = tf.keras.Model(inputs=inp,outputs=x)\n        opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n        loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05) \n        model.compile(optimizer=opt,loss=loss,metrics=['AUC'])\n        return model\n    else:\n        model_path = \"/kaggle/input/model-data-pipeline-v18/model.h5\"\n        model = tf.keras.models.load_model(model_path)\n      \n        opt = tf.keras.optimizers.Adam(lr=0.001)\n        loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05, name='binary_crossentropy')\n        model.compile(optimizer=opt, loss=loss, metrics=['accuracy',tf.keras.metrics.AUC()])\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Callbacks: Learning Rate scheduler, Early Stopping Callback and Best Model Save","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr_callback(batch_size):\n    lr_start   = 0.000005\n    lr_max     = 0.000020 * strategy.num_replicas_in_sync\n    lr_min     = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback\n\n\nlr = get_lr_callback(BATCH_SIZE)\n\nes = tf.keras.callbacks.EarlyStopping(monitor='val_auc', mode='max', verbose=1, patience=3)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_test = tf.data.TFRecordDataset(tfrec_files_test).map(parse_rec_test, num_parallel_calls=AUTO)\nimg_name_list_test = list(dataset_test.map(get_img_name, num_parallel_calls=AUTO).as_numpy_iterator())\ndataset_test_raw = dataset_test.map(get_img, num_parallel_calls=AUTO)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":false},"cell_type":"code","source":"arr_test_pred = []\narr_test_pred_min = []\narr_test_pred_max = []\n\nif BLN_TRAIN_MODEL:\n    from sklearn.model_selection import KFold\n    kf = KFold(n_splits=KFOLD_SPLITS, shuffle=True,random_state=SEED)\n    for fold , (train_idx, val_idx) in enumerate(kf.split(range(len(tfrec_files_train_all)))):\n        print(train_idx, val_idx)\n        tfrec_files_train = np.array(tfrec_files_train_all)[list(train_idx)]\n        if fold >= 3:\n            additional_files = np.array(tfrec_files_train_2019)[list(train_idx)]\n        else:\n            additional_files = np.array(tfrec_files_train_2018)[list(train_idx)]\n        tfrec_files_train = np.concatenate([tfrec_files_train, additional_files])\n        np.random.shuffle(tfrec_files_train)\n        tfrec_files_valid = np.array(tfrec_files_train_all)[list(val_idx)]\n        print(\"num_steps:\", count_data_items(tfrec_files_train)/BATCH_SIZE)\n        temp_dataset = tf.data.TFRecordDataset(tfrec_files_train).shuffle(1024*8).map(parse_rec_train, num_parallel_calls=AUTO).map(get_img_label, num_parallel_calls=AUTO)\n        temp_dataset = temp_dataset.map(aug_img_label, num_parallel_calls=AUTO)\n        dataset_train = temp_dataset.batch(BATCH_SIZE, drop_remainder=True).prefetch(tf.data.experimental.AUTOTUNE)\n        dataset_valid = tf.data.TFRecordDataset(tfrec_files_valid).map(parse_rec_train, num_parallel_calls=AUTO).map(get_img_label, num_parallel_calls=AUTO).batch(BATCH_SIZE, drop_remainder=True).prefetch(tf.data.experimental.AUTOTUNE)\n        K.clear_session()\n        with strategy.scope():\n            model = build_model(fold)\n            \n        sv = tf.keras.callbacks.ModelCheckpoint('fold-%i.h5'%fold, monitor='val_auc', verbose=0, save_best_only=True,\n            save_weights_only=True, mode='max', save_freq='epoch')\n        model.fit(dataset_train, epochs=EPOCHS, verbose=1, callbacks=[sv, lr, es], validation_data = dataset_valid)    #steps_per_epoch=steps_per_epoch, \n        model.load_weights('fold-%i.h5'%fold)\n        arr_pred = []\n        for i in range(TTA):\n            dataset_test = dataset_test_raw.map(aug_img, num_parallel_calls=AUTO).batch(BATCH_SIZE).prefetch(tf.data.experimental.AUTOTUNE)\n            arr_pred.append( model.predict(dataset_test, verbose=1) )\n        arr_test_pred.append(np.stack(arr_pred, axis=1).mean(axis=1))\n        filename = 'test_pred-%i.pkl'%fold\n        outfile = open(filename,'wb')\n        pickle.dump(arr_pred,outfile)\n        outfile.close()\nelse:\n    for fold in range(KFOLD_SPLITS):\n        filename = '../input/dsamazinglyfastkernel09402/test_pred-%i.pkl'%fold\n        outfile = open(filename,'rb')\n        arr_pred = pickle.load(outfile)\n        arr_test_pred.append(np.stack(arr_pred, axis=1).mean(axis=1))\n        arr_test_pred_min.append(np.stack(arr_pred, axis=1).min(axis=1))\n        arr_test_pred_max.append(np.stack(arr_pred, axis=1).max(axis=1))\n        outfile.close()\n            ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Make predictions for test data and submit","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nfile_1 = pd.read_csv(\"../input/melanoma-submission-9402/submission.csv\") # this is the model with eb6 and data from 2019 in kfold on 384 size\nfile_2 = pd.read_csv(\"../input/amazingly-fast-kernel/submission.csv\") # this is combined model on 384 b0,b4, b6, data just current year\nfile_3 = pd.read_csv(\"../input/amazingly-fast-kernel-256/submission.csv\")\nfile_4 = pd.read_csv(\"../input/amazingly-fast-kernel-512/submission.csv\")\nfile_5 = pd.read_csv(\"../input/amazingly-fast-kernel-192/submission.csv\")\nfile_6 = pd.read_csv(\"../input/amazingly-fast-kernel-128/submission.csv\")\ndf = pd.DataFrame({\"image_name\":img_name_list_test})\ndf[\"image_name\"] = df[\"image_name\"].map(lambda x: x.decode(\"utf-8\"))\n\nfile_1 = pd.merge(df, file_1, on=\"image_name\")\nfile_2 = pd.merge(df, file_2, on=\"image_name\")\nfile_3 = pd.merge(df, file_3, on=\"image_name\")\nfile_4 = pd.merge(df, file_4, on=\"image_name\")\nfile_5 = pd.merge(df, file_5, on=\"image_name\")\nfile_6 = pd.merge(df, file_6, on=\"image_name\")\ndf[\"target_1\"] = list(file_1[\"target\"].values)\ndf[\"target_2\"] = list(file_2[\"target\"].values)\ndf[\"target_3\"] = list(file_3[\"target\"].values)\ndf[\"target_4\"] = list(file_4[\"target\"].values)\ndf[\"target_5\"] = list(arr_test_pred[0][:,0])\ndf[\"target_6\"] = list(arr_test_pred[1][:,0])\ndf[\"target_7\"] = list(arr_test_pred[2][:,0])\ndf[\"target_8\"] = list(arr_test_pred[3][:,0])\ndf[\"target_9\"] = list(arr_test_pred[4][:,0])\ndf[\"target_10\"] = list(file_5[\"target\"].values)\ndf[\"target_11\"] = list(file_6[\"target\"].values)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"meta = pd.read_csv(\"../input/meta-data-model/submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[\"target\"] = df.iloc[:,1:12].mean(axis=1)\ndf[\"target_min\"] = df.iloc[:,1:12].min(axis=1)\ndf[\"target_max\"] = df.iloc[:,1:12].max(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if 1==2:\n    cutoff_lo = 0.88\n    cutoff_hi = 0.11\n    df['target'] = np.where(np.all(df.iloc[:, 1:10] > cutoff_lo, axis=1),\n                                            df['target_max'],\n                                            np.where(np.all(df.iloc[:, 1:10] < cutoff_hi, axis=1),\n                                                     df['target_min'],\n                                                     df['target']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if 1==2:\n    meta = meta[[\"image_name\",\"target\"]]\n    meta.columns = [\"image_name\",\"target_meta\"]\n    df = df[[\"image_name\",\"target\"]]\n    df.columns = [\"image_name\",\"target_model\"]\n    df = pd.merge(df[[\"image_name\",\"target_model\"]], meta[[\"image_name\",\"target_meta\"]], on=\"image_name\")\n    df[\"target\"] = df.apply(lambda row: 0.9*row[\"target_model\"] + 0.1*row[\"target_meta\"], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#df[\"target\"] = df.apply(lambda row: row[\"target_model\"] if (row[\"target\"]<0.11 and row[\"target_model\"] < row[\"target\"]) else row[\"target\"], axis=1)\n#df[\"target\"] = df.apply(lambda row: row[\"target_model\"] if (row[\"target\"]>0.66 and row[\"target_model\"] > row[\"target\"]) else row[\"target\"], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndf[[\"image_name\",\"target\"]].to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df.sort_values(\"target\").tail(5))\nsns.distplot(df[\"target\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}