{"cells":[{"metadata":{},"cell_type":"markdown","source":"**MODEL**\n* external data\n* data balancing\n* stochastic weight averaging\n\n**UTILS**\n* time budget to skip the last folds in case we are risking to exceed 3 hours of computation","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install -q efficientnet\n!pip install -q git+https://github.com/AmedeoBiolatti/dsqol","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os, re, time, tqdm\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn import metrics, model_selection\n\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom tensorflow import keras \nfrom tensorflow.keras import backend as K\nfrom efficientnet import tfkeras as efnet\n\nfrom kaggle_datasets import KaggleDatasets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# my github for trivial but useful functions\nfrom dsqol.tf import imgaug\nfrom dsqol.tf.data import balance\nfrom dsqol.tf.utils import average\nfrom dsqol.tf import losses","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# USE DIFFERENT SEED FOR DIFFERENT STRATIFIED KFOLD\nSEED = 42\ntf.random.set_seed(SEED)\nnp.random.seed(SEED)\n\nTIME_BUDGET = 2.5 * 3600 # time budget allocated for the training (Kaggle has 3 hours max time)\n\n# \nFOLDS = 5\nINCLUDE_2019 = 0\nINCLUDE_2018 = 1\nINCLUDE_MALIGNANT = 0\n\n# DATA PARAMS\nIMG_READ_SIZE     = 384 # 512\nIMG_SIZE          = 384 # 512\nBALANCE_POS_RATIO = 0.08\n\n# MODEL PARAMS\nEFF_NET      = 6\n# loss and loss params\nLOSS_TYPE    = 'BCE' # 'BCE', 'FOCAL'\nLOSS_PARAMS  = dict(label_smoothing=0.05)\n\n# TRAINING PARAMS\nBATCH_SIZE  = 32\nEPOCHS      = 12\n# lr schedule\n\n# VALID AND TEST PARAMS\nTBM        = 6\nTTA        = 15\nVALID_FREQ = 1\nN_SWA      = 3\nSWA_DECAY  = 0.9","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = \"TPU\"\nprint(\"connecting to TPU...\")\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    print(\"Could not connect to TPU\")\n    tpu = None\nif tpu:\n    try:\n        print(\"initializing  TPU ...\")\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"TPU initialized\")\n    except _:\n        print(\"failed to initialize TPU\")\nelse:\n    DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \nAUTO              = tf.data.experimental.AUTOTUNE\nREPLICAS          = strategy.num_replicas_in_sync\nGLOBAL_BATCH_SIZE = BATCH_SIZE * REPLICAS\nprint(\"REPLICAS: %d\" % REPLICAS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/169139","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_READ_SIZE     = 512\nIMG_SIZE          = 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH1 = KaggleDatasets().get_gcs_path('melanoma-%ix%i' % (IMG_READ_SIZE, IMG_READ_SIZE))\nGCS_PATH2 = KaggleDatasets().get_gcs_path('isic2019-%ix%i' % (IMG_READ_SIZE, IMG_READ_SIZE))\nGCS_PATH3 = KaggleDatasets().get_gcs_path('malignant-v2-%ix%i' % (IMG_READ_SIZE, IMG_READ_SIZE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_base_train = pd.read_csv(\"../input/siim-isic-melanoma-classification/train.csv\")\ndf_base_test = pd.read_csv(\"../input/siim-isic-melanoma-classification/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_files = tf.io.gfile.glob(os.path.join(GCS_PATH1, \"train*.tfrec\"))\n#\nif INCLUDE_2019:\n    train_files += tf.io.gfile.glob([os.path.join(GCS_PATH2, \"train%.2i*.tfrec\" % i) for i in range(1, 30, 2)])\nif INCLUDE_2018:\n    train_files += tf.io.gfile.glob([os.path.join(GCS_PATH2, \"train%.2i*.tfrec\" % i) for i in range(0, 30, 2)])\n#\nif INCLUDE_MALIGNANT:\n    train_files += tf.io.gfile.glob([os.path.join(GCS_PATH3, \"train%.2i*.tfrec\" % i) for i in range(15, 30, 1)])\nprint(\"%d train files found\" % len(train_files))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_files = tf.io.gfile.glob(os.path.join(GCS_PATH1, \"test*.tfrec\"))\nprint(\"%d test files found\" % len(test_files))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data pipeline","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n        #'patient_id'                   : tf.io.FixedLenFeature([], tf.int64),\n        #'sex'                          : tf.io.FixedLenFeature([], tf.int64),\n        #'age_approx'                   : tf.io.FixedLenFeature([], tf.int64),\n        #'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),\n        #'diagnosis'                    : tf.io.FixedLenFeature([], tf.int64),\n        'target'                       : tf.io.FixedLenFeature([], tf.int64)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], example['target']\n\n\ndef read_unlabeled_tfrecord(example, return_image_name=True):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], example['image_name'] if return_image_name else 0\n\n\ndef prepare_image(img):    \n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.cast(img, tf.float32) / 255.0           \n    return img\n\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/cdeotte/tfrecord-experiments-upsample-and-coarse-dropout\ndef dropout(image, DIM=256, PROBABILITY = 0.75, CT = 8, SZ = 0.2):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(CT==0)|(SZ==0): \n        return image\n    \n    for k in range(CT):\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        # COMPUTE SQUARE \n        WIDTH = tf.cast( SZ*DIM,tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # DROPOUT IMAGE\n        one = image[ya:yb,0:xa,:]\n        two = tf.zeros([yb-ya,xb-xa,3]) \n        three = image[ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM,DIM,3])\n    return image\n\n\ndef ClasswiseMixup(probability: float=1.0, batch_size: int = 16, alpha: float=1.0, beta: float=1.0):\n    def classwise_mixup(img, tar):\n        # input image as batch of same class\n        do = tf.cast(tf.random.uniform([batch_size], 0, 1) < probability, tf.float32)\n        gamma = tf.reshape(tf.random.uniform([batch_size], 0, 1) * do, [-1, 1, 1, 1])\n        shuffled_img = tf.random.shuffle(img)\n        img_new = (1 - gamma) * img + gamma * shuffled_img\n        return img_new\n    return classwise_mixup","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUG_BS = 64\n\ndef base_aug(img):\n    img = tf.image.random_flip_left_right(img)\n    # img = tf.image.random_hue(img, 0.01)\n    img = tf.image.random_saturation(img, 0.7, 1.3)\n    img = tf.image.random_contrast(img, 0.8, 1.2)\n    img = tf.image.random_brightness(img, 0.1)\n    return img\n\n\ndropout_aug = lambda img, o: dropout(img, DIM=IMG_READ_SIZE, PROBABILITY=0.75, CT=8, SZ=0.15)\ntransform_aug = imgaug.Transform(dim=IMG_READ_SIZE, hzoom_mult=10.0, wzoom_mult=10.0)\ncw_mixup_aug = ClasswiseMixup(probability=0.1, batch_size=AUG_BS)\n\n\ndef basic_augmentation_pipeline(ds: tf.data.Dataset, dim=None, batch_size=None) -> tf.data.Dataset:\n    ds = ds.map(lambda i, o: (transform_aug(i), o), num_parallel_calls=AUTO)\n    ds = ds.map(lambda i, o: (base_aug(i), o), num_parallel_calls=AUTO)\n    return ds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset(files, augment=False, repeat=False, shuffle=False, labeled=True, batch_size=16, drop_remainder=False, \n                dim=256, read_dim=None\n               ) -> tf.data.Dataset:\n    if read_dim is None:\n        read_dim = dim\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024 * 8)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    ds = ds.map(lambda i, o: (prepare_image(i), o), num_parallel_calls=AUTO)\n\n    if augment:\n        ds = basic_augmentation_pipeline(ds, batch_size=8 * batch_size, dim=read_dim) \n        if isinstance(augment, list):\n            for a in augment:\n                ds = ds.map(lambda i, o: (a(i, o), o), num_parallel_calls=AUTO)\n        \n    ds = ds.map(lambda i, o: (tf.image.resize(i, [dim, dim]), o), num_parallel_calls=AUTO)\n    ds = ds.batch(batch_size, drop_remainder=drop_remainder)\n    ds = ds.prefetch(AUTO)\n    return ds\n\n\ndef get_balanced_dataset(files, augment=False, cw_augment=False, repeat=False, shuffle=False, batch_size=16, drop_remainder=False, \n                         dim=256, read_dim=None, pos_ratio=False) -> tf.data.Dataset:\n    if read_dim is None:\n        read_dim = dim\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n        \n    ds0, ds1 = balance.separate_by_target(ds)\n    ds0 = ds.map(lambda i, o: (prepare_image(i), o), num_parallel_calls=AUTO)\n    ds1 = ds.map(lambda i, o: (prepare_image(i), o), num_parallel_calls=AUTO)\n    if cw_augment:\n        ds0 = ds0.batch(AUG_BS, drop_remainder=drop_remainder); ds1 = ds1.batch(AUG_BS, drop_remainder=drop_remainder)\n        for a in cw_augment:\n            ds0 = ds0.map(lambda i, o: (a(i, o), o), num_parallel_calls=AUTO)\n            ds1 = ds1.map(lambda i, o: (a(i, o), o), num_parallel_calls=AUTO)\n        ds0 = ds0.unbatch(); ds1 = ds1.unbatch()\n    \n    ds = balance.merge_ds(ds0, ds1, pos_ratio)\n    if shuffle: \n        ds = ds.shuffle(1024)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n    \n    if augment:\n        ds = basic_augmentation_pipeline(ds, batch_size=8 * batch_size, dim=read_dim) \n        if isinstance(augment, list):\n            for a in augment:\n                ds = ds.map(lambda i, o: (a(i, o), o), num_parallel_calls=AUTO)\n    ds = ds.map(lambda i, o: (tf.image.resize(i, [dim, dim]), o), num_parallel_calls=AUTO)\n        \n    ds = ds.batch(batch_size, drop_remainder=drop_remainder)\n    ds = ds.prefetch(AUTO)\n    return ds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import PIL, cv2\n\ndef show_dataset(thumb_size, cols, rows, ds):\n    mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols + (cols-1), \n                                             thumb_size*rows + (rows-1)))\n    for idx, data in enumerate(iter(ds)):\n        img, target_or_imgid = data\n        ix  = idx % cols\n        iy  = idx // cols\n        img = np.clip(img.numpy() * 255, 0, 255).astype(np.uint8)\n        img = PIL.Image.fromarray(img)\n        img = img.resize((thumb_size, thumb_size), resample=PIL.Image.BILINEAR)\n        mosaic.paste(img, (ix*thumb_size + ix, \n                           iy*thumb_size + iy))\n        #nn = target_or_imgid.numpy().decode(\"utf-8\")\n\n    display(mosaic)\n    pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_dataset(128, 8, 2, get_balanced_dataset(train_files, augment=[dropout_aug], cw_augment=[cw_mixup_aug]).take(10).unbatch())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model building","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(dim=128, ef=0):\n    inp = keras.layers.Input(shape=(dim,dim,3))\n    base = getattr(efnet, 'EfficientNetB%d' % ef)(input_shape=(dim, dim, 3), weights='imagenet', include_top=False)\n    x = base(inp)\n    x = keras.layers.GlobalAveragePooling2D()(x)\n    x = keras.layers.Dense(1)(x)\n    x = keras.layers.Activation('sigmoid', dtype='float32')(x)\n    model = keras.Model(inputs=inp,outputs=x)\n    opt = keras.optimizers.Adam(learning_rate=1e-3)\n    if LOSS_TYPE.upper() == 'BCE':\n        loss = keras.losses.BinaryCrossentropy(**LOSS_PARAMS)\n    elif LOSS_TYPE.upper() == 'FOCAL':\n        loss = losses.BinaryFocalLoss(**LOSS_PARAMS)\n    model.compile(optimizer=opt, loss=loss, metrics=['AUC'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Learning schedule","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"mult       = 1\nlr_start   = 5e-6\nlr_max     = 1.25e-6 * GLOBAL_BATCH_SIZE\nlr_min     = 1e-6\nlr_ramp_ep = 5\nlr_sus_ep  = 0\nlr_decay   = 0.8\n\n\ndef lrfn(epoch):\n    if epoch < lr_ramp_ep:\n        lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n    elif epoch < lr_ramp_ep + lr_sus_ep:\n        lr = lr_max\n    else:\n        lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n    return lr * mult\n    \n\nplt.figure(figsize=(10, 5))\nplt.subplot(1, 2, 1)\nplt.plot([lrfn(e) for e in range(EPOCHS)])\nplt.xlabel('Epoch'); plt.ylabel('LR')\nplt.subplot(1, 2, 2)\nplt.plot([lrfn(e) for e in range(EPOCHS)])\nplt.xlabel('Epoch'); plt.ylabel('Log LR')\nplt.yscale('log');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# build checkpoint folder\nCKPT_FOLDER = \"../working/ckpt\"\nif not os.path.exists(CKPT_FOLDER):\n    os.mkdir(CKPT_FOLDER)\n# build folds\nfolds = list(model_selection.KFold(n_splits=FOLDS, shuffle=True, random_state=SEED).split(np.arange(15)))\ntestiness = pd.read_csv(\"../input/spicv-spicy-vi-make-your-cv-more-testy/testiness.csv\")\n#\nTOTAL_POS = 581 + 2858 * INCLUDE_2019 + 1651 * INCLUDE_2018 + 580 * INCLUDE_MALIGNANT","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VERBOSE = 1\nPLOT    = 1\n\nhistories = []\ndf_oof = pd.DataFrame(); df_res = pd.DataFrame()\nt_start = time.time()\nfor fold, (idTrain, idValid) in enumerate(folds):\n    print(\"#\" * 68)\n    print((\"#\" * 20 + \"\\t\\tFold %d\\t\\t\" + \"#\" * 20) % fold)\n    print(\"#\" * 68)\n    # prepare TPU\n    if DEVICE == 'TPU':\n        if tpu: \n            tf.tpu.experimental.initialize_tpu_system(tpu)\n    # build fold train-valid split   \n    fold_valid_files = [f for f in train_files if any([int(re.match(\"^train([0-9]+)\", f.split(\"/\")[-1]).group(1)) % 15 == i for i in idValid])]\n    fold_valid_files = [f for f in fold_valid_files if GCS_PATH1 in f] # only data from the original dataset\n    # fold_train_files = [f for f in train_files if any([int(re.match(\"^train([0-9]+)\", f.split(\"/\")[-1]).group(1)) % 15 == i for i in idTrain])]\n    fold_train_files = [f for f in train_files if f not in fold_valid_files]\n    np.random.shuffle(fold_train_files)\n    print(\"Train files: %d\\t\\t Valid files: %d\" % (len(fold_train_files), len(fold_valid_files)))\n    # build model and set precision policy\n    K.clear_session()   \n    if DEVICE == 'TPU':\n        keras.mixed_precision.experimental.set_policy('mixed_bfloat16')\n    with strategy.scope():\n        model = build_model(dim=IMG_SIZE, ef=EFF_NET)\n    # callbacks\n    FOLD_CKPT_FOLDER = os.path.join(CKPT_FOLDER, \"fold%d\" % fold)\n    if not os.path.exists(FOLD_CKPT_FOLDER):\n        os.mkdir(FOLD_CKPT_FOLDER)\n    callbacks =[\n        keras.callbacks.ModelCheckpoint(os.path.join(FOLD_CKPT_FOLDER, \"model_fold%d_e{epoch:02d}.h5\" % fold), save_weights_only=True),\n        keras.callbacks.LearningRateScheduler(lrfn)\n    ]    \n    # build ds\n    if BALANCE_POS_RATIO:\n        print(\"Using balanced dataset with pos_ratio = %d%%\" % int(100 * BALANCE_POS_RATIO))\n        ds_train = get_balanced_dataset(fold_train_files, repeat=True,  augment=[dropout_aug],  drop_remainder=True,  shuffle=True,  \n                                        pos_ratio=BALANCE_POS_RATIO,\n                                        dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE)\n        FOLD_POS = TOTAL_POS * (FOLDS - 1) / FOLDS\n        STEPS = int(FOLD_POS / BALANCE_POS_RATIO / GLOBAL_BATCH_SIZE)\n    else:\n        print(\"Using unbalanced dataset\")\n        ds_train = get_dataset(fold_train_files, repeat=True,  augment=[dropout_aug],  drop_remainder=True,  shuffle=True,  \n                               dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE)\n        STEPS = int(count_data_items(fold_train_files) / GLOBAL_BATCH_SIZE)\n    ds_valid = get_dataset(fold_valid_files, repeat=False, augment=False, drop_remainder=False, shuffle=False, \n                           dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM)\n    # train\n    if fold==4:\n        model.load_weights('/kaggle/input/pretraineffi65fold/model_fold%s.h5'%fold)\n    else:\n        model.load_weights('/kaggle/input/pretrained/model_fold%s.h5'%fold)    \n    print(\"Training...\")\n    history = model.fit(\n                            ds_train,\n        validation_data   = ds_valid,\n        epochs            = EPOCHS,\n        steps_per_epoch   = STEPS,\n        verbose           = VERBOSE,\n        callbacks         = callbacks,\n        validation_freq   = VALID_FREQ\n    )\n    histories.append(history)    \n    \n    # SWA\n    ckpt_files = np.sort(tf.io.gfile.glob(os.path.join(FOLD_CKPT_FOLDER, \"*.h5\")))\n    ckpt_files_fow_swa = ckpt_files[-N_SWA:]\n    if len(ckpt_files_fow_swa) > 1:\n        with strategy.scope():\n            model = average.average_weights(ckpt_files_fow_swa, decay=SWA_DECAY, model=model)\n    for f in ckpt_files:\n        os.remove(f)\n    model.save(os.path.join(CKPT_FOLDER, \"model_fold%d.h5\" % fold))\n    # VALID\n    ds_valid = get_dataset(fold_valid_files, augment=TTA >= 1, repeat=True, dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=True)\n    ct_valid = count_data_items(fold_valid_files); STEPS = int(np.ceil(TTA * ct_valid / GLOBAL_BATCH_SIZE / TBM))\n    fold_valid_pred = model.predict(ds_valid, steps=STEPS, verbose=1)\n    fold_valid_pred = fold_valid_pred[:ct_valid * TTA,]\n    ds_valid = get_dataset(fold_valid_files, augment=False, repeat=False, dim=IMG_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=False, labeled=False)\n    fold_valid_names = np.concatenate([np.array([ni.decode(\"utf-8\") for ni in n.numpy()]) for n in ds_valid.map(lambda i, n: n)], 0)\n    \n    fold_df = pd.DataFrame({'image_name': np.tile(fold_valid_names, [TTA]), 'pred': fold_valid_pred.squeeze(), 'fold': fold})\n    df_oof = pd.concat([df_oof, fold_df])\n    fold_df['image_name'] = fold_df['image_name'].str.replace('_downsampled', '')\n    fold_df = fold_df.groupby('image_name').mean().reset_index()\n    fold_df = fold_df.merge(df_base_train[['image_name', 'patient_id', 'target']], on='image_name').merge(testiness, on='image_name')\n    fold_df['fold'] = fold\n    auc  = metrics.roc_auc_score(fold_df.target, fold_df.pred)\n    \n    # TEST\n    ds_test = get_dataset(test_files, augment=TTA >= 1, repeat=True, dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=True, labeled=False)\n    ct_test = count_data_items(test_files); STEPS = int(np.ceil(TTA * ct_test / GLOBAL_BATCH_SIZE / TBM))\n    fold_test_pred = model.predict(ds_test.map(lambda i, l: i), steps=STEPS, verbose=1)\n    fold_test_pred = fold_test_pred[:ct_test * TTA,]\n    ds_test = get_dataset(test_files, augment=False, repeat=False, dim=IMG_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=False, labeled=False)\n    fold_test_names = np.concatenate([np.array([ni.decode(\"utf-8\") for ni in n.numpy()]) for n in ds_test.map(lambda i, n: n)], 0)\n    \n    fold_res = pd.DataFrame({'image_name': np.tile(fold_test_names, [TTA]), 'pred': fold_test_pred.squeeze(), 'fold': fold})\n    df_res = pd.concat([df_res, fold_res])\n    \n    # time\n    used_time_till_now = time.time() - t_start\n    time_per_fold = used_time_till_now / (fold + 1)\n    print(\"Validation AUC last epoch = %.4f\" % history.history['val_auc'][-1])\n    print(\"Validation AUC  (TTA %2d) = %.4f\" % (TTA, auc))\n    print(\"Total time = %ds\\t\\tTime per fold = %ds\" % (int(used_time_till_now), int(time_per_fold)))\n    \n    # plot\n    if PLOT:\n        plt.figure(figsize=(16, 4))\n        plt.subplot(1, 2, 1)\n        plt.plot(history.history['loss'], color='tab:blue', marker='o')\n        plt.plot(range(VALID_FREQ - 1, EPOCHS, VALID_FREQ), history.history['val_loss'], color='tab:blue', marker='x', linestyle=':')\n        plt.yscale('log')\n        plt.subplot(1, 2, 2)\n        plt.plot(history.history['auc'], color='tab:red', marker='o')\n        plt.plot(range(VALID_FREQ - 1, EPOCHS, VALID_FREQ), history.history['val_auc'], color='tab:red', marker='x', linestyle=':')\n        plt.show()\n    \n    del model, ds_train, ds_valid, ds_test\n    print(\"\\n\\n\")\n    \n    if (fold + 1) < FOLDS:\n        # time: if next iteration will exceed the time budget, abort\n        time_till_next_fold = used_time_till_now + time_per_fold\n        if time_till_next_fold > TIME_BUDGET:\n            print(\"Forecasted total time till next fold completion = %d (budget = %d)\" % (time_till_next_fold, TIME_BUDGET))\n            break\n    pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"avgh = dict()\nfor history in histories:\n    for k in history.history.keys():\n        if k in avgh.keys():\n            avgh[k] = np.concatenate([avgh[k], np.array(history.history[k]).reshape(-1, 1)], 1)\n        else:\n            avgh[k] = np.array(history.history[k]).reshape(-1, 1)\nplt.figure(figsize=(16, 4))\nplt.subplot(1, 2, 1)\nplt.title('Log Loss')\nplt.plot(avgh['loss'], marker='o', color='tab:blue', alpha=0.2)\nplt.plot(avgh['loss'].mean(1), color='tab:blue')\nplt.plot(range(VALID_FREQ - 1, EPOCHS, VALID_FREQ), avgh['val_loss'], color='tab:blue', alpha=0.2, linestyle=\":\")\nplt.plot(range(VALID_FREQ - 1, EPOCHS, VALID_FREQ), avgh['val_loss'].mean(1), marker='x', color='tab:blue', linestyle=\":\")\nplt.yscale('log')\nplt.subplot(1, 2, 2)\nplt.title('AUC')\nplt.plot(avgh['auc'], marker='o', color='tab:red', alpha=0.2)\nplt.plot(avgh['auc'].mean(1), color='tab:red')\nplt.plot(range(VALID_FREQ - 1, EPOCHS, VALID_FREQ), avgh['val_auc'], color='tab:red', alpha=0.2, linestyle=\":\")\nplt.plot(range(VALID_FREQ - 1, EPOCHS, VALID_FREQ), avgh['val_auc'].mean(1), marker='x', color='tab:red', linestyle=\":\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TTA Analysis","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"R = 5\ndf = df_oof.merge(df_base_train[['image_name', 'patient_id', 'target']], on='image_name').merge(testiness, on='image_name')\nplt.figure(figsize=(15, 5))\nfor N in range(1, TTA + 1):\n    aucs = []; caucs = []\n    for r in range(R):\n        df_i = df.sample(frac=1.0).groupby('image_name').tail(N).groupby('image_name').mean().reset_index()\n        aucs.append(metrics.roc_auc_score(df_i.target, df_i.pred))\n        caucs.append(metrics.roc_auc_score(df_i.target, df_i.pred, sample_weight=df_i.testiness))\n    plt.scatter([N] * R, aucs, color='tab:blue', alpha=0.05)\n    plt.scatter(N, sum(aucs) / R, color='tab:blue', alpha=0.9)\nplt.xlabel(\"TTA\"); plt.ylabel(\"AUC\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xxx = df_oof.groupby('image_name').mean().reset_index().merge(df_base_train, on='image_name')\nprint(\"OOF AUC (TTA %d) = %.4f\" % (TTA, metrics.roc_auc_score(xxx.target, xxx.pred)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_res.to_csv('../working/test_res_all.csv', index=False)\ndf_oof.to_csv('../working/oof_res_all.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_res[['image_name', 'pred']].groupby('image_name').mean().reset_index().rename({'pred': 'target'}, axis=1).to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\neffnet6 = pd.read_csv('/kaggle/input/good-submission/submission_effi6.csv')\neffnet6.to_csv('../working/effnet6.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}