{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Experimenting On single Model to get As good as Possible LB score\n\n---\n### NOOB Stuff\n\n* v2: Baseline model --> 0.710\n* v3: Model with extra layers and Dropout --> 0.755 \n* V4: Learning rate Schedular + IMG_SIZE 1024 --> 0.793\n* **V6: Learning rate Schedular + IMG_SIZE 1024 + Augmentation + EfficientNetB3 --> 0.875**\n* V8: Learning rate Schedular + IMG_SIZE 1024 + Augmentation++ + EfficientNetB4 + classweights--> 0.754\n* V9: Learning rate Schedular + IMG_SIZE 1024 + Augmentation++ + EfficientNetB3 + classweights --> 0.647 [class weights doesn't work]\n* V10: Learning rate Schedular + IMG_SIZE 1024 + Augmentation++ + EfficientNetB3 --> 0.697 [not all augmenation works]\n* **V11: Learning rate Schedular + IMG_SIZE 1024 + two augmentation + EfficientNetB3 --> 0.882** [Increase due to extra augmentation]\n* V16: LRS + IMG_SIZE 512 [external data] + two augmentation + EfficientNetB3 + 30 epochs --> 0.837 [Over-fit model]\n* V18: LRS + IMG_SIZE 512 [external data] + two augmentation + EfficientNetB3 + 30 epochs[EARLY stopping] --> 0.877\n\n--- \n\n### CV and LB AUC got Quite stable from here; now incorprating following things\n\n* Model Tuning\n* LOSS, OPTIMIZER, SCHEDULER TUNING\n* Playing with data and augmentation\n---\n* V21: LRS + IMG_SIZE 512 [external data] + two augmentation + EfficientNetB6 + 30 epochs[EARLY stopping] --> 0.915\n* V23: LRS + IMG_SIZE 256 [external data] + two augmentation + EfficientNetB6 + 30 epochs[EARLY stopping] --> 0.893\n\n\n\n---\n\n### Switched to Smaller Nets and Small Image Sizes For Fast Experimentation\n\n#### For Results Table See Comment Below [JUMP to Comment](https://www.kaggle.com/orionpax00/melanoma-detection-single-model-gpu-tpu-yapl#923921)\n\nNote: I Moved Result Table to the comment section due to ease in updation after training.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install yapl==0.1.2 efficientnet > /dev/null\n!rm -rf *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import yapl\nimport os\nimport re\nimport random\nimport pandas as pd\nimport numpy as np\nimport math\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom kaggle_datasets import KaggleDatasets\nimport efficientnet.tfkeras as efn\n\nfrom yapl.config.config import Config\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.backend.clear_session()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class configSetup(Config):\n    def __init__(self):\n        super().__init__()\n        self.IS_EXT = True\n        self.DO_VAL_SPLIT = True\n        self.EXPERIMENT_NAME = 'melanoma-detection-yapl-testing'\n        \n        if self.IS_EXT:\n            self.GCS_DS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')\n            self.TOTAL_FILES = tf.io.gfile.glob(self.GCS_DS_PATH + '/train*.tfrec')\n            self.TEST_FILES = tf.io.gfile.glob(self.GCS_DS_PATH + '/test*.tfrec')\n            self.TRAIN_CSV = '../input/melanoma-128x128/train.csv'\n        else:\n            self.GCS_DS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')\n            self.TOTAL_FILES = tf.io.gfile.glob(self.GCS_DS_PATH + '/train/*.tfrec')\n            self.TEST_FILES = tf.io.gfile.glob(self.GCS_DS_PATH + '/test/*.tfrec')\n            self.TRAIN_CSV = '../input/siim-isic-melanoma-classification/train.csv'\n        \n        self.TEST_CSV = '../input/siim-isic-melanoma-classification/test.csv'\n        \n        self.VALIDATION_CSV = \"\"\n        \n        self.TOTAL_TRAIN_IMG = 0\n        self.TOTAL_TEST_IMG = 0\n        self.NEG_SMAPLE = 32542\n        self.POS_SMAPLE = 584\n        \n        self.IMG_SIZE = [384, 384]\n        self.IMG_TRAIN_SHAPE = [384,384]\n        self.IMG_SHAPE = (384, 384, 3)\n        self.DO_FINETUNE = True\n        \n        self.BATCH_SIZE = 16 # 16\n        self.EPOCHES = 10 \n        self.SEED = 127\n        self.FOLDS = 3\n        \n        self.LOSS = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)\n        self.OPTIMIZER = tf.keras.optimizers.Adam()\n        self.ACCURACY = ['AUC']\n        \n        self.CALLBACKS = [\n            tf.keras.callbacks.EarlyStopping(\n                monitor='val_auc', restore_best_weights=True,  mode = 'max', patience = 3, verbose = 1, min_delta = 0.001,\n            )\n        ]\n        \n        self.STRATEGY = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# store config in yapl global variable        \nconfigSetup().make_global()\n\nyapl.backend = 'tf'\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    yapl.config.STRATEGY = strategy\n    yapl.config.BATCH_SIZE = 32 * strategy.num_replicas_in_sync \nelse:\n    strategy = tf.distribute.get_strategy() \n\n#configuration\n# yapl.config.TOTAL_TRAIN_IMG = len(pd.read_csv(yapl.config.TRAIN_CSV).image_name)\nyapl.config.TOTAL_TEST_IMG = len(pd.read_csv(yapl.config.TEST_CSV).image_name)\n\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Using Stratified Folds\nThis method uses pre build stratified tfrecord \n[Dataset](https://www.kaggle.com/cdeotte/melanoma-128x128?select=train.csv)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(yapl.config.TRAIN_CSV)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FOLDS_DICT = {}\n\nskf = KFold(n_splits=yapl.config.FOLDS,shuffle=True,random_state=yapl.config.SEED)\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(15))):\n    FOLDS_DICT['fold_{}'.format(fold+1)] = {\n                                            \"trainfiles\" : [yapl.config.TOTAL_FILES[x] for x in idxT],\n                                            \"valfiles\"   : [yapl.config.TOTAL_FILES[x] for x in idxV]\n                                            }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything():\n    np.random.seed(yapl.config.SEED)\n    tf.random.set_seed(yapl.config.SEED)\n    random.seed(a=yapl.config.SEED)\n    os.environ['PYTHONHASHSEED'] = str(yapl.config.SEED)\n\nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROT_ = 180.0\nSHR_ = 2.0\nHZOOM_ = 8.0\nWZOOM_ = 8.0\nHSHIFT_ = 8.0\nWSHIFT_ = 8.0\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, DIM=256):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Helper Functions\ndef data_augment(image, label):\n    image = transform(image, DIM=yapl.config.IMG_SIZE[0])\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.rot90(image)\n    \n    return image, label  \n\ndef process_training_data(data_file):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        \"target\": tf.io.FixedLenFeature([], tf.int64),\n    }\n    data = tf.io.parse_single_example(data_file, LABELED_TFREC_FORMAT)\n    img = tf.image.decode_jpeg(data['image'], channels=3)\n#     img = tf.image.resize(img, yapl.config.IMG_TRAIN_SHAPE)\n    img = tf.cast(img, tf.float32) / 255.0\n    img = tf.reshape(img, [*yapl.config.IMG_TRAIN_SHAPE, 3])\n\n    label = tf.cast(data['target'], tf.int32)\n\n    return img, label\n\ndef process_test_data(data_file):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n    }\n    data = tf.io.parse_single_example(data_file, LABELED_TFREC_FORMAT)\n    img = tf.image.decode_jpeg(data['image'], channels=3)\n#     img = tf.image.resize(img, yapl.config.IMG_TRAIN_SHAPE)\n    img = tf.cast(img, tf.float32) / 255.0\n    img = tf.reshape(img, [*yapl.config.IMG_TRAIN_SHAPE, 3])\n\n    idnum = data['image_name']\n\n    return img, idnum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#learning rate Scheduler\ndef build_lrfn(lr_start=0.000005, lr_max=0.000020, \n               lr_min=0.000001, lr_rampup_epochs=5, \n               lr_sustain_epochs=0, lr_exp_decay=.8):\n    if yapl.config.STRATEGY is not None:\n        lr_max = lr_max * yapl.config.STRATEGY.num_replicas_in_sync\n\n    def lrfn(epoch):\n        if epoch < lr_rampup_epochs:\n            lr = (lr_max - lr_start) / lr_rampup_epochs * epoch + lr_start\n        elif epoch < lr_rampup_epochs + lr_sustain_epochs:\n            lr = lr_max\n        else:\n            lr = (lr_max - lr_min) * lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs) + lr_min\n        return lr\n    \n    return lrfn\n\n#learning rate Scheduler\ndef build_lrfn_l(lr_start=0.00001, lr_max=0.000075, \n               lr_min=0.000001, lr_rampup_epochs=20, \n               lr_sustain_epochs=0, lr_exp_decay=.8):\n    if yapl.config.STRATEGY is not None:\n        lr_max = lr_max * yapl.config.STRATEGY.num_replicas_in_sync\n\n    def lrfn(epoch):\n        if epoch < lr_rampup_epochs:\n            lr = (lr_max - lr_start) / lr_rampup_epochs * epoch + lr_start\n        elif epoch < lr_rampup_epochs + lr_sustain_epochs:\n            lr = lr_max\n        else:\n            lr = (lr_max - lr_min) * lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs) + lr_min\n        return lr\n    \n    return lrfn\n\ndef build_lrfn_2():\n    lr_start   = 0.000005\n    lr_max     = 0.000020 \n    lr_min     = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n    if yapl.config.STRATEGY is not None:\n        lr_max = lr_max * yapl.config.BATCH_SIZE\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    return lrfn\n\n\ndef get_cosine_schedule_with_warmup(lr = 0.00004, num_warmup_steps = 5 , num_training_steps = yapl.config.EPOCHES, num_cycles=0.5):\n    def lrfn(epoch):\n        if epoch < num_warmup_steps:\n            return (float(epoch) / float(max(1, num_warmup_steps))) * lr\n        progress = float(epoch - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr\n\n    return lrfn\n\nlrfn = get_cosine_schedule_with_warmup()\nlr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=0)\nyapl.config.CALLBACKS.append(lr_schedule)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def efficientnetbx():\n    return tf.keras.Sequential([\n                    efn.EfficientNetB0(\n                        input_shape=(*yapl.config.IMG_TRAIN_SHAPE, 3),\n                        weights='imagenet',\n                        include_top=False\n                    ),\n                    tf.keras.layers.GlobalAveragePooling2D(),\n                    tf.keras.layers.Dense(1, activation='sigmoid')\n                ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fitengine(model, traindataset, valdataset = None, istraining = True):\n    model.compile(\n        optimizer   =  yapl.config.OPTIMIZER, \n        loss        =  yapl.config.LOSS, \n        metrics     =  yapl.config.ACCURACY\n    )\n    weight_for_0 = (1 / yapl.config.NEG_SMAPLE)*(yapl.config.TOTAL_TRAIN_IMG)/2.0 \n    weight_for_1 = (1 / yapl.config.POS_SMAPLE)*(yapl.config.TOTAL_TRAIN_IMG)/2.0\n\n    class_weight = {0: weight_for_0, 1: weight_for_1}\n    \n    history = model.fit(\n                traindataset, \n                epochs            =   yapl.config.EPOCHES, \n                steps_per_epoch   =   yapl.config.TOTAL_TRAIN_IMG//yapl.config.BATCH_SIZE,\n                callbacks         =   yapl.config.CALLBACKS,\n                validation_data   =   valdataset,\n                validation_steps  =   yapl.config.TOTAL_VAL_IMG//yapl.config.BATCH_SIZE,\n                verbose           =   0\n            )\n\n    return history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODELS = {}\nHISTORY = {}\nTTA = 11\n\nprint('##'*30)\nprint(\"#### IMAGE SIZE {} \".format(yapl.config.IMG_SHAPE))\nprint(\"#### BATCH SIZE {} \".format(yapl.config.BATCH_SIZE))\nprint(\"#### TRAINING RECORDS {} \".format(15 - 15/yapl.config.FOLDS))\nprint(\"#### VALIDATION RECORDS {} \".format(15/yapl.config.FOLDS))\nprint('##'*30)\nprint(\"\\n\\n\")\n\nfor fold in range(yapl.config.FOLDS):\n    K.clear_session()\n    yapl.config.CALLBACKS[0] = tf.keras.callbacks.ModelCheckpoint(\n                                    'fold-%i.h5'%(fold+1), monitor='val_loss', verbose=0, save_best_only=True,\n                                    save_weights_only=True, mode='min', save_freq='epoch')\n    \n    yapl.config.TOTAL_TRAIN_IMG = count_data_items(FOLDS_DICT['fold_{}'.format(fold+1)]['trainfiles'])\n    yapl.config.TOTAL_VAL_IMG = count_data_items(FOLDS_DICT['fold_{}'.format(fold+1)]['valfiles'])\n        \n    # training data\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False\n    train_dataset = (\n        tf.data.TFRecordDataset(\n            FOLDS_DICT['fold_{}'.format(fold+1)]['trainfiles'],  \n            num_parallel_reads=tf.data.experimental.AUTOTUNE\n        ).with_options(\n            ignore_order\n        ).map(\n            process_training_data,\n            num_parallel_calls=tf.data.experimental.AUTOTUNE\n        ).map(\n            data_augment, \n            num_parallel_calls=tf.data.experimental.AUTOTUNE\n        ).repeat(\n        ).shuffle(\n            yapl.config.SEED\n        ).batch(\n            yapl.config.BATCH_SIZE\n        ).prefetch(\n            tf.data.experimental.AUTOTUNE\n        )\n    )\n\n    #Validation data\n    ignore_order = tf.data.Options()\n    val_dataset = (\n        tf.data.TFRecordDataset(\n            FOLDS_DICT['fold_{}'.format(fold+1)]['valfiles'],  \n            num_parallel_reads=tf.data.experimental.AUTOTUNE\n        ).with_options(\n            ignore_order\n        ).map(\n            process_training_data,\n            num_parallel_calls=tf.data.experimental.AUTOTUNE\n        ).batch(\n            yapl.config.BATCH_SIZE\n        ).prefetch(\n            tf.data.experimental.AUTOTUNE\n        )\n    )\n\n    if yapl.config.STRATEGY is not None:\n        with strategy.scope():\n            model = efficientnetbx()\n    else:\n        model = efficientnetbx()\n\n    print(\"##\"*30)\n    print(\"#### FOLD {} \".format(fold+1))\n    \n    \n    history = fitengine(model, train_dataset, val_dataset); #trining model\n    HISTORY['fold_{}'.format(fold+1)] = history #storing history\n    \n    \n    model.load_weights('fold-%i.h5'%(fold+1)) #loading saved model\n    \n    max_val_auc = np.max( history.history['val_auc'])\n    print('#### Validation AUC without TTA: %.4f'%max_val_auc)\n    \n    ## Prediction Setup\n    ignore_order = tf.data.Options()\n    test_dataset = (\n        tf.data.TFRecordDataset(\n            yapl.config.TEST_FILES,  \n            num_parallel_reads=tf.data.experimental.AUTOTUNE\n        ).with_options(\n            ignore_order\n        ).map(\n            process_test_data,\n            num_parallel_calls=tf.data.experimental.AUTOTUNE\n        ).map(\n            data_augment, \n            num_parallel_calls=tf.data.experimental.AUTOTUNE\n        ).repeat(\n        ).batch(\n            yapl.config.BATCH_SIZE * 4 \n        ).prefetch(\n            tf.data.experimental.AUTOTUNE\n        )\n    )      \n\n    test_imgs = test_dataset.map(lambda images, ids: images)\n    img_ids_ds = test_dataset.map(lambda images, ids: ids).unbatch()\n        \n    ct_test = count_data_items(yapl.config.TEST_FILES)\n    STEPS = TTA * ct_test/(yapl.config.BATCH_SIZE * 4)\n    pred = model.predict(test_imgs,steps=STEPS,verbose=0)[:TTA*ct_test,] \n    predictions = np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1) \n    \n    test_ids = next(iter(img_ids_ds.batch(yapl.config.TOTAL_TEST_IMG))).numpy().astype('U') \n\n    pd.DataFrame({\n         'image_name'  : test_ids, \n         'target'      : predictions\n        }).to_csv('prediction_fold_{}.csv'.format(fold+1), index=False)   \n    \n    \n    #Predicting on validation Data with TTA\n    ignore_order = tf.data.Options()\n    val_dataset = (\n        tf.data.TFRecordDataset(\n            FOLDS_DICT['fold_{}'.format(fold+1)]['valfiles'], \n            num_parallel_reads=tf.data.experimental.AUTOTUNE\n        ).with_options(\n            ignore_order\n        ).map(\n            process_training_data,\n            num_parallel_calls=tf.data.experimental.AUTOTUNE\n        ).map(\n            data_augment, \n            num_parallel_calls=tf.data.experimental.AUTOTUNE\n        ).repeat(\n        ).batch(\n            yapl.config.BATCH_SIZE * 4\n        ).prefetch(\n            tf.data.experimental.AUTOTUNE\n        )\n    )\n    \n    test_imgs = val_dataset.map(lambda images, labels: images)\n    img_labels_ds = val_dataset.map(lambda images, labels: labels).unbatch()\n    \n    ct_test = yapl.config.TOTAL_VAL_IMG\n    STEPS = TTA * ct_test/(yapl.config.BATCH_SIZE * 4)\n    pred = model.predict(test_imgs,steps=STEPS,verbose=0)[:TTA*ct_test,] \n    predictions = np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1)\n    \n    test_labels = next(iter(img_labels_ds.batch(yapl.config.TOTAL_VAL_IMG))).numpy()\n\n    pd.DataFrame(\n        {\n         'target'    : predictions,\n         'label'     : test_labels\n        }\n    ).to_csv('oof_{}.csv'.format(fold+1), index=False)  \n    \n    \n    ttavalauc = roc_auc_score(test_labels,predictions)\n    print('#### Validation AUC with TTA: %.4f'%ttavalauc)\n    \n    print('##'*30)\n    #Displaying Training\n    plt.figure(figsize=(15,5))\n    plt.plot(np.arange(yapl.config.EPOCHES),history.history['auc'],'-o',label='Train AUC',color='#ff7f0e')\n    plt.plot(np.arange(yapl.config.EPOCHES),history.history['val_auc'],'-o',label='Val AUC',color='#1f77b4')\n    x = np.argmax( history.history['val_auc'] ); y = np.max( history.history['val_auc'] )\n    xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n    plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n    plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)\n    plt.legend(loc=2)\n    plt2 = plt.gca().twinx()\n    plt2.plot(np.arange(yapl.config.EPOCHES),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n    plt2.plot(np.arange(yapl.config.EPOCHES),history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n    x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n    ydist = plt.ylim()[1] - plt.ylim()[0]\n    plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n    plt.ylabel('Loss',size=14)\n    plt.legend(loc=3)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Analysing Training ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"## Learning Rate plot\nimport matplotlib.pyplot as plt\n\nlrfn = build_lrfn_2()\n\nplt.plot([lrfn(epoch) for epoch in range(yapl.config.EPOCHES)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}