{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Experiment-10: Transfer training a highend model (efficientnetv2-s-21k-ft1k) with original images, traditional image augmentation, stylegan generated image augmentation & transformation techniques","metadata":{}},{"cell_type":"code","source":"import itertools\nimport os\nimport random\nimport gc\nimport numpy as np\nimport pandas as pd\nimport math\nimport pprint\nimport matplotlib.pylab as plt\nimport seaborn as sns\nsns.set(rc={\"axes.titlesize\":15, \"axes.labelsize\":9,\"axes.titlepad\":15,\n            \"axes.labelpad\":12, \"legend.fontsize\":9,\n            \"legend.title_fontsize\":9, \"figure.titlesize\":15,\n            \"axes.grid\":False})\n\nfrom sklearn.model_selection import train_test_split, GroupKFold\nimport tensorflow as tf\nimport tensorflow_hub as tfhub\nimport tensorflow.keras.backend as K\nfrom kaggle_datasets import KaggleDatasets\n\nprint('TF version:', tf.__version__)\nprint('Hub version:', tfhub.__version__)\nprint('Physical devices:', tf.config.list_physical_devices())","metadata":{"execution":{"iopub.status.busy":"2021-08-27T18:20:06.235668Z","iopub.execute_input":"2021-08-27T18:20:06.236048Z","iopub.status.idle":"2021-08-27T18:20:06.249403Z","shell.execute_reply.started":"2021-08-27T18:20:06.236017Z","shell.execute_reply":"2021-08-27T18:20:06.24832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #027fc1; font-family: Segoe UI; font-size: 1.9em; font-weight: 300;\">Data Loaders & Augmentations on TPU</span>\n\nThanks to [@cdeotte](https://www.kaggle.com/cdeotte) & [@xhlulu](https://www.kaggle.com/xhlulu)","metadata":{}},{"cell_type":"code","source":"SATURATION  = (0.9, 1.1)\nCONTRAST = (0.9, 1.1)\nBRIGHTNESS  =  0.1\nROTATION    = 10.0\nSHEAR    = 2.0\nHZOOM  = 8.0\nWZOOM  = 4.0\nHSHIFT = 4.0\nWSHIFT = 4.0\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = cfg.input_dims[0]\n    XDIM = DIM%2\n    \n    rot = ROTATION * tf.random.normal([1], dtype='float32')\n    shr = SHEAR * tf.random.normal([1], dtype='float32')\n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM\n    h_shift = HSHIFT * tf.random.normal([1], dtype='float32')\n    w_shift = WSHIFT * tf.random.normal([1], dtype='float32')\n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot, shr, h_zoom, w_zoom, h_shift, w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3]), label","metadata":{"execution":{"iopub.status.busy":"2021-08-27T18:20:08.741057Z","iopub.execute_input":"2021-08-27T18:20:08.741395Z","iopub.status.idle":"2021-08-27T18:20:08.762307Z","shell.execute_reply.started":"2021-08-27T18:20:08.741366Z","shell.execute_reply":"2021-08-27T18:20:08.76129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy\n\n\ndef build_decoder(with_labels=True, target_size=(256, 256), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True):\n    def augment(img):\n        img = tf.image.random_flip_left_right(img)\n        #img = tf.image.random_flip_up_down(img)\n        img = tf.image.random_saturation(img, SATURATION[0], SATURATION[1])\n        img = tf.image.random_contrast(img, CONTRAST[0], CONTRAST[1])\n        img = tf.image.random_brightness(img, BRIGHTNESS)\n        return img\n    \n    def augment_with_labels(img, label):\n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, bsize=128, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024,\n                  seed=None, cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    \n    # Map the functions to perform Augmentations\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.map(transform, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle, seed=seed) if shuffle else dset\n    dset = dset.batch(bsize).prefetch(AUTO)\n    \n    return dset","metadata":{"execution":{"iopub.status.busy":"2021-08-27T18:20:12.844486Z","iopub.execute_input":"2021-08-27T18:20:12.844816Z","iopub.status.idle":"2021-08-27T18:20:12.86323Z","shell.execute_reply.started":"2021-08-27T18:20:12.844788Z","shell.execute_reply":"2021-08-27T18:20:12.862394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #027fc1; font-family: Segoe UI; font-size: 1.9em; font-weight: 300;\">Global Config & Seeding</span>","metadata":{}},{"cell_type":"code","source":"def seed_everything(SEED):\n    os.environ['PYTHONHASHSEED']=str(SEED)\n    random.seed(SEED)\n    np.random.seed(SEED)\n    tf.random.set_seed(SEED)\n    os.environ['TF_CUDNN_DETERMINISTIC'] = str(SEED)\n    \nclass Config:\n    seed = 2021\n    job = 1\n    num_classes = 2\n    input_dims = (256, 256)\n    model_arch = \"efficientnetv2-s-21k-ft1k\" #\"efficientnetv2-b0\" #\"efficientnetv2-s-21k-ft1k\" ## Choose model architecture\n    batch_size = 8*16\n    kfold = 5\n    n_epochs = 15\n    lr = 0.0001\n    loss_func = 'binary_crossentropy'\n    # Whether to finetune the whole model or just the top layer.\n    fine_tune = True\n    wandb_project = 'lmju-covid-exp10-aug-sty-trans'\n    dataset = \"ljmusiimcovid19exp9\"\n    GAN_TRAIN_FOLDER = \"stylegan-train\"\n    seed_everything(seed)\n    \ncfg = Config()\n","metadata":{"execution":{"iopub.status.busy":"2021-08-27T18:23:12.156186Z","iopub.execute_input":"2021-08-27T18:23:12.156554Z","iopub.status.idle":"2021-08-27T18:23:12.166244Z","shell.execute_reply.started":"2021-08-27T18:23:12.156518Z","shell.execute_reply":"2021-08-27T18:23:12.165093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #027fc1; font-family: Segoe UI; font-size: 1.9em; font-weight: 300;\">KFold Split</span>","metadata":{}},{"cell_type":"code","source":"#study_df = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\n\n# Has Id, Covid columns. \ntrain_df1 = pd.read_csv('/kaggle/input/ljmusiimcovid19exp9/train_df.csv')\ntrain_df1.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T18:21:52.094377Z","iopub.execute_input":"2021-08-27T18:21:52.094732Z","iopub.status.idle":"2021-08-27T18:21:52.123375Z","shell.execute_reply.started":"2021-08-27T18:21:52.094701Z","shell.execute_reply":"2021-08-27T18:21:52.122441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df1['id']= train_df1.apply(lambda row: 'original-train/'+ row['id'], axis=1)\ntrain_df1.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T18:22:30.686519Z","iopub.execute_input":"2021-08-27T18:22:30.686864Z","iopub.status.idle":"2021-08-27T18:22:30.743656Z","shell.execute_reply.started":"2021-08-27T18:22:30.686836Z","shell.execute_reply":"2021-08-27T18:22:30.74258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df2 = pd.DataFrame(columns=['id','label'])\nfiles = [file for file in os.listdir('/kaggle/input/ljmusiimcovid19exp9/'+cfg.GAN_TRAIN_FOLDER)]\n\ntrain_df2['id'] = files\ntrain_df2.loc[train_df2['id'].astype(str).str.startswith(\"c-\"), ['label']] = 1\ntrain_df2.loc[train_df2['id'].astype(str).str.startswith(\"n-\"), ['label']] = 0\n\ntrain_df2.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T18:23:20.748991Z","iopub.execute_input":"2021-08-27T18:23:20.749278Z","iopub.status.idle":"2021-08-27T18:23:21.076832Z","shell.execute_reply.started":"2021-08-27T18:23:20.749253Z","shell.execute_reply":"2021-08-27T18:23:21.076149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df2['id']= train_df2.apply(lambda row: cfg.GAN_TRAIN_FOLDER +'/'+ row['id'], axis=1)\ntrain_df2.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T18:24:27.499519Z","iopub.execute_input":"2021-08-27T18:24:27.499837Z","iopub.status.idle":"2021-08-27T18:24:27.579058Z","shell.execute_reply.started":"2021-08-27T18:24:27.499792Z","shell.execute_reply":"2021-08-27T18:24:27.578296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#concat both dataset, drop index, shuffle\ntrain_df = pd.concat([train_df1, train_df2], ignore_index=True)\ntrain_df = train_df.sample(frac = 1)\nprint('Total train images = '+str(len(train_df)))\ntrain_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T18:24:55.537237Z","iopub.execute_input":"2021-08-27T18:24:55.537578Z","iopub.status.idle":"2021-08-27T18:24:55.554504Z","shell.execute_reply.started":"2021-08-27T18:24:55.537548Z","shell.execute_reply":"2021-08-27T18:24:55.553699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For one-hot encoding Covid label, add Negative column and flip the value. 1 0 <-- using this\n# For label encoding, just keep label column 1 or 0\n\ntrain_df.rename(columns={'id':'Id'}, inplace=True)\n\ntrain_df.rename(columns={'label':'Covid'}, inplace=True)\ntrain_df['Covid'] = train_df['Covid'].astype('int64')\n\ntrain_df['Negative'] = 0\ntrain_df['Negative'] = train_df['Negative'].astype('int64')\n\ntrain_df['Fold'] = -1\ntrain_df['Fold'] = train_df['Fold'].astype('int64')\n\ntrain_df.loc[train_df.Covid == 0, 'Negative'] = 1\nlabel_cols = ['Covid', 'Negative']\n\ngkf  = GroupKFold(n_splits=cfg.kfold)\n\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(train_df, groups=train_df.Id.tolist())):\n    train_df.loc[val_idx, 'Fold'] = fold\n    \ntrain_df.to_csv('train_df_with_fold.csv')\ntrain_df.sample(10)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T18:25:27.992107Z","iopub.execute_input":"2021-08-27T18:25:27.992409Z","iopub.status.idle":"2021-08-27T18:25:28.109506Z","shell.execute_reply.started":"2021-08-27T18:25:27.99238Z","shell.execute_reply":"2021-08-27T18:25:28.108793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2021-08-27T18:25:41.567187Z","iopub.execute_input":"2021-08-27T18:25:41.567512Z","iopub.status.idle":"2021-08-27T18:25:41.575389Z","shell.execute_reply.started":"2021-08-27T18:25:41.567482Z","shell.execute_reply":"2021-08-27T18:25:41.574399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #027fc1; font-family: Segoe UI; font-size: 1.9em; font-weight: 300;\">🚀 Training</span>\n\nWe first find the GCS path of the selected EffNetV2 architecture from the EffNetV2 weights Kaggle dataset.","metadata":{}},{"cell_type":"markdown","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nuser_credential = user_secrets.get_gcloud_credential()\nuser_secrets.set_tensorflow_credential(user_credential)","metadata":{"execution":{"iopub.status.busy":"2021-08-13T20:51:15.477772Z","iopub.execute_input":"2021-08-13T20:51:15.478046Z","iopub.status.idle":"2021-08-13T20:51:16.811907Z","shell.execute_reply.started":"2021-08-13T20:51:15.478013Z","shell.execute_reply":"2021-08-13T20:51:16.810812Z"}}},{"cell_type":"code","source":"# Get the TensorFlow Hub model URL\nhub_type = 'feature_vector' # ['classification', 'feature_vector']\n# Get the GCS path of EfficientNet Models\nDS_GCS_PATH = KaggleDatasets().get_gcs_path(\"efficientnetv2-tfhub-weight-files\")\nMODEL_GCS_PATH = f'{DS_GCS_PATH}/tfhub_models/{cfg.model_arch}/{hub_type}'\n\n# Get the GCS path of the images from the Kaggle dataset\n#GCS_DS_PATH =  KaggleDatasets().get_gcs_path(cfg.dataset)+\"/ljmu-siim-covid19/exp1/train/\"\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(cfg.dataset)+\"/\"\n\nprint(DS_GCS_PATH)\nprint()\nprint(GCS_DS_PATH)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T18:26:04.697147Z","iopub.execute_input":"2021-08-27T18:26:04.697488Z","iopub.status.idle":"2021-08-27T18:27:36.994997Z","shell.execute_reply.started":"2021-08-27T18:26:04.697442Z","shell.execute_reply":"2021-08-27T18:27:36.994338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cloud Storage\nfrom google.cloud import storage\nstorage_client = storage.Client(project='YOUR PROJECT ID')","metadata":{}},{"cell_type":"code","source":"import wandb\nfrom kaggle_secrets import UserSecretsClient\n\nuser_secrets = UserSecretsClient()\nwandb_api = user_secrets.get_secret(\"wandb_apikey\") \nwandb.login(key=wandb_api)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T18:28:34.034273Z","iopub.execute_input":"2021-08-27T18:28:34.03464Z","iopub.status.idle":"2021-08-27T18:28:34.210756Z","shell.execute_reply.started":"2021-08-27T18:28:34.034606Z","shell.execute_reply":"2021-08-27T18:28:34.209688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DISPLAY_PLOT = True\noof_aucs = dict()\n\nAUG_COUNT=0\n\nfor fold in range(cfg.kfold):\n    \n    print(\"\\nFold:\", fold)\n    \n    # Define TPU strategy and clear TPU\n    strategy = auto_select_accelerator()\n    \n    # Converting global config class object to a dictionary to log using Wandb\n    config_dict = dict(vars(Config))\n    config_dict = {k:(v if type(v)==int else str(v)) for (k,v) in config_dict.items() if '__' not in k}\n    config_dict['fold'] = fold\n    config_dict['job_name'] = f\"{config_dict['model_arch']}_fold{fold}_job{config_dict['job']}\"\n    print(\"Train Job:\", config_dict['job_name'], \"\\nConfig\")\n    pprint.pprint(config_dict)\n\n    wandb.init(project=cfg.wandb_project, name=config_dict['job_name'], config=config_dict)\n\n    valid_paths = GCS_DS_PATH + train_df[train_df['Fold']==fold]['Id'] # gs://.../00abxyz.jpg\n    train_paths = GCS_DS_PATH + train_df[train_df['Fold']!=fold]['Id']\n    \n    valid_labels = train_df[train_df['Fold']==fold][label_cols].values\n    train_labels = train_df[train_df['Fold']!=fold][label_cols].values\n\n    decoder = build_decoder(with_labels=True, target_size=cfg.input_dims, ext='jpg')\n    test_decoder = build_decoder(with_labels=False, target_size=cfg.input_dims, ext='jpg')\n\n    train_dataset = build_dataset(\n        train_paths, train_labels, bsize=cfg.batch_size, decode_fn=decoder\n        #repeat=False, shuffle=False, augment=False\n    )\n\n    valid_dataset = build_dataset(\n        valid_paths, valid_labels, bsize=cfg.batch_size, decode_fn=decoder,\n        repeat=False, shuffle=False, augment=False\n    )\n\n    num_classes = cfg.num_classes if cfg.num_classes else train_labels.shape[1]\n\n    with strategy.scope():\n        model = tf.keras.Sequential([\n            # Explicitly define the input shape so the model can be properly\n            # loaded by the TFLiteConverter\n            tf.keras.layers.InputLayer(input_shape=[cfg.input_dims[0], cfg.input_dims[1], 3]),\n            tfhub.KerasLayer(MODEL_GCS_PATH, trainable=cfg.fine_tune),\n            tf.keras.layers.Dropout(rate=0.1),\n            tf.keras.layers.Dense(num_classes,\n                                  activation='sigmoid')\n        ])\n\n        model.build((None, cfg.input_dims[0], cfg.input_dims[1], 3))\n        model.summary()\n\n        metrics = ['accuracy', tf.keras.metrics.AUC(name='auc', multi_label=True)]\n    \n        model.compile(\n          optimizer=tf.keras.optimizers.Adam(learning_rate=cfg.lr),\n          loss=cfg.loss_func,\n          metrics=metrics)\n\n    steps_per_epoch = train_paths.shape[0] // cfg.batch_size\n    callbacks = [\n        tf.keras.callbacks.ModelCheckpoint(f'model{fold}.h5',\n                                           save_best_only=True,\n                                           monitor='val_loss',\n                                           mode='min'),\n        wandb.keras.WandbCallback(save_model=True, monitor='val_loss',  mode='min'),\n        tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\",\n                                             patience=3,\n                                             min_lr=1e-6,\n                                             mode='min'),\n                ]\n\n    history = model.fit(\n        train_dataset, \n        epochs=cfg.n_epochs,\n        verbose=1,\n        callbacks=callbacks,\n        steps_per_epoch=steps_per_epoch,\n        validation_data=valid_dataset)\n\n    history_df = pd.DataFrame(history.history)\n    history_df.to_csv(f'history{fold}.csv')\n\n    del decoder, test_decoder, train_dataset, valid_dataset, model\n    gc.collect()\n\n    oof_aucs[fold] = float(np.max(history.history['val_auc']))\n    print(\"oof_aucs\", oof_aucs)\n    \n    wandb.finish()\n    \n    # Plot Training History\n    if DISPLAY_PLOT:\n        print (\"\\n\\n\")\n        plt.figure(figsize=(15,5))\n        plt.plot(np.arange(len(history.history['auc'])), history.history['auc'],\n                 '-o', label='Train auc', color='#ff7f0e')\n        plt.plot(np.arange(len(history.history['auc'])), history.history['val_auc'],\n                 '-o', label='Val auc', color='#1f77b4')\n        x = np.argmax(history.history['val_auc'])\n        y = np.max(history.history['val_auc'])\n        xdist = plt.xlim()[1] - plt.xlim()[0]\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200, color='#1f77b4')\n        plt.text(x-0.03*xdist, y-0.13*ydist, 'Max AUC\\n%.2f'%y, size=10)\n        plt.ylabel('auc', size=14)\n        plt.xlabel('Epoch', size=14)\n        plt.legend(loc=2)\n\n        plt2 = plt.gca().twinx()\n        plt2.plot(np.arange(len(history.history['auc'])),\n                  history.history['loss'],'-o', label='Train Loss', color='#2ca02c')\n        plt2.plot(np.arange(len(history.history['auc'])),\n                  history.history['val_loss'],'-o', label='Val Loss', color='#d62728')\n        x = np.argmin(history.history['val_loss'])\n        y = np.min(history.history['val_loss'])\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x, y, s=200, color='#d62728')\n        plt.text(x-0.03*xdist, y+0.05*ydist,'Min Loss', size=10)\n        plt.ylabel('Loss', size=14)\n        plt.title(f\"{config_dict['job_name']} Size - {cfg.input_dims}\")\n        plt.legend(loc=3)\n        plt.savefig(f'fig{fold}.png')\n        plt.show()\n        \nprint(\"\\n\"+\"-\"*40)\nprint(\"CV AUC:\", sum(list(oof_aucs.values()))/cfg.kfold)\nprint(\"-\"*40)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T18:29:55.070334Z","iopub.execute_input":"2021-08-27T18:29:55.070667Z","iopub.status.idle":"2021-08-27T18:40:10.940994Z","shell.execute_reply.started":"2021-08-27T18:29:55.070638Z","shell.execute_reply":"2021-08-27T18:40:10.9355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!rm -r /kaggle/working/wandb\nprint(\"Completed ...\")","metadata":{"execution":{"iopub.status.busy":"2021-08-26T21:35:11.622966Z","iopub.status.idle":"2021-08-26T21:35:11.62394Z"},"trusted":true},"execution_count":null,"outputs":[]}]}