{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"![](https://i.ibb.co/Zm9Rmdb/lung-nb4-short.jpg)\n\n

SIIM COVID-19 EffNetV2 Keras Study Level Train [TPU]

\n\nOverview\n\n  โœ…  Training Official TF EfficientNetV2 Models on TPU [**CV AUC 0.804+**]
\n  โœ…  Minimal Overhead TPU Augmentations
\n  โœ…  Weights & Biases Integration
\n
\n\n**Please use v6 or above of this notebook, v4 or above of the TFHub Weight Files dataset and v6 or above of the inference notebook as some of the weight files are fixed.**\n\n
\n\n

๐Ÿท๏ธ Dataset with EffNetV2 TfHub Weights used in this notebook:

\n\n\n> [EfficientNetV2 TFHub Weight Files](https://www.kaggle.com/sreevishnudamodaran/efficientnetv2-tfhub-weight-files?select=tfhub_models)
\n Official EfficientNetV2 Saved Model Files from tfhub.dev\n\n
\n\n

๐Ÿท๏ธ Inference notebook for EffNetV2_L and Cascade RCNN:

\n\n>[SIIM EffNetV2_L CascadeRCNN MMDetection Infer](https://www.kaggle.com/sreevishnudamodaran/siim-effnetv2-l-cascadercnn-mmdetection-infer)\n
\n\nReferences:\n\n- https://www.kaggle.com/xhlulu/ranzcr-efficientnet-tpu-training\n- https://www.kaggle.com/h053473666/siim-covid19-efnb7-train-study\n- https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\n- https://github.com/google/automl/blob/master/efficientnetv2\n\n
\nRelated Notebooks:\n\nEfficientNetV2 with Fastai:\n[SIIM Covid19 Fastai+EfficientNetV2+TIMM Models๐Ÿ’ ๐Ÿฎ](https://www.kaggle.com/sreevishnudamodaran/siim-covid19-fastai-efficientnetv2-timm-models)\n\n
\n
\"Ask
\n
\n
\"Ask
","metadata":{}},{"cell_type":"code","source":"import itertools\nimport os\nimport random\nimport gc\nimport numpy as np\nimport pandas as pd\nimport math\nimport pprint\nimport matplotlib.pylab as plt\nimport seaborn as sns\nsns.set(rc={\"axes.titlesize\":15, \"axes.labelsize\":9,\"axes.titlepad\":15,\n \"axes.labelpad\":12, \"legend.fontsize\":9,\n \"legend.title_fontsize\":9, \"figure.titlesize\":15,\n \"axes.grid\":False})\n\nfrom sklearn.model_selection import train_test_split, GroupKFold\nimport tensorflow as tf\nimport tensorflow_hub as tfhub\nimport tensorflow.keras.backend as K\nfrom kaggle_datasets import KaggleDatasets\n\nprint('TF version:', tf.__version__)\nprint('Hub version:', tfhub.__version__)\nprint('Physical devices:', tf.config.list_physical_devices())","metadata":{"execution":{"iopub.status.busy":"2021-07-22T06:46:02.072899Z","iopub.execute_input":"2021-07-22T06:46:02.073316Z","iopub.status.idle":"2021-07-22T06:46:09.1009Z","shell.execute_reply.started":"2021-07-22T06:46:02.073234Z","shell.execute_reply":"2021-07-22T06:46:09.099297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data Loaders & Augmentations on TPU\n\nThanks to [@cdeotte](https://www.kaggle.com/cdeotte) & [@xhlulu](https://www.kaggle.com/xhlulu)","metadata":{}},{"cell_type":"code","source":"SATURATION = (0.9, 1.1)\nCONTRAST = (0.9, 1.1)\nBRIGHTNESS = 0.1\nROTATION = 10.0\nSHEAR = 2.0\nHZOOM = 8.0\nWZOOM = 4.0\nHSHIFT = 4.0\nWSHIFT = 4.0\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n # returns 3x3 transformmatrix which transforms indicies\n \n # CONVERT DEGREES TO RADIANS\n rotation = math.pi * rotation / 180.\n shear = math.pi * shear / 180.\n\n def get_3x3_mat(lst):\n return tf.reshape(tf.concat([lst],axis=0), [3,3])\n \n # ROTATION MATRIX\n c1 = tf.math.cos(rotation)\n s1 = tf.math.sin(rotation)\n one = tf.constant([1],dtype='float32')\n zero = tf.constant([0],dtype='float32')\n \n rotation_matrix = get_3x3_mat([c1, s1, zero, \n -s1, c1, zero, \n zero, zero, one]) \n # SHEAR MATRIX\n c2 = tf.math.cos(shear)\n s2 = tf.math.sin(shear) \n \n shear_matrix = get_3x3_mat([one, s2, zero, \n zero, c2, zero, \n zero, zero, one]) \n # ZOOM MATRIX\n zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero, \n zero, one/width_zoom, zero, \n zero, zero, one]) \n # SHIFT MATRIX\n shift_matrix = get_3x3_mat([one, zero, height_shift, \n zero, one, width_shift, \n zero, zero, one])\n \n return K.dot(K.dot(rotation_matrix, shear_matrix), \n K.dot(zoom_matrix, shift_matrix))\n\n\ndef transform(image, label):\n # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n # output - image randomly rotated, sheared, zoomed, and shifted\n DIM = cfg.input_dims[0]\n XDIM = DIM%2\n \n rot = ROTATION * tf.random.normal([1], dtype='float32')\n shr = SHEAR * tf.random.normal([1], dtype='float32')\n h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM\n w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM\n h_shift = HSHIFT * tf.random.normal([1], dtype='float32')\n w_shift = WSHIFT * tf.random.normal([1], dtype='float32')\n \n # GET TRANSFORMATION MATRIX\n m = get_mat(rot, shr, h_zoom, w_zoom, h_shift, w_shift) \n\n # LIST DESTINATION PIXEL INDICES\n x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n z = tf.ones([DIM*DIM],dtype='int32')\n idx = tf.stack( [x,y,z] )\n \n # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n idx2 = K.cast(idx2,dtype='int32')\n idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n \n # FIND ORIGIN PIXEL VALUES \n idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n d = tf.gather_nd(image,tf.transpose(idx3))\n \n return tf.reshape(d,[DIM,DIM,3]), label","metadata":{"execution":{"iopub.status.busy":"2021-07-22T06:46:09.102336Z","iopub.execute_input":"2021-07-22T06:46:09.102636Z","iopub.status.idle":"2021-07-22T06:46:09.122814Z","shell.execute_reply.started":"2021-07-22T06:46:09.102608Z","shell.execute_reply":"2021-07-22T06:46:09.121755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n try:\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n tf.config.experimental_connect_to_cluster(tpu)\n tf.tpu.experimental.initialize_tpu_system(tpu)\n strategy = tf.distribute.experimental.TPUStrategy(tpu)\n print(\"Running on TPU:\", tpu.master())\n except ValueError:\n strategy = tf.distribute.get_strategy()\n print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n \n return strategy\n\n\ndef build_decoder(with_labels=True, target_size=(256, 256), ext='jpg'):\n def decode(path):\n file_bytes = tf.io.read_file(path)\n\n if ext == 'png':\n img = tf.image.decode_png(file_bytes, channels=3)\n elif ext in ['jpg', 'jpeg']:\n img = tf.image.decode_jpeg(file_bytes, channels=3)\n else:\n raise ValueError(\"Image extension not supported\")\n img = tf.cast(img, tf.float32) / 255.0\n img = tf.image.resize(img, target_size)\n\n return img\n \n def decode_with_labels(path, label):\n return decode(path), label\n \n return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True):\n def augment(img):\n img = tf.image.random_flip_left_right(img)\n# img = tf.image.random_flip_up_down(img)\n img = tf.image.random_saturation(img, SATURATION[0], SATURATION[1])\n img = tf.image.random_contrast(img, CONTRAST[0], CONTRAST[1])\n img = tf.image.random_brightness(img, BRIGHTNESS)\n return img\n \n def augment_with_labels(img, label):\n return augment(img), label\n \n return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, bsize=128, cache=True,\n decode_fn=None, augment_fn=None,\n augment=True, repeat=True, shuffle=1024,\n seed=None, cache_dir=\"\"):\n if cache_dir != \"\" and cache is True:\n os.makedirs(cache_dir, exist_ok=True)\n \n if decode_fn is None:\n decode_fn = build_decoder(labels is not None)\n \n if augment_fn is None:\n augment_fn = build_augmenter(labels is not None)\n \n AUTO = tf.data.experimental.AUTOTUNE\n slices = paths if labels is None else (paths, labels)\n \n dset = tf.data.Dataset.from_tensor_slices(slices)\n dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n dset = dset.cache(cache_dir) if cache else dset\n \n # Map the functions to perform Augmentations\n dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n dset = dset.map(transform, num_parallel_calls=AUTO) if augment else dset\n dset = dset.repeat() if repeat else dset\n dset = dset.shuffle(shuffle, seed=seed) if shuffle else dset\n dset = dset.batch(bsize).prefetch(AUTO)\n \n return dset","metadata":{"execution":{"iopub.status.busy":"2021-07-22T06:46:09.124635Z","iopub.execute_input":"2021-07-22T06:46:09.124967Z","iopub.status.idle":"2021-07-22T06:46:09.144053Z","shell.execute_reply.started":"2021-07-22T06:46:09.124936Z","shell.execute_reply":"2021-07-22T06:46:09.143058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"๐Ÿ“ฅ EffNetV2 Architecture Selection\n\n
\n\n\n\n \n \n \n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n
Model Architecture
efficientnetv2-b0
efficientnetv2-b1
efficientnetv2-b2
efficientnetv2-b3
efficientnetv2-s
efficientnetv2-m
efficientnetv2-l
efficientnetv2-s-21k
efficientnetv2-m-21k
efficientnetv2-l-21k
efficientnetv2-s-21k-ft1k
efficientnetv2-m-21k-ft1k
efficientnetv2-l-21k-ft1k
\n\n
\n\n๐Ÿ“Œ The model architectures with no suffixes are pretrained on ImageNet1K. The ones with the '21k' as the suffix are pretrained on ImageNet21K and the ones with '21k-ft1k' as the suffix are pretrained on ImageNet21K and then finetuned on ImageNet1K.\n\n
\n\n **ImageNet1K pretrained and finetuned models:**\n\n| ImageNet1K | Top1 Acc. | Params | FLOPs | Inference Latency | links |\n| ---------- | ------ | ------ | ------ | ------ | ------ |\n| EffNetV2-S | 83.9% | 21.5M | 8.4B | [V100/A100](g3doc/effnetv2-s-gpu.png) | [ckpt](https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/v2/efficientnetv2-s.tgz), [tensorboard](https://tensorboard.dev/experiment/wozwYcXkRPia76RopgCLlg)\n| EffNetV2-M | 85.2% | 54.1M | 24.7B | [V100/A100](g3doc/effnetv2-m-gpu.png) | [ckpt](https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/v2/efficientnetv2-m.tgz), [tensorboard](https://tensorboard.dev/experiment/syoaqB2gTP6Vr0KRlrezmg)\n| EffNetV2-L | 85.7% | 119.5M | 56.3B | [V100/A100](g3doc/effnetv2-l-gpu.png) | [ckpt](https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/v2/efficientnetv2-l.tgz), [tensorboard](https://tensorboard.dev/experiment/qgnTQ5JZQ92nSex6ZlWBbQ)\n\n
\n\n**Models Pretrained on ImageNet21K pretrained and finetuned with ImageNet1K:**\n\n\n| ImageNet21K | Pretrained models | Finetuned ImageNet1K |\n| ---------- | ------ | ------ |\n| EffNetV2-S | [pretrain ckpt](https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/v2/efficientnetv2-s-21k.tgz) | top1=84.9%, [ckpt](https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/v2/efficientnetv2-s-21k-ft1k.tgz), [tensorboard](https://tensorboard.dev/experiment/7sga2olqTBeH4ioydel0hg/) |\n| EffNetV2-M | [pretrain ckpt](https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/v2/efficientnetv2-m-21k.tgz) | top1=86.2%, [ckpt](https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/v2/efficientnetv2-m-21k-ft1k.tgz), [tensorboard](https://tensorboard.dev/experiment/HkV6ANZSQ6WI5GhlZa48xQ/) |\n| EffNetV2-L | [pretrain ckpt](https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/v2/efficientnetv2-l-21k.tgz) | top1=86.9%, [ckpt](https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/v2/efficientnetv2-l-21k-ft1k.tgz), [tensorboard](https://tensorboard.dev/experiment/m9ZHx1L6SQu5iBYhXO5jOw/) |","metadata":{}},{"cell_type":"markdown","source":"Global Config & Seeding","metadata":{}},{"cell_type":"code","source":"def seed_everything(SEED):\n os.environ['PYTHONHASHSEED']=str(SEED)\n random.seed(SEED)\n np.random.seed(SEED)\n tf.random.set_seed(SEED)\n os.environ['TF_CUDNN_DETERMINISTIC'] = str(SEED)","metadata":{"execution":{"iopub.status.busy":"2021-07-22T06:46:09.14559Z","iopub.execute_input":"2021-07-22T06:46:09.145892Z","iopub.status.idle":"2021-07-22T06:46:09.157475Z","shell.execute_reply.started":"2021-07-22T06:46:09.145864Z","shell.execute_reply":"2021-07-22T06:46:09.156403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n seed = 333\n job = 1\n num_classes = 4\n input_dims = (768, 768)\n model_arch = \"efficientnetv2-l-21k-ft1k\" ## Choose model architecture\n batch_size = 8*16\n kfold = 5\n n_epochs = 15\n lr = 0.001\n loss_func = 'categorical_crossentropy'\n # Whether to finetune the whole model or just the top layer.\n fine_tune = True\n wandb_project = 'SIIM_classifier_public'\n dataset = \"siim-covid19-images-metadata-256-512-768\"\n \n seed_everything(seed)\n \ncfg = Config()","metadata":{"execution":{"iopub.status.busy":"2021-07-22T06:46:09.159039Z","iopub.execute_input":"2021-07-22T06:46:09.159471Z","iopub.status.idle":"2021-07-22T06:46:09.170994Z","shell.execute_reply.started":"2021-07-22T06:46:09.159425Z","shell.execute_reply":"2021-07-22T06:46:09.169828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"KFold Split","metadata":{}},{"cell_type":"code","source":"# df = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\n# df = pd.read_csv('../input/siim-covid19-images-metadata-256-512-768/images_metadata_256_512_768/train_meta_768x768.csv')\ndf = pd.read_csv('../input/siim-covid19-images-metadata-256-512-768/images_metadata_256_512_768/train_meta_768x768.csv')\nstudy_df = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\nstudy_df = study_df.rename({'id':'study_id'}, axis=1)\ndf['study_id'] = df['study_id']+ '_study'\ndf = pd.merge(df, study_df, on = 'study_id', how = 'left')\n\nlabel_cols = ['Negative for Pneumonia','Typical Appearance',\n 'Indeterminate Appearance','Atypical Appearance']\n\ngkf = GroupKFold(n_splits=cfg.kfold)\ndf['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(df, groups=df.study_id.tolist())):\n df.loc[val_idx, 'fold'] = fold\n \ndf.to_csv('study_train_df.csv')\ndf.sample(3)","metadata":{"execution":{"iopub.status.busy":"2021-07-22T06:46:09.172389Z","iopub.execute_input":"2021-07-22T06:46:09.172697Z","iopub.status.idle":"2021-07-22T06:46:09.382388Z","shell.execute_reply.started":"2021-07-22T06:46:09.172669Z","shell.execute_reply":"2021-07-22T06:46:09.381492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"๐Ÿš€ Training\n\nWe first find the GCS path of the selected EffNetV2 architecture from the EffNetV2 weights Kaggle dataset.","metadata":{}},{"cell_type":"code","source":"# Get the TensorFlow Hub model URL\nhub_type = 'feature_vector' # ['classification', 'feature_vector']\n# Get the GCS path of EfficientNet Models\nDS_GCS_PATH = KaggleDatasets().get_gcs_path(\"efficientnetv2-tfhub-weight-files\")\nMODEL_GCS_PATH = f'{DS_GCS_PATH}/tfhub_models/{cfg.model_arch}/{hub_type}'\nMODEL_GCS_PATH","metadata":{"execution":{"iopub.status.busy":"2021-07-22T06:46:09.383826Z","iopub.execute_input":"2021-07-22T06:46:09.384222Z","iopub.status.idle":"2021-07-22T06:46:09.813257Z","shell.execute_reply.started":"2021-07-22T06:46:09.384178Z","shell.execute_reply":"2021-07-22T06:46:09.812232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\nfrom kaggle_secrets import UserSecretsClient\n\nuser_secrets = UserSecretsClient()\nwandb_api = user_secrets.get_secret(\"wandb_key\") \nwandb.login(key=wandb_api)","metadata":{"execution":{"iopub.status.busy":"2021-07-22T06:46:09.815872Z","iopub.execute_input":"2021-07-22T06:46:09.81632Z","iopub.status.idle":"2021-07-22T06:46:11.294149Z","shell.execute_reply.started":"2021-07-22T06:46:09.816275Z","shell.execute_reply":"2021-07-22T06:46:11.293115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DISPLAY_PLOT = True\noof_aucs = dict()\n\n# Get the GCS path of the images from the Kaggle dataset\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(cfg.dataset)\n\nfor fold in range(cfg.kfold):\n \n print(\"\\nFold:\", fold)\n \n # Define TPU strategy and clear TPU\n strategy = auto_select_accelerator()\n \n # Converting global config class object to a dictionary to log using Wandb\n config_dict = dict(vars(Config))\n config_dict = {k:(v if type(v)==int else str(v)) for (k,v) in config_dict.items() if '__' not in k}\n config_dict['fold'] = fold\n config_dict['job_name'] = f\"{config_dict['model_arch']}_fold{fold}_job{config_dict['job']}\"\n print(\"Train Job:\", config_dict['job_name'], \"\\nConfig\")\n pprint.pprint(config_dict)\n\n wandb.init(project=cfg.wandb_project, name=config_dict['job_name'], config=config_dict)\n\n valid_paths = GCS_DS_PATH + '/images_metadata_256_512_768/train_768x768/' + df[df['fold']==fold]['id'] + '.png' #\"/train/\"\n train_paths = GCS_DS_PATH + '/images_metadata_256_512_768/train_768x768/' + df[df['fold']!=fold]['id'] + '.png' #\"/train/\" \n valid_labels = df[df['fold']==fold][label_cols].values\n train_labels = df[df['fold']!=fold][label_cols].values\n\n decoder = build_decoder(with_labels=True, target_size=cfg.input_dims, ext='png')\n test_decoder = build_decoder(with_labels=False, target_size=cfg.input_dims, ext='png')\n\n train_dataset = build_dataset(\n train_paths, train_labels, bsize=cfg.batch_size, decode_fn=decoder\n )\n\n valid_dataset = build_dataset(\n valid_paths, valid_labels, bsize=cfg.batch_size, decode_fn=decoder,\n repeat=False, shuffle=False, augment=False\n )\n\n num_classes = cfg.num_classes if cfg.num_classes else train_labels.shape[1]\n\n with strategy.scope():\n model = tf.keras.Sequential([\n # Explicitly define the input shape so the model can be properly\n # loaded by the TFLiteConverter\n tf.keras.layers.InputLayer(input_shape=[cfg.input_dims[0], cfg.input_dims[1], 3]),\n tfhub.KerasLayer(MODEL_GCS_PATH, trainable=cfg.fine_tune),\n tf.keras.layers.Dropout(rate=0.1),\n tf.keras.layers.Dense(num_classes,\n activation='softmax')\n ])\n\n model.build((None, cfg.input_dims[0], cfg.input_dims[1], 3))\n# model.summary()\n\n metrics = ['accuracy', tf.keras.metrics.AUC(name='auc', multi_label=True)]\n model.compile(\n optimizer=tf.keras.optimizers.Adam(learning_rate=cfg.lr),\n loss=cfg.loss_func,\n metrics=metrics)\n\n steps_per_epoch = train_paths.shape[0] // cfg.batch_size\n callbacks = [\n tf.keras.callbacks.ModelCheckpoint(f'model{fold}.h5',\n save_best_only=True,\n monitor='val_loss',\n mode='min'),\n wandb.keras.WandbCallback(save_model=True,\n monitor='val_loss',\n mode='min'),\n tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\",\n patience=3,\n min_lr=1e-6,\n mode='min'),\n ]\n\n history = model.fit(\n train_dataset, \n epochs=cfg.n_epochs,\n verbose=1,\n callbacks=callbacks,\n steps_per_epoch=steps_per_epoch,\n validation_data=valid_dataset)\n\n history_df = pd.DataFrame(history.history)\n history_df.to_csv(f'history{fold}.csv')\n\n del decoder, test_decoder, train_dataset, valid_dataset, model\n gc.collect()\n\n oof_aucs[fold] = float(np.max(history.history['val_auc']))\n print(\"oof_aucs\", oof_aucs)\n \n wandb.finish()\n \n # Plot Training History\n if DISPLAY_PLOT:\n print (\"\\n\\n\")\n plt.figure(figsize=(15,5))\n plt.plot(np.arange(len(history.history['auc'])), history.history['auc'],\n '-o', label='Train auc', color='#ff7f0e')\n plt.plot(np.arange(len(history.history['auc'])), history.history['val_auc'],\n '-o', label='Val auc', color='#1f77b4')\n x = np.argmax(history.history['val_auc'])\n y = np.max(history.history['val_auc'])\n xdist = plt.xlim()[1] - plt.xlim()[0]\n ydist = plt.ylim()[1] - plt.ylim()[0]\n plt.scatter(x,y,s=200, color='#1f77b4')\n plt.text(x-0.03*xdist, y-0.13*ydist, 'Max AUC\\n%.2f'%y, size=10)\n plt.ylabel('auc', size=14)\n plt.xlabel('Epoch', size=14)\n plt.legend(loc=2)\n\n plt2 = plt.gca().twinx()\n plt2.plot(np.arange(len(history.history['auc'])),\n history.history['loss'],'-o', label='Train Loss', color='#2ca02c')\n plt2.plot(np.arange(len(history.history['auc'])),\n history.history['val_loss'],'-o', label='Val Loss', color='#d62728')\n x = np.argmin(history.history['val_loss'])\n y = np.min(history.history['val_loss'])\n ydist = plt.ylim()[1] - plt.ylim()[0]\n plt.scatter(x, y, s=200, color='#d62728')\n plt.text(x-0.03*xdist, y+0.05*ydist,'Min Loss', size=10)\n plt.ylabel('Loss', size=14)\n plt.title(f\"{config_dict['job_name']} Size - {cfg.input_dims}\")\n plt.legend(loc=3)\n plt.savefig(f'fig{fold}.png')\n plt.show()\n \nprint(\"\\n\"+\"-\"*40)\nprint(\"CV AUC:\", sum(list(oof_aucs.values()))/cfg.kfold)\nprint(\"-\"*40)","metadata":{"execution":{"iopub.status.busy":"2021-07-22T06:46:11.295887Z","iopub.execute_input":"2021-07-22T06:46:11.296204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r /kaggle/working/wandb","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

HAVE A GREAT DAY!

\n\n

Let me know if you have any suggestions!

","metadata":{}}]}