{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This kernel builds a \"study-level\" classifier. It's ideally the first approach that many would have tried so far. I have been using this kernel to train study-level classifier and after refining it have decided to share it. I hope you all find it useful. It's written using TensorFlow and uses Weights and Biases for experiment tracking. \n\nI have also written three other kernels at the start of this competition: \n\n* [Visualize Bounding Boxes Interactively](https://www.kaggle.com/ayuraj/visualize-bounding-boxes-interactively)\n\n* [[Train] COVID-19 Detection using YOLOv5](https://www.kaggle.com/ayuraj/train-covid-19-detection-using-yolov5)\n\n* [Submission Covid19](https://www.kaggle.com/ayuraj/submission-covid19)\n\nThe Submission Covid19 kernel uses the study level model trained using this kernel. \n\nIf you like the work consider showing some love. :)","metadata":{}},{"cell_type":"markdown","source":"# 🧰 Imports and Setups","metadata":{}},{"cell_type":"code","source":"# Import the latest version of wandb\n!pip install -q --upgrade wandb","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:36:25.998990Z","iopub.execute_input":"2021-06-30T15:36:25.999310Z","iopub.status.idle":"2021-06-30T15:36:32.475864Z","shell.execute_reply.started":"2021-06-30T15:36:25.999281Z","shell.execute_reply":"2021-06-30T15:36:32.474802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nprint(tf.__version__)\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import models\nimport tensorflow_addons as tfa\nfrom tensorflow.keras import mixed_precision\n\n\nimport tensorflow_probability as tfp\ntfd = tfp.distributions\n\nimport os\nimport gc\nimport json\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.utils.class_weight import compute_class_weight\n\n# Imports for augmentations. \nfrom albumentations import (Compose, RandomResizedCrop, Cutout, Rotate, HorizontalFlip, \n                            VerticalFlip, RandomBrightnessContrast, ShiftScaleRotate, \n                            CenterCrop, Resize)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:36:58.464164Z","iopub.execute_input":"2021-06-30T15:36:58.464496Z","iopub.status.idle":"2021-06-30T15:36:58.477315Z","shell.execute_reply.started":"2021-06-30T15:36:58.464464Z","shell.execute_reply":"2021-06-30T15:36:58.476145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# W&B related imports\nimport wandb\nprint(wandb.__version__)\nfrom wandb.keras import WandbCallback\n\nwandb.login()","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2021-06-30T15:37:00.441116Z","iopub.execute_input":"2021-06-30T15:37:00.441444Z","iopub.status.idle":"2021-06-30T15:37:00.458779Z","shell.execute_reply.started":"2021-06-30T15:37:00.441414Z","shell.execute_reply":"2021-06-30T15:37:00.454737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Increase GPU memory as per the need.\ngpus = tf.config.list_physical_devices('GPU')\nif gpus:\n  try:\n    # Currently, memory growth needs to be the same across GPUs\n    for gpu in gpus:\n      tf.config.experimental.set_memory_growth(gpu, True)\n    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n  except RuntimeError as e:\n    # Memory growth must be set before GPUs have been initialized\n    print(e)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:37:04.607439Z","iopub.execute_input":"2021-06-30T15:37:04.607778Z","iopub.status.idle":"2021-06-30T15:37:04.617808Z","shell.execute_reply.started":"2021-06-30T15:37:04.607744Z","shell.execute_reply":"2021-06-30T15:37:04.616950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📀 Hyperparameters","metadata":{}},{"cell_type":"code","source":"TRAIN_PATH = '../input/siim-covid19-resized-to-224px-png/train/'\nAUTOTUNE = tf.data.AUTOTUNE\n\nCONFIG = dict (\n    seed = 42,\n    num_labels = 4,\n    num_folds = 5,\n    img_width = 224, # If you change the resolution to 512 reduce batch size. \n    img_height = 224,\n    batch_size = 32,\n    epochs = 100,\n    learning_rate = 1e-3,\n    architecture = \"CNN\",\n    competition = 'siim-covid',\n    _wandb_kernel = 'ayut',\n    infra = \"GCP\",\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:37:05.946371Z","iopub.execute_input":"2021-06-30T15:37:05.946698Z","iopub.status.idle":"2021-06-30T15:37:05.951948Z","shell.execute_reply.started":"2021-06-30T15:37:05.946668Z","shell.execute_reply":"2021-06-30T15:37:05.950964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔨 Build Input Pipeline","metadata":{}},{"cell_type":"code","source":"# read training csv file\ndf = pd.read_csv('../input/siim-covid-merged-train-labels/image_study_total.csv')\ndf['path'] = df.apply(lambda row: TRAIN_PATH+row.id+'.png', axis=1)\n\n# Group by Study Ids and remove images that are \"assumed\" to be mislabeled\n# Ref: https://www.kaggle.com/c/siim-covid19-detection/discussion/246597\nfor grp_df in df.groupby('StudyInstanceUID'):\n    grp_id, grp_df = grp_df[0], grp_df[1]\n    if len(grp_df) == 1:\n        pass\n    else:\n        for i in range(len(grp_df)):\n            row = grp_df.loc[grp_df.index.values[i]]\n            if row.study_level > 0 and row.boxes is np.nan:\n                df = df.drop(grp_df.index.values[i])\n                \ndf = df.drop('boxes', axis=1).reset_index()\nFold = StratifiedKFold(n_splits=CONFIG['num_folds'], shuffle=True, random_state=CONFIG['seed'])\nfor n, (train_index, val_index) in enumerate(Fold.split(df, df['study_level'])):\n    df.loc[val_index, 'fold'] = int(n)\ndf['fold'] = df['fold'].astype(int)\ndf.groupby(['fold', 'study_level']).size()","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:37:09.601969Z","iopub.execute_input":"2021-06-30T15:37:09.602292Z","iopub.status.idle":"2021-06-30T15:37:10.194768Z","shell.execute_reply.started":"2021-06-30T15:37:09.602261Z","shell.execute_reply":"2021-06-30T15:37:10.193823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compute Class weights to mitigate class imbalance to some extent. \nfold_samp = df.loc[df.fold!=0]\nprint(len(fold_samp))\n\nclass_weights = compute_class_weight('balanced', \n                                    classes=np.unique(fold_samp['study_level'].values),\n                                    y=fold_samp['study_level'].values)\n\nclass_weights_dict = {key: val for key, val in zip(np.unique(fold_samp['study_level'].values), class_weights)}\nclass_weights_dict                                                            ","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:37:11.085035Z","iopub.execute_input":"2021-06-30T15:37:11.085360Z","iopub.status.idle":"2021-06-30T15:37:11.100208Z","shell.execute_reply.started":"2021-06-30T15:37:11.085330Z","shell.execute_reply":"2021-06-30T15:37:11.099224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef decode_image(image):\n    # convert the compressed string to a 3D uint8 tensor\n    image = tf.image.decode_png(image, channels=3)\n    # Normalize image\n    image = tf.image.convert_image_dtype(image, dtype=tf.float32)\n    return image\n\ndef load_image(df_dict):\n    # Load image\n    image = tf.io.read_file(df_dict['path'])\n    image = decode_image(image)\n    \n    # Parse label\n    label = df_dict['study_level']\n    label = tf.one_hot(indices=label, depth=CONFIG['num_labels'])\n    \n    return image, label\n\n# Mixup Augmentation policy\n@tf.function\ndef mixup(a, b):\n    alpha = [1.0]\n    beta = [1.0]\n\n    # unpack (image, label) pairs\n    (image1, label1), (image2, label2) = a, b\n\n    # define beta distribution\n    dist = tfd.Beta(alpha, beta)\n    # sample from this distribution\n    l = dist.sample(1)[0][0]\n\n    # mixup augmentation\n    img = l*image1+(1-l)*image2\n    lab = l*label1+(1-l)*label2\n\n    return img, lab","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:37:12.146772Z","iopub.execute_input":"2021-06-30T15:37:12.147111Z","iopub.status.idle":"2021-06-30T15:37:12.158334Z","shell.execute_reply.started":"2021-06-30T15:37:12.147081Z","shell.execute_reply":"2021-06-30T15:37:12.156080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CROP_SIZE = CONFIG['img_height']\n\n# Random Resized Crop\ntransforms = Compose([\n            HorizontalFlip(p=0.6),\n#             Rotate(limit=6, p=0.3),\n            ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=5, p=0.6),\n            RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, brightness_by_max=False, p=0.5),\n#             Cutout(num_holes=2, max_h_size=int(0.4*CONFIG['img_height']), max_w_size=int(0.4*CONFIG['img_height']), fill_value=0, always_apply=False, p=1.0)\n        ])\n\ndef aug_fn(image):\n    data = {\"image\":image}\n    aug_data = transforms(**data)\n    aug_img = aug_data[\"image\"]\n\n    return aug_img.astype(np.float32) \n\ndef augmentations(image, label):\n    aug_img = tf.numpy_function(func=aug_fn, inp=[image], Tout=tf.float32)\n    aug_img.set_shape((CROP_SIZE, CROP_SIZE, 3))\n\n    return aug_img, label","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:37:20.789396Z","iopub.execute_input":"2021-06-30T15:37:20.789733Z","iopub.status.idle":"2021-06-30T15:37:20.796255Z","shell.execute_reply.started":"2021-06-30T15:37:20.789700Z","shell.execute_reply":"2021-06-30T15:37:20.795482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\n\n# Simple dataloader\ndef get_dataloaders(train_df, valid_df):\n    trainloader = tf.data.Dataset.from_tensor_slices(dict(train_df))\n    validloader = tf.data.Dataset.from_tensor_slices(dict(valid_df))\n\n    trainloader = (\n        trainloader\n        .shuffle(1024)\n        .map(load_image, num_parallel_calls=AUTOTUNE)\n        .map(augmentations, num_parallel_calls=AUTOTUNE)\n        .batch(CONFIG['batch_size'])\n        .prefetch(AUTOTUNE)\n    )\n\n    validloader = (\n        validloader\n        .map(load_image, num_parallel_calls=AUTOTUNE)\n        .batch(CONFIG['batch_size'])\n        .prefetch(AUTOTUNE)\n    )\n    \n    return trainloader, validloader\n\n# Mixup\ndef get_mixup_dataloaders(train_df, valid_df):\n    trainloader1 = tf.data.Dataset.from_tensor_slices(dict(train_df)).shuffle(1024).map(load_image, num_parallel_calls=AUTOTUNE)\n    trainloader2 = tf.data.Dataset.from_tensor_slices(dict(train_df)).shuffle(1024).map(load_image, num_parallel_calls=AUTOTUNE)\n\n    trainloader = tf.data.Dataset.zip((trainloader1, trainloader2))\n\n    # Valid Loader\n    validloader = tf.data.Dataset.from_tensor_slices(dict(valid_df))\n\n    trainloader = (\n        trainloader\n        .shuffle(1024)\n        .map(mixup, num_parallel_calls=AUTOTUNE)\n        .map(augmentations, num_parallel_calls=AUTOTUNE)\n        .batch(CONFIG['batch_size'])\n        .prefetch(AUTOTUNE)\n    )\n\n    validloader = (\n        validloader\n        .map(load_image, num_parallel_calls=AUTOTUNE)\n        .batch(CONFIG['batch_size'])\n        .prefetch(AUTOTUNE)\n    )\n    \n    return trainloader, validloader","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:37:21.206609Z","iopub.execute_input":"2021-06-30T15:37:21.206935Z","iopub.status.idle":"2021-06-30T15:37:21.216178Z","shell.execute_reply.started":"2021-06-30T15:37:21.206906Z","shell.execute_reply":"2021-06-30T15:37:21.215099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_batch(image_batch, label_batch):\n    plt.figure(figsize=(20,20))\n    for n in range(25):\n        ax = plt.subplot(5,5,n+1)\n        plt.imshow(image_batch[n])\n        plt.title(np.argmax(label_batch[n].numpy()))\n        plt.axis('off')\n\n# Prepare train and valid df\ntrain_df = df.loc[df.fold != 0].reset_index(drop=True)\nvalid_df = df.loc[df.fold == 0].reset_index(drop=True)\n\n# Prepare dataloader\ntrainloader, validloader = get_mixup_dataloaders(train_df, valid_df)\n\n# Visualize\nimage_batches, label_batches = [], []\nfor _ in range(32//CONFIG['batch_size']):\n    image_batch, label_batch = next(iter(trainloader))    \n    image_batches.extend(image_batch)\n    label_batches.extend(label_batch)\n    \nshow_batch(image_batches, label_batches)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:37:21.938241Z","iopub.execute_input":"2021-06-30T15:37:21.938655Z","iopub.status.idle":"2021-06-30T15:37:28.519611Z","shell.execute_reply.started":"2021-06-30T15:37:21.938596Z","shell.execute_reply":"2021-06-30T15:37:28.518635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🐤 Model","metadata":{}},{"cell_type":"code","source":"def get_model():\n    base_model = tf.keras.applications.EfficientNetB0(include_top=False, weights='imagenet')\n    base_model.trainabe = True\n\n    inputs = layers.Input((CONFIG['img_height'], CONFIG['img_width'], 3))\n    x = base_model(inputs, training=True)\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.Dropout(0.5)(x)\n    \n    outputs = layers.Dense(CONFIG['num_labels'], kernel_regularizer=tf.keras.regularizers.l2(0.001))(x)\n    outputs = layers.Activation('softmax', dtype='float32', name='predictions')(outputs)\n    \n    return models.Model(inputs, outputs)\n\ntf.keras.backend.clear_session() \nmodel = get_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:37:28.521207Z","iopub.execute_input":"2021-06-30T15:37:28.521537Z","iopub.status.idle":"2021-06-30T15:37:30.602643Z","shell.execute_reply.started":"2021-06-30T15:37:28.521503Z","shell.execute_reply":"2021-06-30T15:37:30.601860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG['model_name'] = 'effnetb0_mixup'\nCONFIG['group'] = 'Effnetb0-Mixup-224'","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:37:30.605000Z","iopub.execute_input":"2021-06-30T15:37:30.605334Z","iopub.status.idle":"2021-06-30T15:37:30.611316Z","shell.execute_reply.started":"2021-06-30T15:37:30.605298Z","shell.execute_reply":"2021-06-30T15:37:30.610466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🚄 Train","metadata":{}},{"cell_type":"code","source":"# Early stopping regularization\nearlystopper = tf.keras.callbacks.EarlyStopping(\n    monitor='val_loss', patience=6, verbose=0, mode='min',\n    restore_best_weights=True\n)\n\n# Reduce learning rate when validation loss gets plateau \nreduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2,\n                              patience=3, min_lr=CONFIG['learning_rate'])","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:37:44.073886Z","iopub.execute_input":"2021-06-30T15:37:44.074212Z","iopub.status.idle":"2021-06-30T15:37:44.078777Z","shell.execute_reply.started":"2021-06-30T15:37:44.074183Z","shell.execute_reply":"2021-06-30T15:37:44.077944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# utility to run prediction on out-of-fold validation data. \ndef get_predictions(model, validloader, valid_df):\n    y_pred = []\n    for image_batch, label_batch in tqdm(validloader):\n        preds = model.predict(image_batch)\n        y_pred.extend(preds)\n        \n    valid_df['preds'] = y_pred\n    \n    return valid_df \n\n# dataframe to collect oof predictions\noof_df = pd.DataFrame()","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:37:45.626706Z","iopub.execute_input":"2021-06-30T15:37:45.627034Z","iopub.status.idle":"2021-06-30T15:37:45.633404Z","shell.execute_reply.started":"2021-06-30T15:37:45.627005Z","shell.execute_reply":"2021-06-30T15:37:45.632405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model for 5 folds.\nfor fold in range(CONFIG['num_folds']):\n    print('FOLD: ', fold)\n    # Prepare train and valid df\n    train_df = df.loc[df.fold != fold].reset_index(drop=True)\n    valid_df = df.loc[df.fold == fold].reset_index(drop=True)\n\n    # Prepare dataloaders\n    trainloader, validloader = get_mixup_dataloaders(train_df, valid_df)\n    \n    # Initialize model\n    tf.keras.backend.clear_session()\n    model = get_model()\n\n    # Compile model\n    optimizer = tf.keras.optimizers.Adam(learning_rate=CONFIG['learning_rate'])\n    model.compile(optimizer, \n                  loss='categorical_crossentropy', \n                  metrics=['acc', tf.keras.metrics.AUC(curve='ROC')])\n\n\n    # Update CONFIG dict with the name of the model.\n    print('Training configuration: ', CONFIG)\n\n    # Initialize W&B run\n    run = wandb.init(project='siim-study-level', \n                     config=CONFIG,\n                     group=CONFIG['group'], \n                     job_type='train',\n                     tags=['train', 'kfold', 'mixup', 'effnetb0'],\n                     notes='training effnetb0 with mixup')\n    \n    # Train\n    _ = model.fit(trainloader, \n                  epochs=CONFIG['epochs'],\n                  validation_data=validloader,\n                  class_weight=class_weights_dict,\n                  callbacks=[WandbCallback(),\n                             earlystopper,\n                             reduce_lr])\n    \n    # Evaluate\n    loss, acc, auc = model.evaluate(validloader)\n    wandb.log({'Val Acc': acc, 'Val AUC-ROC': auc})\n    \n    # Save model\n    model_name = CONFIG['model_name']\n    MODEL_PATH = f'models/study_level/{model_name}'\n    os.makedirs(MODEL_PATH, exist_ok=True)\n    count_models = len(os.listdir(MODEL_PATH))\n    \n    model.save(f'{MODEL_PATH}/{model_name}_{count_models}.h5')\n\n    # Get Prediction on validation set\n    _oof_df = get_predictions(model, validloader, valid_df)\n    oof_df = pd.concat([oof_df, _oof_df])\n\n    # Close W&B run\n    run.finish()\n    \n    del model, trainloader, validloader, _oof_df\n    _ = gc.collect()\n    \nos.makedirs('study_level_oof', exist_ok=True)\noof_df.to_csv('study_level_oof/oof_preds.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The W&B dashboard shown in the GIF below shows the metrics for EfficientNetB0 trained on 512x512 image size on a single V100 GPU. \n\nThe models weights can be found in [here](https://www.kaggle.com/ayuraj/siim-study-level-models).\n\nIn the GIF you can:\n\n* See the loss and accuracy metrics. Note that due to Mixup augmentation the validation loss is lower than training loss. This is the case of strong regularization. <br>\n* In the table mode of the dashboard, I sorted the VAL ROC-AUC score in descending order. Clearly there is some variance in the score between diffrent folds. \n\n![img](https://i.imgur.com/kCJNhbp.gif)","metadata":{}},{"cell_type":"markdown","source":"# 🦄 CV Score","metadata":{}},{"cell_type":"code","source":"oof_df = pd.read_csv('study_level_oof/oof_preds.csv')\noof_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:27:18.376053Z","iopub.execute_input":"2021-06-25T05:27:18.376465Z","iopub.status.idle":"2021-06-25T05:27:18.432579Z","shell.execute_reply.started":"2021-06-25T05:27:18.376432Z","shell.execute_reply":"2021-06-25T05:27:18.431271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_argmax(row):\n    return [float(val) for val in row.preds.strip('[]').split(' ') if val!='']\n\noof_df['preds'] = oof_df.apply(lambda row: get_argmax(row), axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:28:17.0589Z","iopub.execute_input":"2021-06-25T05:28:17.059375Z","iopub.status.idle":"2021-06-25T05:28:17.205374Z","shell.execute_reply.started":"2021-06-25T05:28:17.059344Z","shell.execute_reply":"2021-06-25T05:28:17.204315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric = tf.keras.metrics.CategoricalAccuracy()\nmetric.update_state(tf.one_hot(oof_df.study_level.values, depth=4), tf.cast(np.array(list(map(np.array, oof_df.preds.values))), tf.float32))\nprint(f'CV Score: {metric.result().numpy()}')","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:38:22.929792Z","iopub.execute_input":"2021-06-30T15:38:22.930117Z","iopub.status.idle":"2021-06-30T15:38:22.934740Z","shell.execute_reply.started":"2021-06-30T15:38:22.930088Z","shell.execute_reply":"2021-06-30T15:38:22.933464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric = tf.keras.metrics.AUC(curve='ROC')\nmetric.update_state(tf.one_hot(oof_df.study_level.values, depth=4), tf.cast(np.array(list(map(np.array, oof_df.preds.values))), tf.float32))\nprint(f'CV Score: {metric.result().numpy()}')","metadata":{"execution":{"iopub.status.busy":"2021-06-30T15:38:43.200798Z","iopub.execute_input":"2021-06-30T15:38:43.201110Z","iopub.status.idle":"2021-06-30T15:38:43.205290Z","shell.execute_reply.started":"2021-06-30T15:38:43.201082Z","shell.execute_reply":"2021-06-30T15:38:43.204499Z"},"trusted":true},"execution_count":null,"outputs":[]}]}