{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-28T14:34:36.041165Z","iopub.execute_input":"2022-10-28T14:34:36.041809Z","iopub.status.idle":"2022-10-28T14:34:53.486606Z","shell.execute_reply.started":"2022-10-28T14:34:36.041700Z","shell.execute_reply":"2022-10-28T14:34:53.485250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Setting","metadata":{}},{"cell_type":"code","source":"EPOCHS = 10\nIMG_SIZE = 128\nMODEL_NAME = \"tf_efficientnetv2_m_in21ft1k\"\nDEBUG = False\nBATCH_SIZE = 16","metadata":{"execution":{"iopub.status.busy":"2022-10-28T14:34:53.491850Z","iopub.execute_input":"2022-10-28T14:34:53.494360Z","iopub.status.idle":"2022-10-28T14:34:53.502193Z","shell.execute_reply.started":"2022-10-28T14:34:53.494317Z","shell.execute_reply":"2022-10-28T14:34:53.501255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport numpy as np\nimport cv2\nfrom sklearn.metrics import recall_score, precision_score, f1_score, roc_auc_score\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport pandas as pd\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport timm\n\nimport matplotlib.pyplot as plt\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-10-28T14:46:30.490234Z","iopub.execute_input":"2022-10-28T14:46:30.490828Z","iopub.status.idle":"2022-10-28T14:46:30.512675Z","shell.execute_reply.started":"2022-10-28T14:46:30.490777Z","shell.execute_reply":"2022-10-28T14:46:30.511481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed: int):\n    import random, os\n    import numpy as np\n    import torch\n    \n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \nseed_everything(42)","metadata":{"execution":{"iopub.status.busy":"2022-10-28T14:34:58.574401Z","iopub.execute_input":"2022-10-28T14:34:58.577957Z","iopub.status.idle":"2022-10-28T14:34:58.590018Z","shell.execute_reply.started":"2022-10-28T14:34:58.577893Z","shell.execute_reply":"2022-10-28T14:34:58.588965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare Dataset","metadata":{}},{"cell_type":"code","source":"training_sd_filepaths = [f\"../input/stable-diffusion-dataset/stable-diffusion-dataset/landmark/{i:05}.png\" for i in range(5000)]\nvalidation_sd_filepaths = [f\"../input/stable-diffusion-dataset/stable-diffusion-dataset/landmark/{i:05}.png\" for i in range(5000, 10000)]\ntest_sd_filepaths = [f\"../input/stable-diffusion-dataset/stable-diffusion-dataset/landmark/{i:05}.png\" for i in range(10000, 15000)]\nunknown_sd_filepaths = [f\"../input/stable-diffusion-dataset/stable-diffusion-dataset/car/{i:05}.png\" for i in range(10000)]\n\nif DEBUG:\n    training_sd_filepaths = training_sd_filepaths[:100]\n    validation_sd_filepaths = validation_sd_filepaths[:100]\n    test_sd_filepaths = test_sd_filepaths[:100]\n    unknown_sd_filepaths = unknown_sd_filepaths[:100]","metadata":{"execution":{"iopub.status.busy":"2022-10-28T14:34:58.594864Z","iopub.execute_input":"2022-10-28T14:34:58.597211Z","iopub.status.idle":"2022-10-28T14:34:58.622069Z","shell.execute_reply.started":"2022-10-28T14:34:58.597174Z","shell.execute_reply":"2022-10-28T14:34:58.621223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_get_short_size(path):\n    img = cv2.imread(path)\n    return img.shape[0] if img.shape[0] < img.shape[1] else img.shape[1]","metadata":{"execution":{"iopub.status.busy":"2022-10-28T14:34:58.625937Z","iopub.execute_input":"2022-10-28T14:34:58.628008Z","iopub.status.idle":"2022-10-28T14:34:58.634502Z","shell.execute_reply.started":"2022-10-28T14:34:58.627973Z","shell.execute_reply":"2022-10-28T14:34:58.633486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/landmark-recognition-2020/train.csv\")\ntrain_landmark_df = train_df[train_df[\"landmark_id\"].isin([i for i in range(2000)])]\ntraining_landmark_filepaths = [f\"../input/landmark-recognition-2020/train/{image_id[0]}/{image_id[1]}/{image_id[2]}/{image_id}.jpg\" for image_id in  train_landmark_df[\"id\"]]\ntrain_landmark_df[\"short_size\"] = [process_get_short_size(filepath) for filepath in training_landmark_filepaths]\n\ntrain_landmark_df = train_landmark_df[train_landmark_df[\"short_size\"] > IMG_SIZE]\n\nif DEBUG:\n    train_landmark_df = train_landmark_df.head(100)\nelse:\n    train_landmark_df = train_landmark_df.head(5000)\ntraining_landmark_filepaths = [f\"../input/landmark-recognition-2020/train/{image_id[0]}/{image_id[1]}/{image_id[2]}/{image_id}.jpg\" for image_id in  train_landmark_df[\"id\"]]\n\nvalid_landmark_df = train_df[train_df[\"landmark_id\"].isin([i for i in range(2000, 4000)])]\nvalidation_landmark_filepaths = [f\"../input/landmark-recognition-2020/train/{image_id[0]}/{image_id[1]}/{image_id[2]}/{image_id}.jpg\" for image_id in  valid_landmark_df[\"id\"]]\nvalid_landmark_df[\"short_size\"] = [process_get_short_size(filepath) for filepath in validation_landmark_filepaths]\n\nvalid_landmark_df = valid_landmark_df[valid_landmark_df[\"short_size\"] > IMG_SIZE]\n\nif DEBUG:\n    valid_landmark_df = valid_landmark_df.head(100)\nelse:\n    valid_landmark_df = valid_landmark_df.head(5000)\nvalidation_landmark_filepaths = [f\"../input/landmark-recognition-2020/train/{image_id[0]}/{image_id[1]}/{image_id[2]}/{image_id}.jpg\" for image_id in  valid_landmark_df[\"id\"]]\n\ntest_landmark_df = train_df[train_df[\"landmark_id\"].isin([i for i in range(4000, 6000)])]\ntest_landmark_filepaths = [f\"../input/landmark-recognition-2020/train/{image_id[0]}/{image_id[1]}/{image_id[2]}/{image_id}.jpg\" for image_id in  test_landmark_df[\"id\"]]\ntest_landmark_df[\"short_size\"] = [process_get_short_size(filepath) for filepath in test_landmark_filepaths]\n\ntest_landmark_df = test_landmark_df[test_landmark_df[\"short_size\"] > IMG_SIZE]\n\nif DEBUG:\n    test_landmark_df = test_landmark_df.head(100)\nelse:\n    test_landmark_df = test_landmark_df.head(5000)\ntest_landmark_filepaths = [f\"../input/landmark-recognition-2020/train/{image_id[0]}/{image_id[1]}/{image_id[2]}/{image_id}.jpg\" for image_id in  test_landmark_df[\"id\"]]","metadata":{"execution":{"iopub.status.busy":"2022-10-28T14:34:58.639255Z","iopub.execute_input":"2022-10-28T14:34:58.641807Z","iopub.status.idle":"2022-10-28T14:46:29.206859Z","shell.execute_reply.started":"2022-10-28T14:34:58.641771Z","shell.execute_reply":"2022-10-28T14:46:29.205810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEBUG:\n    unknown_car_filepaths = glob.glob(\"../input/myautoge-cars-dataset/**/*.jpg\", recursive=True)[:100]\nelse:\n    unknown_car_filepaths = glob.glob(\"../input/myautoge-cars-dataset/**/*.jpg\", recursive=True)[:10000]","metadata":{"execution":{"iopub.status.busy":"2022-10-28T14:51:53.254956Z","iopub.execute_input":"2022-10-28T14:51:53.255653Z","iopub.status.idle":"2022-10-28T15:02:53.863571Z","shell.execute_reply.started":"2022-10-28T14:51:53.255614Z","shell.execute_reply":"2022-10-28T15:02:53.861956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tile_images(img_paths):\n    tile = np.zeros((1536, 1536, 3), dtype=np.uint8)\n    for h in range(3):\n        for w in range(3):\n            tile[h * 512: (h + 1) * 512, w * 512: (w + 1) * 512, :] = cv2.resize(cv2.imread(img_paths[h * 3 + w])[:, :, ::-1], (512, 512))\n    return tile","metadata":{"execution":{"iopub.status.busy":"2022-10-28T14:47:54.391726Z","iopub.execute_input":"2022-10-28T14:47:54.392111Z","iopub.status.idle":"2022-10-28T14:47:54.398644Z","shell.execute_reply.started":"2022-10-28T14:47:54.392081Z","shell.execute_reply":"2022-10-28T14:47:54.397480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\ntile = tile_images(random.choices(training_landmark_filepaths, k=9))\nplt.figure(figsize=(8, 8))\nplt.imshow(tile)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\ntile = tile_images(random.choices(training_sd_filepaths, k=9))\nplt.figure(figsize=(8, 8))\nplt.imshow(tile)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\ntile = tile_images(random.choices(unknown_car_filepaths, k=9))\nplt.figure(figsize=(8, 8))\nplt.imshow(tile)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\ntile = tile_images(random.choices(unknown_sd_filepaths, k=9))\nplt.figure(figsize=(8, 8))\nplt.imshow(tile)","metadata":{"execution":{"iopub.status.busy":"2022-10-28T14:50:44.560850Z","iopub.execute_input":"2022-10-28T14:50:44.561758Z","iopub.status.idle":"2022-10-28T14:50:45.472419Z","shell.execute_reply.started":"2022-10-28T14:50:44.561712Z","shell.execute_reply":"2022-10-28T14:50:45.470571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-10-28T15:02:53.865755Z","iopub.execute_input":"2022-10-28T15:02:53.866141Z","iopub.status.idle":"2022-10-28T15:02:54.812724Z","shell.execute_reply.started":"2022-10-28T15:02:53.866105Z","shell.execute_reply":"2022-10-28T15:02:54.811899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\ntile = tile_images(random.choices(unknown_sd_filepaths, k=9))\nplt.figure(figsize=(8, 8))\nplt.imshow(tile)","metadata":{"execution":{"iopub.status.busy":"2022-10-28T14:51:46.931277Z","iopub.execute_input":"2022-10-28T14:51:46.931632Z","iopub.status.idle":"2022-10-28T14:51:47.810484Z","shell.execute_reply.started":"2022-10-28T14:51:46.931602Z","shell.execute_reply":"2022-10-28T14:51:47.808550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-10-28T14:51:09.071395Z","iopub.execute_input":"2022-10-28T14:51:09.072346Z","iopub.status.idle":"2022-10-28T14:51:44.228610Z","shell.execute_reply.started":"2022-10-28T14:51:09.072298Z","shell.execute_reply":"2022-10-28T14:51:44.226922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_full_filepaths = training_sd_filepaths + training_landmark_filepaths\ntraining_labels = [1 for _ in range(len(training_sd_filepaths))] + [0 for _ in range(len(training_landmark_filepaths))] \n\nvalidation_full_filepaths = validation_sd_filepaths + validation_landmark_filepaths\nvalidation_labels = [1 for _ in range(len(validation_sd_filepaths))] + [0 for _ in range(len(validation_landmark_filepaths))] \n\ntest_full_filepaths = test_sd_filepaths + test_landmark_filepaths\ntest_labels = [1 for _ in range(len(test_sd_filepaths))] + [0 for _ in range(len(test_landmark_filepaths))] \n\nunknown_test_full_filepaths = unknown_sd_filepaths + unknown_car_filepaths\nunknown_test_labels = [1 for _ in range(len(unknown_sd_filepaths))] + [0 for _ in range(len(unknown_car_filepaths))] ","metadata":{"execution":{"iopub.status.busy":"2022-10-14T01:52:42.891901Z","iopub.execute_input":"2022-10-14T01:52:42.892492Z","iopub.status.idle":"2022-10-14T01:52:42.905006Z","shell.execute_reply.started":"2022-10-14T01:52:42.892454Z","shell.execute_reply":"2022-10-14T01:52:42.904179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_transform():\n    return A.Compose([\n        A.RandomCrop(IMG_SIZE, IMG_SIZE),\n#         A.Resize(IMG_SIZE, IMG_SIZE),\n        A.Normalize(),\n        ToTensorV2()\n    ])\n    \n\ndef get_test_transform():\n    return A.Compose([\n        A.CenterCrop(IMG_SIZE, IMG_SIZE),\n#         A.Resize(IMG_SIZE, IMG_SIZE),\n        A.Normalize(),\n        ToTensorV2()\n    ])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T01:52:42.910510Z","iopub.execute_input":"2022-10-14T01:52:42.910900Z","iopub.status.idle":"2022-10-14T01:52:42.917996Z","shell.execute_reply.started":"2022-10-14T01:52:42.910836Z","shell.execute_reply":"2022-10-14T01:52:42.917085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImageDataset(Dataset):\n    def __init__(self, x, y=None, transform=None) -> None:\n        super().__init__()\n        self.transform = transform\n\n        self.x = x\n        self.y = y\n    \n    def __getitem__(self, index):\n        img = cv2.imread(self.x[index])\n        if self.y is not None:\n            label = self.y[index]\n\n        if img is None:\n            print(self.x[index])\n        img = img[:, :, ::-1]\n\n        transformed_img = self.transform(image=img)[\"image\"]\n        if self.y is None:\n            return transformed_img\n        else:\n            return transformed_img, np.array([label], dtype=np.float32)\n        \n    def __len__(self):\n        return len(self.x)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T01:52:42.919352Z","iopub.execute_input":"2022-10-14T01:52:42.919690Z","iopub.status.idle":"2022-10-14T01:52:42.929608Z","shell.execute_reply.started":"2022-10-14T01:52:42.919653Z","shell.execute_reply":"2022-10-14T01:52:42.928572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_dataloader = DataLoader(ImageDataset(training_full_filepaths, training_labels, get_train_transform()), shuffle=True, batch_size=BATCH_SIZE)\nvalidation_dataloader = DataLoader(ImageDataset(validation_full_filepaths, validation_labels, get_test_transform()), shuffle=False, batch_size=BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T01:52:42.930992Z","iopub.execute_input":"2022-10-14T01:52:42.931335Z","iopub.status.idle":"2022-10-14T01:52:42.947753Z","shell.execute_reply.started":"2022-10-14T01:52:42.931300Z","shell.execute_reply":"2022-10-14T01:52:42.946833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"model = timm.create_model(MODEL_NAME, pretrained=True, num_classes=1)\nmodel.cuda()\noptimizer = optim.Adam(model.parameters(), lr=1e-4)\nscheduler = optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=EPOCHS, eta_min=1e-7)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T01:52:42.949177Z","iopub.execute_input":"2022-10-14T01:52:42.949543Z","iopub.status.idle":"2022-10-14T01:52:52.350389Z","shell.execute_reply.started":"2022-10-14T01:52:42.949505Z","shell.execute_reply":"2022-10-14T01:52:52.349376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.BCEWithLogitsLoss()\nfor epoch in range(EPOCHS):\n    # Training Step\n    model.train()\n    for x, y in training_dataloader:\n        x, y = x.cuda(), y.cuda()\n        out = model(x)\n        loss = criterion(out, y)\n        \n        loss.backward()\n        optimizer.step()\n        optimizer.zero_grad()\n        \n    scheduler.step()\n    \n    # Valid Step\n    total_loss = 0.0\n    n_images = 0\n    model.eval()\n    with torch.no_grad():\n        for x, y in validation_dataloader:\n            x, y = x.cuda(), y.cuda()\n            out = model(x)\n            loss = criterion(out, y)\n            total_loss += loss * x.size(0)\n            n_images += x.size(0)\n    mean_loss = total_loss / n_images\n    print (f\"Epoch {epoch} Loss: {mean_loss}\")","metadata":{"execution":{"iopub.status.busy":"2022-10-14T01:52:52.351690Z","iopub.execute_input":"2022-10-14T01:52:52.353735Z","iopub.status.idle":"2022-10-14T04:10:32.701392Z","shell.execute_reply.started":"2022-10-14T01:52:52.353696Z","shell.execute_reply":"2022-10-14T04:10:32.698509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Standard Test Set","metadata":{}},{"cell_type":"code","source":"def inference(model ,test_dataloader):\n    out_list = []\n    model.eval()\n    with torch.no_grad():\n        for x, _ in test_dataloader:\n            x = x.cuda()\n            out = model(x)\n            out = F.sigmoid(out).detach().cpu().numpy()\n            out_list.append(out)\n    return np.vstack(out_list)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T04:10:32.705530Z","iopub.execute_input":"2022-10-14T04:10:32.707245Z","iopub.status.idle":"2022-10-14T04:10:32.720390Z","shell.execute_reply.started":"2022-10-14T04:10:32.707194Z","shell.execute_reply":"2022-10-14T04:10:32.718979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataloader = DataLoader(ImageDataset(test_full_filepaths, test_labels, get_test_transform()), shuffle=False, batch_size=BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T04:10:32.721990Z","iopub.execute_input":"2022-10-14T04:10:32.722339Z","iopub.status.idle":"2022-10-14T04:10:32.753030Z","shell.execute_reply.started":"2022-10-14T04:10:32.722304Z","shell.execute_reply":"2022-10-14T04:10:32.751768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outputs = inference(model, test_dataloader).ravel()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T04:10:32.754889Z","iopub.execute_input":"2022-10-14T04:10:32.755465Z","iopub.status.idle":"2022-10-14T04:13:17.184741Z","shell.execute_reply.started":"2022-10-14T04:10:32.755400Z","shell.execute_reply":"2022-10-14T04:13:17.181183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"auc_value = roc_auc_score(test_labels, outputs)\nf1_value = f1_score(test_labels, outputs > 0.5)\nrecall_value = recall_score(test_labels, outputs > 0.5)\nprecision_value = precision_score(test_labels, outputs > 0.5)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T04:13:17.185600Z","iopub.status.idle":"2022-10-14T04:13:17.185981Z","shell.execute_reply.started":"2022-10-14T04:13:17.185787Z","shell.execute_reply":"2022-10-14T04:13:17.185804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f\"Standard Test SetClassification Score AUC: {auc_value} F1: {f1_value} Recall: {recall_value} Precision: {precision_value}\"","metadata":{"execution":{"iopub.status.busy":"2022-10-14T04:13:17.188772Z","iopub.status.idle":"2022-10-14T04:13:17.189621Z","shell.execute_reply.started":"2022-10-14T04:13:17.189350Z","shell.execute_reply":"2022-10-14T04:13:17.189376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Unknown Test Set","metadata":{}},{"cell_type":"code","source":"test_dataloader = DataLoader(ImageDataset(unknown_test_full_filepaths, unknown_test_labels, get_test_transform()), shuffle=False, batch_size=BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T04:13:17.191376Z","iopub.status.idle":"2022-10-14T04:13:17.192171Z","shell.execute_reply.started":"2022-10-14T04:13:17.191919Z","shell.execute_reply":"2022-10-14T04:13:17.191945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outputs = inference(model, test_dataloader).ravel()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T04:13:17.193647Z","iopub.status.idle":"2022-10-14T04:13:17.194470Z","shell.execute_reply.started":"2022-10-14T04:13:17.194201Z","shell.execute_reply":"2022-10-14T04:13:17.194228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"auc_value = roc_auc_score(unknown_test_labels, outputs)\nf1_value = f1_score(unknown_test_labels, outputs > 0.5)\nrecall_value = recall_score(unknown_test_labels, outputs > 0.5)\nprecision_value = precision_score(unknown_test_labels, outputs > 0.5)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T04:13:17.195896Z","iopub.status.idle":"2022-10-14T04:13:17.196761Z","shell.execute_reply.started":"2022-10-14T04:13:17.196512Z","shell.execute_reply":"2022-10-14T04:13:17.196538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f\"Unknown Test Set Classification Score AUC: {auc_value} F1: {f1_value} Recall: {recall_value} Precision: {precision_value}\"","metadata":{"execution":{"iopub.status.busy":"2022-10-14T04:13:17.198666Z","iopub.status.idle":"2022-10-14T04:13:17.199712Z","shell.execute_reply.started":"2022-10-14T04:13:17.199441Z","shell.execute_reply":"2022-10-14T04:13:17.199467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}