{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport cv2\n!pip install timm\nimport timm\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader, Dataset, random_split\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\n\nimport time \nfrom PIL import Image\ntrain_on_gpu = True\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\n\n!pip install torchsummary\nfrom torchsummary import summary\n\nimport copy\n\ntorch.manual_seed(0)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-10T09:41:57.387763Z","iopub.execute_input":"2023-04-10T09:41:57.388079Z","iopub.status.idle":"2023-04-10T09:42:22.049232Z","shell.execute_reply.started":"2023-04-10T09:41:57.387996Z","shell.execute_reply":"2023-04-10T09:42:22.048133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Deep Learning Classification Flow\n\n1. Dataset, DataLoader 정의\n    - Data augmentation 정의\n    - 데이터 셋의 특성 파악을 위한 visual inspection\n2. Model (Encoder) 정의\n    - 본인이 직접 디자인\n    - 유명한 모델의 구조를 이용하여 scratch부터 학습\n    - ImageNet 등의 대규모 이미지 데이터 셋으로 학습된 weight 가져와서 fine-tuning \n3. Training & validation function\n    - Optimizer, loss function, learning rate, learning rate scheduler 등등\n4. Testing & performance evaluation\n    - Generalization performance 측정, 어떤 evaluation metrics 를 사용할 것인지?\n        - AUROC, AUPRC, F1-score, sensitivity, specificity, PPV, NPV, etc.\n5. XAI\n    - Explainable AI\n    - 모델의 학습 과정이 black box라 불릴 정도로 내부는 복잡하고 알 수 없음\n    - 결과를 통해 간접적으로 모델을 해석하려는 시도\n    - 대표적으로 LIME, SHAP, Grad-CAM 등이 있음\n    - 여기에서는 Grad-CAM 실습","metadata":{}},{"cell_type":"code","source":"path2labels = \"/kaggle/input/histopathologic-cancer-detection/train_labels.csv\"\nlabels_df = pd.read_csv(path2labels)\nlabels_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T09:43:21.808932Z","iopub.execute_input":"2023-04-10T09:43:21.809349Z","iopub.status.idle":"2023-04-10T09:43:22.228211Z","shell.execute_reply.started":"2023-04-10T09:43:21.809312Z","shell.execute_reply":"2023-04-10T09:43:22.227294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_theme()\nsns.countplot(data=labels_df, x='label')\nplt.xticks(range(2), ['Normal', 'Tumor'], fontsize=10)\nplt.xlabel(\"Labels\", fontsize=12)\nplt.ylabel(\"Count\", fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T09:43:22.230103Z","iopub.execute_input":"2023-04-10T09:43:22.230488Z","iopub.status.idle":"2023-04-10T09:43:22.467585Z","shell.execute_reply.started":"2023-04-10T09:43:22.230451Z","shell.execute_reply":"2023-04-10T09:43:22.466653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(10, 10))\npath2data = \"/kaggle/input/histopathologic-cancer-detection/train\"\ntrain_imgs = os.listdir(path2data)\nfor idx, img in enumerate(np.random.choice(train_imgs, 20)):\n    ax = fig.add_subplot(4, 20//4, idx+1)\n    im = Image.open(path2data + \"/\" + img)\n    plt.imshow(im)\n    plt.xticks([])\n    plt.yticks([])\n    \n    lab = labels_df.loc[labels_df[\"id\"] == img.split('.')[0], 'label'].values[0]\n    ax.set_title(f'Label: {[\"Tumor\" if lab == 1 else \"Normal\"][0]}')","metadata":{"execution":{"iopub.status.busy":"2023-04-10T09:43:22.469491Z","iopub.execute_input":"2023-04-10T09:43:22.469865Z","iopub.status.idle":"2023-04-10T09:43:26.360995Z","shell.execute_reply.started":"2023-04-10T09:43:22.469829Z","shell.execute_reply":"2023-04-10T09:43:26.360183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preparation\n- Dataset, Dataloader 정의\n- Data augmentation 정의","metadata":{}},{"cell_type":"code","source":"class cancer_dataset(Dataset):\n    def __init__(self, data_dir, transform=None, data_type=\"train\"):\n        # path to images\n        path2data = os.path.join(data_dir, data_type)\n        \n        # list of images in directory\n        filenames = os.listdir(path2data)\n        \n        # get full path to images\n        self.full_filenames = [os.path.join(path2data, f) for f in filenames]\n        \n        # get labels\n        path2labels = os.path.join(data_dir, \"train_labels.csv\")\n        labels_df = pd.read_csv(path2labels)\n        \n        # seg dataframe index to id\n        labels_df.set_index(\"id\", inplace=True)\n        \n        # obtain labels from df\n        self.labels = [labels_df.loc[filename[:-4]].values[0] for filename in filenames]\n        \n        # For image data augmentions\n        if transform is not None:\n            self.transform = transform\n        else:\n            self.transform = None\n        \n        \n    def __len__(self):\n        # return size of dataset\n        return len(self.full_filenames)\n    \n    def __getitem__(self, idx):\n        # open image, apply transforms and return with label\n        img = Image.open(self.full_filenames[idx]) # PIL image\n        if self.transform is not None:\n            img = self.transform(img)\n            \n        return img, self.labels[idx].squeeze(-1)\n    \ndata_transformer = transforms.Compose([transforms.ToTensor(),\n                                       transforms.Resize((224, 224))])","metadata":{"execution":{"iopub.status.busy":"2023-04-10T09:43:26.362384Z","iopub.execute_input":"2023-04-10T09:43:26.364365Z","iopub.status.idle":"2023-04-10T09:43:26.376429Z","shell.execute_reply.started":"2023-04-10T09:43:26.364326Z","shell.execute_reply":"2023-04-10T09:43:26.375546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = \"/kaggle/input/histopathologic-cancer-detection\"\nimg_dataset = cancer_dataset(data_dir, transform=data_transformer, data_type=\"train\")","metadata":{"execution":{"iopub.status.busy":"2023-04-10T09:43:26.378169Z","iopub.execute_input":"2023-04-10T09:43:26.379045Z","iopub.status.idle":"2023-04-10T09:43:43.119203Z","shell.execute_reply.started":"2023-04-10T09:43:26.378993Z","shell.execute_reply":"2023-04-10T09:43:43.118233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Splitting the Dataset\n- Train - Validation - Test 스플릿\n- Train - Test 가 주어졌으므로 우리는 Train을 valid만 스플릿\n- 8:2 비율로 진행 / 일반적으로 6:2:2","metadata":{}},{"cell_type":"code","source":"len_dataset = len(img_dataset)\nlen_train = int(0.8 * len_dataset)\nlen_val = len_dataset - len_train\n\ntrain_ds, val_ds = random_split(img_dataset, [len_train, len_val])\n\nprint(f'train dataset length: {len(train_ds)}')\nprint(f'validation dataset length: {len(val_ds)}')","metadata":{"execution":{"iopub.status.busy":"2023-04-10T09:43:43.121939Z","iopub.execute_input":"2023-04-10T09:43:43.122338Z","iopub.status.idle":"2023-04-10T09:43:43.148933Z","shell.execute_reply.started":"2023-04-10T09:43:43.122298Z","shell.execute_reply":"2023-04-10T09:43:43.147797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image Transformation\nWe will define some transformers for image augmentation","metadata":{}},{"cell_type":"code","source":"# transformer for trainging dataset\ntrain_transf = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Resize((224, 224)),\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.RandomVerticalFlip(p=0.5),\n    transforms.RandomRotation(45),\n#     transforms.RandomResizedCrop(96, scale=(0.8, 1.0), ratio=(1.0, 1.0)),\n#     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n    ])\n\n# No augmentation for validation dataset\nval_transf = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Resize((224, 224)),\n#     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n    ])\n\n# Overwrite the transforms functions\ntrain_ds.transform = train_transf\nval_ds.transform = val_transf\n\n# Dataloaders\ntrain_dl = DataLoader(train_ds, batch_size=256, pin_memory=True, num_workers=2, shuffle=True)\nval_dl = DataLoader(val_ds, batch_size=256, pin_memory=True, num_workers=2, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T09:43:43.150287Z","iopub.execute_input":"2023-04-10T09:43:43.150735Z","iopub.status.idle":"2023-04-10T09:43:43.158785Z","shell.execute_reply.started":"2023-04-10T09:43:43.150697Z","shell.execute_reply":"2023-04-10T09:43:43.157600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Classifier","metadata":{}},{"cell_type":"markdown","source":"## Simple CNN model\nThis model has: \n    - Four CNN layers with poolin layer for each layer.\n    - Two fully connected layers, with a dropout layer\n    - activation function is log_softmax","metadata":{}},{"cell_type":"code","source":"timm.list_models()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T09:43:43.160432Z","iopub.execute_input":"2023-04-10T09:43:43.161073Z","iopub.status.idle":"2023-04-10T09:43:43.391216Z","shell.execute_reply.started":"2023-04-10T09:43:43.161038Z","shell.execute_reply":"2023-04-10T09:43:43.389997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = timm.create_model(\"resnet18\", pretrained=True, num_classes=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T09:43:43.393472Z","iopub.execute_input":"2023-04-10T09:43:43.394274Z","iopub.status.idle":"2023-04-10T09:43:44.072727Z","shell.execute_reply.started":"2023-04-10T09:43:43.394234Z","shell.execute_reply":"2023-04-10T09:43:44.071658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_func = nn.BCEWithLogitsLoss()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T09:43:44.075925Z","iopub.execute_input":"2023-04-10T09:43:44.078222Z","iopub.status.idle":"2023-04-10T09:43:44.083910Z","shell.execute_reply.started":"2023-04-10T09:43:44.078190Z","shell.execute_reply":"2023-04-10T09:43:44.082991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opt = optim.Adam(model.parameters(), lr=3e-4)\nlr_scheduler = ReduceLROnPlateau(opt, mode='min', factor=0.5, patience=20, verbose=0)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T09:43:44.087513Z","iopub.execute_input":"2023-04-10T09:43:44.087827Z","iopub.status.idle":"2023-04-10T09:43:44.094307Z","shell.execute_reply.started":"2023-04-10T09:43:44.087790Z","shell.execute_reply":"2023-04-10T09:43:44.093350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Model","metadata":{}},{"cell_type":"markdown","source":"## Helper functions","metadata":{}},{"cell_type":"code","source":"\n# Function to get the learning rate\ndef get_lr(opt):\n    for param_group in opt.param_groups:\n        return param_group['lr']\n\n# Function to compute the loss value per batch of data\ndef loss_batch(loss_func, output, target, opt=None):\n    \n    loss = loss_func(output, target) # get loss\n    pred = output.argmax(dim=1, keepdim=True) # Get Output Class\n    metric_b=pred.eq(target.view_as(pred)).sum().item() # get performance metric\n    \n    if opt is not None:\n        opt.zero_grad()\n        loss.backward()\n        opt.step()\n\n    return loss.item(), metric_b\n\n# Compute the loss value & performance metric for the entire dataset (epoch)\ndef loss_epoch(model, loss_func, dataset_dl, opt=None, device=\"cpu\"):\n    \n    run_loss=0.0 \n    t_metric=0.0\n    len_data=len(dataset_dl.dataset)\n\n    # internal loop over dataset\n    for xb, yb in tqdm(dataset_dl, leave=False):\n        # move batch to device\n        xb=xb.to(device)\n        yb=yb.to(device).unsqueeze(-1).float()\n        output=model(xb) # get model output\n        loss_b,metric_b=loss_batch(loss_func, output, yb, opt) # get loss per batch\n        run_loss+=loss_b        # update running loss\n\n        if metric_b is not None: # update running metric\n            t_metric+=metric_b    \n    \n    loss=run_loss/float(len_data)  # average loss value\n    metric=t_metric/float(len_data) # average metric value\n    \n    return loss, metric\n","metadata":{"execution":{"iopub.status.busy":"2023-04-10T09:43:44.097250Z","iopub.execute_input":"2023-04-10T09:43:44.097759Z","iopub.status.idle":"2023-04-10T09:43:44.109104Z","shell.execute_reply.started":"2023-04-10T09:43:44.097724Z","shell.execute_reply":"2023-04-10T09:43:44.108035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training function","metadata":{}},{"cell_type":"code","source":"from tqdm.notebook import trange, tqdm\n\ndef train_val(model, params, verbose=False):\n    \n    # Get parameters\n    epochs = params[\"epochs\"]\n    opt = params[\"optimiser\"]\n    loss_func = params[\"f_loss\"]\n    train_dl = params[\"train\"]\n    val_dl = params[\"val\"]\n    lr_scheduler = params[\"lr_change\"]\n    weight_path = params[\"weight_path\"]\n    \n    # history of loss and metric values in each epoch\n    loss_history = {\"train\": [], \"val\": []}\n    metric_history = {\"train\": [], \"val\": []}\n    \n    # a deep copy of weights for the best model\n    best_model_wts = copy.deepcopy(model.state_dict())\n    \n    best_loss = float('inf')      # init loss\n    \n    # Train loop\n    for epoch in tqdm(range(epochs), leave=False):\n        \n        # get lr\n        current_lr = get_lr(opt)\n        if(verbose):\n            print(f'Epoch {epoch +1}/{epochs}, current lr={current_lr}')\n        \n        # train model\n        model.train()\n        train_loss, train_metric = loss_epoch(model, loss_func, train_dl, opt, device=params[\"device\"])\n        \n        loss_history[\"train\"].append(train_loss)\n        metric_history[\"train\"].append(train_metric)\n        \n        # evaluate model\n        model.eval()\n        with torch.no_grad():\n            val_loss, val_metric = loss_epoch(model, loss_func, val_dl, opt=None, device=params[\"device\"])\n        \n        # store best model\n        if val_loss < best_loss:\n            best_loss = val_loss\n            best_model_wts = copy.deepcopy(model.state_dict())\n            \n            # save weights in a local file\n            torch.save(model.state_dict(), weight_path)\n            if verbose:\n                print(\"Saved best model weights\")\n            \n        loss_history[\"val\"].append(val_loss)\n        metric_history[\"val\"].append(val_metric)\n        \n        # lr schedule\n        lr_scheduler.step(val_loss)\n        if current_lr != get_lr(opt):\n            if verbose:\n                print(\"Loading best model weights\")\n            model.load_state_dict(best_model_wts)\n            \n        if verbose:\n            print(f\"train loss: {train_loss:.6f}, dev loss: {val_loss:.6f}, accuracy: {100*val_metric:.2f}\")\n            print(\"-\"*20)\n            \n    model.load_state_dict(best_model_wts)\n    \n    return model, loss_history, metric_history","metadata":{"execution":{"iopub.status.busy":"2023-04-10T09:43:45.636341Z","iopub.execute_input":"2023-04-10T09:43:45.637108Z","iopub.status.idle":"2023-04-10T09:43:45.662964Z","shell.execute_reply.started":"2023-04-10T09:43:45.637061Z","shell.execute_reply":"2023-04-10T09:43:45.661838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel.to(device)\n\nparams_train={\n    \"train\": train_dl,\"val\": val_dl,\n    \"epochs\": 1,\n    \"optimiser\": opt,\n    \"lr_change\": lr_scheduler,\n    \"f_loss\": loss_func,\n    \"device\": device,\n    \"weight_path\": \"weights.pt\",\n}\n\n# Train model\n\nmodel, loss_hist, metric_hist = train_val(model, params_train, verbose=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T09:43:47.135645Z","iopub.execute_input":"2023-04-10T09:43:47.136370Z","iopub.status.idle":"2023-04-10T09:59:30.241520Z","shell.execute_reply.started":"2023-04-10T09:43:47.136331Z","shell.execute_reply":"2023-04-10T09:59:30.240455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loss & Metrics visual\n","metadata":{}},{"cell_type":"code","source":"import seaborn as sns; sns.set(style='whitegrid')\n\nepochs=params_train[\"epochs\"]\n\nfig,ax = plt.subplots(1,2,figsize=(12,5))\n\nsns.lineplot(x=[*range(1,epochs+1)],y=loss_hist[\"train\"],ax=ax[0],label='loss_hist[\"train\"]')\nsns.lineplot(x=[*range(1,epochs+1)],y=loss_hist[\"val\"],ax=ax[0],label='loss_hist[\"val\"]')\nsns.lineplot(x=[*range(1,epochs+1)],y=metric_hist[\"train\"],ax=ax[1],label='metric_hist[\"train\"]')\nsns.lineplot(x=[*range(1,epochs+1)],y=metric_hist[\"val\"],ax=ax[1],label='metric_hist[\"val\"]')\nplt.title('Convergence History')","metadata":{"execution":{"iopub.status.busy":"2023-04-10T10:37:57.038427Z","iopub.execute_input":"2023-04-10T10:37:57.038829Z","iopub.status.idle":"2023-04-10T10:37:57.559006Z","shell.execute_reply.started":"2023-04-10T10:37:57.038792Z","shell.execute_reply":"2023-04-10T10:37:57.558071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference\nWe will use our trained model to classify the test dataset","metadata":{}},{"cell_type":"code","source":"class cancerdata_test(Dataset):\n    \n    def __init__(self, data_dir, transform,data_type=\"train\"):\n        \n        path2data = os.path.join(data_dir,data_type)\n        filenames = os.listdir(path2data)\n        self.full_filenames = [os.path.join(path2data, f) for f in filenames]\n        \n        # labels are in a csv file named train_labels.csv\n        csv_filename=\"sample_submission.csv\"\n        path2csvLabels=os.path.join(data_dir,csv_filename)\n        labels_df=pd.read_csv(path2csvLabels)\n        \n        # set data frame index to id\n        labels_df.set_index(\"id\", inplace=True)\n        \n        # obtain labels from data frame\n        self.labels = [labels_df.loc[filename[:-4]].values[0] for filename in filenames]\n        self.transform = transform       \n        \n    def __len__(self):\n        # return size of dataset\n        return len(self.full_filenames)\n    \n    def __getitem__(self, idx):\n        # open image, apply transforms and return with label\n        image = Image.open(self.full_filenames[idx]) # PIL image\n        image = self.transform(image)\n        return image, self.labels[idx]","metadata":{"execution":{"iopub.status.busy":"2023-04-10T10:38:05.889312Z","iopub.execute_input":"2023-04-10T10:38:05.889681Z","iopub.status.idle":"2023-04-10T10:38:05.899253Z","shell.execute_reply.started":"2023-04-10T10:38:05.889648Z","shell.execute_reply":"2023-04-10T10:38:05.898092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load any model weights for the model\nmodel.load_state_dict(torch.load('weights.pt'))","metadata":{"execution":{"iopub.status.busy":"2023-04-10T10:38:06.649943Z","iopub.execute_input":"2023-04-10T10:38:06.650520Z","iopub.status.idle":"2023-04-10T10:38:06.720845Z","shell.execute_reply.started":"2023-04-10T10:38:06.650471Z","shell.execute_reply":"2023-04-10T10:38:06.719913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path2sub = \"/kaggle/input/histopathologic-cancer-detection/sample_submission.csv\"\nlabels_df = pd.read_csv(path2sub)\ndata_dir = '/kaggle/input/histopathologic-cancer-detection/'\n\ndata_transformer = transforms.Compose([transforms.ToTensor(),\n                                       transforms.Resize((224, 224))])\n\nimg_dataset_test = cancerdata_test(data_dir,data_transformer,data_type=\"test\")\nprint(len(img_dataset_test), 'samples found')","metadata":{"execution":{"iopub.status.busy":"2023-04-10T10:38:17.792337Z","iopub.execute_input":"2023-04-10T10:38:17.792743Z","iopub.status.idle":"2023-04-10T10:38:22.287185Z","shell.execute_reply.started":"2023-04-10T10:38:17.792706Z","shell.execute_reply":"2023-04-10T10:38:22.286081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference(model, dataset, device, num_classes=1):\n    \n    len_data=len(dataset)\n    y_out=torch.zeros(len_data,num_classes) # initialize output tensor on CPU\n    y_gt=np.zeros((len_data),dtype=\"uint8\") # initialize ground truth on CPU\n    model=model.to(device) # move model to device\n    model.eval()\n    \n    with torch.no_grad():\n        for i in tqdm(range(len_data)):\n            x,y=dataset[i]\n            y_gt[i]=y\n            y_out[i]=model(x.unsqueeze(0).to(device))\n\n    return y_out.numpy(),y_gt           \n\ny_test_out,_ = inference(model,img_dataset_test, device)  \ny_test_pred=np.argmax(y_test_out,axis=1)\n\ntest_ids = [name.split('/')[-1].split('.')[0] for name in img_dataset_test.full_filenames]\ntest_preds = pd.DataFrame({\"img\": test_ids, \"preds\": y_test_pred})\nsubmission = pd.merge(labels_df, test_preds, left_on='id', right_on='img')\nsubmission = submission[['id', 'preds']]\nsubmission.columns = ['id', 'label']\nsubmission.head()\n","metadata":{"execution":{"iopub.status.busy":"2023-04-10T10:38:22.372554Z","iopub.execute_input":"2023-04-10T10:38:22.372929Z","iopub.status.idle":"2023-04-10T10:49:11.750840Z","shell.execute_reply.started":"2023-04-10T10:38:22.372897Z","shell.execute_reply":"2023-04-10T10:49:11.749871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T10:59:38.135048Z","iopub.execute_input":"2023-04-10T10:59:38.136104Z","iopub.status.idle":"2023-04-10T10:59:38.216351Z","shell.execute_reply.started":"2023-04-10T10:59:38.136044Z","shell.execute_reply":"2023-04-10T10:59:38.215444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}