{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7418759,"sourceType":"datasetVersion","datasetId":4315969},{"sourceId":7424438,"sourceType":"datasetVersion","datasetId":4319837},{"sourceId":7427536,"sourceType":"datasetVersion","datasetId":4322021},{"sourceId":7431944,"sourceType":"datasetVersion","datasetId":4324907},{"sourceId":7440582,"sourceType":"datasetVersion","datasetId":4330648},{"sourceId":7454625,"sourceType":"datasetVersion","datasetId":4339114},{"sourceId":7461677,"sourceType":"datasetVersion","datasetId":4342849},{"sourceId":7581078,"sourceType":"datasetVersion","datasetId":4413083},{"sourceId":7581090,"sourceType":"datasetVersion","datasetId":4413090}],"dockerImageVersionId":30636,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### **ℹ️Update Info(2024/01/23)**\n\n* **forked original great work kernels**\n    * [Inference] https://www.kaggle.com/code/ttahara/hms-hbac-resnet34d-baseline-inference\n    * [Training] https://www.kaggle.com/code/ttahara/hms-hbac-resnet34d-baseline-training/\n\n\n* My Train Info**\n    * tf_efficientnetv2_s\n    * Split 10Fold(SGKF)\n    * resize x512\n    * CV:0.6753573036121466\n    \n* My Train Info(2024/02/01)\n    * tf_efficientnetv2_s\n    * tf_efficientnetv2_m\n    * tf_efficientnetv2_l\n    \n* My Train Info(2024/02/06)[Improve LB.42(Notebook Version8)]\n    * Resize x600 x800 Model\n    ","metadata":{}},{"cell_type":"markdown","source":"## Import","metadata":{}},{"cell_type":"code","source":"import sys\nimport os\nimport gc\nimport copy\nimport yaml\nimport random\nimport shutil\nfrom time import time\nimport typing as tp\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import StratifiedGroupKFold\n\nimport torch\nfrom torch import nn\nfrom torch import optim\nfrom torch.optim import lr_scheduler\nfrom torch.cuda import amp\n\nimport timm\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-07T14:45:22.102308Z","iopub.execute_input":"2024-02-07T14:45:22.102589Z","iopub.status.idle":"2024-02-07T14:45:29.547401Z","shell.execute_reply.started":"2024-02-07T14:45:22.102563Z","shell.execute_reply":"2024-02-07T14:45:29.546296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"","metadata":{"execution":{"iopub.status.busy":"2024-02-07T14:45:29.549044Z","iopub.execute_input":"2024-02-07T14:45:29.549507Z","iopub.status.idle":"2024-02-07T14:45:29.553982Z","shell.execute_reply.started":"2024-02-07T14:45:29.549480Z","shell.execute_reply":"2024-02-07T14:45:29.552918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT = ROOT / \"input\"\nOUTPUT = ROOT / \"output\"\nSRC = ROOT / \"src\"\n\nDATA = INPUT / \"hms-harmful-brain-activity-classification\"\nTRAIN_SPEC = DATA / \"train_spectrograms\"\nTEST_SPEC = DATA / \"test_spectrograms\"\nTRAINED_MODEL = INPUT / \"hms-train-a-20111-20240118113811\"\nTRAINED_MODEL_2 = INPUT / \"hms-train-a-2011-20240117074649\"\nTRAINED_MODEL_3 = INPUT / \"hms-train-a-2031-20240118024348\"\nTRAINED_MODEL_4 = INPUT / \"hms-train-a-311-20240119031117\"\nTRAINED_MODEL_5 = INPUT / \"hms-train-a-416-20240120103942\"\nTRAINED_MODEL_6 = INPUT / \"hms-train-a-520-20240122104431\"\nTRAINED_MODEL_7 = INPUT / \"hms-train-a-5221-20240123070754\" # [SGKF] 0.6593463688787448\nTRAINED_MODEL_8 = INPUT / \"hms-train-a-5201-20240207125207\"# [SKF][v2s] 0.6963048410932463\nTRAINED_MODEL_9 = INPUT / \"hms-train-a-5202-20240207125310\"# [SKF][v2m] 0.6786888934385632\n\nTMP = ROOT / \"tmp\"\nTRAIN_SPEC_SPLIT = TMP / \"train_spectrograms_split\"\nTEST_SPEC_SPLIT = TMP / \"test_spectrograms_split\"\nTMP.mkdir(exist_ok=True)\nTRAIN_SPEC_SPLIT.mkdir(exist_ok=True)\nTEST_SPEC_SPLIT.mkdir(exist_ok=True)\n\n\nRANDAM_SEED = 1086\nCLASSES = [\"seizure_vote\", \"lpd_vote\", \"gpd_vote\", \"lrda_vote\", \"grda_vote\", \"other_vote\"]\nN_CLASSES = len(CLASSES)\nFOLDS = [0, 2, 3, 5, 6, 8, 9]\nN_FOLDS = len(FOLDS)","metadata":{"execution":{"iopub.status.busy":"2024-02-07T14:45:29.555157Z","iopub.execute_input":"2024-02-07T14:45:29.555499Z","iopub.status.idle":"2024-02-07T14:45:29.566573Z","shell.execute_reply.started":"2024-02-07T14:45:29.555468Z","shell.execute_reply":"2024-02-07T14:45:29.565869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read Data, Convert Spectrograms to Numpy file","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(DATA / \"test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-02-07T14:45:29.568592Z","iopub.execute_input":"2024-02-07T14:45:29.568848Z","iopub.status.idle":"2024-02-07T14:45:29.587649Z","shell.execute_reply.started":"2024-02-07T14:45:29.568826Z","shell.execute_reply":"2024-02-07T14:45:29.586967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-07T14:45:29.588497Z","iopub.execute_input":"2024-02-07T14:45:29.588720Z","iopub.status.idle":"2024-02-07T14:45:29.603283Z","shell.execute_reply.started":"2024-02-07T14:45:29.588700Z","shell.execute_reply":"2024-02-07T14:45:29.602303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### convert sepectogram files to numpy files","metadata":{}},{"cell_type":"code","source":"for spec_id in test[\"spectrogram_id\"]:\n    spec = pd.read_parquet(TEST_SPEC / f\"{spec_id}.parquet\")\n    \n    spec_arr = spec.fillna(0).values[:, 1:].T.astype(\"float32\")  # (Hz, Time) = (400, 300)\n    \n    np.save(TEST_SPEC_SPLIT / f\"{spec_id}.npy\", spec_arr)","metadata":{"execution":{"iopub.status.busy":"2024-02-07T14:45:29.604335Z","iopub.execute_input":"2024-02-07T14:45:29.604849Z","iopub.status.idle":"2024-02-07T14:45:29.819860Z","shell.execute_reply.started":"2024-02-07T14:45:29.604825Z","shell.execute_reply":"2024-02-07T14:45:29.819075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Difinition, Model, Dataset","metadata":{}},{"cell_type":"markdown","source":"### model","metadata":{}},{"cell_type":"code","source":"class HMSHBACSpecModel(nn.Module):\n\n    def __init__(\n            self,\n            model_name: str,\n            pretrained: bool,\n            in_channels: int,\n            num_classes: int,\n        ):\n        super().__init__()\n        self.model = timm.create_model(\n            model_name=model_name, pretrained=pretrained,\n            num_classes=num_classes, in_chans=in_channels)\n\n    def forward(self, x):\n        h = self.model(x)      \n\n        return h","metadata":{"execution":{"iopub.status.busy":"2024-02-07T14:45:29.820947Z","iopub.execute_input":"2024-02-07T14:45:29.821217Z","iopub.status.idle":"2024-02-07T14:45:29.826764Z","shell.execute_reply.started":"2024-02-07T14:45:29.821195Z","shell.execute_reply":"2024-02-07T14:45:29.826021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## dataset","metadata":{}},{"cell_type":"code","source":"FilePath = tp.Union[str, Path]\nLabel = tp.Union[int, float, np.ndarray]\n\nclass HMSHBACSpecDataset(torch.utils.data.Dataset):\n\n    def __init__(\n        self,\n        image_paths: tp.Sequence[FilePath],\n        labels: tp.Sequence[Label],\n        transform: A.Compose,\n    ):\n        self.image_paths = image_paths\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_paths)\n\n    def __getitem__(self, index: int):\n        img_path = self.image_paths[index]\n        label = self.labels[index]\n\n        img = np.load(img_path)  # shape: (Hz, Time) = (400, 300)\n        \n        # log transform\n        img = np.clip(img,np.exp(-4), np.exp(8))\n        img = np.log(img)\n        \n        # normalize per image\n        eps = 1e-6\n        img_mean = img.mean(axis=(0, 1))\n        img = img - img_mean\n        img_std = img.std(axis=(0, 1))\n        img = img / (img_std + eps)\n\n        img = img[..., None] # shape: (Hz, Time) -> (Hz, Time, Channel)\n        img = self._apply_transform(img)\n\n        return {\"data\": img, \"target\": label}\n\n    def _apply_transform(self, img: np.ndarray):\n        \"\"\"apply transform to image and mask\"\"\"\n        transformed = self.transform(image=img)\n        img = transformed[\"image\"]\n        return img","metadata":{"execution":{"iopub.status.busy":"2024-02-07T14:45:29.827921Z","iopub.execute_input":"2024-02-07T14:45:29.828216Z","iopub.status.idle":"2024-02-07T14:45:29.838288Z","shell.execute_reply.started":"2024-02-07T14:45:29.828193Z","shell.execute_reply":"2024-02-07T14:45:29.837533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference Test Data","metadata":{}},{"cell_type":"code","source":"class CFG:\n    model_name = \"tf_efficientnetv2_s.in21k_ft_in1k\"\n    model_name2 = \"tf_efficientnetv2_l.in21k_ft_in1k\"\n    model_name3 = \"tf_efficientnetv2_m.in21k_ft_in1k\"\n    img_size_h = 512\n    img_size_w = 512\n    max_epoch = 9\n    batch_size = 32\n    lr = 1.0e-03\n    weight_decay = 1.0e-02\n    es_patience =  5\n    seed = 1086\n    deterministic = True\n    enable_amp = True\n    device = \"cuda\"\nclass CFG2:\n    model_name = \"tf_efficientnetv2_s.in21k_ft_in1k\"\n    model_name2 = \"tf_efficientnetv2_l.in21k_ft_in1k\"\n    model_name3 = \"tf_efficientnetv2_m.in21k_ft_in1k\"\n    img_size_h = 1024\n    img_size_w = 1024\n    max_epoch = 9\n    batch_size = 32\n    lr = 1.0e-03\n    weight_decay = 1.0e-02\n    es_patience =  5\n    seed = 1086\n    deterministic = True\n    enable_amp = True\n    device = \"cuda\"\nclass CFG3:\n    model_name = \"tf_efficientnetv2_s.in21k_ft_in1k\"\n    model_name2 = \"tf_efficientnetv2_l.in21k_ft_in1k\"\n    model_name3 = \"tf_efficientnetv2_m.in21k_ft_in1k\"\n    img_size_h = 600\n    img_size_w = 800\n    max_epoch = 9\n    batch_size = 32\n    lr = 1.0e-03\n    weight_decay = 1.0e-02\n    es_patience =  5\n    seed = 1086\n    deterministic = True\n    enable_amp = True\n    device = \"cuda\"","metadata":{"execution":{"iopub.status.busy":"2024-02-07T14:45:29.839513Z","iopub.execute_input":"2024-02-07T14:45:29.840092Z","iopub.status.idle":"2024-02-07T14:45:29.849495Z","shell.execute_reply.started":"2024-02-07T14:45:29.840060Z","shell.execute_reply":"2024-02-07T14:45:29.848587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_device(\n    tensors: tp.Union[tp.Tuple[torch.Tensor], tp.Dict[str, torch.Tensor]],\n    device: torch.device, *args, **kwargs\n):\n    if isinstance(tensors, tuple):\n        return (t.to(device, *args, **kwargs) for t in tensors)\n    elif isinstance(tensors, dict):\n        return {\n            k: t.to(device, *args, **kwargs) for k, t in tensors.items()}\n    else:\n        return tensors.to(device, *args, **kwargs)\n\n    \ndef get_test_path_label(test: pd.DataFrame):\n    \"\"\"Get file path and dummy target info.\"\"\"\n    \n    img_paths = []\n    labels = np.full((len(test), 6), -1, dtype=\"float32\")\n    for spec_id in test[\"spectrogram_id\"].values:\n        img_path = TEST_SPEC_SPLIT / f\"{spec_id}.npy\"\n        img_paths.append(img_path)\n        \n    test_data = {\n        \"image_paths\": img_paths,\n        \"labels\": [l for l in labels]}\n    \n    return test_data\n\ndef get_test_transforms(CFG):\n    test_transform = A.Compose([\n        A.Resize(p=1.0, height=CFG.img_size_h, width=CFG.img_size_w),\n        ToTensorV2(p=1.0)\n    ])\n    return test_transform","metadata":{"execution":{"iopub.status.busy":"2024-02-07T14:45:29.852652Z","iopub.execute_input":"2024-02-07T14:45:29.853409Z","iopub.status.idle":"2024-02-07T14:45:29.863555Z","shell.execute_reply.started":"2024-02-07T14:45:29.853377Z","shell.execute_reply":"2024-02-07T14:45:29.862781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_inference_loop(model, loader, device):\n    model.to(device)\n    model.eval()\n    pred_list = []\n    with torch.no_grad():\n        for batch in tqdm(loader):\n            x = to_device(batch[\"data\"], device)\n            y = model(x)\n            pred_list.append(y.softmax(dim=1).detach().cpu().numpy())\n        \n    pred_arr = np.concatenate(pred_list)\n    del pred_list\n    return pred_arr","metadata":{"execution":{"iopub.status.busy":"2024-02-07T14:45:29.864506Z","iopub.execute_input":"2024-02-07T14:45:29.864737Z","iopub.status.idle":"2024-02-07T14:45:29.875986Z","shell.execute_reply.started":"2024-02-07T14:45:29.864717Z","shell.execute_reply":"2024-02-07T14:45:29.875113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds_arr = np.zeros((N_FOLDS*5, len(test), N_CLASSES))\n\ntest_path_label = get_test_path_label(test)\n\n\"\"\" Dataset x512 \"\"\"\ntest_transform = get_test_transforms(CFG)\ntest_dataset = HMSHBACSpecDataset(**test_path_label, transform=test_transform)\ntest_loader = torch.utils.data.DataLoader(\n    test_dataset, batch_size=CFG.batch_size, num_workers=4, shuffle=False, drop_last=False)\n\n\"\"\" Dataset x1024 \"\"\"\ntest_transform_1024 = get_test_transforms(CFG2)\ntest_dataset_1024 = HMSHBACSpecDataset(**test_path_label, transform=test_transform_1024)\ntest_loader_1024 = torch.utils.data.DataLoader(\n    test_dataset_1024, batch_size=CFG.batch_size, num_workers=4, shuffle=False, drop_last=False)\n\n\"\"\" Dataset x600x800 \"\"\"\ntest_transform_x600x800 = get_test_transforms(CFG3)\ntest_dataset_x600x800 = HMSHBACSpecDataset(**test_path_label, transform=test_transform_x600x800)\ntest_loader_x600x800 = torch.utils.data.DataLoader(\n    test_dataset_x600x800, batch_size=CFG.batch_size, num_workers=4, shuffle=False, drop_last=False)\n\n\ndevice = torch.device(CFG.device)\n\nfor fold_id in range(N_FOLDS):\n    print(f\"\\n[fold {fold_id}]\")\n    \n    # ---------------------------------------------------------------- #\n    # Model Pred \n    # ---------------------------------------------------------------- #\n    model_path = TRAINED_MODEL / f\"best_model_fold{fold_id}.pth\"\n    model = HMSHBACSpecModel(\n        model_name=CFG.model_name, pretrained=False, num_classes=6, in_channels=1)\n    model.load_state_dict(torch.load(model_path, map_location=device))\n    \n    # # inference\n    test_pred = run_inference_loop(model, test_loader, device)\n    test_preds_arr[fold_id] = test_pred\n    \n    del model\n    torch.cuda.empty_cache()\n    gc.collect()\n    \n    # ---------------------------------------------------------------- #\n    # Model Pred \n    # ---------------------------------------------------------------- #\n#     model_path = TRAINED_MODEL_2 / f\"best_model_fold{fold_id}.pth\"\n#     model = HMSHBACSpecModel(\n#         model_name=CFG.model_name, pretrained=False, num_classes=6, in_channels=1)\n#     model.load_state_dict(torch.load(model_path, map_location=device))\n    \n#     # # inference\n#     test_pred = run_inference_loop(model, test_loader, device)\n#     test_preds_arr[fold_id+10] = test_pred\n    \n#     del model\n#     torch.cuda.empty_cache()\n#     gc.collect()\n    \n    # ---------------------------------------------------------------- #\n    # Model Pred \n    # ---------------------------------------------------------------- #\n    model_path = TRAINED_MODEL_3 / f\"best_model_fold{fold_id}.pth\"\n    model = HMSHBACSpecModel(\n        model_name=CFG.model_name2, pretrained=False, num_classes=6, in_channels=1)\n    model.load_state_dict(torch.load(model_path, map_location=device))\n    \n    # # inference\n    test_pred = run_inference_loop(model, test_loader, device)\n    test_preds_arr[fold_id+N_FOLDS*1] = test_pred\n    \n    del model\n    torch.cuda.empty_cache()\n    gc.collect()\n    \n    # ---------------------------------------------------------------- #\n    # Model Pred \n    # ---------------------------------------------------------------- #\n#     model_path = TRAINED_MODEL_4 / f\"best_model_fold{fold_id}.pth\"\n#     model = HMSHBACSpecModel(\n#         model_name=CFG.model_name, pretrained=False, num_classes=6, in_channels=1)\n#     model.load_state_dict(torch.load(model_path, map_location=device))\n    \n#     # # inference\n#     test_pred = run_inference_loop(model, test_loader_1024, device)\n#     test_preds_arr[fold_id+20] = test_pred\n    \n#     del model\n#     torch.cuda.empty_cache()\n#     gc.collect()\n    \n    # ---------------------------------------------------------------- #\n    # Model Pred \n    # ---------------------------------------------------------------- #\n    model_path = TRAINED_MODEL_5 / f\"best_model_fold{fold_id}.pth\"\n    model = HMSHBACSpecModel(\n        model_name=CFG.model_name3, pretrained=False, num_classes=6, in_channels=1)\n    model.load_state_dict(torch.load(model_path, map_location=device))\n    \n    # # inference\n    test_pred = run_inference_loop(model, test_loader, device)\n    test_preds_arr[fold_id+N_FOLDS*2] = test_pred\n    \n    del model\n    torch.cuda.empty_cache()\n    gc.collect()\n    \n    # ---------------------------------------------------------------- #\n    # Model Pred \n    # ---------------------------------------------------------------- #\n    model_path = TRAINED_MODEL_6 / f\"best_model_fold{fold_id}.pth\"\n    model = HMSHBACSpecModel(\n        model_name=CFG.model_name, pretrained=False, num_classes=6, in_channels=1)\n    model.load_state_dict(torch.load(model_path, map_location=device))\n    \n    # # inference\n    test_pred = run_inference_loop(model, test_loader_x600x800, device)\n    test_preds_arr[fold_id+N_FOLDS*3] = test_pred\n    \n    del model\n    torch.cuda.empty_cache()\n    gc.collect()\n    \n    # ---------------------------------------------------------------- #\n    # Model Pred \n    # ---------------------------------------------------------------- #\n#     model_path = TRAINED_MODEL_7 / f\"best_model_fold{fold_id}.pth\"\n#     model = HMSHBACSpecModel(\n#         model_name=CFG.model_name3, pretrained=False, num_classes=6, in_channels=1)\n#     model.load_state_dict(torch.load(model_path, map_location=device))\n    \n#     # # inference\n#     test_pred = run_inference_loop(model, test_loader_x600x800, device)\n#     test_preds_arr[fold_id+40] = test_pred\n    \n#     del model\n#     torch.cuda.empty_cache()\n#     gc.collect()\n    \n    # ---------------------------------------------------------------- #\n    # Model Pred \n    # ---------------------------------------------------------------- #\n#     model_path = TRAINED_MODEL_8 / f\"best_model_fold{fold_id}.pth\"\n#     model = HMSHBACSpecModel(\n#         model_name=CFG.model_name, pretrained=False, num_classes=6, in_channels=1)\n#     model.load_state_dict(torch.load(model_path, map_location=device))\n    \n#     # # inference\n#     test_pred = run_inference_loop(model, test_loader_x600x800, device)\n#     test_preds_arr[fold_id+40] = test_pred\n    \n#     del model\n#     torch.cuda.empty_cache()\n#     gc.collect()\n    \n    # ---------------------------------------------------------------- #\n    # Model Pred \n    # ---------------------------------------------------------------- #\n    model_path = TRAINED_MODEL_9 / f\"best_model_fold{fold_id}.pth\"\n    model = HMSHBACSpecModel(\n        model_name=CFG.model_name3, pretrained=False, num_classes=6, in_channels=1)\n    model.load_state_dict(torch.load(model_path, map_location=device))\n    \n    # # inference\n    test_pred = run_inference_loop(model, test_loader_x600x800, device)\n    test_preds_arr[fold_id+N_FOLDS*4] = test_pred\n    \n    del model\n    torch.cuda.empty_cache()\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-07T14:45:29.878751Z","iopub.execute_input":"2024-02-07T14:45:29.879042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Submission","metadata":{}},{"cell_type":"code","source":"test_pred = test_preds_arr.mean(axis=0)\n\ntest_pred_df = pd.DataFrame(\n    test_pred, columns=CLASSES\n)\n\ntest_pred_df = pd.concat([test[[\"eeg_id\"]], test_pred_df], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"smpl_sub = pd.read_csv(DATA / \"sample_submission.csv\")\n\nsub = pd.merge(\n    smpl_sub[[\"eeg_id\"]], test_pred_df, on=\"eeg_id\", how=\"left\")\n\nsub.to_csv(\"submission.csv\", index=False)\n\nsub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EOF","metadata":{}}]}