{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# import manipulation\nimport numpy as np\nimport pandas as pd\n\n# import Pytorch\nimport torch \nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.utils.checkpoint import checkpoint\nfrom torch.autograd import Variable\n\n# import wandb\nimport wandb\n\n# import Transformer model\nimport transformers\nfrom transformers import AutoTokenizer, AutoModel, AutoConfig, AdamW\nfrom transformers import DataCollatorWithPadding\nfrom transformers.models.deberta_v2.modeling_deberta_v2 import StableDropout, ContextPooler\n\n# import SKLearn\nfrom sklearn.model_selection import  KFold, GroupKFold, StratifiedKFold, StratifiedGroupKFold\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import log_loss\n\n\n# import ...\nimport string\nimport random\nimport os\nimport joblib\nimport gc\nimport copy\nimport time\n\n\n# other\nfrom tqdm import tqdm\nfrom collections import defaultdict\n\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:37:25.185897Z","iopub.execute_input":"2022-07-18T02:37:25.186535Z","iopub.status.idle":"2022-07-18T02:37:34.372257Z","shell.execute_reply.started":"2022-07-18T02:37:25.186443Z","shell.execute_reply":"2022-07-18T02:37:34.370904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    seed = 2022\n    max_length = 512\n    epoch = 4\n    train_batch_size = 16\n    valid_batch_size = 32\n\n    model_name = \"../input/debertav3base\"\n    token_name = \"../input/debertav3base\"\n\n    scheduler = \"CosineAnnealingLR\"\n    learning_rate = 1e-5\n    min_lr = 1e-6\n    T_max = 500\n    weight_decay = 0.005\n    \n    num_classes = 3\n    n_fold = 3\n    n_accumulate = 2\n    freezing = True\n    gradient_checkpoint = True\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n    \n    wandb_id = f\"PL{round(time.time())}\" # ID on WandB\n    group = f'{wandb_id}-Baseline'\n    competition = \"FeedBack\"\n    _wandb_kernel = \"deb\"\n\n    \nCFG.tokenizer = AutoTokenizer.from_pretrained(CFG.token_name, use_fast=True)\nCFG.tokenizer.model_max_length = CFG.max_length\nCFG.tokenizer.is_fast","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:37:34.379599Z","iopub.execute_input":"2022-07-18T02:37:34.383060Z","iopub.status.idle":"2022-07-18T02:37:35.223662Z","shell.execute_reply.started":"2022-07-18T02:37:34.383018Z","shell.execute_reply":"2022-07-18T02:37:35.222437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def freeze(module):\n    \"\"\"\n    Freezes module's parameters.\n    \"\"\"\n    for parameter in module.parameters():\n        parameter.requires_grad = False","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:37:35.225330Z","iopub.execute_input":"2022-07-18T02:37:35.225892Z","iopub.status.idle":"2022-07-18T02:37:35.232203Z","shell.execute_reply.started":"2022-07-18T02:37:35.225854Z","shell.execute_reply":"2022-07-18T02:37:35.230929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeedbackDataset(Dataset):\n    def __init__(self,df, max_length, tokenizer, training=True):\n        self.df = df\n        self.max_len = max_length\n        self.tokenizer = tokenizer\n        self.discourse_type = self.df['discourse_type'].values\n        self.discourse_text = self.df['discourse_text'].values\n        self.essays = self.df['essay_text'].values\n        self.training = training\n        \n        if self.training:\n            self.targets = self.df['discourse_effectiveness'].values\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        discourse_type = self.discourse_type[index]\n        discourse_text = self.discourse_text[index]\n        essay = self.essays[index]\n        type_text = discourse_type + ' ' + discourse_text\n        \n        inputs = self.tokenizer.encode_plus(\n            type_text, \n            essay,\n            truncation = True,\n            add_special_tokens = True,\n            return_token_type_ids = True,\n            max_length = self.max_len\n        )\n        \n        samples = {\n            'input_ids': inputs['input_ids'],\n            'attention_mask': inputs['attention_mask'],\n        }\n        \n        if 'token_type_ids' in inputs:\n            samples['token_type_ids'] = inputs['token_type_ids']\n          \n        if self.training:\n            samples['target'] = self.targets[index]\n        \n        return samples\n","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:37:35.235253Z","iopub.execute_input":"2022-07-18T02:37:35.236325Z","iopub.status.idle":"2022-07-18T02:37:35.250761Z","shell.execute_reply.started":"2022-07-18T02:37:35.236286Z","shell.execute_reply":"2022-07-18T02:37:35.249692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dynamic Padding (Collate)\nclass Collate:\n    def __init__(self, tokenizer, isTrain=True):\n        self.tokenizer = tokenizer\n        self.isTrain = isTrain\n        # self.args = args\n\n    def __call__(self, batch):\n        output = dict()\n        output[\"input_ids\"] = [sample[\"input_ids\"] for sample in batch]\n        output[\"attention_mask\"] = [sample[\"attention_mask\"] for sample in batch]\n        if self.isTrain:\n            output[\"target\"] = [sample[\"target\"] for sample in batch]\n\n        # calculate max token length of this batch\n        batch_max = max([len(ids) for ids in output[\"input_ids\"]])\n\n        # add padding\n        if self.tokenizer.padding_side == \"right\":\n            output[\"input_ids\"] = [s + (batch_max - len(s)) * [self.tokenizer.pad_token_id] for s in output[\"input_ids\"]]\n            output[\"attention_mask\"] = [s + (batch_max - len(s)) * [0] for s in output[\"attention_mask\"]]\n        else:\n            output[\"input_ids\"] = [(batch_max - len(s)) * [self.tokenizer.pad_token_id] + s for s in output[\"input_ids\"]]\n            output[\"attention_mask\"] = [(batch_max - len(s)) * [0] + s for s in output[\"attention_mask\"]]\n\n        # convert to tensors\n        output[\"input_ids\"] = torch.tensor(output[\"input_ids\"], dtype=torch.long)\n        output[\"attention_mask\"] = torch.tensor(output[\"attention_mask\"], dtype=torch.long)\n        if self.isTrain:\n            output[\"target\"] = torch.tensor(output[\"target\"], dtype=torch.long)\n\n        return output\n\n# collate_fn = DataCollatorWithPadding(tokenizer=CFG.tokenizer)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:37:35.252431Z","iopub.execute_input":"2022-07-18T02:37:35.253222Z","iopub.status.idle":"2022-07-18T02:37:35.268890Z","shell.execute_reply.started":"2022-07-18T02:37:35.253179Z","shell.execute_reply":"2022-07-18T02:37:35.267910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MeanPooling(nn.Module):\n    def __init__(self):\n        super(MeanPooling, self).__init__()\n        \n    def forward(self, last_hidden_state, attention_mask):\n        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n        sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n        sum_mask = input_mask_expanded.sum(1)\n        sum_mask = torch.clamp(sum_mask, min=1e-9)\n        mean_embeddings = sum_embeddings / sum_mask\n\n        return mean_embeddings\n\n    \nclass MeanMaxPooling(nn.Module):\n    def __init__(self):\n        super(MeanMaxPooling, self).__init__()\n        \n    def forward(self, last_hidden_state, attention_mask):\n        mean_pooling_embeddings = torch.mean(last_hidden_state, 1)\n        _, max_pooling_embeddings = torch.max(last_hidden_state, 1)\n        mean_max_embeddings = torch.cat((mean_pooling_embeddings, max_pooling_embeddings), 1)\n        return mean_max_embeddings\n\n    \nclass LSTMPooling(nn.Module):\n    def __init__(self, num_layers, hidden_size, hiddendim_lstm):\n        super(LSTMPooling, self).__init__()\n        self.num_hidden_layers = num_layers\n        self.hidden_size = hidden_size\n        self.hiddendim_lstm = hiddendim_lstm\n        self.lstm = nn.LSTM(self.hidden_size, self.hiddendim_lstm, batch_first=True)\n        self.dropout = nn.Dropout(0.1)\n    \n    def forward(self, all_hidden_states):\n        ## forward\n        hidden_states = torch.stack([all_hidden_states[layer_i][:, 0].squeeze()\n                                     for layer_i in range(1, self.num_hidden_layers+1)], dim=-1)\n        hidden_states = hidden_states.view(-1, self.num_hidden_layers, self.hidden_size)\n        out, _ = self.lstm(hidden_states, None)\n        out = self.dropout(out[:, -1, :])\n        return out\n    \nclass WeightedLayerPooling(nn.Module):\n    def __init__(self, num_hidden_layers, layer_start: int = 4, layer_weights = None):\n        super(WeightedLayerPooling, self).__init__()\n        self.layer_start = layer_start\n        self.num_hidden_layers = num_hidden_layers\n        self.layer_weights = layer_weights if layer_weights is not None \\\n            else nn.Parameter(\n                torch.tensor([1] * (num_hidden_layers+1 - layer_start), dtype=torch.float)\n            )\n\n    def forward(self, all_hidden_states):\n        all_layer_embedding = torch.stack(list(all_hidden_states), dim=0)\n        all_layer_embedding = all_layer_embedding[self.layer_start:, :, :, :]\n        weight_factor = self.layer_weights.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).expand(all_layer_embedding.size())\n        weighted_average = (weight_factor*all_layer_embedding).sum(dim=0) / self.layer_weights.sum()\n        return weighted_average","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:37:35.271974Z","iopub.execute_input":"2022-07-18T02:37:35.272750Z","iopub.status.idle":"2022-07-18T02:37:35.293967Z","shell.execute_reply.started":"2022-07-18T02:37:35.272713Z","shell.execute_reply":"2022-07-18T02:37:35.292565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MultiSampleDropout(nn.Module):\n    # Multisample Dropout: https://arxiv.org/abs/1905.09788\n    def __init__(self, classifier, start_prob=0.2, num_samples=8, increment=0.01):\n        super(MultiSampleDropout, self).__init__()\n        #self.dropout = nn.Dropout\n        self.dropouts = [StableDropout(start_prob + (increment*i)) for i in range(num_samples)] \n        self.classifier = classifier\n        \n    def forward(self, out):\n        return torch.mean(torch.stack([\n            self.classifier(dropout(out)) for dropout in self.dropouts\n        ], dim=0), dim=0)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:37:35.295348Z","iopub.execute_input":"2022-07-18T02:37:35.295768Z","iopub.status.idle":"2022-07-18T02:37:35.308390Z","shell.execute_reply.started":"2022-07-18T02:37:35.295731Z","shell.execute_reply":"2022-07-18T02:37:35.307352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeedbackModel(nn.Module):\n    def __init__(self, model_name):\n        super(FeedbackModel, self).__init__()\n        \n        # DeBERTa\n        self.model = AutoModel.from_pretrained(model_name)\n        self.config = AutoConfig.from_pretrained(model_name)\n        \n        # gradient checkpointing\n        if CFG.gradient_checkpoint:\n            self.model.gradient_checkpointing_enable()\n            print(f\"Gradient Checkpointing: {self.model.is_gradient_checkpointing}\")\n\n        # freezing embeddings and first 6 layers of encoder\n        if  CFG.freezing:\n            freeze(self.model.embeddings)\n            freeze(self.model.encoder.layer[:6])\n            \n        # Pooling\n        #self.weighted_pooler = WeightedLayerPooling(num_hidden_layers=self.config.num_hidden_layers, layer_start=4)\n        #self.pooler = MeanPooling()\n        \n        self.context_pooler = ContextPooler(self.config)\n        \n        #self.bilstm = nn.LSTM(self.config.hidden_size, self.config.hidden_size//2, num_layers=2, \n        #                      dropout=self.config.hidden_dropout_prob, batch_first=True,\n        #                      bidirectional=False)\n        \n        #self.drop = nn.Dropout(p=0.2)\n        \n        # Multi Sample Dropout\n        self.fc = nn.Linear(self.config.hidden_size, CFG.num_classes)\n        self.multi_sample_dropout = MultiSampleDropout(self.fc, start_prob=0.2, num_samples=8, increment=0.01)\n\n    def forward(self, ids, mask):        \n        out = self.model(input_ids=ids,attention_mask=mask,\n                        output_hidden_states=True)\n        \n        # out = self.weighted_pooler(out.hidden_states) # For WeightedLayerPooling\n        # out = self.pooler(out, mask) # For MeanPooling\n                \n        #out = self.context_pooler(torch.stack(list(out.hidden_states), dim=0)) # For ContextPooler\n        out = self.context_pooler(out[0]) # For ContextPooler\n\n        # out = self.pooler(out.last_hidden_state, mask)\n\n        outputs = self.multi_sample_dropout(out)\n        \n        #out = self.pooler(out.last_hidden_state, mask)\n        #out = self.bilstm(out)[0]\n        #out = self.drop(out)\n        #outputs = self.fc(out)\n\n        return outputs\n    \n    def set_optimizer_scheduler(self, option=\"Adam8bit\"):\n        if option == \"AdamW\":\n            model_parameters = filter(lambda parameter: parameter.requires_grad, self.parameters())\n\n            # Optimizer and scheduler\n            optimizer = AdamW(model_parameters, lr=CFG.learning_rate, weight_decay = CFG.weight_decay)\n            scheduler = fetch_scheduler(optimizer)\n        elif option == \"Adam8bit\":\n            # Adam 8-bits optimizer\n            no_decay = [\"bias\", \"LayerNorm.weight\"]\n            optimizer_grouped_parameters = [\n                {\n                    \"params\": [p for n, p in self.named_parameters() if not any(nd in n for nd in no_decay) and p[1].requires_grad],\n                    \"weight_decay\": CFG.weight_decay,\n                },\n                {\n                    \"params\": [p for n, p in self.named_parameters() if any(nd in n for nd in no_decay) and p[1].requires_grad],\n                    \"weight_decay\": 0.0,\n                },\n            ]\n\n            # initializing optimizer \n            # bnb_optimizer = bnb.optim.AdamW(params=model_parameters, lr=CFG.learning_rate, weight_decay=CFG.weight_decay, optim_bits=8)\n            optimizer = bnb.optim.Adam8bit(optimizer_grouped_parameters, lr=CFG.learning_rate)\n            print(f\"8-bit Optimizer:\\n\\n{optimizer}\")\n\n            # setting embeddings parameters\n            # set_embedding_parameters_bits(embeddings_path=self.model.embeddings)\n\n            scheduler = fetch_scheduler(optimizer)\n        else:\n            embedding_parameters = filter(lambda parameter: parameter.requires_grad, self.model.parameters())\n\n            optimizer_model = AdamW(embedding_parameters, lr=5e-6, weight_decay = CFG.weight_decay)\n            optimizer_linear = AdamW(model.fc.parameters(), lr=1e-4, weight_decay = CFG.weight_decay)\n\n            scheduler_model = fetch_scheduler(optimizer_model)\n            scheduler_linear = fetch_scheduler(optimizer_linear)\n\n            optimizer = [optimizer_model, optimizer_linear]\n            scheduler = [scheduler_model, scheduler_linear]\n        \n        self.optimizer = optimizer\n        self.scheduler = scheduler\n        return True\n    \n    def get_optimizer_scheduler(self):\n        return self.optimizer, self.scheduler","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:37:35.310919Z","iopub.execute_input":"2022-07-18T02:37:35.311762Z","iopub.status.idle":"2022-07-18T02:37:35.331821Z","shell.execute_reply.started":"2022-07-18T02:37:35.311725Z","shell.execute_reply":"2022-07-18T02:37:35.330801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(\"../input/feedback-prize-effectiveness/test.csv\")\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:37:35.333510Z","iopub.execute_input":"2022-07-18T02:37:35.333954Z","iopub.status.idle":"2022-07-18T02:37:35.367958Z","shell.execute_reply.started":"2022-07-18T02:37:35.333917Z","shell.execute_reply":"2022-07-18T02:37:35.366667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR = \"/kaggle/input/feedback-prize-effectiveness\"\nTEST_DIR = os.path.join(INPUT_DIR, \"test\")\nTEST_CSV = os.path.join(INPUT_DIR, \"test.csv\")\n\ndef get_essay_test(essay_id):\n    path = os.path.join(TEST_DIR, f'{essay_id}.txt')\n    essay_text = open(path, 'r').read()\n    return essay_text\n\ntest_df = pd.read_csv(TEST_CSV)\n\ntest_df['essay_text']= test_df['essay_id'].apply(get_essay_test)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:37:35.373238Z","iopub.execute_input":"2022-07-18T02:37:35.373513Z","iopub.status.idle":"2022-07-18T02:37:35.404442Z","shell.execute_reply.started":"2022-07-18T02:37:35.373489Z","shell.execute_reply":"2022-07-18T02:37:35.403283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"collate_fn = Collate(tokenizer=CFG.tokenizer, isTrain=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:37:35.406019Z","iopub.execute_input":"2022-07-18T02:37:35.406474Z","iopub.status.idle":"2022-07-18T02:37:35.411536Z","shell.execute_reply.started":"2022-07-18T02:37:35.406395Z","shell.execute_reply":"2022-07-18T02:37:35.410458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_test_loader(test_df):    \n    test_dataset = FeedbackDataset(test_df, \n                                   tokenizer=CFG.tokenizer, \n                                   max_length=CFG.max_length,\n                                    training=False)\n    \n    test_loader = DataLoader(test_dataset, \n                             batch_size=CFG.valid_batch_size, \n                             collate_fn=collate_fn, \n                             num_workers=2, \n                             shuffle=False, \n                             pin_memory=True, \n                             drop_last=False)\n    return test_loader\n\ntest_loader = prepare_test_loader(test_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:37:35.413074Z","iopub.execute_input":"2022-07-18T02:37:35.414078Z","iopub.status.idle":"2022-07-18T02:37:35.423449Z","shell.execute_reply.started":"2022-07-18T02:37:35.414042Z","shell.execute_reply":"2022-07-18T02:37:35.422243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.no_grad()\ndef inference(test_loader, model, device):\n    preds = []\n    model.eval()\n    model.to(device)\n    \n    bar = tqdm(enumerate(test_loader), total=len(test_loader))\n    \n    for step, data in bar: \n        ids = data['input_ids'].to(device, dtype = torch.long)\n        mask = data['attention_mask'].to(device, dtype = torch.long)\n        \n        output = model(ids, mask)\n        y_preds = nn.Softmax(dim=1)(torch.tensor(output.to('cpu'))).numpy()\n        \n        preds.append(y_preds)\n         \n    predictions = np.concatenate(preds)\n    return predictions","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:37:35.425124Z","iopub.execute_input":"2022-07-18T02:37:35.425518Z","iopub.status.idle":"2022-07-18T02:37:35.436104Z","shell.execute_reply.started":"2022-07-18T02:37:35.425448Z","shell.execute_reply":"2022-07-18T02:37:35.434645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deberta_predictions = []\n\nfor fold in range(CFG.n_fold):\n    print(\"Fold {}\".format(fold))\n\n    model = FeedbackModel(CFG.model_name)\n    state = torch.load(f'../input/deberta-v3-training/LossFold-{fold}.bin')\n\n    model.load_state_dict(state)\n\n    prediction = inference(test_loader, model, CFG.device)\n    deberta_predictions.append(prediction)\n    del model, state, prediction\n    gc.collect()\n    torch.cuda.empty_cache()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-18T02:37:35.438238Z","iopub.execute_input":"2022-07-18T02:37:35.438778Z","iopub.status.idle":"2022-07-18T02:38:12.309572Z","shell.execute_reply.started":"2022-07-18T02:37:35.438739Z","shell.execute_reply":"2022-07-18T02:38:12.308410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(deberta_predictions)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:38:12.311181Z","iopub.execute_input":"2022-07-18T02:38:12.311794Z","iopub.status.idle":"2022-07-18T02:38:12.320267Z","shell.execute_reply.started":"2022-07-18T02:38:12.311754Z","shell.execute_reply":"2022-07-18T02:38:12.319054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = np.mean(deberta_predictions, axis=0)\npredictions","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:38:12.321829Z","iopub.execute_input":"2022-07-18T02:38:12.322628Z","iopub.status.idle":"2022-07-18T02:38:12.332442Z","shell.execute_reply.started":"2022-07-18T02:38:12.322591Z","shell.execute_reply":"2022-07-18T02:38:12.331416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR = \"../input/feedback-prize-effectiveness/\"\nsubmission = pd.read_csv(os.path.join(INPUT_DIR, 'sample_submission.csv'))\n\nsubmission['Adequate'] = predictions[:, 0]\nsubmission['Effective'] = predictions[:, 1]\nsubmission['Ineffective'] = predictions[:, 2]\n\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:38:12.334696Z","iopub.execute_input":"2022-07-18T02:38:12.335073Z","iopub.status.idle":"2022-07-18T02:38:12.373779Z","shell.execute_reply.started":"2022-07-18T02:38:12.335038Z","shell.execute_reply":"2022-07-18T02:38:12.372793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T02:38:12.374977Z","iopub.execute_input":"2022-07-18T02:38:12.375337Z","iopub.status.idle":"2022-07-18T02:38:12.383118Z","shell.execute_reply.started":"2022-07-18T02:38:12.375302Z","shell.execute_reply":"2022-07-18T02:38:12.382244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}