{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-13T03:12:49.725115Z","iopub.execute_input":"2022-08-13T03:12:49.725508Z","iopub.status.idle":"2022-08-13T03:12:49.730907Z","shell.execute_reply.started":"2022-08-13T03:12:49.725474Z","shell.execute_reply":"2022-08-13T03:12:49.729192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/quora-question-pairs/train.csv.zip', keep_default_na=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:49.753838Z","iopub.execute_input":"2022-08-13T03:12:49.754734Z","iopub.status.idle":"2022-08-13T03:12:51.881303Z","shell.execute_reply.started":"2022-08-13T03:12:49.754700Z","shell.execute_reply":"2022-08-13T03:12:51.880180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating a validation set\n\nThe training set has about 37% positive labels, but the public test set has [about 17.4%](https://www.kaggle.com/code/davidthaler/how-many-1-s-are-in-the-public-lb).\nOne hypothesis is that there is a [declining positive label rate with increasing qid](https://www.kaggle.com/code/ashhafez/temporal-pattern-in-train-response-rates/notebook) which is related to [date posted](https://www.kaggle.com/code/tezdhar/temporal-pattern-verified-with-quora-dates) but when comparing with [time sensitive topics](https://www.kaggle.com/code/mikevdickson/are-qid-and-time-related/notebook) there doesn't seem to be a correlation.\n\nLet's try each way","metadata":{}},{"cell_type":"code","source":"val_1_pct = 0.1","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:51.883198Z","iopub.execute_input":"2022-08-13T03:12:51.883598Z","iopub.status.idle":"2022-08-13T03:12:51.889141Z","shell.execute_reply.started":"2022-08-13T03:12:51.883560Z","shell.execute_reply":"2022-08-13T03:12:51.887939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qid_cutoff = (\n    df.assign(qid_max=lambda df: np.maximum(df['qid1'], df['qid2']))\n    ['qid_max']\n    .value_counts(sort=False)\n    .rename('n')\n    .to_frame()\n    .sort_index()\n    .reset_index()\n    .assign(cp=lambda _: _.n.cumsum() / _.n.sum())\n    .query('cp>=1-@val_1_pct')\n    .iloc[0,0]\n)\nqid_cutoff","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:51.890597Z","iopub.execute_input":"2022-08-13T03:12:51.891483Z","iopub.status.idle":"2022-08-13T03:12:52.006114Z","shell.execute_reply.started":"2022-08-13T03:12:51.891448Z","shell.execute_reply":"2022-08-13T03:12:52.005042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['val1'] = (df['qid1'] > qid_cutoff) | (df['qid2'] > qid_cutoff)\n\ndf.groupby('val1').agg(n=('id', 'count'), p=('is_duplicate', 'mean')).assign(pct=lambda _: _.n/_.n.sum())","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:52.008543Z","iopub.execute_input":"2022-08-13T03:12:52.008928Z","iopub.status.idle":"2022-08-13T03:12:52.049611Z","shell.execute_reply.started":"2022-08-13T03:12:52.008894Z","shell.execute_reply":"2022-08-13T03:12:52.048558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now a random split of the rest","metadata":{}},{"cell_type":"code","source":"val_pct = 0.1\n\nval_pos = 0.174","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:52.051259Z","iopub.execute_input":"2022-08-13T03:12:52.051895Z","iopub.status.idle":"2022-08-13T03:12:52.056939Z","shell.execute_reply.started":"2022-08-13T03:12:52.051859Z","shell.execute_reply":"2022-08-13T03:12:52.055947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dft = df.loc[~df['val1']]","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:52.058495Z","iopub.execute_input":"2022-08-13T03:12:52.059092Z","iopub.status.idle":"2022-08-13T03:12:52.096069Z","shell.execute_reply.started":"2022-08-13T03:12:52.059054Z","shell.execute_reply":"2022-08-13T03:12:52.095004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pos_prob = val_pos / dft['is_duplicate'].sum()\nneg_prob = (1-val_pos) / (dft['is_duplicate'] == 0).sum()\nsample_prob = np.where(df['val1'], 0, np.where(df['is_duplicate'] == 1, pos_prob, neg_prob))\nsample_prob.sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:52.097346Z","iopub.execute_input":"2022-08-13T03:12:52.097707Z","iopub.status.idle":"2022-08-13T03:12:52.117444Z","shell.execute_reply.started":"2022-08-13T03:12:52.097672Z","shell.execute_reply":"2022-08-13T03:12:52.116608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_id = np.random.choice(len(df),\n                             size=int(len(df) * val_pct),\n                             replace=False,\n                             p=sample_prob)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:52.118734Z","iopub.execute_input":"2022-08-13T03:12:52.119148Z","iopub.status.idle":"2022-08-13T03:12:52.158492Z","shell.execute_reply.started":"2022-08-13T03:12:52.119115Z","shell.execute_reply":"2022-08-13T03:12:52.157563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_qid = set(np.concatenate([df.loc[sample_id, 'qid1'], df.loc[sample_id, 'qid2']]))","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:52.159743Z","iopub.execute_input":"2022-08-13T03:12:52.160160Z","iopub.status.idle":"2022-08-13T03:12:52.181732Z","shell.execute_reply.started":"2022-08-13T03:12:52.160125Z","shell.execute_reply":"2022-08-13T03:12:52.180611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['val2'] = False\ndf.loc[sample_id, 'val2'] = True\ndf['train'] = ~df['val1'] & ~(df['qid1'].isin(valid_qid) | df['qid2'].isin(valid_qid))","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:52.185643Z","iopub.execute_input":"2022-08-13T03:12:52.185898Z","iopub.status.idle":"2022-08-13T03:12:52.245031Z","shell.execute_reply.started":"2022-08-13T03:12:52.185875Z","shell.execute_reply":"2022-08-13T03:12:52.244217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert (df['val1'] & df['val2']).sum() == 0\nassert (df['train'] & df['val1']).sum() == 0\nassert (df['train'] & df['val2']).sum() == 0","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:52.246232Z","iopub.execute_input":"2022-08-13T03:12:52.246718Z","iopub.status.idle":"2022-08-13T03:12:52.256122Z","shell.execute_reply.started":"2022-08-13T03:12:52.246597Z","shell.execute_reply":"2022-08-13T03:12:52.255040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['split'] = np.where(df['train'], 'train',\n                       np.where(df['val1'], 'val1',\n                                np.where(df['val2'], 'val2', 'other')))\n\n(\n    df\n    .groupby(['split'])\n    .agg(n=('id', 'count'),\n         p=('is_duplicate', 'mean'))\n    .assign(pct=lambda _: _.n/sum(_.n))\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:52.257679Z","iopub.execute_input":"2022-08-13T03:12:52.258508Z","iopub.status.idle":"2022-08-13T03:12:52.377316Z","shell.execute_reply.started":"2022-08-13T03:12:52.258475Z","shell.execute_reply":"2022-08-13T03:12:52.376306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_val1 = df.query('val1').copy()\ndf_val2 = df.query('val2').copy()\ndf_train = df.query('train').copy()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:52.378756Z","iopub.execute_input":"2022-08-13T03:12:52.379332Z","iopub.status.idle":"2022-08-13T03:12:52.529200Z","shell.execute_reply.started":"2022-08-13T03:12:52.379295Z","shell.execute_reply":"2022-08-13T03:12:52.528269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train a model","metadata":{}},{"cell_type":"code","source":"from transformers import AutoTokenizer, AutoModelForSequenceClassification\nfrom datasets import Dataset","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:52.530567Z","iopub.execute_input":"2022-08-13T03:12:52.531019Z","iopub.status.idle":"2022-08-13T03:12:53.577510Z","shell.execute_reply.started":"2022-08-13T03:12:52.530974Z","shell.execute_reply":"2022-08-13T03:12:53.576586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['label'] = df['is_duplicate']","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:53.578936Z","iopub.execute_input":"2022-08-13T03:12:53.579519Z","iopub.status.idle":"2022-08-13T03:12:53.586564Z","shell.execute_reply.started":"2022-08-13T03:12:53.579474Z","shell.execute_reply":"2022-08-13T03:12:53.585430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = Dataset.from_pandas(df, split='split')","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:53.588593Z","iopub.execute_input":"2022-08-13T03:12:53.588998Z","iopub.status.idle":"2022-08-13T03:12:53.732430Z","shell.execute_reply.started":"2022-08-13T03:12:53.588964Z","shell.execute_reply":"2022-08-13T03:12:53.731521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name = 'nreimers/MiniLMv2-L6-H384-distilled-from-BERT-Base'","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:53.734437Z","iopub.execute_input":"2022-08-13T03:12:53.735185Z","iopub.status.idle":"2022-08-13T03:12:53.740077Z","shell.execute_reply.started":"2022-08-13T03:12:53.735142Z","shell.execute_reply":"2022-08-13T03:12:53.738860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(model_name)\nmodel = AutoModelForSequenceClassification.from_pretrained(model_name)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:12:53.742982Z","iopub.execute_input":"2022-08-13T03:12:53.743886Z","iopub.status.idle":"2022-08-13T03:13:10.959713Z","shell.execute_reply.started":"2022-08-13T03:12:53.743852Z","shell.execute_reply":"2022-08-13T03:13:10.958717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pyarrow.compute as pc\n\ndef preprocess_function(examples):\n    return tokenizer(pc.binary_join_element_wise(examples[\"question1\"], examples['question2'], tokenizer.sep_token).to_pylist(),\n                     truncation=True,\n                     padding=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:13:10.963073Z","iopub.execute_input":"2022-08-13T03:13:10.963360Z","iopub.status.idle":"2022-08-13T03:13:10.971520Z","shell.execute_reply.started":"2022-08-13T03:13:10.963328Z","shell.execute_reply":"2022-08-13T03:13:10.970464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_tokenized = data.map(preprocess_function, batched=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:13:10.973075Z","iopub.execute_input":"2022-08-13T03:13:10.973450Z","iopub.status.idle":"2022-08-13T03:14:18.573544Z","shell.execute_reply.started":"2022-08-13T03:13:10.973416Z","shell.execute_reply":"2022-08-13T03:14:18.572673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.iloc[0].question1, df.iloc[0].question2","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:14:18.574828Z","iopub.execute_input":"2022-08-13T03:14:18.575207Z","iopub.status.idle":"2022-08-13T03:14:18.583191Z","shell.execute_reply.started":"2022-08-13T03:14:18.575170Z","shell.execute_reply":"2022-08-13T03:14:18.582019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'_'.join(tokenizer.convert_ids_to_tokens(data_tokenized[0]['input_ids']))","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:14:18.584573Z","iopub.execute_input":"2022-08-13T03:14:18.585084Z","iopub.status.idle":"2022-08-13T03:14:18.594518Z","shell.execute_reply.started":"2022-08-13T03:14:18.585049Z","shell.execute_reply":"2022-08-13T03:14:18.593552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import TrainingArguments, Trainer\n\ntraining_args = TrainingArguments(\n    output_dir = './output',\n    learning_rate=1e-4,\n    per_device_train_batch_size=16,\n    per_device_eval_batch_size=16,\n    num_train_epochs=3,\n    weight_decay=0.01,\n    report_to='none',\n    save_steps=100_000,\n    \n)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:14:18.596095Z","iopub.execute_input":"2022-08-13T03:14:18.596851Z","iopub.status.idle":"2022-08-13T03:14:22.916341Z","shell.execute_reply.started":"2022-08-13T03:14:18.596816Z","shell.execute_reply":"2022-08-13T03:14:22.915189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_indices = np.where(data_tokenized['train'])[0]\nval_indices = np.where(data_tokenized['val2'])[0]\nval1_indices = np.where(data_tokenized['val1'])[0]","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:14:22.917702Z","iopub.execute_input":"2022-08-13T03:14:22.918482Z","iopub.status.idle":"2022-08-13T03:14:23.456416Z","shell.execute_reply.started":"2022-08-13T03:14:22.918443Z","shell.execute_reply":"2022-08-13T03:14:23.455443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_indices)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:14:23.457944Z","iopub.execute_input":"2022-08-13T03:14:23.458301Z","iopub.status.idle":"2022-08-13T03:14:23.466874Z","shell.execute_reply.started":"2022-08-13T03:14:23.458263Z","shell.execute_reply":"2022-08-13T03:14:23.463877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = df.iloc[train_indices]['label'].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:14:23.468437Z","iopub.execute_input":"2022-08-13T03:14:23.469678Z","iopub.status.idle":"2022-08-13T03:14:23.555704Z","shell.execute_reply.started":"2022-08-13T03:14:23.469628Z","shell.execute_reply":"2022-08-13T03:14:23.554724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_pos","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:14:23.557211Z","iopub.execute_input":"2022-08-13T03:14:23.557592Z","iopub.status.idle":"2022-08-13T03:14:23.564493Z","shell.execute_reply.started":"2022-08-13T03:14:23.557557Z","shell.execute_reply":"2022-08-13T03:14:23.563424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nn_train = 250_000\n\ntrain_indices = np.random.choice(train_indices, n_train, replace=True,\n                                 p=np.where(train_labels==1,\n                                            val_pos/train_labels.sum(),\n                                            (1-val_pos)/(len(train_labels) - train_labels.sum())))","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:14:23.571351Z","iopub.execute_input":"2022-08-13T03:14:23.572013Z","iopub.status.idle":"2022-08-13T03:14:23.607845Z","shell.execute_reply.started":"2022-08-13T03:14:23.571977Z","shell.execute_reply":"2022-08-13T03:14:23.606872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer = Trainer(\n    model=model,\n    args=training_args,\n    train_dataset=data_tokenized.select(train_indices),\n    eval_dataset=data_tokenized.select(val_indices),\n    tokenizer=tokenizer,\n   # data_collator=data_collator,\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:14:23.609125Z","iopub.execute_input":"2022-08-13T03:14:23.609555Z","iopub.status.idle":"2022-08-13T03:14:28.134554Z","shell.execute_reply.started":"2022-08-13T03:14:23.609521Z","shell.execute_reply":"2022-08-13T03:14:28.133540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.train()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:14:28.136002Z","iopub.execute_input":"2022-08-13T03:14:28.136514Z","iopub.status.idle":"2022-08-13T03:44:16.180441Z","shell.execute_reply.started":"2022-08-13T03:14:28.136458Z","shell.execute_reply":"2022-08-13T03:44:16.179094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_preds = trainer.predict(data_tokenized.select(val_indices))","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:44:16.182185Z","iopub.execute_input":"2022-08-13T03:44:16.183641Z","iopub.status.idle":"2022-08-13T03:45:00.539209Z","shell.execute_reply.started":"2022-08-13T03:44:16.183604Z","shell.execute_reply":"2022-08-13T03:45:00.538314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val1_preds = trainer.predict(data_tokenized.select(val1_indices))","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:45:00.540748Z","iopub.execute_input":"2022-08-13T03:45:00.541025Z","iopub.status.idle":"2022-08-13T03:45:52.473718Z","shell.execute_reply.started":"2022-08-13T03:45:00.541000Z","shell.execute_reply":"2022-08-13T03:45:52.472830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"40k balanced","metadata":{}},{"cell_type":"code","source":"from scipy.special import expit","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:45:52.475044Z","iopub.execute_input":"2022-08-13T03:45:52.475883Z","iopub.status.idle":"2022-08-13T03:45:52.480588Z","shell.execute_reply.started":"2022-08-13T03:45:52.475844Z","shell.execute_reply":"2022-08-13T03:45:52.479641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Negative log loss","metadata":{}},{"cell_type":"code","source":"-np.log(expit(np.where(val_preds.label_ids==0, val_preds.predictions[:,0], val_preds.predictions[:,1]))).mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:45:52.481954Z","iopub.execute_input":"2022-08-13T03:45:52.482959Z","iopub.status.idle":"2022-08-13T03:45:52.494536Z","shell.execute_reply.started":"2022-08-13T03:45:52.482919Z","shell.execute_reply":"2022-08-13T03:45:52.493607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"-np.log(expit(np.where(val1_preds.label_ids==0, val1_preds.predictions[:,0], val1_preds.predictions[:,1]))).mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:45:52.497849Z","iopub.execute_input":"2022-08-13T03:45:52.498099Z","iopub.status.idle":"2022-08-13T03:45:52.507853Z","shell.execute_reply.started":"2022-08-13T03:45:52.498076Z","shell.execute_reply":"2022-08-13T03:45:52.506834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit","metadata":{}},{"cell_type":"code","source":"df_test = pd.read_csv('../input/quora-question-pairs/test.csv', keep_default_na=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:45:52.509353Z","iopub.execute_input":"2022-08-13T03:45:52.509691Z","iopub.status.idle":"2022-08-13T03:46:01.165573Z","shell.execute_reply.started":"2022-08-13T03:45:52.509659Z","shell.execute_reply":"2022-08-13T03:46:01.164595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_test = Dataset.from_pandas(df_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:46:01.167194Z","iopub.execute_input":"2022-08-13T03:46:01.167594Z","iopub.status.idle":"2022-08-13T03:46:02.089072Z","shell.execute_reply.started":"2022-08-13T03:46:01.167558Z","shell.execute_reply":"2022-08-13T03:46:02.088047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_test_tokenized = data_test.map(preprocess_function, batched=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T03:46:02.090447Z","iopub.execute_input":"2022-08-13T03:46:02.090806Z","iopub.status.idle":"2022-08-13T03:52:26.943468Z","shell.execute_reply.started":"2022-08-13T03:46:02.090772Z","shell.execute_reply":"2022-08-13T03:52:26.942555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_preds = trainer.predict(data_test_tokenized)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T05:02:42.023796Z","iopub.execute_input":"2022-08-13T05:02:42.024176Z","iopub.status.idle":"2022-08-13T05:46:51.479833Z","shell.execute_reply.started":"2022-08-13T05:02:42.024145Z","shell.execute_reply":"2022-08-13T05:46:51.478754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probs = expit(test_preds.predictions[:,1])","metadata":{"execution":{"iopub.status.busy":"2022-08-13T05:46:51.481815Z","iopub.execute_input":"2022-08-13T05:46:51.482257Z","iopub.status.idle":"2022-08-13T05:46:51.508699Z","shell.execute_reply.started":"2022-08-13T05:46:51.482221Z","shell.execute_reply":"2022-08-13T05:46:51.507866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['is_duplicate'] = probs","metadata":{"execution":{"iopub.status.busy":"2022-08-13T05:46:51.511098Z","iopub.execute_input":"2022-08-13T05:46:51.511607Z","iopub.status.idle":"2022-08-13T05:46:51.519333Z","shell.execute_reply.started":"2022-08-13T05:46:51.511581Z","shell.execute_reply":"2022-08-13T05:46:51.517435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.to_csv('submission.csv', columns=['test_id', 'is_duplicate'], index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T05:46:51.523126Z","iopub.execute_input":"2022-08-13T05:46:51.523381Z","iopub.status.idle":"2022-08-13T05:46:55.249824Z","shell.execute_reply.started":"2022-08-13T05:46:51.523359Z","shell.execute_reply":"2022-08-13T05:46:55.248487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-08-13T05:46:55.251572Z","iopub.execute_input":"2022-08-13T05:46:55.251943Z","iopub.status.idle":"2022-08-13T05:46:56.358259Z","shell.execute_reply.started":"2022-08-13T05:46:55.251897Z","shell.execute_reply":"2022-08-13T05:46:56.356975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wc -l submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-08-13T05:46:56.360196Z","iopub.execute_input":"2022-08-13T05:46:56.360547Z","iopub.status.idle":"2022-08-13T05:46:57.498797Z","shell.execute_reply.started":"2022-08-13T05:46:56.360515Z","shell.execute_reply":"2022-08-13T05:46:57.497607Z"},"trusted":true},"execution_count":null,"outputs":[]}]}