{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 5 millions of rows are missing in the validation set","metadata":{}},{"cell_type":"markdown","source":"I was trying to a random split on the first 3 weeks of the training set, like Radek did on the 4th week of the training set, so that I can train the LGBMRanker model with a larger dataset (3 weeks) and use the 4th week as my validation set to test how good is my model.\n\nI don't remember how exactly I came to this. During the data exploration, I just could not add up the number together. So, I used this notebook and Radek's code from the local validation framework notebook to check numbers of the validation set (test.parquet, test_labels.parquet).\n\nWhat I found is that the 4th week of training set have 5 million rows more than ","metadata":{}},{"cell_type":"markdown","source":"# Load Libraries and Train Data","metadata":{}},{"cell_type":"code","source":"# LOAD LIBRARIES\nimport pandas as pd, numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as mpatches\n# import cudf, cupy\n# print('Using RAPIDS version',cudf.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-02T10:17:21.026860Z","iopub.execute_input":"2022-12-02T10:17:21.027285Z","iopub.status.idle":"2022-12-02T10:17:21.033643Z","shell.execute_reply.started":"2022-12-02T10:17:21.027254Z","shell.execute_reply":"2022-12-02T10:17:21.032353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = pd.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:17:29.706935Z","iopub.execute_input":"2022-12-02T10:17:29.707445Z","iopub.status.idle":"2022-12-02T10:17:49.321100Z","shell.execute_reply.started":"2022-12-02T10:17:29.707410Z","shell.execute_reply":"2022-12-02T10:17:49.319719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:18:09.878557Z","iopub.execute_input":"2022-12-02T10:18:09.880021Z","iopub.status.idle":"2022-12-02T10:18:09.889917Z","shell.execute_reply.started":"2022-12-02T10:18:09.879979Z","shell.execute_reply":"2022-12-02T10:18:09.888518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_cv = pd.read_parquet('/kaggle/input/otto-train-and-test-data-for-local-validation/train.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:17:49.325802Z","iopub.execute_input":"2022-12-02T10:17:49.326242Z","iopub.status.idle":"2022-12-02T10:18:09.876839Z","shell.execute_reply.started":"2022-12-02T10:17:49.326209Z","shell.execute_reply":"2022-12-02T10:18:09.875768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cv.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:18:09.892572Z","iopub.execute_input":"2022-12-02T10:18:09.893832Z","iopub.status.idle":"2022-12-02T10:18:09.906662Z","shell.execute_reply.started":"2022-12-02T10:18:09.893788Z","shell.execute_reply":"2022-12-02T10:18:09.905546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_cv = pd.read_parquet('/kaggle/input/otto-train-and-test-data-for-local-validation/test.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-02T11:19:41.952818Z","iopub.execute_input":"2022-12-02T11:19:41.955753Z","iopub.status.idle":"2022-12-02T11:19:42.491854Z","shell.execute_reply.started":"2022-12-02T11:19:41.955677Z","shell.execute_reply":"2022-12-02T11:19:42.490317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_cv.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:18:10.700865Z","iopub.execute_input":"2022-12-02T10:18:10.701224Z","iopub.status.idle":"2022-12-02T10:18:10.708870Z","shell.execute_reply.started":"2022-12-02T10:18:10.701192Z","shell.execute_reply":"2022-12-02T10:18:10.707686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_labels_cv = pd.read_parquet('/kaggle/input/otto-train-and-test-data-for-local-validation/test_labels.parquet')\ntest_labels_cv.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-02T11:20:11.798667Z","iopub.execute_input":"2022-12-02T11:20:11.799216Z","iopub.status.idle":"2022-12-02T11:20:14.472953Z","shell.execute_reply.started":"2022-12-02T11:20:11.799178Z","shell.execute_reply":"2022-12-02T11:20:14.470852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_labels_cv = test_labels_cv.explode('ground_truth')\ntest_labels_cv.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-02T11:20:39.194943Z","iopub.execute_input":"2022-12-02T11:20:39.195523Z","iopub.status.idle":"2022-12-02T11:20:42.885432Z","shell.execute_reply.started":"2022-12-02T11:20:39.195480Z","shell.execute_reply":"2022-12-02T11:20:42.882966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### TRAIN_4_WEEKS: 12_899_799 unqiue sessions","metadata":{}},{"cell_type":"code","source":"%%time\nunique_sessions = set(train.session)\nnum_unique_sessions = len(unique_sessions)\nnum_unique_sessions","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:18:31.269297Z","iopub.execute_input":"2022-12-02T10:18:31.269761Z","iopub.status.idle":"2022-12-02T10:18:57.251361Z","shell.execute_reply.started":"2022-12-02T10:18:31.269726Z","shell.execute_reply":"2022-12-02T10:18:57.249976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nunique_sessions_train_cv = set(train_cv.session)\nnum_unique_sessions_train_cv = len(unique_sessions_train_cv)\nnum_unique_sessions_train_cv","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:19:09.575018Z","iopub.execute_input":"2022-12-02T10:19:09.575575Z","iopub.status.idle":"2022-12-02T10:19:30.008649Z","shell.execute_reply.started":"2022-12-02T10:19:09.575534Z","shell.execute_reply":"2022-12-02T10:19:30.006907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nunique_sessions_test_cv = set(test_cv.session)\nnum_unique_sessions_test_cv = len(unique_sessions_test_cv)\nnum_unique_sessions_test_cv","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:19:35.297845Z","iopub.execute_input":"2022-12-02T10:19:35.298301Z","iopub.status.idle":"2022-12-02T10:19:36.368992Z","shell.execute_reply.started":"2022-12-02T10:19:35.298268Z","shell.execute_reply":"2022-12-02T10:19:36.367573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nunique_sessions_test_labels_cv = set(test_labels_cv.session)\nnum_unique_sessions_test_labels_cv = len(unique_sessions_test_labels_cv)\nnum_unique_sessions_test_labels_cv","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:19:40.448906Z","iopub.execute_input":"2022-12-02T10:19:40.449334Z","iopub.status.idle":"2022-12-02T10:19:40.860386Z","shell.execute_reply.started":"2022-12-02T10:19:40.449302Z","shell.execute_reply":"2022-12-02T10:19:40.858949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### train_cv and test_cv have no shared sessions (1)","metadata":{}},{"cell_type":"code","source":"%%time\nunique_sessions_train_cv.intersection(unique_sessions_test_cv)","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:19:49.489599Z","iopub.execute_input":"2022-12-02T10:19:49.490184Z","iopub.status.idle":"2022-12-02T10:19:49.521253Z","shell.execute_reply.started":"2022-12-02T10:19:49.490129Z","shell.execute_reply":"2022-12-02T10:19:49.519463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### unique sessions of (train_cv + test_cv) == unique sessions of train (2)","metadata":{}},{"cell_type":"code","source":"num_unique_sessions_test_cv  + num_unique_sessions_train_cv == num_unique_sessions","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:19:58.633632Z","iopub.execute_input":"2022-12-02T10:19:58.634086Z","iopub.status.idle":"2022-12-02T10:19:58.643496Z","shell.execute_reply.started":"2022-12-02T10:19:58.634052Z","shell.execute_reply":"2022-12-02T10:19:58.642069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Based on (1) and (2), test_cv and test_labels_cv must have dropped all the intersected sessions with train_cv already","metadata":{"execution":{"iopub.status.busy":"2022-11-30T11:43:16.142166Z","iopub.execute_input":"2022-11-30T11:43:16.143352Z","iopub.status.idle":"2022-11-30T11:43:16.147545Z","shell.execute_reply.started":"2022-11-30T11:43:16.143309Z","shell.execute_reply":"2022-11-30T11:43:16.146509Z"}}},{"cell_type":"code","source":"import datetime","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:20:00.419358Z","iopub.execute_input":"2022-12-02T10:20:00.419826Z","iopub.status.idle":"2022-12-02T10:20:00.425662Z","shell.execute_reply.started":"2022-12-02T10:20:00.419791Z","shell.execute_reply":"2022-12-02T10:20:00.424579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### test_cv and test_labels_cv are the 4th week of train set (3)","metadata":{}},{"cell_type":"code","source":"datetime.datetime.fromtimestamp(test_cv['ts'].min()), datetime.datetime.fromtimestamp(test_cv['ts'].max())","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:20:02.630579Z","iopub.execute_input":"2022-12-02T10:20:02.631136Z","iopub.status.idle":"2022-12-02T10:20:02.657996Z","shell.execute_reply.started":"2022-12-02T10:20:02.631094Z","shell.execute_reply":"2022-12-02T10:20:02.656703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Based on (3), train_4th_week should have the same num of rows of test_cv, test_labels_cv and the dropped intersected sessions of train_4th_week combined","metadata":{}},{"cell_type":"markdown","source":"### The start and end date of train set: 2022-7-31, 22:00 to 2022-8-28, 21:59:59","metadata":{}},{"cell_type":"code","source":"import datetime\ndatetime.datetime.fromtimestamp(train.ts.max()), datetime.datetime.fromtimestamp(train.ts.min())","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:20:07.367134Z","iopub.execute_input":"2022-12-02T10:20:07.368896Z","iopub.status.idle":"2022-12-02T10:20:07.787888Z","shell.execute_reply.started":"2022-12-02T10:20:07.368814Z","shell.execute_reply":"2022-12-02T10:20:07.786152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nseven_days = 7*24*60*60\n\ntrain_cutoff = train.ts.max() - seven_days","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:20:09.373083Z","iopub.execute_input":"2022-12-02T10:20:09.373834Z","iopub.status.idle":"2022-12-02T10:20:09.596940Z","shell.execute_reply.started":"2022-12-02T10:20:09.373782Z","shell.execute_reply":"2022-12-02T10:20:09.594822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### the split datetime is 2022-8-21, 21:59:59","metadata":{}},{"cell_type":"code","source":"datetime.datetime.fromtimestamp(train_cutoff)","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:20:12.794490Z","iopub.execute_input":"2022-12-02T10:20:12.795442Z","iopub.status.idle":"2022-12-02T10:20:12.804632Z","shell.execute_reply.started":"2022-12-02T10:20:12.795396Z","shell.execute_reply":"2022-12-02T10:20:12.802562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_4th_weeks = train[train.ts > train_cutoff]","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:20:17.638028Z","iopub.execute_input":"2022-12-02T10:20:17.639002Z","iopub.status.idle":"2022-12-02T10:20:21.803517Z","shell.execute_reply.started":"2022-12-02T10:20:17.638943Z","shell.execute_reply":"2022-12-02T10:20:21.802370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_3_weeks = train[train.ts <= train_cutoff]","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:20:21.805552Z","iopub.execute_input":"2022-12-02T10:20:21.806298Z","iopub.status.idle":"2022-12-02T10:20:28.680407Z","shell.execute_reply.started":"2022-12-02T10:20:21.806257Z","shell.execute_reply":"2022-12-02T10:20:28.679164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\noverlapping_sessions = set(train_3_weeks.session).intersection(set(train_4th_weeks.session))","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:20:53.200396Z","iopub.execute_input":"2022-12-02T10:20:53.200940Z","iopub.status.idle":"2022-12-02T10:21:20.956023Z","shell.execute_reply.started":"2022-12-02T10:20:53.200899Z","shell.execute_reply":"2022-12-02T10:21:20.954683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3_521_833 sessions shared between train_3_weeks and train_4th_week","metadata":{}},{"cell_type":"code","source":"len(overlapping_sessions)","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:21:23.615231Z","iopub.execute_input":"2022-12-02T10:21:23.615668Z","iopub.status.idle":"2022-12-02T10:21:23.625481Z","shell.execute_reply.started":"2022-12-02T10:21:23.615636Z","shell.execute_reply":"2022-12-02T10:21:23.623777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_4th_weeks_intersected_sessions = train_4th_weeks[train_4th_weeks.session.isin(overlapping_sessions)]","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:21:28.568861Z","iopub.execute_input":"2022-12-02T10:21:28.569264Z","iopub.status.idle":"2022-12-02T10:21:32.291879Z","shell.execute_reply.started":"2022-12-02T10:21:28.569234Z","shell.execute_reply":"2022-12-02T10:21:32.290423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### train_4th_week has 37_376_338 rows inside the 3_521_833 sessions","metadata":{}},{"cell_type":"code","source":"train_4th_weeks_intersected_sessions.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:21:39.298025Z","iopub.execute_input":"2022-12-02T10:21:39.298514Z","iopub.status.idle":"2022-12-02T10:21:39.307416Z","shell.execute_reply.started":"2022-12-02T10:21:39.298478Z","shell.execute_reply":"2022-12-02T10:21:39.305780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_4th_weeks.shape[0] == train_4th_weeks_intersected_sessions.shape[0] + test_cv.shape[0] + test_labels_cv.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:21:46.476576Z","iopub.execute_input":"2022-12-02T10:21:46.477086Z","iopub.status.idle":"2022-12-02T10:21:46.486449Z","shell.execute_reply.started":"2022-12-02T10:21:46.477036Z","shell.execute_reply":"2022-12-02T10:21:46.484855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### There seems to be 5 million rows in local validation set are missing from train_4th_week","metadata":{}},{"cell_type":"code","source":"train_4th_weeks.shape[0] - (train_4th_weeks_intersected_sessions.shape[0] + test_cv.shape[0] + test_labels_cv.shape[0])","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:21:53.343890Z","iopub.execute_input":"2022-12-02T10:21:53.344431Z","iopub.status.idle":"2022-12-02T10:21:53.355886Z","shell.execute_reply.started":"2022-12-02T10:21:53.344394Z","shell.execute_reply":"2022-12-02T10:21:53.353663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### But the missing rows could be duplicated rows from test_labels_cv, I need to recreate test_labels_cv to prove it.","metadata":{}},{"cell_type":"markdown","source":"### train_3_weeks has same number of rows as train_cv (1 row difference)","metadata":{}},{"cell_type":"code","source":"train_3_weeks.shape[0] - train_cv.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:22:06.742834Z","iopub.execute_input":"2022-12-02T10:22:06.743371Z","iopub.status.idle":"2022-12-02T10:22:06.752624Z","shell.execute_reply.started":"2022-12-02T10:22:06.743332Z","shell.execute_reply":"2022-12-02T10:22:06.751123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_cv, test_cv, test_labels_cv, train","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:22:08.421853Z","iopub.execute_input":"2022-12-02T10:22:08.422349Z","iopub.status.idle":"2022-12-02T10:22:08.429354Z","shell.execute_reply.started":"2022-12-02T10:22:08.422314Z","shell.execute_reply":"2022-12-02T10:22:08.428050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:22:12.674885Z","iopub.execute_input":"2022-12-02T10:22:12.676429Z","iopub.status.idle":"2022-12-02T10:22:13.526894Z","shell.execute_reply.started":"2022-12-02T10:22:12.676376Z","shell.execute_reply":"2022-12-02T10:22:13.525581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_4th_weeks_drop = train_4th_weeks[~train_4th_weeks.session.isin(overlapping_sessions)]","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:22:38.707385Z","iopub.execute_input":"2022-12-02T10:22:38.708365Z","iopub.status.idle":"2022-12-02T10:22:41.031947Z","shell.execute_reply.started":"2022-12-02T10:22:38.708316Z","shell.execute_reply":"2022-12-02T10:22:41.030058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nlen(set(train_4th_weeks_drop.session)), len(set(train_3_weeks.session))","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:22:59.378352Z","iopub.execute_input":"2022-12-02T10:22:59.378788Z","iopub.status.idle":"2022-12-02T10:23:21.383991Z","shell.execute_reply.started":"2022-12-02T10:22:59.378754Z","shell.execute_reply":"2022-12-02T10:23:21.382835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nlen(train_4th_weeks_drop.session.unique())","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:23:21.385859Z","iopub.execute_input":"2022-12-02T10:23:21.386222Z","iopub.status.idle":"2022-12-02T10:23:21.580215Z","shell.execute_reply.started":"2022-12-02T10:23:21.386190Z","shell.execute_reply":"2022-12-02T10:23:21.578874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Do random split on train_4th_weeks_drop\n\nThere are 1.8MM unique sessions, so it actually takes about 10 minutes to run in the following cell.","metadata":{}},{"cell_type":"code","source":"%%time\nnew_test = []\ndata_to_calculate_validation_score = []\n\nfor grp in train_4th_weeks_drop.groupby('session'):\n    cutoff = np.random.randint(1, grp[1].shape[0]) # we want at least a single item in our validation data for each session\n    new_test.append(grp[1].iloc[:cutoff])\n    data_to_calculate_validation_score.append(grp[1].iloc[cutoff:])","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:24:06.844343Z","iopub.execute_input":"2022-12-02T10:24:06.844944Z","iopub.status.idle":"2022-12-02T10:27:36.787485Z","shell.execute_reply.started":"2022-12-02T10:24:06.844902Z","shell.execute_reply":"2022-12-02T10:27:36.785859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_4th_weeks_test = pd.concat(new_test).reset_index(drop=True)\ntrain_4th_weeks_test_labels = pd.concat(data_to_calculate_validation_score).reset_index(drop=True)\n\ntrain_4th_weeks_test.to_parquet('train_4th_weeks_test.parquet')\ntrain_4th_weeks_test_labels.to_parquet('train_4th_weeks_test_labels.parquet')\n\ndel new_test, data_to_calculate_validation_score","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:28:15.080090Z","iopub.execute_input":"2022-12-02T10:28:15.081407Z","iopub.status.idle":"2022-12-02T10:35:23.592251Z","shell.execute_reply.started":"2022-12-02T10:28:15.081297Z","shell.execute_reply":"2022-12-02T10:35:23.590827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:35:23.683711Z","iopub.execute_input":"2022-12-02T10:35:23.684250Z","iopub.status.idle":"2022-12-02T10:35:24.620959Z","shell.execute_reply.started":"2022-12-02T10:35:23.684211Z","shell.execute_reply":"2022-12-02T10:35:24.619478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_4th_weeks_test_labels.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:41:41.309008Z","iopub.execute_input":"2022-12-02T10:41:41.309523Z","iopub.status.idle":"2022-12-02T10:41:41.319726Z","shell.execute_reply.started":"2022-12-02T10:41:41.309479Z","shell.execute_reply":"2022-12-02T10:41:41.318137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_4th_weeks_test_labels_drop_duplicate = train_4th_weeks_test_labels.drop_duplicates(['session', 'type', 'aid'])","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:42:06.806536Z","iopub.execute_input":"2022-12-02T10:42:06.807004Z","iopub.status.idle":"2022-12-02T10:42:08.467071Z","shell.execute_reply.started":"2022-12-02T10:42:06.806962Z","shell.execute_reply":"2022-12-02T10:42:08.465727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_4th_weeks_test_labels_drop_duplicate.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:42:17.776117Z","iopub.execute_input":"2022-12-02T10:42:17.776659Z","iopub.status.idle":"2022-12-02T10:42:17.784438Z","shell.execute_reply.started":"2022-12-02T10:42:17.776620Z","shell.execute_reply":"2022-12-02T10:42:17.783280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_4th_weeks_drop.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:43:54.601770Z","iopub.execute_input":"2022-12-02T10:43:54.602280Z","iopub.status.idle":"2022-12-02T10:43:54.611423Z","shell.execute_reply.started":"2022-12-02T10:43:54.602238Z","shell.execute_reply":"2022-12-02T10:43:54.609898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### the number of unique rows","metadata":{}},{"cell_type":"code","source":"train_4th_weeks_drop.drop_duplicates(['session', 'type', 'aid']).shape","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:44:25.582072Z","iopub.execute_input":"2022-12-02T10:44:25.582451Z","iopub.status.idle":"2022-12-02T10:44:29.440372Z","shell.execute_reply.started":"2022-12-02T10:44:25.582419Z","shell.execute_reply":"2022-12-02T10:44:29.438918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number of duplicated rows in the 4th week of train","metadata":{}},{"cell_type":"code","source":"15384577 - 11161117","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:44:46.377786Z","iopub.execute_input":"2022-12-02T10:44:46.378250Z","iopub.status.idle":"2022-12-02T10:44:46.387223Z","shell.execute_reply.started":"2022-12-02T10:44:46.378214Z","shell.execute_reply":"2022-12-02T10:44:46.385573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_cv.drop_duplicates(['session', 'type', 'aid']).shape","metadata":{"execution":{"iopub.status.busy":"2022-12-02T11:24:48.059109Z","iopub.execute_input":"2022-12-02T11:24:48.059571Z","iopub.status.idle":"2022-12-02T11:24:50.591691Z","shell.execute_reply.started":"2022-12-02T11:24:48.059538Z","shell.execute_reply":"2022-12-02T11:24:50.589844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_cv.drop_duplicates(['session', 'type', 'aid']).head()","metadata":{"execution":{"iopub.status.busy":"2022-12-02T11:25:23.973636Z","iopub.execute_input":"2022-12-02T11:25:23.974662Z","iopub.status.idle":"2022-12-02T11:25:26.186997Z","shell.execute_reply.started":"2022-12-02T11:25:23.974610Z","shell.execute_reply":"2022-12-02T11:25:26.185574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_cv.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-02T11:23:28.969518Z","iopub.execute_input":"2022-12-02T11:23:28.970103Z","iopub.status.idle":"2022-12-02T11:23:28.980464Z","shell.execute_reply.started":"2022-12-02T11:23:28.970063Z","shell.execute_reply":"2022-12-02T11:23:28.979046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_cv.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-02T11:22:57.862909Z","iopub.execute_input":"2022-12-02T11:22:57.863427Z","iopub.status.idle":"2022-12-02T11:22:57.881738Z","shell.execute_reply.started":"2022-12-02T11:22:57.863390Z","shell.execute_reply":"2022-12-02T11:22:57.879980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_4th_weeks = train_4th_weeks.groupby('session')['aid'].apply(list)","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:16:58.036600Z","iopub.status.idle":"2022-12-02T10:16:58.037035Z","shell.execute_reply.started":"2022-12-02T10:16:58.036825Z","shell.execute_reply":"2022-12-02T10:16:58.036843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(dict.fromkeys(AIDs[::-1]))","metadata":{"execution":{"iopub.status.busy":"2022-12-02T10:16:58.038622Z","iopub.status.idle":"2022-12-02T10:16:58.039089Z","shell.execute_reply.started":"2022-12-02T10:16:58.038869Z","shell.execute_reply":"2022-12-02T10:16:58.038888Z"},"trusted":true},"execution_count":null,"outputs":[]}]}