{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-27T14:45:11.446487Z","iopub.execute_input":"2022-11-27T14:45:11.447191Z","iopub.status.idle":"2022-11-27T14:45:11.455895Z","shell.execute_reply.started":"2022-11-27T14:45:11.447154Z","shell.execute_reply":"2022-11-27T14:45:11.454368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom pathlib import Path\nimport os\nimport random\nimport numpy as np\nimport json\nimport datetime as dt\nfrom argparse import Namespace\nfrom datetime import timedelta\nfrom collections import Counter, defaultdict\nfrom tqdm.notebook import tqdm, trange\n\ntqdm.pandas()\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_theme()","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:45:11.458751Z","iopub.execute_input":"2022-11-27T14:45:11.459179Z","iopub.status.idle":"2022-11-27T14:45:12.627355Z","shell.execute_reply.started":"2022-11-27T14:45:11.459129Z","shell.execute_reply":"2022-11-27T14:45:12.625835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_PATH = Path('../input/otto-recommender-system')\nTRAIN_PATH = DATA_PATH/'train.jsonl'\nTEST_PATH = DATA_PATH/'test.jsonl'\nSAMPLE_SUB_PATH = Path('../input/otto-recommender-system/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:45:12.630360Z","iopub.execute_input":"2022-11-27T14:45:12.631038Z","iopub.status.idle":"2022-11-27T14:45:12.639363Z","shell.execute_reply.started":"2022-11-27T14:45:12.630982Z","shell.execute_reply":"2022-11-27T14:45:12.637483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(TRAIN_PATH, 'r') as f:\n    print(f\"We have {len(f.readlines()):,} lines in the training data\")","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:45:12.642471Z","iopub.execute_input":"2022-11-27T14:45:12.643050Z","iopub.status.idle":"2022-11-27T14:47:56.516937Z","shell.execute_reply.started":"2022-11-27T14:45:12.642982Z","shell.execute_reply":"2022-11-27T14:47:56.514096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_size = 20000\n\nchunks = pd.read_json(TRAIN_PATH, lines=True, chunksize = sample_size)\n\nfor c in chunks:\n    sample_train_df = c\n    break","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:47:56.522207Z","iopub.execute_input":"2022-11-27T14:47:56.522811Z","iopub.status.idle":"2022-11-27T14:47:58.414910Z","shell.execute_reply.started":"2022-11-27T14:47:56.522764Z","shell.execute_reply":"2022-11-27T14:47:58.413453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_size = 5000\n\nchunks = pd.read_json(TEST_PATH, lines=True, chunksize = sample_size)\n\nfor c in chunks:\n    sample_test_df = c\n    break","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:47:58.417588Z","iopub.execute_input":"2022-11-27T14:47:58.418154Z","iopub.status.idle":"2022-11-27T14:47:58.537134Z","shell.execute_reply.started":"2022-11-27T14:47:58.418101Z","shell.execute_reply":"2022-11-27T14:47:58.535703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:47:58.539291Z","iopub.execute_input":"2022-11-27T14:47:58.539710Z","iopub.status.idle":"2022-11-27T14:47:58.649437Z","shell.execute_reply.started":"2022-11-27T14:47:58.539674Z","shell.execute_reply":"2022-11-27T14:47:58.647915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_train_df.iloc[0, 1]","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:47:58.651216Z","iopub.execute_input":"2022-11-27T14:47:58.651705Z","iopub.status.idle":"2022-11-27T14:47:58.695066Z","shell.execute_reply.started":"2022-11-27T14:47:58.651663Z","shell.execute_reply":"2022-11-27T14:47:58.693338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.to_datetime(1659304800025, unit='ms')","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:47:58.697197Z","iopub.execute_input":"2022-11-27T14:47:58.697565Z","iopub.status.idle":"2022-11-27T14:47:58.716275Z","shell.execute_reply.started":"2022-11-27T14:47:58.697533Z","shell.execute_reply":"2022-11-27T14:47:58.714417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_train_df['max_date'] = sample_train_df.progress_apply(lambda x: pd.to_datetime(x.events[-1]['ts'], unit='ms'), axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:47:58.718326Z","iopub.execute_input":"2022-11-27T14:47:58.719293Z","iopub.status.idle":"2022-11-27T14:48:03.181465Z","shell.execute_reply.started":"2022-11-27T14:47:58.719245Z","shell.execute_reply":"2022-11-27T14:48:03.179893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_train_df['min_date'] = sample_train_df.progress_apply(lambda x: pd.to_datetime(x.events[0]['ts'], unit='ms'), axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:48:03.183217Z","iopub.execute_input":"2022-11-27T14:48:03.183583Z","iopub.status.idle":"2022-11-27T14:48:07.350729Z","shell.execute_reply.started":"2022-11-27T14:48:03.183551Z","shell.execute_reply":"2022-11-27T14:48:07.349495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:48:07.352784Z","iopub.execute_input":"2022-11-27T14:48:07.353533Z","iopub.status.idle":"2022-11-27T14:48:07.462971Z","shell.execute_reply.started":"2022-11-27T14:48:07.353493Z","shell.execute_reply":"2022-11-27T14:48:07.461511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:48:07.465063Z","iopub.execute_input":"2022-11-27T14:48:07.465780Z","iopub.status.idle":"2022-11-27T14:48:07.534349Z","shell.execute_reply.started":"2022-11-27T14:48:07.465739Z","shell.execute_reply":"2022-11-27T14:48:07.529881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_test_df['max_date'] = sample_test_df.progress_apply(lambda x: pd.to_datetime(x.events[-1]['ts'], unit='ms'), axis=1)\nsample_test_df['min_date'] = sample_test_df.progress_apply(lambda x: pd.to_datetime(x.events[0]['ts'], unit='ms'), axis=1)\nsample_test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:48:07.540180Z","iopub.execute_input":"2022-11-27T14:48:07.540797Z","iopub.status.idle":"2022-11-27T14:48:09.896785Z","shell.execute_reply.started":"2022-11-27T14:48:07.540762Z","shell.execute_reply":"2022-11-27T14:48:09.895374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_test_df.max_date.max(), sample_train_df.max_date.max()","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:48:09.898603Z","iopub.execute_input":"2022-11-27T14:48:09.899689Z","iopub.status.idle":"2022-11-27T14:48:09.910481Z","shell.execute_reply.started":"2022-11-27T14:48:09.899645Z","shell.execute_reply":"2022-11-27T14:48:09.908938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_train_df['seq_len'] = sample_train_df.progress_apply(lambda x: len(x.events), axis=1)\nsample_test_df['seq_len'] = sample_test_df.progress_apply(lambda x: len(x.events), axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:48:09.912844Z","iopub.execute_input":"2022-11-27T14:48:09.913251Z","iopub.status.idle":"2022-11-27T14:48:10.860026Z","shell.execute_reply.started":"2022-11-27T14:48:09.913203Z","shell.execute_reply":"2022-11-27T14:48:10.858601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=sample_train_df, x='seq_len')","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:48:10.861690Z","iopub.execute_input":"2022-11-27T14:48:10.862736Z","iopub.status.idle":"2022-11-27T14:48:11.435277Z","shell.execute_reply.started":"2022-11-27T14:48:10.862665Z","shell.execute_reply":"2022-11-27T14:48:11.434330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=sample_test_df, x='seq_len')","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:48:11.436948Z","iopub.execute_input":"2022-11-27T14:48:11.437587Z","iopub.status.idle":"2022-11-27T14:48:12.677359Z","shell.execute_reply.started":"2022-11-27T14:48:11.437548Z","shell.execute_reply":"2022-11-27T14:48:12.676519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_train_df.seq_len.quantile(0.5), sample_test_df.seq_len.quantile(0.5)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:48:12.679528Z","iopub.execute_input":"2022-11-27T14:48:12.679995Z","iopub.status.idle":"2022-11-27T14:48:12.694127Z","shell.execute_reply.started":"2022-11-27T14:48:12.679950Z","shell.execute_reply":"2022-11-27T14:48:12.692474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_train_df.seq_len.quantile(0.95), sample_test_df.seq_len.quantile(0.95)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:48:12.696128Z","iopub.execute_input":"2022-11-27T14:48:12.696595Z","iopub.status.idle":"2022-11-27T14:48:12.712076Z","shell.execute_reply.started":"2022-11-27T14:48:12.696555Z","shell.execute_reply":"2022-11-27T14:48:12.710755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.to_datetime(1659304800025, unit='ms') < pd.to_datetime('20220819')","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:48:12.714344Z","iopub.execute_input":"2022-11-27T14:48:12.714775Z","iopub.status.idle":"2022-11-27T14:48:12.723083Z","shell.execute_reply.started":"2022-11-27T14:48:12.714739Z","shell.execute_reply":"2022-11-27T14:48:12.721949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.to_datetime('20220822'), dt.datetime.strftime(pd.to_datetime(1659304800025, unit='ms'), '%Y-%m-%d')","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:48:12.724605Z","iopub.execute_input":"2022-11-27T14:48:12.724998Z","iopub.status.idle":"2022-11-27T14:48:12.740597Z","shell.execute_reply.started":"2022-11-27T14:48:12.724965Z","shell.execute_reply":"2022-11-27T14:48:12.739546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = []\nvalid = []\nsplit_date = pd.to_datetime('20220825')\nfor i, row in tqdm(sample_train_df.iterrows(), total=len(sample_train_df), desc='Train-valid-split'):\n    split_idx = len(row.events)\n    for i, event in enumerate(row.events):\n        if pd.to_datetime(event['ts'], unit='ms') >= split_date:\n            split_idx = i\n            break\n    train_part = row.events[:split_idx] if row.events[:split_idx] else None\n    valid_part = row.events[split_idx:] if row.events[split_idx:] else None\n    if train_part is not None:\n        train.append({'session': i, 'events': train_part})\n    if valid_part is not None:\n        valid.append({'session': i, 'events': valid_part})","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:48:12.741892Z","iopub.execute_input":"2022-11-27T14:48:12.742338Z","iopub.status.idle":"2022-11-27T14:50:32.289379Z","shell.execute_reply.started":"2022-11-27T14:48:12.742299Z","shell.execute_reply":"2022-11-27T14:50:32.287867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\ntime.time(), pd.to_datetime(time.time(), unit='s'), pd.to_datetime(time.time(), unit='s').timestamp() * 1000","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:32.291294Z","iopub.execute_input":"2022-11-27T14:50:32.291691Z","iopub.status.idle":"2022-11-27T14:50:32.303312Z","shell.execute_reply.started":"2022-11-27T14:50:32.291651Z","shell.execute_reply":"2022-11-27T14:50:32.301938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.DataFrame(train)\nvalid_df = pd.DataFrame(valid)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:32.304495Z","iopub.execute_input":"2022-11-27T14:50:32.304869Z","iopub.status.idle":"2022-11-27T14:50:32.349653Z","shell.execute_reply.started":"2022-11-27T14:50:32.304830Z","shell.execute_reply":"2022-11-27T14:50:32.348405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:32.351493Z","iopub.execute_input":"2022-11-27T14:50:32.351971Z","iopub.status.idle":"2022-11-27T14:50:32.373365Z","shell.execute_reply.started":"2022-11-27T14:50:32.351913Z","shell.execute_reply":"2022-11-27T14:50:32.371852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:32.375678Z","iopub.execute_input":"2022-11-27T14:50:32.376238Z","iopub.status.idle":"2022-11-27T14:50:32.397855Z","shell.execute_reply.started":"2022-11-27T14:50:32.376145Z","shell.execute_reply":"2022-11-27T14:50:32.396783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['seq_len'] = train_df.progress_apply(lambda x: len(x.events), axis=1)\nvalid_df['seq_len'] = valid_df.progress_apply(lambda x: len(x.events), axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:32.400409Z","iopub.execute_input":"2022-11-27T14:50:32.401035Z","iopub.status.idle":"2022-11-27T14:50:32.982226Z","shell.execute_reply.started":"2022-11-27T14:50:32.400984Z","shell.execute_reply":"2022-11-27T14:50:32.980850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=train_df, x='seq_len', kde=True, color='green')","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:32.983742Z","iopub.execute_input":"2022-11-27T14:50:32.984054Z","iopub.status.idle":"2022-11-27T14:50:33.655406Z","shell.execute_reply.started":"2022-11-27T14:50:32.984025Z","shell.execute_reply":"2022-11-27T14:50:33.653804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=sample_test_df, x='seq_len', kde=True, color='red')\nsns.histplot(data=valid_df, x='seq_len', kde=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:33.657064Z","iopub.execute_input":"2022-11-27T14:50:33.657393Z","iopub.status.idle":"2022-11-27T14:50:35.448489Z","shell.execute_reply.started":"2022-11-27T14:50:33.657364Z","shell.execute_reply":"2022-11-27T14:50:35.446982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_df.seq_len.quantile(0.75), sample_test_df.seq_len.quantile(0.75)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:35.450175Z","iopub.execute_input":"2022-11-27T14:50:35.450568Z","iopub.status.idle":"2022-11-27T14:50:35.463065Z","shell.execute_reply.started":"2022-11-27T14:50:35.450534Z","shell.execute_reply":"2022-11-27T14:50:35.461693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['max_date'] = train_df.progress_apply(lambda x: pd.to_datetime(x.events[-1]['ts'], unit='ms'), axis=1)\ntrain_df.max_date.max()","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:35.465458Z","iopub.execute_input":"2022-11-27T14:50:35.466680Z","iopub.status.idle":"2022-11-27T14:50:39.363766Z","shell.execute_reply.started":"2022-11-27T14:50:35.466628Z","shell.execute_reply":"2022-11-27T14:50:39.362395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.loc[train_df.seq_len > 1]\ntrain_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:39.365956Z","iopub.execute_input":"2022-11-27T14:50:39.366362Z","iopub.status.idle":"2022-11-27T14:50:39.392121Z","shell.execute_reply.started":"2022-11-27T14:50:39.366330Z","shell.execute_reply":"2022-11-27T14:50:39.390483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_last_aid_set = defaultdict(list)\nfor _, row in tqdm(train_df.iterrows(), desc='Build last aids', total=len(train_df)):\n    train_last_aid_set[row.events[-2]['aid']].append(row.events[-1]['aid'])","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:39.393987Z","iopub.execute_input":"2022-11-27T14:50:39.394385Z","iopub.status.idle":"2022-11-27T14:50:40.944339Z","shell.execute_reply.started":"2022-11-27T14:50:39.394348Z","shell.execute_reply":"2022-11-27T14:50:40.943151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last_aids = list(train_last_aid_set.keys())","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:40.945924Z","iopub.execute_input":"2022-11-27T14:50:40.946268Z","iopub.status.idle":"2022-11-27T14:50:40.953977Z","shell.execute_reply.started":"2022-11-27T14:50:40.946236Z","shell.execute_reply":"2022-11-27T14:50:40.952480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pos_neg = []\nfor _, row in tqdm(train_df.iterrows(), desc='Build last aids', total=len(train_df)):\n    events = row.events\n    last = events[-2]['aid']\n    next_pos = events[-1]['aid']\n    next_negs = []\n    for _ in range(100):\n        next_neg = random.choice(last_aids)\n        if next_neg != last and next_neg != next_pos:\n            next_negs.append(next_neg)\n        if len(next_negs) > 3:\n            break\n    x = ' '.join(str(event['aid']) for event in events[:-1])\n    train_pos_neg.append({'session': row.session, 'hist': x, 'item': next_pos, 'label': 1})\n    for next_neg in next_negs:\n        train_pos_neg.append({'session': row.session, 'hist': x, 'item': next_neg, 'label': 0})\n\ntrain_pos_neg_df = pd.DataFrame(train_pos_neg)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:40.955649Z","iopub.execute_input":"2022-11-27T14:50:40.956008Z","iopub.status.idle":"2022-11-27T14:50:44.489541Z","shell.execute_reply.started":"2022-11-27T14:50:40.955977Z","shell.execute_reply":"2022-11-27T14:50:44.488306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_df = valid_df.loc[valid_df.seq_len > 1]\nvalid_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:44.491574Z","iopub.execute_input":"2022-11-27T14:50:44.492148Z","iopub.status.idle":"2022-11-27T14:50:44.512406Z","shell.execute_reply.started":"2022-11-27T14:50:44.492057Z","shell.execute_reply":"2022-11-27T14:50:44.510551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_last_aid_set = defaultdict(list)\nfor _, row in tqdm(valid_df.iterrows(), desc='Build last aids', total=len(valid_df)):\n    valid_last_aid_set[row.events[-2]['aid']].append(row.events[-1]['aid'])\nlast_aids = list(valid_last_aid_set.keys())","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:44.515124Z","iopub.execute_input":"2022-11-27T14:50:44.515638Z","iopub.status.idle":"2022-11-27T14:50:45.012913Z","shell.execute_reply.started":"2022-11-27T14:50:44.515576Z","shell.execute_reply":"2022-11-27T14:50:45.011886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_pos_neg = []\nfor _, row in tqdm(valid_df.iterrows(), desc='Build last aids', total=len(valid_df)):\n    events = row.events\n    last = events[-2]['aid']\n    next_pos = events[-1]['aid']\n    next_negs = []\n    for _ in range(100):\n        next_neg = random.choice(last_aids)\n        if next_neg != last and next_neg != next_pos:\n            next_negs.append(next_neg)\n        if len(next_negs) > 3:\n            break\n    x = ' '.join(str(event['aid']) for event in events[:-1])\n    valid_pos_neg.append({'session': row.session, 'hist': x, 'item': next_pos, 'label': 1})\n    for next_neg in next_negs:\n        valid_pos_neg.append({'session': row.session, 'hist': x, 'item': next_neg, 'label': 0})\n\nvalid_pos_neg_df = pd.DataFrame(valid_pos_neg)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:45.014583Z","iopub.execute_input":"2022-11-27T14:50:45.015014Z","iopub.status.idle":"2022-11-27T14:50:45.988150Z","shell.execute_reply.started":"2022-11-27T14:50:45.014976Z","shell.execute_reply":"2022-11-27T14:50:45.986925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_pos_neg_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:45.990107Z","iopub.execute_input":"2022-11-27T14:50:45.990473Z","iopub.status.idle":"2022-11-27T14:50:46.004422Z","shell.execute_reply.started":"2022-11-27T14:50:45.990439Z","shell.execute_reply":"2022-11-27T14:50:46.003391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pos_neg_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:50:46.015441Z","iopub.execute_input":"2022-11-27T14:50:46.016209Z","iopub.status.idle":"2022-11-27T14:50:46.035970Z","shell.execute_reply.started":"2022-11-27T14:50:46.016171Z","shell.execute_reply":"2022-11-27T14:50:46.032997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pos_neg_df['hist'].apply(lambda x: len(x.split())).quantile(0.95), valid_pos_neg_df['hist'].apply(lambda x: len(x.split())).quantile(0.95)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T14:51:12.904325Z","iopub.execute_input":"2022-11-27T14:51:12.905069Z","iopub.status.idle":"2022-11-27T14:51:13.238645Z","shell.execute_reply.started":"2022-11-27T14:51:12.905022Z","shell.execute_reply":"2022-11-27T14:51:13.237221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pos_neg_df.to_csv('./train.csv', index=False, encoding='utf8'), valid_pos_neg_df.to_csv('./valid.csv', index=False, encoding='utf8')","metadata":{"execution":{"iopub.status.busy":"2022-11-19T14:26:41.228606Z","iopub.execute_input":"2022-11-19T14:26:41.229504Z","iopub.status.idle":"2022-11-19T14:26:42.263287Z","shell.execute_reply.started":"2022-11-19T14:26:41.229464Z","shell.execute_reply":"2022-11-19T14:26:42.262152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}