{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Create your own validation dataset in jsonl format by running organizer script on Kaggle\n\nIn this notebook, I made it possible to run otto organizer's [scripts](https://github.com/otto-de/recsys-dataset/tree/main/src) on Kaggle so that you can create your own validation set in jsonl file.\n\nIf you want to convert your validation jsonl dataset into parquet in Radek's style optimized dataset, you can run another [notebook](https://www.kaggle.com/code/danielliao/otto-validation-optimized-jsonl2parquet/) with your jsonl dataset.","metadata":{}},{"cell_type":"code","source":"!pip install beartype","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:48:53.826263Z","iopub.execute_input":"2022-12-27T12:48:53.826871Z","iopub.status.idle":"2022-12-27T12:49:07.322573Z","shell.execute_reply.started":"2022-12-27T12:48:53.826767Z","shell.execute_reply":"2022-12-27T12:49:07.321525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import argparse\nimport json\nimport random\nfrom copy import deepcopy\nfrom pathlib import Path\n\nimport pandas as pd\nfrom beartype import beartype\nfrom pandas.io.json._json import JsonReader\nfrom tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:49:11.372673Z","iopub.execute_input":"2022-12-27T12:49:11.373113Z","iopub.status.idle":"2022-12-27T12:49:11.528241Z","shell.execute_reply.started":"2022-12-27T12:49:11.373075Z","shell.execute_reply":"2022-12-27T12:49:11.527077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"```python\nfrom src.labels import ground_truth\n```\n\nThis line can be replaced by the cell below","metadata":{}},{"cell_type":"code","source":"from __future__ import annotations","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:49:14.905300Z","iopub.execute_input":"2022-12-27T12:49:14.905724Z","iopub.status.idle":"2022-12-27T12:49:14.910961Z","shell.execute_reply.started":"2022-12-27T12:49:14.905687Z","shell.execute_reply":"2022-12-27T12:49:14.909911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\n# def ground_truth(events: list[dict]):\ndef ground_truth(events):    \n    prev_labels = {\"clicks\": None, \"carts\": set(), \"orders\": set()}\n\n    for event in reversed(events):\n        event[\"labels\"] = {}\n\n        for label in ['clicks', 'carts', 'orders']:\n            if prev_labels[label]:\n                if label != 'clicks':\n                    event[\"labels\"][label] = prev_labels[label].copy()\n                else:\n                    event[\"labels\"][label] = prev_labels[label]\n\n        if event[\"type\"] == \"clicks\":\n            prev_labels['clicks'] = event[\"aid\"]\n        if event[\"type\"] == \"carts\":\n            prev_labels['carts'].add(event[\"aid\"])\n        elif event[\"type\"] == \"orders\":\n            prev_labels['orders'].add(event[\"aid\"])\n\n    return events[:-1]","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:49:51.348857Z","iopub.execute_input":"2022-12-27T12:49:51.349287Z","iopub.status.idle":"2022-12-27T12:49:51.358386Z","shell.execute_reply.started":"2022-12-27T12:49:51.349251Z","shell.execute_reply":"2022-12-27T12:49:51.357251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class setEncoder(json.JSONEncoder):\n\n    def default(self, obj):\n        return list(obj)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:50:04.266460Z","iopub.execute_input":"2022-12-27T12:50:04.266857Z","iopub.status.idle":"2022-12-27T12:50:04.272632Z","shell.execute_reply.started":"2022-12-27T12:50:04.266824Z","shell.execute_reply":"2022-12-27T12:50:04.271139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\n# def split_events(events: list[dict], split_idx=None):\ndef split_events(events, split_idx=None):\n    test_events = ground_truth(deepcopy(events))\n    if not split_idx:\n        split_idx = random.randint(1, len(test_events))\n    test_events = test_events[:split_idx]\n    labels = test_events[-1]['labels']\n    for event in test_events:\n        del event['labels']\n    return test_events, labels","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:50:27.254869Z","iopub.execute_input":"2022-12-27T12:50:27.255316Z","iopub.status.idle":"2022-12-27T12:50:27.261784Z","shell.execute_reply.started":"2022-12-27T12:50:27.255259Z","shell.execute_reply":"2022-12-27T12:50:27.260976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\ndef create_kaggle_testset(sessions: pd.DataFrame, sessions_output: Path, labels_output: Path):\n    last_labels = []\n    splitted_sessions = []\n\n    for _, session in tqdm(sessions.iterrows(), desc=\"Creating trimmed testset\", total=len(sessions)):\n        session = session.to_dict()\n        splitted_events, labels = split_events(session['events'])\n        last_labels.append({'session': session['session'], 'labels': labels})\n        splitted_sessions.append({'session': session['session'], 'events': splitted_events})\n\n    with open(sessions_output, 'w') as f:\n        for session in splitted_sessions:\n            f.write(json.dumps(session) + '\\n')\n\n    with open(labels_output, 'w') as f:\n        for label in last_labels:\n            f.write(json.dumps(label, cls=setEncoder) + '\\n')","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:50:40.944747Z","iopub.execute_input":"2022-12-27T12:50:40.945311Z","iopub.status.idle":"2022-12-27T12:50:40.955932Z","shell.execute_reply.started":"2022-12-27T12:50:40.945262Z","shell.execute_reply":"2022-12-27T12:50:40.954616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\ndef trim_session(session: dict, max_ts: int) -> dict:\n    session['events'] = [event for event in session['events'] if event['ts'] < max_ts]\n    return session","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:50:49.538158Z","iopub.execute_input":"2022-12-27T12:50:49.539266Z","iopub.status.idle":"2022-12-27T12:50:49.545257Z","shell.execute_reply.started":"2022-12-27T12:50:49.539220Z","shell.execute_reply":"2022-12-27T12:50:49.544394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\ndef get_max_ts(sessions_path: Path) -> int:\n    max_ts = float('-inf')\n    with open(sessions_path) as f:\n        for line in tqdm(f, desc=\"Finding max timestamp\"):\n            session = json.loads(line)\n            max_ts = max(max_ts, session['events'][-1]['ts'])\n    return max_ts","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:50:54.103630Z","iopub.execute_input":"2022-12-27T12:50:54.104072Z","iopub.status.idle":"2022-12-27T12:50:54.111326Z","shell.execute_reply.started":"2022-12-27T12:50:54.104032Z","shell.execute_reply":"2022-12-27T12:50:54.110219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\n# def filter_unknown_items(session_path: Path, known_items: set[int]):\ndef filter_unknown_items(session_path: Path, known_items):    \n    filtered_sessions = []\n    with open(session_path) as f:\n        for line in tqdm(f, desc=\"Filtering unknown items\"):\n            session = json.loads(line)\n            session['events'] = [event for event in session['events'] if event['aid'] in known_items]\n            if len(session['events']) >= 2:\n                filtered_sessions.append(session)\n    with open(session_path, 'w') as f:\n        for session in filtered_sessions:\n            f.write(json.dumps(session) + '\\n')","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:51:13.730719Z","iopub.execute_input":"2022-12-27T12:51:13.731301Z","iopub.status.idle":"2022-12-27T12:51:13.740193Z","shell.execute_reply.started":"2022-12-27T12:51:13.731260Z","shell.execute_reply":"2022-12-27T12:51:13.738958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\ndef train_test_split(session_chunks: JsonReader, train_path: Path, test_path: Path, max_ts: int, test_days: int):\n    split_millis = test_days * 24 * 60 * 60 * 1000\n    split_ts = max_ts - split_millis\n    train_items = set()\n    Path(train_path).parent.mkdir(parents=True, exist_ok=True)\n    train_file = open(train_path, \"w\")\n    Path(test_path).parent.mkdir(parents=True, exist_ok=True)\n    test_file = open(test_path, \"w\")\n    for chunk in tqdm(session_chunks, desc=\"Splitting sessions\"):\n        for _, session in chunk.iterrows():\n            session = session.to_dict()\n            if session['events'][0]['ts'] > split_ts:\n                test_file.write(json.dumps(session) + \"\\n\")\n            else:\n                session = trim_session(session, split_ts)\n                if len(session['events']) >= 2:\n                    train_items.update([event['aid'] for event in session['events']])\n                    train_file.write(json.dumps(session) + \"\\n\")\n    train_file.close()\n    test_file.close()\n    filter_unknown_items(test_path, train_items)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:51:24.347872Z","iopub.execute_input":"2022-12-27T12:51:24.348923Z","iopub.status.idle":"2022-12-27T12:51:24.359642Z","shell.execute_reply.started":"2022-12-27T12:51:24.348884Z","shell.execute_reply":"2022-12-27T12:51:24.358401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\ndef main(train_set: Path, output_path: Path, days: int, seed: int):\n    random.seed(seed)\n    max_ts = get_max_ts(train_set)\n\n    session_chunks = pd.read_json(train_set, lines=True, chunksize=100000)\n    train_file = output_path / 'train_sessions.jsonl'\n    test_file_full = output_path / 'test_sessions_full.jsonl'\n    train_test_split(session_chunks, train_file, test_file_full, max_ts, days)\n\n    test_sessions = pd.read_json(test_file_full, lines=True)\n    test_sessions_file = output_path / 'test_sessions.jsonl'\n    test_labels_file = output_path / 'test_labels.jsonl'\n    create_kaggle_testset(test_sessions, test_sessions_file, test_labels_file)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:51:35.336648Z","iopub.execute_input":"2022-12-27T12:51:35.337190Z","iopub.status.idle":"2022-12-27T12:51:35.345039Z","shell.execute_reply.started":"2022-12-27T12:51:35.337143Z","shell.execute_reply":"2022-12-27T12:51:35.344164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pipenv run python -m src.testset --train-set train.jsonl --days 2 --output-path 'out/' --seed 42 \npath = Path('/kaggle/input/otto-recommender-system/train.jsonl')\nmain(train_set = path, output_path=Path('.'), days=7, seed=42)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T13:06:22.316064Z","iopub.execute_input":"2022-12-27T13:06:22.316480Z","iopub.status.idle":"2022-12-27T13:51:26.263436Z","shell.execute_reply.started":"2022-12-27T13:06:22.316447Z","shell.execute_reply":"2022-12-27T13:51:26.262275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"At first, I could never imagine it is possible to running organizer's script on Kaggle, because it is possible then amazing Kagglers like @radek1 and @cdeotte would have done so long time ago, but it turns out it can be done on Kaggle! How amazing!\n\nThe 30GB RAM was nearly blown up, but luckily kaggle notebook managed to finish the running.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}