{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook, I will try to convert otto organizer's [scripts](https://github.com/otto-de/recsys-dataset/tree/main/src) in a way that they can be run on kaggle notebooks","metadata":{}},{"cell_type":"code","source":"!pip install beartype","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:48:53.826263Z","iopub.execute_input":"2022-12-27T12:48:53.826871Z","iopub.status.idle":"2022-12-27T12:49:07.322573Z","shell.execute_reply.started":"2022-12-27T12:48:53.826767Z","shell.execute_reply":"2022-12-27T12:49:07.321525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import argparse\nimport json\nimport random\nfrom copy import deepcopy\nfrom pathlib import Path\n\nimport pandas as pd\nfrom beartype import beartype\nfrom pandas.io.json._json import JsonReader\nfrom tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:49:11.372673Z","iopub.execute_input":"2022-12-27T12:49:11.373113Z","iopub.status.idle":"2022-12-27T12:49:11.528241Z","shell.execute_reply.started":"2022-12-27T12:49:11.373075Z","shell.execute_reply":"2022-12-27T12:49:11.527077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"```python\nfrom src.labels import ground_truth\n```\n\nThis line can be replaced by the cell below","metadata":{}},{"cell_type":"code","source":"from __future__ import annotations","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:49:14.9053Z","iopub.execute_input":"2022-12-27T12:49:14.905724Z","iopub.status.idle":"2022-12-27T12:49:14.910961Z","shell.execute_reply.started":"2022-12-27T12:49:14.905687Z","shell.execute_reply":"2022-12-27T12:49:14.909911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\n# def ground_truth(events: list[dict]):\ndef ground_truth(events):    \n    prev_labels = {\"clicks\": None, \"carts\": set(), \"orders\": set()}\n\n    for event in reversed(events):\n        event[\"labels\"] = {}\n\n        for label in ['clicks', 'carts', 'orders']:\n            if prev_labels[label]:\n                if label != 'clicks':\n                    event[\"labels\"][label] = prev_labels[label].copy()\n                else:\n                    event[\"labels\"][label] = prev_labels[label]\n\n        if event[\"type\"] == \"clicks\":\n            prev_labels['clicks'] = event[\"aid\"]\n        if event[\"type\"] == \"carts\":\n            prev_labels['carts'].add(event[\"aid\"])\n        elif event[\"type\"] == \"orders\":\n            prev_labels['orders'].add(event[\"aid\"])\n\n    return events[:-1]","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:49:51.348857Z","iopub.execute_input":"2022-12-27T12:49:51.349287Z","iopub.status.idle":"2022-12-27T12:49:51.358386Z","shell.execute_reply.started":"2022-12-27T12:49:51.349251Z","shell.execute_reply":"2022-12-27T12:49:51.357251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class setEncoder(json.JSONEncoder):\n\n    def default(self, obj):\n        return list(obj)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:50:04.26646Z","iopub.execute_input":"2022-12-27T12:50:04.266857Z","iopub.status.idle":"2022-12-27T12:50:04.272632Z","shell.execute_reply.started":"2022-12-27T12:50:04.266824Z","shell.execute_reply":"2022-12-27T12:50:04.271139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\n# def split_events(events: list[dict], split_idx=None):\ndef split_events(events, split_idx=None):\n    test_events = ground_truth(deepcopy(events))\n    if not split_idx:\n        split_idx = random.randint(1, len(test_events))\n    test_events = test_events[:split_idx]\n    labels = test_events[-1]['labels']\n    for event in test_events:\n        del event['labels']\n    return test_events, labels","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:50:27.254869Z","iopub.execute_input":"2022-12-27T12:50:27.255316Z","iopub.status.idle":"2022-12-27T12:50:27.261784Z","shell.execute_reply.started":"2022-12-27T12:50:27.255259Z","shell.execute_reply":"2022-12-27T12:50:27.260976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\ndef create_kaggle_testset(sessions: pd.DataFrame, sessions_output: Path, labels_output: Path):\n    last_labels = []\n    splitted_sessions = []\n\n    for _, session in tqdm(sessions.iterrows(), desc=\"Creating trimmed testset\", total=len(sessions)):\n        session = session.to_dict()\n        splitted_events, labels = split_events(session['events'])\n        last_labels.append({'session': session['session'], 'labels': labels})\n        splitted_sessions.append({'session': session['session'], 'events': splitted_events})\n\n    with open(sessions_output, 'w') as f:\n        for session in splitted_sessions:\n            f.write(json.dumps(session) + '\\n')\n\n    with open(labels_output, 'w') as f:\n        for label in last_labels:\n            f.write(json.dumps(label, cls=setEncoder) + '\\n')","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:50:40.944747Z","iopub.execute_input":"2022-12-27T12:50:40.945311Z","iopub.status.idle":"2022-12-27T12:50:40.955932Z","shell.execute_reply.started":"2022-12-27T12:50:40.945262Z","shell.execute_reply":"2022-12-27T12:50:40.954616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\ndef trim_session(session: dict, max_ts: int) -> dict:\n    session['events'] = [event for event in session['events'] if event['ts'] < max_ts]\n    return session","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:50:49.538158Z","iopub.execute_input":"2022-12-27T12:50:49.539266Z","iopub.status.idle":"2022-12-27T12:50:49.545257Z","shell.execute_reply.started":"2022-12-27T12:50:49.53922Z","shell.execute_reply":"2022-12-27T12:50:49.544394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\ndef get_max_ts(sessions_path: Path) -> int:\n    max_ts = float('-inf')\n    with open(sessions_path) as f:\n        for line in tqdm(f, desc=\"Finding max timestamp\"):\n            session = json.loads(line)\n            max_ts = max(max_ts, session['events'][-1]['ts'])\n    return max_ts","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:50:54.10363Z","iopub.execute_input":"2022-12-27T12:50:54.104072Z","iopub.status.idle":"2022-12-27T12:50:54.111326Z","shell.execute_reply.started":"2022-12-27T12:50:54.104032Z","shell.execute_reply":"2022-12-27T12:50:54.110219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\n# def filter_unknown_items(session_path: Path, known_items: set[int]):\ndef filter_unknown_items(session_path: Path, known_items):    \n    filtered_sessions = []\n    with open(session_path) as f:\n        for line in tqdm(f, desc=\"Filtering unknown items\"):\n            session = json.loads(line)\n            session['events'] = [event for event in session['events'] if event['aid'] in known_items]\n            if len(session['events']) >= 2:\n                filtered_sessions.append(session)\n    with open(session_path, 'w') as f:\n        for session in filtered_sessions:\n            f.write(json.dumps(session) + '\\n')","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:51:13.730719Z","iopub.execute_input":"2022-12-27T12:51:13.731301Z","iopub.status.idle":"2022-12-27T12:51:13.740193Z","shell.execute_reply.started":"2022-12-27T12:51:13.73126Z","shell.execute_reply":"2022-12-27T12:51:13.738958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\ndef train_test_split(session_chunks: JsonReader, train_path: Path, test_path: Path, max_ts: int, test_days: int):\n    split_millis = test_days * 24 * 60 * 60 * 1000\n    split_ts = max_ts - split_millis\n    train_items = set()\n    Path(train_path).parent.mkdir(parents=True, exist_ok=True)\n    train_file = open(train_path, \"w\")\n    Path(test_path).parent.mkdir(parents=True, exist_ok=True)\n    test_file = open(test_path, \"w\")\n    for chunk in tqdm(session_chunks, desc=\"Splitting sessions\"):\n        for _, session in chunk.iterrows():\n            session = session.to_dict()\n            if session['events'][0]['ts'] > split_ts:\n                test_file.write(json.dumps(session) + \"\\n\")\n            else:\n                session = trim_session(session, split_ts)\n                if len(session['events']) >= 2:\n                    train_items.update([event['aid'] for event in session['events']])\n                    train_file.write(json.dumps(session) + \"\\n\")\n    train_file.close()\n    test_file.close()\n    filter_unknown_items(test_path, train_items)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:51:24.347872Z","iopub.execute_input":"2022-12-27T12:51:24.348923Z","iopub.status.idle":"2022-12-27T12:51:24.359642Z","shell.execute_reply.started":"2022-12-27T12:51:24.348884Z","shell.execute_reply":"2022-12-27T12:51:24.358401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@beartype\ndef main(train_set: Path, output_path: Path, days: int, seed: int):\n    random.seed(seed)\n    max_ts = get_max_ts(train_set)\n\n    session_chunks = pd.read_json(train_set, lines=True, chunksize=100000)\n    train_file = output_path / 'train_sessions.jsonl'\n    test_file_full = output_path / 'test_sessions_full.jsonl'\n    train_test_split(session_chunks, train_file, test_file_full, max_ts, days)\n\n    test_sessions = pd.read_json(test_file_full, lines=True)\n    test_sessions_file = output_path / 'test_sessions.jsonl'\n    test_labels_file = output_path / 'test_labels.jsonl'\n    create_kaggle_testset(test_sessions, test_sessions_file, test_labels_file)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T12:51:35.336648Z","iopub.execute_input":"2022-12-27T12:51:35.33719Z","iopub.status.idle":"2022-12-27T12:51:35.345039Z","shell.execute_reply.started":"2022-12-27T12:51:35.337143Z","shell.execute_reply":"2022-12-27T12:51:35.344164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pipenv run python -m src.testset --train-set train.jsonl --days 2 --output-path 'out/' --seed 42 \npath = Path('/kaggle/input/otto-recommender-system/train.jsonl')\nmain(train_set = path, output_path=Path('.'), days=7, seed=42)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T13:06:22.316064Z","iopub.execute_input":"2022-12-27T13:06:22.31648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}