{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Input dataset","metadata":{}},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-02T16:49:00.811688Z","iopub.execute_input":"2022-12-02T16:49:00.812222Z","iopub.status.idle":"2022-12-02T16:49:00.823737Z","shell.execute_reply.started":"2022-12-02T16:49:00.812183Z","shell.execute_reply":"2022-12-02T16:49:00.822204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Script\nRunning train/test split script from official repo https://github.com/otto-de/recsys-dataset","metadata":{}},{"cell_type":"code","source":"import argparse\nimport json\nimport random\nfrom copy import deepcopy\nfrom pathlib import Path\n\nimport pandas as pd\nfrom pandas.io.json._json import JsonReader\nfrom tqdm.auto import tqdm\n\nfrom typing import List, Set\n\n\n\ndef ground_truth(events: List[dict]):\n    prev_labels = {\"clicks\": None, \"carts\": set(), \"orders\": set()}\n\n    for event in reversed(events):\n        event[\"labels\"] = {}\n\n        for label in ['clicks', 'carts', 'orders']:\n            if prev_labels[label]:\n                if label != 'clicks':\n                    event[\"labels\"][label] = prev_labels[label].copy()\n                else:\n                    event[\"labels\"][label] = prev_labels[label]\n\n        if event[\"type\"] == \"clicks\":\n            prev_labels['clicks'] = event[\"aid\"]\n        if event[\"type\"] == \"carts\":\n            prev_labels['carts'].add(event[\"aid\"])\n        elif event[\"type\"] == \"orders\":\n            prev_labels['orders'].add(event[\"aid\"])\n\n    return events[:-1]\n\n\nclass setEncoder(json.JSONEncoder):\n\n    def default(self, obj):\n        return list(obj)\n\n\ndef split_events(events: List[dict], split_idx=None):\n    test_events = ground_truth(deepcopy(events))\n    if not split_idx:\n        split_idx = random.randint(1, len(test_events))\n    test_events = test_events[:split_idx]\n    labels = test_events[-1]['labels']\n    for event in test_events:\n        del event['labels']\n    return test_events, labels\n\n\ndef create_kaggle_testset(sessions: pd.DataFrame, sessions_output: Path, labels_output: Path):\n    last_labels = []\n    splitted_sessions = []\n\n    for _, session in tqdm(sessions.iterrows(), desc=\"Creating trimmed testset\", total=len(sessions)):\n        session = session.to_dict()\n        splitted_events, labels = split_events(session['events'])\n        last_labels.append({'session': session['session'], 'labels': labels})\n        splitted_sessions.append({'session': session['session'], 'events': splitted_events})\n\n    with open(sessions_output, 'w') as f:\n        for session in splitted_sessions:\n            f.write(json.dumps(session) + '\\n')\n\n    with open(labels_output, 'w') as f:\n        for label in last_labels:\n            f.write(json.dumps(label, cls=setEncoder) + '\\n')\n\n\ndef trim_session(session: dict, max_ts: int) -> dict:\n    session['events'] = [event for event in session['events'] if event['ts'] < max_ts]\n    return session\n\n\ndef get_max_ts(sessions_path: Path) -> int:\n    max_ts = float('-inf')\n    with open(sessions_path) as f:\n        for line in tqdm(f, desc=\"Finding max timestamp\"):\n            session = json.loads(line)\n            max_ts = max(max_ts, session['events'][-1]['ts'])\n    return max_ts\n\n\ndef filter_unknown_items(session_path: Path, known_items: Set[int]):\n    filtered_sessions = []\n    with open(session_path) as f:\n        for line in tqdm(f, desc=\"Filtering unknown items\"):\n            session = json.loads(line)\n            session['events'] = [event for event in session['events'] if event['aid'] in known_items]\n            if len(session['events']) >= 2:\n                filtered_sessions.append(session)\n    with open(session_path, 'w') as f:\n        for session in filtered_sessions:\n            f.write(json.dumps(session) + '\\n')\n\n\ndef train_test_split(session_chunks: JsonReader, train_path: Path, test_path: Path, max_ts: int, test_days: int):\n    split_millis = int(test_days * 24 * 60 * 60 * 1000)\n    split_ts = max_ts - split_millis\n    train_items = set()\n    Path(train_path).parent.mkdir(parents=True, exist_ok=True)\n    train_file = open(train_path, \"w\")\n    Path(test_path).parent.mkdir(parents=True, exist_ok=True)\n    test_file = open(test_path, \"w\")\n    for chunk in tqdm(session_chunks, desc=\"Splitting sessions\"):\n        for _, session in chunk.iterrows():\n            session = session.to_dict()\n            if session['events'][0]['ts'] > split_ts:\n                test_file.write(json.dumps(session) + \"\\n\")\n            else:\n                session = trim_session(session, split_ts)\n                if len(session['events']) >= 2:\n                    train_items.update([event['aid'] for event in session['events']])\n                    train_file.write(json.dumps(session) + \"\\n\")\n    train_file.close()\n    test_file.close()\n    filter_unknown_items(test_path, train_items)\n\n\ndef main(train_set: Path, output_path: Path, days: int, seed: int):\n    random.seed(seed)\n    max_ts = get_max_ts(train_set)\n\n    session_chunks = pd.read_json(train_set, lines=True, chunksize=100000)\n    train_file = output_path / 'train_sessions.jsonl'\n    test_file_full = output_path / 'test_sessions_full.jsonl'\n    train_test_split(session_chunks, train_file, test_file_full, max_ts, days)\n\n    test_sessions = pd.read_json(test_file_full, lines=True)\n    test_sessions_file = output_path / 'test_sessions.jsonl'\n    test_labels_file = output_path / 'test_labels.jsonl'\n    create_kaggle_testset(test_sessions, test_sessions_file, test_labels_file)","metadata":{"execution":{"iopub.status.busy":"2022-12-02T16:49:02.939149Z","iopub.execute_input":"2022-12-02T16:49:02.939539Z","iopub.status.idle":"2022-12-02T16:49:02.972322Z","shell.execute_reply.started":"2022-12-02T16:49:02.939510Z","shell.execute_reply":"2022-12-02T16:49:02.971143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Choosing 6 days as closest to 28 days * 0.2 to respect split size","metadata":{}},{"cell_type":"code","source":"main(Path('/kaggle/input/otto-recommender-system/train.jsonl'),\n     Path('/kaggle/working/otto-validation'),\n     days=6,\n     seed=42)","metadata":{"execution":{"iopub.status.busy":"2022-12-02T17:05:14.363590Z","iopub.execute_input":"2022-12-02T17:05:14.365281Z","iopub.status.idle":"2022-12-02T17:53:49.432375Z","shell.execute_reply.started":"2022-12-02T17:05:14.365226Z","shell.execute_reply":"2022-12-02T17:53:49.430580Z"},"trusted":true},"execution_count":null,"outputs":[]}]}