{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.17","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":38760,"databundleVersionId":4493939,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# OTTO Recommender System","metadata":{}},{"cell_type":"code","source":"import torch\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T21:40:40.274939Z","iopub.execute_input":"2025-06-18T21:40:40.275288Z","iopub.status.idle":"2025-06-18T21:40:40.286815Z","shell.execute_reply.started":"2025-06-18T21:40:40.275254Z","shell.execute_reply":"2025-06-18T21:40:40.282390Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install polars","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T21:40:40.287489Z","iopub.execute_input":"2025-06-18T21:40:40.287691Z","iopub.status.idle":"2025-06-18T21:40:43.802262Z","shell.execute_reply.started":"2025-06-18T21:40:40.287671Z","shell.execute_reply":"2025-06-18T21:40:43.796766Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install numba","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport pandas as pd\nimport numpy as np\nimport json\nfrom tqdm import tqdm\nfrom collections import defaultdict, Counter\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import HistGradientBoostingClassifier\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import recall_score\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport json\nfrom tqdm import tqdm\nfrom torch.utils.data import Dataset, DataLoader\nimport polars as pl\nimport gc\nfrom numba import jit\nfrom multiprocessing import Pool\nimport csv\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")\n\ndef read_jsonl_chunk(file, n=100000):\n    with open(file, 'r') as f:\n        for i, line in enumerate(f):\n            if i >= n: \n                break\n            yield json.loads(line)\n\ndef count_lines_fast(filename):\n    with open(filename, 'r') as f:\n        return sum(1 for _ in f)\n\n@jit(nopython=True)\ndef fast_feature_extraction(aid, session_aids_array, session_counts_array):\n    return [int(aid in session_aids_array), session_counts_array[aid]]\n\ndef get_popular_aids_fast(train_file, top_k=169):\n    aid_counts = Counter()\n    \n    print(\"Подсчет популярных aid...\")\n    for row in read_jsonl_chunk(train_file, n=1_000_000):\n        session = row['events']\n        for event in session:\n            aid_counts[event['aid']] += 1\n    \n    popular_aids = [aid for aid, _ in aid_counts.most_common(top_k)]\n    print(f\"Найдено {len(popular_aids)} популярных aid\")\n    print(f\"Размер aid_counts: {len(aid_counts)}\")\n    return popular_aids, aid_counts\n\ndef get_candidates_fast(session, popular_aids, top_k=100):\n    session_aids = set()\n    session_counts = Counter()\n    \n    for event in session:\n        aid = event['aid']\n        session_aids.add(aid)\n        session_counts[aid] += 1\n    \n    candidates = list(session_aids) + [aid for aid in popular_aids if aid not in session_aids]\n    return candidates[:top_k], session_aids, session_counts\n\ndef extract_features_fast(aid, session_aids, session_counts, global_scores=None):\n    if global_scores is None:\n        global_scores = {}\n    features = [\n        int(aid in session_aids),\n        session_counts.get(aid, 0),\n        global_scores.get(aid, 0),\n    ]\n    return features\n\ndef train_model_fast(X, y):\n    model = HistGradientBoostingClassifier(\n        max_iter=16,\n        learning_rate=0.1,\n        max_depth=6,\n        random_state=42,\n        verbose=0\n    )\n    model.fit(X, y)\n    print(f\"Размер X: {X.shape}\")\n    print(f\"Размер y: {y.shape}\")\n    return model\n\ndef prepare_training_data(train_file, popular_aids, max_samples=1_000_000):\n    X_train, y_train = [], []\n    sample_count = 0\n    \n    print(\"Подготовка данных для обучения...\")\n    for row in read_jsonl_chunk(train_file, n=max_samples):\n        if sample_count >= max_samples:\n            break\n            \n        session = row['events']\n        candidates, session_aids, session_counts = get_candidates_fast(\n            session, popular_aids, top_k=69\n        )\n        \n        for event_type in ['clicks', 'carts', 'orders']:\n            gt_aid = next((e['aid'] for e in session if e['type'] == event_type), None)\n            if gt_aid and gt_aid in candidates:\n                features = extract_features_fast(gt_aid, session_aids, session_counts)\n                X_train.append(features)\n                y_train.append(1)\n                \n                neg_candidates = [aid for aid in candidates if aid != gt_aid][:3]\n                for neg_aid in neg_candidates:\n                    features = extract_features_fast(neg_aid, session_aids, session_counts)\n                    X_train.append(features)\n                    y_train.append(0)\n                    sample_count += 1\n                    \n        sample_count += 1\n        \n        if sample_count >= max_samples:\n            break\n    \n    print(f\"Подготовлено {len(X_train)} образцов\")\n    print(f\"Размер X_train: {len(X_train)}\")\n    print(f\"Размер y_train: {len(y_train)}\")\n    return np.array(X_train, dtype=np.float32), np.array(y_train, dtype=np.int32)\n\ndef train_pipeline():\n    popular_aids, aid_counts = get_popular_aids_fast('/kaggle/input/otto-recommender-system/train.jsonl', top_k=1000)\n    \n    X_train, y_train = prepare_training_data('/kaggle/input/otto-recommender-system/train.jsonl', popular_aids, max_samples=69_000)\n    \n    print(\"Обучение модели...\")\n    model = train_model_fast(X_train, y_train)\n    \n    del X_train, y_train\n    gc.collect()\n    \n    return model, popular_aids, aid_counts\n\ndef inference_and_save(model, popular_aids, test_file='/kaggle/input/otto-recommender-system/test.jsonl', submission_file='submission.csv'):\n    test_len = count_lines_fast(test_file)\n    total_rows_written = 0\n    \n    print(f\"Инференс на {test_len} сессиях...\")\n    \n    with open(submission_file, 'w', newline='') as f:\n        writer = csv.DictWriter(f, fieldnames=['session_type', 'labels'])\n        writer.writeheader()\n        \n        submission_batch = []\n        batch_size = 100_000\n        \n        for row in tqdm(read_jsonl_chunk(test_file, n=test_len), total=test_len, desc=\"Processing test\"):\n            session = row['events']\n            session_id = row['session']\n            \n            candidates, session_aids, session_counts = get_candidates_fast(\n                session, popular_aids, top_k=100\n            )\n            \n            for event_type in ['clicks', 'carts', 'orders']:\n                features_list = [\n                    extract_features_fast(aid, session_aids, session_counts)\n                    for aid in candidates\n                ]\n                \n                if features_list:\n                    scores = model.predict_proba(features_list)[:, 1]\n                    top_20 = [candidates[i] for i in np.argsort(scores)[-20:][::-1]]\n                else:\n                    top_20 = popular_aids[:20]\n                \n                submission_batch.append({\n                    'session_type': f\"{session_id}_{event_type}\",\n                    'labels': ' '.join(map(str, top_20))\n                })\n                \n                if len(submission_batch) >= batch_size:\n                    writer.writerows(submission_batch)\n                    total_rows_written += len(submission_batch)\n                    submission_batch = []\n        \n        if submission_batch:\n            writer.writerows(submission_batch)\n            total_rows_written += len(submission_batch)\n    \n    print(f\"Submission saved: {total_rows_written} rows\")\n    \n    expected_rows = 5015409\n    if total_rows_written != expected_rows:\n        print(f\"WARNING: Expected {expected_rows} rows, got {total_rows_written}\")\n    else:\n        print(\"✓ Correct number of rows in submission\")\n    \n    return total_rows_written\n\nprint(\"Starting optimized pipeline...\")\n\nmodel, popular_aids, aid_counts = train_pipeline()\n\ntotal_rows = inference_and_save(model, popular_aids)\n\nprint(\"Pipeline completed successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T21:40:43.804515Z","iopub.execute_input":"2025-06-18T21:40:43.804774Z","iopub.status.idle":"2025-06-18T21:40:43.815784Z","shell.execute_reply.started":"2025-06-18T21:40:43.804746Z","shell.execute_reply":"2025-06-18T21:40:43.812045Z"}},"outputs":[],"execution_count":null}]}