{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":4461402,"sourceType":"datasetVersion","datasetId":2611514},{"sourceId":4474043,"sourceType":"datasetVersion","datasetId":2601572},{"sourceId":271510179,"sourceType":"kernelVersion"},{"sourceId":271735638,"sourceType":"kernelVersion"},{"sourceId":271807222,"sourceType":"kernelVersion"}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom collections import defaultdict\nfrom tqdm import tqdm\nimport gc\nimport polars as pl\nfrom typing import List\nimport os\nimport lightgbm as lgb\nfrom catboost import CatBoostRanker, Pool","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T07:26:09.405460Z","iopub.execute_input":"2025-10-30T07:26:09.405729Z","iopub.status.idle":"2025-10-30T07:26:15.150131Z","shell.execute_reply.started":"2025-10-30T07:26:09.405708Z","shell.execute_reply":"2025-10-30T07:26:15.149542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def split_preserving_items(df: pl.DataFrame, n_parts: int = 10) -> List[pl.DataFrame]:\n    \"\"\"\n    将数据分成多个部分，确保同一商品的所有行在同一分块中\n    \"\"\"\n    # 获取所有唯一的商品ID\n    all_aids = df.select('aid').unique().to_series().to_list()\n    \n    # 计算每个分块应该包含的商品数量\n    chunk_size = len(all_aids) // n_parts\n    df_parts = []\n    \n    for i in range(n_parts):\n        if i == n_parts - 1:\n            # 最后一个分块包含剩余的所有商品\n            aid_chunk = all_aids[i * chunk_size:]\n        else:\n            aid_chunk = all_aids[i * chunk_size:(i + 1) * chunk_size]\n        \n        # 过滤出包含这些商品的所有行\n        chunk_df = df.filter(pl.col('aid').is_in(aid_chunk))\n        df_parts.append(chunk_df)\n    \n    return df_parts\n\ndef process_chunk(chunk: pl.DataFrame, chunk_id: int, output_dir: str = \"./\"):\n    \"\"\"\n    处理单个数据分块，生成商品特征\n    \"\"\"\n    item_features_chunk = chunk.lazy().group_by('aid').agg([\n        pl.count('aid').alias('item_item_count'),\n        pl.col('session').n_unique().alias('item_user_count'),\n        pl.col('type').mean().alias('item_buy_ratio')\n    ]).collect()\n    \n    # 转换数据类型\n    item_features_chunk = item_features_chunk.with_columns([\n        pl.col('item_item_count').cast(pl.Int32),\n        pl.col('item_user_count').cast(pl.Int32),\n        pl.col('item_buy_ratio').cast(pl.Float32)\n    ])\n    \n    # 保存分块结果\n    output_path = os.path.join(output_dir, f'item_features_part_{chunk_id}.parquet')\n    item_features_chunk.write_parquet(output_path)\n    \n    return item_features_chunk\n\ndef merge_all_parts(n_parts: int, input_dir: str = \"./\") -> pl.DataFrame:\n    \"\"\"\n    合并所有分块的特征文件\n    \"\"\"\n    all_parts = []\n    \n    for i in range(n_parts):\n        file_path = os.path.join(input_dir, f'item_features_part_{i}.parquet')\n        if os.path.exists(file_path):\n            part_df = pl.read_parquet(file_path)\n            all_parts.append(part_df)\n    \n    # 合并所有分块\n    if all_parts:\n        final_item_features = pl.concat(all_parts)\n        \n        # 最终的数据类型转换（确保一致性）\n        final_item_features = final_item_features.with_columns([\n            pl.col('item_item_count').cast(pl.Int32),\n            pl.col('item_user_count').cast(pl.Int32),\n            pl.col('item_buy_ratio').cast(pl.Float32)\n        ])\n        \n        return final_item_features\n    else:\n        return pl.DataFrame()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T07:26:15.151127Z","iopub.execute_input":"2025-10-30T07:26:15.151679Z","iopub.status.idle":"2025-10-30T07:26:15.160434Z","shell.execute_reply.started":"2025-10-30T07:26:15.151659Z","shell.execute_reply":"2025-10-30T07:26:15.159649Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def generate_user_features_chunked(test_data: str, output_dir: str = \"./user_features_chunks\", n_parts: int = 5):\n    \"\"\"\n    分块生成用户特征，每个分块保存为单独文件\n    \"\"\"\n    # 读取数据\n    combined_data = test_data\n    \n    # 获取所有唯一的用户ID并分成5块\n    all_sessions = combined_data.select('session').unique().to_series().to_list()\n    chunk_size = len(all_sessions) // n_parts\n    \n    # 创建输出目录\n    os.makedirs(output_dir, exist_ok=True)\n    \n    # 处理每个分块\n    for i in range(n_parts):\n        print(f\"Processing user chunk {i+1}/{n_parts}...\")\n        \n        # 确定当前分块的用户ID范围\n        if i == n_parts - 1:\n            session_chunk = all_sessions[i * chunk_size:]\n        else:\n            session_chunk = all_sessions[i * chunk_size:(i + 1) * chunk_size]\n        \n        # 处理当前分块\n        chunk_features = (combined_data\n            .lazy()\n            .filter(pl.col('session').is_in(session_chunk))\n            .group_by('session')\n            .agg([\n                pl.count('session').alias('user_user_count'),\n                pl.col('aid').n_unique().alias('user_item_count'),\n                pl.col('type').mean().alias('user_buy_ratio')\n            ])\n            .collect()\n        )\n        \n        # 转换数据类型\n        chunk_features = chunk_features.with_columns([\n            pl.col('user_user_count').cast(pl.Int32),\n            pl.col('user_item_count').cast(pl.Int32),\n            pl.col('user_buy_ratio').cast(pl.Float32)\n        ])\n        \n        # 保存当前分块\n        output_path = os.path.join(output_dir, f'user_features_chunk_{i}.parquet')\n        chunk_features.write_parquet(output_path)\n        print(f\"User chunk {i+1} saved with {len(chunk_features)} users\")\n    \n    print(f\"All {n_parts} user chunks saved to {output_dir}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T07:26:15.161317Z","iopub.execute_input":"2025-10-30T07:26:15.162259Z","iopub.status.idle":"2025-10-30T07:26:15.178168Z","shell.execute_reply.started":"2025-10-30T07:26:15.162222Z","shell.execute_reply":"2025-10-30T07:26:15.177480Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## train the model","metadata":{}},{"cell_type":"code","source":"%%time\ntest_df = pl.read_parquet('/kaggle/input/otto-train-and-test-data-for-local-validation/test.parquet')\ntrain_df = pl.read_parquet('/kaggle/input/otto-train-and-test-data-for-local-validation/train.parquet')\n\nprint(f\"Test data shape: {test_df.shape}\")\nprint(f\"Train data shape: {train_df.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T07:26:15.179555Z","iopub.execute_input":"2025-10-30T07:26:15.180284Z","iopub.status.idle":"2025-10-30T07:26:23.298407Z","shell.execute_reply.started":"2025-10-30T07:26:15.180266Z","shell.execute_reply":"2025-10-30T07:26:23.297615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CHUNK_PIECES = 3\ncand_dir = \"/kaggle/input/otto-train\"\n\ncandidates = pl.concat([\n    pl.read_parquet(f'{cand_dir}/session_clicks_candidates_chunk_{k}.parquet')\n    for k in range(1, CHUNK_PIECES + 1)\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T07:26:23.299135Z","iopub.execute_input":"2025-10-30T07:26:23.299401Z","iopub.status.idle":"2025-10-30T07:26:27.667554Z","shell.execute_reply.started":"2025-10-30T07:26:23.299384Z","shell.execute_reply":"2025-10-30T07:26:27.667014Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ncombined_data = pl.concat([train_df, test_df])\n\nn_parts = 1\ndata_parts = split_preserving_items(combined_data, n_parts=n_parts)\noutput_dir = \"./item_features_chunks\"\nos.makedirs(output_dir, exist_ok=True)\nfor i, part in enumerate(data_parts):\n    print(f\"Processing chunk {i+1}/{n_parts}\")\n    item_features_chunk = process_chunk(part, i, output_dir)\n\ngenerate_user_features_chunked(\n    test_data=test_df,\n    output_dir='./user_features_chunks',\n    n_parts=1\n)    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T07:26:27.668268Z","iopub.execute_input":"2025-10-30T07:26:27.668480Z","iopub.status.idle":"2025-10-30T07:26:46.036040Z","shell.execute_reply.started":"2025-10-30T07:26:27.668463Z","shell.execute_reply":"2025-10-30T07:26:46.035274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"item_features = pl.read_parquet('/kaggle/working/item_features_chunks/item_features_part_0.parquet')\ncandidates_expanded = candidates.select([\n    pl.col('session'),\n    pl.col('candidates').alias('aid'),\n    pl.col('weights').alias('wgt')\n]).explode(['aid', 'wgt'])\n\ncandidates1 = candidates_expanded.join(item_features, on='aid', how='left').fill_null(-1)\nuser_features = pl.read_parquet('/kaggle/working/user_features_chunks/user_features_chunk_0.parquet')\ncandidates2 = candidates1.join(user_features, on='session', how='left').fill_null(-1)\noutput_path = f'training_candidates_features.parquet'\ncandidates2.write_parquet(output_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T07:26:46.036713Z","iopub.execute_input":"2025-10-30T07:26:46.036957Z","iopub.status.idle":"2025-10-30T07:27:04.093362Z","shell.execute_reply.started":"2025-10-30T07:26:46.036933Z","shell.execute_reply":"2025-10-30T07:27:04.092488Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfeature_cols = [ 'wgt','item_item_count', 'item_user_count', 'item_buy_ratio', #'wgt',\n                'user_user_count', 'user_item_count', 'user_buy_ratio']\n\nval_labels = pl.read_parquet('/kaggle/input/otto-train-and-test-data-for-local-validation/test_labels.parquet')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T07:27:04.094178Z","iopub.execute_input":"2025-10-30T07:27:04.094927Z","iopub.status.idle":"2025-10-30T07:27:04.369956Z","shell.execute_reply.started":"2025-10-30T07:27:04.094892Z","shell.execute_reply":"2025-10-30T07:27:04.369173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nval_labels_clicks = val_labels.filter(\n    pl.col('type') == 'clicks'\n).select([\n    pl.col('session'),\n    pl.col('ground_truth')\n]).explode('ground_truth').rename({\n    'ground_truth': 'aid'\n}).with_columns([\n    pl.col('session').cast(pl.Int32),\n    pl.col('aid').cast(pl.Int32),\n    pl.lit(1).alias('click')\n])\n\n\nval_candidates_click = candidates2.join(val_labels_clicks, on=['session', 'aid'], how='left').fill_null(0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T07:27:04.370777Z","iopub.execute_input":"2025-10-30T07:27:04.371013Z","iopub.status.idle":"2025-10-30T07:27:08.197575Z","shell.execute_reply.started":"2025-10-30T07:27:04.370990Z","shell.execute_reply":"2025-10-30T07:27:08.196997Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = val_candidates_click\nX = df[feature_cols].to_pandas()\ny = df['click'].to_pandas()\ngroups = df['session'].to_pandas()\n\ntrain_pool = Pool(X, y, group_id=groups)\nmodel_click = CatBoostRanker(iterations=100, task_type='GPU', verbose=False)\nmodel_click.fit(train_pool)\nmodel_click.save_model('ranker_model_click.cbm')\nprint(\"click model saved\")\n\n\n# # Create group sizes (number of candidates per session)\n# group_sizes = groups.value_counts().sort_index().values\n# # Create LightGBM dataset\n# train_data = lgb.Dataset(X, label=y, group=group_sizes)\n# # Parameters\n# params = {\n#     'objective': 'lambdarank',\n#     'metric': 'ndcg',\n#     'device': 'gpu',\n#     'num_leaves': 31,\n#     'learning_rate': 0.05,\n#     'verbose': -1\n# }\n\n# # Train model\n# model_click = lgb.train(params, train_data, num_boost_round=100)\n# model_click.save_model('ranker_model_click_lgb.txt')\n# print(\"click model saved\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T07:48:58.078004Z","iopub.execute_input":"2025-10-30T07:48:58.078314Z","execution_failed":"2025-10-30T08:05:58.977Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_labels_orders = val_labels.filter(\n    pl.col('type') == 'orders'\n).select([\n    pl.col('session'),\n    pl.col('ground_truth')\n]).explode('ground_truth').rename({\n    'ground_truth': 'aid'\n}).with_columns([\n    pl.col('session').cast(pl.Int32),\n    pl.col('aid').cast(pl.Int32),\n    pl.lit(1).alias('order')\n])\nval_candidates_order =  candidates2.join(val_labels_orders, on=['session', 'aid'], how='left').fill_null(0)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-30T08:05:58.979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = val_candidates_order\nX = df[feature_cols].to_pandas()\ny = df['order'].to_pandas()\ngroups = df['session'].to_pandas()\n\ntrain_pool = Pool(X, y, group_id=groups)\nmodel_order = CatBoostRanker(iterations=100, task_type='GPU', verbose=False)\nmodel_order.fit(train_pool)\nmodel_order.save_model('ranker_model_order.cbm')\nprint(\"order model saved\")\n\n# # Create group sizes (number of candidates per session)\n# group_sizes = groups.value_counts().sort_index().values\n# # Create LightGBM dataset\n# train_data = lgb.Dataset(X, label=y, group=group_sizes)\n# # Parameters\n# params = {\n#     'objective': 'lambdarank',\n#     'metric': 'ndcg',\n#     'device': 'gpu',\n#     'num_leaves': 31,\n#     'learning_rate': 0.05,\n#     'verbose': -1\n# }\n\n# # Train model\n# model_order = lgb.train(params, train_data, num_boost_round=100)\n# model_order.save_model('ranker_model_order_lgb.txt')\n# print(\"order model saved\")","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-30T08:05:58.979Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## prepare test feature","metadata":{}},{"cell_type":"code","source":"test_df = pl.read_parquet('/kaggle/input/otto-full-optimized-memory-footprint/test.parquet')\ntrain_df = pl.read_parquet('/kaggle/input/otto-full-optimized-memory-footprint/train.parquet')\n\nprint(f\"Test data shape: {test_df.shape}\")\nprint(f\"Train data shape: {train_df.shape}\")","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-30T08:05:58.979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CHUNK_PIECES = 3\ncand_dir = \"/kaggle/input/otto-test\"\n\ntest_candidates_click = pl.concat([\n    pl.read_parquet(f'{cand_dir}/session_clicks_candidates_chunk_{k}.parquet')\n    for k in range(1, CHUNK_PIECES + 1)\n])\ntest_candidates_order = pl.concat([\n    pl.read_parquet(f'{cand_dir}/session_orders_candidates_chunk_{k}.parquet')\n    for k in range(1, CHUNK_PIECES + 1)\n])","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-30T08:05:58.979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ncombined_data = pl.concat([train_df, test_df])\n\nn_parts = 1\ndata_parts = split_preserving_items(combined_data, n_parts=n_parts)\noutput_dir = \"./item_features_chunks\"\nos.makedirs(output_dir, exist_ok=True)\nfor i, part in enumerate(data_parts):\n    print(f\"Processing chunk {i+1}/{n_parts}\")\n    item_features_chunk = process_chunk(part, i, output_dir)\n\ngenerate_user_features_chunked(\n    test_data=test_df,\n    output_dir='./user_features_chunks',\n    n_parts=1\n)    ","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-30T08:05:58.979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"item_features = pl.read_parquet('/kaggle/working/item_features_chunks/item_features_part_0.parquet')\ncandidates_expanded = test_candidates_click.select([\n    pl.col('session'),\n    pl.col('candidates').alias('aid'),\n    pl.col('weights').alias('wgt')\n]).explode(['aid', 'wgt'])\n\ntest_candidates_click1 = candidates_expanded.join(item_features, on='aid', how='left').fill_null(-1)\nuser_features = pl.read_parquet('/kaggle/working/user_features_chunks/user_features_chunk_0.parquet')\ntest_candidates_click2 = test_candidates_click1.join(user_features, on='session', how='left').fill_null(-1)\n\noutput_path = f'test_candidates_click_features.parquet'\ntest_candidates_click2.write_parquet(output_path)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-30T08:05:58.980Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"candidates_expanded = test_candidates_order.select([\n    pl.col('session'),\n    pl.col('candidates').alias('aid'),\n    pl.col('weights').alias('wgt')\n]).explode(['aid', 'wgt'])\ntest_candidates_order1 = candidates_expanded.join(item_features, on='aid', how='left').fill_null(-1)\ntest_candidates_order2 = test_candidates_order1.join(user_features, on='session', how='left').fill_null(-1)\n\noutput_path = f'test_candidates_order_features.parquet'\ntest_candidates_order2.write_parquet(output_path)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-30T08:05:58.980Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## prediction","metadata":{}},{"cell_type":"code","source":"# Predict scores\ntest_df = test_candidates_click2\nX_test = test_df[feature_cols].to_pandas()\n\ngroups_test = test_df['session'].to_pandas()\npred_pool = Pool(X_test, group_id=groups_test)\ntest_predictions = model_click.predict(pred_pool)\n\ntest_predictions = model_click.predict(X_test)\ntest_df = test_df.with_columns(pl.Series('pred_score', test_predictions))\n\ntop_20_candidates = (\n    test_df\n    .sort(['session', 'pred_score'], descending=[False, True])\n    .group_by('session')\n    .agg([\n        pl.col('aid').head(20).alias('candidates')\n    ])\n)\n\nsubmission_click = top_20_candidates.with_columns([\n    (pl.col('session').cast(pl.Utf8) + '_clicks').alias('session_type'),\n    pl.col('candidates').cast(pl.List(pl.Utf8)).list.join(' ').alias('labels')\n]).select(['session_type', 'labels'])\n\n\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-30T08:05:58.980Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = test_candidates_order2\nX_test = test_df[feature_cols].to_pandas()\n\ngroups_test = test_df['session'].to_pandas()\npred_pool = Pool(X_test, group_id=groups_test)\ntest_predictions = model_order.predict(pred_pool)\n\ntest_predictions = model_order.predict(X_test)\ntest_df = test_df.with_columns(pl.Series('pred_score', test_predictions))\n\ntop_20_candidates = (\n    test_df\n    .sort(['session', 'pred_score'], descending=[False, True])\n    .group_by('session')\n    .agg([\n        pl.col('aid').head(20).alias('candidates')\n    ])\n)\n\nsubmission_order = top_20_candidates.with_columns([\n    (pl.col('session').cast(pl.Utf8) + '_orders').alias('session_type'),\n    pl.col('candidates').cast(pl.List(pl.Utf8)).list.join(' ').alias('labels')\n]).select(['session_type', 'labels'])\n\nsubmission_cart = top_20_candidates.with_columns([\n    (pl.col('session').cast(pl.Utf8) + '_carts').alias('session_type'),\n    pl.col('candidates').cast(pl.List(pl.Utf8)).list.join(' ').alias('labels')\n]).select(['session_type', 'labels'])\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-30T08:05:58.980Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Concatenate all three submission types\nsubmission = pl.concat([\n    submission_click,\n    submission_cart,\n    submission_order\n])\n\n# Save to CSV\nsubmission.write_csv('submission.csv')","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-30T08:05:58.980Z"}},"outputs":[],"execution_count":null}]}