{"metadata":{"kernelspec":{"display_name":"pytorch_310","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.16"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# data processing libraries\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport shap\n\nfrom datetime import datetime\nimport os\n\n\nfrom sklearn.model_selection import KFold\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom xgboost  import XGBRegressor\nimport joblib   \n\n# for monitoring progress\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport seaborn as sns # plots for statistical analysis\nimport matplotlib.pyplot as plt # for data visualization\n\n# define default colors for plots in notebook\nfrom matplotlib import cycler\nfrom matplotlib.colors import LinearSegmentedColormap\ncolors = [\"#068D9D\", \"#53599A\", \"#607BB0\", \"#6D9DC5\", \"#77BECF\", \"#80DED9\", \"#AEECEF\"]\n\nplt.rc('axes', facecolor='#E6E6E6', edgecolor='none', axisbelow=True, grid=True, prop_cycle=cycler('color', colors))\n\nSEED = 42","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def reduce_mem_usage(dataframe, dataset):\n    \"\"\"\n    参数:\n        dataframe: 需要优化内存使用的pandas数据框\n        dataset: 数据集名称(用于打印信息)\n        \n    返回:\n        优化后的数据框\n        \n    功能:\n        1. 遍历数据框的每一列\n        2. 根据列的数据类型和取值范围,将其转换为占用内存更小的数据类型\n        3. 对于整数类型,尝试转换为int8/int16/int32/int64\n        4. 对于浮点类型,尝试转换为float16/float32/float64\n        5. 打印内存使用前后的对比信息\n    \"\"\"\n    print('Reducing memory usage for:', dataset)\n    initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    \n    for col in dataframe.columns:\n        if col == 'timestamp':\n            continue\n        col_type = dataframe[col].dtype\n\n        c_min = dataframe[col].min()\n        c_max = dataframe[col].max()\n        if str(col_type)[:3] == 'int':\n            # np.iinfo()：NumPy的一个函数，用来获取整数类型的信息，包括它的最小值（.min）和最大值（.max）。\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                dataframe[col] = dataframe[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                dataframe[col] = dataframe[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                dataframe[col] = dataframe[col].astype(np.int32)\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                dataframe[col] = dataframe[col].astype(np.int64)\n        else:\n            if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                dataframe[col] = dataframe[col].astype(np.float16)\n            elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                dataframe[col] = dataframe[col].astype(np.float32)\n            else:\n                dataframe[col] = dataframe[col].astype(np.float64)\n\n    final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    print('--- Memory usage before: {:.2f} MB'.format(initial_mem_usage))\n    print('--- Memory usage after: {:.2f} MB'.format(final_mem_usage))\n    print('--- Decreased memory usage by {:.1f}%\\n'.format(100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage))\n\n    return dataframe","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# pl_train = pl.read_parquet('./drw-crypto-market-prediction/train.parquet')\n# pl_test = pl.read_parquet('./drw-crypto-market-prediction/test.parquet')\n# sample = pd.read_csv(\"./drw-crypto-market-prediction/sample_submission.csv\")\n\npl_train = pl.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\npl_test = pl.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')\nsample=pd.read_csv(\"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = pl_train.to_pandas()\ndf_test = pl_test.to_pandas()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = reduce_mem_usage(df_train, \"train\")\ndf_test = reduce_mem_usage(df_test, \"test\")\n\ndf_train = df_train.reset_index()\n\ndf_train = df_train.replace([np.inf, -np.inf], np.nan)\ndf_test = df_test.replace([np.inf, -np.inf], np.nan).drop('label',axis=1)\n\nproprietary_features = [col for col in df_train.columns if col.startswith('X')]\nprint(f\"There are {len(proprietary_features)} anonymized market proprietary features.\")\n\nbasic_features = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\nprint(f\"There are {len(basic_features)} basic features.\\n\")\n\nall_features = proprietary_features + basic_features\ntarget = 'label'\ndf_train = df_train[ ['timestamp'] + basic_features + proprietary_features + ['label']]\n\nprint(f\"Train dataset contains {df_train.shape[0]} rows and {df_train.shape[1]} columns.\" )\nprint(f\"Test dataset contains {df_test.shape[0]} rows and {df_test.shape[1]} columns.\" )","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train[['X697','X698','X699','X700','X701','X702','X703']] ","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# <span><h1 style = \"font-family: garamond; font-size: 40px; font-style: normal; letter-spcaing: 3px; background-color: #f6f5f5; color :#fe346e; border-radius: 100px 100px; text-align:center\">Model training</h1></span>","metadata":{}},{"cell_type":"code","source":"lgb_params={\"boosting_type\": \"gbdt\",\n            \"metric\": 'rmse', # 评估指标 均方根误差（用于回归任务）\n            'random_state': 2025,\n            \"max_depth\": 10, # 限制树的最大深度\n            \"n_estimators\": 120, # 弱学习树的个数\n            \"learning_rate\": 0.1,\n            'num_leaves':64, # 叶子节点的最大数量\n            \"max_bin\":255, # 最大分箱数\n            \"colsample_bytree\": 0.6,   # 构建每棵树时随机选择的特征比例。\n            \"colsample_bynode\": 0.6,  # 在每个节点分裂时随机选择的特征比例。\n            \"verbose\": 0,  # 控制日志信息的输出级别。-1禁止输出，0输出最少得日志信息，1输出详细的日志信息\n            \"reg_alpha\": 0.2, \"reg_lambda\": 5, # L1和L2正则化系数\n            \"extra_trees\":True, # 当为 True 时，每个树的特征分裂点会随机选择，而不是通过贪心搜索\n            'device':'gpu', 'gpu_use_dp':True,\n            }\n\ncat_params={'task_type':'GPU',\n            'random_state':2025,\n            'eval_metric'         :'RMSE',  # 评估指标\n            'bagging_temperature' : 0.50,    # 控制随机采样的强度，用于训练不同的弱学习器。\n            'iterations'          : 200,     # 弱学习器（树）的最大迭代次数，即模型训练的轮数\n            'learning_rate'       : 0.1,\n            'max_depth'           : 12,\n            'l2_leaf_reg'         : 1.25,\n            'min_data_in_leaf'    : 24,      # 叶节点中的最小数据量\n            'random_strength'     : 0.25,    # 随机性强度，控制模型训练中随机特征分裂点的权重\n            'verbose'             : 0,\n            'loss_function'       :'RMSE',   # 多目标预测\n            # 'od_wait'             :50,     # 如果 50 轮没有提升，则停止\n          }\n\nxgb_params={'random_state': 2025, \n            'n_estimators': 125,       # 最大树的数量，即迭代的轮次\n            'learning_rate': 0.1, \n            'max_depth': 10,           # 每棵树的最大深度，控制树的复杂度。\n            'reg_alpha': 0.08,         # L1 正则化参数\n            'reg_lambda': 0.8,         # L2 正则化参数\n            'subsample': 0.95,         # 用于训练每棵树的样本比例\n            'colsample_bytree': 0.6,   # 每棵树训练时随机选择的特征比例。\n            'min_child_weight': 3,     # 子节点所需的最小权重（样本量的加权总和）\n            'device': 'cuda',          # 1. 明确指定使用 CUDA 设备 (GPU)\n            'tree_method': 'hist',     # 2. 配合使用 'hist' 算法\n            'verbose': 0,\n            # 'early_stopping_rounds': 10,\n           }\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# 1. 加载示例数据\nX, y = df_train[all_features], df_train[target]\n# 2. 划分训练集和测试集\n# 对于时间序列数据，我们应该按时间顺序分割，而不是随机分割\n# 使用前80%的数据作为训练集，后20%的数据作为测试集\ntrain_size = int(len(X) * 0.8)\nX_train, X_test = X[:train_size], X[train_size:]\ny_train, y_test = y[:train_size], y[train_size:]\n\n# 3. 训练XGBoost模型\nmodel = XGBRegressor(**xgb_params)\nmodel.fit(X_train, y_train)\n\nprint(\"XGBoost模型训练完成！\")\n# 1. 创建一个解释器（Explainer）\n# 对于树模型，使用 TreeExplainer 效率最高\nexplainer = shap.TreeExplainer(model)\n\n# 2. 计算SHAP值\n# 我们可以对测试集进行计算，以了解模型在未知数据上的表现\nshap_values = explainer.shap_values(X_test)\n\n# 3. 绘制条形图进行排序可视化\nprint(\"\\n方法一：SHAP特征重要性排序图（条形图）\")\nshap.summary_plot(shap_values, X_test, plot_type=\"bar\")\n\n# 1. 计算每个特征的平均绝对SHAP值\nmean_abs_shap = np.abs(shap_values).mean(axis=0)\n\n# 2. 将结果整理成Pandas DataFrame，方便排序和查看\nfeature_names = X_test.columns\nshap_summary = pd.DataFrame({\n    'feature': feature_names,\n    'shap_importance': mean_abs_shap\n})\n\n# 3. 对DataFrame按SHAP重要性进行降序排序\nshap_summary_sorted = shap_summary.sort_values('shap_importance', ascending=False)\n\n# 4. 打印排序后的结果\nprint(\"\\n方法二：排序后的SHAP特征重要性DataFrame\")\n# 将SHAP特征重要性数据保存为Excel文件\n# shap_summary_sorted.to_excel('shap_feature_importance.xlsx', index=False)\n# print(\"SHAP特征重要性已保存到 'shap_feature_importance.xlsx'\")\nshap_summary_sorted\n\n# 筛选出importance >= 0.01的特征\n# important_features = shap_summary_sorted[shap_summary_sorted['shap_importance'] != 0]['feature'].tolist()\nimportant_features = [\n    \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n    \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n    \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"\n]\n# # 打印筛选出的特征数量\n# print(f\"重要性大于等于0.01的特征数量: {len(important_features)}\")\n\n# # 打印筛选出的特征列表\n# print(\"\\n重要性大于等于0.01的特征:\")\n# print(important_features)\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create a directory to store the trained models\nif not os.path.exists('models'):\n    os.mkdir('models')\n\n# Define the path to load pre-trained models (if not in training mode)\nmodel_path = './models'\n\n# Initialize a list to store trained models\nmodels = []\n\nTRAINING = True\n# Function to train a model or load a pre-trained model\ndef Train(model_dict, model_name, train=None):\n    if TRAINING:\n        train_ = train\n\n         # Get the model from the dictionary\n        model = model_dict[model_name]\n\n        # Train the model based on the type (LightGBM, XGBoost, or CatBoost)\n        if model_name == 'lgb':\n            # Train LightGBM model with early stopping and evaluation logging\n            model.fit(train_[important_features],\n                      train_[target],\n                    #   eval_set=[(valid_[all_features], valid_[target])],\n                      )\n            \n        elif model_name == 'cat':\n            # Prepare evaluation set for CatBoost\n            # evalset = cat.Pool( valid_[all_features], valid_[target] )\n            \n            # Train CatBoost model with early stopping and verbose logging\n            model.fit(train_[important_features],\n                      train_[target],\n                    #   eval_set=[(evalset)]\n                     )\n            \n        else:\n            # Train XGBoost model with early stopping and verbose logging\n            model.fit(train_[important_features],\n                      train_[target], \n                    #   eval_set=[(valid_[all_features], valid_[target])],\n                      )\n\n        # Append the trained model to the list\n        models.append(model)\n        \n        # Save the trained model to a file\n        joblib.dump(model, f'{model_path}/{model_name}.model')\n        \n        # Delete training data to free up memory\n        del train\n        \n        # Collect garbage to free up memory\n        import gc\n        gc.collect()\n        \n    else:\n        # If not in training mode, load the pre-trained model from the specified path\n        models.append(joblib.load(f'{model_path}/{model_name}.model'))\n        \n    return \n\n# Dictionary to store different models with their configurations\nmodel_dict = {\n    'lgb': LGBMRegressor(**lgb_params),\n    'xgb': XGBRegressor(**xgb_params),\n    'cat': CatBoostRegressor(**cat_params),\n}\n\nprint(f'lgb')\nTrain(model_dict, 'lgb', df_train.sample(100_000)) if TRAINING else Train(model_dict, 'lgb')\nprint(f'xgb')\nTrain(model_dict, 'xgb', df_train.sample(100_000)) if TRAINING else Train(model_dict, 'xgb')\nprint(f'cat')\nTrain(model_dict, 'cat', df_train.sample(100_000)) if TRAINING else Train(model_dict, 'cat')\nprint(f'Finished')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict(test: pl.DataFrame):\n    \"\"\"Make a prediction.\"\"\"\n    # All the responders from the previous day are passed in at time_id == 0. We save them in a global variable for access at every time_id.\n    # Use them as extra features, if you like.\n    test = test[important_features]\n    test_preds = 0.55 * models[0].predict(test) + 0.25 * models[1].predict(test) + 0.2 * models[2].predict(test)\n    \n    return test_preds","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample[\"prediction\"] = predict(df_test)\nsample.to_csv(\"submission.csv\", index=False)\nsample.head()","metadata":{},"outputs":[],"execution_count":null}]}