{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n\npath = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\nsamples = [] \nr = range(10)\nfor i in r:\n    file_path = f\"{path}/train.parquet/partition_id={i}/part-0.parquet\"\n    chunk = pd.read_parquet(file_path)\n    sample_chunk = chunk.sample(n=100000, random_state=42)  \n    samples.append(sample_chunk)\n\nsample_df = pd.concat(samples, ignore_index=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:23:31.003848Z","iopub.execute_input":"2024-12-29T05:23:31.005119Z","iopub.status.idle":"2024-12-29T05:24:27.904208Z","shell.execute_reply.started":"2024-12-29T05:23:31.005078Z","shell.execute_reply":"2024-12-29T05:24:27.902920Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_squared_error, r2_score\nimport os\nimport polars as pl\nimport kaggle_evaluation.jane_street_inference_server\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nfrom os.path import exists, join\nimport plotly.express as px\nimport polars.selectors as cs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:24:45.922975Z","iopub.execute_input":"2024-12-29T05:24:45.923357Z","iopub.status.idle":"2024-12-29T05:24:48.752659Z","shell.execute_reply.started":"2024-12-29T05:24:45.923322Z","shell.execute_reply":"2024-12-29T05:24:48.751441Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:24:54.828877Z","iopub.execute_input":"2024-12-29T05:24:54.830131Z","iopub.status.idle":"2024-12-29T05:24:54.865039Z","shell.execute_reply.started":"2024-12-29T05:24:54.830089Z","shell.execute_reply":"2024-12-29T05:24:54.863970Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features = sample_df.filter(regex='^feature_') \nresponders = sample_df.filter(regex='^responder_') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:26:28.292198Z","iopub.execute_input":"2024-12-29T05:26:28.292978Z","iopub.status.idle":"2024-12-29T05:26:28.421829Z","shell.execute_reply.started":"2024-12-29T05:26:28.292931Z","shell.execute_reply":"2024-12-29T05:26:28.420602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:27:38.000826Z","iopub.execute_input":"2024-12-29T05:27:38.001236Z","iopub.status.idle":"2024-12-29T05:27:38.150520Z","shell.execute_reply.started":"2024-12-29T05:27:38.001200Z","shell.execute_reply":"2024-12-29T05:27:38.149218Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"可以看出，feature_21是连续型的，由于feature_20——feature_31的tag0都是false，所以推测这几个特征之间可能是有关联的，通过上面的数据分析可以看出feature_21与feature_26\tfeature_27  feature_31关联度相对较高","metadata":{}},{"cell_type":"code","source":"# 计算每个特征的最小值\nmin_values = features.min()\n\n# 将最小值和特征名称组合成一个DataFrame\nmin_df = pd.DataFrame({\n    \"feature\": min_values.index,  # 特征名称\n    \"min_value\": min_values.values  # 每个特征的最小值\n})\n\n# 设置pandas显示选项，避免省略显示\npd.set_option('display.max_rows', None)  # 显示所有行\npd.set_option('display.max_columns', None)  # 显示所有列\n\n# 显示表格\nprint(min_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:38:29.026337Z","iopub.execute_input":"2024-12-29T05:38:29.026784Z","iopub.status.idle":"2024-12-29T05:38:29.342782Z","shell.execute_reply.started":"2024-12-29T05:38:29.026743Z","shell.execute_reply":"2024-12-29T05:38:29.341623Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 计算每个特征的最小值\nmin_values = features.min()\n\n# 将最小值和特征名称组合成一个DataFrame\nmin_df = pd.DataFrame({\n    \"feature\": min_values.index,  # 特征名称\n    \"min_value\": min_values.values  # 每个特征的最小值\n})\n\n# 设置pandas显示选项，避免省略显示\npd.set_option('display.max_rows', None)  # 显示所有行\npd.set_option('display.max_columns', None)  # 显示所有列\n\n# 显示表格\nprint(max_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:38:39.286735Z","iopub.execute_input":"2024-12-29T05:38:39.287744Z","iopub.status.idle":"2024-12-29T05:38:39.600048Z","shell.execute_reply.started":"2024-12-29T05:38:39.287684Z","shell.execute_reply":"2024-12-29T05:38:39.598738Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"feature_21的范围大致为-0.558637-161.486771\n\nfeature_31的范围大致为-0.503498-127.98037\n\n这两个特征的区间相对来说很接近","metadata":{}},{"cell_type":"code","source":"# Checking distributions for feature_21 to feature_31\nfeature_cols = [f'feature_{i}' for i in range(21, 32)]  # select features from feature_21 to feature_31\n\n# Create subplots\nfig, axs = plt.subplots(1, len(feature_cols), figsize=(25, 5))\nfor i, col in enumerate(feature_cols):\n    sns.histplot(features[col].dropna(), ax=axs[i], kde=True, bins=30)\n    axs[i].set_title(f\"Distribution of {col}\")\n    axs[i].set_xlabel(col)\n    axs[i].set_ylabel('Frequency')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:53:59.449982Z","iopub.execute_input":"2024-12-29T06:53:59.450461Z","iopub.status.idle":"2024-12-29T06:54:46.718397Z","shell.execute_reply.started":"2024-12-29T06:53:59.450420Z","shell.execute_reply":"2024-12-29T06:54:46.717013Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"可以看出feature_21,feature_29,feature_30,feature_31分布很像","metadata":{}},{"cell_type":"code","source":"# 计算 feature_21 和 feature_31 之间的相关性\ncorrelation = features['feature_21'].corr(features['feature_31'])\n\n# 输出相关性结果\nprint(f\"feature_21 与 feature_31 的相关性: {correlation}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:45:27.671454Z","iopub.execute_input":"2024-12-29T05:45:27.671999Z","iopub.status.idle":"2024-12-29T05:45:27.768048Z","shell.execute_reply.started":"2024-12-29T05:45:27.671957Z","shell.execute_reply":"2024-12-29T05:45:27.766647Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 计算相关性矩阵\ncorrelation_matrix = features[['feature_21', 'feature_31']].corr()\n\n# 绘制热力图\nplt.figure(figsize=(8, 6))\nsns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', fmt='.2f', cbar=True)\n\n# 设置标题\nplt.title(\"Correlation Heatmap between feature_21 and feature_31\")\n\n# 显示图表\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:46:01.165228Z","iopub.execute_input":"2024-12-29T05:46:01.165700Z","iopub.status.idle":"2024-12-29T05:46:01.564329Z","shell.execute_reply.started":"2024-12-29T05:46:01.165660Z","shell.execute_reply":"2024-12-29T05:46:01.562796Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"通过皮尔逊系数和热力图可以看出来feature_21与feature_31的相关性非常高","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(8, 6))\nsns.scatterplot(x=features['feature_21'], y=features['feature_31'])\nplt.title('Scatter plot between feature_21 and feature_31')\nplt.xlabel('feature_21')\nplt.ylabel('feature_31')\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:23:21.657219Z","iopub.execute_input":"2024-12-29T06:23:21.657703Z","iopub.status.idle":"2024-12-29T06:23:24.012281Z","shell.execute_reply.started":"2024-12-29T06:23:21.657663Z","shell.execute_reply":"2024-12-29T06:23:24.011127Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"特征的值都集中在一个非常小的范围内，下面进行标准化或归一化","metadata":{}},{"cell_type":"code","source":"features['feature_21_log'] = np.log1p(features['feature_21'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:23:28.044262Z","iopub.execute_input":"2024-12-29T06:23:28.044717Z","iopub.status.idle":"2024-12-29T06:23:28.062159Z","shell.execute_reply.started":"2024-12-29T06:23:28.044676Z","shell.execute_reply":"2024-12-29T06:23:28.060873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 对数变换后的 feature_21 分布\nplt.figure(figsize=(10, 6))\nsns.histplot(features['feature_21_log'], kde=True, bins=30)\nplt.title('Distribution of Log-transformed feature_21')\nplt.xlabel('Log-transformed feature_21')\nplt.ylabel('Frequency')\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:23:28.659892Z","iopub.execute_input":"2024-12-29T06:23:28.660369Z","iopub.status.idle":"2024-12-29T06:23:32.625046Z","shell.execute_reply.started":"2024-12-29T06:23:28.660332Z","shell.execute_reply":"2024-12-29T06:23:32.623639Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 原始 feature_21 分布\nplt.figure(figsize=(10, 6))\nsns.histplot(features['feature_21'], kde=True, bins=30)\nplt.title('Distribution of Original feature_21')\nplt.xlabel('Original feature_21')\nplt.ylabel('Frequency')\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:23:32.627175Z","iopub.execute_input":"2024-12-29T06:23:32.627597Z","iopub.status.idle":"2024-12-29T06:23:36.491261Z","shell.execute_reply.started":"2024-12-29T06:23:32.627529Z","shell.execute_reply":"2024-12-29T06:23:36.490000Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**对数变换后，数据偏态分布应该得到改善而且更接近正态分布****","metadata":{}},{"cell_type":"markdown","source":"**将特征与feature_31进行比例交互：**","metadata":{}},{"cell_type":"code","source":"features['interaction_21_div_31'] = features['feature_21'] / (features['feature_31'] + 1e-6)\ncorrelation_with_responders = responders.corrwith(features['interaction_21_div_31'])\nprint(\"Correlation with responders:\")\nprint(correlation_with_responders)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T07:01:21.020238Z","iopub.execute_input":"2024-12-29T07:01:21.020745Z","iopub.status.idle":"2024-12-29T07:01:21.193206Z","shell.execute_reply.started":"2024-12-29T07:01:21.020704Z","shell.execute_reply":"2024-12-29T07:01:21.191985Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 计算 feature_21 和 responder_6 之间的相关性\ncorrelation = features['feature_21'].corr(responders['responder_6'])\n\n# 输出相关性结果\nprint(f\" feature_21 与 responder_6的相关性: {correlation}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T07:19:43.661888Z","iopub.execute_input":"2024-12-29T07:19:43.662362Z","iopub.status.idle":"2024-12-29T07:19:43.685874Z","shell.execute_reply.started":"2024-12-29T07:19:43.662328Z","shell.execute_reply":"2024-12-29T07:19:43.684649Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 创建更多交互特征\nfeatures['interaction_21_plus_23'] = features['feature_21'] + features['feature_23']\nfeatures['interaction_21_minus_23'] = features['feature_21'] - features['feature_23']\nfeatures['interaction_21_mult_23'] = features['feature_21'] * features['feature_23']\nfeatures['interaction_23_div_21'] = features['feature_22'] / (features['feature_23'] + 1e-6) \n\n# 创建非线性特征：平方\nfeatures['feature_21_square'] = features['feature_23'] ** 2\nfeatures['feature_23_square'] = features['feature_23'] ** 2\n\n# 绘制这些新特征的分布\nnew_features = [\n    'interaction_21_plus_23', 'interaction_21_minus_23', 'interaction_21_mult_23',\n    'interaction_23_div_21', 'feature_21_square', 'feature_23_square'\n]\n\nfig, axs = plt.subplots(2, 5, figsize=(20, 8))\n\nfor i, feature in enumerate(new_features):\n    ax = axs[i // 5, i % 5] \n    sns.histplot(features[feature].dropna(), kde=True, ax=ax, bins=30)\n    ax.set_title(f\"Distribution of {feature}\")\n\nplt.tight_layout()\nplt.show()\n\n# 进一步探索交互特征与目标变量（responder_6）的相关性\ninteraction_columns = [\n    'interaction_21_plus_23', 'interaction_21_minus_23', 'interaction_21_mult_23',\n    'interaction_23_div_21', 'feature_21_square', 'feature_23_square'\n]\n\n# 计算各个交互特征与 responder_6 的相关性\ncorrelations = {col: features[col].corr(responders['responder_6']) for col in interaction_columns}\n\n# 输出相关性结果\ncorrelation_df = pd.DataFrame(correlations, index=['Correlation with responder_6']).T\nprint(correlation_df)\n\n# 可视化各个交互特征与 responder_6 的散点图\nplt.figure(figsize=(15, 10))\nfor i, feature in enumerate(interaction_columns):\n    plt.subplot(3, 4, i + 1)\n    sns.scatterplot(x=features[feature], y=responders['responder_6'])\n    plt.title(f\"{feature} vs responder_6\")\n    plt.xlabel(feature)\n    plt.ylabel('responder_6')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T07:15:36.271645Z","iopub.execute_input":"2024-12-29T07:15:36.272200Z","iopub.status.idle":"2024-12-29T07:16:13.551183Z","shell.execute_reply.started":"2024-12-29T07:15:36.272158Z","shell.execute_reply":"2024-12-29T07:16:13.549941Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n**上述结果可以看出，feature_21 与 responder_6的相关性: -0.001961585358078289**\n\n**但是经过特征交互之后，一些新特征与responder6之间的相关性增强：interaction_23_div_21的相关性最高**\n","metadata":{}},{"cell_type":"markdown","source":"**不断尝试后，发现interaction_21_div_31与responder6的相关性最高，但是0.003292 的相关性接近于零，这意味着交互特征 ( interaction_21_div_31 ) 和responder_6之间的关系非常弱。**","metadata":{}},{"cell_type":"code","source":"features['interaction_21_div_31'] = features['feature_21'] / (features['feature_31'] + 1e-6)\ncorrelation_with_responders = responders.corrwith(features['interaction_21_div_31'])\nprint(\"Correlation with responders:\")\nprint(correlation_with_responders)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T07:24:57.544314Z","iopub.execute_input":"2024-12-29T07:24:57.544773Z","iopub.status.idle":"2024-12-29T07:24:57.708381Z","shell.execute_reply.started":"2024-12-29T07:24:57.544733Z","shell.execute_reply":"2024-12-29T07:24:57.707173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}