{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30839,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nfrom matplotlib.ticker import PercentFormatter\nimport seaborn as sns\nimport warnings\n\nwarnings.filterwarnings('ignore')\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:20:47.054070Z","iopub.execute_input":"2025-01-16T07:20:47.054509Z","iopub.status.idle":"2025-01-16T07:20:47.061288Z","shell.execute_reply.started":"2025-01-16T07:20:47.054481Z","shell.execute_reply":"2025-01-16T07:20:47.060110Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 数据集基本情况","metadata":{}},{"cell_type":"markdown","source":"EDA主要思路以及部分代码参考来源：[ANTONINA DOLGORUKOVA](https://www.kaggle.com/code/antoninadolgorukova/cmi-piu-features-eda)\n\n这是非常优秀的一篇大佬文，学习为主。By the way, Kaggle上学习氛围真的很好，大佬们很乐于分享。\n\n本文主要根据数据测量方式对变量进行分类EDA，但没有分析全部变量，只是挑了比较常见的一些，主要是学习分析思路啦。\n另外，本文也会定义许多有用的可以复用的函数，帮助大家提高数据探索效率。","metadata":{}},{"cell_type":"code","source":"#导入数据\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\ndata_dict = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:29:14.990892Z","iopub.execute_input":"2025-01-16T07:29:14.991262Z","iopub.status.idle":"2025-01-16T07:29:15.046740Z","shell.execute_reply.started":"2025-01-16T07:29:14.991235Z","shell.execute_reply":"2025-01-16T07:29:15.045573Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#定义函数，查看数据的维度及前若干行信息\ndef display_dataset_info(dataset, num_rows=5, max_columns=10):\n    \"\"\"\n    输出数据集的前若干行和数据集的形状信息\n\n    参数:\n    dataset (pd.DataFrame): 输入的Pandas数据集\n    num_rows (int): 显示的行数，默认为5\n    max_columns (int or None): 显示的列数，默认为10（显示最多10列）\n    \"\"\"\n    # 显示前 num_rows 行，并控制列显示数\n    with pd.option_context('display.max_columns', max_columns):\n        display(dataset.head(num_rows))\n    # 打印形状\n    print(f\"Dataset shape: {dataset.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:20:53.591133Z","iopub.execute_input":"2025-01-16T07:20:53.591493Z","iopub.status.idle":"2025-01-16T07:20:53.596586Z","shell.execute_reply.started":"2025-01-16T07:20:53.591431Z","shell.execute_reply":"2025-01-16T07:20:53.595364Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"max_columns = train.shape[1]\ndisplay_dataset_info(train, max_columns=max_columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:20:56.941634Z","iopub.execute_input":"2025-01-16T07:20:56.941963Z","iopub.status.idle":"2025-01-16T07:20:57.010959Z","shell.execute_reply.started":"2025-01-16T07:20:56.941937Z","shell.execute_reply":"2025-01-16T07:20:57.009830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"max_columns = test.shape[1]\ndisplay_dataset_info(test, max_columns=max_columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:21:00.232316Z","iopub.execute_input":"2025-01-16T07:21:00.232714Z","iopub.status.idle":"2025-01-16T07:21:00.287113Z","shell.execute_reply.started":"2025-01-16T07:21:00.232680Z","shell.execute_reply":"2025-01-16T07:21:00.286009Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_rows = data_dict.shape[0]\ndisplay_dataset_info(data_dict, num_rows=num_rows)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:21:03.191066Z","iopub.execute_input":"2025-01-16T07:21:03.191393Z","iopub.status.idle":"2025-01-16T07:21:03.204830Z","shell.execute_reply.started":"2025-01-16T07:21:03.191366Z","shell.execute_reply":"2025-01-16T07:21:03.203727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#查看训练集中存在而测试集中不存在的变量\n#一般来说，训练集和数据集的差别在于一个目标变量，但这里却有许多，但都是和目标变量相关的，也有许多大佬用不同的目标变量构建最后的模型\ntrain_cols = set(train.columns)\ntest_cols = set(test.columns)\ncolumns_not_in_test = sorted(list(train_cols - test_cols)) \ndata_dict[data_dict['Field'].isin(columns_not_in_test)].iloc[:, 1]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:21:08.988399Z","iopub.execute_input":"2025-01-16T07:21:08.988823Z","iopub.status.idle":"2025-01-16T07:21:08.997472Z","shell.execute_reply.started":"2025-01-16T07:21:08.988778Z","shell.execute_reply":"2025-01-16T07:21:08.996333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#定义缺失值统计及可视化函数\ndef display_missing_values_info(dataset):\n    \"\"\"\n    显示数据集中每列的缺失值总数和缺失值比例\n\n    参数:\n    dataset (pd.DataFrame): 输入的Pandas数据集\n\n    返回:\n    pd.DataFrame: 包含缺失值总数和比例的信息\n    \"\"\"\n    # 计算缺失值总数\n    missing_count = dataset.isnull().sum()\n    # 计算缺失值比例\n    missing_percentage = missing_count / len(dataset)\n    # 合并到一个 DataFrame 中\n    missing_info = pd.DataFrame({\n        'Missing Count': missing_count,\n        'Missing Percentage': missing_percentage\n    }).sort_values(by='Missing Count', ascending=False)\n\n    return missing_info\n\ndef plot_missing_values_bar(missing_dataframe):\n    \"\"\"\n    根据缺失值 DataFrame 绘制缺失值和可用值比例的水平堆叠条形图\n\n    参数:\n    missing_dataframe (pd.DataFrame): 包含缺失值信息的 DataFrame，\n                                       需要包含 'Missing Count' 和 'Missing Percentage' 列\n    \"\"\"\n    # 过滤掉没有缺失值的列\n    missing_dataframe = missing_dataframe[missing_dataframe['Missing Count'] > 0]\n\n    if missing_dataframe.empty:\n        print(\"没有缺失值，图形无法绘制。\")\n        return\n\n    # 提取特征名、缺失值比例和可用值比例\n    features = missing_dataframe.index\n    missing_ratios = missing_dataframe['Missing Percentage']\n    available_ratios = 1 - missing_ratios\n\n    # 设置图形尺寸\n    plt.figure(figsize=(10, 8))\n\n    # 绘制缺失值比例（左部分）\n    plt.barh(np.arange(len(features)), missing_ratios, color='coral', label='Missing', edgecolor='black')\n\n    # 绘制可用值比例（右部分，堆叠在缺失值之后）\n    plt.barh(np.arange(len(features)), available_ratios, left=missing_ratios, color='darkseagreen', label='Available', edgecolor='black')\n\n    # 设置 y 轴为特征名\n    plt.yticks(np.arange(len(features)), features)\n\n    # 设置 x 轴为百分比格式\n    plt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\n\n    # 设置轴范围和图例\n    plt.xlim(0, 1)\n    plt.xlabel('Percentage', fontsize=12)\n    plt.ylabel('Features', fontsize=12)\n    plt.title('Missing vs Available Values', fontsize=14)\n    plt.legend()\n\n    # 显示图形\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:21:12.519119Z","iopub.execute_input":"2025-01-16T07:21:12.519509Z","iopub.status.idle":"2025-01-16T07:21:12.529868Z","shell.execute_reply.started":"2025-01-16T07:21:12.519468Z","shell.execute_reply":"2025-01-16T07:21:12.528465Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_dataframe = display_missing_values_info(train)\nmissing_dataframe","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:21:19.968155Z","iopub.execute_input":"2025-01-16T07:21:19.968542Z","iopub.status.idle":"2025-01-16T07:21:19.986428Z","shell.execute_reply.started":"2025-01-16T07:21:19.968510Z","shell.execute_reply":"2025-01-16T07:21:19.985188Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_missing_values_bar(missing_dataframe)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:21:24.538408Z","iopub.execute_input":"2025-01-16T07:21:24.538806Z","iopub.status.idle":"2025-01-16T07:21:25.701328Z","shell.execute_reply.started":"2025-01-16T07:21:24.538764Z","shell.execute_reply":"2025-01-16T07:21:25.700110Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 目标变量分析","metadata":{}},{"cell_type":"markdown","source":"我们需要明白一点，那就是目标变量sii是从变量PCIAT-PCIAT_Total score衍生而来的，而PCIAT-PCIAT_Total score\n又是PCIAT-PCIAT_01到PCIAT-PCIAT_20这二十个变量分数的加总，但会有一些细节上需要考虑的问题，下面会详细说明。","metadata":{},"attachments":{"bdadaf83-4cd1-4030-a525-9c7c4e3439af.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAFEAAAAYCAYAAACC2BGSAAAB2ElEQVRYCe2XyRHDIAxFqUsFqR6qoZLcKEYZQbBZJGIM9onMeIKxWP7js5nP50P7mWNgGOD+3SfA/DbE+/xCyQ1xEiAX3xA3xAUEFlSxnfgoRIdkjCkfAELn5Wa9IwTI4oEALeXRDg0ZdEp5S8DtNd89Waj6UffLANm8IbmFmPuALt2J3BiUELyLQqHucehYDdiTQ4aKlLD1IHoLZBAJs3iZhSM0hhrWcnCb+4CuIYihR55BnmAovOuiGGSCrkOMbmMwHNMHtB7irK5xiMQizukTHZS81g58nqNCzAeGndKl+BDECV2TEKODktNyYFJag8j5Zx3lILX1vAFxTNc4xLD+pel8TsNWbJsjQ2yhlVDreh6COKFrCGK7sbQAasn5uwhRWOhJyjsqWg9xVlcfYn2UACRbHHHmnSi7rjc4CyAu1tWHWB1xDjMcibG1o3Hib2dvzqNJpLjB9CDGb2V9aen5dbrr8iRsTNckRKKZ3TmU1QaqWKOSOP7vQczjlPQliGO6piH+E6WfE/+NtrZUvANxRNcCiGwOviIO3ljys6FiGtnlb0G8rmsNRIbgrXB3durdWQZU0RRBvwjxoi4dYqVnv+oENkSdzeUvG+JlVHrgAZET+7nP4AvNopUR1GFbVAAAAABJRU5ErkJggg=="},"e8ce4768-2485-4c55-b227-3e9035de6cef.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAFEAAAAYCAYAAACC2BGSAAAB2ElEQVRYCe2XyRHDIAxFqUsFqR6qoZLcKEYZQbBZJGIM9onMeIKxWP7js5nP50P7mWNgGOD+3SfA/DbE+/xCyQ1xEiAX3xA3xAUEFlSxnfgoRIdkjCkfAELn5Wa9IwTI4oEALeXRDg0ZdEp5S8DtNd89Waj6UffLANm8IbmFmPuALt2J3BiUELyLQqHucehYDdiTQ4aKlLD1IHoLZBAJs3iZhSM0hhrWcnCb+4CuIYihR55BnmAovOuiGGSCrkOMbmMwHNMHtB7irK5xiMQizukTHZS81g58nqNCzAeGndKl+BDECV2TEKODktNyYFJag8j5Zx3lILX1vAFxTNc4xLD+pel8TsNWbJsjQ2yhlVDreh6COKFrCGK7sbQAasn5uwhRWOhJyjsqWg9xVlcfYn2UACRbHHHmnSi7rjc4CyAu1tWHWB1xDjMcibG1o3Hib2dvzqNJpLjB9CDGb2V9aen5dbrr8iRsTNckRKKZ3TmU1QaqWKOSOP7vQczjlPQliGO6piH+E6WfE/+NtrZUvANxRNcCiGwOviIO3ljys6FiGtnlb0G8rmsNRIbgrXB3durdWQZU0RRBvwjxoi4dYqVnv+oENkSdzeUvG+JlVHrgAZET+7nP4AvNopUR1GFbVAAAAABJRU5ErkJggg=="},"b3a24f5c-2435-4dac-9ae2-b80ca280d1bf.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAFEAAAAYCAYAAACC2BGSAAAB2ElEQVRYCe2XyRHDIAxFqUsFqR6qoZLcKEYZQbBZJGIM9onMeIKxWP7js5nP50P7mWNgGOD+3SfA/DbE+/xCyQ1xEiAX3xA3xAUEFlSxnfgoRIdkjCkfAELn5Wa9IwTI4oEALeXRDg0ZdEp5S8DtNd89Waj6UffLANm8IbmFmPuALt2J3BiUELyLQqHucehYDdiTQ4aKlLD1IHoLZBAJs3iZhSM0hhrWcnCb+4CuIYihR55BnmAovOuiGGSCrkOMbmMwHNMHtB7irK5xiMQizukTHZS81g58nqNCzAeGndKl+BDECV2TEKODktNyYFJag8j5Zx3lILX1vAFxTNc4xLD+pel8TsNWbJsjQ2yhlVDreh6COKFrCGK7sbQAasn5uwhRWOhJyjsqWg9xVlcfYn2UACRbHHHmnSi7rjc4CyAu1tWHWB1xDjMcibG1o3Hib2dvzqNJpLjB9CDGb2V9aen5dbrr8iRsTNckRKKZ3TmU1QaqWKOSOP7vQczjlPQliGO6piH+E6WfE/+NtrZUvANxRNcCiGwOviIO3ljys6FiGtnlb0G8rmsNRIbgrXB3durdWQZU0RRBvwjxoi4dYqVnv+oENkSdzeUvG+JlVHrgAZET+7nP4AvNopUR1GFbVAAAAABJRU5ErkJggg=="},"fdd1a597-98f0-4812-8af2-5b7fe3bbfcc4.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAFEAAAAYCAYAAACC2BGSAAAB2ElEQVRYCe2XyRHDIAxFqUsFqR6qoZLcKEYZQbBZJGIM9onMeIKxWP7js5nP50P7mWNgGOD+3SfA/DbE+/xCyQ1xEiAX3xA3xAUEFlSxnfgoRIdkjCkfAELn5Wa9IwTI4oEALeXRDg0ZdEp5S8DtNd89Waj6UffLANm8IbmFmPuALt2J3BiUELyLQqHucehYDdiTQ4aKlLD1IHoLZBAJs3iZhSM0hhrWcnCb+4CuIYihR55BnmAovOuiGGSCrkOMbmMwHNMHtB7irK5xiMQizukTHZS81g58nqNCzAeGndKl+BDECV2TEKODktNyYFJag8j5Zx3lILX1vAFxTNc4xLD+pel8TsNWbJsjQ2yhlVDreh6COKFrCGK7sbQAasn5uwhRWOhJyjsqWg9xVlcfYn2UACRbHHHmnSi7rjc4CyAu1tWHWB1xDjMcibG1o3Hib2dvzqNJpLjB9CDGb2V9aen5dbrr8iRsTNckRKKZ3TmU1QaqWKOSOP7vQczjlPQliGO6piH+E6WfE/+NtrZUvANxRNcCiGwOviIO3ljys6FiGtnlb0G8rmsNRIbgrXB3durdWQZU0RRBvwjxoi4dYqVnv+oENkSdzeUvG+JlVHrgAZET+7nP4AvNopUR1GFbVAAAAABJRU5ErkJggg=="}}},{"cell_type":"code","source":"#定义数值变量直方图和分类变量柱状图的可视化函数\ndef plot_categorical_distribution(train, categorical_col, include_na=False, ax=None):\n    \"\"\"\n    绘制分类变量的分布及其百分比，并可选择是否包含缺失值\n\n    参数:\n    train (pd.DataFrame): 输入数据集\n    categorical_col (str): 分类变量的列名\n    include_na (bool): 是否统计缺失值，默认为 False\n    \"\"\"\n    # 计算分类变量的计数\n    if include_na:\n        counts = train[categorical_col].value_counts(dropna=False).reset_index()\n        counts[categorical_col] = counts[categorical_col].fillna('Missing')  # 将缺失值标记为 'Missing'\n    else:\n        counts = train[categorical_col].value_counts().reset_index()\n\n    counts.columns = [categorical_col, 'count']\n    total = counts['count'].sum()\n    counts['percentage'] = (counts['count'] / total) * 100\n\n    # 如果 ax 是 None，则创建新的图形和轴\n    if ax is None:\n        fig, ax = plt.subplots(1, 1, figsize=(10, 6))\n\n    # 绘制分类变量的分布柱状图\n    sns.barplot(x=categorical_col, y='count', data=counts, palette='Blues_d', ax=ax)\n    ax.set_title(f'Distribution of {categorical_col} (Include Missing: {include_na})', fontsize=14)\n\n    # 在柱状图上显示具体数值和百分比\n    for p in ax.patches:\n        height = p.get_height()\n        # 找到当前高度对应的百分比\n        percentage = counts.loc[counts['count'] == height, 'percentage'].values[0]\n        ax.text(\n            p.get_x() + p.get_width() / 2,\n            height + 5, f'{int(height)} ({percentage:.1f}%)',\n            ha=\"center\", fontsize=12\n        )\n\n    # 设置标签\n    ax.set_xlabel(categorical_col, fontsize=12)\n    ax.set_ylabel('Count', fontsize=12)\n\n    # 调整布局并显示图形\n    plt.tight_layout()\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:21:32.031813Z","iopub.execute_input":"2025-01-16T07:21:32.032216Z","iopub.status.idle":"2025-01-16T07:21:32.040776Z","shell.execute_reply.started":"2025-01-16T07:21:32.032170Z","shell.execute_reply":"2025-01-16T07:21:32.039554Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_numeric_distribution(train, numeric_col, bins=20, ax=None):\n    \"\"\"\n    绘制数值型变量的分布直方图，自动处理缺失值和无穷值\n\n    参数:\n    train (pd.DataFrame): 输入数据集\n    numeric_col (str): 数值型变量的列名\n    bins (int): 直方图的分箱数量，默认为 20\n    \"\"\"\n    # 检查列是否存在\n    if numeric_col not in train.columns:\n        raise ValueError(f\"列 '{numeric_col}' 不存在于数据集中\")\n\n    # 处理无穷值，将 inf 和 -inf 替换为 NaN\n    numeric_data = train[numeric_col].replace([np.inf, -np.inf], np.nan).dropna()\n\n    # 检查是否有有效数据\n    if numeric_data.empty:\n        print(f\"列 '{numeric_col}' 不包含有效数值数据\")\n        return\n\n    # 如果 ax 是 None，则创建新的图形和轴\n    if ax is None:\n        fig, ax = plt.subplots(figsize=(10, 6))\n\n    # 绘制直方图\n    ax.hist(numeric_data, bins=bins, color='skyblue', edgecolor='black', alpha=0.7)\n\n    # 设置标题和标签\n    ax.set_title(f'Distribution of {numeric_col}', fontsize=14)\n    ax.set_xlabel(f'{numeric_col}', fontsize=12)\n    ax.set_ylabel('Frequency', fontsize=12)\n\n    # 调整布局并显示图形\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:21:36.128823Z","iopub.execute_input":"2025-01-16T07:21:36.129186Z","iopub.status.idle":"2025-01-16T07:21:36.135836Z","shell.execute_reply.started":"2025-01-16T07:21:36.129155Z","shell.execute_reply":"2025-01-16T07:21:36.134635Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#对目标变量的含义进行定义后映射\nsii_map = {0.0: '0 (None)', 1.0: '1 (Mild)', 2.0: '2 (Moderate)', 3.0: '3 (Severe)'}\ntrain['sii'] = train['sii'].map(sii_map)\n# 将非缺失值转换为int类型\ntrain['PCIAT-PCIAT_Total'] = train['PCIAT-PCIAT_Total'].apply(lambda x: int(x) if not pd.isna(x) else x)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:24:52.731672Z","iopub.execute_input":"2025-01-16T07:24:52.732076Z","iopub.status.idle":"2025-01-16T07:24:52.741956Z","shell.execute_reply.started":"2025-01-16T07:24:52.732045Z","shell.execute_reply":"2025-01-16T07:24:52.741012Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n# 调用函数绘制类别变量的分布\nplot_categorical_distribution(train, 'sii', include_na=True, ax=axes[0])\n# 调用函数绘制数值变量的分布\nplot_numeric_distribution(train, 'PCIAT-PCIAT_Total', bins=10, ax=axes[1])\n\n#这里绘制数值型变量直方图时除了点问题，但是单独执行这个代码是没有问题的，不知到为什么设置成子图后就不显示了，本地测试也是没有问题的，\n#心累~ ~","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:26:47.755447Z","iopub.execute_input":"2025-01-16T07:26:47.755842Z","iopub.status.idle":"2025-01-16T07:26:48.355049Z","shell.execute_reply.started":"2025-01-16T07:26:47.755813Z","shell.execute_reply":"2025-01-16T07:26:48.354018Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1.目标变量数据极度不平衡，极端的数据样本案例比较少\n2.目标变量存在大量的缺失值","metadata":{}},{"cell_type":"code","source":"#探究目标变量sii和PCIAT-PCIAT_Total之间的关系\npciat_min_max = train.groupby('sii')['PCIAT-PCIAT_Total'].agg(['min', 'max'])\npciat_min_max = pciat_min_max.rename(\n    columns={'min': 'Minimum PCIAT total Score', 'max': 'Maximum total PCIAT Score'})\npciat_min_max","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:21:49.333949Z","iopub.execute_input":"2025-01-16T07:21:49.334288Z","iopub.status.idle":"2025-01-16T07:21:49.347856Z","shell.execute_reply.started":"2025-01-16T07:21:49.334259Z","shell.execute_reply":"2025-01-16T07:21:49.346574Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"变量PCIAT-PCIAT_Total的数值与变量sii的类别是相对应的","metadata":{}},{"cell_type":"code","source":"#探究变量PCIAT-PCIAT_Total与PCIAT-PCIAT_01 to PCIAT-PCIAT_20之间的关系\n#在上面的缺失值情况统计时，我们知道变量PCIAT-PCIAT_Total与PCIAT-PCIAT_01 to PCIAT-PCIAT_20的缺失情况不一致，这说明\n#有的人可能只进行了部分问题测试，其中没有回答被记为NA Values,这样就会导致没有完成所有测试问题的孩子分可能被低估\nfiltered_columns = [col for col in train.columns if 'PCIAT-PCIAT' in col or 'sii' in col]\ntrain_with_sii = train[~train['sii'].isnull()][filtered_columns]\n\n#显示在剔除所有缺失值列后，所有PCIAT-PCIAT系列变量至少含有一个缺失数值的样本\ntrain_with_sii[train_with_sii.isnull().any(axis=1)].style.applymap(\n    lambda x: 'background-color: #FFC0CB' if pd.isna(x) else '')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:32:13.457563Z","iopub.execute_input":"2025-01-16T07:32:13.457930Z","iopub.status.idle":"2025-01-16T07:32:13.503941Z","shell.execute_reply.started":"2025-01-16T07:32:13.457903Z","shell.execute_reply":"2025-01-16T07:32:13.502861Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#确定这些数据中的缺失值是否直接被忽略或是当作0值来处理\nPCIAT_cols = [f'PCIAT-PCIAT_{i+1:02d}' for i in range(20)]\nrecalc_total_score = train_with_sii[PCIAT_cols].sum(\n    axis=1, skipna=True)\n(recalc_total_score == train_with_sii['PCIAT-PCIAT_Total']).all()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:32:22.847738Z","iopub.execute_input":"2025-01-16T07:32:22.848074Z","iopub.status.idle":"2025-01-16T07:32:22.857680Z","shell.execute_reply.started":"2025-01-16T07:32:22.848046Z","shell.execute_reply":"2025-01-16T07:32:22.856654Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#确定这些数据中哪些缺失值可能会影响变量SII的值，思路是对于每个测试中的问题，最高得分为5分，如果\n#加上最高分的情况下，SII的值仍然不变（未达到阈值），这说明不会影响SII的结果，反之，则可能影响SII的结果\n\ndef recalculate_sii(row):\n    max_possible = row['PCIAT-PCIAT_Total'] + row[PCIAT_cols].isna().sum() * 5\n    if row['PCIAT-PCIAT_Total'] <= 30 and max_possible <= 30:\n        return 0\n    elif 31 <= row['PCIAT-PCIAT_Total'] <= 49 and max_possible <= 49:\n        return 1\n    elif 50 <= row['PCIAT-PCIAT_Total'] <= 79 and max_possible <= 79:\n        return 2\n    elif row['PCIAT-PCIAT_Total'] >= 80 and max_possible >= 80:\n        return 3\n    return np.nan\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:30:41.632187Z","iopub.execute_input":"2025-01-16T07:30:41.632597Z","iopub.status.idle":"2025-01-16T07:30:41.638345Z","shell.execute_reply.started":"2025-01-16T07:30:41.632564Z","shell.execute_reply":"2025-01-16T07:30:41.637084Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_with_sii['recalc_sii'] = train_with_sii.apply(recalculate_sii, axis=1)\n\nreversed_sii_map = {v: k for k, v in sii_map.items()}\ntrain_with_sii['sii'] = train_with_sii['sii'].map(reversed_sii_map)\n\nmismatch_rows = train_with_sii.query('recalc_sii != sii')\nmismatch_rows\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:32:29.887200Z","iopub.execute_input":"2025-01-16T07:32:29.887599Z","iopub.status.idle":"2025-01-16T07:32:31.367522Z","shell.execute_reply.started":"2025-01-16T07:32:29.887563Z","shell.execute_reply":"2025-01-16T07:32:31.366303Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"存在17行可能错误的SII值","metadata":{}},{"cell_type":"markdown","source":"#  Age&Sex","metadata":{}},{"cell_type":"code","source":"#借鉴非常好用的统计函数\ndef calculate_stats(data, columns):\n    if isinstance(columns, str):\n        columns = [columns]\n\n    stats = []\n    for col in columns:\n        if data[col].dtype in ['object', 'category']:\n            counts = data[col].value_counts(dropna=False, sort=False)\n            percents = data[col].value_counts(normalize=True, dropna=False, sort=False) * 100\n            formatted = counts.astype(str) + ' (' + percents.round(2).astype(str) + '%)'\n            stats_col = pd.DataFrame({'count (%)': formatted})\n            stats.append(stats_col)\n        else:\n            stats_col = data[col].describe().to_frame().transpose()\n            stats_col['missing'] = data[col].isnull().sum()\n            stats_col.index.name = col\n            stats.append(stats_col)\n\n    return pd.concat(stats, axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:22:16.042340Z","iopub.execute_input":"2025-01-16T07:22:16.042725Z","iopub.status.idle":"2025-01-16T07:22:16.049287Z","shell.execute_reply.started":"2025-01-16T07:22:16.042691Z","shell.execute_reply":"2025-01-16T07:22:16.048109Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Age&Sex\ntrain['sii'].fillna('Missing', inplace=True)\ndemo_features = [col for col in train.columns if 'Basic_Demos' in col]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:22:21.425765Z","iopub.execute_input":"2025-01-16T07:22:21.426105Z","iopub.status.idle":"2025-01-16T07:22:21.431641Z","shell.execute_reply.started":"2025-01-16T07:22:21.426077Z","shell.execute_reply":"2025-01-16T07:22:21.430248Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#人口统计特征变量基本情况\nsex_map = {0: 'Male', 1: 'Female'}\ntrain['Basic_Demos-Sex'] = train['Basic_Demos-Sex'].map(sex_map)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:22:24.455057Z","iopub.execute_input":"2025-01-16T07:22:24.455397Z","iopub.status.idle":"2025-01-16T07:22:24.461188Z","shell.execute_reply.started":"2025-01-16T07:22:24.455371Z","shell.execute_reply":"2025-01-16T07:22:24.460003Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, ['Basic_Demos-Sex'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:22:27.586879Z","iopub.execute_input":"2025-01-16T07:22:27.587224Z","iopub.status.idle":"2025-01-16T07:22:27.598557Z","shell.execute_reply.started":"2025-01-16T07:22:27.587196Z","shell.execute_reply":"2025-01-16T07:22:27.597313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#注意，cut()默认是左开右闭，而不是左闭右开\ntrain['Age_Group'] = pd.cut(\n    train['Basic_Demos-Age'],\n    bins=[4, 12, 18, 22],\n    labels=['Children (5-12)', 'Adolescents (13-18)', 'Adults (19-22)']\n)\n\ncalculate_stats(train, ['Basic_Demos-Sex', 'Age_Group', 'Basic_Demos-Enroll_Season'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:22:30.422588Z","iopub.execute_input":"2025-01-16T07:22:30.422938Z","iopub.status.idle":"2025-01-16T07:22:30.441504Z","shell.execute_reply.started":"2025-01-16T07:22:30.422906Z","shell.execute_reply":"2025-01-16T07:22:30.440388Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"季节系列的变量，我认为是不相关的预测变量，不知道大家有没有不同的看法。","metadata":{}},{"cell_type":"code","source":"#sii by Age & PCIAT_PCIAT_Total by Age\nfig, axes = plt.subplots(2, 2, figsize=(16, 10))\n\n#PCIAT-PCIAT_Total by Age_Group\nsns.boxplot(\n    x='Age_Group', y='PCIAT-PCIAT_Total',\n    data=train, palette=\"Set3\", ax=axes[0, 0])\naxes[0, 0].set_title('PCIAT_Total by Age Group')\naxes[0, 0].set_ylabel('PCIAT_Total values')\naxes[0, 0].set_xlabel('Age Group')\n\n#SII by Age_Group\nsns.countplot(\n    x='Age_Group', hue='sii',\n    data=train, palette=\"Set3\", ax=axes[0, 1])\naxes[0, 1].set_title('SII by Age Group')\naxes[0, 1].set_ylabel('SII Counts')\naxes[0, 1].set_xlabel('Age Group')\n\n#PCIAT-PCIAT_Total by Sex\nsns.boxplot(\n    x='Basic_Demos-Sex', y='PCIAT-PCIAT_Total',\n    data=train, palette=\"Set3\", ax=axes[1, 0])\naxes[1, 0].set_title('PCIAT_Total by Sex')\naxes[1, 0].set_ylabel('PCIAT_Total values')\naxes[1, 0].set_xlabel('Sex')\n\n#SII by Sex\nsns.countplot(\n    x='Basic_Demos-Sex', hue='sii',\n    data=train, palette=\"Set3\", ax=axes[1, 1])\naxes[1, 1].set_title('SII by Sex')\naxes[1, 1].set_ylabel('SII Counts')\naxes[1, 1].set_xlabel('Sex')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:22:34.638787Z","iopub.execute_input":"2025-01-16T07:22:34.639231Z","iopub.status.idle":"2025-01-16T07:22:35.617966Z","shell.execute_reply.started":"2025-01-16T07:22:34.639195Z","shell.execute_reply":"2025-01-16T07:22:35.616170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n#SII by Age_Group\nstats = train.groupby(['Age_Group', 'sii']).size().unstack(fill_value=0)\nstats_prop = stats.div(stats.sum(axis=1), axis=0) * 100\nstats = stats.astype(str) +' (' + stats_prop.round(1).astype(str) + '%)'\n\nmax_columns = stats.shape[1]\nwith pd.option_context('display.max_columns', max_columns):\n    display(stats)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:33:05.052228Z","iopub.execute_input":"2025-01-16T07:33:05.052643Z","iopub.status.idle":"2025-01-16T07:33:05.078827Z","shell.execute_reply.started":"2025-01-16T07:33:05.052608Z","shell.execute_reply":"2025-01-16T07:33:05.077508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#SII by Sex\nstats = train.groupby(['Basic_Demos-Sex', 'sii']).size().unstack(fill_value=0)\nstats_prop = stats.div(stats.sum(axis=1), axis=0) * 100\nstats = stats.astype(str) +' (' + stats_prop.round(1).astype(str) + '%)'\nstats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:40:37.212044Z","iopub.execute_input":"2025-01-16T06:40:37.212400Z","iopub.status.idle":"2025-01-16T06:40:37.229561Z","shell.execute_reply.started":"2025-01-16T06:40:37.212369Z","shell.execute_reply":"2025-01-16T06:40:37.228372Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1.PIU impairment和年龄之间存在倒U型关系  2.PIU impairment和性别之间关系并不明显","metadata":{}},{"cell_type":"markdown","source":"# CGAS-CGAS_Score","metadata":{}},{"cell_type":"code","source":"#明显存在异常值\ncalculate_stats(train, 'CGAS-CGAS_Score')\nplot_numeric_distribution(train, 'CGAS-CGAS_Score', bins=100)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:44:18.487008Z","iopub.execute_input":"2025-01-16T06:44:18.487338Z","iopub.status.idle":"2025-01-16T06:44:18.866471Z","shell.execute_reply.started":"2025-01-16T06:44:18.487311Z","shell.execute_reply":"2025-01-16T06:44:18.865445Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#这条异常数据绝大部分特征都是缺失的，直接剔除即可\noutlier = train.query('`CGAS-CGAS_Score` == 999.0')\ntrain.query('`CGAS-CGAS_Score` != 999.0', inplace=True)\n#剔除后的数据分布\nplot_numeric_distribution(train, 'CGAS-CGAS_Score', bins=20)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:44:45.582212Z","iopub.execute_input":"2025-01-16T06:44:45.582607Z","iopub.status.idle":"2025-01-16T06:44:45.876637Z","shell.execute_reply.started":"2025-01-16T06:44:45.582576Z","shell.execute_reply":"2025-01-16T06:44:45.875581Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#CGAS-CGAS_Score与目标变量的关系\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\n#CGAS-CGAS_Score by sii\nsns.boxplot(\n    x='sii', y='CGAS-CGAS_Score',\n    data=train, palette=\"Set3\", ax=axes[0])\naxes[0].set_title('CGAS-CGAS_Score by sii')\naxes[0].set_ylabel('CGAS-CGAS_Score')\naxes[0].set_xlabel('sii')\n\n#回归趋势图\nsns.regplot(x='CGAS-CGAS_Score', y='PCIAT-PCIAT_Total', data=train, scatter_kws={'alpha': 0.5}, ax=axes[1])\naxes[1].set_title('Regression Plot')\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:45:36.727391Z","iopub.execute_input":"2025-01-16T06:45:36.727796Z","iopub.status.idle":"2025-01-16T06:45:37.331017Z","shell.execute_reply.started":"2025-01-16T06:45:36.727765Z","shell.execute_reply":"2025-01-16T06:45:37.329894Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"CGAS-CGAS_Score与目标变量sii之间并没有存在明显的关系 ","metadata":{}},{"cell_type":"markdown","source":"# Internet Use","metadata":{}},{"cell_type":"code","source":"train.rename(columns={'PreInt_EduHx-computerinternet_hoursday': 'hours of internet_use'}, inplace=True)\n\nhours_map = {0.0: '< 1h/day', 1.0: '~ 1h/day', 2.0: '~ 2hs/day', 3.0: '> 3hs/day'}\ntrain['hours of internet_use'] = train['hours of internet_use'].map(hours_map)\ntrain['hours of internet_use'].fillna('Missing', inplace=True)\ncalculate_stats(train, 'hours of internet_use')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:48:04.319258Z","iopub.execute_input":"2025-01-16T06:48:04.319866Z","iopub.status.idle":"2025-01-16T06:48:04.345399Z","shell.execute_reply.started":"2025-01-16T06:48:04.319829Z","shell.execute_reply":"2025-01-16T06:48:04.344202Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#CGAS-CGAS_Score与目标变量的关系\n\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\n#CGAS-CGAS_Score by sii\nsns.boxplot(\n    x='hours of internet_use', y='PCIAT-PCIAT_Total',\n    data=train, palette=\"Set3\", ax=axes[0])\naxes[0].set_title('PCIAT-PCIAT_Total by hours of internet_use')\naxes[0].set_ylabel('PCIAT-PCIAT_Total')\naxes[0].set_xlabel('hours of internet_use')\n\n#SII by hours of internet_use\nsns.countplot(\n    x='hours of internet_use', hue='sii',\n    data=train, palette=\"Set3\", ax=axes[1])\naxes[1].set_title('SII by hours of internet_use')\naxes[1].set_ylabel('SII Counts')\naxes[1].set_xlabel('Age Group')\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:48:37.646674Z","iopub.execute_input":"2025-01-16T06:48:37.647077Z","iopub.status.idle":"2025-01-16T06:48:38.312306Z","shell.execute_reply.started":"2025-01-16T06:48:37.647035Z","shell.execute_reply":"2025-01-16T06:48:38.310935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#hours of internet_use by sex\nstats = train.groupby(['Basic_Demos-Sex', 'hours of internet_use']).size().unstack(fill_value=0)\nstats_prop = stats.div(stats.sum(axis=1), axis=0) * 100\nstats = stats.astype(str) +' (' + stats_prop.round(1).astype(str) + '%)'\nstats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:48:59.492215Z","iopub.execute_input":"2025-01-16T06:48:59.492612Z","iopub.status.idle":"2025-01-16T06:48:59.509101Z","shell.execute_reply.started":"2025-01-16T06:48:59.492574Z","shell.execute_reply":"2025-01-16T06:48:59.508058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#hours of internet_use by Age_Group\nstats = train.groupby(['Age_Group', 'hours of internet_use']).size().unstack(fill_value=0)\n\nfig, axes = plt.subplots(1, len(stats), figsize=(18, 5))\nfor i, age_group in enumerate(stats.index):\n    group_counts = stats.loc[age_group] / stats.loc[age_group].sum()\n    axes[i].pie(group_counts, labels=group_counts.index, autopct='%1.1f%%',\n                startangle=90, colors=sns.color_palette(\"Set3\"), labeldistance=1.1)\n    axes[i].set_title(f'Distribution of Hours of Internet Use\\n{age_group}')\n    axes[i].axis('equal')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:49:41.155730Z","iopub.execute_input":"2025-01-16T06:49:41.156102Z","iopub.status.idle":"2025-01-16T06:49:41.548485Z","shell.execute_reply.started":"2025-01-16T06:49:41.156072Z","shell.execute_reply":"2025-01-16T06:49:41.547398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"reversed_hours_map = {v: k for k, v in hours_map.items()}\ntrain['hours of internet_use'] = train['hours of internet_use'].map(reversed_hours_map)\n\n# 自定义四分位数函数\ndef q25(x):\n    return x.quantile(0.25)\ndef q75(x):\n    return x.quantile(0.75)\n# 分组统计\ntrain.groupby('Age_Group')['hours of internet_use'].agg(\n    mean=lambda x: round(x.mean(), 2),\n    median=lambda x: round(x.median(), 1),\n    q25=lambda x: round(q25(x), 1),\n    q75=lambda x: round(q75(x), 1))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:50:05.596576Z","iopub.execute_input":"2025-01-16T06:50:05.596910Z","iopub.status.idle":"2025-01-16T06:50:05.621570Z","shell.execute_reply.started":"2025-01-16T06:50:05.596884Z","shell.execute_reply":"2025-01-16T06:50:05.620343Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":" 1.每天上网的时长和目标sii存在明显的正向线性关系 2.男性女性每天上网的时间分布很相似\n 3.不同年龄组之间的每日上网时长区分明显，成年组平均时长为2.23小时，青少年组平均时长为1.76小时，儿童组平均时长为0.81小时。结果表明年纪越大的对象往往会花更多的时间上网。这里我有一个猜想，是否可以将年龄分组与上网时长结合排列组合形成一个新的交互变量，比如变量值Children (5-12)-~ 2h/day与变量值Adults (19-22) -~ 2h/day，虽然上网时间都是一小时，但对于不同年龄组别来说，对身心健康的影响程度可能存在不同（仅作参考，没有考证是否有相关的文献资料支持）。","metadata":{}},{"cell_type":"markdown","source":"# Sleep Disturbance Scale","metadata":{}},{"cell_type":"code","source":"SDS_features = [col for col in train.columns if 'SDS-SDS' in col]\n\ncalculate_stats(train, SDS_features)\n\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\nplot_numeric_distribution(train, SDS_features[0], bins=20, ax=axes[0])\nplot_numeric_distribution(train, SDS_features[1], bins=20, ax=axes[1])\n\n#为什么第二张子图老是有问题呢 ，本地是没有问题的。。。","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:51:40.348964Z","iopub.execute_input":"2025-01-16T06:51:40.349286Z","iopub.status.idle":"2025-01-16T06:51:40.775123Z","shell.execute_reply.started":"2025-01-16T06:51:40.349263Z","shell.execute_reply":"2025-01-16T06:51:40.774090Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#SDS与目标变量的关系\nfig, axes = plt.subplots(2, 2, figsize=(16, 10))\n#SDS by sii\nsns.boxplot(\n    x='sii', y=SDS_features[0],\n    data=train, palette=\"Set3\", ax=axes[0, 0])\naxes[0, 0].set_title(f'{SDS_features[0]} by sii')\naxes[0, 0].set_ylabel('CGAS-CGAS_Score')\naxes[0, 0].set_xlabel(f'{SDS_features[0]}')\n\nsns.boxplot(\n    x='sii', y=SDS_features[1],\n    data=train, palette=\"Set3\", ax=axes[0, 1])\naxes[0, 1].set_title(f'{SDS_features[1]} by sii')\naxes[0, 1].set_ylabel('CGAS-CGAS_Score')\naxes[0, 1].set_xlabel(f'{SDS_features[1]}')\n\n#回归趋势图\nsns.regplot(x='CGAS-CGAS_Score', y=SDS_features[0], data=train, scatter_kws={'alpha': 0.5}, ax=axes[1, 0])\naxes[1, 0].set_title('Regression Plot')\n\nsns.regplot(x='CGAS-CGAS_Score', y=SDS_features[1], data=train, scatter_kws={'alpha': 0.5}, ax=axes[1, 1])\naxes[1, 1].set_title('Regression Plot')\n\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:53:35.899167Z","iopub.execute_input":"2025-01-16T06:53:35.899562Z","iopub.status.idle":"2025-01-16T06:53:37.032348Z","shell.execute_reply.started":"2025-01-16T06:53:35.899530Z","shell.execute_reply":"2025-01-16T06:53:37.031228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 分组统计\ntrain.groupby('Age_Group')[SDS_features[0]].agg(\n    mean=lambda x: round(x.mean(), 2),\n    median=lambda x: round(x.median(), 1),\n    q25=lambda x: round(q25(x), 1),\n    q75=lambda x: round(q75(x), 1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:54:16.436386Z","iopub.execute_input":"2025-01-16T06:54:16.436774Z","iopub.status.idle":"2025-01-16T06:54:16.457421Z","shell.execute_reply.started":"2025-01-16T06:54:16.436743Z","shell.execute_reply":"2025-01-16T06:54:16.456364Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.groupby('Basic_Demos-Sex')[SDS_features[0]].agg(\n    mean=lambda x: round(x.mean(), 2),\n    median=lambda x: round(x.median(), 1),\n    q25=lambda x: round(q25(x), 1),\n    q75=lambda x: round(q75(x), 1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:54:24.027765Z","iopub.execute_input":"2025-01-16T06:54:24.028123Z","iopub.status.idle":"2025-01-16T06:54:24.047807Z","shell.execute_reply.started":"2025-01-16T06:54:24.028092Z","shell.execute_reply":"2025-01-16T06:54:24.046579Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.groupby('Age_Group')[SDS_features[1]].agg(\n    mean=lambda x: round(x.mean(), 2),\n    median=lambda x: round(x.median(), 1),\n    q25=lambda x: round(q25(x), 1),\n    q75=lambda x: round(q75(x), 1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:54:31.573846Z","iopub.execute_input":"2025-01-16T06:54:31.574172Z","iopub.status.idle":"2025-01-16T06:54:31.593902Z","shell.execute_reply.started":"2025-01-16T06:54:31.574146Z","shell.execute_reply":"2025-01-16T06:54:31.592983Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.groupby('Basic_Demos-Sex')[SDS_features[1]].agg(\n    mean=lambda x: round(x.mean(), 2),\n    median=lambda x: round(x.median(), 1),\n    q25=lambda x: round(q25(x), 1),\n    q75=lambda x: round(q75(x), 1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:54:38.242630Z","iopub.execute_input":"2025-01-16T06:54:38.243002Z","iopub.status.idle":"2025-01-16T06:54:38.262526Z","shell.execute_reply.started":"2025-01-16T06:54:38.242966Z","shell.execute_reply":"2025-01-16T06:54:38.261306Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1.SDS score为右偏分布，数值主要集中在较低值附近\n2.考虑不同性别，不同年龄组，发现SDS score没有显著的区别\n3.CGAS-CGAS_Score和目标变量存在明显的负相关关系，从这个角度来看，SDS score是具有区分度的预测变量","metadata":{}},{"cell_type":"markdown","source":"# Physical Activity Questionnaire","metadata":{}},{"cell_type":"code","source":"PAQ_features = ['PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total']\ncalculate_stats(train, PAQ_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:55:52.010626Z","iopub.execute_input":"2025-01-16T06:55:52.011436Z","iopub.status.idle":"2025-01-16T06:55:52.036615Z","shell.execute_reply.started":"2025-01-16T06:55:52.011371Z","shell.execute_reply":"2025-01-16T06:55:52.035347Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#问卷年龄的划分存在问题\nA_max_min = train[~train['PAQ_A-PAQ_A_Total'].isnull()]['Basic_Demos-Age'].describe()[['max', 'min']]\nA_max_min.name = \"Age of Adolescents (13-18)\"\nC_max_min = train[~train['PAQ_C-PAQ_C_Total'].isnull()]['Basic_Demos-Age'].describe()[['max', 'min']]\nC_max_min.name = \"Children (5-12)\"\npd.concat([A_max_min, C_max_min], axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:56:33.987927Z","iopub.execute_input":"2025-01-16T06:56:33.988266Z","iopub.status.idle":"2025-01-16T06:56:34.010214Z","shell.execute_reply.started":"2025-01-16T06:56:33.988239Z","shell.execute_reply":"2025-01-16T06:56:34.009025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#存在争议的样本数据\nwrong_age_group = train[~train['PAQ_A-PAQ_A_Total'].isnull()].query('`Basic_Demos-Age` >= 13')\nwrong_age_group","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:56:53.675737Z","iopub.execute_input":"2025-01-16T06:56:53.676134Z","iopub.status.idle":"2025-01-16T06:56:53.727002Z","shell.execute_reply.started":"2025-01-16T06:56:53.676103Z","shell.execute_reply":"2025-01-16T06:56:53.725910Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#是否存在同时具有PAQ_A-PAQ_A_Total和PAQ_C-PAQ_C_Total值的数据\ndouble_PAQs = train[train['PAQ_A-PAQ_A_Total'].notnull() & train['PAQ_C-PAQ_C_Total'].notnull()]\ndouble_PAQs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:57:15.940574Z","iopub.execute_input":"2025-01-16T06:57:15.940928Z","iopub.status.idle":"2025-01-16T06:57:15.963484Z","shell.execute_reply.started":"2025-01-16T06:57:15.940901Z","shell.execute_reply":"2025-01-16T06:57:15.962446Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#可以考虑剔除上述数据并且将变量PAQ_A-PAQ_A_Total和PAQ_C-PAQ_C_Total进行合并\ntrain = train[~train['id'].isin(double_PAQs['id'])]\ntrain['PAQ_Total_Combined'] = train['PAQ_A-PAQ_A_Total'].combine_first(train['PAQ_C-PAQ_C_Total'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:57:40.356219Z","iopub.execute_input":"2025-01-16T06:57:40.356622Z","iopub.status.idle":"2025-01-16T06:57:40.366316Z","shell.execute_reply.started":"2025-01-16T06:57:40.356587Z","shell.execute_reply":"2025-01-16T06:57:40.365278Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_numeric_distribution(train, 'PAQ_Total_Combined')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:57:48.378886Z","iopub.execute_input":"2025-01-16T06:57:48.379256Z","iopub.status.idle":"2025-01-16T06:57:48.631675Z","shell.execute_reply.started":"2025-01-16T06:57:48.379223Z","shell.execute_reply":"2025-01-16T06:57:48.630351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#PAQ_Total_Combined与目标变量的关系\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n#CGAS-CGAS_Score by sii\nsns.boxplot(\n    x='sii', y='PAQ_Total_Combined',\n    data=train, palette=\"Set3\", ax=axes[0])\naxes[0].set_title('PAQ_Total_Combined by sii')\naxes[0].set_ylabel('PAQ_Total_Combined_Score')\naxes[0].set_xlabel('sii')\n\n#回归趋势图\nsns.regplot(x='PAQ_Total_Combined', y='PCIAT-PCIAT_Total', data=train, scatter_kws={'alpha': 0.5}, ax=axes[1])\naxes[1].set_title('Regression Plot')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:58:10.188042Z","iopub.execute_input":"2025-01-16T06:58:10.188391Z","iopub.status.idle":"2025-01-16T06:58:10.722615Z","shell.execute_reply.started":"2025-01-16T06:58:10.188363Z","shell.execute_reply":"2025-01-16T06:58:10.721567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#这里注意，是否剔除具有争议的列直接影响统计结论\ntrain.groupby('Age_Group')['PAQ_Total_Combined'].agg(\n    mean=lambda x: round(x.mean(), 2),\n    median=lambda x: round(x.median(), 1),\n    q25=lambda x: round(q25(x), 1),\n    q75=lambda x: round(q75(x), 1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:58:36.372100Z","iopub.execute_input":"2025-01-16T06:58:36.372439Z","iopub.status.idle":"2025-01-16T06:58:36.391852Z","shell.execute_reply.started":"2025-01-16T06:58:36.372412Z","shell.execute_reply":"2025-01-16T06:58:36.390731Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[~train['id'].isin(wrong_age_group['id'])].groupby('Age_Group')['PAQ_Total_Combined'].agg(\n    mean=lambda x: round(x.mean(), 2),\n    median=lambda x: round(x.median(), 1),\n    q25=lambda x: round(q25(x), 1),\n    q75=lambda x: round(q75(x), 1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:58:48.843738Z","iopub.execute_input":"2025-01-16T06:58:48.844093Z","iopub.status.idle":"2025-01-16T06:58:48.865059Z","shell.execute_reply.started":"2025-01-16T06:58:48.844063Z","shell.execute_reply":"2025-01-16T06:58:48.864064Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.groupby('Basic_Demos-Sex')['PAQ_Total_Combined'].agg(\n    mean=lambda x: round(x.mean(), 2),\n    median=lambda x: round(x.median(), 1),\n    q25=lambda x: round(q25(x), 1),\n    q75=lambda x: round(q75(x), 1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T06:59:00.578678Z","iopub.execute_input":"2025-01-16T06:59:00.579020Z","iopub.status.idle":"2025-01-16T06:59:00.596911Z","shell.execute_reply.started":"2025-01-16T06:59:00.578992Z","shell.execute_reply":"2025-01-16T06:59:00.595749Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1.PAQ数据接近正态分布，对于Adults来说，该变量值为缺失\n2.PAQ_Total_Combined与目标变量不存在明显关系\n3.男性的Physical Activity score 整体水平高于女性。值得注意是，在不剔除组别划分有问题的数据的情况下，儿童的Physical Activity score整体水平明显高于青年，这是比较反常识的，但在剔除之后，两者的差异明显消除，同时，我们可以推测，是否将这些错误分组的数据归到青年组比较合适？如果将这些错分数据归为青年组，那么青年组的Physical Activity score整体水平会明显高于儿童，这是比较符合理论的。","metadata":{}},{"cell_type":"markdown","source":"# Physical Activity Questionnaire","metadata":{}},{"cell_type":"code","source":"#转化度量单位\nPhysical_features = [col for col in train.columns if 'Physical' in col]\nPhysical_features.remove('Physical-Season')\n\nlbs_to_kg = 0.453592\ninches_to_cm = 2.54\n\ntrain['Physical-Weight'] = train['Physical-Weight'] * lbs_to_kg\ntrain['Physical-Height'] = train['Physical-Height'] * inches_to_cm\ntrain['Physical-Waist_Circumference'] = train['Physical-Waist_Circumference'] * inches_to_cm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:00:36.652120Z","iopub.execute_input":"2025-01-16T07:00:36.652530Z","iopub.status.idle":"2025-01-16T07:00:36.660053Z","shell.execute_reply.started":"2025-01-16T07:00:36.652488Z","shell.execute_reply":"2025-01-16T07:00:36.658976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#根据体重和身高计算BMI并于原始数据中的Physical-BMI对比\ntrain['Calculated-BMI'] = np.where(\n    train['Physical-Weight'].isna() | train['Physical-Height'].isna(),\n    np.nan,\n    train['Physical-Weight'] / ((train['Physical-Height'] / 100) ** 2)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:00:58.075332Z","iopub.execute_input":"2025-01-16T07:00:58.075694Z","iopub.status.idle":"2025-01-16T07:00:58.082916Z","shell.execute_reply.started":"2025-01-16T07:00:58.075665Z","shell.execute_reply":"2025-01-16T07:00:58.081598Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#计算两者的误差\ntrain['BMI-Absolute-Error'] = abs(train['Calculated-BMI'] - train['Physical-BMI'])\ntrain['BMI-Relative-Error'] = np.where(\n    train['Physical-BMI'] != 0,\n    train['BMI-Absolute-Error'] / train['Physical-BMI'],\n    np.nan  # 避免除以零\n)\n# 统计误差\nmean_abs_error = train['BMI-Absolute-Error'].mean()\nstd_abs_error = train['BMI-Absolute-Error'].std()\nprint(f\"平均绝对误差: {mean_abs_error:.4f}, 绝对误差标准差: {std_abs_error:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:01:13.452137Z","iopub.execute_input":"2025-01-16T07:01:13.452512Z","iopub.status.idle":"2025-01-16T07:01:13.461723Z","shell.execute_reply.started":"2025-01-16T07:01:13.452478Z","shell.execute_reply":"2025-01-16T07:01:13.460498Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 可视化误差分布\nplt.figure(figsize=(10, 6))\nplt.hist(train['BMI-Absolute-Error'].dropna(), bins=20, alpha=0.7, edgecolor='black')\nplt.title('Distribution of Absolute Errors')\nplt.xlabel('Absolute Error')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:01:30.998750Z","iopub.execute_input":"2025-01-16T07:01:30.999166Z","iopub.status.idle":"2025-01-16T07:01:31.217505Z","shell.execute_reply.started":"2025-01-16T07:01:30.999130Z","shell.execute_reply":"2025-01-16T07:01:31.216402Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"重新计算的BMI与原始BMI的平均绝对误差和标准差均处于千分位范围，说明原始BMI数据准确性是没问题的","metadata":{}},{"cell_type":"code","source":"#异常值分类处理\nmax_columns = 9\nwith pd.option_context('display.max_columns', max_columns):\n    display(calculate_stats(train, Physical_features))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:02:13.891397Z","iopub.execute_input":"2025-01-16T07:02:13.891824Z","iopub.status.idle":"2025-01-16T07:02:13.925297Z","shell.execute_reply.started":"2025-01-16T07:02:13.891792Z","shell.execute_reply":"2025-01-16T07:02:13.923982Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Physical-BMI,Physical-Height,Physical-Weight,Physical-Waist_Circumference\n#找出所有异常值为0的列,因为在正常情况下，这些指标都不可能为0\nBCHW_with_zeros = train.query('`Physical-BMI`==0.00 | `Physical-Weight`==0.00')\nBCHW_with_zeros","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:03:02.428031Z","iopub.execute_input":"2025-01-16T07:03:02.428426Z","iopub.status.idle":"2025-01-16T07:03:02.480511Z","shell.execute_reply.started":"2025-01-16T07:03:02.428396Z","shell.execute_reply":"2025-01-16T07:03:02.479316Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BCHW_with_zeros[Physical_features].isnull().sum().sort_values(ascending=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:04:42.481775Z","iopub.execute_input":"2025-01-16T07:04:42.482189Z","iopub.status.idle":"2025-01-16T07:04:42.491420Z","shell.execute_reply.started":"2025-01-16T07:04:42.482157Z","shell.execute_reply":"2025-01-16T07:04:42.490264Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1.“0”值的特点：当Physical-Weight的值为0时，如果Physical-Height值缺失，那么Physical-BMI的值也会缺失，如果\nPhysical-Height值存在，则Physical-BMI的值为0。\n\n2.Physical-Weight为0值时，其它的Physical类型变量存在具体缺失的情况，猜测Physical-Weight值为0的数据原本可能也是缺失的，\n但是被错误标记成了0值。","metadata":{}},{"cell_type":"code","source":"#Physical-Diastolic_BP,Physical-HeartRate,Physical-Systolic_BP\nDHS_with_zeros = train.query('`Physical-Diastolic_BP`==0.00 | `Physical-Systolic_BP`==0.00')\nDHS_with_zeros\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:06:29.577659Z","iopub.execute_input":"2025-01-16T07:06:29.578015Z","iopub.status.idle":"2025-01-16T07:06:29.619381Z","shell.execute_reply.started":"2025-01-16T07:06:29.577982Z","shell.execute_reply":"2025-01-16T07:06:29.618141Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"只有一个样本存在这种情况，该样本其它Physical类型数据值正常，猜测Physical-Diastolic_BP和Physical-Systolic_BP属于统计错误，\n或者该对象并没有参加心血管相关检测","metadata":{}},{"cell_type":"code","source":"#脉压（Pulse Pressure）是收缩压（Systolic BP）和舒张压（Diastolic BP）之间的差值,正常情况下，脉压是不可能小于0的\ntrain['Physical-Pulse_Pressure'] = train['Physical-Systolic_BP'] - train['Physical-Diastolic_BP']\nPP_below_zeros = train.query('`Physical-Pulse_Pressure` < 0')\nPP_below_zeros","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:07:20.316111Z","iopub.execute_input":"2025-01-16T07:07:20.316438Z","iopub.status.idle":"2025-01-16T07:07:20.363136Z","shell.execute_reply.started":"2025-01-16T07:07:20.316412Z","shell.execute_reply":"2025-01-16T07:07:20.361936Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"存在3例不正常样本","metadata":{}},{"cell_type":"code","source":"#探究Physical变量与目标变量之间的联系\n\n# 创建子图：2行4列，留出额外空间\nfig, axes = plt.subplots(2, 4, figsize=(18, 9))  # 2行4列布局，宽度和高度调整\n\n# 遍历特征并绘制\nfor i, Physical_feature in enumerate(Physical_features):\n    ax = axes.flat[i]  # 获取对应的子图位置\n    sns.regplot(x=f'{Physical_feature}', y='PCIAT-PCIAT_Total', data=train, scatter_kws={'alpha': 0.5}, ax=ax)\n    ax.set_title(f'Regplot: {Physical_feature}')  # 设置标题\n\n# 隐藏多余的子图（第8个空白）\nif len(Physical_features) < len(axes.flat):\n    for j in range(len(Physical_features), len(axes.flat)):\n        axes.flat[j].axis('off')\n\n# 优化布局\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:08:21.349408Z","iopub.execute_input":"2025-01-16T07:08:21.349848Z","iopub.status.idle":"2025-01-16T07:08:23.992949Z","shell.execute_reply.started":"2025-01-16T07:08:21.349813Z","shell.execute_reply":"2025-01-16T07:08:23.991770Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#SII by Physical_features\n# 创建子图：2行4列，留出额外空间\nfig, axes = plt.subplots(2, 4, figsize=(18, 9))  # 2行4列布局，宽度和高度调整\n\n# 遍历特征并绘制\nfor i, Physical_feature in enumerate(Physical_features):\n    ax = axes.flat[i]  # 获取对应的子图位置\n    sns.boxplot(y=f'{Physical_feature}', x='sii', data=train, palette=\"Set3\", ax=ax)\n    ax.set_title(f'Boxplot: {Physical_feature}')  # 设置标题\n\n# 隐藏多余的子图（第8个空白）\nif len(Physical_features) < len(axes.flat):\n    for j in range(len(Physical_features), len(axes.flat)):\n        axes.flat[j].axis('off')\n\n# 优化布局\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:08:57.266242Z","iopub.execute_input":"2025-01-16T07:08:57.266625Z","iopub.status.idle":"2025-01-16T07:08:59.002927Z","shell.execute_reply.started":"2025-01-16T07:08:57.266593Z","shell.execute_reply":"2025-01-16T07:08:59.001723Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Physical变量与目标变量之间的相关关系\ndata_subset = train[Physical_features + ['PCIAT-PCIAT_Total']]\n\ncorr_matrix = data_subset.corr()\n\nplt.figure(figsize=(10, 8))\nsns.heatmap(corr_matrix, annot=True, cmap='coolwarm', fmt='.2f', vmin=-1, vmax=1)\nplt.title('Correlation Heatmap')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-16T07:09:16.416404Z","iopub.execute_input":"2025-01-16T07:09:16.416792Z","iopub.status.idle":"2025-01-16T07:09:16.888257Z","shell.execute_reply.started":"2025-01-16T07:09:16.416759Z","shell.execute_reply":"2025-01-16T07:09:16.887217Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1.不论是从箱线图还是相关系数图上来看，Physical-BMI,Physical-Height,Physical-Weight,Physical-Waist_Circumference\n与目标变量之间的关系更加密切，区分度也更高。而Physical-Diastolic_BP和Physical-Systolic_BP与目标变量之间的关系则弱很多。\n关于原因，文章初始推荐的大佬的EDA说了一点自己的想法。\n2.上述变量其实还有许多可以探索的点，上面只是讨论了不可能为0值的情况，但上述指标低于某个阈值或高于某个阈值很大可能在生理上是不可能的\n以此可以进一步考虑异常值范围","metadata":{}},{"cell_type":"markdown","source":"到此，其实只是一部分变量的EDA，本篇文章的目的其实主要是和大佬们学习分析思维，以后有时间的话，再继续更新吧。\n\n**新年快乐！！！**","metadata":{}}]}