{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":10149029,"sourceType":"datasetVersion","datasetId":6265202},{"sourceId":10151560,"sourceType":"datasetVersion","datasetId":6267167}],"dockerImageVersionId":30804,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 讀入檔案\nimport pandas as pd\ntrain = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\")\ndictionary = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv')\nprint(train.shape)\nprint(test.shape)\nprint(dictionary.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:29.705488Z","iopub.execute_input":"2024-12-13T12:56:29.705827Z","iopub.status.idle":"2024-12-13T12:56:30.195162Z","shell.execute_reply.started":"2024-12-13T12:56:29.705783Z","shell.execute_reply":"2024-12-13T12:56:30.193472Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)\ntrain","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:30.198890Z","iopub.execute_input":"2024-12-13T12:56:30.199402Z","iopub.status.idle":"2024-12-13T12:56:30.303151Z","shell.execute_reply.started":"2024-12-13T12:56:30.199348Z","shell.execute_reply":"2024-12-13T12:56:30.302023Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"SUMMARY","metadata":{}},{"cell_type":"code","source":"def display_data_statistics(data):\n    # 數值型欄位的描述性統計\n    print(\"數值欄位的描述性統計:\")\n    display(data.describe())\n\n    # 類別型欄位的值分佈\n    categorical_columns = data.select_dtypes(include=['object']).columns\n    if len(categorical_columns) > 0:\n        print(\"\\n類別欄位的值分佈:\")\n        for column in categorical_columns:\n            print(f\"\\n{column} 的值分佈:\")\n            display(data[column].value_counts(dropna=False))\n\n# 使用優化函數來顯示 train 資料集的統計數據\ndisplay_data_statistics(train)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:30.305097Z","iopub.execute_input":"2024-12-13T12:56:30.305564Z","iopub.status.idle":"2024-12-13T12:56:30.566904Z","shell.execute_reply.started":"2024-12-13T12:56:30.305513Z","shell.execute_reply":"2024-12-13T12:56:30.565753Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"看sii的各個值的分佈","metadata":{}},{"cell_type":"code","source":"# 計算每個 'sii' 分組的 PCIAT-PCIAT_Total 的最小值和最大值，並重命名列名\npciat_min_max = (\n    train.groupby('sii')['PCIAT-PCIAT_Total']\n    .agg(Minimum_Total_Score='min', Maximum_Total_Score='max')\n    .reset_index()\n)\n\n# 顯示結果\npciat_min_max\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:30.568581Z","iopub.execute_input":"2024-12-13T12:56:30.568972Z","iopub.status.idle":"2024-12-13T12:56:30.584889Z","shell.execute_reply.started":"2024-12-13T12:56:30.568936Z","shell.execute_reply":"2024-12-13T12:56:30.583470Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 找出 train 中有而 test 中沒有的列\ncolumns_not_in_test = sorted(set(train.columns) - set(test.columns))\n# 以列表形式顯示結果\ncolumns_not_in_test\nlist(columns_not_in_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:30.586864Z","iopub.execute_input":"2024-12-13T12:56:30.587334Z","iopub.status.idle":"2024-12-13T12:56:30.599703Z","shell.execute_reply.started":"2024-12-13T12:56:30.587282Z","shell.execute_reply":"2024-12-13T12:56:30.598554Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 選擇 'sii' 不為缺失值的行並篩選指定列\ntrain_with_sii = train.loc[train['sii'].notna(), columns_not_in_test]\n\n# 篩選有缺失值的行，並高亮顯示缺失值\nhighlighted_missing = (\n    train_with_sii[train_with_sii.isna().any(axis=1)]\n     .style.map(lambda x: 'background-color: #778877' if pd.isna(x) else '')\n)\n\nhighlighted_missing\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:30.601446Z","iopub.execute_input":"2024-12-13T12:56:30.601953Z","iopub.status.idle":"2024-12-13T12:56:30.742126Z","shell.execute_reply.started":"2024-12-13T12:56:30.601901Z","shell.execute_reply":"2024-12-13T12:56:30.740739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PCIAT_cols = [f'PCIAT-PCIAT_{i+1:02d}' for i in range(20)]\nrecalc_total_score = train_with_sii[PCIAT_cols].sum(\n    axis=1, skipna=True\n)\n(recalc_total_score == train_with_sii['PCIAT-PCIAT_Total']).all()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:30.746868Z","iopub.execute_input":"2024-12-13T12:56:30.748007Z","iopub.status.idle":"2024-12-13T12:56:30.762352Z","shell.execute_reply.started":"2024-12-13T12:56:30.747945Z","shell.execute_reply":"2024-12-13T12:56:30.760924Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 數值型變數與sii之間的關係","metadata":{}},{"cell_type":"code","source":"import scipy.stats as stats\n\ndef calculate_spearman_corr(data, target_var):\n    \"\"\"\n    計算數值型變數與指定目標變數之間的 Spearman 相關係數\n\n    參數：\n    - data:資料集\n    - target_var:目標變數名稱\n\n    返回：\n    - spearman_corr:與目標變數的 Spearman 相關係數\n    \"\"\"\n    # 篩選數值型變數\n    numeric_vars = data.select_dtypes(include=['float64', 'int64']).columns\n    numeric_vars = [col for col in numeric_vars if col != target_var]\n    \n    # 計算 Spearman 相關係數\n    spearman_corr = data[numeric_vars + [target_var]].corr(method='spearman')[target_var].drop(target_var)\n    return spearman_corr\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:30.764362Z","iopub.execute_input":"2024-12-13T12:56:30.764730Z","iopub.status.idle":"2024-12-13T12:56:31.231281Z","shell.execute_reply.started":"2024-12-13T12:56:30.764694Z","shell.execute_reply":"2024-12-13T12:56:31.229971Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nspearman_corr = calculate_spearman_corr(train, 'sii')\n# 繪製 Spearman 相關性的條形圖，並在每個長條旁邊添加數值\nspearman_corr_sorted = spearman_corr.sort_values()\nspearman_corr_sorted.plot(kind='barh' ,figsize=(13, 13))\n\nplt.title(\"Spearman Correlation of Variables with sii\")\nplt.xlabel(\"Spearman Correlation\")\nplt.ylabel(\"Variables\")\n\n# 在長條圖旁添加數值\nfor index, value in enumerate(spearman_corr_sorted):\n    if value > 0:\n        plt.text(value + 0.005, index, f\"{value:.3f}\")  # 正值數字放在長條右側\n    else:\n        plt.text(value - 0.05, index, f\"{value:.3f}\")  # 負值數字放在長條左側\n\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:31.233205Z","iopub.execute_input":"2024-12-13T12:56:31.233930Z","iopub.status.idle":"2024-12-13T12:56:33.338276Z","shell.execute_reply.started":"2024-12-13T12:56:31.233873Z","shell.execute_reply":"2024-12-13T12:56:33.336823Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"拆成兩張","metadata":{}},{"cell_type":"code","source":"spearman_corr = calculate_spearman_corr(train, 'sii')\nspearman_corr_sorted = spearman_corr.sort_values()\n\n# 分為兩組：大於 0.3 和小於等於 0.3\nhigh_corr = spearman_corr_sorted[spearman_corr_sorted > 0.3]\nlow_corr = spearman_corr_sorted[spearman_corr_sorted <= 0.3]\n\n# 繪製大於 0.3 的相關性條形圖\nplt.figure(figsize=(10, len(high_corr) * 0.3))\nhigh_corr.plot(kind='barh')\nplt.title(\"Spearman Correlation of Variables with sii (Correlation > 0.3)\")\nplt.xlabel(\"Spearman Correlation\")\nplt.ylabel(\"Variables\")\n\n# 在長條圖旁添加數值\nfor index, value in enumerate(high_corr):\n    plt.text(value + 0.005, index, f\"{value:.3f}\")\n\nplt.show()\n\n# 繪製小於等於 0.3 的相關性條形圖\nplt.figure(figsize=(10, len(low_corr) * 0.3))\nlow_corr.plot(kind='barh')\nplt.title(\"Spearman Correlation of Variables with sii (Correlation <= 0.3)\")\nplt.xlabel(\"Spearman Correlation\")\nplt.ylabel(\"Variables\")\n\n# 在長條圖旁添加數值\nfor index, value in enumerate(low_corr):\n    plt.text(value + 0.003 if value > 0 else value - 0.027, index, f\"{value:.3f}\")\n\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:33.339959Z","iopub.execute_input":"2024-12-13T12:56:33.340402Z","iopub.status.idle":"2024-12-13T12:56:35.247162Z","shell.execute_reply.started":"2024-12-13T12:56:33.340354Z","shell.execute_reply":"2024-12-13T12:56:35.245942Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\nnumeric_vars = train.select_dtypes(include=['float64', 'int64']).columns\n# 將 \"BIA\" 或 \"Physical\" 開頭的變數以及 sii 加入選定變數\nselected_vars = [col for col in numeric_vars if col.startswith(\"BIA\") or col.startswith(\"Physical\")]\nselected_vars.append('sii')  # 加入 sii\n\n# 計算選定變數之間的相關係數矩陣\nselected_corr = train[selected_vars].corr()\n\n# 繪製熱力圖\nplt.figure(figsize=(14, 10))\nsns.heatmap(selected_corr, annot=True, fmt=\".2f\", cmap=\"coolwarm\", center=0)\nplt.title(\"Correlation Heatmap of BIA, Physical Variables, and sii\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:35.248995Z","iopub.execute_input":"2024-12-13T12:56:35.249825Z","iopub.status.idle":"2024-12-13T12:56:37.270709Z","shell.execute_reply.started":"2024-12-13T12:56:35.249769Z","shell.execute_reply":"2024-12-13T12:56:37.269398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 篩選以 \"Fitness\"、\"FGC\" 和 \"SDS\" 開頭的變數\nselected_vars = [col for col in numeric_vars if col.startswith(\"Fitness\") or col.startswith(\"FGC\") or col.startswith(\"SDS\")]\n\n# 檢查是否包含選定變數\nif 'sii' not in selected_vars:\n    selected_vars.append('sii')  # 加入目標變數 sii\n\n# 計算選定變數之間的相關係數矩陣\nselected_corr = train[selected_vars].corr()\n\n# 繪製熱力圖\nplt.figure(figsize=(12, 10))\nsns.heatmap(selected_corr, annot=True, fmt=\".2f\", cmap=\"coolwarm\", center=0)\nplt.title(\"Correlation Heatmap of Fitness, FGC, SDS Variables, and sii\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:37.272369Z","iopub.execute_input":"2024-12-13T12:56:37.272966Z","iopub.status.idle":"2024-12-13T12:56:38.490104Z","shell.execute_reply.started":"2024-12-13T12:56:37.272925Z","shell.execute_reply":"2024-12-13T12:56:38.488812Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 類別型變數與sii之間的關係","metadata":{}},{"cell_type":"code","source":"from pprint import pprint\n\n\n# 類別型變數（object 型）與 sii 的 Kruskal-Wallis 檢定\ncategorical_vars = train.select_dtypes(include=['object']).columns\nkruskal_results = {}\n\nfor cat in categorical_vars:\n    try:\n        # 排除 NaN 值進行 Kruskal-Wallis 檢定\n        groups = [train['sii'][train[cat] == lvl].dropna() for lvl in train[cat].dropna().unique()]\n        if len(groups) > 1:\n            kruskal_results[cat] = stats.kruskal(*groups)\n        else:\n            kruskal_results[cat] = \"Not enough unique groups\"\n    except ValueError as e:\n        kruskal_results[cat] = str(e)\n\nprint(\"Kruskal-Wallis test results for categorical variables with sii:\")\npprint(kruskal_results, width=80)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:38.491736Z","iopub.execute_input":"2024-12-13T12:56:38.492125Z","iopub.status.idle":"2024-12-13T12:56:41.380011Z","shell.execute_reply.started":"2024-12-13T12:56:38.492088Z","shell.execute_reply":"2024-12-13T12:56:41.378788Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Missing value","metadata":{}},{"cell_type":"code","source":"# 算出每個變數的遺失值比率\nmissing = train.isnull().sum()/len(train) \n\n# 從大排到小\norder = missing.sort_values(ascending=False).index\nprint(missing[order])\n\nno_missing_values = train.columns[train.isnull().sum() == 0]\nprint(\"沒有遺失值的變數有：\", no_missing_values.tolist())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:41.381381Z","iopub.execute_input":"2024-12-13T12:56:41.381733Z","iopub.status.idle":"2024-12-13T12:56:41.398535Z","shell.execute_reply.started":"2024-12-13T12:56:41.381699Z","shell.execute_reply":"2024-12-13T12:56:41.397207Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[\"sii\"].isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:41.399879Z","iopub.execute_input":"2024-12-13T12:56:41.400202Z","iopub.status.idle":"2024-12-13T12:56:41.414639Z","shell.execute_reply.started":"2024-12-13T12:56:41.400170Z","shell.execute_reply":"2024-12-13T12:56:41.413266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 過濾 sii 未知的樣本\nsubtrain = train[train[\"sii\"].isnull() == False]\nsubtrain","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:41.416253Z","iopub.execute_input":"2024-12-13T12:56:41.417159Z","iopub.status.idle":"2024-12-13T12:56:41.612226Z","shell.execute_reply.started":"2024-12-13T12:56:41.417096Z","shell.execute_reply":"2024-12-13T12:56:41.611153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 算出每個變數的遺失值比率\nmissing = subtrain.isnull().sum()/len(subtrain)\n# 從大排到小\norder = missing.sort_values(ascending=False).index\n\n# 畫出遺失值比率\nmissing[order].plot(kind=\"barh\", \n                    figsize=(9, 18),\n                    color=\"skyblue\",\n                    alpha=0.8)\nplt.title(\"Proportion of missing values ​​for variables with target({}sample)\".format(len(subtrain)), fontsize=16)\n\n# 設定 x 軸和 y 軸刻度字體大小\nplt.xticks(fontsize=12)\nplt.yticks(fontsize=12)\nplt.gca().invert_yaxis()\n\n# 顯示網格線以輔助對齊\nplt.grid(axis=\"x\", linestyle=\"--\", alpha=0.5)\nplt.tight_layout()\n# 顯示圖表\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:41.613797Z","iopub.execute_input":"2024-12-13T12:56:41.614833Z","iopub.status.idle":"2024-12-13T12:56:42.883402Z","shell.execute_reply.started":"2024-12-13T12:56:41.614778Z","shell.execute_reply":"2024-12-13T12:56:42.882281Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"subtrain中有缺失值>40%的欄位","metadata":{}},{"cell_type":"code","source":"# 計算每個變數的缺失值比率\nmissing_ratio = subtrain.isnull().mean()\n\n# 篩選出缺失值比率超過 40% 的變數\nhigh_missing_columns = missing_ratio[missing_ratio > 0.4].index\n\n# 顯示結果\nprint(\"缺失值超過 40% 的變數有：\", high_missing_columns.tolist())\n\nlen(high_missing_columns)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:42.885076Z","iopub.execute_input":"2024-12-13T12:56:42.885552Z","iopub.status.idle":"2024-12-13T12:56:42.899050Z","shell.execute_reply.started":"2024-12-13T12:56:42.885497Z","shell.execute_reply":"2024-12-13T12:56:42.897673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 刪除這些變數\nsubtrain_cleaned = subtrain.drop(columns=high_missing_columns)\n\n# 確認變數已被刪除\nprint(\"subtrain_cleaned 的維度：\", subtrain_cleaned.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:42.900642Z","iopub.execute_input":"2024-12-13T12:56:42.901079Z","iopub.status.idle":"2024-12-13T12:56:42.913099Z","shell.execute_reply.started":"2024-12-13T12:56:42.901010Z","shell.execute_reply":"2024-12-13T12:56:42.911941Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 找出有缺失值的變數\nmissing_columns = subtrain_cleaned.columns[subtrain_cleaned.isnull().any()]\n\n# 列出這些變數的資料型態\nmissing_data_types = subtrain_cleaned[missing_columns].dtypes\n\n# 顯示完整結果\nprint(\"有缺失值的變數及其資料型態：\")\nprint(missing_data_types.to_string())\nlen(missing_columns)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:42.914682Z","iopub.execute_input":"2024-12-13T12:56:42.915088Z","iopub.status.idle":"2024-12-13T12:56:42.933971Z","shell.execute_reply.started":"2024-12-13T12:56:42.915052Z","shell.execute_reply":"2024-12-13T12:56:42.932592Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 找出 subtrain_cleaned 中有缺失值的變數\nmissing_columns = subtrain_cleaned.columns[subtrain_cleaned.isnull().any()]\n\n# 篩選出 data_dict 中這些變數的解釋\nmissing_columns_dict = dictionary[dictionary['Field'].isin(missing_columns)]\n# 顯示結果\nprint(\"有缺失值的變數解釋表格：\")\nmissing_columns_dict.drop(columns=['Instrument'])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:42.935495Z","iopub.execute_input":"2024-12-13T12:56:42.935935Z","iopub.status.idle":"2024-12-13T12:56:42.959303Z","shell.execute_reply.started":"2024-12-13T12:56:42.935893Z","shell.execute_reply":"2024-12-13T12:56:42.957905Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 有缺失值的變數及其資料型態：\n\n#### 填補unknown:\nCGAS-Season                                object\nPhysical-Season                            object\nFGC-Season                                 object\nBIA-Season                                 object\nSDS-Season                                 object\nPreInt_EduHx-Season                        object\n\n#### 用mice填補缺失值:\nBIA-BIA_BMC                               float64\nBIA-BIA_BMI                               float64\nBIA-BIA_BMR                               float64\nBIA-BIA_DEE                               float64\nBIA-BIA_ECW                               float64\nBIA-BIA_FFM                               float64\nBIA-BIA_FFMI                              float64\nBIA-BIA_FMI                               float64\nBIA-BIA_Fat                               float64\nBIA-BIA_ICW                               float64\nBIA-BIA_LDM                               float64\nBIA-BIA_LST                               float64\nBIA-BIA_SMM                               float64\nBIA-BIA_TBW                               float64\n\n\n#### 用mice填補缺失值:(zone不要\nFGC-FGC_CU                                float64\nFGC-FGC_CU_Zone                           float64\nFGC-FGC_PU                                float64\nFGC-FGC_PU_Zone                           float64\nFGC-FGC_SRL                               float64\nFGC-FGC_SRL_Zone                          float64\nFGC-FGC_SRR                               float64\nFGC-FGC_SRR_Zone                          float64\nFGC-FGC_TL                                float64\nFGC-FGC_TL_Zone                           float64\n\n#### 用mice填補缺失值:\nPCIAT-PCIAT_01                            float64\nPCIAT-PCIAT_02                            float64\nPCIAT-PCIAT_03                            float64\nPCIAT-PCIAT_04                            float64\nPCIAT-PCIAT_05                            float64\nPCIAT-PCIAT_06                            float64\nPCIAT-PCIAT_07                            float64\nPCIAT-PCIAT_08                            float64\nPCIAT-PCIAT_09                            float64\nPCIAT-PCIAT_10                            float64\nPCIAT-PCIAT_11                            float64\nPCIAT-PCIAT_12                            float64\nPCIAT-PCIAT_13                            float64\nPCIAT-PCIAT_14                            float64\nPCIAT-PCIAT_15                            float64\nPCIAT-PCIAT_16                            float64\nPCIAT-PCIAT_17                            float64\nPCIAT-PCIAT_18                            float64\nPCIAT-PCIAT_19                            float64\nPCIAT-PCIAT_20                            float64\n\n#### 用mice填補缺失值:\nPhysical-Diastolic_BP                     float64\nPhysical-HeartRate                        float64\nPhysical-Systolic_BP                      float64\n\n#### 隨機森林處理:\nPhysical-BMI                              float64\nPhysical-Height                           float64\nPhysical-Weight                           float64\n\n#### 看偏態、峰度決定均值or中位數填補:\nSDS-SDS_Total_Raw                         float64\nSDS-SDS_Total_T                           float64\nCGAS-CGAS_Score                           float64\nBIA-BIA_Activity_Level_num                float64\n\n#### 用眾數:\nPreInt_EduHx-computerinternet_hoursday    float64\n\n#### 用BMI分組後的眾數:\nBIA-BIA_Frame_num                         float64","metadata":{}},{"cell_type":"code","source":"missing_summary = subtrain_cleaned[['Physical-BMI', 'BIA-BIA_BMI']].isnull()\n\n# 查看重疊遺失的行數\noverlap_missing = missing_summary.all(axis=1).sum()\nprint(f\"兩個變數均有遺失值的行數: {overlap_missing}\")\n\n# 查看只有其中一個變數有遺失值的行數\nphysical_missing_only = (missing_summary['Physical-BMI'] & ~missing_summary['BIA-BIA_BMI']).sum()\nbia_missing_only = (~missing_summary['Physical-BMI'] & missing_summary['BIA-BIA_BMI']).sum()\n\nprint(f\"只有 Physical-BMI 遺失值的行數: {physical_missing_only}\")\nprint(f\"只有 BIA-BIA_BMI 遺失值的行數: {bia_missing_only}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:42.967156Z","iopub.execute_input":"2024-12-13T12:56:42.967905Z","iopub.status.idle":"2024-12-13T12:56:42.979043Z","shell.execute_reply.started":"2024-12-13T12:56:42.967824Z","shell.execute_reply":"2024-12-13T12:56:42.977776Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 篩選出 Physical-BMI 為空且 BIA-BIA_BMI 有值的行\nphysical_missing_bia_present = subtrain_cleaned[subtrain_cleaned['Physical-BMI'].isnull() & subtrain_cleaned['BIA-BIA_BMI'].notnull()]\n\n# 查看篩選後的行數\nprint(f\"Physical-BMI 沒有值但 BIA-BIA_BMI 有值的行數: {len(physical_missing_bia_present)}\")\n\n# 顯示部分數據\nprint(physical_missing_bia_present.loc[:, ['Physical-BMI', 'BIA-BIA_BMI']])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:42.980549Z","iopub.execute_input":"2024-12-13T12:56:42.980960Z","iopub.status.idle":"2024-12-13T12:56:43.004397Z","shell.execute_reply.started":"2024-12-13T12:56:42.980922Z","shell.execute_reply":"2024-12-13T12:56:43.002958Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"因為Physical-Weight有0，所以改成nan","metadata":{}},{"cell_type":"code","source":"import numpy as np\n# 將 Physical-Weight 為 0 的值設為 NaN\nsubtrain_cleaned.loc[subtrain_cleaned['Physical-Weight'] == 0, 'Physical-Weight'] = np.nan\n\n# 確認修改結果\nprint(\"將 Physical-Weight 為 0 的值設為 NaN 已完成。\")\nprint(subtrain_cleaned['Physical-Weight'].isnull().sum())  # 檢查遺失值數量\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:43.006201Z","iopub.execute_input":"2024-12-13T12:56:43.006926Z","iopub.status.idle":"2024-12-13T12:56:43.020544Z","shell.execute_reply.started":"2024-12-13T12:56:43.006873Z","shell.execute_reply":"2024-12-13T12:56:43.019242Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 篩選出體重和身高都有值的數據\nweight_height_available = subtrain_cleaned[(subtrain_cleaned['Physical-Weight'].notnull()) & \n                                           (subtrain_cleaned['Physical-Height'].notnull())]\n\n# 檢查在體重和身高都有值的情況下，BMI 是否有缺失值\nbmi_missing_count = weight_height_available['Physical-BMI'].isnull().sum()\n\nif bmi_missing_count == 0:\n    print(\"在體重和身高都有值的情況下，BMI 一定有值。\")\nelse:\n    print(f\"在體重和身高都有值的情況下，BMI 仍然有 {bmi_missing_count} 個缺失值。\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:43.022171Z","iopub.execute_input":"2024-12-13T12:56:43.022604Z","iopub.status.idle":"2024-12-13T12:56:43.037604Z","shell.execute_reply.started":"2024-12-13T12:56:43.022568Z","shell.execute_reply":"2024-12-13T12:56:43.036283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 計算身高和體重同時缺失的數量\nheight_weight_both_missing = subtrain_cleaned[(subtrain_cleaned['Physical-Height'].isnull()) & \n                                              (subtrain_cleaned['Physical-Weight'].isnull())].shape[0]\n\nprint(f\"身高和體重同時遺失的數量：{height_weight_both_missing}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:43.039017Z","iopub.execute_input":"2024-12-13T12:56:43.039384Z","iopub.status.idle":"2024-12-13T12:56:43.053457Z","shell.execute_reply.started":"2024-12-13T12:56:43.039327Z","shell.execute_reply":"2024-12-13T12:56:43.052158Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\n\n# 假設 subtrain_cleaned 是你的原始 DataFrame，並且 Basic_Demos-Age 和 Basic_Demos-Sex 沒有缺失值\n# 分離身高缺失和非缺失數據\nheight_missing = subtrain_cleaned[subtrain_cleaned['Physical-Height'].isnull()]\nheight_non_missing = subtrain_cleaned[subtrain_cleaned['Physical-Height'].notnull()]\n\n# 使用非缺失數據訓練模型來預測身高\nX_train_height = height_non_missing[['Basic_Demos-Age', 'Basic_Demos-Sex']]  # 特徵\ny_train_height = height_non_missing['Physical-Height']  # 目標變數\n\n# 建立隨機森林回歸模型並進行訓練\nheight_model = RandomForestRegressor(random_state=0)\nheight_model.fit(X_train_height, y_train_height)\n\n# 使用訓練好的模型來預測缺失的身高數據\nX_pred_height = height_missing[['Basic_Demos-Age', 'Basic_Demos-Sex']]\nsubtrain_cleaned.loc[height_missing.index, 'Physical-Height'] = height_model.predict(X_pred_height)\n\n\n# 分離體重缺失和非缺失數據\nweight_missing = subtrain_cleaned[subtrain_cleaned['Physical-Weight'].isnull()]\nweight_non_missing = subtrain_cleaned[subtrain_cleaned['Physical-Weight'].notnull()]\n\n# 使用非缺失數據訓練模型來預測體重\nX_train_weight = weight_non_missing[['Basic_Demos-Age', 'Basic_Demos-Sex']]  # 特徵\ny_train_weight = weight_non_missing['Physical-Weight']  # 目標變數\n\n# 建立隨機森林回歸模型並進行訓練\nweight_model = RandomForestRegressor(random_state=0)\nweight_model.fit(X_train_weight, y_train_weight)\n\n# 使用訓練好的模型來預測缺失的體重數據\nX_pred_weight = weight_missing[['Basic_Demos-Age', 'Basic_Demos-Sex']]\nsubtrain_cleaned.loc[weight_missing.index, 'Physical-Weight'] = weight_model.predict(X_pred_weight)\n\n# 查看 Physical-Height 和 Physical-Weight 列的缺失值數量\nprint(\"Physical-Height 缺失值數量：\", subtrain_cleaned['Physical-Height'].isnull().sum())\nprint(\"Physical-Weight 缺失值數量：\", subtrain_cleaned['Physical-Weight'].isnull().sum())\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:43.055088Z","iopub.execute_input":"2024-12-13T12:56:43.055449Z","iopub.status.idle":"2024-12-13T12:56:43.554021Z","shell.execute_reply.started":"2024-12-13T12:56:43.055415Z","shell.execute_reply":"2024-12-13T12:56:43.552829Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary_table = subtrain_cleaned[['Physical-Height', 'Physical-Weight']].describe()\nsummary_table\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:43.555461Z","iopub.execute_input":"2024-12-13T12:56:43.555820Z","iopub.status.idle":"2024-12-13T12:56:43.576411Z","shell.execute_reply.started":"2024-12-13T12:56:43.555786Z","shell.execute_reply":"2024-12-13T12:56:43.574922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 將身高從英吋轉換為米\nsubtrain_cleaned['Physical-Height-m'] = subtrain_cleaned['Physical-Height'] * 0.0254\n\n# 將體重從磅轉換為公斤\nsubtrain_cleaned['Physical-Weight-kg'] = subtrain_cleaned['Physical-Weight'] * 0.453592\n\n# 計算 BMI\nsubtrain_cleaned['BMI'] = subtrain_cleaned['Physical-Weight-kg'] / (subtrain_cleaned['Physical-Height-m'] ** 2)\n\n# 檢查計算結果\nprint(subtrain_cleaned[['Physical-Height', 'Physical-Height-m', \n                        'Physical-Weight', 'Physical-Weight-kg', 'BMI']].head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:43.578319Z","iopub.execute_input":"2024-12-13T12:56:43.579517Z","iopub.status.idle":"2024-12-13T12:56:43.593343Z","shell.execute_reply.started":"2024-12-13T12:56:43.579457Z","shell.execute_reply":"2024-12-13T12:56:43.591767Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 繪製身高的分布\nsubtrain_cleaned['Physical-Height-m'].plot(kind='hist', bins=30, alpha=0.7,edgecolor = 'black', label='Height')\nplt.title('Physical-Height Distribution')\nplt.xlabel('Height')\nplt.legend()\nplt.show()\n\n# 繪製體重的分布\nsubtrain_cleaned['Physical-Weight-kg'].plot(kind='hist', bins=30, alpha=0.7,edgecolor = 'black', label='Weight', color='orange')\nplt.title('Physical-Weight Distribution')\nplt.xlabel('Weight')\nplt.legend()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:43.595023Z","iopub.execute_input":"2024-12-13T12:56:43.595396Z","iopub.status.idle":"2024-12-13T12:56:44.145337Z","shell.execute_reply.started":"2024-12-13T12:56:43.595360Z","shell.execute_reply":"2024-12-13T12:56:44.143947Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 將Season插補為眾數","metadata":{}},{"cell_type":"code","source":"# 選取包含 \"Season\" 的變數\nseason_columns = [col for col in subtrain_cleaned.columns if \"Season\" in col]\n\n# 將這些變數中的缺失值填補為該變數的眾數\nfor col in season_columns:\n    mode_value = subtrain_cleaned[col].mode()[0]  # 計算眾數\n    subtrain_cleaned[col].fillna(mode_value, inplace=True)\n\n# 檢查結果\nprint(\"已將 Season 相關變數的遺失值填補為眾數:\")\nprint(subtrain_cleaned[season_columns].head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:44.146834Z","iopub.execute_input":"2024-12-13T12:56:44.147236Z","iopub.status.idle":"2024-12-13T12:56:44.167771Z","shell.execute_reply.started":"2024-12-13T12:56:44.147200Z","shell.execute_reply":"2024-12-13T12:56:44.166687Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\n\n# 選擇需要填補的變數\npciat_columns = [\n    'PCIAT-PCIAT_01', 'PCIAT-PCIAT_02', 'PCIAT-PCIAT_03', 'PCIAT-PCIAT_04', 'PCIAT-PCIAT_05',\n    'PCIAT-PCIAT_06', 'PCIAT-PCIAT_07', 'PCIAT-PCIAT_08', 'PCIAT-PCIAT_09', 'PCIAT-PCIAT_10',\n    'PCIAT-PCIAT_11', 'PCIAT-PCIAT_12', 'PCIAT-PCIAT_13', 'PCIAT-PCIAT_14', 'PCIAT-PCIAT_15',\n    'PCIAT-PCIAT_16', 'PCIAT-PCIAT_17', 'PCIAT-PCIAT_18', 'PCIAT-PCIAT_19', 'PCIAT-PCIAT_20'\n]\n\n# 初始化 MICE 插補器\nimputer = IterativeImputer(random_state=0, max_iter=20)\n\n# 對選定的變數進行插補\nimputed_data = imputer.fit_transform(subtrain_cleaned[pciat_columns])\n\n# 將插補結果四捨五入並轉換為整數類型\nsubtrain_cleaned[pciat_columns] = imputed_data.round()\n\n# 檢查插補後的結果\nprint(\"插補完成且轉為整數後的數據：\")\nprint(subtrain_cleaned[pciat_columns].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:44.169251Z","iopub.execute_input":"2024-12-13T12:56:44.169618Z","iopub.status.idle":"2024-12-13T12:56:52.831216Z","shell.execute_reply.started":"2024-12-13T12:56:44.169582Z","shell.execute_reply":"2024-12-13T12:56:52.829273Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\n\n# 選擇需要填補的變數\nbia_columns = [\n    'BIA-BIA_BMC', 'BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW',\n    'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_ICW',\n    'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW'\n]\n\n# 初始化插補器\nimputer = IterativeImputer(random_state=0, max_iter=20)\n\n# 對選定的變數進行插補\nsubtrain_cleaned[bia_columns] = imputer.fit_transform(subtrain_cleaned[bia_columns])\n\n# 檢查插補後的結果\nprint(\"插補完成後的數據：\")\nprint(subtrain_cleaned[bia_columns].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:52.832490Z","iopub.execute_input":"2024-12-13T12:56:52.832931Z","iopub.status.idle":"2024-12-13T12:56:54.004525Z","shell.execute_reply.started":"2024-12-13T12:56:52.832886Z","shell.execute_reply":"2024-12-13T12:56:54.003418Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\n\n# 選擇需要填補的變數\nFGC_columns = [\n    \"FGC-FGC_CU\", \n    \"FGC-FGC_PU\", \n    \"FGC-FGC_SRL\",\n    \"FGC-FGC_SRR\", \n    \"FGC-FGC_TL\"\n]\n\n# 初始化插補器\nimputer = IterativeImputer(random_state=0, max_iter=20)\n\n# 對選定的變數進行插補\nsubtrain_cleaned[FGC_columns] = imputer.fit_transform(subtrain_cleaned[FGC_columns])\n\n# 四捨五入到整數\nsubtrain_cleaned[FGC_columns] = subtrain_cleaned[FGC_columns].round(0)\n\n# 檢查插補後的結果\nprint(\"插補完成後並四捨五入的數據：\")\nprint(subtrain_cleaned[FGC_columns].head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:54.006629Z","iopub.execute_input":"2024-12-13T12:56:54.007243Z","iopub.status.idle":"2024-12-13T12:56:54.193357Z","shell.execute_reply.started":"2024-12-13T12:56:54.007175Z","shell.execute_reply":"2024-12-13T12:56:54.192070Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\n\n# 選擇需要填補的變數\nphy_columns = [\n    \"Physical-Diastolic_BP\", \n    \"Physical-HeartRate\", \n    \"Physical-Systolic_BP\"\n]\n\n# 初始化插補器\nimputer = IterativeImputer(random_state=0, max_iter=20)\n\n# 對選定的變數進行插補\nsubtrain_cleaned[phy_columns] = imputer.fit_transform(subtrain_cleaned[phy_columns])\n\n# 四捨五入到整數\nsubtrain_cleaned[phy_columns] = subtrain_cleaned[phy_columns].round(0)\n\n# 檢查插補後的結果\nprint(\"插補完成後並四捨五入的數據：\")\nprint(subtrain_cleaned[phy_columns].head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:54.195347Z","iopub.execute_input":"2024-12-13T12:56:54.195863Z","iopub.status.idle":"2024-12-13T12:56:54.244369Z","shell.execute_reply.started":"2024-12-13T12:56:54.195786Z","shell.execute_reply":"2024-12-13T12:56:54.243029Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 定義 BMI 分群的條件和標籤\nbins = [0, 18.5, 24, 27, float('inf')]  # 定義分群的邊界\nlabels = ['過輕', '健康體重', '過重', '肥胖']  # 定義分群的標籤\n\n# 使用 pandas 的 cut 方法進行分群\nsubtrain_cleaned['BMI_Group'] = pd.cut(subtrain_cleaned['BMI'], bins=bins, labels=labels, right=False)\n\n# 檢查分群結果\nprint(subtrain_cleaned[['BMI', 'BMI_Group']].head())\n\n# 統計每個群體的人數\ngroup_counts = subtrain_cleaned['BMI_Group'].value_counts()\nprint(\"各群體人數：\")\nprint(group_counts)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:54.246003Z","iopub.execute_input":"2024-12-13T12:56:54.247267Z","iopub.status.idle":"2024-12-13T12:56:54.262175Z","shell.execute_reply.started":"2024-12-13T12:56:54.247221Z","shell.execute_reply":"2024-12-13T12:56:54.260677Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"subtrain_cleaned.groupby(\"BMI_Group\",observed=False)[\"BIA-BIA_Frame_num\"].value_counts().unstack()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:54.263973Z","iopub.execute_input":"2024-12-13T12:56:54.264471Z","iopub.status.idle":"2024-12-13T12:56:54.304305Z","shell.execute_reply.started":"2024-12-13T12:56:54.264418Z","shell.execute_reply":"2024-12-13T12:56:54.302659Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 計算每組的眾數，顯式設置 observed=False\nmode_per_group = (\n    subtrain_cleaned.groupby(\"BMI_Group\", observed=False)[\"BIA-BIA_Frame_num\"]\n    .agg(lambda x: x.mode().iloc[0] if not x.mode().empty else None)\n)\n\nprint(\"每組的眾數:\")\nprint(mode_per_group)\n\n# 定義填補函數\ndef fill_with_mode(row):\n    if pd.isnull(row[\"BIA-BIA_Frame_num\"]):  # 如果缺失值\n        return mode_per_group[row[\"BMI_Group\"]]  # 使用對應的眾數填補\n    return row[\"BIA-BIA_Frame_num\"]  # 否則返回原值\n\n# 使用 apply 填補缺失值\nsubtrain_cleaned[\"BIA-BIA_Frame_num\"] = subtrain_cleaned.apply(fill_with_mode, axis=1)\n\n# 確認填補後是否還有缺失值\nprint(\"填補完成，剩餘缺失值數量:\")\nprint(subtrain_cleaned[\"BIA-BIA_Frame_num\"].isnull().sum())\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:54.306118Z","iopub.execute_input":"2024-12-13T12:56:54.306506Z","iopub.status.idle":"2024-12-13T12:56:54.366788Z","shell.execute_reply.started":"2024-12-13T12:56:54.306467Z","shell.execute_reply":"2024-12-13T12:56:54.365336Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"變數與CGAS-CGAS_Score的相關性","metadata":{}},{"cell_type":"code","source":"spearman_corr2 = calculate_spearman_corr(train, 'CGAS-CGAS_Score')\n# 繪製 Spearman 相關性的條形圖，並在每個長條旁邊添加數值\nspearman_corr_sorted2 = spearman_corr2.sort_values()\nspearman_corr_sorted2.plot(kind='barh' ,figsize=(13, 13))\n\nplt.title(\"Spearman Correlation of Variables with CGAS-CGAS_Score\")\nplt.xlabel(\"Spearman Correlation\")\nplt.ylabel(\"Variables\")\n\n# 在長條圖旁添加數值\nfor index, value in enumerate(spearman_corr_sorted2):\n    if value > 0:\n        plt.text(value + 0.005, index, f\"{value:.3f}\")  # 正值數字放在長條右側\n    else:\n        plt.text(value - 0.02, index, f\"{value:.3f}\")  # 負值數字放在長條左側\n\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:54.368521Z","iopub.execute_input":"2024-12-13T12:56:54.369039Z","iopub.status.idle":"2024-12-13T12:56:56.170323Z","shell.execute_reply.started":"2024-12-13T12:56:54.368985Z","shell.execute_reply":"2024-12-13T12:56:56.168921Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 定義要檢查的變數名稱列表\nfgc_columns = [\n    'SDS-SDS_Total_Raw',\n    'SDS-SDS_Total_T',\n    'CGAS-CGAS_Score',\n    'BIA-BIA_Activity_Level_num'\n]\n\n# 計算每個變數的偏態和峰度，並決定填補方式\nfor col in fgc_columns:\n    skewness = subtrain_cleaned[col].skew()\n    kurtosis = subtrain_cleaned[col].kurt()\n    \n    print(f\"變數: {col}\")\n    print(f\"  偏態: {skewness}\")\n    print(f\"  峰度: {kurtosis}\")\n    \n    # 判斷插補方式並進行填補\n    if abs(skewness) < 0.5:\n        print(\"  建議填補方式: 使用均值填補（分佈相對對稱）\")\n        # 使用均值填補\n        mean_value = round(subtrain_cleaned[col].mean())\n        subtrain_cleaned[col] = subtrain_cleaned[col].fillna(mean_value)\n        print(f\"  使用均值填補，填補值: {mean_value}\")\n    else:\n        print(\"  建議填補方式: 使用中位數填補（分佈偏斜）\")\n        # 使用中位數填補\n        median_value = subtrain_cleaned[col].median()\n        subtrain_cleaned[col] = subtrain_cleaned[col].fillna(median_value)\n        print(f\"  使用中位數填補，填補值: {median_value}\")\n    print(\"-\" * 30)\n\n# 確認填補後是否還有缺失值\nprint(\"填補完成，剩餘缺失值情況：\")\nprint(subtrain_cleaned[fgc_columns].isnull().sum())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:56.172244Z","iopub.execute_input":"2024-12-13T12:56:56.172739Z","iopub.status.idle":"2024-12-13T12:56:56.194377Z","shell.execute_reply.started":"2024-12-13T12:56:56.172689Z","shell.execute_reply":"2024-12-13T12:56:56.192969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 計算列的眾數\nmode_value = subtrain_cleaned['PreInt_EduHx-computerinternet_hoursday'].mode()\n\n# 檢查是否有眾數（防止空列導致問題）\nif not mode_value.empty:\n    mode_value = mode_value.iloc[0]  # 提取第一個眾數\n    print(f\"PreInt_EduHx-computerinternet_hoursday 的眾數為: {mode_value}\")\n\n    # 填補缺失值\n    subtrain_cleaned['PreInt_EduHx-computerinternet_hoursday'].fillna(mode_value, inplace=False)\n\n    # 確認填補後的缺失值數量\n    print(\"填補完成，剩餘缺失值數量:\")\n    print(subtrain_cleaned['PreInt_EduHx-computerinternet_hoursday'].isnull().sum())\nelse:\n    print(\"PreInt_EduHx-computerinternet_hoursday 沒有眾數，請檢查數據。\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:56.196029Z","iopub.execute_input":"2024-12-13T12:56:56.196412Z","iopub.status.idle":"2024-12-13T12:56:56.216030Z","shell.execute_reply.started":"2024-12-13T12:56:56.196373Z","shell.execute_reply":"2024-12-13T12:56:56.214587Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 找出有缺失值的變數\nmissing_columns = subtrain_cleaned.columns[subtrain_cleaned.isnull().any()]\n\n# 列出這些變數的資料型態\nmissing_data_types = subtrain_cleaned[missing_columns].dtypes\n\n# 顯示完整結果\nprint(\"有缺失值的變數及其資料型態：\")\nprint(missing_data_types.to_string())\nlen(missing_columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:56.217983Z","iopub.execute_input":"2024-12-13T12:56:56.218495Z","iopub.status.idle":"2024-12-13T12:56:56.242200Z","shell.execute_reply.started":"2024-12-13T12:56:56.218430Z","shell.execute_reply":"2024-12-13T12:56:56.240949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 定義要刪除的變數列表\ncolumns_to_drop = [\n    'Physical-BMI',\n    'FGC-FGC_CU_Zone',\n    'FGC-FGC_PU_Zone',\n    'FGC-FGC_SRL_Zone',\n    'FGC-FGC_SRR_Zone',\n    'FGC-FGC_TL_Zone',\n    'BIA-BIA_BMI',\n    'BMI_Group'\n]\n\n# 創建一個新的 DataFrame，刪除指定變數\nnew_subtrain_cleaned = subtrain_cleaned.drop(columns=columns_to_drop)\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:56.244052Z","iopub.execute_input":"2024-12-13T12:56:56.244491Z","iopub.status.idle":"2024-12-13T12:56:56.259875Z","shell.execute_reply.started":"2024-12-13T12:56:56.244444Z","shell.execute_reply":"2024-12-13T12:56:56.258561Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"newtrain = new_subtrain_cleaned\nnewtrain","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:56.261569Z","iopub.execute_input":"2024-12-13T12:56:56.262089Z","iopub.status.idle":"2024-12-13T12:56:56.343800Z","shell.execute_reply.started":"2024-12-13T12:56:56.262037Z","shell.execute_reply":"2024-12-13T12:56:56.342548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n# 創建 StandardScaler 實例\nscaler = StandardScaler()\n\n# 擬合並轉換數據\nX_scaled = pd.DataFrame(scaler.fit_transform(newtrain.select_dtypes(include=[float])), columns=newtrain.select_dtypes(include=[float]).columns)\n\nimport matplotlib.pyplot as plt\n\n# 設置圖形大小\nplt.figure(figsize=(12, 10))\n\n# 使用 DataFrame 的 boxplot 方法直接繪製箱形圖\nX_scaled.boxplot(vert=False)\n\n\n# 添加標題和軸標籤\nplt.title(\"Box Plot of Each Feature (Standardized)\")\nplt.ylabel(\"Value (Standardized)\")\nplt.xlabel(\"Features\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:56.345377Z","iopub.execute_input":"2024-12-13T12:56:56.345995Z","iopub.status.idle":"2024-12-13T12:56:57.330656Z","shell.execute_reply.started":"2024-12-13T12:56:56.345953Z","shell.execute_reply":"2024-12-13T12:56:57.329538Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 定義一個函數檢測離群值\ndef detect_outliers_iqr(df):\n    outliers = {}\n    for col in df.select_dtypes(include=['float64', 'int64']).columns:  # 選擇數值型變數\n        Q1 = df[col].quantile(0.25)\n        Q3 = df[col].quantile(0.75)\n        IQR = Q3 - Q1\n        lower_bound = Q1 - 1.5 * IQR\n        upper_bound = Q3 + 1.5 * IQR\n        outlier_values = df[(df[col] < lower_bound) | (df[col] > upper_bound)][col]\n        if not outlier_values.empty:\n            outliers[col] = outlier_values\n    return outliers\n\n# 檢測離群值\noutliers_iqr = detect_outliers_iqr(newtrain)\n\n# 輸出結果\nif outliers_iqr:\n    print(\"檢測到的離群值:\")\n    for col, outlier_values in outliers_iqr.items():\n        print(f\"{col}: {len(outlier_values)} 個離群值\")\n        print(f\"離群值為: {outlier_values.tolist()}\")\n        print(\"-\" * 50)\nelse:\n    print(\"沒有檢測到任何離群值。\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:57.332370Z","iopub.execute_input":"2024-12-13T12:56:57.332859Z","iopub.status.idle":"2024-12-13T12:56:57.502628Z","shell.execute_reply.started":"2024-12-13T12:56:57.332794Z","shell.execute_reply":"2024-12-13T12:56:57.501365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 找出滿足條件的行索引\nrows_to_update = newtrain[\n    newtrain['Physical-Diastolic_BP'] > newtrain['Physical-Systolic_BP']\n].index\n\n# 計算兩個變數的中位數\nmedian_diastolic = newtrain['Physical-Diastolic_BP'].median()\nmedian_systolic = newtrain['Physical-Systolic_BP'].median()\n\nprint(f\"Physical-Diastolic_BP 中位數: {median_diastolic}\")\nprint(f\"Physical-Systolic_BP 中位數: {median_systolic}\")\n\n# 將這兩個變數改為中位數\nnewtrain.loc[rows_to_update, 'Physical-Diastolic_BP'] = median_diastolic\nnewtrain.loc[rows_to_update, 'Physical-Systolic_BP'] = median_systolic\n\n# 確認結果\nprint(\"修改後的數據：\")\nprint(newtrain.loc[rows_to_update, ['Physical-Diastolic_BP', 'Physical-Systolic_BP']])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:57.504302Z","iopub.execute_input":"2024-12-13T12:56:57.504788Z","iopub.status.idle":"2024-12-13T12:56:57.525580Z","shell.execute_reply.started":"2024-12-13T12:56:57.504739Z","shell.execute_reply":"2024-12-13T12:56:57.524241Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 定義需要修改的行和列\nrows_to_update = [2221, 2417]\ncolumns_to_update = [\n    'BIA-BIA_BMC', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW',\n    'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat',\n    'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW'\n]\n\n# 計算每列的中位數\nmedians = newtrain[columns_to_update].median()\nprint(\"各列中位數：\")\nprint(medians)\n\n# 強制將指定行的這些列替換為中位數\nnewtrain.loc[rows_to_update, columns_to_update] = medians.values\n\n# 查看修改結果\nprint(f\"已成功將第 {rows_to_update} 行的指定列改為中位數：\")\nprint(newtrain.loc[rows_to_update].filter(regex='BIA'))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:57.527119Z","iopub.execute_input":"2024-12-13T12:56:57.527479Z","iopub.status.idle":"2024-12-13T12:56:57.557942Z","shell.execute_reply.started":"2024-12-13T12:56:57.527447Z","shell.execute_reply":"2024-12-13T12:56:57.556668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n# 創建 StandardScaler 實例\nscaler = StandardScaler()\n\n# 擬合並轉換數據\nX_scaled = pd.DataFrame(scaler.fit_transform(newtrain.select_dtypes(include=[float])), columns=newtrain.select_dtypes(include=[float]).columns)\n\nimport matplotlib.pyplot as plt\n\n# 設置圖形大小\nplt.figure(figsize=(12, 10))\n\n# 使用 DataFrame 的 boxplot 方法直接繪製箱形圖\nX_scaled.boxplot(vert=False)\n\n\n# 添加標題和軸標籤\nplt.title(\"Box Plot of Each Feature (Standardized)\")\nplt.ylabel(\"Value (Standardized)\")\nplt.xlabel(\"Features\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:57.559358Z","iopub.execute_input":"2024-12-13T12:56:57.559684Z","iopub.status.idle":"2024-12-13T12:56:58.730417Z","shell.execute_reply.started":"2024-12-13T12:56:57.559653Z","shell.execute_reply":"2024-12-13T12:56:58.729232Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report, accuracy_score\nimport scipy.stats as stats\nimport keras\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:56:58.731933Z","iopub.execute_input":"2024-12-13T12:56:58.732276Z","iopub.status.idle":"2024-12-13T12:57:02.003242Z","shell.execute_reply.started":"2024-12-13T12:56:58.732242Z","shell.execute_reply":"2024-12-13T12:57:02.002115Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"parquet_train = \"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\"\nparquet_test = \"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\"\ntest_csv_path = \"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\"\ntrain_df = newtrain\ntest_df = pd.read_csv(test_csv_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:02.004537Z","iopub.execute_input":"2024-12-13T12:57:02.005191Z","iopub.status.idle":"2024-12-13T12:57:02.016284Z","shell.execute_reply.started":"2024-12-13T12:57:02.005153Z","shell.execute_reply":"2024-12-13T12:57:02.015045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define function to calculate mode with safe handling for scalar values\ndef calculate_mode(series):\n    mode_result = stats.mode(series)\n    if isinstance(mode_result.mode, np.ndarray) and len(mode_result.mode) > 0:\n        return mode_result.mode[0]\n    else:\n        return mode_result.mode\n\n# List of columns to calculate statistics for\ncolumns_to_check = ['X', 'Y', 'Z', 'ENMO', 'anglez', 'light', 'battery_voltage', 'step', 'non-wear-flag', 'weekday', 'quarter', 'relative_date_PCIAT']\n\n# Initialize list to hold processed dataframes\nprocessed_dataframes = []\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:02.017758Z","iopub.execute_input":"2024-12-13T12:57:02.018152Z","iopub.status.idle":"2024-12-13T12:57:02.031663Z","shell.execute_reply.started":"2024-12-13T12:57:02.018117Z","shell.execute_reply":"2024-12-13T12:57:02.030218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Iterate through each folder in the parquet directory\nfor folder in os.listdir(parquet_train):\n    folder_path = os.path.join(parquet_train, folder)\n    if os.path.isdir(folder_path) and folder.startswith(\"id=\"):\n        for file in os.listdir(folder_path):\n            if file.endswith('.parquet'):\n                file_path = os.path.join(folder_path, file)\n                parquet_df = pd.read_parquet(file_path)\n                # Filter out columns that are available in the current parquet file\n                available_columns = [col for col in columns_to_check if col in parquet_df.columns]                 \n                data = {'id': folder.split('=')[1]} # Initialize a dictionary to hold the calculated statistics\n                \n                for col in ['X', 'Y', 'Z', 'ENMO', 'anglez', 'light', 'battery_voltage']: # Calculate statistics for available columns\n                    if col in available_columns:\n                        data[f'{col}_mean'] = parquet_df[col].mean()\n                        data[f'{col}_std'] = parquet_df[col].std()\n                        data[f'{col}_range'] = parquet_df[col].max() - parquet_df[col].min()\n                if 'step' in available_columns:\n                    data['step_sum'] = parquet_df['step'].sum()\n                if 'non-wear-flag' in available_columns:\n                    data['non_wear_flag_percentage'] = parquet_df['non-wear-flag'].mean() * 100\n                if 'weekday' in available_columns:\n                    data['weekday_mode'] = calculate_mode(parquet_df['weekday'])\n                if 'quarter' in available_columns:\n                    data['quarter_mode'] = calculate_mode(parquet_df['quarter'])\n                if 'relative_date_PCIAT' in available_columns:\n                    data['relative_date_PCIAT_mean'] = parquet_df['relative_date_PCIAT'].mean()\n                    data['relative_date_PCIAT_range'] = parquet_df['relative_date_PCIAT'].max() - parquet_df['relative_date_PCIAT'].min()\n\n                # Create a DataFrame with the calculated statistics\n                avg_df = pd.DataFrame([data])\n                processed_dataframes.append(avg_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:02.033318Z","iopub.execute_input":"2024-12-13T12:57:02.033864Z","iopub.status.idle":"2024-12-13T12:57:59.354629Z","shell.execute_reply.started":"2024-12-13T12:57:02.033791Z","shell.execute_reply":"2024-12-13T12:57:59.353275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_avg_df = pd.concat(processed_dataframes, ignore_index=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.356272Z","iopub.execute_input":"2024-12-13T12:57:59.356640Z","iopub.status.idle":"2024-12-13T12:57:59.440165Z","shell.execute_reply.started":"2024-12-13T12:57:59.356605Z","shell.execute_reply":"2024-12-13T12:57:59.438995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_avg_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.441774Z","iopub.execute_input":"2024-12-13T12:57:59.442145Z","iopub.status.idle":"2024-12-13T12:57:59.470531Z","shell.execute_reply.started":"2024-12-13T12:57:59.442112Z","shell.execute_reply":"2024-12-13T12:57:59.469324Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Concatenate all processed DataFrames and merge with training data\nall_avg_df = pd.concat(processed_dataframes, ignore_index=True)\n\n# Identify intersecting columns between train and test data\nintersecting_columns = train_df.columns.intersection(test_df.columns).tolist()\n\nfinal_df = all_avg_df.merge(train_df[intersecting_columns + ['Physical-Height-m'] + ['Physical-Weight-kg'] + ['BMI'] +['sii']], on='id', how='right')\nfinal_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.472067Z","iopub.execute_input":"2024-12-13T12:57:59.472469Z","iopub.status.idle":"2024-12-13T12:57:59.620210Z","shell.execute_reply.started":"2024-12-13T12:57:59.472364Z","shell.execute_reply":"2024-12-13T12:57:59.618944Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.621864Z","iopub.execute_input":"2024-12-13T12:57:59.622314Z","iopub.status.idle":"2024-12-13T12:57:59.693223Z","shell.execute_reply.started":"2024-12-13T12:57:59.622267Z","shell.execute_reply":"2024-12-13T12:57:59.691883Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 計算眾數\nmode_value = final_df['PreInt_EduHx-computerinternet_hoursday'].mode()[0]\n\n# 用眾數填補遺失值\nfinal_df['PreInt_EduHx-computerinternet_hoursday'].fillna(mode_value, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.694765Z","iopub.execute_input":"2024-12-13T12:57:59.695271Z","iopub.status.idle":"2024-12-13T12:57:59.703595Z","shell.execute_reply.started":"2024-12-13T12:57:59.695231Z","shell.execute_reply":"2024-12-13T12:57:59.702357Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.705048Z","iopub.execute_input":"2024-12-13T12:57:59.705420Z","iopub.status.idle":"2024-12-13T12:57:59.730778Z","shell.execute_reply.started":"2024-12-13T12:57:59.705384Z","shell.execute_reply":"2024-12-13T12:57:59.729362Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.732385Z","iopub.execute_input":"2024-12-13T12:57:59.732892Z","iopub.status.idle":"2024-12-13T12:57:59.792154Z","shell.execute_reply.started":"2024-12-13T12:57:59.732820Z","shell.execute_reply":"2024-12-13T12:57:59.790890Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the season mapping\nseason_mapping = {\n    'Winter': 1,\n    'Spring': 2,\n    'Summer': 3,\n    'Fall': 4\n}\n\n# Iterate through columns in the DataFrame\nfor column in final_df.columns:\n    # Check if the column name ends with 'Season' and is of object data type\n    if column.endswith('Season') and final_df[column].dtype == 'object':\n        # Map the values in the column using the defined mapping\n        final_df[column] = final_df[column].map(season_mapping)\n\n# Optionally, check the updated DataFrame to ensure mapping is applied\nprint(final_df[[col for col in final_df.columns if col.endswith('Season')]].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.793698Z","iopub.execute_input":"2024-12-13T12:57:59.794174Z","iopub.status.idle":"2024-12-13T12:57:59.815890Z","shell.execute_reply.started":"2024-12-13T12:57:59.794123Z","shell.execute_reply":"2024-12-13T12:57:59.814676Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for column in final_df.columns:\n    if column.endswith('Season'):\n        final_df[column].fillna(method='ffill', inplace=True) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.817629Z","iopub.execute_input":"2024-12-13T12:57:59.818112Z","iopub.status.idle":"2024-12-13T12:57:59.830595Z","shell.execute_reply.started":"2024-12-13T12:57:59.818051Z","shell.execute_reply":"2024-12-13T12:57:59.829214Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.832178Z","iopub.execute_input":"2024-12-13T12:57:59.832568Z","iopub.status.idle":"2024-12-13T12:57:59.857796Z","shell.execute_reply.started":"2024-12-13T12:57:59.832531Z","shell.execute_reply":"2024-12-13T12:57:59.856480Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_df.filter(like='Season').head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.859829Z","iopub.execute_input":"2024-12-13T12:57:59.860319Z","iopub.status.idle":"2024-12-13T12:57:59.873081Z","shell.execute_reply.started":"2024-12-13T12:57:59.860271Z","shell.execute_reply":"2024-12-13T12:57:59.871854Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ncolumns_to_impute = [\n    'X_mean', 'X_std', 'X_range', 'Y_mean', 'Y_std', 'Y_range',\n    'Z_mean', 'Z_std', 'Z_range', 'anglez_mean', 'anglez_std',\n    'anglez_range', 'light_mean', 'light_std', 'light_range',\n    'battery_voltage_mean', 'battery_voltage_std', 'battery_voltage_range',\n    'step_sum', 'weekday_mode', 'quarter_mode', 'relative_date_PCIAT_mean',\n    'relative_date_PCIAT_range'\n]\n\n# 提取需要補全的數據\ndata_to_impute = final_df[columns_to_impute]\n\n# 填補平均數\nfor column in columns_to_impute:\n    final_df[column] = final_df[column].fillna(final_df[column].mean())\n\n# 確認補全後是否還有缺失值\nprint(final_df[columns_to_impute].isnull().sum())  # 應輸出全為 0\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.874726Z","iopub.execute_input":"2024-12-13T12:57:59.875139Z","iopub.status.idle":"2024-12-13T12:57:59.902227Z","shell.execute_reply.started":"2024-12-13T12:57:59.875104Z","shell.execute_reply":"2024-12-13T12:57:59.900928Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define your features and target variable\nX = final_df.drop(columns=['sii'])  # Features\ny = final_df['sii']  # Target\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.903504Z","iopub.execute_input":"2024-12-13T12:57:59.903866Z","iopub.status.idle":"2024-12-13T12:57:59.911896Z","shell.execute_reply.started":"2024-12-13T12:57:59.903814Z","shell.execute_reply":"2024-12-13T12:57:59.910675Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 查看 X 的數據類型\nprint(X.dtypes)\n\n# 找出非數字列\nnon_numeric_columns = X.select_dtypes(include=['object']).columns\nprint(f\"Non-numeric columns: {non_numeric_columns}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.922936Z","iopub.execute_input":"2024-12-13T12:57:59.923336Z","iopub.status.idle":"2024-12-13T12:57:59.932401Z","shell.execute_reply.started":"2024-12-13T12:57:59.923301Z","shell.execute_reply":"2024-12-13T12:57:59.931170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = X.drop(columns=non_numeric_columns)\nprint(X.dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.933910Z","iopub.execute_input":"2024-12-13T12:57:59.934264Z","iopub.status.idle":"2024-12-13T12:57:59.947868Z","shell.execute_reply.started":"2024-12-13T12:57:59.934228Z","shell.execute_reply":"2024-12-13T12:57:59.946697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# 目標變數分布\ndistribution = y.value_counts()\nprint(distribution)\n\n# 視覺化分布\nplt.figure(figsize=(8, 5))\ndistribution.plot(kind='bar')\nplt.title('Class Distribution Before SMOTE')\nplt.xlabel('Class')\nplt.ylabel('Frequency')\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:57:59.949465Z","iopub.execute_input":"2024-12-13T12:57:59.949867Z","iopub.status.idle":"2024-12-13T12:58:00.224173Z","shell.execute_reply.started":"2024-12-13T12:57:59.949809Z","shell.execute_reply":"2024-12-13T12:58:00.222910Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from imblearn.over_sampling import SMOTE\nsmote = SMOTE(sampling_strategy={0.0: 1594, 1.0: 1460, 2.0: 756, 3.0: 68}, random_state=42)\nX_res, y_res = smote.fit_resample(X, y)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:00.225990Z","iopub.execute_input":"2024-12-13T12:58:00.226489Z","iopub.status.idle":"2024-12-13T12:58:00.307945Z","shell.execute_reply.started":"2024-12-13T12:58:00.226436Z","shell.execute_reply":"2024-12-13T12:58:00.306741Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"處理test_df","metadata":{}},{"cell_type":"code","source":"test_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:00.309034Z","iopub.execute_input":"2024-12-13T12:58:00.309575Z","iopub.status.idle":"2024-12-13T12:58:00.393112Z","shell.execute_reply.started":"2024-12-13T12:58:00.309538Z","shell.execute_reply":"2024-12-13T12:58:00.391983Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 將身高從英吋轉換為米\ntest_df['Physical-Height-m'] = test_df['Physical-Height'] * 0.0254\n\n# 將體重從磅轉換為公斤\ntest_df['Physical-Weight-kg'] = test_df['Physical-Weight'] * 0.453592\n\n# 計算 BMI\ntest_df['BMI'] = test_df['Physical-Weight-kg'] / (test_df['Physical-Height-m'] ** 2)\n\n# 檢查計算結果\nprint(test_df[['Physical-Height', 'Physical-Height-m', \n                        'Physical-Weight', 'Physical-Weight-kg', 'BMI']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:00.394425Z","iopub.execute_input":"2024-12-13T12:58:00.394746Z","iopub.status.idle":"2024-12-13T12:58:00.408159Z","shell.execute_reply.started":"2024-12-13T12:58:00.394716Z","shell.execute_reply":"2024-12-13T12:58:00.406926Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:00.409652Z","iopub.execute_input":"2024-12-13T12:58:00.410159Z","iopub.status.idle":"2024-12-13T12:58:00.504932Z","shell.execute_reply.started":"2024-12-13T12:58:00.410121Z","shell.execute_reply":"2024-12-13T12:58:00.503590Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 初始化保存處理後數據的列表\nprocessed_test_dataframes = []\n\n# 遍歷 parquet_test 資料夾並計算統計數據\nfor folder in os.listdir(parquet_test):\n    folder_path = os.path.join(parquet_test, folder)\n    if os.path.isdir(folder_path) and folder.startswith(\"id=\"):\n        for file in os.listdir(folder_path):\n            if file.endswith('.parquet'):\n                file_path = os.path.join(folder_path, file)\n                parquet_df = pd.read_parquet(file_path)\n                # 過濾當前 parquet 文件中可用的列\n                available_columns = [col for col in columns_to_check if col in parquet_df.columns]\n                data = {'id': folder.split('=')[1]}  # 初始化字典用於保存統計結果\n                \n                # 計算可用列的統計數據\n                for col in ['X', 'Y', 'Z', 'ENMO', 'anglez', 'light', 'battery_voltage']:\n                    if col in available_columns:\n                        data[f'{col}_mean'] = parquet_df[col].mean()\n                        data[f'{col}_std'] = parquet_df[col].std()\n                        data[f'{col}_range'] = parquet_df[col].max() - parquet_df[col].min()\n                if 'step' in available_columns:\n                    data['step_sum'] = parquet_df['step'].sum()\n                if 'non-wear-flag' in available_columns:\n                    data['non_wear_flag_percentage'] = parquet_df['non-wear-flag'].mean() * 100\n                if 'weekday' in available_columns:\n                    data['weekday_mode'] = calculate_mode(parquet_df['weekday'])\n                if 'quarter' in available_columns:\n                    data['quarter_mode'] = calculate_mode(parquet_df['quarter'])\n                if 'relative_date_PCIAT' in available_columns:\n                    data['relative_date_PCIAT_mean'] = parquet_df['relative_date_PCIAT'].mean()\n                    data['relative_date_PCIAT_range'] = parquet_df['relative_date_PCIAT'].max() - parquet_df['relative_date_PCIAT'].min()\n\n                # 創建一個包含統計數據的 DataFrame\n                avg_test_df = pd.DataFrame([data])\n                processed_test_dataframes.append(avg_test_df)\n\n# 合併所有處理後的 DataFrame\nif processed_test_dataframes:\n    all_avg_test_df = pd.concat(processed_test_dataframes, ignore_index=True)\nelse:\n    all_avg_test_df = pd.DataFrame(columns=['id'] + [f'{col}_mean' for col in ['X', 'Y', 'Z', 'ENMO', 'anglez', 'light', 'battery_voltage']] +\n                                   [f'{col}_std' for col in ['X', 'Y', 'Z', 'ENMO', 'anglez', 'light', 'battery_voltage']] +\n                                   [f'{col}_range' for col in ['X', 'Y', 'Z', 'ENMO', 'anglez', 'light', 'battery_voltage']] +\n                                   ['step_sum', 'non_wear_flag_percentage', 'weekday_mode', 'quarter_mode', \n                                    'relative_date_PCIAT_mean', 'relative_date_PCIAT_range'])\n\n# 確保 `id` 列的格式與 test_df 一致\nall_avg_test_df['id'] = all_avg_test_df['id'].astype(str)\ntest_df['id'] = test_df['id'].astype(str)\n\n# 進行左合併，保留 test_df 的所有數據\nfinal_test_df = test_df.merge(all_avg_test_df, on='id', how='left')\n\n# 檢查結果\nprint(final_test_df.shape)\nfinal_test_df.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:00.506674Z","iopub.execute_input":"2024-12-13T12:58:00.507079Z","iopub.status.idle":"2024-12-13T12:58:00.679506Z","shell.execute_reply.started":"2024-12-13T12:58:00.507040Z","shell.execute_reply":"2024-12-13T12:58:00.678196Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 保存訓練時使用的特徵名稱\nfeature_columns = X_res.columns.tolist()  # 獲取訓練數據中的特徵列名稱\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:00.681035Z","iopub.execute_input":"2024-12-13T12:58:00.681401Z","iopub.status.idle":"2024-12-13T12:58:00.686676Z","shell.execute_reply.started":"2024-12-13T12:58:00.681365Z","shell.execute_reply":"2024-12-13T12:58:00.685316Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 確保 final_test_df 包含所有訓練模型時的特徵\nmissing_features = set(feature_columns) - set(final_test_df.columns)\nextra_features = set(final_test_df.columns) - set(feature_columns)\n\n# 打印不一致的特徵\nif missing_features:\n    print(f\"Missing features in final_test_df: {missing_features}\")\nif extra_features:\n    print(f\"Extra features in final_test_df: {extra_features}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:00.688747Z","iopub.execute_input":"2024-12-13T12:58:00.689304Z","iopub.status.idle":"2024-12-13T12:58:00.708405Z","shell.execute_reply.started":"2024-12-13T12:58:00.689248Z","shell.execute_reply":"2024-12-13T12:58:00.706092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 定義訓練時的特徵名稱（feature_columns）\n# 假設 feature_columns 是在訓練過程中保存的\nfeature_columns = X_res.columns.tolist()\n\n# 移除多餘的特徵\nfinal_test_df = final_test_df[feature_columns]\n\n# 確認移除後的數據\nprint(f\"Updated columns in final_test_df: {final_test_df.columns.tolist()}\")\n\n# 確保最終測試數據特徵與訓練一致\nassert list(final_test_df.columns) == feature_columns, \"Features in final_test_df do not match training data!\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:00.710078Z","iopub.execute_input":"2024-12-13T12:58:00.710451Z","iopub.status.idle":"2024-12-13T12:58:00.725621Z","shell.execute_reply.started":"2024-12-13T12:58:00.710415Z","shell.execute_reply":"2024-12-13T12:58:00.724259Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_test_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:00.727730Z","iopub.execute_input":"2024-12-13T12:58:00.728274Z","iopub.status.idle":"2024-12-13T12:58:00.827274Z","shell.execute_reply.started":"2024-12-13T12:58:00.728218Z","shell.execute_reply":"2024-12-13T12:58:00.825952Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the season mapping\nseason_mapping = {\n    'Winter': 1,\n    'Spring': 2,\n    'Summer': 3,\n    'Fall': 4\n}\n\n# Iterate through columns in the DataFrame\nfor column in final_test_df.columns:\n    # Check if the column name ends with 'Season' and is of object data type\n    if column.endswith('Season') and final_test_df[column].dtype == 'object':\n        # Map the values in the column using the defined mapping\n        final_test_df[column] = final_test_df[column].map(season_mapping)\n\n# Optionally, check the updated DataFrame to ensure mapping is applied\nprint(final_test_df[[col for col in final_test_df.columns if col.endswith('Season')]].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:00.829018Z","iopub.execute_input":"2024-12-13T12:58:00.829399Z","iopub.status.idle":"2024-12-13T12:58:00.850451Z","shell.execute_reply.started":"2024-12-13T12:58:00.829362Z","shell.execute_reply":"2024-12-13T12:58:00.849012Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_test_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:00.852363Z","iopub.execute_input":"2024-12-13T12:58:00.852964Z","iopub.status.idle":"2024-12-13T12:58:00.869611Z","shell.execute_reply.started":"2024-12-13T12:58:00.852901Z","shell.execute_reply":"2024-12-13T12:58:00.868238Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 過濾數值類型的列\nnumeric_columns = final_test_df.select_dtypes(include=[np.number]).columns.tolist()\n\n# 檢查數值類型列中的 NaN 或 inf\nfor col in numeric_columns:\n    if final_test_df[col].isnull().any() or not np.isfinite(final_test_df[col]).all():\n        print(f\"Column {col} contains NaN or inf values.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:00.871092Z","iopub.execute_input":"2024-12-13T12:58:00.871458Z","iopub.status.idle":"2024-12-13T12:58:00.890327Z","shell.execute_reply.started":"2024-12-13T12:58:00.871413Z","shell.execute_reply":"2024-12-13T12:58:00.889180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ncolumns_to_impute = [\n    'X_mean', 'X_std', 'X_range', 'Y_mean', 'Y_std', 'Y_range',\n    'Z_mean', 'Z_std', 'Z_range', 'anglez_mean', 'anglez_std',\n    'anglez_range', 'light_mean', 'light_std', 'light_range',\n    'battery_voltage_mean', 'battery_voltage_std', 'battery_voltage_range',\n    'step_sum', 'weekday_mode', 'quarter_mode', 'relative_date_PCIAT_mean',\n    'relative_date_PCIAT_range'\n]\n\n# 提取需要補全的數據\ndata_to_impute = final_test_df[columns_to_impute]\n\n# 填補平均數\nfor column in columns_to_impute:\n    final_test_df[column] = final_test_df[column].fillna(final_test_df[column].mean())\n\n# 確認補全後是否還有缺失值\nprint(final_test_df[columns_to_impute].isnull().sum())  # 應輸出全為 0\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:00.891875Z","iopub.execute_input":"2024-12-13T12:58:00.892292Z","iopub.status.idle":"2024-12-13T12:58:00.920614Z","shell.execute_reply.started":"2024-12-13T12:58:00.892257Z","shell.execute_reply":"2024-12-13T12:58:00.919180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 選擇需要填補的變數\nbia_columns = [\n    'BIA-BIA_BMC', 'BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW',\n    'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_ICW',\n    'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW'\n]\n\n# 初始化插補器\nimputer = IterativeImputer(random_state=0, max_iter=20)\n\n# 對選定的變數進行插補\nfinal_test_df[bia_columns] = imputer.fit_transform(final_test_df[bia_columns])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:00.922181Z","iopub.execute_input":"2024-12-13T12:58:00.922547Z","iopub.status.idle":"2024-12-13T12:58:01.001906Z","shell.execute_reply.started":"2024-12-13T12:58:00.922509Z","shell.execute_reply":"2024-12-13T12:58:01.000457Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\n\n# 選擇需要填補的變數\nFGC_columns = [\n    \"FGC-FGC_CU\", \n    \"FGC-FGC_PU\", \n    \"FGC-FGC_SRL\",\n    \"FGC-FGC_SRR\", \n    \"FGC-FGC_TL\"\n]\n\n# 初始化插補器\nimputer = IterativeImputer(random_state=0, max_iter=20)\n\n# 對選定的變數進行插補\nfinal_test_df[FGC_columns] = imputer.fit_transform(final_test_df[FGC_columns])\n\n# 四捨五入到整數\nfinal_test_df[FGC_columns] = final_test_df[FGC_columns].round(0)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:01.003998Z","iopub.execute_input":"2024-12-13T12:58:01.004498Z","iopub.status.idle":"2024-12-13T12:58:01.031460Z","shell.execute_reply.started":"2024-12-13T12:58:01.004432Z","shell.execute_reply":"2024-12-13T12:58:01.030073Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\n\n# 選擇需要填補的變數\nphy_columns = [\n    \"Physical-Diastolic_BP\", \n    \"Physical-HeartRate\", \n    \"Physical-Systolic_BP\"\n]\n\n# 初始化插補器\nimputer = IterativeImputer(random_state=0, max_iter=20)\n\n# 對選定的變數進行插補\nfinal_test_df[phy_columns] = imputer.fit_transform(final_test_df[phy_columns])\n\n# 四捨五入到整數\nfinal_test_df[phy_columns] = final_test_df[phy_columns].round(0)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:01.032908Z","iopub.execute_input":"2024-12-13T12:58:01.033266Z","iopub.status.idle":"2024-12-13T12:58:01.054788Z","shell.execute_reply.started":"2024-12-13T12:58:01.033231Z","shell.execute_reply":"2024-12-13T12:58:01.053122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 定義 BMI 分群的條件和標籤\nbins = [0, 18.5, 24, 27, float('inf')]  # 定義分群的邊界\nlabels = ['過輕', '健康體重', '過重', '肥胖']  # 定義分群的標籤\n\n# 使用 pandas 的 cut 方法進行分群\nfinal_test_df['BMI_Group'] = pd.cut(final_test_df['BMI'], bins=bins, labels=labels, right=False)\n\n# 檢查分群結果\nprint(final_test_df[['BMI', 'BMI_Group']].head())\n\n# 統計每個群體的人數\ngroup_counts = final_test_df['BMI_Group'].value_counts()\nprint(\"各群體人數：\")\nprint(group_counts)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:01.056589Z","iopub.execute_input":"2024-12-13T12:58:01.057155Z","iopub.status.idle":"2024-12-13T12:58:01.074372Z","shell.execute_reply.started":"2024-12-13T12:58:01.057101Z","shell.execute_reply":"2024-12-13T12:58:01.072918Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 計算每組的眾數，顯式設置 observed=False\nmode_per_group = (\n    final_test_df.groupby(\"BMI_Group\", observed=False)[\"BIA-BIA_Frame_num\"]\n    .agg(lambda x: x.mode().iloc[0] if not x.mode().empty else None)\n)\n\nprint(\"每組的眾數:\")\nprint(mode_per_group)\n\n# 定義填補函數\ndef fill_with_mode(row):\n    if pd.isnull(row[\"BIA-BIA_Frame_num\"]):  # 如果缺失值\n        return mode_per_group[row[\"BMI_Group\"]]  # 使用對應的眾數填補\n    return row[\"BIA-BIA_Frame_num\"]  # 否則返回原值\n\n# 使用 apply 填補缺失值\nfinal_test_df[\"BIA-BIA_Frame_num\"] = final_test_df.apply(fill_with_mode, axis=1)\n\n# 確認填補後是否還有缺失值\nprint(\"填補完成，剩餘缺失值數量:\")\nprint(final_test_df[\"BIA-BIA_Frame_num\"].isnull().sum())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:01.076109Z","iopub.execute_input":"2024-12-13T12:58:01.076819Z","iopub.status.idle":"2024-12-13T12:58:01.103041Z","shell.execute_reply.started":"2024-12-13T12:58:01.076774Z","shell.execute_reply":"2024-12-13T12:58:01.101443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 定義要檢查的變數名稱列表\nfgc_columns = [\n    'SDS-SDS_Total_Raw',\n    'SDS-SDS_Total_T',\n    'CGAS-CGAS_Score',\n    'BIA-BIA_Activity_Level_num'\n]\n\n# 計算每個變數的偏態和峰度，並決定填補方式\nfor col in fgc_columns:\n    skewness = final_test_df[col].skew()\n    kurtosis = final_test_df[col].kurt()\n    \n    print(f\"變數: {col}\")\n    print(f\"  偏態: {skewness}\")\n    print(f\"  峰度: {kurtosis}\")\n    \n    # 判斷插補方式並進行填補\n    if abs(skewness) < 0.5:\n        print(\"  建議填補方式: 使用均值填補（分佈相對對稱）\")\n        # 使用均值填補\n        mean_value = round(final_test_df[col].mean())\n        final_test_df[col] = final_test_df[col].fillna(mean_value)\n        print(f\"  使用均值填補，填補值: {mean_value}\")\n    else:\n        print(\"  建議填補方式: 使用中位數填補（分佈偏斜）\")\n        # 使用中位數填補\n        median_value = final_test_df[col].median()\n        final_test_df[col] = final_test_df[col].fillna(median_value)\n        print(f\"  使用中位數填補，填補值: {median_value}\")\n    print(\"-\" * 30)\n\n# 確認填補後是否還有缺失值\nprint(\"填補完成，剩餘缺失值情況：\")\nprint(final_test_df[fgc_columns].isnull().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:01.105170Z","iopub.execute_input":"2024-12-13T12:58:01.106208Z","iopub.status.idle":"2024-12-13T12:58:01.131531Z","shell.execute_reply.started":"2024-12-13T12:58:01.106147Z","shell.execute_reply":"2024-12-13T12:58:01.130287Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 計算列的眾數\nmode_value = final_test_df['PreInt_EduHx-computerinternet_hoursday'].mode()\n\n# 檢查是否有眾數（防止空列導致問題）\nif not mode_value.empty:\n    mode_value = mode_value.iloc[0]  # 提取第一個眾數\n    print(f\"PreInt_EduHx-computerinternet_hoursday 的眾數為: {mode_value}\")\n\n    # 填補缺失值\n    final_test_df['PreInt_EduHx-computerinternet_hoursday'].fillna(mode_value, inplace=True)\n\n    # 確認填補後的缺失值數量\n    print(\"填補完成，剩餘缺失值數量:\")\n    print(final_test_df['PreInt_EduHx-computerinternet_hoursday'].isnull().sum())\nelse:\n    print(\"PreInt_EduHx-computerinternet_hoursday 沒有眾數，請檢查數據。\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:01.133008Z","iopub.execute_input":"2024-12-13T12:58:01.133465Z","iopub.status.idle":"2024-12-13T12:58:01.158976Z","shell.execute_reply.started":"2024-12-13T12:58:01.133426Z","shell.execute_reply":"2024-12-13T12:58:01.157911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 選取包含 \"Season\" 的變數\nseason_columns = [col for col in final_test_df.columns if \"Season\" in col]\n\n# 將這些變數中的缺失值填補為該變數的眾數\nfor col in season_columns:\n    mode_value = final_test_df[col].mode()[0]  # 計算眾數\n    final_test_df[col].fillna(mode_value, inplace=True)\n\n# 檢查結果\nprint(\"已將 Season 相關變數的遺失值填補為眾數:\")\nprint(final_test_df[season_columns].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:01.160382Z","iopub.execute_input":"2024-12-13T12:58:01.160892Z","iopub.status.idle":"2024-12-13T12:58:01.190625Z","shell.execute_reply.started":"2024-12-13T12:58:01.160821Z","shell.execute_reply":"2024-12-13T12:58:01.189391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 找出有缺失值的變數\nmissing_columns = final_test_df.columns[final_test_df.isnull().any()]\n\n# 列出這些變數的資料型態\nmissing_data_types = final_test_df[missing_columns].dtypes\n\n# 顯示完整結果\nprint(\"有缺失值的變數及其資料型態：\")\nprint(missing_data_types.to_string())\nlen(missing_columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:01.192118Z","iopub.execute_input":"2024-12-13T12:58:01.192522Z","iopub.status.idle":"2024-12-13T12:58:01.218574Z","shell.execute_reply.started":"2024-12-13T12:58:01.192468Z","shell.execute_reply":"2024-12-13T12:58:01.217228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.ensemble import RandomForestRegressor\n\n# 假設 subtrain_cleaned 是你的原始 DataFrame，並且 Basic_Demos-Age 和 Basic_Demos-Sex 沒有缺失值\n# 分離身高缺失和非缺失數據\nheight_missing = final_test_df[final_test_df['Physical-Height'].isnull()]\nheight_non_missing = final_test_df[final_test_df['Physical-Height'].notnull()]\n\n# 使用非缺失數據訓練模型來預測身高\nX_test_height = height_non_missing[['Basic_Demos-Age', 'Basic_Demos-Sex']]  # 特徵\ny_test_height = height_non_missing['Physical-Height']  # 目標變數\n\n# 建立隨機森林回歸模型並進行訓練\nheight_model = RandomForestRegressor(random_state=0)\nheight_model.fit(X_test_height, y_test_height)\n\n# 使用訓練好的模型來預測缺失的身高數據\nX_pred_height = height_missing[['Basic_Demos-Age', 'Basic_Demos-Sex']]\nfinal_test_df.loc[height_missing.index, 'Physical-Height'] = height_model.predict(X_pred_height)\n\n\n# 分離體重缺失和非缺失數據\nweight_missing = final_test_df[final_test_df['Physical-Weight'].isnull()]\nweight_non_missing = final_test_df[final_test_df['Physical-Weight'].notnull()]\n\n# 使用非缺失數據訓練模型來預測體重\nX_test_weight = weight_non_missing[['Basic_Demos-Age', 'Basic_Demos-Sex']]  # 特徵\ny_test_weight = weight_non_missing['Physical-Weight']  # 目標變數\n\n# 建立隨機森林回歸模型並進行訓練\nweight_model = RandomForestRegressor(random_state=0)\nweight_model.fit(X_test_weight, y_test_weight)\n\n# 使用訓練好的模型來預測缺失的體重數據\nX_pred_weight = weight_missing[['Basic_Demos-Age', 'Basic_Demos-Sex']]\nfinal_test_df.loc[weight_missing.index, 'Physical-Weight'] = weight_model.predict(X_pred_weight)\n\n# 查看 Physical-Height 和 Physical-Weight 列的缺失值數量\nprint(\"Physical-Height 缺失值數量：\", final_test_df['Physical-Height'].isnull().sum())\nprint(\"Physical-Weight 缺失值數量：\", final_test_df['Physical-Weight'].isnull().sum())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:01.220149Z","iopub.execute_input":"2024-12-13T12:58:01.220697Z","iopub.status.idle":"2024-12-13T12:58:01.520536Z","shell.execute_reply.started":"2024-12-13T12:58:01.220643Z","shell.execute_reply":"2024-12-13T12:58:01.519153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 將身高從英吋轉換為米\nfinal_test_df['Physical-Height-m'] = final_test_df['Physical-Height'] * 0.0254\n\n# 將體重從磅轉換為公斤\nfinal_test_df['Physical-Weight-kg'] = final_test_df['Physical-Weight'] * 0.453592\n\n# 計算 BMI\nfinal_test_df['BMI'] = final_test_df['Physical-Weight-kg'] / (final_test_df['Physical-Height-m'] ** 2)\n\n# 檢查計算結果\nprint(final_test_df[['Physical-Height', 'Physical-Height-m', \n                        'Physical-Weight', 'Physical-Weight-kg', 'BMI']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:01.521913Z","iopub.execute_input":"2024-12-13T12:58:01.522234Z","iopub.status.idle":"2024-12-13T12:58:01.535354Z","shell.execute_reply.started":"2024-12-13T12:58:01.522202Z","shell.execute_reply":"2024-12-13T12:58:01.533963Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 定義要刪除的變數列表\ncolumns_to_drop = ['BMI_Group']\n\n# 創建一個新的 DataFrame，刪除指定變數\nfinal_test_df = final_test_df.drop(columns=columns_to_drop)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:01.537076Z","iopub.execute_input":"2024-12-13T12:58:01.537652Z","iopub.status.idle":"2024-12-13T12:58:01.556657Z","shell.execute_reply.started":"2024-12-13T12:58:01.537596Z","shell.execute_reply":"2024-12-13T12:58:01.555530Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_test_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:01.558467Z","iopub.execute_input":"2024-12-13T12:58:01.558976Z","iopub.status.idle":"2024-12-13T12:58:01.587762Z","shell.execute_reply.started":"2024-12-13T12:58:01.558934Z","shell.execute_reply":"2024-12-13T12:58:01.586494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from xgboost import XGBClassifier\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score\nimport numpy as np\nimport pandas as pd\nfrom scipy.optimize import minimize\nfrom tqdm import tqdm\nfrom sklearn.metrics import cohen_kappa_score\n\n# 定義 quadratic_weighted_kappa\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n# 定義 threshold_Rounder\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\n# 定義 evaluate_predictions\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\n# 修改後的 TrainKFoldWithTestPrediction\ndef TrainKFoldWithTestPrediction(final_df, final_test_df, feature_columns, n_splits=20):\n    \"\"\"\n    使用 K-Fold 訓練模型並對測試集進行預測，適用於前面已處理過的不平衡數據。\n    \"\"\"\n    # 提取特徵和目標變數\n    X = final_df[feature_columns]\n    y = final_df['sii']\n    X_test = final_test_df[feature_columns]\n\n    # 初始化 Stratified K-Fold\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    oof_non_rounded = np.zeros(len(y))  # Out-of-Fold 預測值（未四捨五入）\n    test_preds = np.zeros((len(X_test), n_splits))  # 每個摺疊的測試集預測\n    fold_metrics = []  # 每個摺疊的 QWK 分數\n\n    print(f\"Start Training on {n_splits} Folds...\\n\")\n\n    # K-Fold 訓練\n    for fold, (train_idx, val_idx) in enumerate(tqdm(SKF.split(X, y), total=n_splits, desc=\"Training Folds\")):\n        # 分割訓練集和驗證集\n        X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n\n        # 定義模型\n        model = XGBClassifier(\n            n_estimators=400,\n            learning_rate=0.05,\n            max_depth=6,\n            colsample_bytree=0.8,\n            tree_method='exact',\n            use_label_encoder=False,\n            random_state=666,\n            reg_alpha=0.1,  # L1 正則化\n            reg_lambda=1.0,  # L2 正則化\n            gamma=1.0  # 節點分裂的最小損失減少\n        )\n\n        # 模型訓練\n        model.fit(\n            X_train, y_train,\n            eval_set=[(X_val, y_val)],\n            verbose=False\n        )\n\n        # 驗證集預測\n        val_preds_non_rounded = model.predict_proba(X_val).dot(np.arange(4))  # 非四捨五入預測\n        oof_non_rounded[val_idx] = val_preds_non_rounded\n        val_preds_rounded = threshold_Rounder(val_preds_non_rounded, [0.5, 1.5, 2.5])  # 四捨五入後的預測\n\n        # 計算 QWK 分數\n        kappa_score = quadratic_weighted_kappa(y_val, val_preds_rounded)\n        fold_metrics.append(kappa_score)\n        print(f\"Fold {fold + 1} - Validation QWK: {kappa_score:.4f}\")\n\n        # 測試集預測\n        test_preds[:, fold] = model.predict_proba(X_test).dot(np.arange(4))\n\n    print(\"\\nTraining Completed.\")\n    print(f\"Mean Validation QWK: {np.mean(fold_metrics):.4f}\")\n\n    # 優化閾值\n    optimal_thresholds = minimize(\n        evaluate_predictions, [0.5, 1.5, 2.5],\n        args=(y, oof_non_rounded),\n        method='Nelder-Mead'\n    ).x\n\n    print(f\"Optimized Thresholds: {optimal_thresholds}\")\n\n    # 使用最佳閾值對測試集進行預測\n    final_test_df['sii'] = threshold_Rounder(test_preds.mean(axis=1), optimal_thresholds)\n\n    return final_test_df\n\n# 使用函數進行 K-Fold 訓練和測試集預測\nfeature_columns = [col for col in final_df.columns if col != 'sii' and col in final_test_df.columns]\nfinal_test_df = TrainKFoldWithTestPrediction(final_df, final_test_df, feature_columns, n_splits=20)\n\n# 檢查測試集預測結果\nprint(final_test_df[['sii']].head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T12:58:01.589998Z","iopub.execute_input":"2024-12-13T12:58:01.590414Z","iopub.status.idle":"2024-12-13T13:02:06.014720Z","shell.execute_reply.started":"2024-12-13T12:58:01.590376Z","shell.execute_reply":"2024-12-13T13:02:06.012997Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 確保 `original_test_df` 中有 `id`\nif 'id' in test_df.columns:\n    # 從原始測試數據中提取 id\n    final_test_df['id'] = test_df['id'].values\nelse:\n    print(\"Error: `id` column is missing in the original test data.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T13:02:06.016611Z","iopub.execute_input":"2024-12-13T13:02:06.017097Z","iopub.status.idle":"2024-12-13T13:02:06.025134Z","shell.execute_reply.started":"2024-12-13T13:02:06.017044Z","shell.execute_reply":"2024-12-13T13:02:06.023891Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 提取 id 和預測的 sii\nresult_df = final_test_df[['id', 'sii']]\n\n# 檢查結果\nprint(result_df.head(20))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T13:02:06.026708Z","iopub.execute_input":"2024-12-13T13:02:06.027137Z","iopub.status.idle":"2024-12-13T13:02:06.048128Z","shell.execute_reply.started":"2024-12-13T13:02:06.027100Z","shell.execute_reply":"2024-12-13T13:02:06.046937Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 保存結果到文件\nresult_df.to_csv(\"submission.csv\", index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T13:02:06.049636Z","iopub.execute_input":"2024-12-13T13:02:06.050134Z","iopub.status.idle":"2024-12-13T13:02:06.065187Z","shell.execute_reply.started":"2024-12-13T13:02:06.050084Z","shell.execute_reply":"2024-12-13T13:02:06.063922Z"}},"outputs":[],"execution_count":null}]}