{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Data Import","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:47.623238Z","iopub.execute_input":"2024-12-03T03:36:47.623864Z","iopub.status.idle":"2024-12-03T03:36:51.314918Z","shell.execute_reply.started":"2024-12-03T03:36:47.623812Z","shell.execute_reply":"2024-12-03T03:36:51.313565Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport seaborn as sns\nimport warnings","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:51.317242Z","iopub.execute_input":"2024-12-03T03:36:51.317927Z","iopub.status.idle":"2024-12-03T03:36:52.195745Z","shell.execute_reply.started":"2024-12-03T03:36:51.317882Z","shell.execute_reply":"2024-12-03T03:36:52.194771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"warnings.filterwarnings('ignore', category=FutureWarning)\n\nsns.set(style=\"whitegrid\")\n%matplotlib inline","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:52.197089Z","iopub.execute_input":"2024-12-03T03:36:52.197797Z","iopub.status.idle":"2024-12-03T03:36:52.205349Z","shell.execute_reply.started":"2024-12-03T03:36:52.197745Z","shell.execute_reply":"2024-12-03T03:36:52.204158Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\ndata_dict = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:52.207785Z","iopub.execute_input":"2024-12-03T03:36:52.208144Z","iopub.status.idle":"2024-12-03T03:36:52.304928Z","shell.execute_reply.started":"2024-12-03T03:36:52.208109Z","shell.execute_reply":"2024-12-03T03:36:52.303787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(train.head())\nprint(f\"Train shape: {train.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:52.306613Z","iopub.execute_input":"2024-12-03T03:36:52.307015Z","iopub.status.idle":"2024-12-03T03:36:52.356187Z","shell.execute_reply.started":"2024-12-03T03:36:52.306979Z","shell.execute_reply":"2024-12-03T03:36:52.354780Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\ndata type별로 기초적인 기술통계 보여주는 함수 정의\n''' \ndef calculate_stats(data, columns):\n    if isinstance(columns, str): #str인지\n        columns = [columns] #열 이름 리스트\n\n    stats = []\n    for col in columns:\n        if data[col].dtype in ['object', 'category']: #현재 열의 data type\n            counts = data[col].value_counts(dropna=False, sort=False) #고유값 기준, NA 포함\n            percents = data[col].value_counts(normalize=True, dropna=False, sort=False) * 100\n            formatted = counts.astype(str) + ' (' + percents.round(2).astype(str) + '%)'\n            stats_col = pd.DataFrame({'count (%)': formatted})\n            stats.append(stats_col)\n        else: #cate. 아닌 경우\n            stats_col = data[col].describe().to_frame().transpose()\n            stats_col['missing'] = data[col].isnull().sum() #null 값 수\n            stats_col.index.name = col\n            stats.append(stats_col)\n\n    return pd.concat(stats, axis=0) #하나의 DF로 결합","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:52.357512Z","iopub.execute_input":"2024-12-03T03:36:52.357960Z","iopub.status.idle":"2024-12-03T03:36:52.367587Z","shell.execute_reply.started":"2024-12-03T03:36:52.357919Z","shell.execute_reply":"2024-12-03T03:36:52.366297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nPCIAT-PCIAT_Total 기술통계 확인\n''' \ncalculate_stats(train, 'PCIAT-PCIAT_Total')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:52.369121Z","iopub.execute_input":"2024-12-03T03:36:52.369474Z","iopub.status.idle":"2024-12-03T03:36:52.403964Z","shell.execute_reply.started":"2024-12-03T03:36:52.369439Z","shell.execute_reply":"2024-12-03T03:36:52.402758Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nsii == 3인 참가자들 개별 확인\n'''\n\ntrain[train['sii'] == 3]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:52.405406Z","iopub.execute_input":"2024-12-03T03:36:52.406011Z","iopub.status.idle":"2024-12-03T03:36:52.469311Z","shell.execute_reply.started":"2024-12-03T03:36:52.405952Z","shell.execute_reply":"2024-12-03T03:36:52.468271Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# CGAS","metadata":{}},{"cell_type":"code","source":"'''\nCGAS-CGAS_Season 기술통계 확인\n''' \ncalculate_stats(train, 'CGAS-Season')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:52.470461Z","iopub.execute_input":"2024-12-03T03:36:52.470887Z","iopub.status.idle":"2024-12-03T03:36:52.492822Z","shell.execute_reply.started":"2024-12-03T03:36:52.470842Z","shell.execute_reply":"2024-12-03T03:36:52.491157Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nCGAS-CGAS_Score 기술통계 확인\n''' \ncalculate_stats(train, 'CGAS-CGAS_Score')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:52.498065Z","iopub.execute_input":"2024-12-03T03:36:52.498435Z","iopub.status.idle":"2024-12-03T03:36:52.528252Z","shell.execute_reply.started":"2024-12-03T03:36:52.498397Z","shell.execute_reply":"2024-12-03T03:36:52.526329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nCGAS 점수 변수의 age 최소-최대 확인\n'''\ndata = train[train['CGAS-CGAS_Score'].notnull()] #condi.\nage_range = data['Basic_Demos-Age']\nprint(\n    f\"Age range for participants with CGAS-CGAS_Score data:\"\n    f\" {age_range.min()} - {age_range.max()} years\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:52.530263Z","iopub.execute_input":"2024-12-03T03:36:52.530823Z","iopub.status.idle":"2024-12-03T03:36:52.539987Z","shell.execute_reply.started":"2024-12-03T03:36:52.530761Z","shell.execute_reply":"2024-12-03T03:36:52.538604Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CGAS 이상치 처리","metadata":{}},{"cell_type":"code","source":"'''\n1~100점이므로 100점이 넘어가는 행 확인\n''' \ntrain[train['CGAS-CGAS_Score'] > 100]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:52.541579Z","iopub.execute_input":"2024-12-03T03:36:52.542060Z","iopub.status.idle":"2024-12-03T03:36:52.569154Z","shell.execute_reply.started":"2024-12-03T03:36:52.542010Z","shell.execute_reply":"2024-12-03T03:36:52.567147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n이상치 1개는 결측치 처리 후 결측치 비율 계산 및 Season와 (이상치 제거한) Score의 그래프 그리기\n'''\ntrain.loc[train['CGAS-CGAS_Score'] == 999, 'CGAS-CGAS_Score'] = np.nan\n\nmissing_ratio = train['CGAS-Season'].isna().mean()\nprint(f\"{'CGAS-Season'}의 결측치 비율: {missing_ratio:.2%}\")\n\nmissing_ratio = train['CGAS-CGAS_Score'].isna().mean()\nprint(f\"{'CGAS-CGAS_Score'}의 결측치 비율: {missing_ratio:.2%}\")\n\nplt.figure(figsize=(12, 5))\n\n# CGAS-Season\nplt.subplot(1, 2, 1)\ncgas_season_counts = train['CGAS-Season'].value_counts(normalize=True)\nplt.pie(\n    cgas_season_counts,\n    labels=cgas_season_counts.index,\n    autopct='%1.1f%%',\n    startangle=90,\n    colors=sns.color_palette(\"Set3\")\n)\nplt.title('CGAS-Season')\nplt.axis('equal')\n\n# CGAS-CGAS_Score without outliers (score == 999)\nplt.subplot(1, 2, 2)\nsns.histplot(\n    train['CGAS-CGAS_Score'].dropna(),\n    bins=20, kde=True #Kernel Density Estimation\n)\nplt.title('CGAS-CGAS_Score (Without Outlier)')\nplt.xlabel('CGAS Score')\nplt.ylabel('Count')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:52.570743Z","iopub.execute_input":"2024-12-03T03:36:52.571203Z","iopub.status.idle":"2024-12-03T03:36:53.246766Z","shell.execute_reply.started":"2024-12-03T03:36:52.571150Z","shell.execute_reply":"2024-12-03T03:36:53.245517Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nCGAS reference에 맞춰 범주형 변수(new_CGAS_Score_Bin) 생성 및 분포 확인\n'''\nbins = np.arange(0, 101, 10)\nlabels = [\n    \"1-10: Needs constant supervision (24 hour care)\",\n    \"11-20: Needs considerable supervision\",\n    \"21-30: Unable to function in almost all areas\",\n    \"31-40: Major impairment in functioning in several areas\",\n    \"41-50: Moderate degree of interference in functioning\",\n    \"51-60: Variable functioning with sporadic difficulties\",\n    \"61-70: Some difficulty in a single area\",\n    \"71-80: No more than slight impairment in functioning\",\n    \"81-90: Good functioning in all areas\",\n    \"91-100: Superior functioning\" #refer.\n]\n\ntrain['new_CGAS_Score_Bin'] = pd.cut(\n    train['CGAS-CGAS_Score'], bins=bins, labels=labels\n) #new var.\n\ncounts = train['new_CGAS_Score_Bin'].value_counts().reindex(labels)\nprop = (counts / counts.sum() * 100).round(1)\ncount_prop_labels = counts.astype(str) + \" (\" + prop.astype(str) + \"%)\"\n\nplt.figure(figsize=(18, 6))\nbars = plt.barh(labels, counts)\nplt.xlabel('Count')\nplt.title('CGAS Score Distribution')\n\nfor bar, label in zip(bars, count_prop_labels):\n    plt.text(\n        bar.get_width(), bar.get_y() + bar.get_height() / 2, label, va='center'\n    ) #text code\n\nplt.gca().invert_yaxis()\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:53.248023Z","iopub.execute_input":"2024-12-03T03:36:53.248336Z","iopub.status.idle":"2024-12-03T03:36:53.853652Z","shell.execute_reply.started":"2024-12-03T03:36:53.248305Z","shell.execute_reply":"2024-12-03T03:36:53.852034Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SII과 상관관계 확인","metadata":{}},{"cell_type":"markdown","source":"아래 전처리들은 PAQ, SDSC에도 동일하게 적용","metadata":{}},{"cell_type":"code","source":"'''\nPCIAT 20개의 문항 중 하나라도 NA면 PCIAT-PCIAT_Total 변수도 NA인 변수(complete_resp_total) 생성\n'''\nPCIAT_cols = [f'PCIAT-PCIAT_{i+1:02d}' for i in range(20)] #PCIAT 문항 변수 이름 모음\n\ntrain['complete_resp_total'] = train['PCIAT-PCIAT_Total'].where(\n    train[PCIAT_cols].notna().all(axis=1), np.nan\n) #문항의 값 중 하나라도 NA면 np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:53.855300Z","iopub.execute_input":"2024-12-03T03:36:53.855633Z","iopub.status.idle":"2024-12-03T03:36:53.865906Z","shell.execute_reply.started":"2024-12-03T03:36:53.855602Z","shell.execute_reply":"2024-12-03T03:36:53.864709Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSII 변수를 ordered 범주형 변수(new_sii_ordered)로 만들기\n'''\nsii_map = {0: '0 (None)', 1: '1 (Mild)', 2: '2 (Moderate)', 3: '3 (Severe)'}\ntrain['new_sii_ordered'] = train['sii'].map(sii_map).fillna('Missing') #dictionary 이용해 값 대응\n\nsii_order = ['Missing', '0 (None)', '1 (Mild)', '2 (Moderate)', '3 (Severe)']\ntrain['new_sii_ordered'] = pd.Categorical(train['new_sii_ordered'], categories=sii_order, ordered=True) #cate.로 변환","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:53.867242Z","iopub.execute_input":"2024-12-03T03:36:53.867608Z","iopub.status.idle":"2024-12-03T03:36:53.886161Z","shell.execute_reply.started":"2024-12-03T03:36:53.867571Z","shell.execute_reply":"2024-12-03T03:36:53.884838Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nnew_CGAS_Score_Bin, complete_resp_total에서 NA 행 제거하고 안쓰는 범주는 제거한 다음에 총 data 개수 보기\n'''\ntrain_filt = train.dropna(subset=['new_CGAS_Score_Bin', 'complete_resp_total'])\ntrain_filt.loc[:, 'new_CGAS_Score_Bin'] = train_filt['new_CGAS_Score_Bin'].cat.remove_unused_categories() #안쓰는 범주는 제거\ntrain_filt.loc[:, 'new_sii_ordered'] = train_filt['new_sii_ordered'].cat.remove_unused_categories()\nlen(train_filt)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:53.888056Z","iopub.execute_input":"2024-12-03T03:36:53.888521Z","iopub.status.idle":"2024-12-03T03:36:53.915928Z","shell.execute_reply.started":"2024-12-03T03:36:53.888466Z","shell.execute_reply":"2024-12-03T03:36:53.914547Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nage 변수를 범주화하기\n'''\ntrain['Age_Group'] = pd.cut(\n    train['Basic_Demos-Age'],\n    bins=[4, 12, 18, 22], #age 나눌 구간 지정\n    labels=['Children (5-12)', 'Adolescents (13-18)', 'Adults (19-22)']\n)\ncalculate_stats(train, 'Age_Group')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:53.917315Z","iopub.execute_input":"2024-12-03T03:36:53.917713Z","iopub.status.idle":"2024-12-03T03:36:53.935216Z","shell.execute_reply.started":"2024-12-03T03:36:53.917632Z","shell.execute_reply":"2024-12-03T03:36:53.933940Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nsex 변수 라벨링\n'''\nsex_map = {0: 'Male', 1: 'Female'} #value labels 참고\ntrain['Basic_Demos-Sex'] = train['Basic_Demos-Sex'].map(sex_map)\ncalculate_stats(train, 'Basic_Demos-Sex')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:53.936765Z","iopub.execute_input":"2024-12-03T03:36:53.937121Z","iopub.status.idle":"2024-12-03T03:36:53.955186Z","shell.execute_reply.started":"2024-12-03T03:36:53.937075Z","shell.execute_reply":"2024-12-03T03:36:53.954088Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n1. CGAS-CGAS_Score vs sii\n2. complete_resp_total vs CGAS_Score_Bin\n3. CGAS-CGAS_Score vs Age Group\n4. CGAS-CGAS_Score vs Basic_Demos-Sex\n'''\nfig, axes = plt.subplots(2, 2, figsize=(16, 12))\n\ncorr = train[['CGAS-CGAS_Score', 'complete_resp_total']].corr().iloc[0, 1]\nprint(f\"CGAS-CGAS_Score와 complete_resp_total 간의 상관 계수: {corr:.2f}\")\n\n# CGAS-CGAS_Score vs sii\nsns.boxplot(\n    data=train_filt,\n    x='new_sii_ordered', y='CGAS-CGAS_Score',\n    palette='Set3', ax=axes[0,0]\n)\naxes[0,0].set_xlabel('SII Score')\naxes[0,0].set_ylabel('CGAS Score')\naxes[0,0].set_title('Distribution of CGAS Scores by SII')\n\n# complete_resp_total vs CGAS_Score_Bin\nsns.boxplot(\n    data=train_filt,\n    x='new_CGAS_Score_Bin', y='complete_resp_total',\n    ax=axes[0,1], palette='Set3'\n)\n\n# Get the tick positions and match the labels\nrange_labels = [label.split(\":\")[0] for label in train_filt['new_CGAS_Score_Bin'].cat.categories] #기존 label : 기준으로 쪼개서 앞 부분 사용\naxes[0,1].set_xticklabels(range_labels)\n\naxes[0,1].set_xlabel('CGAS Score category')\naxes[0,1].set_ylabel('PCIAT_Total for Complete PCIAT Responses')\naxes[0,1].set_title('Distribution of PCIAT_Total by CGAS Score categories')\n\n# CGAS-CGAS_Score vs Age Group\nsns.boxplot(\n    x='Age_Group', y='CGAS-CGAS_Score',\n    data=train, palette=\"Set3\", ax=axes[1,0]\n)\naxes[1,0].set_title('CGAS-CGAS_Score by Age Group')\naxes[1,0].set_ylabel('CGAS-CGAS_Score')\naxes[1,0].set_xlabel('Age Group')\n\n# CGAS-CGAS_Score vs Sex\nsns.boxplot(\n    x='Basic_Demos-Sex', y='CGAS-CGAS_Score',\n    data=train, palette=\"Set3\", ax=axes[1,1]\n)\naxes[1,1].set_title('CGAS-CGAS_Score by Basic_Demos-Sex')\naxes[1,1].set_ylabel('CGAS-CGAS_Score')\naxes[1,1].set_xlabel('Basic_Demos-Sex')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:53.956520Z","iopub.execute_input":"2024-12-03T03:36:53.956859Z","iopub.status.idle":"2024-12-03T03:36:55.174730Z","shell.execute_reply.started":"2024-12-03T03:36:53.956825Z","shell.execute_reply":"2024-12-03T03:36:55.173547Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSII와 동시에 결측인 행의 수 확인\n'''\ntrain[(train['PCIAT-PCIAT_Total'].isnull()) &\n      (train['CGAS-CGAS_Score'].isnull())][\n    ['id','Basic_Demos-Age','Basic_Demos-Sex','CGAS-CGAS_Score','PCIAT-PCIAT_Total','sii']\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:55.176094Z","iopub.execute_input":"2024-12-03T03:36:55.176418Z","iopub.status.idle":"2024-12-03T03:36:55.196103Z","shell.execute_reply.started":"2024-12-03T03:36:55.176384Z","shell.execute_reply":"2024-12-03T03:36:55.194906Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SII 결측 없는 data","metadata":{}},{"cell_type":"code","source":"'''\nSII 변수 결측 없는 data 만들기\n'''\ntrain_sii_not_NA=train[(train['PCIAT-PCIAT_Total'].notnull())]\ntrain_sii_not_NA","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:55.197726Z","iopub.execute_input":"2024-12-03T03:36:55.198592Z","iopub.status.idle":"2024-12-03T03:36:55.236600Z","shell.execute_reply.started":"2024-12-03T03:36:55.198528Z","shell.execute_reply":"2024-12-03T03:36:55.235491Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n결측 없는 data에서 CGAS가 결측인지 아닌지에 따른 sii 분포 차이 확인\n'''\nplt.figure(figsize=(18, 5))\n\nplt.subplot(1, 3, 1)\nsns.scatterplot(x=train_sii_not_NA['CGAS-CGAS_Score'], y=train_sii_not_NA['complete_resp_total'], alpha=0.7)\nplt.title('Scatter Plot of CGAS-CGAS_Score vs PCIAT')\nplt.xlabel('PCGAS-CGAS_Score')\nplt.ylabel('PCIAT')\n\nplt.subplot(1, 3, 2)\nsns.histplot(train_sii_not_NA[train_sii_not_NA['CGAS-CGAS_Score'].isnull()]['new_sii_ordered'])\nplt.title('CGAS-CGAS_Score (all NA)')\n\nplt.subplot(1, 3, 3)\nsns.histplot(train_sii_not_NA[train_sii_not_NA['CGAS-CGAS_Score'].notnull()]['new_sii_ordered'])\nplt.title('CGAS-CGAS_Score (not NA)')\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T04:03:38.589557Z","iopub.execute_input":"2024-12-03T04:03:38.590142Z","iopub.status.idle":"2024-12-03T04:03:39.325628Z","shell.execute_reply.started":"2024-12-03T04:03:38.590079Z","shell.execute_reply":"2024-12-03T04:03:39.324473Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSII 결측 없는 data에서 CGAS 점수 결측치 개수 보기\n'''\ntrain_sii_not_NA[(train_sii_not_NA['CGAS-CGAS_Score'].isnull())][\n    ['id','Basic_Demos-Age','Basic_Demos-Sex','CGAS-CGAS_Score','PCIAT-PCIAT_Total','sii']\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:55.238265Z","iopub.execute_input":"2024-12-03T03:36:55.238849Z","iopub.status.idle":"2024-12-03T03:36:55.259021Z","shell.execute_reply.started":"2024-12-03T03:36:55.238793Z","shell.execute_reply":"2024-12-03T03:36:55.257826Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n결측치 비율 계산 및 (이상치 제거한) 그래프 그리기\n'''\n\nmissing_ratio = train_sii_not_NA['CGAS-CGAS_Score'].isna().mean()\nprint(f\"{'CGAS-CGAS_Score'}의 결측치 비율: {missing_ratio:.2%}\")\n\nplt.figure(figsize=(12, 5))\n\n# CGAS-Season\nplt.subplot(1, 2, 1)\ncgas_season_counts = train_sii_not_NA['CGAS-Season'].value_counts(normalize=True)\nplt.pie(\n    cgas_season_counts,\n    labels=cgas_season_counts.index,\n    autopct='%1.1f%%',\n    startangle=90,\n    colors=sns.color_palette(\"Set3\")\n)\nplt.title('CGAS-Season')\nplt.axis('equal')\n\n# CGAS-CGAS_Score without outliers (score == 999)\nplt.subplot(1, 2, 2)\nsns.histplot(\n    train_sii_not_NA['CGAS-CGAS_Score'].dropna(),\n    bins=20, kde=True #Kernel Density Estimation\n)\nplt.title('CGAS-CGAS_Score (Without Outlier)')\nplt.xlabel('CGAS Score')\nplt.ylabel('Count')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:55.260497Z","iopub.execute_input":"2024-12-03T03:36:55.261490Z","iopub.status.idle":"2024-12-03T03:36:55.830001Z","shell.execute_reply.started":"2024-12-03T03:36:55.261430Z","shell.execute_reply":"2024-12-03T03:36:55.828956Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr = train_sii_not_NA[['CGAS-CGAS_Score', 'sii']].corr().iloc[0, 1]\nprint(f\"CGAS-CGAS_Score와 sii 간의 상관 계수: {corr:.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:55.831282Z","iopub.execute_input":"2024-12-03T03:36:55.831605Z","iopub.status.idle":"2024-12-03T03:36:55.839589Z","shell.execute_reply.started":"2024-12-03T03:36:55.831574Z","shell.execute_reply":"2024-12-03T03:36:55.838404Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n범주형 변수를 제외하고 CGAS를 기준으로 다른 변수와의 상관계수를 구한 후 내림차순으로 확인\n'''\nnumeric_cols = train_sii_not_NA.select_dtypes(exclude=['object', 'category'])\ncorr_train = numeric_cols.corr()\ncorrelations = corr_train['CGAS-CGAS_Score'].drop('CGAS-CGAS_Score')\n\nsorted_correlations = correlations.sort_values(ascending=False)\n\nfiltered_corr = sorted_correlations[(sorted_correlations > 0.1) | (sorted_correlations < -0.1)]\nprint(filtered_corr)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:55.841167Z","iopub.execute_input":"2024-12-03T03:36:55.841606Z","iopub.status.idle":"2024-12-03T03:36:55.900683Z","shell.execute_reply.started":"2024-12-03T03:36:55.841556Z","shell.execute_reply":"2024-12-03T03:36:55.899475Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nnew_CGAS_Score_Bin와 new_sii_ordered의 상관 보기\n'''\n\nfrom scipy.stats import chi2_contingency\n\ncontingency_table = pd.crosstab(train_sii_not_NA['new_CGAS_Score_Bin'], train_sii_not_NA['new_sii_ordered'])\n\nchi2, p, dof, expected = chi2_contingency(contingency_table)\n\nprint(f\"Chi2: {chi2}, p-value: {p}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T04:27:49.236714Z","iopub.execute_input":"2024-12-03T04:27:49.237221Z","iopub.status.idle":"2024-12-03T04:27:49.265384Z","shell.execute_reply.started":"2024-12-03T04:27:49.237179Z","shell.execute_reply":"2024-12-03T04:27:49.264110Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cramers_v(chi2, n, r, c):\n    return np.sqrt(chi2 / (n * (min(r, c) - 1)))\n\nchi2, p, dof, _ = chi2_contingency(contingency_table)\n\nn = contingency_table.sum().sum()\nr, c = contingency_table.shape\ncramer_v = cramers_v(chi2, n, r, c)\n\nprint(f\"Cramér's V: {cramer_v}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T04:28:59.800112Z","iopub.execute_input":"2024-12-03T04:28:59.800541Z","iopub.status.idle":"2024-12-03T04:28:59.809603Z","shell.execute_reply.started":"2024-12-03T04:28:59.800506Z","shell.execute_reply":"2024-12-03T04:28:59.808351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nnew_CGAS_Score_Bin_Numeric와 sii의 상관 보기\n'''\nfrom scipy.stats import spearmanr\n\ntrain_sii_not_NA.loc[:, 'new_CGAS_Score_Bin_Numeric'] = pd.Categorical(train_sii_not_NA['new_CGAS_Score_Bin']).codes\ncorr, p_value = spearmanr(train_sii_not_NA['new_CGAS_Score_Bin_Numeric'], train_sii_not_NA['sii'])\nprint(f\"Spearman Correlation: {corr}, p-value: {p_value}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T04:37:30.808783Z","iopub.execute_input":"2024-12-03T04:37:30.809210Z","iopub.status.idle":"2024-12-03T04:37:30.821387Z","shell.execute_reply.started":"2024-12-03T04:37:30.809175Z","shell.execute_reply":"2024-12-03T04:37:30.820158Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# PAQ-A","metadata":{}},{"cell_type":"code","source":"'''\nPAQ_A-Season 기술통계 확인\n'''\ncalculate_stats(train, 'PAQ_A-Season')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:55.902257Z","iopub.execute_input":"2024-12-03T03:36:55.902730Z","iopub.status.idle":"2024-12-03T03:36:55.915181Z","shell.execute_reply.started":"2024-12-03T03:36:55.902674Z","shell.execute_reply":"2024-12-03T03:36:55.914146Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nPAQ_A-PAQ_A_Total 기술통계 확인\n'''\ncalculate_stats(train, 'PAQ_A-PAQ_A_Total')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:55.923913Z","iopub.execute_input":"2024-12-03T03:36:55.924300Z","iopub.status.idle":"2024-12-03T03:36:55.945195Z","shell.execute_reply.started":"2024-12-03T03:36:55.924265Z","shell.execute_reply":"2024-12-03T03:36:55.943787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nPAQ-A 점수 변수의 age 최소-최대 확인\n'''\ndata = train[train['PAQ_A-PAQ_A_Total'].notnull()] #condi.\nage_range = data['Basic_Demos-Age']\nprint(\n    f\"Age range for participants with PAQ_C-PAQ_C_Total data:\"\n    f\" {age_range.min()} - {age_range.max()} years\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:55.946773Z","iopub.execute_input":"2024-12-03T03:36:55.947227Z","iopub.status.idle":"2024-12-03T03:36:55.957337Z","shell.execute_reply.started":"2024-12-03T03:36:55.947177Z","shell.execute_reply":"2024-12-03T03:36:55.956106Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## PAQ-A 이상치 처리","metadata":{}},{"cell_type":"code","source":"'''\n1~5점이므로 1점보다 작은 행 확인\n''' \ntrain.loc[train['PAQ_A-PAQ_A_Total'] < 1, ['id','Basic_Demos-Age','Basic_Demos-Sex','PAQ_A-Season','PAQ_A-PAQ_A_Total']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:55.958575Z","iopub.execute_input":"2024-12-03T03:36:55.958985Z","iopub.status.idle":"2024-12-03T03:36:55.977152Z","shell.execute_reply.started":"2024-12-03T03:36:55.958950Z","shell.execute_reply":"2024-12-03T03:36:55.975947Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n이상치 7개는 결측치 처리 후 결측치 비율 계산 및 Season와 (이상치 제거한) Score의 그래프 그리기\n'''\ntrain.loc[train['PAQ_A-PAQ_A_Total'] < 1, 'PAQ_A-PAQ_A_Total'] = np.nan\n\nmissing_ratio = train['PAQ_A-Season'].isna().mean()\nprint(f\"{'PAQ_A-Season'}의 결측치 비율: {missing_ratio:.2%}\")\n\nmissing_ratio = train['PAQ_A-PAQ_A_Total'].isna().mean()\nprint(f\"{'PAQ_A-PAQ_A_Total'}의 결측치 비율: {missing_ratio:.2%}\")\n\nplt.figure(figsize=(18, 5))\n\n# PAQ_A-Season\nplt.subplot(1, 3, 1)\ntrain['PAQ_A-Season'].value_counts(normalize=True).plot.pie(\n    autopct='%1.1f%%', colors=plt.cm.Set3.colors\n)\nplt.title('PAQ_A-Season (Adolescents)')\n\n# PAQ_A-PAQ_A_Total\nplt.subplot(1, 3, 2)\nsns.histplot(train['PAQ_A-PAQ_A_Total'], bins=20, kde=True)\nplt.title('PAQ_A-PAQ_A_Total (Adolescents)')\n\n# PAQ_A_Total by Season\nplt.subplot(1, 3, 3)\nsns.violinplot(x='PAQ_A-Season', y='PAQ_A-PAQ_A_Total', data=train, palette=\"Set3\")\nplt.title('PAQ_A_Total by Season (Adolescents)')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:55.978797Z","iopub.execute_input":"2024-12-03T03:36:55.979255Z","iopub.status.idle":"2024-12-03T03:36:56.896730Z","shell.execute_reply.started":"2024-12-03T03:36:55.979203Z","shell.execute_reply":"2024-12-03T03:36:56.895504Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SII과 상관관계 확인","metadata":{}},{"cell_type":"code","source":"'''\nPAQ_A-PAQ_A_Total, complete_resp_total에서 NA 행 제거하고 안쓰는 범주는 제거한 다음에 총 data 개수 보기\n'''\ntrain_filt2 = train.dropna(subset=['PAQ_A-PAQ_A_Total', 'complete_resp_total'])\ntrain_filt2.loc[:, 'new_sii_ordered'] = train_filt['new_sii_ordered'].cat.remove_unused_categories()\nlen(train_filt2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:56.898284Z","iopub.execute_input":"2024-12-03T03:36:56.898771Z","iopub.status.idle":"2024-12-03T03:36:56.914241Z","shell.execute_reply.started":"2024-12-03T03:36:56.898720Z","shell.execute_reply":"2024-12-03T03:36:56.912921Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n1. PAQ_A-PAQ_A_Total vs sii\n2. PAQ_A-PAQ_A_Total vs Age Group\n3. PAQ_A-PAQ_A_Total vs Basic_Demos-Sex\n'''\nfig, axes = plt.subplots(1, 3, figsize=(16, 5))\n\ncorr = train[['PAQ_A-PAQ_A_Total', 'complete_resp_total']].corr().iloc[0, 1]\nprint(f\"PAQ_A-PAQ_A_Total와 complete_resp_total 간의 상관 계수: {corr:.2f}\")\n\n# PAQ_A-PAQ_A_Total vs sii\nsns.boxplot(\n    data=train_filt2,\n    x='new_sii_ordered', y='PAQ_A-PAQ_A_Total',\n    palette='Set3', ax=axes[0]\n)\naxes[0].set_xlabel('SII Score')\naxes[0].set_ylabel('PAQ_A-PAQ_A_Total')\naxes[0].set_title('Distribution of PAQ_A-PAQ_A_Total by SII')\n\n# PAQ_A-PAQ_A_Total vs Age Group\nsns.boxplot(\n    x='Age_Group', y='PAQ_A-PAQ_A_Total',\n    data=train_filt2, palette=\"Set3\", ax=axes[1]\n)\naxes[1].set_title('PAQ_A-PAQ_A_Total by Age Group')\naxes[1].set_ylabel('PAQ_A-PAQ_A_Total')\naxes[1].set_xlabel('Age Group')\n\n# PAQ_A-PAQ_A_Total vs Sex\nsns.boxplot(\n    x='Basic_Demos-Sex', y='PAQ_A-PAQ_A_Total',\n    data=train_filt2, palette=\"Set3\", ax=axes[2]\n)\naxes[2].set_title('PAQ_A-PAQ_A_Total by Basic_Demos-Sex')\naxes[2].set_ylabel('PAQ_A-PAQ_A_Total')\naxes[2].set_xlabel('Basic_Demos-Sex')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:56.915787Z","iopub.execute_input":"2024-12-03T03:36:56.916230Z","iopub.status.idle":"2024-12-03T03:36:57.780079Z","shell.execute_reply.started":"2024-12-03T03:36:56.916186Z","shell.execute_reply":"2024-12-03T03:36:57.778980Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSII와 동시에 결측인 행의 수 확인\n'''\ntrain[(train['PCIAT-PCIAT_Total'].isnull()) &\n      (train['PAQ_A-PAQ_A_Total'].isnull())][\n    ['id','Basic_Demos-Age','Basic_Demos-Sex','PAQ_A-PAQ_A_Total','PCIAT-PCIAT_Total','sii']\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:57.781532Z","iopub.execute_input":"2024-12-03T03:36:57.781903Z","iopub.status.idle":"2024-12-03T03:36:57.798895Z","shell.execute_reply.started":"2024-12-03T03:36:57.781862Z","shell.execute_reply":"2024-12-03T03:36:57.797742Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SII 결측 없는 data","metadata":{}},{"cell_type":"code","source":"'''\n이상치 처리를 train에서 하고 있기 때문에 다시 정의\n'''\ntrain_sii_not_NA=train[(train['PCIAT-PCIAT_Total'].notnull())]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:57.800470Z","iopub.execute_input":"2024-12-03T03:36:57.800986Z","iopub.status.idle":"2024-12-03T03:36:57.811917Z","shell.execute_reply.started":"2024-12-03T03:36:57.800935Z","shell.execute_reply":"2024-12-03T03:36:57.810932Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n결측 없는 data에서 PAQ-A가 결측인지 아닌지에 따른 sii 분포 차이 확인\n'''\nplt.figure(figsize=(18, 5))\n\nplt.subplot(1, 3, 1)\nsns.scatterplot(x=train_sii_not_NA['PAQ_A-PAQ_A_Total'], y=train_sii_not_NA['complete_resp_total'], alpha=0.7)\nplt.title('Scatter Plot of PAQ_A-PAQ_A_Total vs PCIAT')\nplt.xlabel('PAQ_A-PAQ_A_Total')\nplt.ylabel('PCIAT')\n\nplt.subplot(1, 3, 2)\nsns.histplot(train_sii_not_NA[train_sii_not_NA['PAQ_A-PAQ_A_Total'].isnull()]['new_sii_ordered'])\nplt.title('PAQ_A-PAQ_A_Total (all NA)')\n\nplt.subplot(1, 3, 3)\nsns.histplot(train_sii_not_NA[train_sii_not_NA['PAQ_A-PAQ_A_Total'].notnull()]['new_sii_ordered'])\nplt.title('PAQ_A-PAQ_A_Total (not NA)')\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:50:54.498991Z","iopub.execute_input":"2024-12-03T03:50:54.500007Z","iopub.status.idle":"2024-12-03T03:50:55.611927Z","shell.execute_reply.started":"2024-12-03T03:50:54.499951Z","shell.execute_reply":"2024-12-03T03:50:55.610755Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(18, 5))\n\nplt.subplot(1, 3, 1)\nsns.histplot(train_sii_not_NA[train_sii_not_NA['Age_Group'] == 'Children (5-12)']['new_sii_ordered'])\nplt.title('sii (Children)')\n\nplt.subplot(1, 3, 2)\nsns.histplot(train_sii_not_NA[train_sii_not_NA['Age_Group'] == 'Adolescents (13-18)']['new_sii_ordered'])\nplt.title('sii (Adolescents)')\n\nplt.subplot(1, 3, 3)\nsns.histplot(train_sii_not_NA[train_sii_not_NA['Age_Group'] == 'Adults (19-22)']['new_sii_ordered'])\nplt.title('sii (Adults)')\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:51:57.954929Z","iopub.execute_input":"2024-12-03T03:51:57.955409Z","iopub.status.idle":"2024-12-03T03:51:58.715820Z","shell.execute_reply.started":"2024-12-03T03:51:57.955369Z","shell.execute_reply":"2024-12-03T03:51:58.714419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSII 결측 없는 data에서 PAQ_A 점수 결측치 개수 보기\n'''\ntrain_sii_not_NA[(train_sii_not_NA['PAQ_A-PAQ_A_Total'].isnull())][\n    ['id','Basic_Demos-Age','Basic_Demos-Sex','PAQ_A-PAQ_A_Total','PCIAT-PCIAT_Total','sii']\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:58.342383Z","iopub.execute_input":"2024-12-03T03:36:58.342908Z","iopub.status.idle":"2024-12-03T03:36:58.363341Z","shell.execute_reply.started":"2024-12-03T03:36:58.342856Z","shell.execute_reply":"2024-12-03T03:36:58.362060Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr = train_sii_not_NA[['PAQ_A-PAQ_A_Total', 'sii']].corr().iloc[0, 1]\nprint(f\"PAQ_A-PAQ_A_Total와 sii 간의 상관 계수: {corr:.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:58.364788Z","iopub.execute_input":"2024-12-03T03:36:58.365146Z","iopub.status.idle":"2024-12-03T03:36:58.377028Z","shell.execute_reply.started":"2024-12-03T03:36:58.365107Z","shell.execute_reply":"2024-12-03T03:36:58.375605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n범주형 변수를 제외하고 PAQ-A를 기준으로 다른 변수와의 상관계수를 구한 후 내림차순으로 확인\n'''\nnumeric_cols = train_sii_not_NA.select_dtypes(exclude=['object', 'category'])\ncorr_train = numeric_cols.corr()\ncorrelations = corr_train['PAQ_A-PAQ_A_Total'].drop('PAQ_A-PAQ_A_Total')\n\nsorted_correlations = correlations.sort_values(ascending=False)\n\nfiltered_corr = sorted_correlations[(sorted_correlations > 0.1) | (sorted_correlations < -0.1)]\nprint(filtered_corr)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:58.378579Z","iopub.execute_input":"2024-12-03T03:36:58.379105Z","iopub.status.idle":"2024-12-03T03:36:58.433123Z","shell.execute_reply.started":"2024-12-03T03:36:58.379049Z","shell.execute_reply":"2024-12-03T03:36:58.431960Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# PAQ-C","metadata":{}},{"cell_type":"code","source":"'''\nPAQ_C-Season 기술통계 확인\n'''\ncalculate_stats(train, 'PAQ_C-Season')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:58.435001Z","iopub.execute_input":"2024-12-03T03:36:58.435451Z","iopub.status.idle":"2024-12-03T03:36:58.447839Z","shell.execute_reply.started":"2024-12-03T03:36:58.435405Z","shell.execute_reply":"2024-12-03T03:36:58.446633Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nPAQ_C-PAQ_C_Total 기술통계 확인\n'''\ncalculate_stats(train, 'PAQ_C-PAQ_C_Total')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:58.449488Z","iopub.execute_input":"2024-12-03T03:36:58.449937Z","iopub.status.idle":"2024-12-03T03:36:58.471574Z","shell.execute_reply.started":"2024-12-03T03:36:58.449888Z","shell.execute_reply":"2024-12-03T03:36:58.470490Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nPAQ-C 점수 변수의 age 최소-최대 확인\n'''\ndata = train[train['PAQ_C-PAQ_C_Total'].notnull()] #condi.\nage_range = data['Basic_Demos-Age']\nprint(\n    f\"Age range for participants with PAQ_C-PAQ_C_Total data:\"\n    f\" {age_range.min()} - {age_range.max()} years\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:58.472908Z","iopub.execute_input":"2024-12-03T03:36:58.473205Z","iopub.status.idle":"2024-12-03T03:36:58.482240Z","shell.execute_reply.started":"2024-12-03T03:36:58.473174Z","shell.execute_reply":"2024-12-03T03:36:58.481149Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## PAQ-C 이상치 처리","metadata":{}},{"cell_type":"code","source":"'''\n1~5점이므로 1점보다 작은 행 확인\n''' \ntrain.loc[train['PAQ_C-PAQ_C_Total'] < 1, ['id','Basic_Demos-Age','Basic_Demos-Sex','PAQ_C-Season','PAQ_C-PAQ_C_Total']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:58.483935Z","iopub.execute_input":"2024-12-03T03:36:58.485119Z","iopub.status.idle":"2024-12-03T03:36:58.500271Z","shell.execute_reply.started":"2024-12-03T03:36:58.485065Z","shell.execute_reply":"2024-12-03T03:36:58.499113Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n이상치 11개는 결측치 처리 후 결측치 비율 계산 및 Season와 (이상치 제거한) Score의 그래프 그리기\n'''\ntrain.loc[train['PAQ_C-PAQ_C_Total'] < 1, 'PAQ_C-PAQ_C_Total'] = np.nan\n\nmissing_ratio = train['PAQ_C-Season'].isna().mean()\nprint(f\"{'PAQ_C-Season'}의 결측치 비율: {missing_ratio:.2%}\")\n\nmissing_ratio = train['PAQ_C-PAQ_C_Total'].isna().mean()\nprint(f\"{'PAQ_C-PAQ_C_Total'}의 결측치 비율: {missing_ratio:.2%}\")\n\nplt.figure(figsize=(18, 5))\n\n# PAQ_C-Season\nplt.subplot(1, 3, 1)\ntrain['PAQ_C-Season'].value_counts(normalize=True).plot.pie(\n    autopct='%1.1f%%', colors=plt.cm.Set3.colors\n)\nplt.title('PAQ_C-Season (Children)')\n\n# PAQ_C-PAQ_C_Total\nplt.subplot(1, 3, 2)\nsns.histplot(train['PAQ_C-PAQ_C_Total'], bins=20, kde=True)\nplt.title('PAQ_C-PAQ_C_Total (Children)')\n\n# PAQ_C_Total by Season\nplt.subplot(1, 3, 3)\nsns.violinplot(x='PAQ_C-Season', y='PAQ_C-PAQ_C_Total', data=train, palette=\"Set3\")\nplt.title('PAQ_C_Total by Season (Children)')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:58.501856Z","iopub.execute_input":"2024-12-03T03:36:58.502994Z","iopub.status.idle":"2024-12-03T03:36:59.484144Z","shell.execute_reply.started":"2024-12-03T03:36:58.502947Z","shell.execute_reply":"2024-12-03T03:36:59.482996Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SII과 상관관계 확인","metadata":{}},{"cell_type":"code","source":"'''\nPAQ_C-PAQ_C_Total, complete_resp_total에서 NA 행 제거하고 안쓰는 범주는 제거한 다음에 총 data 개수 보기\n'''\ntrain_filt3 = train.dropna(subset=['PAQ_C-PAQ_C_Total', 'complete_resp_total'])\ntrain_filt3.loc[:, 'new_sii_ordered'] = train_filt['new_sii_ordered'].cat.remove_unused_categories()\nlen(train_filt3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:59.485783Z","iopub.execute_input":"2024-12-03T03:36:59.486230Z","iopub.status.idle":"2024-12-03T03:36:59.500782Z","shell.execute_reply.started":"2024-12-03T03:36:59.486182Z","shell.execute_reply":"2024-12-03T03:36:59.499680Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n1. PAQ_C-PAQ_C_Total vs sii\n2. PAQ_C-PAQ_C_Total vs Age Group\n3. PAQ_C-PAQ_C_Total vs Basic_Demos-Sex\n'''\nfig, axes = plt.subplots(1, 3, figsize=(16, 5))\n\ncorr = train[['PAQ_C-PAQ_C_Total', 'complete_resp_total']].corr().iloc[0, 1]\nprint(f\"PAQ_C-PAQ_C_Total와 complete_resp_total 간의 상관 계수: {corr:.2f}\")\n\n# PAQ_C-PAQ_C_Total vs sii\nsns.boxplot(\n    data=train_filt3,\n    x='new_sii_ordered', y='PAQ_C-PAQ_C_Total',\n    palette='Set3', ax=axes[0]\n)\naxes[0].set_xlabel('SII Score')\naxes[0].set_ylabel('PAQ_C-PAQ_C_Total')\naxes[0].set_title('Distribution of PAQ_C-PAQ_C_Total by SII')\n\n# PAQ_C-PAQ_C_Total vs Age Group\nsns.boxplot(\n    x='Age_Group', y='PAQ_C-PAQ_C_Total',\n    data=train_filt3, palette=\"Set3\", ax=axes[1]\n)\naxes[1].set_title('PAQ_C-PAQ_C_Total by Age Group')\naxes[1].set_ylabel('PAQ_C-PAQ_C_Total')\naxes[1].set_xlabel('Age Group')\n\n# PAQ_C-PAQ_C_Total vs Sex\nsns.boxplot(\n    x='Basic_Demos-Sex', y='PAQ_C-PAQ_C_Total',\n    data=train_filt3, palette=\"Set3\", ax=axes[2]\n)\naxes[2].set_title('PAQ_C-PAQ_C_Total by Basic_Demos-Sex')\naxes[2].set_ylabel('PAQ_C-PAQ_C_Total')\naxes[2].set_xlabel('Basic_Demos-Sex')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:36:59.502099Z","iopub.execute_input":"2024-12-03T03:36:59.502452Z","iopub.status.idle":"2024-12-03T03:37:00.363301Z","shell.execute_reply.started":"2024-12-03T03:36:59.502417Z","shell.execute_reply":"2024-12-03T03:37:00.362019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSII와 동시에 결측인 행의 수 확인\n'''\ntrain[(train['PCIAT-PCIAT_Total'].isnull()) &\n      (train['PAQ_C-PAQ_C_Total'].isnull())][\n    ['id','Basic_Demos-Age','Basic_Demos-Sex','PAQ_C-PAQ_C_Total','PCIAT-PCIAT_Total','sii']\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:00.365136Z","iopub.execute_input":"2024-12-03T03:37:00.365597Z","iopub.status.idle":"2024-12-03T03:37:00.386822Z","shell.execute_reply.started":"2024-12-03T03:37:00.365547Z","shell.execute_reply":"2024-12-03T03:37:00.385524Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SII 결측 없는 data","metadata":{}},{"cell_type":"code","source":"'''\n이상치 처리를 train에서 하고 있기 때문에 다시 정의\n'''\ntrain_sii_not_NA=train[(train['PCIAT-PCIAT_Total'].notnull())]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:00.388383Z","iopub.execute_input":"2024-12-03T03:37:00.388792Z","iopub.status.idle":"2024-12-03T03:37:00.396925Z","shell.execute_reply.started":"2024-12-03T03:37:00.388751Z","shell.execute_reply":"2024-12-03T03:37:00.395749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n결측 없는 data에서 PAQ-C가 결측인지 아닌지에 따른 sii 분포 차이 확인\n'''\nplt.figure(figsize=(18, 5))\n\nplt.subplot(1, 3, 1)\nsns.scatterplot(x=train_sii_not_NA['PAQ_C-PAQ_C_Total'], y=train_sii_not_NA['complete_resp_total'], alpha=0.7)\nplt.title('Scatter Plot of PAQ_C-PAQ_C_Total vs PCIAT')\nplt.xlabel('PAQ_A-PAQ_A_Total')\nplt.ylabel('PCIAT')\n\nplt.subplot(1, 3, 2)\nsns.histplot(train_sii_not_NA[train_sii_not_NA['PAQ_C-PAQ_C_Total'].isnull()]['new_sii_ordered'])\nplt.title('PAQ_C-PAQ_C_Total (all NA)')\n\nplt.subplot(1, 3, 3)\nsns.histplot(train_sii_not_NA[train_sii_not_NA['PAQ_C-PAQ_C_Total'].notnull()]['new_sii_ordered'])\nplt.title('PAQ_C-PAQ_C_Total (not NA)')\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:40:35.862796Z","iopub.execute_input":"2024-12-03T03:40:35.863313Z","iopub.status.idle":"2024-12-03T03:40:36.658793Z","shell.execute_reply.started":"2024-12-03T03:40:35.863265Z","shell.execute_reply":"2024-12-03T03:40:36.657559Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSII 결측 없는 data에서 PAQ_C 점수 결측치 개수 보기\n'''\ntrain_sii_not_NA[(train_sii_not_NA['PAQ_C-PAQ_C_Total'].isnull())][\n    ['id','Basic_Demos-Age','Basic_Demos-Sex','PAQ_C-PAQ_C_Total','PCIAT-PCIAT_Total','sii']\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:00.398288Z","iopub.execute_input":"2024-12-03T03:37:00.398597Z","iopub.status.idle":"2024-12-03T03:37:00.423418Z","shell.execute_reply.started":"2024-12-03T03:37:00.398567Z","shell.execute_reply":"2024-12-03T03:37:00.421642Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr = train_sii_not_NA[['PAQ_C-PAQ_C_Total', 'sii']].corr().iloc[0, 1]\nprint(f\"PAQ_C-PAQ_C_Total와 sii 간의 상관 계수: {corr:.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:00.425576Z","iopub.execute_input":"2024-12-03T03:37:00.426151Z","iopub.status.idle":"2024-12-03T03:37:00.434209Z","shell.execute_reply.started":"2024-12-03T03:37:00.426109Z","shell.execute_reply":"2024-12-03T03:37:00.432966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n범주형 변수를 제외하고 PAQ-C를 기준으로 다른 변수와의 상관계수를 구한 후 내림차순으로 확인\n'''\nnumeric_cols = train_sii_not_NA.select_dtypes(exclude=['object', 'category'])\ncorr_train = numeric_cols.corr()\ncorrelations = corr_train['PAQ_C-PAQ_C_Total'].drop('PAQ_C-PAQ_C_Total')\n\nsorted_correlations = correlations.sort_values(ascending=False)\n\nfiltered_corr = sorted_correlations[(sorted_correlations > 0.1) | (sorted_correlations < -0.1)]\nprint(filtered_corr)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:00.435795Z","iopub.execute_input":"2024-12-03T03:37:00.436226Z","iopub.status.idle":"2024-12-03T03:37:00.494341Z","shell.execute_reply.started":"2024-12-03T03:37:00.436158Z","shell.execute_reply":"2024-12-03T03:37:00.493164Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## PAQ-A & PAQ-C 병합","metadata":{}},{"cell_type":"code","source":"'''\nPAQ_A & PAQ_C 동시에 있는 행 확인\n'''\npaq_columns = [col for col in train.columns if 'PAQ' in col]\ntrain[(train['PAQ_A-PAQ_A_Total'].notnull()) &\n      (train['PAQ_C-PAQ_C_Total'].notnull())][\n    paq_columns + ['Basic_Demos-Age']\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:00.495766Z","iopub.execute_input":"2024-12-03T03:37:00.496103Z","iopub.status.idle":"2024-12-03T03:37:00.511384Z","shell.execute_reply.started":"2024-12-03T03:37:00.496068Z","shell.execute_reply":"2024-12-03T03:37:00.510078Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n값이 동시에 있는 1개의 행은 두 열의 평균값을 계산하고, 하나만 존재할 경우 해당 값을 사용(병합)\n'''\ntrain['new_PAQ_Total'] = train[['PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total']].mean(axis=1, skipna=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:00.512624Z","iopub.execute_input":"2024-12-03T03:37:00.512939Z","iopub.status.idle":"2024-12-03T03:37:00.527774Z","shell.execute_reply.started":"2024-12-03T03:37:00.512907Z","shell.execute_reply":"2024-12-03T03:37:00.526644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nnew_PAQ_Total 기술통계 확인\n'''\ncalculate_stats(train, 'new_PAQ_Total')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:00.529077Z","iopub.execute_input":"2024-12-03T03:37:00.529428Z","iopub.status.idle":"2024-12-03T03:37:00.554553Z","shell.execute_reply.started":"2024-12-03T03:37:00.529374Z","shell.execute_reply":"2024-12-03T03:37:00.553403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n이미 이상치를 A, C 둘 다 처리했기 때문에 이상치 없음\n'''\ntrain.loc[train['new_PAQ_Total'] < 1, ['id','Basic_Demos-Age','Basic_Demos-Sex','new_PAQ_Total']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:00.556180Z","iopub.execute_input":"2024-12-03T03:37:00.556562Z","iopub.status.idle":"2024-12-03T03:37:00.568888Z","shell.execute_reply.started":"2024-12-03T03:37:00.556525Z","shell.execute_reply":"2024-12-03T03:37:00.567811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nnew_PAQ_Total, complete_resp_total에서 NA 행 제거하고 안쓰는 범주는 제거한 다음에 총 data 개수 보기\n'''\ntrain_filt5 = train.dropna(subset=['new_PAQ_Total', 'complete_resp_total'])\ntrain_filt5.loc[:, 'new_sii_ordered'] = train_filt['new_sii_ordered'].cat.remove_unused_categories()\nlen(train_filt5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:00.570432Z","iopub.execute_input":"2024-12-03T03:37:00.570929Z","iopub.status.idle":"2024-12-03T03:37:00.590532Z","shell.execute_reply.started":"2024-12-03T03:37:00.570877Z","shell.execute_reply":"2024-12-03T03:37:00.589449Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n결측치 비율 계산 및 Season와 (이상치 제거한) Score의 그래프 그리기\n\n1. new_PAQ_Total 분포\n2. new_PAQ_Total vs sii\n3. new_PAQ_Total vs Age Group\n4. new_PAQ_Total vs Basic_Demos-Sex\n'''\n\nmissing_ratio = train['new_PAQ_Total'].isna().mean()\nprint(f\"{'new_PAQ_Total'}의 결측치 비율: {missing_ratio:.2%}\")\n\ncorr = train[['new_PAQ_Total', 'complete_resp_total']].corr().iloc[0, 1]\nprint(f\"new_PAQ_Total와 complete_resp_total 간의 상관 계수: {corr:.2f}\")\n\nfig, axes = plt.subplots(2, 2, figsize=(16, 12))\n\n# new_PAQ_Total\nsns.histplot(train['new_PAQ_Total'], bins=20, kde=True, ax=axes[0,0])\naxes[0,0].set_title('new_PAQ_Total')\n\n# new_PAQ_Total vs sii\nsns.boxplot(\n    data=train_filt5,\n    x='new_sii_ordered', y='new_PAQ_Total',\n    palette='Set3', ax=axes[0,1]\n)\naxes[0,1].set_xlabel('SII Score')\naxes[0,1].set_ylabel('new_PAQ_Total')\naxes[0,1].set_title('Distribution of PAQ_C-PAQ_C_Total by SII')\n\n# new_PAQ_Total vs Age Group\nsns.boxplot(\n    x='Age_Group', y='new_PAQ_Total',\n    data=train_filt5, palette=\"Set3\", ax=axes[1,0]\n)\naxes[1,0].set_title('new_PAQ_Totalby Age Group')\naxes[1,0].set_ylabel('new_PAQ_Total')\naxes[1,0].set_xlabel('Age Group')\n\n# new_PAQ_Total vs Sex\nsns.boxplot(\n    x='Basic_Demos-Sex', y='new_PAQ_Total',\n    data=train_filt5, palette=\"Set3\", ax=axes[1,1]\n)\naxes[1,1].set_title('new_PAQ_Total by Basic_Demos-Sex')\naxes[1,1].set_ylabel('new_PAQ_Total')\naxes[1,1].set_xlabel('Basic_Demos-Sex')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:00.592781Z","iopub.execute_input":"2024-12-03T03:37:00.593250Z","iopub.status.idle":"2024-12-03T03:37:02.088576Z","shell.execute_reply.started":"2024-12-03T03:37:00.593198Z","shell.execute_reply":"2024-12-03T03:37:02.087515Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSII와 동시에 결측인 행의 수 확인\n'''\ntrain[(train['PCIAT-PCIAT_Total'].isnull()) &\n      (train['new_PAQ_Total'].isnull())][\n    ['id','Basic_Demos-Age','Basic_Demos-Sex','new_PAQ_Total','PCIAT-PCIAT_Total','sii']\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:02.090424Z","iopub.execute_input":"2024-12-03T03:37:02.091446Z","iopub.status.idle":"2024-12-03T03:37:02.111252Z","shell.execute_reply.started":"2024-12-03T03:37:02.091391Z","shell.execute_reply":"2024-12-03T03:37:02.109957Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SII 결측 없는 data","metadata":{}},{"cell_type":"code","source":"'''\n새로운 변수를 train에서 넣었기 때문에 다시 정의\n'''\ntrain_sii_not_NA=train[(train['PCIAT-PCIAT_Total'].notnull())]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:02.112492Z","iopub.execute_input":"2024-12-03T03:37:02.112931Z","iopub.status.idle":"2024-12-03T03:37:02.121393Z","shell.execute_reply.started":"2024-12-03T03:37:02.112889Z","shell.execute_reply":"2024-12-03T03:37:02.120331Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n결측 없는 data에서 PAQ_Total가 결측인지 아닌지에 따른 sii 분포 차이 확인\n'''\nplt.figure(figsize=(18, 5))\n\nplt.subplot(1, 3, 1)\nsns.scatterplot(x=train_sii_not_NA['new_PAQ_Total'], y=train_sii_not_NA['complete_resp_total'], alpha=0.7)\nplt.title('Scatter Plot of new_PAQ_Total vs PCIAT')\nplt.xlabel('new_PAQ_Total')\nplt.ylabel('PCIAT')\n\nplt.subplot(1, 3, 2)\nsns.histplot(train_sii_not_NA[train_sii_not_NA['new_PAQ_Total'].isnull()]['new_sii_ordered'])\nplt.title('new_PAQ_Total (all NA)')\n\nplt.subplot(1, 3, 3)\nsns.histplot(train_sii_not_NA[train_sii_not_NA['new_PAQ_Total'].notnull()]['new_sii_ordered'])\nplt.title('new_PAQ_Total (not NA)')\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:43:04.320556Z","iopub.execute_input":"2024-12-03T03:43:04.321048Z","iopub.status.idle":"2024-12-03T03:43:05.100227Z","shell.execute_reply.started":"2024-12-03T03:43:04.321006Z","shell.execute_reply":"2024-12-03T03:43:05.098979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSII 결측 없는 data에서 PAQ_Total 점수 결측 개수 보기\n'''\ntrain_sii_not_NA[(train_sii_not_NA['new_PAQ_Total'].isnull())][\n    ['id','Basic_Demos-Age','Basic_Demos-Sex','new_PAQ_Total','PCIAT-PCIAT_Total','sii']\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:02.123083Z","iopub.execute_input":"2024-12-03T03:37:02.123472Z","iopub.status.idle":"2024-12-03T03:37:02.146012Z","shell.execute_reply.started":"2024-12-03T03:37:02.123435Z","shell.execute_reply":"2024-12-03T03:37:02.144574Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n결측치 비율 계산 및 그래프 그리기\n'''\n\nmissing_ratio = train_sii_not_NA['new_PAQ_Total'].isna().mean()\nprint(f\"{'new_PAQ_Total'}의 결측치 비율: {missing_ratio:.2%}\")\n\nfig, axes = plt.subplots(1, 1, figsize=(8, 5))\n\n# new_PAQ_Total\nsns.histplot(train_sii_not_NA['new_PAQ_Total'], bins=20, kde=True)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:02.147541Z","iopub.execute_input":"2024-12-03T03:37:02.148026Z","iopub.status.idle":"2024-12-03T03:37:02.712906Z","shell.execute_reply.started":"2024-12-03T03:37:02.147972Z","shell.execute_reply":"2024-12-03T03:37:02.711708Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr = train_sii_not_NA[['new_PAQ_Total', 'sii']].corr().iloc[0, 1]\nprint(f\"new_PAQ_Total와 sii 간의 상관 계수: {corr:.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:02.714344Z","iopub.execute_input":"2024-12-03T03:37:02.714729Z","iopub.status.idle":"2024-12-03T03:37:02.722743Z","shell.execute_reply.started":"2024-12-03T03:37:02.714680Z","shell.execute_reply":"2024-12-03T03:37:02.721513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n범주형 변수를 제외하고 new_PAQ_Total를 기준으로 다른 변수와의 상관계수를 구한 후 내림차순으로 확인\n'''\nnumeric_cols = train_sii_not_NA.select_dtypes(exclude=['object', 'category'])\ncorr_train = numeric_cols.corr()\ncorrelations = corr_train['new_PAQ_Total'].drop('new_PAQ_Total')\n\nsorted_correlations = correlations.sort_values(ascending=False)\n\nfiltered_corr = sorted_correlations[(sorted_correlations > 0.1) | (sorted_correlations < -0.1)]\nprint(filtered_corr)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:02.724216Z","iopub.execute_input":"2024-12-03T03:37:02.724572Z","iopub.status.idle":"2024-12-03T03:37:02.780177Z","shell.execute_reply.started":"2024-12-03T03:37:02.724538Z","shell.execute_reply":"2024-12-03T03:37:02.779004Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# SDSC","metadata":{}},{"cell_type":"code","source":"'''\nSDS-Season 기술통계 확인\n'''\ncalculate_stats(train, 'SDS-Season')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:02.781702Z","iopub.execute_input":"2024-12-03T03:37:02.782034Z","iopub.status.idle":"2024-12-03T03:37:02.794123Z","shell.execute_reply.started":"2024-12-03T03:37:02.782001Z","shell.execute_reply":"2024-12-03T03:37:02.792991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSDS-SDS_Total_Raw, T 기술통계 확인\n'''\ncalculate_stats(train, ['SDS-SDS_Total_Raw', 'SDS-SDS_Total_T'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:02.795485Z","iopub.execute_input":"2024-12-03T03:37:02.795877Z","iopub.status.idle":"2024-12-03T03:37:02.822391Z","shell.execute_reply.started":"2024-12-03T03:37:02.795842Z","shell.execute_reply":"2024-12-03T03:37:02.821256Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSDS-SDS_Total_Raw, T 결측치 3개 차이에 대한 확인\n'''\nsds_columns = [col for col in train.columns if 'SDS' in col]\ntrain[(train['SDS-SDS_Total_Raw'].notnull()) &\n      (train['SDS-SDS_Total_T'].isnull())][\n    sds_columns + ['Basic_Demos-Age', 'sii']\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:02.823813Z","iopub.execute_input":"2024-12-03T03:37:02.824147Z","iopub.status.idle":"2024-12-03T03:37:02.840986Z","shell.execute_reply.started":"2024-12-03T03:37:02.824111Z","shell.execute_reply":"2024-12-03T03:37:02.839550Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SDS Raw 값 있는 T 값 결측치 채우기","metadata":{}},{"cell_type":"code","source":"'''\n위 행들의 T 결측치 채우는 함수 (Raw 값 기반)\nRaw 값이 74 이상인 경우 T 값은 100으로 설정\n'''\ndef fill_missing_t_values(df, raw_col, t_col):\n    # 매핑 테이블 정의\n    mapping_table = {\n        t_col: [100, 99, 98, 97, 95, 94, 93, 91, 90, 89, 88, 86, 85, 84, 82, 81, 80, 79, 77, 76, 75, 73, 72, 71, 69, 68, 67, 66, 64, 63, 62, 60, 59, 58, 56, 55, 54, 53, 51, 50, 49, 47, 46, 45, 43, 42, 41, 40, 38],\n        raw_col: [74, 73, 72, 71, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26]\n    }\n\n    # 매핑 테이블을 데이터프레임으로 변환\n    mapping_df = pd.DataFrame(mapping_table)\n    \n    # 결측된 T 값에 대해서만 처리\n    is_na = df[t_col].isnull()\n    \n    # Total 값이 74 이상인 경우 T 값을 100으로 설정\n    df.loc[is_na & (df[raw_col] > 74), t_col] = 100\n\n    # 매핑 테이블과 병합하여 T 값을 설정\n    merged = df[is_na].merge(mapping_df, on=raw_col, how=\"left\", suffixes=(\"_existing\", \"_mapped\"))\n    \n    # 결측된 T 값만 매핑된 T 값을 대입\n    df.loc[is_na, t_col] = df.loc[is_na].apply(\n        lambda row: merged.loc[merged[raw_col] == row[raw_col], t_col + \"_mapped\"].values[0]\n        if row[raw_col] <= 74 else row[t_col], axis=1\n    )\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:02.842288Z","iopub.execute_input":"2024-12-03T03:37:02.842695Z","iopub.status.idle":"2024-12-03T03:37:02.855389Z","shell.execute_reply.started":"2024-12-03T03:37:02.842634Z","shell.execute_reply":"2024-12-03T03:37:02.854119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n실제로 결측값을 채울 때는 _new 때고 기존 data에 적용\n'''\ntrain_new = fill_missing_t_values(train, 'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:02.856868Z","iopub.execute_input":"2024-12-03T03:37:02.857246Z","iopub.status.idle":"2024-12-03T03:37:02.906166Z","shell.execute_reply.started":"2024-12-03T03:37:02.857209Z","shell.execute_reply":"2024-12-03T03:37:02.905129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n다시 Raw와 T의 결측치 개수 차이 확인\n'''\nprint(train_new[(train_new['SDS-SDS_Total_Raw'].notnull()) &\n      (train_new['SDS-SDS_Total_T'].isnull())][\nsds_columns + ['Basic_Demos-Age', 'sii']\n])\ncalculate_stats(train_new, ['SDS-SDS_Total_Raw', 'SDS-SDS_Total_T'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:02.907514Z","iopub.execute_input":"2024-12-03T03:37:02.907887Z","iopub.status.idle":"2024-12-03T03:37:02.936161Z","shell.execute_reply.started":"2024-12-03T03:37:02.907851Z","shell.execute_reply":"2024-12-03T03:37:02.935120Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSDS 원점수 변수의 age 최소-최대 확인\n'''\ndata = train[train['SDS-SDS_Total_Raw'].notnull()] #condi.\nage_range = data['Basic_Demos-Age']\nprint(\n    f\"Age range for participants with SDS-SDS_Total_Raw data:\"\n    f\" {age_range.min()} - {age_range.max()} years\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:02.937530Z","iopub.execute_input":"2024-12-03T03:37:02.937888Z","iopub.status.idle":"2024-12-03T03:37:02.947869Z","shell.execute_reply.started":"2024-12-03T03:37:02.937853Z","shell.execute_reply":"2024-12-03T03:37:02.946491Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SDS 이상치 처리","metadata":{}},{"cell_type":"code","source":"'''\n26~130점이므로 26점보다 작은 행 확인\n''' \ntrain.loc[train['SDS-SDS_Total_Raw'] < 26, ['id','Basic_Demos-Age','Basic_Demos-Sex','SDS-Season','SDS-SDS_Total_Raw','SDS-SDS_Total_T']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:02.949054Z","iopub.execute_input":"2024-12-03T03:37:02.949395Z","iopub.status.idle":"2024-12-03T03:37:02.974035Z","shell.execute_reply.started":"2024-12-03T03:37:02.949360Z","shell.execute_reply":"2024-12-03T03:37:02.972735Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n이상치 1개는 결측치 처리 후 결측치 비율 계산 및 Season와 (이상치 제거한) Score의 그래프 그리기\n'''\ntrain.loc[train['SDS-SDS_Total_Raw'] == 17, 'SDS-SDS_Total_Raw'] = np.nan\n\nmissing_ratio = train['SDS-Season'].isna().mean()\nprint(f\"{'SDS-Season'}의 결측치 비율: {missing_ratio:.2%}\")\n\nmissing_ratio = train['SDS-SDS_Total_Raw'].isna().mean()\nprint(f\"{'SDS-SDS_Total_Raw'}의 결측치 비율: {missing_ratio:.2%}\")\n\nmissing_ratio = train['SDS-SDS_Total_T'].isna().mean()\nprint(f\"{'SDS-SDS_Total_T'}의 결측치 비율: {missing_ratio:.2%}\")\n\nplt.figure(figsize=(18, 5))\n\n# SDS-Season\nplt.subplot(1, 3, 1)\ntrain['SDS-Season'].value_counts(normalize=True).plot.pie(\n    autopct='%1.1f%%', colors=plt.cm.Set3.colors\n)\nplt.title('SDS-Season')\n\n# SDS-SDS_Total_Raw\nplt.subplot(1, 3, 2)\nsns.histplot(train['SDS-SDS_Total_Raw'], bins=20, kde=True)\nplt.title('SDS-SDS_Total_Raw')\n\n# SDS-SDS_Total_T\nplt.subplot(1, 3, 3)\nsns.histplot(train['SDS-SDS_Total_T'], bins=20, kde=True)\nplt.title('SDS-SDS_Total_T')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:02.975398Z","iopub.execute_input":"2024-12-03T03:37:02.975885Z","iopub.status.idle":"2024-12-03T03:37:03.971452Z","shell.execute_reply.started":"2024-12-03T03:37:02.975832Z","shell.execute_reply":"2024-12-03T03:37:03.970083Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SII과 상관관계 확인","metadata":{}},{"cell_type":"code","source":"'''\nSDS-SDS_Total_T, complete_resp_total에서 NA 행 제거하고 안쓰는 범주는 제거한 다음에 총 data 개수 보기\n(T가 있으면 Raw가 있기 때문에 T 기준 제거)\n'''\ntrain_filt4 = train.dropna(subset=['SDS-SDS_Total_T', 'complete_resp_total'])\ntrain_filt4.loc[:, 'new_sii_ordered'] = train_filt['new_sii_ordered'].cat.remove_unused_categories()\nlen(train_filt4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:03.972985Z","iopub.execute_input":"2024-12-03T03:37:03.973328Z","iopub.status.idle":"2024-12-03T03:37:03.987192Z","shell.execute_reply.started":"2024-12-03T03:37:03.973293Z","shell.execute_reply":"2024-12-03T03:37:03.985911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n1. SDS-SDS_Total_Raw vs sii\n2. SDS-SDS_Total_T vs sii\n3. SDS-SDS_Total_T vs Age Group\n4. SDS-SDS_Total_T vs Basic_Demos-Sex\n'''\nfig, axes = plt.subplots(2, 2, figsize=(16, 12))\n\ncorr = train[['SDS-SDS_Total_Raw', 'complete_resp_total']].corr().iloc[0, 1]\nprint(f\"SDS-SDS_Total_Raw와 complete_resp_total 간의 상관 계수: {corr:.2f}\")\n\ncorr = train[['SDS-SDS_Total_T', 'complete_resp_total']].corr().iloc[0, 1]\nprint(f\"SDS-SDS_Total_T와 complete_resp_total 간의 상관 계수: {corr:.2f}\")\n\n# SDS-SDS_Total_Raw vs sii\nsns.boxplot(\n    data=train_filt4,\n    x='new_sii_ordered', y='SDS-SDS_Total_Raw',\n    palette='Set3', ax=axes[0,0]\n)\naxes[0,0].set_xlabel('SII Score')\naxes[0,0].set_ylabel('SDS-SDS_Total_Raw')\naxes[0,0].set_title('Distribution of SDS-SDS_Total_Raw by SII')\n\n# SDS-SDS_Total_T vs sii\nsns.boxplot(\n    data=train_filt4,\n    x='new_sii_ordered', y='SDS-SDS_Total_T',\n    palette='Set3', ax=axes[0,1]\n)\naxes[0,1].set_xlabel('SII Score')\naxes[0,1].set_ylabel('SDS-SDS_Total_T')\naxes[0,1].set_title('Distribution of SDS-SDS_Total_T by SII')\n\n# SDS-SDS_Total_T vs Age Group\nsns.boxplot(\n    x='Age_Group', y='SDS-SDS_Total_T',\n    data=train_filt4, palette=\"Set3\", ax=axes[1,0]\n)\naxes[1,0].set_title('SDS-SDS_Total_T by Age Group')\naxes[1,0].set_ylabel('SDS-SDS_Total_T')\naxes[1,0].set_xlabel('Age Group')\n\n# SDS-SDS_Total_T vs Sex\nsns.boxplot(\n    x='Basic_Demos-Sex', y='SDS-SDS_Total_T',\n    data=train_filt4, palette=\"Set3\", ax=axes[1,1]\n)\naxes[1,1].set_title('SDS-SDS_Total_T by Basic_Demos-Sex')\naxes[1,1].set_ylabel('SDS-SDS_Total_T')\naxes[1,1].set_xlabel('Basic_Demos-Sex')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:03.988591Z","iopub.execute_input":"2024-12-03T03:37:03.988991Z","iopub.status.idle":"2024-12-03T03:37:05.063382Z","shell.execute_reply.started":"2024-12-03T03:37:03.988954Z","shell.execute_reply":"2024-12-03T03:37:05.062201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSII와 동시에 결측인 행의 수 확인\n'''\ntrain[(train['PCIAT-PCIAT_Total'].isnull()) &\n      (train['SDS-SDS_Total_T'].isnull())][\n    ['id','Basic_Demos-Age','Basic_Demos-Sex','SDS-SDS_Total_T','PCIAT-PCIAT_Total','sii']\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:05.065141Z","iopub.execute_input":"2024-12-03T03:37:05.066186Z","iopub.status.idle":"2024-12-03T03:37:05.085890Z","shell.execute_reply.started":"2024-12-03T03:37:05.066131Z","shell.execute_reply":"2024-12-03T03:37:05.084619Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SII 결측 없는 data","metadata":{}},{"cell_type":"code","source":"'''\n이상치 처리를 train에서 하고 있기 때문에 다시 정의\n'''\ntrain_sii_not_NA=train[(train['PCIAT-PCIAT_Total'].notnull())]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:05.087427Z","iopub.execute_input":"2024-12-03T03:37:05.088009Z","iopub.status.idle":"2024-12-03T03:37:05.096827Z","shell.execute_reply.started":"2024-12-03T03:37:05.087958Z","shell.execute_reply":"2024-12-03T03:37:05.095550Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSII 결측 없는 data에서 SDS 점수 결측치 개수 보기\n'''\ntrain_sii_not_NA[(train_sii_not_NA['SDS-SDS_Total_T'].isnull())][\n    ['id','Basic_Demos-Age','Basic_Demos-Sex','SDS-SDS_Total_T','PCIAT-PCIAT_Total','sii']\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:05.098332Z","iopub.execute_input":"2024-12-03T03:37:05.098829Z","iopub.status.idle":"2024-12-03T03:37:05.125015Z","shell.execute_reply.started":"2024-12-03T03:37:05.098778Z","shell.execute_reply":"2024-12-03T03:37:05.123648Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n결측치 비율 계산 및 그래프 그리기\n'''\nmissing_ratio = train_sii_not_NA['SDS-SDS_Total_T'].isna().mean()\nprint(f\"{'SDS-SDS_Total_T'}의 결측치 비율: {missing_ratio:.2%}\")\n\nfig, axes = plt.subplots(1, 1, figsize=(8, 5))\n\n# new_PAQ_Total\nsns.histplot(train_sii_not_NA['SDS-SDS_Total_T'], bins=20, kde=True)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:05.126383Z","iopub.execute_input":"2024-12-03T03:37:05.126755Z","iopub.status.idle":"2024-12-03T03:37:05.607317Z","shell.execute_reply.started":"2024-12-03T03:37:05.126718Z","shell.execute_reply":"2024-12-03T03:37:05.605857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr = train_sii_not_NA[['SDS-SDS_Total_Raw', 'sii']].corr().iloc[0, 1]\nprint(f\"SDS-SDS_Total_Raw와 sii 간의 상관 계수: {corr:.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:05.608793Z","iopub.execute_input":"2024-12-03T03:37:05.609138Z","iopub.status.idle":"2024-12-03T03:37:05.617179Z","shell.execute_reply.started":"2024-12-03T03:37:05.609103Z","shell.execute_reply":"2024-12-03T03:37:05.615992Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr = train_sii_not_NA[['SDS-SDS_Total_T', 'sii']].corr().iloc[0, 1]\nprint(f\"SDS-SDS_Total_T와 sii 간의 상관 계수: {corr:.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:05.619122Z","iopub.execute_input":"2024-12-03T03:37:05.619573Z","iopub.status.idle":"2024-12-03T03:37:05.635366Z","shell.execute_reply.started":"2024-12-03T03:37:05.619523Z","shell.execute_reply":"2024-12-03T03:37:05.634107Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n범주형 변수를 제외하고 SDS를 기준으로 다른 변수와의 상관계수를 구한 후 내림차순으로 확인\n'''\nnumeric_cols = train_sii_not_NA.select_dtypes(exclude=['object', 'category'])\ncorr_train = numeric_cols.corr()\ncorrelations = corr_train['SDS-SDS_Total_T'].drop('SDS-SDS_Total_T')\n\nsorted_correlations = correlations.sort_values(ascending=False)\n\nfiltered_corr = sorted_correlations[(sorted_correlations > 0.1) | (sorted_correlations < -0.1)]\nprint(filtered_corr)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:05.636909Z","iopub.execute_input":"2024-12-03T03:37:05.637249Z","iopub.status.idle":"2024-12-03T03:37:05.692137Z","shell.execute_reply.started":"2024-12-03T03:37:05.637216Z","shell.execute_reply":"2024-12-03T03:37:05.691000Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SDS 결측치 처리","metadata":{}},{"cell_type":"code","source":"'''\nSDS 결측치 자체가 sii에 어떤 관계가 있을지 확인 (SDS 결측치인 행에서만 target 분포 확인)\n'''\nplt.figure(figsize=(18, 5))\n\nplt.subplot(1, 3, 1)\nsns.histplot(train_sii_not_NA[(train_sii_not_NA['SDS-SDS_Total_T'].isnull())]['PCIAT-PCIAT_Total'], bins=20, kde=True)\nplt.title('PCIAT-PCIAT_Total')\n\nplt.subplot(1, 3, 2)\ntrain_sii_not_NA[(train_sii_not_NA['SDS-SDS_Total_T'].isnull())]['new_sii_ordered'].value_counts(normalize=True).plot.pie(\n    autopct='%1.1f%%', colors=plt.cm.Set3.colors\n)\nplt.title('new_sii_ordered')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:05.702812Z","iopub.execute_input":"2024-12-03T03:37:05.703329Z","iopub.status.idle":"2024-12-03T03:37:06.358135Z","shell.execute_reply.started":"2024-12-03T03:37:05.703292Z","shell.execute_reply":"2024-12-03T03:37:06.356881Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSDS 결측치 자체가 sii에 어떤 관계가 있을지 확인 (SDS 결측치가 아닌 행에서만 target 분포 확인)\n'''\nplt.figure(figsize=(18, 5))\n\n# SDS-SDS_Total_Raw\nplt.subplot(1, 3, 1)\nsns.histplot(train_sii_not_NA[(train_sii_not_NA['SDS-SDS_Total_T'].notnull())]['PCIAT-PCIAT_Total'], bins=20, kde=True)\nplt.title('PCIAT-PCIAT_Total')\n\nplt.subplot(1, 3, 2)\ntrain_sii_not_NA[(train_sii_not_NA['SDS-SDS_Total_T'].notnull())]['new_sii_ordered'].value_counts(normalize=True).plot.pie(\n    autopct='%1.1f%%', colors=plt.cm.Set3.colors\n)\nplt.title('new_sii_ordered')\n\nplt.tight_layout()\nplt.show() #거의 유사함, 부모가 일부러 응답을 피했다 같은 어떤 '특성'이 있는 것은 아닌 듯","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:06.359711Z","iopub.execute_input":"2024-12-03T03:37:06.360217Z","iopub.status.idle":"2024-12-03T03:37:07.062933Z","shell.execute_reply.started":"2024-12-03T03:37:06.360155Z","shell.execute_reply":"2024-12-03T03:37:07.061727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSex, Age의 중앙값 경향성 확인 (위 그래프도 참고)\n'''\ntrain_sii_not_NA.groupby(['Basic_Demos-Sex','Basic_Demos-Age'])['SDS-SDS_Total_T'].median()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:07.064574Z","iopub.execute_input":"2024-12-03T03:37:07.065030Z","iopub.status.idle":"2024-12-03T03:37:07.080911Z","shell.execute_reply.started":"2024-12-03T03:37:07.064989Z","shell.execute_reply":"2024-12-03T03:37:07.079698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\ntarget을 제외하고 상관이 높은 변수인 CGAS와의 plot 확인\n'''\nbins = np.arange(0, 101, 10)\nlabels = [\n    \"1-10\", \"11-20\", \"21-30\", \"31-40\", \"41-50\", \"51-60\", \"61-70\", \"71-80\", \"81-90\", \"91-100\"\n]\n\ntrain_sii_not_NA['CGAS_Score_Bin'] = pd.cut(\n    train_sii_not_NA['CGAS-CGAS_Score'], bins=bins, labels=labels\n) #new var.\n\nplt.figure(figsize=(18, 5))\n\nplt.subplot(1, 3, 1)\nsns.scatterplot(x=train_sii_not_NA['CGAS-CGAS_Score'], y=train_sii_not_NA['SDS-SDS_Total_T'], alpha=0.7)\nplt.title('Scatter Plot of CGAS vs SDS')\nplt.xlabel('CGAS-CGAS_Score')\nplt.ylabel('SDS')\n\nplt.subplot(1, 3, 2)\nsns.boxplot(data=train_sii_not_NA, x='CGAS_Score_Bin', y='SDS-SDS_Total_T', palette='Set3')\nplt.title('SDS by CGAS Category')\nplt.xlabel('CGAS Category')\nplt.ylabel('SDS')\nplt.xticks(rotation=45)\nplt.show() #선형, 비선형의 관계 자체는 안보임","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:07.082738Z","iopub.execute_input":"2024-12-03T03:37:07.083216Z","iopub.status.idle":"2024-12-03T03:37:07.925075Z","shell.execute_reply.started":"2024-12-03T03:37:07.083164Z","shell.execute_reply":"2024-12-03T03:37:07.923971Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nCGAS의 중앙값 경향성 확인\n'''\ntrain_sii_not_NA.groupby('new_CGAS_Score_Bin')['SDS-SDS_Total_T'].median()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:07.926580Z","iopub.execute_input":"2024-12-03T03:37:07.927048Z","iopub.status.idle":"2024-12-03T03:37:07.939958Z","shell.execute_reply.started":"2024-12-03T03:37:07.926999Z","shell.execute_reply":"2024-12-03T03:37:07.938623Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nSDS와 CGAS와 동시에 결측인 행의 수 확인\n'''\ntrain_sii_not_NA[(train_sii_not_NA['SDS-SDS_Total_Raw'].isnull()) &\n      (train_sii_not_NA['CGAS-CGAS_Score'].isnull()) ][\n    ['id','Basic_Demos-Age','Basic_Demos-Sex','SDS-SDS_Total_Raw','CGAS-CGAS_Score','PCIAT-PCIAT_Total','sii']\n].count()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:07.941379Z","iopub.execute_input":"2024-12-03T03:37:07.941789Z","iopub.status.idle":"2024-12-03T03:37:07.956960Z","shell.execute_reply.started":"2024-12-03T03:37:07.941753Z","shell.execute_reply":"2024-12-03T03:37:07.955848Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n① SDS의 결측치를 SDS의 중앙값으로만 채우기\n'''\n#train['SDS-SDS_Total_Raw'].fillna(train['SDS-SDS_Total_Raw'].median(), inplace=True)\n#train['SDS-SDS_Total_T'].fillna(train['SDS-SDS_Total_T'].median(), inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:07.958325Z","iopub.execute_input":"2024-12-03T03:37:07.958708Z","iopub.status.idle":"2024-12-03T03:37:07.974868Z","shell.execute_reply.started":"2024-12-03T03:37:07.958630Z","shell.execute_reply":"2024-12-03T03:37:07.973546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n② new_CGAS_Score_Bin에서의 SDS 중앙값을 활용하여 결측 채우기\n'''\ngroup_medians = train.groupby('new_CGAS_Score_Bin')['SDS-SDS_Total_Raw'].median()\noverall_median = train['SDS-SDS_Total_Raw'].median()\n\ntrain['SDS-SDS_Total_Raw'] = train.apply(\n    lambda row: (\n        group_medians[row['new_CGAS_Score_Bin']] if pd.notna(row['new_CGAS_Score_Bin']) \n        else overall_median\n    ) if pd.isna(row['SDS-SDS_Total_Raw']) else row['SDS-SDS_Total_Raw'],\n    axis=1\n)\n\ngroup_medians_T = train.groupby('new_CGAS_Score_Bin')['SDS-SDS_Total_T'].median()\noverall_median_T = train['SDS-SDS_Total_T'].median()\n\ntrain['SDS-SDS_Total_T'] = train.apply(\n    lambda row: (\n        group_medians[row['new_CGAS_Score_Bin']] if pd.notna(row['new_CGAS_Score_Bin']) \n        else overall_median\n    ) if pd.isna(row['SDS-SDS_Total_T']) else row['SDS-SDS_Total_T'],\n    axis=1\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:07.976152Z","iopub.execute_input":"2024-12-03T03:37:07.976534Z","iopub.status.idle":"2024-12-03T03:37:08.110526Z","shell.execute_reply.started":"2024-12-03T03:37:07.976498Z","shell.execute_reply":"2024-12-03T03:37:08.109235Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[(train['SDS-SDS_Total_Raw'].isnull())|\n    (train['SDS-SDS_Total_T'].isnull())]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:08.112295Z","iopub.execute_input":"2024-12-03T03:37:08.112737Z","iopub.status.idle":"2024-12-03T03:37:08.131806Z","shell.execute_reply.started":"2024-12-03T03:37:08.112645Z","shell.execute_reply":"2024-12-03T03:37:08.130385Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Actigraphy","metadata":{}},{"cell_type":"code","source":"'''\nstep에 따라 각 변수의 plot 그리는 함수\n'''\ndef plot_series_data(df, col='non-wear_flag',\n                     label='Worn (0 = Worn, 1 = Not Worn)',\n                     title='Non-Wear Flag',\n                     x_col='day_time', x_label='Day Relative to PCIAT + Time'):\n    plt.figure(figsize=(18, 12))\n\n    # ENMO\n    plt.subplot(4, 1, 1)\n    plt.scatter(df[x_col], df['enmo'], label='ENMO', color='green', s=1)\n    plt.title('ENMO (Euclidean Norm Minus One)')\n    plt.ylabel('Movement Intensity')\n\n    # Angle Z\n    plt.subplot(4, 1, 2)\n    plt.scatter(df[x_col], df['anglez'], label='Angle Z', color='blue', s=1)\n    plt.title('Angle Z')\n    plt.ylabel('Angle (degrees)')\n\n    # Light\n    plt.subplot(4, 1, 3)\n    plt.scatter(df[x_col], df['light'], label='Light', color='orange', s=1)\n    plt.title('Ambient Light')\n    plt.ylabel('Light (lux)')\n\n    # Any other column\n    plt.subplot(4, 1, 4)\n    plt.scatter(df[x_col], df[col], label=col, color='red', s=1)\n    plt.title(f'{title}')\n    plt.ylabel(f'{label}')\n    plt.xlabel(f'{x_label}')\n\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:08.133477Z","iopub.execute_input":"2024-12-03T03:37:08.133986Z","iopub.status.idle":"2024-12-03T03:37:08.150898Z","shell.execute_reply.started":"2024-12-03T03:37:08.133924Z","shell.execute_reply":"2024-12-03T03:37:08.149398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n한 참가자만 데이터 확인(/id={id}/)\n'''\n\npath = '/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id=00115b9f/part-0.parquet'\nseries_train = pd.read_parquet(path)\nseries_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:08.152339Z","iopub.execute_input":"2024-12-03T03:37:08.152781Z","iopub.status.idle":"2024-12-03T03:37:08.351065Z","shell.execute_reply.started":"2024-12-03T03:37:08.152739Z","shell.execute_reply":"2024-12-03T03:37:08.349954Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n해당 참가자의 정형 데이터 확인\n'''\nparticipant_id = path.split('/')[-2].split('=')[-1]\nparticipant_id\n\ntrain[train['id'] == participant_id].T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:08.352560Z","iopub.execute_input":"2024-12-03T03:37:08.353027Z","iopub.status.idle":"2024-12-03T03:37:08.368153Z","shell.execute_reply.started":"2024-12-03T03:37:08.352973Z","shell.execute_reply":"2024-12-03T03:37:08.367050Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_series_data(series_train, x_col='step', x_label='Step')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T03:37:08.370069Z","iopub.execute_input":"2024-12-03T03:37:08.370553Z","iopub.status.idle":"2024-12-03T03:37:09.762171Z","shell.execute_reply.started":"2024-12-03T03:37:08.370500Z","shell.execute_reply":"2024-12-03T03:37:09.761010Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}