{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\nsns.set(style=\"whitegrid\")\n%matplotlib inline\n\n\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\ndata_dict = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv')\n\ndisplay(train.head())\nprint(f\"Train shape: {train.shape}\")\n\ndisplay(test.head())\nprint(f\"Test shape: {test.shape}\")\n\ndata_dict.head()\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:28:23.117847Z","iopub.execute_input":"2024-11-24T14:28:23.118290Z","iopub.status.idle":"2024-11-24T14:28:24.521739Z","shell.execute_reply.started":"2024-11-24T14:28:23.118224Z","shell.execute_reply":"2024-11-24T14:28:24.520583Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_stats(data, columns):\n    if isinstance(columns, str):\n        columns = [columns]\n\n    stats = []\n    for col in columns:\n        if data[col].dtype in ['object', 'category']:\n            counts = data[col].value_counts(dropna=False, sort=False)\n            percents = data[col].value_counts(normalize=True, dropna=False, sort=False) * 100\n            formatted = counts.astype(str) + ' (' + percents.round(2).astype(str) + '%)'\n            stats_col = pd.DataFrame({'count (%)': formatted})\n            stats.append(stats_col)\n        else:\n            stats_col = data[col].describe().to_frame().transpose()\n            stats_col['missing'] = data[col].isnull().sum()\n            stats_col.index.name = col\n            stats.append(stats_col)\n\n    return pd.concat(stats, axis=0)\n\n\ntrain_cols = set(train.columns)\ntest_cols = set(test.columns)\ncolumns_not_in_test = sorted(list(train_cols - test_cols))\ndata_dict[data_dict['Field'].isin(columns_not_in_test)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:28:28.728083Z","iopub.execute_input":"2024-11-24T14:28:28.728481Z","iopub.status.idle":"2024-11-24T14:28:28.755137Z","shell.execute_reply.started":"2024-11-24T14:28:28.728446Z","shell.execute_reply":"2024-11-24T14:28:28.753936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pciat_min_max = train.groupby('sii')['PCIAT-PCIAT_Total'].agg(['min', 'max'])\npciat_min_max = pciat_min_max.rename(\n    columns={'min': 'Minimum PCIAT total Score', 'max': 'Maximum total PCIAT Score'}\n)\npciat_min_max","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:28:34.955672Z","iopub.execute_input":"2024-11-24T14:28:34.956075Z","iopub.status.idle":"2024-11-24T14:28:34.977728Z","shell.execute_reply.started":"2024-11-24T14:28:34.956034Z","shell.execute_reply":"2024-11-24T14:28:34.976106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dict[data_dict['Field'] == 'PCIAT-PCIAT_Total']['Value Labels'].iloc[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:28:40.491184Z","iopub.execute_input":"2024-11-24T14:28:40.491559Z","iopub.status.idle":"2024-11-24T14:28:40.499688Z","shell.execute_reply.started":"2024-11-24T14:28:40.491528Z","shell.execute_reply":"2024-11-24T14:28:40.498526Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_with_sii = train[train['sii'].notna()][columns_not_in_test]\ntrain_with_sii[train_with_sii.isna().any(axis=1)].head().style.applymap(\n    lambda x: 'background-color: #FFC0CB' if pd.isna(x) else ''\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:28:42.671401Z","iopub.execute_input":"2024-11-24T14:28:42.671747Z","iopub.status.idle":"2024-11-24T14:28:42.761458Z","shell.execute_reply.started":"2024-11-24T14:28:42.671718Z","shell.execute_reply":"2024-11-24T14:28:42.760004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PCIAT_cols = [f'PCIAT-PCIAT_{i+1:02d}' for i in range(20)]\nrecalc_total_score = train_with_sii[PCIAT_cols].sum(\n    axis=1, skipna=True\n)\n(recalc_total_score == train_with_sii['PCIAT-PCIAT_Total']).all()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:28:45.650543Z","iopub.execute_input":"2024-11-24T14:28:45.651114Z","iopub.status.idle":"2024-11-24T14:28:45.663313Z","shell.execute_reply.started":"2024-11-24T14:28:45.651076Z","shell.execute_reply":"2024-11-24T14:28:45.662221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def recalculate_sii(row):\n    if pd.isna(row['PCIAT-PCIAT_Total']):\n        return np.nan\n    max_possible = row['PCIAT-PCIAT_Total'] + row[PCIAT_cols].isna().sum() * 5\n    if row['PCIAT-PCIAT_Total'] <= 30 and max_possible <= 30:\n        return 0\n    elif 31 <= row['PCIAT-PCIAT_Total'] <= 49 and max_possible <= 49:\n        return 1\n    elif 50 <= row['PCIAT-PCIAT_Total'] <= 79 and max_possible <= 79:\n        return 2\n    elif row['PCIAT-PCIAT_Total'] >= 80 and max_possible >= 80:\n        return 3\n    return np.nan\n\ntrain['recalc_sii'] = train.apply(recalculate_sii, axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:29:02.436819Z","iopub.execute_input":"2024-11-24T14:29:02.437625Z","iopub.status.idle":"2024-11-24T14:29:03.867540Z","shell.execute_reply.started":"2024-11-24T14:29:02.437580Z","shell.execute_reply":"2024-11-24T14:29:03.865999Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatch_rows = train[\n    (train['recalc_sii'] != train['sii']) & train['sii'].notna()\n]\n\nmismatch_rows[PCIAT_cols + [\n    'PCIAT-PCIAT_Total', 'sii', 'recalc_sii'\n]].style.applymap(\n    lambda x: 'background-color: #FFC0CB' if pd.isna(x) else ''\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:29:17.623755Z","iopub.execute_input":"2024-11-24T14:29:17.624105Z","iopub.status.idle":"2024-11-24T14:29:17.652383Z","shell.execute_reply.started":"2024-11-24T14:29:17.624074Z","shell.execute_reply":"2024-11-24T14:29:17.650708Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['sii'] = train['recalc_sii']\ntrain['complete_resp_total'] = train['PCIAT-PCIAT_Total'].where(\n    train[PCIAT_cols].notna().all(axis=1), np.nan\n)\n\nsii_map = {0: '0 (None)', 1: '1 (Mild)', 2: '2 (Moderate)', 3: '3 (Severe)'}\ntrain['sii'] = train['sii'].map(sii_map).fillna('Missing')\n\nsii_order = ['Missing', '0 (None)', '1 (Mild)', '2 (Moderate)', '3 (Severe)']\ntrain['sii'] = pd.Categorical(train['sii'], categories=sii_order, ordered=True)\n\ntrain.drop(columns='recalc_sii', inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:29:35.505549Z","iopub.execute_input":"2024-11-24T14:29:35.506362Z","iopub.status.idle":"2024-11-24T14:29:35.522772Z","shell.execute_reply.started":"2024-11-24T14:29:35.506309Z","shell.execute_reply":"2024-11-24T14:29:35.521421Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sii_counts = train['sii'].value_counts().reset_index()\ntotal = sii_counts['count'].sum()\nsii_counts['percentage'] = (sii_counts['count'] / total) * 100\n\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\n# SII\nsns.barplot(x='sii', y='count', data=sii_counts, palette='Blues_d', ax=axes[0])\naxes[0].set_title('Distribution of Severity Impairment Index (sii)', fontsize=14)\nfor p in axes[0].patches:\n    height = p.get_height()\n    percentage = sii_counts.loc[sii_counts['count'] == height, 'percentage'].values[0]\n    axes[0].text(\n        p.get_x() + p.get_width() / 2,\n        height + 5, f'{int(height)} ({percentage:.1f}%)',\n        ha=\"center\", fontsize=12\n    )\n\n# PCIAT_Total for complete responses\nsns.histplot(train['complete_resp_total'].dropna(), bins=20, ax=axes[1])\naxes[1].set_title('Distribution of PCIAT_Total', fontsize=14)\naxes[1].set_xlabel('PCIAT_Total for Complete PCIAT Responses')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:29:56.412232Z","iopub.execute_input":"2024-11-24T14:29:56.412650Z","iopub.status.idle":"2024-11-24T14:29:57.273417Z","shell.execute_reply.started":"2024-11-24T14:29:56.412615Z","shell.execute_reply":"2024-11-24T14:29:57.272373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train[train['complete_resp_total'] == 0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:30:17.668254Z","iopub.execute_input":"2024-11-24T14:30:17.668690Z","iopub.status.idle":"2024-11-24T14:30:17.677868Z","shell.execute_reply.started":"2024-11-24T14:30:17.668655Z","shell.execute_reply":"2024-11-24T14:30:17.676562Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"assert train['Basic_Demos-Age'].isna().sum() == 0\nassert train['Basic_Demos-Sex'].isna().sum() == 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:30:31.187799Z","iopub.execute_input":"2024-11-24T14:30:31.188196Z","iopub.status.idle":"2024-11-24T14:30:31.194984Z","shell.execute_reply.started":"2024-11-24T14:30:31.188163Z","shell.execute_reply":"2024-11-24T14:30:31.193735Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['Age Group'] = pd.cut(\n    train['Basic_Demos-Age'],\n    bins=[4, 12, 18, 22],\n    labels=['Children (5-12)', 'Adolescents (13-18)', 'Adults (19-22)']\n)\ncalculate_stats(train, 'Age Group')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:30:44.085414Z","iopub.execute_input":"2024-11-24T14:30:44.085790Z","iopub.status.idle":"2024-11-24T14:30:44.104581Z","shell.execute_reply.started":"2024-11-24T14:30:44.085759Z","shell.execute_reply":"2024-11-24T14:30:44.103223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sex_map = {0: 'Male', 1: 'Female'}\ntrain['Basic_Demos-Sex'] = train['Basic_Demos-Sex'].map(sex_map)\ncalculate_stats(train, 'Basic_Demos-Sex')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:30:57.804408Z","iopub.execute_input":"2024-11-24T14:30:57.804759Z","iopub.status.idle":"2024-11-24T14:30:57.818303Z","shell.execute_reply.started":"2024-11-24T14:30:57.804729Z","shell.execute_reply":"2024-11-24T14:30:57.816717Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 3, figsize=(18, 5))\n\n# SII by Age\nsns.boxplot(y=train['Basic_Demos-Age'], x=train['sii'], ax=axes[0], palette=\"Set3\")\naxes[0].set_title('SII by Age')\naxes[0].set_ylabel('Age')\naxes[0].set_xlabel('SII')\n\n# Complete PCIAT Responses by Age Group\nsns.boxplot(\n    x='Age Group', y='complete_resp_total',\n    data=train, palette=\"Set3\", ax=axes[1]\n)\naxes[1].set_title('Complete PCIAT Responses by Age Group')\naxes[1].set_ylabel('PCIAT_Total for Complete Responses')\naxes[1].set_xlabel('Age Group')\n\n# PCIAT_Total by Sex\nsns.histplot(\n    data=train, x='complete_resp_total',\n    hue='Basic_Demos-Sex', multiple='stack',\n    palette=\"Set3\", bins=20, ax=axes[2]\n)\naxes[2].set_title('PCIAT_Total Distribution by Sex')\naxes[2].set_xlabel('PCIAT_Total for Complete Responses')\naxes[2].set_ylabel('Frequency')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:31:16.503747Z","iopub.execute_input":"2024-11-24T14:31:16.504463Z","iopub.status.idle":"2024-11-24T14:31:17.483777Z","shell.execute_reply.started":"2024-11-24T14:31:16.504424Z","shell.execute_reply":"2024-11-24T14:31:17.482604Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = train.groupby(['Age Group', 'sii']).size().unstack(fill_value=0)\nfig, axes = plt.subplots(1, len(stats), figsize=(18, 5))\n\nfor i, age_group in enumerate(stats.index):\n    group_counts = stats.loc[age_group] / stats.loc[age_group].sum()\n    axes[i].pie(\n        group_counts, labels=group_counts.index, autopct='%1.1f%%',\n        startangle=90, colors=sns.color_palette(\"Set3\"),\n        labeldistance=1.05, pctdistance=0.80\n    )\n    axes[i].set_title(f'SII Distribution for {age_group}')\n    axes[i].axis('equal')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:31:40.007109Z","iopub.execute_input":"2024-11-24T14:31:40.007524Z","iopub.status.idle":"2024-11-24T14:31:40.454547Z","shell.execute_reply.started":"2024-11-24T14:31:40.007489Z","shell.execute_reply":"2024-11-24T14:31:40.453491Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = train.groupby(['Age Group', 'sii']).size().unstack(fill_value=0)\nstats_prop = stats.div(stats.sum(axis=1), axis=0) * 100\n\nstats = stats.astype(str) +' (' + stats_prop.round(1).astype(str) + '%)'\nstats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:31:56.327112Z","iopub.execute_input":"2024-11-24T14:31:56.327604Z","iopub.status.idle":"2024-11-24T14:31:56.352511Z","shell.execute_reply.started":"2024-11-24T14:31:56.327565Z","shell.execute_reply":"2024-11-24T14:31:56.351332Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = train[train['sii'] != 'Missing'].groupby(\n    ['Age Group', 'sii']\n).size().unstack(fill_value=0)\nstats_prop = stats.div(stats.sum(axis=1), axis=0) * 100\n\nstats = stats.astype(str) +' (' + stats_prop.round(1).astype(str) + '%)'\nstats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:32:13.646130Z","iopub.execute_input":"2024-11-24T14:32:13.647117Z","iopub.status.idle":"2024-11-24T14:32:13.669329Z","shell.execute_reply.started":"2024-11-24T14:32:13.647075Z","shell.execute_reply":"2024-11-24T14:32:13.668164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = train[train['PreInt_EduHx-computerinternet_hoursday'].notna()]\nage_range = data['Basic_Demos-Age']\nprint(\n    f\"Age range for participants with measured PreInt_EduHx-computerinternet_hoursday data:\"\n    f\" {age_range.min()} - {age_range.max()} years\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:32:39.490686Z","iopub.execute_input":"2024-11-24T14:32:39.491622Z","iopub.status.idle":"2024-11-24T14:32:39.500537Z","shell.execute_reply.started":"2024-11-24T14:32:39.491578Z","shell.execute_reply":"2024-11-24T14:32:39.499425Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['PreInt_EduHx-computerinternet_hoursday'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:32:52.578869Z","iopub.execute_input":"2024-11-24T14:32:52.579315Z","iopub.status.idle":"2024-11-24T14:32:52.587346Z","shell.execute_reply.started":"2024-11-24T14:32:52.579246Z","shell.execute_reply":"2024-11-24T14:32:52.586359Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"param_map = {0: '< 1h/day', 1: '~ 1h/day', 2: '~ 2hs/day', 3: '> 3hs/day'}\ntrain['internet_use_encoded'] = train[\n    'PreInt_EduHx-computerinternet_hoursday'\n].map(param_map).fillna('Missing')\n\nparam_ord = ['Missing', '< 1h/day', '~ 1h/day', '~ 2hs/day', '> 3hs/day']\ntrain['internet_use_encoded'] = pd.Categorical(\n    train['internet_use_encoded'], categories=param_ord,\n    ordered=True\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:33:06.471971Z","iopub.execute_input":"2024-11-24T14:33:06.472487Z","iopub.status.idle":"2024-11-24T14:33:06.484124Z","shell.execute_reply.started":"2024-11-24T14:33:06.472439Z","shell.execute_reply":"2024-11-24T14:33:06.482968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, 'PreInt_EduHx-Season')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:33:19.880130Z","iopub.execute_input":"2024-11-24T14:33:19.880542Z","iopub.status.idle":"2024-11-24T14:33:19.892463Z","shell.execute_reply.started":"2024-11-24T14:33:19.880507Z","shell.execute_reply":"2024-11-24T14:33:19.891022Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 3, figsize=(18, 5))\n\n# Hours of Internet Use\nax1 = sns.countplot(x='internet_use_encoded', data=train, palette=\"Set3\", ax=axes[0])\naxes[0].set_title('Distribution of Hours of Internet Use')\naxes[0].set_xlabel('Hours per Day Group')\naxes[0].set_ylabel('Count')\n\ntotal = len(train['internet_use_encoded'])\nfor p in ax1.patches:\n    count = int(p.get_height())\n    percentage = '{:.1f}%'.format(100 * count / total)\n    ax1.annotate(f'{count} ({percentage})', (p.get_x() + p.get_width() / 2., p.get_height()), \n                 ha='center', va='baseline', fontsize=10, color='black', xytext=(0, 5), \n                 textcoords='offset points')\n\n# Hours of Internet Use by Age\nsns.boxplot(y=train['Basic_Demos-Age'], x=train['internet_use_encoded'], ax=axes[1], palette=\"Set3\")\naxes[1].set_title('Hours of Internet Use by Age')\naxes[1].set_ylabel('Age')\naxes[1].set_xlabel('Hours per Day Group')\n# Hours of Internet Use (numeric) by Age Group\nsns.boxplot(y='PreInt_EduHx-computerinternet_hoursday', x='Age Group', data=train, ax=axes[2], palette=\"Set3\")\naxes[2].set_title('Internet Hours by Age Group')\naxes[2].set_ylabel('Hours per Day (Numeric)')\naxes[2].set_xlabel('Age Group')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:34:06.829535Z","iopub.execute_input":"2024-11-24T14:34:06.829961Z","iopub.status.idle":"2024-11-24T14:34:07.737766Z","shell.execute_reply.started":"2024-11-24T14:34:06.829919Z","shell.execute_reply":"2024-11-24T14:34:07.736676Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = train.groupby(\n    ['Age Group', 'internet_use_encoded']\n).size().unstack(fill_value=0)\nfig, axes = plt.subplots(1, len(stats), figsize=(18, 5))\n\nfor i, age_group in enumerate(stats.index):\n    group_counts = stats.loc[age_group] / stats.loc[age_group].sum()\n    axes[i].pie(group_counts, labels=group_counts.index, autopct='%1.1f%%',\n                startangle=90, colors=sns.color_palette(\"Set3\"), labeldistance=1.1)\n    axes[i].set_title(f'Distribution of Hours of Internet Use\\n{age_group}')\n    axes[i].axis('equal')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:34:25.014864Z","iopub.execute_input":"2024-11-24T14:34:25.015258Z","iopub.status.idle":"2024-11-24T14:34:25.564249Z","shell.execute_reply.started":"2024-11-24T14:34:25.015221Z","shell.execute_reply":"2024-11-24T14:34:25.563148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_non_na = train.dropna(subset=['PreInt_EduHx-computerinternet_hoursday'])\nrows = (train_non_na['PreInt_EduHx-computerinternet_hoursday'] == 3).sum()\nprint(f\"Non-NA Rows - Internet use 3h or more: {(rows / len(train_non_na)) * 100:.2f}%\")\n\nrows = (train_non_na['PreInt_EduHx-computerinternet_hoursday'] == 0).sum()\nprint(f\"Non-NA Rows - Internet use 1h or less: {(rows / len(train_non_na)) * 100:.2f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:34:38.597526Z","iopub.execute_input":"2024-11-24T14:34:38.597933Z","iopub.status.idle":"2024-11-24T14:34:38.610625Z","shell.execute_reply.started":"2024-11-24T14:34:38.597898Z","shell.execute_reply":"2024-11-24T14:34:38.609379Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = train.groupby(['Basic_Demos-Sex', 'internet_use_encoded']\n).size().unstack(fill_value=0)\nstats_prop = stats.div(stats.sum(axis=1), axis=0) * 100\n\nstats = stats.astype(str) +' (' + stats_prop.round(1).astype(str) + '%)'\nstats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:34:56.054092Z","iopub.execute_input":"2024-11-24T14:34:56.054512Z","iopub.status.idle":"2024-11-24T14:34:56.078021Z","shell.execute_reply.started":"2024-11-24T14:34:56.054478Z","shell.execute_reply":"2024-11-24T14:34:56.076958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sii_reported = train[train['sii'] != \"Missing\"]\nsii_reported.loc[:, 'sii'] = sii_reported['sii'].cat.remove_unused_categories()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:35:13.235607Z","iopub.execute_input":"2024-11-24T14:35:13.236786Z","iopub.status.idle":"2024-11-24T14:35:13.249853Z","shell.execute_reply.started":"2024-11-24T14:35:13.236730Z","shell.execute_reply":"2024-11-24T14:35:13.248757Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = sii_reported.groupby(\n    ['internet_use_encoded', 'sii']\n).size().unstack(fill_value=0)\nstats_prop = stats.div(stats.sum(axis=1), axis=0) * 100\n\nstats = stats.astype(str) +' (' + stats_prop.round(1).astype(str) + '%)'\nstats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:35:24.024406Z","iopub.execute_input":"2024-11-24T14:35:24.025504Z","iopub.status.idle":"2024-11-24T14:35:24.048194Z","shell.execute_reply.started":"2024-11-24T14:35:24.025463Z","shell.execute_reply":"2024-11-24T14:35:24.047016Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig = plt.figure(figsize=(12, 10))\ngs = fig.add_gridspec(2, 2, height_ratios=[1, 1.5])\n\n# SII vs Hours of Internet Use\nax1 = fig.add_subplot(gs[0, 0])\nsns.boxplot(\n    x='sii', y='PreInt_EduHx-computerinternet_hoursday',\n    data=sii_reported,\n    ax=ax1, palette=\"Set3\"\n)\nax1.set_title('SII vs Hours of Internet Use')\nax1.set_ylabel('Hours per Day')\nax1.set_xlabel('SII')\n\n# PCIAT_Total for Complete PCIAT Responses by Hours of Internet Use\nax2 = fig.add_subplot(gs[0, 1])\nsns.boxplot(\n    x='internet_use_encoded', y='complete_resp_total',\n    data=sii_reported,\n    palette=\"Set3\", ax=ax2\n)\nax2.set_title('PCIAT_Total by Hours of Internet Use')\nax2.set_ylabel('PCIAT_Total for Complete PCIAT Responses')\nax2.set_xlabel('Hours per Day Group')\n# SII vs Hours of Internet Use by Age Group (Full width)\nax3 = fig.add_subplot(gs[1, :])\nsns.boxplot(\n    x='internet_use_encoded', y='complete_resp_total',\n    data=sii_reported,\n    hue='Age Group', ax=ax3, palette=\"Set3\"\n)\nax3.set_title('PCIAT_Total vs Hours of Internet Use by Age Group')\nax3.set_ylabel('PCIAT_Total for Complete PCIAT Responses')\nax3.set_xlabel('Hours per Day Group')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:36:01.160348Z","iopub.execute_input":"2024-11-24T14:36:01.160766Z","iopub.status.idle":"2024-11-24T14:36:02.165436Z","shell.execute_reply.started":"2024-11-24T14:36:01.160722Z","shell.execute_reply":"2024-11-24T14:36:02.164169Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = sii_reported.groupby(\n    ['sii', 'internet_use_encoded']\n).size().unstack(fill_value=0)\nfig, axes = plt.subplots(1, len(stats), figsize=(18, 5))\n\nfor i, sii_group in enumerate(stats.index):\n    group_counts = stats.loc[sii_group] / stats.loc[sii_group].sum()\n    axes[i].pie(\n        group_counts, labels=group_counts.index, autopct='%1.1f%%',\n        startangle=90, colors=sns.color_palette(\"Set3\"), labeldistance=1.1\n    )\n    axes[i].set_title(f'Hours of using computer/internet\\n for SII = {sii_group}')\n    axes[i].axis('equal')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:36:17.804731Z","iopub.execute_input":"2024-11-24T14:36:17.805566Z","iopub.status.idle":"2024-11-24T14:36:18.470337Z","shell.execute_reply.started":"2024-11-24T14:36:17.805522Z","shell.execute_reply":"2024-11-24T14:36:18.469343Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = sii_reported.groupby(\n    ['sii', 'internet_use_encoded']\n).size().unstack(fill_value=0)\nstats_prop = stats.div(stats.sum(axis=1), axis=0) * 100\n\nstats = stats.astype(str) +' (' + stats_prop.round(1).astype(str) + '%)'\nstats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:36:34.474355Z","iopub.execute_input":"2024-11-24T14:36:34.474740Z","iopub.status.idle":"2024-11-24T14:36:34.496059Z","shell.execute_reply.started":"2024-11-24T14:36:34.474707Z","shell.execute_reply":"2024-11-24T14:36:34.495026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[\n    (train['internet_use_encoded'] == '< 1h/day') & \n    (train['sii'].isin(['2 (Moderate)', '3 (Severe)']))\n]['Basic_Demos-Age'].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:36:48.570397Z","iopub.execute_input":"2024-11-24T14:36:48.570883Z","iopub.status.idle":"2024-11-24T14:36:48.586535Z","shell.execute_reply.started":"2024-11-24T14:36:48.570837Z","shell.execute_reply":"2024-11-24T14:36:48.585323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"groups = data_dict.groupby('Instrument')['Field'].apply(list).to_dict()\n\nfor instrument, features in groups.items():\n    print(f\"{instrument}: {features}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:37:07.167237Z","iopub.execute_input":"2024-11-24T14:37:07.167669Z","iopub.status.idle":"2024-11-24T14:37:07.176248Z","shell.execute_reply.started":"2024-11-24T14:37:07.167631Z","shell.execute_reply":"2024-11-24T14:37:07.175030Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"season_columns = [col for col in train.columns if 'Season' in col]\nseason_df = train[season_columns]\nseason_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:37:20.868208Z","iopub.execute_input":"2024-11-24T14:37:20.868590Z","iopub.status.idle":"2024-11-24T14:37:20.887060Z","shell.execute_reply.started":"2024-11-24T14:37:20.868559Z","shell.execute_reply":"2024-11-24T14:37:20.885750Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[season_columns] = train[season_columns].fillna(\"Missing\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:37:34.602935Z","iopub.execute_input":"2024-11-24T14:37:34.603340Z","iopub.status.idle":"2024-11-24T14:37:34.617193Z","shell.execute_reply.started":"2024-11-24T14:37:34.603303Z","shell.execute_reply":"2024-11-24T14:37:34.615830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dict = data_dict[data_dict['Instrument'] != 'Parent-Child Internet Addiction Test']\ncontinuous_cols = data_dict[data_dict['Type'].str.contains(\n    'float|int', case=False\n)]['Field'].tolist()\n\n# target = train[['sii']]\n# train = train.drop(columns = columns_not_in_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:37:51.235742Z","iopub.execute_input":"2024-11-24T14:37:51.236153Z","iopub.status.idle":"2024-11-24T14:37:51.243771Z","shell.execute_reply.started":"2024-11-24T14:37:51.236118Z","shell.execute_reply":"2024-11-24T14:37:51.242546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"groups.get('Demographics', [])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:41:15.457287Z","iopub.execute_input":"2024-11-24T14:41:15.457700Z","iopub.status.idle":"2024-11-24T14:41:15.464676Z","shell.execute_reply.started":"2024-11-24T14:41:15.457665Z","shell.execute_reply":"2024-11-24T14:41:15.463537Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(12, 5))\n\n# Season of Enrollment\nseason_counts = train['Basic_Demos-Enroll_Season'].value_counts(dropna=False)\n\naxes[0].pie(\n    season_counts, labels=season_counts.index,\n    autopct='%1.1f%%', startangle=90,\n    colors=sns.color_palette(\"Set3\")\n)\naxes[0].set_title('Season of Enrollment')\naxes[0].axis('equal')\n\n# Age Distribution by Sex\nsns.histplot(\n    data=train, x='Basic_Demos-Age',\n    hue='Basic_Demos-Sex', multiple='dodge',\n    palette=\"Set2\", bins=20, ax=axes[1]\n)\naxes[1].set_title('Age Distribution by Sex')\naxes[1].set_xlabel('Age')\naxes[1].set_ylabel('Count')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:41:32.574217Z","iopub.execute_input":"2024-11-24T14:41:32.575118Z","iopub.status.idle":"2024-11-24T14:41:33.288045Z","shell.execute_reply.started":"2024-11-24T14:41:32.575076Z","shell.execute_reply":"2024-11-24T14:41:33.286781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, 'Basic_Demos-Age')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:42:06.208243Z","iopub.execute_input":"2024-11-24T14:42:06.208659Z","iopub.status.idle":"2024-11-24T14:42:06.227931Z","shell.execute_reply.started":"2024-11-24T14:42:06.208624Z","shell.execute_reply":"2024-11-24T14:42:06.226787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"groups.get(\"Children's Global Assessment Scale\", [])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:42:20.954632Z","iopub.execute_input":"2024-11-24T14:42:20.955529Z","iopub.status.idle":"2024-11-24T14:42:20.962135Z","shell.execute_reply.started":"2024-11-24T14:42:20.955486Z","shell.execute_reply":"2024-11-24T14:42:20.960852Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = train[train['CGAS-CGAS_Score'].notnull()]\nage_range = data['Basic_Demos-Age']\nprint(\n    f\"Age range for participants with CGAS-CGAS_Score data:\"\n    f\" {age_range.min()} - {age_range.max()} years\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:42:44.304323Z","iopub.execute_input":"2024-11-24T14:42:44.304713Z","iopub.status.idle":"2024-11-24T14:42:44.314580Z","shell.execute_reply.started":"2024-11-24T14:42:44.304680Z","shell.execute_reply":"2024-11-24T14:42:44.313352Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, 'CGAS-CGAS_Score')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:43:02.308659Z","iopub.execute_input":"2024-11-24T14:43:02.309059Z","iopub.status.idle":"2024-11-24T14:43:02.328648Z","shell.execute_reply.started":"2024-11-24T14:43:02.309025Z","shell.execute_reply":"2024-11-24T14:43:02.327516Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[train['CGAS-CGAS_Score'] > 100]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:43:15.681609Z","iopub.execute_input":"2024-11-24T14:43:15.682021Z","iopub.status.idle":"2024-11-24T14:43:15.706344Z","shell.execute_reply.started":"2024-11-24T14:43:15.681981Z","shell.execute_reply":"2024-11-24T14:43:15.705290Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.loc[train['CGAS-CGAS_Score'] == 999, 'CGAS-CGAS_Score'] = np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:43:30.090064Z","iopub.execute_input":"2024-11-24T14:43:30.090476Z","iopub.status.idle":"2024-11-24T14:43:30.097183Z","shell.execute_reply.started":"2024-11-24T14:43:30.090438Z","shell.execute_reply":"2024-11-24T14:43:30.095830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 5))\n\n# CGAS-Season\nplt.subplot(1, 2, 1)\ncgas_season_counts = train['CGAS-Season'].value_counts(normalize=True)\nplt.pie(\n    cgas_season_counts, \n    labels=cgas_season_counts.index, \n    autopct='%1.1f%%', \n    startangle=90, \n    colors=sns.color_palette(\"Set3\")\n)\nplt.title('CGAS-Season')\nplt.axis('equal')\n\n# CGAS-CGAS_Score without outliers (score == 999)\nplt.subplot(1, 2, 2)\nsns.histplot(\n    train['CGAS-CGAS_Score'].dropna(),\n    bins=20, kde=True\n)\nplt.title('CGAS-CGAS_Score (Without Outlier)')\nplt.xlabel('CGAS Score')\nplt.ylabel('Count')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:43:52.166988Z","iopub.execute_input":"2024-11-24T14:43:52.167418Z","iopub.status.idle":"2024-11-24T14:43:52.733225Z","shell.execute_reply.started":"2024-11-24T14:43:52.167385Z","shell.execute_reply":"2024-11-24T14:43:52.732113Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, 'CGAS-CGAS_Score')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:44:20.871350Z","iopub.execute_input":"2024-11-24T14:44:20.872167Z","iopub.status.idle":"2024-11-24T14:44:20.891195Z","shell.execute_reply.started":"2024-11-24T14:44:20.872132Z","shell.execute_reply":"2024-11-24T14:44:20.889687Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bins = np.arange(0, 101, 10)\nlabels = [\n    \"1-10: Needs constant supervision (24 hour care)\",\n    \"11-20: Needs considerable supervision\",\n    \"21-30: Unable to function in almost all areas\",\n    \"31-40: Major impairment in functioning in several areas\",\n    \"41-50: Moderate degree of interference in functioning\",\n    \"51-60: Variable functioning with sporadic difficulties\",\n    \"61-70: Some difficulty in a single area\",\n    \"71-80: No more than slight impairment in functioning\",\n    \"81-90: Good functioning in all areas\",\n    \"91-100: Superior functioning\"\n]\n\ntrain['CGAS_Score_Bin'] = pd.cut(\n    train['CGAS-CGAS_Score'], bins=bins, labels=labels\n)\n\ncounts = train['CGAS_Score_Bin'].value_counts().reindex(labels)\nprop = (counts / counts.sum() * 100).round(1)\ncount_prop_labels = counts.astype(str) + \" (\" + prop.astype(str) + \"%)\"\n\nplt.figure(figsize=(18, 6))\nbars = plt.barh(labels, counts)\nplt.xlabel('Count')\nplt.title('CGAS Score Distribution')\nfor bar, label in zip(bars, count_prop_labels):\n    plt.text(\n        bar.get_width(), bar.get_y() + bar.get_height() / 2, label, va='center'\n    )\n\nplt.gca().invert_yaxis()\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:45:11.988752Z","iopub.execute_input":"2024-11-24T14:45:11.989178Z","iopub.status.idle":"2024-11-24T14:45:12.551873Z","shell.execute_reply.started":"2024-11-24T14:45:11.989140Z","shell.execute_reply":"2024-11-24T14:45:12.550750Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_filt = train.dropna(subset=['CGAS_Score_Bin', 'complete_resp_total'])\ntrain_filt.loc[:, 'CGAS_Score_Bin'] = train_filt['CGAS_Score_Bin'].cat.remove_unused_categories()\ntrain_filt.loc[:, 'sii'] = train_filt['sii'].cat.remove_unused_categories()\nlen(train_filt)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:45:32.234489Z","iopub.execute_input":"2024-11-24T14:45:32.234951Z","iopub.status.idle":"2024-11-24T14:45:32.250786Z","shell.execute_reply.started":"2024-11-24T14:45:32.234914Z","shell.execute_reply":"2024-11-24T14:45:32.249674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(16, 5))\n\n# CGAS-CGAS_Score vs sii\nsns.boxplot(\n    data=train_filt,\n    x='sii', y='CGAS-CGAS_Score',\n    palette='Set3', ax=axes[0]\n)\naxes[0].set_xlabel('SII Score')\naxes[0].set_ylabel('CGAS Score')\naxes[0].set_title('Distribution of CGAS Scores by SII')\n\n# complete_resp_total vs CGAS_Score_Bin\nsns.boxplot(\n    data=train_filt,\n    x='CGAS_Score_Bin', y='complete_resp_total',\n    ax=axes[1], palette='Set3'\n)\n# Get the tick positions and match the labels\nrange_labels = [label.split(\":\")[0] for label in train_filt['CGAS_Score_Bin'].cat.categories]\naxes[1].set_xticklabels(range_labels)\n\naxes[1].set_xlabel('CGAS Score category')\naxes[1].set_ylabel('PCIAT_Total for Complete PCIAT Responses')\naxes[1].set_title('Distribution of PCIAT_Total by CGAS Score categories')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:46:01.609793Z","iopub.execute_input":"2024-11-24T14:46:01.610209Z","iopub.status.idle":"2024-11-24T14:46:02.487904Z","shell.execute_reply.started":"2024-11-24T14:46:01.610174Z","shell.execute_reply":"2024-11-24T14:46:02.486866Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"score_min_max = train.groupby('sii')['CGAS-CGAS_Score'].agg(['min', 'max'])\nscore_min_max = score_min_max.rename(\n    columns={'min': 'Minimum CGAS Score', 'max': 'Maximum CGAS Score'}\n)\nscore_min_max","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:46:46.872633Z","iopub.execute_input":"2024-11-24T14:46:46.873028Z","iopub.status.idle":"2024-11-24T14:46:46.888303Z","shell.execute_reply.started":"2024-11-24T14:46:46.872995Z","shell.execute_reply":"2024-11-24T14:46:46.887222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_filt[train_filt['CGAS-CGAS_Score'] < 35][\n    ['Basic_Demos-Age', 'Basic_Demos-Sex', 'sii',\n     'CGAS-CGAS_Score',\n     'PreInt_EduHx-computerinternet_hoursday']\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:47:03.518310Z","iopub.execute_input":"2024-11-24T14:47:03.518699Z","iopub.status.idle":"2024-11-24T14:47:03.533958Z","shell.execute_reply.started":"2024-11-24T14:47:03.518666Z","shell.execute_reply":"2024-11-24T14:47:03.532742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[train['CGAS-CGAS_Score'] > 90][\n    ['Basic_Demos-Age', 'Basic_Demos-Sex', 'sii',\n     'CGAS-CGAS_Score',\n     'PreInt_EduHx-computerinternet_hoursday']\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:47:17.226994Z","iopub.execute_input":"2024-11-24T14:47:17.227404Z","iopub.status.idle":"2024-11-24T14:47:17.243995Z","shell.execute_reply.started":"2024-11-24T14:47:17.227373Z","shell.execute_reply":"2024-11-24T14:47:17.242752Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"groups.get('Physical Measures', [])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:47:31.600155Z","iopub.execute_input":"2024-11-24T14:47:31.600558Z","iopub.status.idle":"2024-11-24T14:47:31.608157Z","shell.execute_reply.started":"2024-11-24T14:47:31.600523Z","shell.execute_reply":"2024-11-24T14:47:31.606894Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_physical = groups.get('Physical Measures', [])\ncols = [col for col in features_physical if col in continuous_cols]\n\nplt.figure(figsize=(24, 10))\nn_cols = 4\nn_rows = len(cols) // n_cols + 1\n\nfor i, col in enumerate(cols):\n    plt.subplot(n_rows, n_cols, i + 1)\n    train[col].hist(bins=20)\n    plt.title(col)\n\nplt.subplot(n_rows, n_cols, len(cols) + 1)\nseason_counts = train['Physical-Season'].value_counts(dropna=False)\nplt.pie(\n    season_counts,\n    labels=season_counts.index,\n    autopct='%1.1f%%',\n    startangle=90,\n    colors=sns.color_palette(\"Set3\")\n)\nplt.title('Physical-Season')\n\nplt.suptitle('Histograms for Physical Measures and Physical-Season Pie Chart', y=1.05)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:47:53.240776Z","iopub.execute_input":"2024-11-24T14:47:53.241656Z","iopub.status.idle":"2024-11-24T14:47:55.716180Z","shell.execute_reply.started":"2024-11-24T14:47:53.241607Z","shell.execute_reply":"2024-11-24T14:47:55.715000Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:48:13.143216Z","iopub.execute_input":"2024-11-24T14:48:13.143641Z","iopub.status.idle":"2024-11-24T14:48:13.182318Z","shell.execute_reply.started":"2024-11-24T14:48:13.143606Z","shell.execute_reply":"2024-11-24T14:48:13.181173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wh_cols = [\n    'Physical-BMI', 'Physical-Height',\n    'Physical-Weight', 'Physical-Waist_Circumference'\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:48:28.020208Z","iopub.execute_input":"2024-11-24T14:48:28.020626Z","iopub.status.idle":"2024-11-24T14:48:28.025569Z","shell.execute_reply.started":"2024-11-24T14:48:28.020591Z","shell.execute_reply":"2024-11-24T14:48:28.024471Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(train[wh_cols] == 0).sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:48:42.472196Z","iopub.execute_input":"2024-11-24T14:48:42.472589Z","iopub.status.idle":"2024-11-24T14:48:42.482294Z","shell.execute_reply.started":"2024-11-24T14:48:42.472556Z","shell.execute_reply":"2024-11-24T14:48:42.481163Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[wh_cols] = train[wh_cols].replace(0, np.nan)\ncalculate_stats(train, wh_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:48:54.907880Z","iopub.execute_input":"2024-11-24T14:48:54.908241Z","iopub.status.idle":"2024-11-24T14:48:54.938390Z","shell.execute_reply.started":"2024-11-24T14:48:54.908210Z","shell.execute_reply":"2024-11-24T14:48:54.937162Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lbs_to_kg = 0.453592\ninches_to_cm = 2.54\n\ntrain['Physical-Weight'] = train['Physical-Weight'] * lbs_to_kg\ntrain['Physical-Height'] = train['Physical-Height'] * inches_to_cm\ntrain['Physical-Waist_Circumference'] = train['Physical-Waist_Circumference'] * inches_to_cm\n\n# Recalculate BMI: BMI = weight (kg) / (height (m)^2)\ntrain['Physical-BMI'] = np.where(\n    train['Physical-Weight'].notna() & train['Physical-Height'].notna(),\n    train['Physical-Weight'] / ((train['Physical-Height'] / 100) ** 2),\n    np.nan  # If either is NaN, set BMI to NaN\n)\n\ncalculate_stats(train, wh_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:49:28.383528Z","iopub.execute_input":"2024-11-24T14:49:28.383947Z","iopub.status.idle":"2024-11-24T14:49:28.417738Z","shell.execute_reply.started":"2024-11-24T14:49:28.383911Z","shell.execute_reply":"2024-11-24T14:49:28.416584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(18, 5))\n\n# Physical-Weight by Age\nplt.subplot(1, 3, 1)\nsns.scatterplot(x='Basic_Demos-Age', y='Physical-Weight', data=train)\nplt.title('Physical-Weight by Age')\nplt.xlabel('Age')\nplt.ylabel('Weight (kg)')\n\n# Physical-Height by Age\nplt.subplot(1, 3, 2)\nsns.scatterplot(x='Basic_Demos-Age', y='Physical-Height', data=train)\nplt.title('Physical-Height by Age')\nplt.xlabel('Age')\nplt.ylabel('Height (cm)')\n\n# Physical-Waist_Circumference vs Physical-Weight\nplt.subplot(1, 3, 3)\nsns.scatterplot(x='Physical-Weight', y='Physical-Waist_Circumference', data=train)\nplt.title('Waist Circumference vs Weight')\nplt.xlabel('Weight (kg)')\nplt.ylabel('Waist Circumference (cm)')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:49:50.301002Z","iopub.execute_input":"2024-11-24T14:49:50.301832Z","iopub.status.idle":"2024-11-24T14:49:51.427061Z","shell.execute_reply.started":"2024-11-24T14:49:50.301777Z","shell.execute_reply":"2024-11-24T14:49:51.425942Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bp_hr_cols = [\n    'Physical-Diastolic_BP', 'Physical-Systolic_BP',\n    'Physical-HeartRate'\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:50:06.221386Z","iopub.execute_input":"2024-11-24T14:50:06.221778Z","iopub.status.idle":"2024-11-24T14:50:06.226645Z","shell.execute_reply.started":"2024-11-24T14:50:06.221745Z","shell.execute_reply":"2024-11-24T14:50:06.225473Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(train[bp_hr_cols] < 50).sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:50:17.210814Z","iopub.execute_input":"2024-11-24T14:50:17.211888Z","iopub.status.idle":"2024-11-24T14:50:17.221574Z","shell.execute_reply.started":"2024-11-24T14:50:17.211679Z","shell.execute_reply":"2024-11-24T14:50:17.220437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[train['Physical-Systolic_BP'] <= train['Physical-Diastolic_BP']][bp_hr_cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:50:39.500101Z","iopub.execute_input":"2024-11-24T14:50:39.501050Z","iopub.status.idle":"2024-11-24T14:50:39.514619Z","shell.execute_reply.started":"2024-11-24T14:50:39.501005Z","shell.execute_reply":"2024-11-24T14:50:39.513322Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[cols] = train[cols].replace(0, np.nan)\ntrain.loc[train['Physical-Systolic_BP'] <= train['Physical-Diastolic_BP'], bp_hr_cols] = np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:50:52.296081Z","iopub.execute_input":"2024-11-24T14:50:52.296482Z","iopub.status.idle":"2024-11-24T14:50:52.309479Z","shell.execute_reply.started":"2024-11-24T14:50:52.296449Z","shell.execute_reply":"2024-11-24T14:50:52.308234Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 5))\n\n# Diastolic BP vs Heart Rate\nplt.subplot(1, 2, 1)\nsns.scatterplot(x='Physical-Diastolic_BP', y='Physical-HeartRate', data=train)\nplt.title('Diastolic BP vs Heart Rate')\nplt.xlabel('Diastolic Blood Pressure (mmHg)')\nplt.ylabel('Heart rate (beats/min)')\n\n# Systolic BP vs Heart Rate\nplt.subplot(1, 2, 2)\nsns.scatterplot(x='Physical-Systolic_BP', y='Physical-HeartRate', data=train)\nplt.title('Systolic BP vs Heart Rate')\nplt.xlabel('Systolic Blood Pressure (mmHg)')\nplt.ylabel('Heart rate (beats/min)')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:51:07.782076Z","iopub.execute_input":"2024-11-24T14:51:07.782853Z","iopub.status.idle":"2024-11-24T14:51:08.437333Z","shell.execute_reply.started":"2024-11-24T14:51:07.782816Z","shell.execute_reply":"2024-11-24T14:51:08.436233Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(16, 6))\n\n# BMI vs Systolic Blood Pressure\nsns.scatterplot(x='Physical-BMI', y='Physical-Systolic_BP', data=train, ax=axes[0], color='b')\naxes[0].set_title('BMI vs Systolic Blood Pressure')\naxes[0].set_xlabel('Body Mass Index (BMI) (kg/m^2)')\naxes[0].set_ylabel('Systolic Blood Pressure (mmHg)')\n\n# Systolic Blood Pressure vs Diastolic Blood Pressure\nsns.scatterplot(\n    x='Physical-Systolic_BP', y='Physical-Diastolic_BP',\n    data=train, ax=axes[1], color='g'\n)\naxes[1].set_title('Systolic Blood Pressure vs Diastolic Blood Pressure')\naxes[1].set_xlabel('Systolic Blood Pressure (mmHg)')\naxes[1].set_ylabel('Diastolic Blood Pressure (mmHg)')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:51:28.342286Z","iopub.execute_input":"2024-11-24T14:51:28.342666Z","iopub.status.idle":"2024-11-24T14:51:29.046895Z","shell.execute_reply.started":"2024-11-24T14:51:28.342635Z","shell.execute_reply":"2024-11-24T14:51:29.045808Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"normal_ranges = {\n    'Physical-BMI': (18.5, 24.9),\n    'Physical-Height': (100, 193),\n    'Physical-Weight': (20, 120),\n    'Physical-Waist_Circumference': (50, 90),\n    'Physical-Diastolic_BP': (60, 80),\n    'Physical-HeartRate': (60, 100),\n    'Physical-Systolic_BP': (90, 120)\n}\n\ndef count_out_of_range(data, column, low, high):\n    return ((data[column] < low) | (data[column] > high)).sum","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:52:47.243680Z","iopub.execute_input":"2024-11-24T14:52:47.244105Z","iopub.status.idle":"2024-11-24T14:52:47.250385Z","shell.execute_reply.started":"2024-11-24T14:52:47.244067Z","shell.execute_reply":"2024-11-24T14:52:47.249129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bmi_categories = [\n    ('Underweight', train['Physical-BMI'] < 18.5),\n    ('Normal weight', (train['Physical-BMI'] >= 18.5) & (train['Physical-BMI'] <= 24.9)),\n    ('Overweight', (train['Physical-BMI'] >= 25) & (train['Physical-BMI'] <= 29.9)),\n    ('Obesity', train['Physical-BMI'] >= 30)\n]\nbmi_category_counts = {label: condition.sum() for label, condition in bmi_categories}\n\nplt.figure(figsize=(5, 6))\nplt.pie(bmi_category_counts.values(),\n        labels=bmi_category_counts.keys(),\n        autopct='%1.1f%%', startangle=90,\n        colors=plt.cm.Set3.colors)\nplt.title('BMI Distribution by Category')\nplt.axis('equal')\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:54:18.803489Z","iopub.execute_input":"2024-11-24T14:54:18.803875Z","iopub.status.idle":"2024-11-24T14:54:18.991588Z","shell.execute_reply.started":"2024-11-24T14:54:18.803845Z","shell.execute_reply":"2024-11-24T14:54:18.989982Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[train['Physical-BMI'] < 12][cols + ['Basic_Demos-Age']].sort_values(by = 'Physical-BMI')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:54:50.425241Z","iopub.execute_input":"2024-11-24T14:54:50.426362Z","iopub.status.idle":"2024-11-24T14:54:50.447413Z","shell.execute_reply.started":"2024-11-24T14:54:50.426293Z","shell.execute_reply":"2024-11-24T14:54:50.446258Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[train['Physical-Systolic_BP'] > 160][cols + ['Basic_Demos-Age']].sort_values(by = 'Physical-Systolic_BP')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:55:02.432348Z","iopub.execute_input":"2024-11-24T14:55:02.432753Z","iopub.status.idle":"2024-11-24T14:55:02.457489Z","shell.execute_reply.started":"2024-11-24T14:55:02.432711Z","shell.execute_reply":"2024-11-24T14:55:02.456076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_subset = train[cols + ['complete_resp_total']]\n\ncorr_matrix = data_subset.corr()\n\nplt.figure(figsize=(10, 8))\nsns.heatmap(corr_matrix, annot=True, cmap='coolwarm', fmt='.2f', vmin=-1, vmax=1)\nplt.title('Correlation Heatmap')\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:55:17.655346Z","iopub.execute_input":"2024-11-24T14:55:17.655821Z","iopub.status.idle":"2024-11-24T14:55:18.206054Z","shell.execute_reply.started":"2024-11-24T14:55:17.655782Z","shell.execute_reply":"2024-11-24T14:55:18.204865Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dict[data_dict['Instrument'] == 'Bio-electric Impedance Analysis']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:55:32.194213Z","iopub.execute_input":"2024-11-24T14:55:32.195223Z","iopub.status.idle":"2024-11-24T14:55:32.210381Z","shell.execute_reply.started":"2024-11-24T14:55:32.195183Z","shell.execute_reply":"2024-11-24T14:55:32.209090Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bia_data_dict = data_dict[data_dict['Instrument'] == 'Bio-electric Impedance Analysis']\ncategorical_columns = bia_data_dict[bia_data_dict['Type'] == 'categorical int']['Field'].tolist()\ncontinuous_columns = bia_data_dict[bia_data_dict['Type'] == 'float']['Field'].tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:55:46.232426Z","iopub.execute_input":"2024-11-24T14:55:46.232832Z","iopub.status.idle":"2024-11-24T14:55:46.240525Z","shell.execute_reply.started":"2024-11-24T14:55:46.232798Z","shell.execute_reply":"2024-11-24T14:55:46.239348Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 3, figsize=(18, 5))\n\n# Season\nseason_counts = train['BIA-Season'].value_counts(normalize=True)\naxes[0].pie(\n    season_counts, \n    labels=season_counts.index, \n    autopct='%1.1f%%', \n    startangle=90, \n    colors=sns.color_palette(\"Set3\")\n)\naxes[0].set_title(\n    f\"{bia_data_dict[bia_data_dict['Field'] == 'BIA-Season']['Description'].values[0]}\"\n)\naxes[0].axis('equal')\n\n# Other categorical columns\nfor idx, col in enumerate(categorical_columns):\n    sns.countplot(x=col, data=train, palette=\"Set3\", ax=axes[idx+1])\n    axes[idx+1].set_title(data_dict[data_dict['Field'] == col]['Description'].values[0])\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:56:05.311982Z","iopub.execute_input":"2024-11-24T14:56:05.312510Z","iopub.status.idle":"2024-11-24T14:56:05.979602Z","shell.execute_reply.started":"2024-11-24T14:56:05.312473Z","shell.execute_reply":"2024-11-24T14:56:05.978451Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(24, 20))\n\nfor idx, col in enumerate(continuous_columns):\n    plt.subplot(4, 4, idx + 1)\n    sns.histplot(train[col].dropna(), bins=20, kde=True)\n    plt.title(data_dict[data_dict['Field'] == col]['Description'].values[0])\n    plt.xlabel('Value')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:56:32.730429Z","iopub.execute_input":"2024-11-24T14:56:32.731238Z","iopub.status.idle":"2024-11-24T14:56:38.091224Z","shell.execute_reply.started":"2024-11-24T14:56:32.731198Z","shell.execute_reply":"2024-11-24T14:56:38.089913Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, continuous_columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:56:52.023910Z","iopub.execute_input":"2024-11-24T14:56:52.024478Z","iopub.status.idle":"2024-11-24T14:56:52.091679Z","shell.execute_reply.started":"2024-11-24T14:56:52.024425Z","shell.execute_reply":"2024-11-24T14:56:52.090368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bmi_data = train[['BIA-BIA_BMI', 'Physical-BMI']].dropna()\n\nplt.figure(figsize=(8, 6))\nsns.scatterplot(\n    x='BIA-BIA_BMI', y='Physical-BMI',\n    data=bmi_data,\n    color='b'\n)\nplt.title('Comparison of BIA-BMI vs Physical-BMI')\nplt.xlabel('BIA-BMI')\nplt.ylabel('Physical-BMI')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:57:17.138592Z","iopub.execute_input":"2024-11-24T14:57:17.139025Z","iopub.status.idle":"2024-11-24T14:57:17.586238Z","shell.execute_reply.started":"2024-11-24T14:57:17.138958Z","shell.execute_reply":"2024-11-24T14:57:17.585131Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bmi_measures = train[['BIA-Season', 'Physical-Season']].dropna()\nbmi_measures.groupby(['BIA-Season', 'Physical-Season']).size().reset_index(name='Count')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:57:32.262380Z","iopub.execute_input":"2024-11-24T14:57:32.262781Z","iopub.status.idle":"2024-11-24T14:57:32.281014Z","shell.execute_reply.started":"2024-11-24T14:57:32.262745Z","shell.execute_reply":"2024-11-24T14:57:32.279856Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"groups.get('FitnessGram Vitals and Treadmill', [])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:58:34.329219Z","iopub.execute_input":"2024-11-24T14:58:34.329632Z","iopub.status.idle":"2024-11-24T14:58:34.337163Z","shell.execute_reply.started":"2024-11-24T14:58:34.329596Z","shell.execute_reply":"2024-11-24T14:58:34.335980Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = train[train['Fitness_Endurance-Max_Stage'].notnull()]\nage_range = data['Basic_Demos-Age']\nprint(\n    f\"Age range for participants with Fitness_Endurance-Max_Stage data:\"\n    f\" {age_range.min()} - {age_range.max()} years\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:58:45.800977Z","iopub.execute_input":"2024-11-24T14:58:45.801831Z","iopub.status.idle":"2024-11-24T14:58:45.811808Z","shell.execute_reply.started":"2024-11-24T14:58:45.801790Z","shell.execute_reply":"2024-11-24T14:58:45.810593Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 4, figsize=(24, 5))\n\n# Fitness Endurance Season\ntrain['Fitness_Endurance-Season'].value_counts(normalize=True).plot.pie(\n    autopct='%1.1f%%', colors=plt.cm.Set3.colors, ax=axes[0]\n)\naxes[0].set_title('Fitness Endurance Season')\naxes[0].axis('equal')  # Equal aspect ratio ensures the pie is drawn as a circle.\n\n# Box plot for Max Stage by Season\nsns.violinplot(\n    x='Fitness_Endurance-Season',\n    y='Fitness_Endurance-Max_Stage',\n    data=train, palette=\"Set3\",\n    ax=axes[1]\n)\naxes[1].set_title('Max Stage by Season')\naxes[1].set_xlabel('Season')\naxes[1].set_ylabel('Max Stage')\n# Fitness Endurance Time (Minutes)\nsns.histplot(train['Fitness_Endurance-Time_Mins'], bins=20, kde=True, ax=axes[2])\naxes[2].set_title('Fitness Endurance Time (Minutes)')\naxes[2].set_xlabel('Time (Minutes)')\n\n# Fitness Endurance Time (Seconds)\nsns.histplot(train['Fitness_Endurance-Time_Sec'], bins=20, kde=True, ax=axes[3])\naxes[3].set_title('Fitness Endurance Time (Seconds)')\naxes[3].set_xlabel('Time (Seconds)')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:59:23.613585Z","iopub.execute_input":"2024-11-24T14:59:23.613948Z","iopub.status.idle":"2024-11-24T14:59:24.991360Z","shell.execute_reply.started":"2024-11-24T14:59:23.613916Z","shell.execute_reply":"2024-11-24T14:59:24.990169Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 5))\n\nsns.violinplot(x='Basic_Demos-Age', y='Fitness_Endurance-Max_Stage', data=train, palette=\"Set3\")\nplt.title('Fitness Endurance Max Stage by Age')\nplt.xlabel('Age')\nplt.ylabel('Max Stage')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T14:59:46.076764Z","iopub.execute_input":"2024-11-24T14:59:46.077175Z","iopub.status.idle":"2024-11-24T14:59:46.751439Z","shell.execute_reply.started":"2024-11-24T14:59:46.077139Z","shell.execute_reply":"2024-11-24T14:59:46.750365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols = [\n    'Fitness_Endurance-Max_Stage',\n    'Fitness_Endurance-Time_Mins',\n    'Fitness_Endurance-Time_Sec'\n]\ncalculate_stats(train, cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:00:02.272832Z","iopub.execute_input":"2024-11-24T15:00:02.273507Z","iopub.status.idle":"2024-11-24T15:00:02.298409Z","shell.execute_reply.started":"2024-11-24T15:00:02.273469Z","shell.execute_reply":"2024-11-24T15:00:02.297322Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[\n    (train['Fitness_Endurance-Max_Stage'].notna()) & \n    (train['Fitness_Endurance-Time_Mins'].isna() | \n     train['Fitness_Endurance-Time_Sec'].isna())\n][cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:00:17.195690Z","iopub.execute_input":"2024-11-24T15:00:17.196129Z","iopub.status.idle":"2024-11-24T15:00:17.210716Z","shell.execute_reply.started":"2024-11-24T15:00:17.196090Z","shell.execute_reply":"2024-11-24T15:00:17.209657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.loc[\n    (train['Fitness_Endurance-Max_Stage'].notna()) & \n    (train['Fitness_Endurance-Time_Mins'].isna() | \n     train['Fitness_Endurance-Time_Sec'].isna()), cols\n] = np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:00:29.484714Z","iopub.execute_input":"2024-11-24T15:00:29.485167Z","iopub.status.idle":"2024-11-24T15:00:29.495445Z","shell.execute_reply.started":"2024-11-24T15:00:29.485131Z","shell.execute_reply":"2024-11-24T15:00:29.494097Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['Fitness_Endurance-Total_Time_Sec'] = train[\n    'Fitness_Endurance-Time_Mins'\n] * 60 + train['Fitness_Endurance-Time_Sec']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:00:42.059478Z","iopub.execute_input":"2024-11-24T15:00:42.059874Z","iopub.status.idle":"2024-11-24T15:00:42.066248Z","shell.execute_reply.started":"2024-11-24T15:00:42.059841Z","shell.execute_reply":"2024-11-24T15:00:42.065078Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, ['Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Total_Time_Sec'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:00:52.087245Z","iopub.execute_input":"2024-11-24T15:00:52.087657Z","iopub.status.idle":"2024-11-24T15:00:52.111092Z","shell.execute_reply.started":"2024-11-24T15:00:52.087621Z","shell.execute_reply":"2024-11-24T15:00:52.109995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dict[data_dict['Instrument'] == 'FitnessGram Child']\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:01:17.630486Z","iopub.execute_input":"2024-11-24T15:01:17.630875Z","iopub.status.idle":"2024-11-24T15:01:17.645538Z","shell.execute_reply.started":"2024-11-24T15:01:17.630840Z","shell.execute_reply":"2024-11-24T15:01:17.644356Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fgc_data_dict = data_dict[data_dict['Instrument'] == 'FitnessGram Child']\n\nfgc_columns = []\n\nfor index, row in fgc_data_dict.iterrows():\n    if '_Zone' not in row['Field']:\n        measure_field = row['Field']\n        measure_desc = row['Description']\n        \n        zone_field = measure_field + '_Zone'\n        zone_row = fgc_data_dict[fgc_data_dict['Field'] == zone_field]\n        \n        if not zone_row.empty:\n            zone_desc = zone_row['Description'].values[0]\n            fgc_columns.append((measure_field, zone_field, measure_desc, zone_desc))\n            \nfig, axes = plt.subplots(2, 4, figsize=(24, 10))\nfor idx, (measure, zone, measure_desc, zone_desc) in enumerate(fgc_columns):\n    row = idx // 4\n    col = idx % 4\n    \n    sns.histplot(\n        data=train, x=measure,\n        hue=zone, bins=20, palette='Set2',\n        ax=axes[row, col], kde=True\n    )\n    axes[row, col].set_title(f'{measure_desc}')\n\nseason_counts = train['FGC-Season'].value_counts(normalize=True)\naxes[1, 3].pie(\n    season_counts, labels=season_counts.index,\n    autopct='%1.1f%%', startangle=90,\n    colors=sns.color_palette(\"Set3\")\n)\naxes[1, 3].set_title('Season of participation')\naxes[1, 3].axis('equal') \n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:02:11.083109Z","iopub.execute_input":"2024-11-24T15:02:11.083544Z","iopub.status.idle":"2024-11-24T15:02:14.897505Z","shell.execute_reply.started":"2024-11-24T15:02:11.083508Z","shell.execute_reply":"2024-11-24T15:02:14.896442Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"measurement_columns = [measure for measure, _, _, _ in fgc_columns]\ncalculate_stats(train, measurement_columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:02:31.166313Z","iopub.execute_input":"2024-11-24T15:02:31.166733Z","iopub.status.idle":"2024-11-24T15:02:31.206923Z","shell.execute_reply.started":"2024-11-24T15:02:31.166699Z","shell.execute_reply":"2024-11-24T15:02:31.205787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compute_min_max_by_sex(train, sex, fgc_columns):\n    results = []\n    \n    for measure, zone, _, _ in fgc_columns:\n        sorted_zones = sorted(train[zone].dropna().unique())\n        \n        for zone_value in sorted_zones:\n            data = train[(train[zone] == zone_value) & \n                         (train['Basic_Demos-Sex'] == sex)][measure]\n            \n            if not data.empty:\n                min_val, max_val = data.min(), data.max()\n                results.append({\n                    'Zone': int(zone_value),\n                    'Measure': measure,\n                    'Min-Max': f'{min_val} - {max_val}'\n                })\n    \n    df = pd.DataFrame(results).pivot_table(\n        index='Zone', columns='Measure', values='Min-Max', aggfunc='first'\n    )\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:02:51.862654Z","iopub.execute_input":"2024-11-24T15:02:51.863010Z","iopub.status.idle":"2024-11-24T15:02:51.870663Z","shell.execute_reply.started":"2024-11-24T15:02:51.862980Z","shell.execute_reply":"2024-11-24T15:02:51.869556Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"compute_min_max_by_sex(train, 'Male', fgc_columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:03:04.653176Z","iopub.execute_input":"2024-11-24T15:03:04.654211Z","iopub.status.idle":"2024-11-24T15:03:04.707576Z","shell.execute_reply.started":"2024-11-24T15:03:04.654171Z","shell.execute_reply":"2024-11-24T15:03:04.706524Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"compute_min_max_by_sex(train, 'Female', fgc_columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:03:15.961063Z","iopub.execute_input":"2024-11-24T15:03:15.961441Z","iopub.status.idle":"2024-11-24T15:03:16.015498Z","shell.execute_reply.started":"2024-11-24T15:03:15.961410Z","shell.execute_reply":"2024-11-24T15:03:16.014360Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results_male = []\n\nfor measure, zone, _, _ in fgc_columns:\n    sorted_zones = sorted(train[zone].dropna().unique())\n    for zone_value in sorted_zones:\n        age_sex_data_by_zone = train[train[zone] == zone_value][\n            ['Basic_Demos-Age', 'Basic_Demos-Sex', measure]\n        ]\n        unique_ages = age_sex_data_by_zone['Basic_Demos-Age'].dropna().unique()\n\n        for age in sorted(unique_ages):\n            age_sex_data = age_sex_data_by_zone[\n                (age_sex_data_by_zone['Basic_Demos-Age'] == age) &\n                (age_sex_data_by_zone['Basic_Demos-Sex'] == 'Male')\n            ][measure]\n            \n            if not age_sex_data.empty:\n                min_val, max_val = age_sex_data.min(), age_sex_data.max()\n                results_male.append({\n                    'Age': age,\n                    'Sex': 'Male',\n                    'Zone': zone_value,\n                    'Measure': measure,\n                    'Min-Max': f'{min_val} - {max_val}'\n                })\ndf_male = pd.DataFrame(results_male).pivot_table(\n    index=['Age', 'Sex', 'Zone'], columns='Measure', values='Min-Max', aggfunc='first'\n)\n\ndf_male","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:04:30.952954Z","iopub.execute_input":"2024-11-24T15:04:30.953354Z","iopub.status.idle":"2024-11-24T15:04:31.187924Z","shell.execute_reply.started":"2024-11-24T15:04:30.953322Z","shell.execute_reply":"2024-11-24T15:04:31.186830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"age_ranges = []\n\nfor measure in measurement_columns:\n    valid_rows = train[~train[measure].isna()]\n    \n    min_age = valid_rows['Basic_Demos-Age'].min()\n    max_age = valid_rows['Basic_Demos-Age'].max()\n    \n    age_ranges.append({\n        'Measurement': measure,\n        'Min Age': min_age,\n        'Max Age': max_age\n    })\n\nage_ranges_df = pd.DataFrame(age_ranges)\nage_ranges_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:04:48.261438Z","iopub.execute_input":"2024-11-24T15:04:48.262303Z","iopub.status.idle":"2024-11-24T15:04:48.290798Z","shell.execute_reply.started":"2024-11-24T15:04:48.262236Z","shell.execute_reply":"2024-11-24T15:04:48.289720Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols = [col for col in train.columns if col.startswith('FGC-') \n        and 'Zone' not in col and 'Season' not in col]\ncols.extend(['Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Total_Time_Sec'])\n\ndata_subset = train[cols + ['complete_resp_total']]\n\ncorr_matrix = data_subset.corr()\n\nplt.figure(figsize=(10, 8))\nsns.heatmap(corr_matrix, annot=True, cmap='coolwarm', fmt='.2f', vmin=-1, vmax=1)\nplt.title('Correlation Heatmap')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:05:02.397757Z","iopub.execute_input":"2024-11-24T15:05:02.398120Z","iopub.status.idle":"2024-11-24T15:05:03.108501Z","shell.execute_reply.started":"2024-11-24T15:05:02.398090Z","shell.execute_reply":"2024-11-24T15:05:03.107237Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"age_groups = train['Age Group'].unique()\n\nfig, axes = plt.subplots(1, 3, figsize=(18, 6), sharey=True)\n\nfor i, age_group in enumerate(age_groups):\n    group_data = train[train['Age Group'] == age_group]\n    corr_matrix = group_data[cols + ['complete_resp_total', 'Basic_Demos-Age']].corr()\n    sns.heatmap(corr_matrix, annot=True, cmap='coolwarm', fmt='.1f',\n                vmin=-1, vmax=1, ax=axes[i], cbar=i == 0)\n    axes[i].set_title(f'{age_group}')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:05:18.784556Z","iopub.execute_input":"2024-11-24T15:05:18.784974Z","iopub.status.idle":"2024-11-24T15:05:20.908289Z","shell.execute_reply.started":"2024-11-24T15:05:18.784937Z","shell.execute_reply":"2024-11-24T15:05:20.907107Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[\n    (train['Age Group'] == 'Adults (19-22)') &\n    (train['complete_resp_total'].notna()) &\n    (train[cols].notna().any(axis=1))\n][cols + ['complete_resp_total', 'Basic_Demos-Age']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T15:05:38.639537Z","iopub.execute_input":"2024-11-24T15:05:38.639925Z","iopub.status.idle":"2024-11-24T15:05:38.661395Z","shell.execute_reply.started":"2024-11-24T15:05:38.639890Z","shell.execute_reply":"2024-11-24T15:05:38.660330Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}