{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. **Preprocessing** \n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:12.102278Z","iopub.execute_input":"2024-10-29T09:10:12.102715Z","iopub.status.idle":"2024-10-29T09:10:15.475466Z","shell.execute_reply.started":"2024-10-29T09:10:12.102674Z","shell.execute_reply":"2024-10-29T09:10:15.473849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dictionary = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv')\ndictionary","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:15.478401Z","iopub.execute_input":"2024-10-29T09:10:15.479113Z","iopub.status.idle":"2024-10-29T09:10:15.525265Z","shell.execute_reply.started":"2024-10-29T09:10:15.479061Z","shell.execute_reply":"2024-10-29T09:10:15.523153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:15.527402Z","iopub.execute_input":"2024-10-29T09:10:15.527902Z","iopub.status.idle":"2024-10-29T09:10:15.615832Z","shell.execute_reply.started":"2024-10-29T09:10:15.527855Z","shell.execute_reply":"2024-10-29T09:10:15.614202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = train\nbins = [0,5,7,9,10,12,14,16,18,20,22]\ndata['AgeGroup'] = pd.cut(data['Basic_Demos-Age'], bins)\nmale = data[data['Basic_Demos-Sex'] == 0]\nfemale = data[data['Basic_Demos-Sex'] == 1]\nmale_counts = male['AgeGroup'].value_counts(sort=False)\nfemale_counts = female['AgeGroup'].value_counts(sort=False)\nmale_percent = -100 * male_counts / len(male)  \nfemale_percent = 100 * female_counts / len(female)\n\nfig, ax = plt.subplots(figsize=(10, 8))\n\nax.barh(male_percent.index.astype(str), male_percent, color='blue', label='Male')\n\nax.barh(female_percent.index.astype(str), female_percent, color='green', label='Female')\n\nax.set_xlabel('Percentage')\nax.set_title('Age Distribution by Gender')\nax.legend()\n\nplt.show()\nprint(\"female:\",female.size)\nprint(\"male:\",male.size)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:15.619559Z","iopub.execute_input":"2024-10-29T09:10:15.620477Z","iopub.status.idle":"2024-10-29T09:10:16.118823Z","shell.execute_reply.started":"2024-10-29T09:10:15.620406Z","shell.execute_reply":"2024-10-29T09:10:16.117231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Bio_electric_col_names = dictionary[dictionary['Instrument'] == 'Bio-electric Impedance Analysis']['Field']\nBio_electric_col_value_names = Bio_electric_col_names[1:]","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:16.121128Z","iopub.execute_input":"2024-10-29T09:10:16.121674Z","iopub.status.idle":"2024-10-29T09:10:16.130796Z","shell.execute_reply.started":"2024-10-29T09:10:16.121613Z","shell.execute_reply":"2024-10-29T09:10:16.129298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Bio_electric_col_names","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:16.132497Z","iopub.execute_input":"2024-10-29T09:10:16.133016Z","iopub.status.idle":"2024-10-29T09:10:16.148602Z","shell.execute_reply.started":"2024-10-29T09:10:16.132937Z","shell.execute_reply":"2024-10-29T09:10:16.146410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check Season's relationship with BIA","metadata":{}},{"cell_type":"code","source":"season_index = ['Spring', 'Summer', 'Fall', 'Winter']\nfor name in Bio_electric_col_value_names:\n    avg_score = male.groupby('BIA-Season')[name].mean().reindex(season_index)\n    avg_score.plot(kind='bar')\n    plt.xlabel('age')\n    plt.ylabel(name)\n    plt.title(f'age-{name}')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:16.152856Z","iopub.execute_input":"2024-10-29T09:10:16.154163Z","iopub.status.idle":"2024-10-29T09:10:21.189719Z","shell.execute_reply.started":"2024-10-29T09:10:16.154106Z","shell.execute_reply":"2024-10-29T09:10:21.188664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The season seem to be indifferent","metadata":{}},{"cell_type":"code","source":"for name in Bio_electric_col_value_names:\n    print(sum(male[name].isna())/male.shape[0], sum(female[name].isna())/female.shape[0])\n## 50% of the BIA datas are missing","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:21.191354Z","iopub.execute_input":"2024-10-29T09:10:21.191826Z","iopub.status.idle":"2024-10-29T09:10:21.213666Z","shell.execute_reply.started":"2024-10-29T09:10:21.191773Z","shell.execute_reply":"2024-10-29T09:10:21.212458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"male[Bio_electric_col_value_names]\ncorrelation = male[Bio_electric_col_value_names].corr()\nsns.heatmap(correlation, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix of BIA Metrics')\nplt.show()\ncorrelation = female[Bio_electric_col_value_names].corr()\nsns.heatmap(correlation, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix of BIA Metrics')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:21.215825Z","iopub.execute_input":"2024-10-29T09:10:21.216351Z","iopub.status.idle":"2024-10-29T09:10:23.435055Z","shell.execute_reply.started":"2024-10-29T09:10:21.216295Z","shell.execute_reply":"2024-10-29T09:10:23.433705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For both male and female, it seems that BMR, DEE, ECW, FFM, ICW, LDM, LST, SMM, TBW are 'extremely' correlated\nAnd for female, also Fat and BMC\nMaybe revisit after clipping/removing the outliers","metadata":{}},{"cell_type":"code","source":"for name in Bio_electric_col_value_names:\n    sns.boxplot(data=[male[name], female[name]])\n    plt.title(f'Box Plots of {name}')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:23.440042Z","iopub.execute_input":"2024-10-29T09:10:23.440517Z","iopub.status.idle":"2024-10-29T09:10:26.951593Z","shell.execute_reply.started":"2024-10-29T09:10:23.440459Z","shell.execute_reply":"2024-10-29T09:10:26.949649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"No wonder...","metadata":{}},{"cell_type":"code","source":"male_bia = male.dropna(subset=Bio_electric_col_value_names)\nfemale_bia = female.dropna(subset=Bio_electric_col_value_names)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:26.953161Z","iopub.execute_input":"2024-10-29T09:10:26.953622Z","iopub.status.idle":"2024-10-29T09:10:26.969552Z","shell.execute_reply.started":"2024-10-29T09:10:26.953575Z","shell.execute_reply":"2024-10-29T09:10:26.967883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy import stats\ntrain_bia = train.dropna(subset=Bio_electric_col_value_names)\nz_scores = stats.zscore(train_bia[Bio_electric_col_value_names])\nabs_z_scores = np.abs(z_scores)\ntrain_bia_clean = train_bia[(abs_z_scores < 3).all(axis=1)]\nmale_bia_clean = train_bia_clean[train_bia_clean['Basic_Demos-Sex'] == 0]\nfemale_bia_clean = train_bia_clean[train_bia_clean['Basic_Demos-Sex'] == 1]\nprint(f\"Original male shape: {male_bia.shape}\")\nprint(f\"Male shape after outlier removal: {male_bia_clean.shape}\")\nprint(f\"Original female shape: {female_bia.shape}\")\nprint(f\"Female shape after outlier removal: {female_bia_clean.shape}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:26.971119Z","iopub.execute_input":"2024-10-29T09:10:26.971546Z","iopub.status.idle":"2024-10-29T09:10:26.998233Z","shell.execute_reply.started":"2024-10-29T09:10:26.971504Z","shell.execute_reply":"2024-10-29T09:10:26.996558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# Compute the Z-scores for each column\nz_scores = stats.zscore(male_bia[Bio_electric_col_value_names])\nabs_z_scores = np.abs(z_scores)\nmale_bia_clean = male_bia[(abs_z_scores < 3).all(axis=1)]\nprint(f\"Original data shape: {male_bia.shape}\")\nprint(f\"Data shape after outlier removal: {male_bia_clean.shape}\")\nz_scores = stats.zscore(female_bia[Bio_electric_col_value_names])\nabs_z_scores = np.abs(z_scores)\nfemale_bia_clean = female_bia[(abs_z_scores < 3).all(axis=1)]\nprint(f\"Original data shape: {female_bia.shape}\")\nprint(f\"Data shape after outlier removal: {female_bia_clean.shape}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:26.999757Z","iopub.execute_input":"2024-10-29T09:10:27.000169Z","iopub.status.idle":"2024-10-29T09:10:27.026945Z","shell.execute_reply.started":"2024-10-29T09:10:27.000130Z","shell.execute_reply":"2024-10-29T09:10:27.024937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"BMR should range from 500 to 3000, DEE should be bound within 4000, FFMI should range from 0, 30..., FAT should be 0\\~100, FMI should also be 0\\~","metadata":{}},{"cell_type":"code","source":"for name in Bio_electric_col_value_names:\n    sns.boxplot(data=[male_bia_clean[name], female_bia_clean[name]])\n    plt.title(f'Box Plots of {name}')\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:27.029107Z","iopub.execute_input":"2024-10-29T09:10:27.029538Z","iopub.status.idle":"2024-10-29T09:10:31.291551Z","shell.execute_reply.started":"2024-10-29T09:10:27.029497Z","shell.execute_reply":"2024-10-29T09:10:31.290095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Bio_electric_col_value_names.to_list()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:31.295101Z","iopub.execute_input":"2024-10-29T09:10:31.295543Z","iopub.status.idle":"2024-10-29T09:10:31.304107Z","shell.execute_reply.started":"2024-10-29T09:10:31.295502Z","shell.execute_reply":"2024-10-29T09:10:31.302465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (10, 10))\ncorrelation = male_bia_clean[Bio_electric_col_value_names.to_list() + ['sii']+['Basic_Demos-Age']].corr()\nsns.heatmap(correlation, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix of BIA Metrics')\nplt.show()\nplt.figure(figsize = (10, 10))\ncorrelation = female_bia_clean[Bio_electric_col_value_names.to_list() + ['sii']+['Basic_Demos-Age']].corr()\nsns.heatmap(correlation, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix of BIA Metrics')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:31.306518Z","iopub.execute_input":"2024-10-29T09:10:31.307089Z","iopub.status.idle":"2024-10-29T09:10:34.100726Z","shell.execute_reply.started":"2024-10-29T09:10:31.307031Z","shell.execute_reply":"2024-10-29T09:10:34.099361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(14, 6))  # 1 row, 2 columns\naxes[0].scatter(female['BIA-BIA_Fat'], female['BIA-BIA_BMC'], alpha=0.6, color='blue')\naxes[0].set_xlabel('BIA-BIA_Fat')\naxes[0].set_ylabel('BIA-BIA_BMC')\naxes[0].set_title('Fat-BMC before')\naxes[1].scatter(female_bia_clean['BIA-BIA_Fat'], female_bia_clean['BIA-BIA_BMC'], alpha=0.6, color='blue')\naxes[1].set_xlabel('BIA-BIA_Fat')\naxes[1].set_ylabel('BIA-BIA_BMC')\naxes[1].set_title('Fat-BMC after')\nplt.tight_layout()\nplt.show()\nplt.scatter(male_bia_clean['BIA-BIA_ECW'], male_bia_clean['BIA-BIA_BMR'], alpha=0.6, color='blue', label='Male')\nplt.scatter(female_bia_clean['BIA-BIA_ECW'], female_bia_clean['BIA-BIA_BMR'], alpha=0.6, color='red', label='Female')\nplt.xlabel('BIA-BIA_ECW')\nplt.ylabel('BIA-BIA_BMR')\nplt.title('ECW-BMR Male vs Female')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:34.102527Z","iopub.execute_input":"2024-10-29T09:10:34.103039Z","iopub.status.idle":"2024-10-29T09:10:35.372343Z","shell.execute_reply.started":"2024-10-29T09:10:34.102961Z","shell.execute_reply":"2024-10-29T09:10:35.371091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colors = ['blue', 'red']\ngenders = ['Male', 'Female']\nfor name in Bio_electric_col_value_names:\n    fig, axes = plt.subplots(1, 2, figsize=(14, 6))\n    sii_df = train_bia_clean['sii']\n    temp_df = train_bia_clean\n    for sex in range(2):\n        avg_score = temp_df[temp_df['Basic_Demos-Sex'] == sex].groupby('sii')[name].mean()\n        axes[sex].bar(avg_score.index, avg_score.values, color=colors[sex])\n        axes[sex].set_ylabel(name)\n        axes[sex].set_xlabel('sii')\n        axes[sex].set_title(f'{genders[sex]}\\'s {name}-sii')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:35.373961Z","iopub.execute_input":"2024-10-29T09:10:35.374501Z","iopub.status.idle":"2024-10-29T09:10:45.159390Z","shell.execute_reply.started":"2024-10-29T09:10:35.374448Z","shell.execute_reply":"2024-10-29T09:10:45.158085Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#BMR should range from 500 to 3000, DEE should be bound within 4000, FFMI should range from 0, 30..., Fat should be 0~100, FMI should also be 0~\n#train[\"BIA-BIA_FFMI\"] = np.clip(train[\"BIA-BIA_FFMI\"], 0.0, 25.0)\nclip_targets = ['BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_FFMI', 'BIA-BIA_Fat', 'BIA-BIA_FMI']\nfor name in Bio_electric_col_value_names:\n    print(name, sum(train_bia_clean[name]<0))\ntrain_bia_clean[Bio_electric_col_value_names][train_bia_clean['BIA-BIA_BMC']<0].head()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:45.161291Z","iopub.execute_input":"2024-10-29T09:10:45.162141Z","iopub.status.idle":"2024-10-29T09:10:45.205211Z","shell.execute_reply.started":"2024-10-29T09:10:45.162083Z","shell.execute_reply":"2024-10-29T09:10:45.204015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum((train_bia_clean['BIA-BIA_FMI'] < 0) | (train_bia_clean['BIA-BIA_Fat'] < 0))","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:45.206673Z","iopub.execute_input":"2024-10-29T09:10:45.207091Z","iopub.status.idle":"2024-10-29T09:10:45.217614Z","shell.execute_reply.started":"2024-10-29T09:10:45.207043Z","shell.execute_reply":"2024-10-29T09:10:45.216257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"abs(train_bia_clean['BIA-BIA_BMC'][train_bia_clean['Basic_Demos-Sex'] == sex])","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:45.219216Z","iopub.execute_input":"2024-10-29T09:10:45.219733Z","iopub.status.idle":"2024-10-29T09:10:45.232390Z","shell.execute_reply.started":"2024-10-29T09:10:45.219681Z","shell.execute_reply":"2024-10-29T09:10:45.231096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colors = ['blue', 'red']\ngenders = ['Male', 'Female']\ntemp_df = train_bia_clean[train_bia_clean['BIA-BIA_BMC'] < 0]\nfor name in Bio_electric_col_value_names:\n    fig, axes = plt.subplots(1, 3, figsize=(20, 6))\n    for sex in range(2):\n        axes[0].scatter(train_bia_clean['BIA-BIA_BMC'][train_bia_clean['Basic_Demos-Sex'] == sex], train_bia_clean[name][train_bia_clean['Basic_Demos-Sex'] == sex], color=colors[sex], alpha = 0.6, label=genders[sex])\n        axes[1].sharex(axes[0])\n        axes[1].sharey(axes[0])\n        axes[1].scatter(temp_df['BIA-BIA_BMC'][temp_df['Basic_Demos-Sex'] == sex], temp_df[name][temp_df['Basic_Demos-Sex'] == sex], color=colors[sex], alpha = 0.6, label=genders[sex])\n        axes[2].sharex(axes[0])\n        axes[2].sharey(axes[0])\n        axes[2].scatter(abs(train_bia_clean['BIA-BIA_BMC'][train_bia_clean['Basic_Demos-Sex'] == sex]), train_bia_clean[name][train_bia_clean['Basic_Demos-Sex'] == sex], color=colors[sex], alpha = 0.6, label=genders[sex])\n    for i in range(3):\n        axes[i].set_xlabel('BIA-BIA_BMC')\n        axes[i].set_ylabel(name)\n        axes[i].legend()\n    plt.show()\n#X shaped pattern","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:10:45.234061Z","iopub.execute_input":"2024-10-29T09:10:45.234498Z","iopub.status.idle":"2024-10-29T09:11:00.894910Z","shell.execute_reply.started":"2024-10-29T09:10:45.234441Z","shell.execute_reply":"2024-10-29T09:11:00.893672Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### todo: find more elegent way to remove BIA-BIA_BMC's strange value","metadata":{}},{"cell_type":"code","source":"temp_df = female_bia_clean[(female_bia_clean['BIA-BIA_BMC'] > 5) & (female_bia_clean['BIA-BIA_TBW'] < 45) | (female_bia_clean['BIA-BIA_BMC'] < 1) & (female_bia_clean['BIA-BIA_TBW'] > 50)]\ntemp_df2 = train_bia_clean.drop(temp_df.index)\ncolors = ['blue', 'red']\ngenders = ['Male', 'Female']\n#temp_df = train_bia_clean[train_bia_clean['BIA-BIA_BMC'] < 0]\nfor name in train_bia_clean.columns:\n    if train_bia_clean[name].dtype == object or name == train_bia_clean.columns[-1] or name in Bio_electric_col_value_names:\n        continue\n    fig, axes = plt.subplots(1, 3, figsize=(20, 6))\n    for sex in range(2):\n        axes[0].scatter(train_bia_clean['BIA-BIA_BMC'][train_bia_clean['Basic_Demos-Sex'] == sex], train_bia_clean[name][train_bia_clean['Basic_Demos-Sex'] == sex], color=colors[sex], alpha = 0.6, label=genders[sex])\n        axes[1].sharex(axes[0])\n        axes[1].sharey(axes[0])\n        axes[1].scatter(temp_df['BIA-BIA_BMC'][temp_df['Basic_Demos-Sex'] == sex], temp_df[name][temp_df['Basic_Demos-Sex'] == sex], color=colors[sex], alpha = 0.6, label=genders[sex])\n        axes[2].sharex(axes[0])\n        axes[2].sharey(axes[0])\n        axes[2].scatter(temp_df2['BIA-BIA_BMC'][temp_df2['Basic_Demos-Sex'] == sex], temp_df2[name][temp_df2['Basic_Demos-Sex'] == sex], color=colors[sex], alpha = 0.6, label=genders[sex])\n    for i in range(3):\n        axes[i].set_xlabel('BIA-BIA_BMC')\n        axes[i].set_ylabel(name)\n        axes[i].legend()\n    plt.show()\n# the odd values are missing in Physical-Waist_Circumference? ans: Physical-Waist_Circumference just has lots of missing data","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:11:00.896381Z","iopub.execute_input":"2024-10-29T09:11:00.896733Z","iopub.status.idle":"2024-10-29T09:12:08.404456Z","shell.execute_reply.started":"2024-10-29T09:11:00.896697Z","shell.execute_reply":"2024-10-29T09:12:08.403095Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import RANSACRegressor, LinearRegression\nangles = np.arctan2(female_bia_clean['BIA-BIA_TBW'], female_bia_clean['BIA-BIA_BMC'])\n\n# Separate based on the sign of the angle\nfemale_bia_clean['cluster'] = np.where((angles >= 1.52) | (angles <= 1.465), 1, 0)\n\n# Plot\nplt.scatter(female_bia_clean['BIA-BIA_TBW'], female_bia_clean['BIA-BIA_BMC'], c=female_bia_clean['cluster'], cmap='bwr')\nplt.xlabel('x')\nplt.ylabel('y')\nplt.title('Angle-Based Separation')\nplt.show()\nangles = np.arctan2(male_bia_clean['BIA-BIA_TBW'], male_bia_clean['BIA-BIA_BMC'])\nangles2 = np.arctan2(male_bia_clean['BIA-BIA_LST'], male_bia_clean['BIA-BIA_BMC'])\n# Separate based on the sign of the angle\nmale_bia_clean['cluster'] = np.where((angles >= 1.52) | (angles <= 1.465) | (angles2 >= 1.52) | (angles2 <= 1.495), 1, 0)\n\n# Plot\nplt.scatter(male_bia_clean['BIA-BIA_TBW'], male_bia_clean['BIA-BIA_BMC'], c=male_bia_clean['cluster'], cmap='bwr')\nplt.xlabel('x')\nplt.ylabel('y')\nplt.title('Angle-Based Separation')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T10:31:55.934264Z","iopub.execute_input":"2024-10-29T10:31:55.934707Z","iopub.status.idle":"2024-10-29T10:31:57.485859Z","shell.execute_reply.started":"2024-10-29T10:31:55.934667Z","shell.execute_reply":"2024-10-29T10:31:57.484041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colors = ['blue', 'red']\ngenders = ['Male', 'Female']\nnormal_df = train_bia_clean.drop(male_bia_clean[male_bia_clean['cluster'] == 1].index)\nnormal_df = normal_df.drop(female_bia_clean[female_bia_clean['cluster'] == 1].index)\n#temp_df = train_bia_clean[train_bia_clean['BIA-BIA_BMC'] < 0]\nfor name in train_bia_clean.columns:\n    if train_bia_clean[name].dtype == object or name == train_bia_clean.columns[-1] or name in Bio_electric_col_value_names:\n        continue\n    fig, axes = plt.subplots(1, 3, figsize=(20, 6))\n    for sex in range(2):\n        axes[0].scatter(normal_df['BIA-BIA_BMC'][normal_df['Basic_Demos-Sex'] == sex], normal_df[name][normal_df['Basic_Demos-Sex'] == sex], color=colors[sex], alpha = 0.6, label=genders[sex])\n        axes[1].sharex(axes[0])\n        axes[1].sharey(axes[0])\n        axes[1].scatter(male_bia_clean['BIA-BIA_BMC'][(male_bia_clean['Basic_Demos-Sex'] == sex) & (male_bia_clean['cluster'] == 1)], male_bia_clean[name][(male_bia_clean['Basic_Demos-Sex'] == sex) & (male_bia_clean['cluster'] == 1)], color=colors[sex], alpha = 0.6, label=genders[sex])\n        axes[2].sharex(axes[0])\n        axes[2].sharey(axes[0])\n        axes[2].scatter(female_bia_clean['BIA-BIA_BMC'][(female_bia_clean['Basic_Demos-Sex'] == sex) & (female_bia_clean['cluster'] == 1)], female_bia_clean[name][(female_bia_clean['Basic_Demos-Sex'] == sex) & (female_bia_clean['cluster'] == 1)], color=colors[sex], alpha = 0.6, label=genders[sex])\n    for i in range(3):\n        axes[i].set_xlabel('BIA-BIA_BMC')\n        axes[i].set_ylabel(name)\n        axes[i].legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T10:28:01.986179Z","iopub.execute_input":"2024-10-29T10:28:01.986803Z","iopub.status.idle":"2024-10-29T10:29:08.711129Z","shell.execute_reply.started":"2024-10-29T10:28:01.986746Z","shell.execute_reply":"2024-10-29T10:29:08.709988Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"value_names = [name for name in train_bia_clean.columns if (train_bia_clean[name].dtype != object and not name.startswith('PCIAT-PCIAT'))][:-1]","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:12:08.406280Z","iopub.execute_input":"2024-10-29T09:12:08.406798Z","iopub.status.idle":"2024-10-29T09:12:08.415315Z","shell.execute_reply.started":"2024-10-29T09:12:08.406741Z","shell.execute_reply":"2024-10-29T09:12:08.413910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (20, 20))\ncorrelation2 = female_bia_clean[value_names].corr()\nsns.heatmap(correlation2, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix of BIA Metrics Before removing odd BMC')\nplt.show()\nplt.figure(figsize = (20, 20))\ncorrelation1 = female_bia_clean[female_bia_clean['cluster'] == 0][value_names].corr()\nsns.heatmap(correlation1, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix of BIA Metrics After removing odd BMC')\nplt.show()\nplt.figure(figsize = (20, 20))\ncorrelation3 = correlation1 - correlation2\nsns.heatmap(correlation3, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix of BIA Metrics Diff')\nplt.show()\nplt.figure(figsize = (20, 20))\ncorrelation4 = female_bia_clean[female_bia_clean['cluster'] == 1][value_names].corr()\nsns.heatmap(correlation4, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix of BIA Metrics with only odd BMC')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:53:47.098305Z","iopub.execute_input":"2024-10-29T09:53:47.098874Z","iopub.status.idle":"2024-10-29T09:54:14.869776Z","shell.execute_reply.started":"2024-10-29T09:53:47.098813Z","shell.execute_reply":"2024-10-29T09:54:14.868016Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (20, 20))\ncorrelation2 = male_bia_clean[value_names].corr()\nsns.heatmap(correlation2, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix of BIA Metrics Before removing odd BMC')\nplt.show()\nplt.figure(figsize = (20, 20))\ncorrelation1 = male_bia_clean[male_bia_clean['cluster'] == 0][value_names].corr()\nsns.heatmap(correlation1, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix of BIA Metrics After removing odd BMC')\nplt.show()\nplt.figure(figsize = (20, 20))\ncorrelation3 = correlation1 - correlation2\nsns.heatmap(correlation3, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix of BIA Metrics Diff')\nplt.show()\nplt.figure(figsize = (20, 20))\ncorrelation4 = male_bia_clean[male_bia_clean['cluster'] == 1][value_names].corr()\nsns.heatmap(correlation4, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix of BIA Metrics with only odd BMC')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:54:14.871928Z","iopub.execute_input":"2024-10-29T09:54:14.872754Z","iopub.status.idle":"2024-10-29T09:54:42.685784Z","shell.execute_reply.started":"2024-10-29T09:54:14.872700Z","shell.execute_reply":"2024-10-29T09:54:42.684329Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (10, 10))\ncorrelation1 = female_bia_clean.drop(temp_df.index)[Bio_electric_col_value_names.to_list() + ['sii']+['Basic_Demos-Age']].corr()\nsns.heatmap(correlation1, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix of BIA Metrics')\nplt.show()\nplt.figure(figsize = (10, 10))\ncorrelation2 = female_bia_clean[Bio_electric_col_value_names.to_list() + ['sii']+['Basic_Demos-Age']].corr()\nsns.heatmap(correlation2, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix of BIA Metrics')\nplt.show()\nplt.figure(figsize = (10, 10))\ncorrelation3 = correlation1 - correlation2\nsns.heatmap(correlation3, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix of BIA Metrics')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:12:35.604617Z","iopub.execute_input":"2024-10-29T09:12:35.605096Z","iopub.status.idle":"2024-10-29T09:12:40.244635Z","shell.execute_reply.started":"2024-10-29T09:12:35.605045Z","shell.execute_reply":"2024-10-29T09:12:40.243231Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colors = ['blue', 'red']\ngenders = ['Male', 'Female']\ntemp_df = train_bia_clean[train_bia_clean['BIA-BIA_Fat'] < 0]\nfor name in Bio_electric_col_value_names:\n    fig, axes = plt.subplots(1, 2, figsize=(14, 6))\n    for sex in range(2):\n        axes[0].scatter(train_bia_clean['BIA-BIA_Fat'][train_bia_clean['Basic_Demos-Sex'] == sex], train_bia_clean[name][train_bia_clean['Basic_Demos-Sex'] == sex], color=colors[sex], alpha = 0.6, label=genders[sex])\n        axes[1].sharex(axes[0])\n        axes[1].sharey(axes[0])\n        axes[1].scatter(temp_df['BIA-BIA_Fat'][temp_df['Basic_Demos-Sex'] == sex], temp_df[name][temp_df['Basic_Demos-Sex'] == sex], color=colors[sex], alpha = 0.6, label=genders[sex])\n    for i in range(2):\n        axes[i].set_xlabel('BIA-BIA_Fat')\n        axes[i].set_ylabel(name)\n        axes[i].legend()\n    plt.show()\n# it seems that the distribution where Fat<0 doesn't follow the supposes values anyways, we should simply delete them","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:12:40.246216Z","iopub.execute_input":"2024-10-29T09:12:40.246620Z","iopub.status.idle":"2024-10-29T09:12:51.091434Z","shell.execute_reply.started":"2024-10-29T09:12:40.246573Z","shell.execute_reply":"2024-10-29T09:12:51.090090Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"value_names = [name for name in train_bia_clean.columns if (train_bia_clean[name].dtype != object and not name.startswith('PCIAT-PCIAT'))][:-1]","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:17:54.106754Z","iopub.execute_input":"2024-10-29T11:17:54.107514Z","iopub.status.idle":"2024-10-29T11:17:54.115873Z","shell.execute_reply.started":"2024-10-29T11:17:54.107455Z","shell.execute_reply":"2024-10-29T11:17:54.114395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = train_bia_clean[value_names].fillna(train_bia_clean[value_names].median())","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:17:55.052953Z","iopub.execute_input":"2024-10-29T11:17:55.053447Z","iopub.status.idle":"2024-10-29T11:17:55.087626Z","shell.execute_reply.started":"2024-10-29T11:17:55.053402Z","shell.execute_reply":"2024-10-29T11:17:55.086353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nprint(len(value_names))\npca = PCA(n_components=2)\nreduced_data = pca.fit_transform(temp)\nprint(pca.explained_variance_ratio_)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:20:24.911536Z","iopub.execute_input":"2024-10-29T11:20:24.912066Z","iopub.status.idle":"2024-10-29T11:20:25.064902Z","shell.execute_reply.started":"2024-10-29T11:20:24.912019Z","shell.execute_reply":"2024-10-29T11:20:25.063385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name in Bio_electric_col_value_names:\n    avg_score = male_bia_clean.groupby('Basic_Demos-Age')[name].mean()\n    avg_score.plot(kind='bar')\n    plt.xlabel('age')\n    plt.ylabel(name)\n    plt.title(f'age-{name}')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T09:12:51.099814Z","iopub.execute_input":"2024-10-29T09:12:51.100272Z","iopub.status.idle":"2024-10-29T09:12:57.341598Z","shell.execute_reply.started":"2024-10-29T09:12:51.100221Z","shell.execute_reply":"2024-10-29T09:12:57.340115Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]}]}