{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:04.851767Z","iopub.execute_input":"2024-10-25T17:06:04.852245Z","iopub.status.idle":"2024-10-25T17:06:04.858590Z","shell.execute_reply.started":"2024-10-25T17:06:04.852201Z","shell.execute_reply":"2024-10-25T17:06:04.857117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### CONFIG","metadata":{}},{"cell_type":"code","source":"class Config:\n#     DATA_PATH = \".data/\"\n    DATA_PATH = \"/kaggle/input/child-mind-institute-problematic-internet-use/\"\n    TRAIN_PARQUET_PATH = \"series_train.parquet/\"\n    TEST_PARQUET_PATH = \"series_test.parquet/\"","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:05.638439Z","iopub.execute_input":"2024-10-25T17:06:05.638859Z","iopub.status.idle":"2024-10-25T17:06:05.644461Z","shell.execute_reply.started":"2024-10-25T17:06:05.638820Z","shell.execute_reply":"2024-10-25T17:06:05.643323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Helpers","metadata":{}},{"cell_type":"code","source":"class ParquetManager:\n    def __init__(self, config: Config, dataset_type:str ='train'):\n        self.config = config\n        self.dataset_type = dataset_type\n\n    def load_parquet(self, id:str)->pd.DataFrame:\n        parquet_path = self.config.TRAIN_PARQUET_PATH if self.dataset_type else self.config.TEST_PARQUET_PATH\n        \n        try:\n            return pd.read_parquet(self.config.DATA_PATH+parquet_path+f\"id={id}/\")\n        except Exception as E:\n            print(\"error loading parquet\")\n            raise E\n        \n    def plot_acceleration(self,df:pd.DataFrame):\n        plt.plot(df['step'],df['X'])\n        plt.plot(df['step'],df['Y'])\n        plt.plot(df['step'],df['Z'])","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:05.982346Z","iopub.execute_input":"2024-10-25T17:06:05.982798Z","iopub.status.idle":"2024-10-25T17:06:05.992436Z","shell.execute_reply.started":"2024-10-25T17:06:05.982756Z","shell.execute_reply":"2024-10-25T17:06:05.991102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating a heatmap\ndef plot_heatmap_sii(df_under_study, COL, DIM):\n    \n    # Creating a heatmap\n    # plt.figure(figsize=(7,2))\n    pre_heat_map = df_under_study[[COL,'sii']].groupby([COL,'sii']).size().reset_index(name='count')\n    heatmap = np.zeros(shape=(4,DIM))\n    for i, item in pre_heat_map.iterrows():\n        den = sii_to_max[int(item['sii'])]\n        heatmap[item['sii'].astype('int'), item[COL].astype('int')] = item['count']/den\n    heatmap = plt.pcolor(heatmap, cmap='jet') \n    plt.colorbar(heatmap)\n    plt.title(\"AGE/SII distribution\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:06.148771Z","iopub.execute_input":"2024-10-25T17:06:06.149227Z","iopub.status.idle":"2024-10-25T17:06:06.158392Z","shell.execute_reply.started":"2024-10-25T17:06:06.149186Z","shell.execute_reply":"2024-10-25T17:06:06.156804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA\nEDA goals\n1. Find out the impact of missing values on the dataset\n2. Try to find out correlations between other variables and the target\n3. Try to create new features from series data and find correlations between the target and the created features","metadata":{}},{"cell_type":"markdown","source":"## Categorizing Features","metadata":{}},{"cell_type":"code","source":"COLS_DEMOS = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex']\nCOLS_PHYSICAL = ['Physical-Season', 'Physical-BMI','Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference','Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP']\nCOLS_SEASON = ['CGAS-Season', 'CGAS-CGAS_Score']\n# FitnessGram vitals\nCOLS_FITNESS = ['Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage','Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec']\n# FitnessGram Child\nCOLS_FGC = ['FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND','FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU','FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR','FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone']\n# Bio electric impedence\nCOLS_BIA = ['BIA-Season','BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI','BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM','BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num','BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM','BIA-BIA_TBW', ]\n# Physical Activity Questionnaire\nCOLS_PAQ = ['PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season','PAQ_C-PAQ_C_Total']\nCOLS_PCIAT = ['PCIAT-Season', 'PCIAT-PCIAT_01', 'PCIAT-PCIAT_02','PCIAT-PCIAT_03', 'PCIAT-PCIAT_04', 'PCIAT-PCIAT_05', 'PCIAT-PCIAT_06','PCIAT-PCIAT_07', 'PCIAT-PCIAT_08', 'PCIAT-PCIAT_09', 'PCIAT-PCIAT_10','PCIAT-PCIAT_11', 'PCIAT-PCIAT_12', 'PCIAT-PCIAT_13', 'PCIAT-PCIAT_14','PCIAT-PCIAT_15', 'PCIAT-PCIAT_16', 'PCIAT-PCIAT_17', 'PCIAT-PCIAT_18','PCIAT-PCIAT_19', 'PCIAT-PCIAT_20', 'PCIAT-PCIAT_Total']\n# Sleep disturbance test\nCOLS_SDS = ['SDS-Season','SDS-SDS_Total_Raw', 'SDS-SDS_Total_T']\n# INT\nCOLS_INT = ['PreInt_EduHx-Season','PreInt_EduHx-computerinternet_hoursday']","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:06.708777Z","iopub.execute_input":"2024-10-25T17:06:06.709225Z","iopub.status.idle":"2024-10-25T17:06:06.719938Z","shell.execute_reply.started":"2024-10-25T17:06:06.709184Z","shell.execute_reply":"2024-10-25T17:06:06.718704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Missing sii","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(Config.DATA_PATH+\"train.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:07.094691Z","iopub.execute_input":"2024-10-25T17:06:07.095135Z","iopub.status.idle":"2024-10-25T17:06:07.158539Z","shell.execute_reply.started":"2024-10-25T17:06:07.095091Z","shell.execute_reply":"2024-10-25T17:06:07.157008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sii_hist = df.groupby(['sii'], dropna=False).count()[['id']].rename(columns={'id':'count'}).reset_index()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:07.287415Z","iopub.execute_input":"2024-10-25T17:06:07.287933Z","iopub.status.idle":"2024-10-25T17:06:07.308435Z","shell.execute_reply.started":"2024-10-25T17:06:07.287877Z","shell.execute_reply":"2024-10-25T17:06:07.307157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sii_hist","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:07.501609Z","iopub.execute_input":"2024-10-25T17:06:07.502079Z","iopub.status.idle":"2024-10-25T17:06:07.514282Z","shell.execute_reply.started":"2024-10-25T17:06:07.502036Z","shell.execute_reply":"2024-10-25T17:06:07.513125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.bar(sii_hist['sii'].astype(str), sii_hist['count'].values)","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:07.954592Z","iopub.execute_input":"2024-10-25T17:06:07.955067Z","iopub.status.idle":"2024-10-25T17:06:08.397038Z","shell.execute_reply.started":"2024-10-25T17:06:07.955014Z","shell.execute_reply":"2024-10-25T17:06:08.395727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"SII availability\")","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:08.532173Z","iopub.execute_input":"2024-10-25T17:06:08.532775Z","iopub.status.idle":"2024-10-25T17:06:08.539482Z","shell.execute_reply.started":"2024-10-25T17:06:08.532722Z","shell.execute_reply":"2024-10-25T17:06:08.537928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Conclusion\nWe're missing roughly 31% of the data on sii, the distribution is skewed towards 0, with the least people having severe impairement. Those are the most important ones to predict. Let's see what other variables correlate with this target, I also need to see what PCIAT targets are the most useful. ","metadata":{}},{"cell_type":"markdown","source":"# How different features correlate with PCIAT/SII\n\nIt is to be noted that we won't be getting PCIAT/SII in the test data and we might not get other features at all as well (all might not be available), for our purposes we'll be filtering out nulls before moving on. Since it is said that SII is derived from PCIAT itself\n<a href=\"https://www.kaggle.com/competitions/child-mind-institute-problematic-internet-use/data#:~:text=Note%20in%20particular%20the%20field%20PCIAT%2DPCIAT_Total.%20The%20target%20sii%20for%20this%20competition%20is%20derived%20from%20this%20field%20as%20described%20in%20the%20data%20dictionary%3A%200%20for%20None%2C%201%20for%20Mild%2C%202%20for%20Moderate%2C%20and%203%20for%20Severe.%20Additionally%2C%20each%20participant%20has%20been%20assigned%20a%20unique%20identifier%20id.\" target=\"_blank\">REF</a>, I'm focusing my analysis on the target variable SII for now","metadata":{}},{"cell_type":"markdown","source":"## 1. Impact of Demographics","metadata":{}},{"cell_type":"markdown","source":"### Age Distribution","metadata":{}},{"cell_type":"code","source":"COLS_DEMOS","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:09.674542Z","iopub.execute_input":"2024-10-25T17:06:09.675719Z","iopub.status.idle":"2024-10-25T17:06:09.683518Z","shell.execute_reply.started":"2024-10-25T17:06:09.675653Z","shell.execute_reply":"2024-10-25T17:06:09.682237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_under_study = df[['Basic_Demos-Sex','Basic_Demos-Age', 'sii']].dropna()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:10.010468Z","iopub.execute_input":"2024-10-25T17:06:10.011642Z","iopub.status.idle":"2024-10-25T17:06:10.020213Z","shell.execute_reply.started":"2024-10-25T17:06:10.011577Z","shell.execute_reply":"2024-10-25T17:06:10.018856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(df_under_study['Basic_Demos-Age'], bins=22-4)\nplt.title(\"Age Distribution\")\nNone","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:10.231719Z","iopub.execute_input":"2024-10-25T17:06:10.232169Z","iopub.status.idle":"2024-10-25T17:06:10.754265Z","shell.execute_reply.started":"2024-10-25T17:06:10.232128Z","shell.execute_reply":"2024-10-25T17:06:10.753016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(df_under_study[df_under_study['Basic_Demos-Sex']==0.0]['Basic_Demos-Age'], bins=22-4)\nplt.title(\"Age Distribution Male\")\nNone","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:10.756699Z","iopub.execute_input":"2024-10-25T17:06:10.757190Z","iopub.status.idle":"2024-10-25T17:06:11.221432Z","shell.execute_reply.started":"2024-10-25T17:06:10.757137Z","shell.execute_reply":"2024-10-25T17:06:11.220277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(df_under_study[df_under_study['Basic_Demos-Sex']==1.0]['Basic_Demos-Age'], bins=22-4)\nplt.title(\"Age Distribution Female\")\nNone","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:11.223455Z","iopub.execute_input":"2024-10-25T17:06:11.223981Z","iopub.status.idle":"2024-10-25T17:06:11.734550Z","shell.execute_reply.started":"2024-10-25T17:06:11.223933Z","shell.execute_reply":"2024-10-25T17:06:11.733321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(df_under_study[df_under_study['sii']==3.0]['Basic_Demos-Age'])\nplt.title(\"Age Distribution for the severely impacted\")\nprint(\"min age for sii=3 :\",df_under_study[df_under_study['sii']==3.0]['Basic_Demos-Age'].min())","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:11.736746Z","iopub.execute_input":"2024-10-25T17:06:11.737135Z","iopub.status.idle":"2024-10-25T17:06:12.146386Z","shell.execute_reply.started":"2024-10-25T17:06:11.737093Z","shell.execute_reply":"2024-10-25T17:06:12.144737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating a heatmap\nplt.figure(figsize=(7,2))\npre_heat_map = df_under_study[['Basic_Demos-Age','sii']].groupby(['Basic_Demos-Age','sii']).size().reset_index(name='count')\nsii_to_max = [pre_heat_map[pre_heat_map['sii']==0]['count'].max(),\npre_heat_map[pre_heat_map['sii']==1]['count'].max(),\npre_heat_map[pre_heat_map['sii']==2]['count'].max(),\npre_heat_map[pre_heat_map['sii']==3]['count'].max()\n]\n\nheatmap = np.zeros(shape=(4,22-5+1))\nfor i, item in pre_heat_map.iterrows():\n    den = sii_to_max[int(item['sii'])]\n    heatmap[item['sii'].astype('int'), item['Basic_Demos-Age'].astype('int')-5] = item['count']/den\n# plt.imshow(heatmap, cmap='jet')\nheatmap = plt.pcolor(heatmap, cmap='jet') \nplt.colorbar(heatmap)\nplt.xticks(np.arange(18),np.arange(18)+5)\nplt.title(\"AGE/SII distribution\")\nNone","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:12.499835Z","iopub.execute_input":"2024-10-25T17:06:12.500316Z","iopub.status.idle":"2024-10-25T17:06:13.151084Z","shell.execute_reply.started":"2024-10-25T17:06:12.500270Z","shell.execute_reply":"2024-10-25T17:06:13.149823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.kdeplot(df_under_study['Basic_Demos-Age'], label='Distribution by age', fill=True, color='black')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:13.153367Z","iopub.execute_input":"2024-10-25T17:06:13.153877Z","iopub.status.idle":"2024-10-25T17:06:13.626338Z","shell.execute_reply.started":"2024-10-25T17:06:13.153824Z","shell.execute_reply":"2024-10-25T17:06:13.625157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correlation of Age with Impairement","metadata":{}},{"cell_type":"code","source":"df_under_study.corr()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:14.314672Z","iopub.execute_input":"2024-10-25T17:06:14.315094Z","iopub.status.idle":"2024-10-25T17:06:14.329186Z","shell.execute_reply.started":"2024-10-25T17:06:14.315053Z","shell.execute_reply":"2024-10-25T17:06:14.327727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.kdeplot(df_under_study[df_under_study['sii'] == 0]['Basic_Demos-Age'], label='sii=0', fill=True, bw_adjust=1.0)\nsns.kdeplot(df_under_study[df_under_study['sii'] == 1]['Basic_Demos-Age'], label='sii=1', fill=True, bw_adjust=1.0)\nsns.kdeplot(df_under_study[df_under_study['sii'] == 2]['Basic_Demos-Age'], label='sii=2', fill=True, bw_adjust=1.0)\nsns.kdeplot(df_under_study[df_under_study['sii'] == 3]['Basic_Demos-Age'], label='sii=3', fill=True, bw_adjust=1.0)\n\n\nplt.ylabel(\"Density\")\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:14.750162Z","iopub.execute_input":"2024-10-25T17:06:14.750588Z","iopub.status.idle":"2024-10-25T17:06:15.258250Z","shell.execute_reply.started":"2024-10-25T17:06:14.750549Z","shell.execute_reply":"2024-10-25T17:06:15.256814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Interesting but we shouldn't get too excited\nAs we saw before, age seems to correlate slightly positively with impairement severity. The data for severe impairement is way too less in number for making serious conclusions. It's also possible that either clinicians don't diagnose smaller children with high impairement or teens and adolescents really are at a high risk for problematic internet use.","metadata":{}},{"cell_type":"markdown","source":"### Does gender impact SII?\n// 0 = Male, 1 = Female","metadata":{}},{"cell_type":"code","source":"df_under_study['Basic_Demos-Sex'].hist()\ndf_under_study.groupby(['Basic_Demos-Sex']).size().reset_index(name='count')","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:16.941030Z","iopub.execute_input":"2024-10-25T17:06:16.941448Z","iopub.status.idle":"2024-10-25T17:06:17.366195Z","shell.execute_reply.started":"2024-10-25T17:06:16.941409Z","shell.execute_reply":"2024-10-25T17:06:17.365171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Interesting\nAppears as though there are almost nearly twice as many boys data in the train set when compared to the girls.","metadata":{}},{"cell_type":"code","source":"df_under_study[df_under_study['Basic_Demos-Sex'] == 0]['sii'].hist(\n    bins=20, \n    alpha=0.5, \n    density=True,  # Normalize the histogram\n    color='blue', \n    label='Males (Sex = 0)', \n    edgecolor='black'\n)\n\n# Plot normalized histogram for females\ndf_under_study[df_under_study['Basic_Demos-Sex'] == 1]['sii'].hist(\n    bins=20, \n    alpha=0.5, \n    density=True,  # Normalize the histogram\n    color='red', \n    label='Females (Sex = 1)', \n    edgecolor='black'\n)\nplt.title(\"Normalized Histogram of SII by Gender\")\nplt.xlabel(\"SII Values\")\nplt.ylabel(\"Density\")\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:18.792876Z","iopub.execute_input":"2024-10-25T17:06:18.793645Z","iopub.status.idle":"2024-10-25T17:06:19.433613Z","shell.execute_reply.started":"2024-10-25T17:06:18.793584Z","shell.execute_reply":"2024-10-25T17:06:19.432488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Non significant Correlation\nWe saw that the correlation value we got for sex was around -0.10, it's not enough to make a conclusion, but in impairement scores, there's more boys distributed on the impairement space than girls (normalized), one could argue that the data for girls is half that of the boys in this case and maybe that's the reason for the irregularity or one could also argue that development changes might impact the problematic internet use differently among boys and girls (let's find out next).","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 2, figsize=(12, 10))  \naxes = axes.flatten()\nfor sii in range(4):\n\n    sns.kdeplot(df_under_study[(df_under_study['Basic_Demos-Sex'] == 0) & (df_under_study['sii'] == sii)]['Basic_Demos-Age'], label=f'MALE, sii={sii}', fill=True, bw_adjust=1.0, ax=axes[sii])\n    sns.kdeplot(df_under_study[(df_under_study['Basic_Demos-Sex'] == 1) & (df_under_study['sii'] == sii)]['Basic_Demos-Age'], label=f'FEMALE, sii={sii}', fill=True, bw_adjust=1.0, ax=axes[sii])\n\n\n    axes[sii].set_title(\"Age-Gender-Sii Distribution (density)\")\n    axes[sii].set_xlabel(\"Age\")\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:20.968787Z","iopub.execute_input":"2024-10-25T17:06:20.970145Z","iopub.status.idle":"2024-10-25T17:06:22.745998Z","shell.execute_reply.started":"2024-10-25T17:06:20.970089Z","shell.execute_reply":"2024-10-25T17:06:22.744596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import entropy","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:30.178551Z","iopub.execute_input":"2024-10-25T17:06:30.178992Z","iopub.status.idle":"2024-10-25T17:06:30.184731Z","shell.execute_reply.started":"2024-10-25T17:06:30.178952Z","shell.execute_reply":"2024-10-25T17:06:30.183376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Non-Conclusive?\nThere appears to be more divergence in the distribution as we increase the severity impairement index.\nTODO: Calculate statistical divergence","metadata":{}},{"cell_type":"markdown","source":"#### Divergence","metadata":{}},{"cell_type":"code","source":"divergences = []\neta = 1e-9\nfor sii in range(4):\n    \n    male_ages = df_under_study[(df_under_study['Basic_Demos-Sex'] == 0) & (df_under_study['sii'] == sii)]['Basic_Demos-Age'].values\n    female_ages = df_under_study[(df_under_study['Basic_Demos-Sex'] == 1) & (df_under_study['sii'] == sii)]['Basic_Demos-Age'].values\n# P.S : to the reader, I know this is suboptimal, to ensure I don't get inf in entropy (log(0)->inf) I set the bins way too low... i could replace those with a smaller number but that's at a risk of incorrectness\n# ALSO: I'm using JS divergence to eliminate the non-symmetric property of KL divergence, im not sure if its a good idea and that we shouldve analyzed those separately.\n    bins = 5\n    male_hist, _ = np.histogram(male_ages, bins=bins, density=False)\n    female_hist, _ = np.histogram(female_ages, bins=bins, density=False)\n\n    male_prob_dist = male_hist / male_hist.sum()\n    female_prob_dist = female_hist / female_hist.sum()\n\n    # male_prob_dist = np.where(male_prob_dist == 0, eta, male_prob_dist)\n    # female_prob_dist = np.where(female_prob_dist == 0, eta, female_prob_dist)\n\n    js_div = (entropy(male_prob_dist, female_prob_dist) + entropy(female_prob_dist,male_prob_dist))/2\n    divergences.append(js_div)","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:34.714965Z","iopub.execute_input":"2024-10-25T17:06:34.715447Z","iopub.status.idle":"2024-10-25T17:06:34.745070Z","shell.execute_reply.started":"2024-10-25T17:06:34.715384Z","shell.execute_reply":"2024-10-25T17:06:34.743967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.bar(range(4), divergences)\nplt.xlabel(\"SII\")\nplt.ylabel(\"JS-Divergence\")\nplt.title(\"Divergence between female and male age distributions against Severity of Impairement\")","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:36.794176Z","iopub.execute_input":"2024-10-25T17:06:36.794667Z","iopub.status.idle":"2024-10-25T17:06:37.307325Z","shell.execute_reply.started":"2024-10-25T17:06:36.794583Z","shell.execute_reply":"2024-10-25T17:06:37.306184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. FitnessGram Child and FitnessGram Vitals","metadata":{}},{"cell_type":"markdown","source":"## FitnessGram Vitals","metadata":{}},{"cell_type":"code","source":"df_fitness_vitals = df[[\n 'Fitness_Endurance-Max_Stage',\n 'Fitness_Endurance-Time_Mins',\n 'Fitness_Endurance-Time_Sec', 'sii']].dropna()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:40.667171Z","iopub.execute_input":"2024-10-25T17:06:40.667613Z","iopub.status.idle":"2024-10-25T17:06:40.676415Z","shell.execute_reply.started":"2024-10-25T17:06:40.667570Z","shell.execute_reply":"2024-10-25T17:06:40.675119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COLS_FITNESS","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:41.271299Z","iopub.execute_input":"2024-10-25T17:06:41.271789Z","iopub.status.idle":"2024-10-25T17:06:41.279839Z","shell.execute_reply.started":"2024-10-25T17:06:41.271746Z","shell.execute_reply":"2024-10-25T17:06:41.278532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_fitness_vitals.corr()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:41.815852Z","iopub.execute_input":"2024-10-25T17:06:41.816853Z","iopub.status.idle":"2024-10-25T17:06:41.831258Z","shell.execute_reply.started":"2024-10-25T17:06:41.816804Z","shell.execute_reply":"2024-10-25T17:06:41.830109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Endurance X sii\nAppears to be the case that exercise has negative correlation with sii (meaning less impairement)","metadata":{}},{"cell_type":"code","source":"sns.kdeplot(df_fitness_vitals['Fitness_Endurance-Max_Stage'], label='Distribution by Endurance Stage', fill=True, color='black')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:44.149790Z","iopub.execute_input":"2024-10-25T17:06:44.150290Z","iopub.status.idle":"2024-10-25T17:06:44.651078Z","shell.execute_reply.started":"2024-10-25T17:06:44.150244Z","shell.execute_reply":"2024-10-25T17:06:44.649816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,2))\nplot_heatmap_sii(df_fitness_vitals, 'Fitness_Endurance-Max_Stage',29)\nplt.xlabel(\"Endurance Max Level\")\nplt.ylabel(\"SII level\")","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:44.784555Z","iopub.execute_input":"2024-10-25T17:06:44.785638Z","iopub.status.idle":"2024-10-25T17:06:45.477775Z","shell.execute_reply.started":"2024-10-25T17:06:44.785570Z","shell.execute_reply":"2024-10-25T17:06:45.476459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_fitness_vitals[df_fitness_vitals['Fitness_Endurance-Max_Stage']<=5]['sii'].mean(),\\\ndf_fitness_vitals[df_fitness_vitals['Fitness_Endurance-Max_Stage']>=5]['sii'].mean()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:46.289399Z","iopub.execute_input":"2024-10-25T17:06:46.290431Z","iopub.status.idle":"2024-10-25T17:06:46.300310Z","shell.execute_reply.started":"2024-10-25T17:06:46.290383Z","shell.execute_reply":"2024-10-25T17:06:46.299110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## FitnessGram Child","metadata":{}},{"cell_type":"code","source":"df_fitness_child = df[[*COLS_FGC, 'sii']].dropna()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:48.635845Z","iopub.execute_input":"2024-10-25T17:06:48.636323Z","iopub.status.idle":"2024-10-25T17:06:48.650025Z","shell.execute_reply.started":"2024-10-25T17:06:48.636275Z","shell.execute_reply":"2024-10-25T17:06:48.648595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_fgc_corr = df_fitness_child.drop(columns=['FGC-Season']).corr()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:49.163686Z","iopub.execute_input":"2024-10-25T17:06:49.164582Z","iopub.status.idle":"2024-10-25T17:06:49.172032Z","shell.execute_reply.started":"2024-10-25T17:06:49.164537Z","shell.execute_reply":"2024-10-25T17:06:49.170640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_fgc_corr[df_fgc_corr['sii']>0.10]['sii'].reset_index().rename(columns={'index':\"fgc_feature\", 'sii':\"sii_corr\"})","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:49.634094Z","iopub.execute_input":"2024-10-25T17:06:49.635917Z","iopub.status.idle":"2024-10-25T17:06:49.653211Z","shell.execute_reply.started":"2024-10-25T17:06:49.635857Z","shell.execute_reply":"2024-10-25T17:06:49.651386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_fitness_child[['FGC-FGC_CU']].max()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:50.001771Z","iopub.execute_input":"2024-10-25T17:06:50.002257Z","iopub.status.idle":"2024-10-25T17:06:50.014184Z","shell.execute_reply.started":"2024-10-25T17:06:50.002213Z","shell.execute_reply":"2024-10-25T17:06:50.012720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.kdeplot(df_fitness_child[['FGC-FGC_CU']], label='Curlups distribution', fill=True, color='black')\nplt.legend()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:50.675133Z","iopub.execute_input":"2024-10-25T17:06:50.675553Z","iopub.status.idle":"2024-10-25T17:06:51.173424Z","shell.execute_reply.started":"2024-10-25T17:06:50.675514Z","shell.execute_reply":"2024-10-25T17:06:51.172069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(24,2))\nplot_heatmap_sii(df_fitness_child, 'FGC-FGC_CU',101)\nplt.xlabel(\"N. Curl ups\")\nplt.ylabel(\"SII level\")","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:56.726896Z","iopub.execute_input":"2024-10-25T17:06:56.727871Z","iopub.status.idle":"2024-10-25T17:06:57.274281Z","shell.execute_reply.started":"2024-10-25T17:06:56.727825Z","shell.execute_reply":"2024-10-25T17:06:57.273115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. BIA (Bio Signals)\nLet's explore a few BIA features","metadata":{}},{"cell_type":"code","source":"df_bia = df[[*COLS_BIA,'sii']].drop(columns='BIA-Season').dropna()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:59.026730Z","iopub.execute_input":"2024-10-25T17:06:59.027197Z","iopub.status.idle":"2024-10-25T17:06:59.038251Z","shell.execute_reply.started":"2024-10-25T17:06:59.027151Z","shell.execute_reply":"2024-10-25T17:06:59.036989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_bia['BIA-BIA_BMI_INT'] = df_bia['BIA-BIA_BMI'].astype('int')","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:06:59.656976Z","iopub.execute_input":"2024-10-25T17:06:59.657509Z","iopub.status.idle":"2024-10-25T17:06:59.666291Z","shell.execute_reply.started":"2024-10-25T17:06:59.657462Z","shell.execute_reply":"2024-10-25T17:06:59.664916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_bia['BIA-BIA_BMI_INT'].max()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:07:00.039274Z","iopub.execute_input":"2024-10-25T17:07:00.039798Z","iopub.status.idle":"2024-10-25T17:07:00.049016Z","shell.execute_reply.started":"2024-10-25T17:07:00.039751Z","shell.execute_reply":"2024-10-25T17:07:00.047730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.kdeplot(df_bia[['BIA-BIA_BMI']], label='BMI distribution', fill=True, color='black')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:07:00.834143Z","iopub.execute_input":"2024-10-25T17:07:00.834572Z","iopub.status.idle":"2024-10-25T17:07:01.280430Z","shell.execute_reply.started":"2024-10-25T17:07:00.834533Z","shell.execute_reply":"2024-10-25T17:07:01.279128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,2))\nplot_heatmap_sii(df_bia, 'BIA-BIA_BMI_INT',53)\nplt.xlabel(\"BMI\")\nplt.ylabel(\"SII level\")","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:07:01.607411Z","iopub.execute_input":"2024-10-25T17:07:01.607862Z","iopub.status.idle":"2024-10-25T17:07:02.127138Z","shell.execute_reply.started":"2024-10-25T17:07:01.607820Z","shell.execute_reply":"2024-10-25T17:07:02.125831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Sleep Disturbance","metadata":{}},{"cell_type":"code","source":"df_sds = df[[*COLS_SDS,'sii']].drop(columns='SDS-Season').dropna()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:07:03.181975Z","iopub.execute_input":"2024-10-25T17:07:03.182451Z","iopub.status.idle":"2024-10-25T17:07:03.192967Z","shell.execute_reply.started":"2024-10-25T17:07:03.182404Z","shell.execute_reply":"2024-10-25T17:07:03.191742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sds['SDS-SDS_Total_T'].max()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:07:03.649007Z","iopub.execute_input":"2024-10-25T17:07:03.649502Z","iopub.status.idle":"2024-10-25T17:07:03.658568Z","shell.execute_reply.started":"2024-10-25T17:07:03.649460Z","shell.execute_reply":"2024-10-25T17:07:03.656833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.kdeplot(df_sds[['SDS-SDS_Total_T']], label='SDS Score total', fill=True, color='black')\n# higher is high sleep disturbance\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:07:04.587388Z","iopub.execute_input":"2024-10-25T17:07:04.587847Z","iopub.status.idle":"2024-10-25T17:07:05.156410Z","shell.execute_reply.started":"2024-10-25T17:07:04.587804Z","shell.execute_reply":"2024-10-25T17:07:05.155189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,2))\nplot_heatmap_sii(df_sds, 'SDS-SDS_Total_T',101)\nplt.xlabel(\"SDS\")\nplt.ylabel(\"SII level\")","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:07:05.396561Z","iopub.execute_input":"2024-10-25T17:07:05.397537Z","iopub.status.idle":"2024-10-25T17:07:05.957319Z","shell.execute_reply.started":"2024-10-25T17:07:05.397492Z","shell.execute_reply":"2024-10-25T17:07:05.956154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A case can be made when looking at this chart (while keeping the distribution in mind) that higher sleep disturbance scale score is correlating with high impairement level at least to some extent","metadata":{}},{"cell_type":"code","source":"df_sds.corr()\n# Correlation also shows us something similar","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:07:06.824111Z","iopub.execute_input":"2024-10-25T17:07:06.824508Z","iopub.status.idle":"2024-10-25T17:07:06.837694Z","shell.execute_reply.started":"2024-10-25T17:07:06.824470Z","shell.execute_reply":"2024-10-25T17:07:06.836439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Internet Usage","metadata":{}},{"cell_type":"code","source":"df_int = df[[*COLS_INT,'sii']].drop(columns='PreInt_EduHx-Season').dropna()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:07:08.967410Z","iopub.execute_input":"2024-10-25T17:07:08.967886Z","iopub.status.idle":"2024-10-25T17:07:08.978770Z","shell.execute_reply.started":"2024-10-25T17:07:08.967841Z","shell.execute_reply":"2024-10-25T17:07:08.976971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_int['PreInt_EduHx-computerinternet_hoursday'].max()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:07:09.532324Z","iopub.execute_input":"2024-10-25T17:07:09.532798Z","iopub.status.idle":"2024-10-25T17:07:09.542124Z","shell.execute_reply.started":"2024-10-25T17:07:09.532753Z","shell.execute_reply":"2024-10-25T17:07:09.540740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.kdeplot(df_int[['PreInt_EduHx-computerinternet_hoursday']], label='SDS Score total', fill=True, color='black')\n# higher is high sleep disturbance\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:07:10.086878Z","iopub.execute_input":"2024-10-25T17:07:10.087335Z","iopub.status.idle":"2024-10-25T17:07:10.578428Z","shell.execute_reply.started":"2024-10-25T17:07:10.087293Z","shell.execute_reply":"2024-10-25T17:07:10.577136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_int.min()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:07:11.142298Z","iopub.execute_input":"2024-10-25T17:07:11.142814Z","iopub.status.idle":"2024-10-25T17:07:11.154916Z","shell.execute_reply.started":"2024-10-25T17:07:11.142767Z","shell.execute_reply":"2024-10-25T17:07:11.153046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,2))\nplot_heatmap_sii(df_int, 'PreInt_EduHx-computerinternet_hoursday',4)\nplt.xlabel(\"Internet Usage\")\nplt.ylabel(\"SII level\")","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:07:12.018756Z","iopub.execute_input":"2024-10-25T17:07:12.019221Z","iopub.status.idle":"2024-10-25T17:07:12.585373Z","shell.execute_reply.started":"2024-10-25T17:07:12.019177Z","shell.execute_reply":"2024-10-25T17:07:12.584016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_int.corr()\n# Though we couldn't make a conclusion from the heatmap, I think we can say that there is a somewhat weak positive correlation of Internet usage with SII (Corr = 0.33)","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:07:13.556832Z","iopub.execute_input":"2024-10-25T17:07:13.557984Z","iopub.status.idle":"2024-10-25T17:07:13.570388Z","shell.execute_reply.started":"2024-10-25T17:07:13.557931Z","shell.execute_reply":"2024-10-25T17:07:13.569148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Correlation with other variables","metadata":{}},{"cell_type":"code","source":"df.drop(columns=[\"id\",\"Basic_Demos-Enroll_Season\",\"CGAS-Season\",\"Physical-Season\",\n\"Fitness_Endurance-Season\",\"FGC-Season\",\"BIA-Season\",\"PAQ_A-Season\",\n\"PCIAT-Season\",\"SDS-Season\",\"PreInt_EduHx-Season\",\"PAQ_C-Season\"]).corr()[[\"sii\"]].reset_index().rename({\"index\":\"features\"}).head(60).sort_values(by='sii', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-25T17:07:15.700265Z","iopub.execute_input":"2024-10-25T17:07:15.700749Z","iopub.status.idle":"2024-10-25T17:07:15.786208Z","shell.execute_reply.started":"2024-10-25T17:07:15.700705Z","shell.execute_reply":"2024-10-25T17:07:15.785002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conclusion Part 1/3\nWe focused on what impacts the sii as a target variable directly. We also need to know what impacts PCIAT, we'll do that in the next ipynb (probably). We'll also study the main feature analysis of series data in the third part (after that we'll move on to the solutions building). \n\n## What did we learn?\nA lot of these variables correlate with the target sii, which means we can use them with a grain of salt (we didn't strong correlation with anything but we did see some interesting stuff). We can also study how these variables correlate with the PCIAT targets. We might find more gems from there.\n\n### P.S\nI'm looking for team mates, untill i find them, I'll work open-source. Let's connect!","metadata":{}}]}