{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30369,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 📋Table of Contents\n* [Import and First Glance](#import)\n* [Distributions of Features](#eda)\n* [Correlation of Features](#corr)\n* [Target vs Features](#target_features)","metadata":{}},{"cell_type":"code","source":"# standard\nimport numpy as np\nimport pandas as pd\nimport time\n\n# plots\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# statistics\nfrom scipy import stats\n\n# H2O\nimport h2o\nfrom h2o.estimators.glm import H2OGeneralizedLinearEstimator","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:44:23.670327Z","iopub.execute_input":"2024-09-20T06:44:23.670923Z","iopub.status.idle":"2024-09-20T06:44:23.680631Z","shell.execute_reply.started":"2024-09-20T06:44:23.670881Z","shell.execute_reply":"2024-09-20T06:44:23.678633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# configs\npd.set_option('display.max_columns', None) # we want to display all columns in this notebook\npd.set_option('display.max_rows', 100) # increase rows to be displayed\npd.set_option('display.max_colwidth', None) # show full cell contents\n\n# random seed\nmy_random_seed = 111\n\n# aesthetics\ndefault_color_1 = 'darkblue'\ndefault_color_2 = 'darkgreen'\ndefault_color_3 = 'darkred'","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:44:23.684061Z","iopub.execute_input":"2024-09-20T06:44:23.684647Z","iopub.status.idle":"2024-09-20T06:44:23.697358Z","shell.execute_reply.started":"2024-09-20T06:44:23.684587Z","shell.execute_reply":"2024-09-20T06:44:23.695222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='import'></a>\n# Import and First Glance","metadata":{}},{"cell_type":"code","source":"# load data\ndf_train = pd.read_csv('../input/child-mind-institute-problematic-internet-use/train.csv')\ndf_test = pd.read_csv('../input/child-mind-institute-problematic-internet-use/test.csv')\ndf_sub = pd.read_csv('../input/child-mind-institute-problematic-internet-use/sample_submission.csv')\ndf_dict = pd.read_csv('../input/child-mind-institute-problematic-internet-use/data_dictionary.csv')","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:44:23.699832Z","iopub.execute_input":"2024-09-20T06:44:23.700771Z","iopub.status.idle":"2024-09-20T06:44:23.786567Z","shell.execute_reply.started":"2024-09-20T06:44:23.700715Z","shell.execute_reply":"2024-09-20T06:44:23.785178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show data dictionary\ndf_dict","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:44:23.788894Z","iopub.execute_input":"2024-09-20T06:44:23.789465Z","iopub.status.idle":"2024-09-20T06:44:23.837132Z","shell.execute_reply.started":"2024-09-20T06:44:23.789412Z","shell.execute_reply":"2024-09-20T06:44:23.834815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview training data\ndf_train.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:44:23.842898Z","iopub.execute_input":"2024-09-20T06:44:23.843647Z","iopub.status.idle":"2024-09-20T06:44:24.012047Z","shell.execute_reply.started":"2024-09-20T06:44:23.843589Z","shell.execute_reply":"2024-09-20T06:44:24.010077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# structure of data - train\ndf_train.info()","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:44:24.014302Z","iopub.execute_input":"2024-09-20T06:44:24.014813Z","iopub.status.idle":"2024-09-20T06:44:24.423046Z","shell.execute_reply.started":"2024-09-20T06:44:24.014761Z","shell.execute_reply":"2024-09-20T06:44:24.421294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# structure of data - test\ndf_test.info()","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:44:24.425427Z","iopub.execute_input":"2024-09-20T06:44:24.426449Z","iopub.status.idle":"2024-09-20T06:44:24.456127Z","shell.execute_reply.started":"2024-09-20T06:44:24.426369Z","shell.execute_reply":"2024-09-20T06:44:24.454374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 💡 Test set does not contain all the PCIAT-PCIAT... features!","metadata":{}},{"cell_type":"markdown","source":"<a id='eda'></a>\n# Distributions of Features","metadata":{}},{"cell_type":"code","source":"# basic stats - train\ndf_train.describe(include='all')","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:44:24.458920Z","iopub.execute_input":"2024-09-20T06:44:24.459566Z","iopub.status.idle":"2024-09-20T06:44:24.893282Z","shell.execute_reply.started":"2024-09-20T06:44:24.459506Z","shell.execute_reply":"2024-09-20T06:44:24.889891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basic stats - test\ndf_test.describe(include='all')","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:44:24.897599Z","iopub.execute_input":"2024-09-20T06:44:24.899425Z","iopub.status.idle":"2024-09-20T06:44:25.191998Z","shell.execute_reply.started":"2024-09-20T06:44:24.899363Z","shell.execute_reply":"2024-09-20T06:44:25.190222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define features and target\n\n# categorical\nfeatures_cat_train_test = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Sex', 'CGAS-Season', \n                           'Physical-Season', 'Fitness_Endurance-Season', 'FGC-Season',\n                           'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', \n                           'SDS-Season', 'PreInt_EduHx-Season',]\n\n# numerical\nfeatures_num_train_test = ['Basic_Demos-Age', 'CGAS-CGAS_Score', 'Physical-BMI',\n                           'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                           'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                           'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins', \n                           'Fitness_Endurance-Time_Sec',\n                           'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                           'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                           'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                           'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', \n                           'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                           'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                           'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                           'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                           'BIA-BIA_TBW',  'PAQ_A-PAQ_A_Total', \n                           'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T', \n                           'PreInt_EduHx-computerinternet_hoursday']\n\nfeatures_num_train_only = ['PCIAT-PCIAT_01', 'PCIAT-PCIAT_02',\n                           'PCIAT-PCIAT_03', 'PCIAT-PCIAT_04', 'PCIAT-PCIAT_05', 'PCIAT-PCIAT_06',\n                           'PCIAT-PCIAT_07', 'PCIAT-PCIAT_08', 'PCIAT-PCIAT_09', 'PCIAT-PCIAT_10',\n                           'PCIAT-PCIAT_11', 'PCIAT-PCIAT_12', 'PCIAT-PCIAT_13', 'PCIAT-PCIAT_14',\n                           'PCIAT-PCIAT_15', 'PCIAT-PCIAT_16', 'PCIAT-PCIAT_17', 'PCIAT-PCIAT_18',\n                           'PCIAT-PCIAT_19', 'PCIAT-PCIAT_20', 'PCIAT-PCIAT_Total']\n\nfeatures_cat_train_only = ['PCIAT-Season']\n\n# target\ntarget = 'sii'","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:44:25.194820Z","iopub.execute_input":"2024-09-20T06:44:25.195576Z","iopub.status.idle":"2024-09-20T06:44:25.218140Z","shell.execute_reply.started":"2024-09-20T06:44:25.195504Z","shell.execute_reply":"2024-09-20T06:44:25.215917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot histograms (train and test)\nfor f in features_num_train_test:\n    plt.figure(figsize=(14,3))\n    ax1 = plt.subplot(1,2,1)\n    df_train[f].plot(kind='hist', bins=15, color=default_color_1)\n    plt.title(f + ' - Train')\n    plt.grid()\n    ax2 = plt.subplot(1,2,2, sharex=ax1)\n    df_test[f].plot(kind='hist', bins=15, color=default_color_2)\n    plt.title(f + ' - Test')\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:44:25.220796Z","iopub.execute_input":"2024-09-20T06:44:25.221397Z","iopub.status.idle":"2024-09-20T06:44:47.080279Z","shell.execute_reply.started":"2024-09-20T06:44:25.221345Z","shell.execute_reply":"2024-09-20T06:44:47.078818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# boxplots (train and test)\nfor f in features_num_train_test:\n    plt.figure(figsize=(14,1))\n    ax1 = plt.subplot(1,2,1)\n    df_temp = df_train[f].dropna() # boxplot does not like missings...\n    plt.boxplot(df_temp, vert=False)\n    plt.title(f + ' - Train')\n    plt.grid()\n    ax2 = plt.subplot(1,2,2, sharex=ax1)\n    df_temp = df_test[f].dropna()\n    plt.boxplot(df_temp, vert=False)\n    plt.title(f + ' - Test')\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:44:47.081910Z","iopub.execute_input":"2024-09-20T06:44:47.082306Z","iopub.status.idle":"2024-09-20T06:45:00.176143Z","shell.execute_reply.started":"2024-09-20T06:44:47.082270Z","shell.execute_reply":"2024-09-20T06:45:00.174542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot categorical feature distributions (train and test)\nfor f in features_cat_train_test:\n    plt.figure(figsize=(15,3))\n    ax1 = plt.subplot(1,2,1)\n    df_train[f].value_counts().sort_index().plot(kind='bar', color=default_color_1)\n    plt.title(f + ' - Train')\n    plt.grid()\n    ax2 = plt.subplot(1,2,2, sharex=ax1)\n    df_test[f].value_counts().sort_index().plot(kind='bar', color=default_color_2)\n    plt.title(f + ' - Test')\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:45:00.178178Z","iopub.execute_input":"2024-09-20T06:45:00.178711Z","iopub.status.idle":"2024-09-20T06:45:03.706696Z","shell.execute_reply.started":"2024-09-20T06:45:00.178660Z","shell.execute_reply":"2024-09-20T06:45:03.705190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='corr'></a>\n# Correlation of Features","metadata":{}},{"cell_type":"code","source":"# calc correlation matrices\ncorr_pearson = df_train[features_num_train_test].corr(method='pearson')\ncorr_spearman = df_train[features_num_train_test].corr(method='spearman')","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:45:03.711906Z","iopub.execute_input":"2024-09-20T06:45:03.712307Z","iopub.status.idle":"2024-09-20T06:45:04.174158Z","shell.execute_reply.started":"2024-09-20T06:45:03.712271Z","shell.execute_reply":"2024-09-20T06:45:04.172835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# and plot them\nplt.figure(figsize=(16,14))\nsns.heatmap(corr_pearson, annot=False, cmap='RdYlGn',\n            fmt='.2f', linecolor='black', linewidths=0.5,\n            vmin=-1, vmax=+1)\nplt.title('Pearson Correlation')\n\nplt.figure(figsize=(16,14))\nsns.heatmap(corr_spearman, annot=False, cmap='RdYlGn', \n            fmt='.2f', linecolor='black', linewidths=0.5,\n            vmin=-1, vmax=+1)\nplt.title('Spearman Correlation')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:45:04.175646Z","iopub.execute_input":"2024-09-20T06:45:04.176004Z","iopub.status.idle":"2024-09-20T06:45:07.172169Z","shell.execute_reply.started":"2024-09-20T06:45:04.175972Z","shell.execute_reply":"2024-09-20T06:45:07.170642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='target_features'></a>\n# Target vs Features","metadata":{}},{"cell_type":"code","source":"# plot target distribution\ndf_train[target].value_counts().sort_index().plot(kind='bar', color=default_color_3)\nplt.title('Target')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:45:07.174308Z","iopub.execute_input":"2024-09-20T06:45:07.174683Z","iopub.status.idle":"2024-09-20T06:45:07.341344Z","shell.execute_reply.started":"2024-09-20T06:45:07.174644Z","shell.execute_reply":"2024-09-20T06:45:07.339775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# target vs numerical features\nfor f in features_num_train_test:\n    plt.figure(figsize=(8,4))\n    sns.violinplot(data=df_train, x=target, y=f)\n    plt.title('Target vs ' + f)\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:45:07.342560Z","iopub.execute_input":"2024-09-20T06:45:07.342933Z","iopub.status.idle":"2024-09-20T06:45:19.680589Z","shell.execute_reply.started":"2024-09-20T06:45:07.342892Z","shell.execute_reply":"2024-09-20T06:45:19.679279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# impact of categorical features - normalized cross tables\nfor f in features_cat_train_test:\n    if f != 'hospital_number': # skip hospital_number due to too many levels\n        ctab = pd.crosstab(df_train[target], df_train[f])\n        ctab_norm = ctab / ctab.sum()\n        plt.figure(figsize=(12,2))\n        g = sns.heatmap(ctab_norm, annot=True,\n                        fmt='.2%', linecolor='black',\n                        linewidths=1, cmap='Greens', \n                        vmin=0, vmax=+1)\n        plt.title(f + ' vs target - train')\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-20T06:45:19.682508Z","iopub.execute_input":"2024-09-20T06:45:19.682964Z","iopub.status.idle":"2024-09-20T06:45:23.271028Z","shell.execute_reply.started":"2024-09-20T06:45:19.682919Z","shell.execute_reply":"2024-09-20T06:45:23.269615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Work in progress...","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-28T18:56:17.135691Z","iopub.execute_input":"2023-09-28T18:56:17.136136Z","iopub.status.idle":"2023-09-28T18:56:17.204689Z","shell.execute_reply.started":"2023-09-28T18:56:17.136101Z","shell.execute_reply":"2023-09-28T18:56:17.20336Z"}}}]}