{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":206675453,"sourceType":"kernelVersion"}],"dockerImageVersionId":30805,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"colab":{"provenance":[{"file_id":"https://storage.googleapis.com/kaggle-colab-exported-notebooks/hms-data-exploration-ce7f528a-9f71-468e-8ad9-0e214724babe.ipynb?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gcp-kaggle-com%40kaggle-161607.iam.gserviceaccount.com/20241202/auto/storage/goog4_request&X-Goog-Date=20241202T141659Z&X-Goog-Expires=259200&X-Goog-SignedHeaders=host&X-Goog-Signature=197f2bfa2cd59d0f362f6533df00995eacc39535ba5b8f2c41a0c6c33e5f10e170feb7efbfce344ec652b038d007b9cbd4703d41b4aa4b1d934207871a41337161dcf57b2820885e61b851197a2209127cbe81fb7b73139a45151da83e3b27327e15a548d36f1bf6b84006b5541061b88acb387988a02aa964d4f24cf7ebc61d1ceca3e1a35c1564178518bf147736f88038f9b23e22e920f824604fbc51edf610ba362c2fe2a88b141775b5423514626721a4302eb028d7fbd25cbee2cdfb20a7797ee97e0e53ff775badbaab09aa010dd409bb118a9ec1f0f236787370c8fa3d151fd02ae3774c3a2b340b8fd788124a631c942321ae0cb1d92d1675f17cd3","timestamp":1733149320630}]}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# IMPORTANT: RUN THIS CELL IN ORDER TO IMPORT YOUR KAGGLE DATA SOURCES,\n\n# THEN FEEL FREE TO DELETE THIS CELL.\n\n# NOTE: THIS NOTEBOOK ENVIRONMENT DIFFERS FROM KAGGLE'S PYTHON\n\n# ENVIRONMENT SO THERE MAY BE MISSING LIBRARIES USED BY YOUR\n\n# NOTEBOOK.\n\n\n\nchild_mind_institute_problematic_internet_use_path = '/kaggle/input/child-mind-institute-problematic-internet-use'\n\n\nprint('Data source import complete.')\n","metadata":{"id":"h_nz4Mtou2UN","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:01:50.863623Z","iopub.execute_input":"2024-12-08T20:01:50.863974Z","iopub.status.idle":"2024-12-08T20:01:50.869172Z","shell.execute_reply.started":"2024-12-08T20:01:50.863934Z","shell.execute_reply":"2024-12-08T20:01:50.868231Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**This notebook is focusing on HMS EDA**","metadata":{"id":"XDHIFr5-u2UN"}},{"cell_type":"code","source":"palette='viridis'","metadata":{"id":"-zuJL2ucPFgb","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:01:50.870613Z","iopub.execute_input":"2024-12-08T20:01:50.870854Z","iopub.status.idle":"2024-12-08T20:01:50.881271Z","shell.execute_reply.started":"2024-12-08T20:01:50.870830Z","shell.execute_reply":"2024-12-08T20:01:50.880479Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Import Libraries","metadata":{"id":"D14pZZaVu2UO"}},{"cell_type":"code","source":"!pip install hyperopt scikit-learn lightgbm xgboost catboost\n!pip install tpot","metadata":{"id":"cRcgCfDnqL6y","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:01:50.882288Z","iopub.execute_input":"2024-12-08T20:01:50.882611Z","iopub.status.idle":"2024-12-08T20:02:00.151118Z","shell.execute_reply.started":"2024-12-08T20:01:50.882575Z","shell.execute_reply":"2024-12-08T20:02:00.150266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom tpot import TPOTClassifier\n\nimport numpy as np\n\npd.set_option('display.max_columns', 500)\n\npd.set_option('display.max_rows', 100)\n\nimport matplotlib.pyplot as plt\n\nimport seaborn as sns\n\nfrom scipy.stats import zscore\n\nfrom sklearn.preprocessing import OneHotEncoder, OrdinalEncoder\n\nfrom sklearn.impute import SimpleImputer\n\n\n\nimport scipy.cluster.hierarchy as sch\n\nfrom sklearn.cluster import AgglomerativeClustering\n\nfrom sklearn.impute import KNNImputer\n\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom sklearn.preprocessing import StandardScaler\n\nfrom sklearn.ensemble import RandomForestClassifier\n\nfrom sklearn.metrics import accuracy_score, classification_report\n\nfrom sklearn.preprocessing import StandardScaler\n\nimport pandas as pd\n\nimport numpy as np\n\nfrom sklearn.model_selection import train_test_split\n\nfrom sklearn.metrics import cohen_kappa_score, make_scorer\n\nfrom hyperopt import fmin, tpe, hp, Trials, STATUS_OK\n\nfrom sklearn.ensemble import RandomForestClassifier\n\nimport lightgbm as lgb\n\nimport xgboost as xgb\n\nfrom catboost import CatBoostClassifier\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:00.153411Z","iopub.execute_input":"2024-12-08T20:02:00.153734Z","iopub.status.idle":"2024-12-08T20:02:04.650552Z","shell.execute_reply.started":"2024-12-08T20:02:00.153688Z","shell.execute_reply":"2024-12-08T20:02:04.649650Z"},"id":"N_lCHM6Xu2UO"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Helper Functions","metadata":{"id":"qvqHEsofVM4K"}},{"cell_type":"code","source":"def filter_by_instrument(df_train, df_dict, instrument_filter):\n\n    \"\"\"\n\n    Filter the training dataset by a specific instrument using the dictionary file.\n\n\n\n    Parameters:\n\n        df_train (pd.DataFrame): The training dataset.\n\n        df_dict (pd.DataFrame): The dictionary file.\n\n        instrument_filter (str): Instrument name to filter columns.\n\n\n\n    Returns:\n\n        pd.DataFrame: Filtered training data for the specified instrument.\n\n        pd.DataFrame: Filtered dictionary for the specified instrument.\n\n    \"\"\"\n\n    df_dict_instrument = df_dict[df_dict['Instrument'] == instrument_filter]\n\n    columns = df_dict_instrument['Field'].tolist()\n\n    df_filtered = df_train[columns]\n\n    return df_filtered, df_dict_instrument\n\n\n\n\n\ndef general_info(df, name=\"Dataset\"):\n\n    \"\"\"\n\n    Display general information about the dataset.\n\n\n\n    Parameters:\n\n        df (pd.DataFrame): The dataset to analyze.\n\n        name (str): The name of the dataset for display purposes.\n\n    \"\"\"\n\n    print(f\"Summary of {name}:\")\n\n    print(df.info())\n\n    print(\"\\nSummary Statistics:\")\n\n    print(df.describe())\n\n    print(\"\\nMissing Values Percentage:\")\n\n    print(df.isnull().sum() / len(df))\n\n    total_rows = df.shape[0]\n\n\n\n    completely_missing_rows = df[df.isnull().all(axis=1)].shape[0]\n\n    # Total missing values\n\n    total_missing_values = df.isnull().sum().sum()\n\n    percentage_completely_missing_rows = (completely_missing_rows / total_rows) * 100\n\n    print(\"\\n Totally Missing Rows Percentage:\")\n\n    print(f\"{percentage_completely_missing_rows:.2f}\")\n\n\n\ndef analyze_categorical(df, categorical_columns):\n\n    \"\"\"\n\n    Analyze categorical columns by printing value counts.\n\n\n\n    Parameters:\n\n        df (pd.DataFrame): The dataset.\n\n        categorical_columns (list): List of categorical column names.\n\n    \"\"\"\n\n    for col in categorical_columns:\n\n        print(f\"\\nValue Counts for {col}:\")\n\n        print(df[col].value_counts())\n\n\n\ndef analyze_numerical(df, numerical_columns):\n\n    \"\"\"\n\n    Analyze numerical columns by generating descriptive statistics.\n\n\n\n    Parameters:\n\n        df (pd.DataFrame): The dataset.\n\n        numerical_columns (list): List of numerical column names.\n\n    \"\"\"\n\n    print(\"\\nDescriptive Statistics for Numerical Columns:\")\n\n    print(df[numerical_columns].describe())\n\n\n\ndef plot_numerical_distributions(df, numerical_columns):\n\n    \"\"\"\n\n    Plot distributions for numerical columns.\n\n\n\n    Parameters:\n\n        df (pd.DataFrame): The dataset.\n\n        numerical_columns (list): List of numerical column names.\n\n    \"\"\"\n\n    for col in numerical_columns:\n\n        plt.figure(figsize=(8, 4))\n\n        sns.histplot(df[col], kde=True, bins=30)\n\n        plt.title(f'Distribution of {col}')\n\n        plt.show()\n\ndef plot_categorical_distributions(df, categorical_columns):\n\n    \"\"\"\n\n    Plot bar charts for categorical columns.\n\n\n\n    Parameters:\n\n        df (pd.DataFrame): The dataset.\n\n        categorical_columns (list): List of categorical column names.\n\n    \"\"\"\n\n    for col in categorical_columns:\n\n        plt.figure(figsize=(8, 4))\n\n        df[col].value_counts().plot(kind='bar', color='skyblue')\n\n        plt.title(f'Distribution of {col}')\n\n        plt.ylabel('Count')\n\n        plt.xlabel('Categories')\n\n        plt.xticks(rotation=45)\n\n        plt.show()","metadata":{"id":"tSwqg1zXVOcm","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:04.651724Z","iopub.execute_input":"2024-12-08T20:02:04.652396Z","iopub.status.idle":"2024-12-08T20:02:04.663376Z","shell.execute_reply.started":"2024-12-08T20:02:04.652366Z","shell.execute_reply":"2024-12-08T20:02:04.662503Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Define Directories","metadata":{"id":"KSYcSBhfu2UP"}},{"cell_type":"code","source":"root_path = child_mind_institute_problematic_internet_use_path\n\ncsv_train_path = f'{root_path}/train.csv'\n\ncsv_dict_path = f'{root_path}/data_dictionary.csv'\n\n\n\ncsv_test_path = f'{root_path}/test.csv'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:04.664403Z","iopub.execute_input":"2024-12-08T20:02:04.664698Z","iopub.status.idle":"2024-12-08T20:02:04.688741Z","shell.execute_reply.started":"2024-12-08T20:02:04.664673Z","shell.execute_reply":"2024-12-08T20:02:04.688006Z"},"id":"Gd41U2K5u2UP"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load CSV Data","metadata":{"id":"ITnsXS3uu2UP"}},{"cell_type":"code","source":"df_train_csv = pd.read_csv(csv_train_path)\n\ndf_test_csv = pd.read_csv(csv_test_path)\n\ndf_dict_csv = pd.read_csv(csv_dict_path)\n\nunique_instruments = np.unique(df_dict_csv['Instrument'])\n\nprint(f\"Unique Instruments are\\n  {unique_instruments}\")\n\nprint(f\"The number of Unique Instruments is  {len(unique_instruments)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:04.689837Z","iopub.execute_input":"2024-12-08T20:02:04.690147Z","iopub.status.idle":"2024-12-08T20:02:04.767868Z","shell.execute_reply.started":"2024-12-08T20:02:04.690121Z","shell.execute_reply":"2024-12-08T20:02:04.767001Z"},"id":"uVZRGgtBu2UP"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_csv.columns","metadata":{"id":"uRapE0sSV4pq","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:04.768851Z","iopub.execute_input":"2024-12-08T20:02:04.769136Z","iopub.status.idle":"2024-12-08T20:02:04.775611Z","shell.execute_reply.started":"2024-12-08T20:02:04.769109Z","shell.execute_reply":"2024-12-08T20:02:04.774711Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_csv_columns = df_train_csv.columns\n\ntest_csv_columns = df_test_csv.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:04.776707Z","iopub.execute_input":"2024-12-08T20:02:04.776970Z","iopub.status.idle":"2024-12-08T20:02:04.784663Z","shell.execute_reply.started":"2024-12-08T20:02:04.776923Z","shell.execute_reply":"2024-12-08T20:02:04.783830Z"},"id":"QgPL0swCu2UQ"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Number of training columns is: \", len(train_csv_columns))\n\nprint(\"Number of test columns is: \", len(test_csv_columns))","metadata":{"id":"9SyZ8HaRntMU","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:04.787588Z","iopub.execute_input":"2024-12-08T20:02:04.787826Z","iopub.status.idle":"2024-12-08T20:02:04.795847Z","shell.execute_reply.started":"2024-12-08T20:02:04.787802Z","shell.execute_reply":"2024-12-08T20:02:04.794892Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# different columns\n\ndiff_cols = set(train_csv_columns) - set(test_csv_columns)\n\ndiff_cols\n\ndiff_cols.remove('sii')","metadata":{"id":"nlcFkUcTn3DK","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:04.796902Z","iopub.execute_input":"2024-12-08T20:02:04.797259Z","iopub.status.idle":"2024-12-08T20:02:04.808387Z","shell.execute_reply.started":"2024-12-08T20:02:04.797234Z","shell.execute_reply":"2024-12-08T20:02:04.807630Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# List of PCIAT columns\n\nPCIAT_cols = [f'PCIAT-PCIAT_{i+1:02d}' for i in range(20)]\n\n\n\ndf_train_csv['Calculated_PCIAT_Total'] = df_train_csv[PCIAT_cols].sum(axis=1)\n\n# convert 0 to NaN\n\ndf_train_csv['Calculated_PCIAT_Total'] = df_train_csv['Calculated_PCIAT_Total'].replace(0, np.nan)\n\ndf_train_csv['Calculated_PCIAT_Total'] = df_train_csv['Calculated_PCIAT_Total'].replace(0.0, np.nan)\n\ndf_train_csv['PCIAT-PCIAT_Total'] = df_train_csv['PCIAT-PCIAT_Total'].replace(0.0, np.nan)\n\n\n\n# Count mismatched totals\n\nmiscalculated_PCIAT_Total = (df_train_csv['PCIAT-PCIAT_Total'] != df_train_csv['Calculated_PCIAT_Total'])\n\n\n\nPCIAT_cols = [f'PCIAT-PCIAT_{i+1:02d}' for i in range(20)]\n\n\n\ndef recalculate_sii(row):\n\n    if pd.isna(row['PCIAT-PCIAT_Total']):\n\n        return np.nan\n\n    max_possible = row['PCIAT-PCIAT_Total'] + row[PCIAT_cols].isna().sum() * 5\n\n    if row['PCIAT-PCIAT_Total'] <= 30 and max_possible <= 30:\n\n        return 0\n\n    elif 31 <= row['PCIAT-PCIAT_Total'] <= 49 and max_possible <= 49:\n\n        return 1\n\n    elif 50 <= row['PCIAT-PCIAT_Total'] <= 79 and max_possible <= 79:\n\n        return 2\n\n    elif row['PCIAT-PCIAT_Total'] >= 80 and max_possible >= 80:\n\n        return 3\n\n    return np.nan\n\ndf_train_csv['Severity Impairment Index (SII)'] = df_train_csv.apply(recalculate_sii, axis=1)\n","metadata":{"id":"OlGgXBcUH_Bt","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:04.809229Z","iopub.execute_input":"2024-12-08T20:02:04.809480Z","iopub.status.idle":"2024-12-08T20:02:05.973890Z","shell.execute_reply.started":"2024-12-08T20:02:04.809457Z","shell.execute_reply":"2024-12-08T20:02:05.973016Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# Define the mapping using pd.cut\n\nbins = [5 ,14, 22]\n\nlabels = ['Child', 'Teen']\n\n\n\ndf_train_csv['Age Group'] = pd.cut(df_train_csv['Basic_Demos-Age'], bins=bins, labels=labels, right=True, include_lowest=True)\n\ndf_test_csv['Age Group'] = pd.cut(df_test_csv['Basic_Demos-Age'], bins=bins, labels=labels, right=True, include_lowest=True)\n","metadata":{"id":"y7roB_CXM8NZ","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:05.975093Z","iopub.execute_input":"2024-12-08T20:02:05.975375Z","iopub.status.idle":"2024-12-08T20:02:05.987648Z","shell.execute_reply.started":"2024-12-08T20:02:05.975348Z","shell.execute_reply":"2024-12-08T20:02:05.987003Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Fixing Labels","metadata":{"id":"AbiCDsUup8Us"}},{"cell_type":"markdown","source":"We have a lot of missing values in the target","metadata":{"id":"Y1lxopj6p--s"}},{"cell_type":"code","source":"pciat_columns = [f'PCIAT-PCIAT_{i+1:02d}' for i in range(20)]","metadata":{"id":"1Bd9RfxWsmtm","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:05.988683Z","iopub.execute_input":"2024-12-08T20:02:05.988927Z","iopub.status.idle":"2024-12-08T20:02:05.997922Z","shell.execute_reply.started":"2024-12-08T20:02:05.988903Z","shell.execute_reply":"2024-12-08T20:02:05.997250Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# List of PCIAT columns\n\nquestion_columns = pciat_columns\n\n# Multiply relevant columns by 5 and calculate the sum across the row\n\ndf_train_csv['Calculated_PCIAT_Total'] = df_train_csv[question_columns].sum(axis=1)\n\n# convert 0 to NaN\n\ndf_train_csv['Calculated_PCIAT_Total'] = df_train_csv['Calculated_PCIAT_Total'].replace(0, np.nan)\n\ndf_train_csv['Calculated_PCIAT_Total'] = df_train_csv['Calculated_PCIAT_Total'].replace(0.0, np.nan)\n\ndf_train_csv['PCIAT-PCIAT_Total'] = df_train_csv['PCIAT-PCIAT_Total'].replace(0.0, np.nan)\n\n\n\n# Count mismatched totals\n\nmiscalculated_PCIAT_Total = (df_train_csv['PCIAT-PCIAT_Total'] != df_train_csv['Calculated_PCIAT_Total']).sum()\n\nprint(f\"Number of Miscalculated PCIAT Totals: {miscalculated_PCIAT_Total}\")\n\n\n","metadata":{"id":"jwcM_UiPtTA9","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:05.998816Z","iopub.execute_input":"2024-12-08T20:02:05.999098Z","iopub.status.idle":"2024-12-08T20:02:06.014013Z","shell.execute_reply.started":"2024-12-08T20:02:05.999075Z","shell.execute_reply":"2024-12-08T20:02:06.013105Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"differences = df_train_csv[\n\n    ~((df_train_csv['Calculated_PCIAT_Total'] == df_train_csv['PCIAT-PCIAT_Total']) |\n\n      (df_train_csv['Calculated_PCIAT_Total'].isna() & df_train_csv['PCIAT-PCIAT_Total'].isna()))\n\n]\n\n(differences[['PCIAT-PCIAT_Total', 'Calculated_PCIAT_Total']])\n","metadata":{"id":"2SALWAjOvZSs","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:06.015125Z","iopub.execute_input":"2024-12-08T20:02:06.015919Z","iopub.status.idle":"2024-12-08T20:02:06.031879Z","shell.execute_reply.started":"2024-12-08T20:02:06.015876Z","shell.execute_reply":"2024-12-08T20:02:06.031025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_csv.iloc[1,:]\n\ndf_train_csv[(df_train_csv['sii']==0.0) & (df_train_csv['Severity Impairment Index (SII)'].isna()) ]","metadata":{"id":"WW1fvbX-xvM8","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:06.032880Z","iopub.execute_input":"2024-12-08T20:02:06.033178Z","iopub.status.idle":"2024-12-08T20:02:06.111833Z","shell.execute_reply.started":"2024-12-08T20:02:06.033154Z","shell.execute_reply":"2024-12-08T20:02:06.110991Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# DATA LEAKAGE","metadata":{"id":"yGJsKWsGonUm"}},{"cell_type":"code","source":"diff_cols","metadata":{"id":"CsFXQUkWoQxY","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:06.112857Z","iopub.execute_input":"2024-12-08T20:02:06.113126Z","iopub.status.idle":"2024-12-08T20:02:06.118948Z","shell.execute_reply.started":"2024-12-08T20:02:06.113100Z","shell.execute_reply":"2024-12-08T20:02:06.118007Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"ums_YDb7pbTS","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_dict_csv = pd.read_csv(csv_dict_path)\n\nunique_instruments = np.unique(df_dict_csv['Instrument'])\n\nprint(f\"Unique Instruments are\\n  {unique_instruments}\")\n\nprint(f\"The number of Unique Instruments is  {len(unique_instruments)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:06.120037Z","iopub.execute_input":"2024-12-08T20:02:06.120333Z","iopub.status.idle":"2024-12-08T20:02:06.131392Z","shell.execute_reply.started":"2024-12-08T20:02:06.120307Z","shell.execute_reply":"2024-12-08T20:02:06.130541Z"},"id":"CZkhGjevu2UQ"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We dropped all PCAIT (Parent-Child Internet Addiction Test)\n\n\n\nSo, we must drop the column from dictonary and all its rows","metadata":{"id":"-mkphfxHobhC"}},{"cell_type":"code","source":"df_dict_csv = df_dict_csv[df_dict_csv['Instrument'] != 'Parent-Child Internet Addiction Test']\n\nunique_instruments = np.unique(df_dict_csv['Instrument'])\n\nprint(f\"Unique Instruments are\\n  {unique_instruments}\")\n\nprint(f\"The number of Unique Instruments is  {len(unique_instruments)}\")","metadata":{"id":"aEWHWGnXo2GA","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:06.132441Z","iopub.execute_input":"2024-12-08T20:02:06.132681Z","iopub.status.idle":"2024-12-08T20:02:06.144339Z","shell.execute_reply.started":"2024-12-08T20:02:06.132658Z","shell.execute_reply":"2024-12-08T20:02:06.143437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_csv.drop(columns=diff_cols, inplace=True)","metadata":{"id":"iOLEmCHODMXN","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:06.145302Z","iopub.execute_input":"2024-12-08T20:02:06.145538Z","iopub.status.idle":"2024-12-08T20:02:06.156923Z","shell.execute_reply.started":"2024-12-08T20:02:06.145509Z","shell.execute_reply":"2024-12-08T20:02:06.156167Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_csv.drop(columns = ['sii', 'Calculated_PCIAT_Total'], inplace=True)","metadata":{"id":"klND4cz0tNs7","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:06.157898Z","iopub.execute_input":"2024-12-08T20:02:06.158235Z","iopub.status.idle":"2024-12-08T20:02:06.167144Z","shell.execute_reply.started":"2024-12-08T20:02:06.158201Z","shell.execute_reply":"2024-12-08T20:02:06.166334Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Show High Correlations","metadata":{"id":"TWZon9sHuysf"}},{"cell_type":"code","source":"# Calculate the correlation matrix for float columns\n\ncorrelation_matrix = df_train_csv.select_dtypes(include='float').corr()\n\n\n\n# Style the correlation matrix to highlight values greater than 0.9\n\nstyled_corr_matrix = correlation_matrix.style.applymap(\n\n    lambda x: 'background-color: yellow' if abs(x) > 0.95 and x != 1 else ''\n\n)\n\n\n\n# Display the styled correlation matrix\n\nstyled_corr_matrix","metadata":{"id":"76p4YAySu16o","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:06.168082Z","iopub.execute_input":"2024-12-08T20:02:06.168391Z","iopub.status.idle":"2024-12-08T20:02:06.278804Z","shell.execute_reply.started":"2024-12-08T20:02:06.168354Z","shell.execute_reply":"2024-12-08T20:02:06.278088Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Describe: Bio-electric Impedance Analysis","metadata":{"id":"b5Wi4uETu2UQ"}},{"cell_type":"markdown","source":"Here, I will perform Exploratory Data Analysis and Handle any missing values or outliers","metadata":{"id":"yfNjPE6tSW_u"}},{"cell_type":"code","source":"# Apply Filters\n\ninstrument_filter = \"Bio-electric Impedance Analysis\"\n\ndf_bia, df_dict_bia = filter_by_instrument(df_train_csv, df_dict_csv, instrument_filter)\n\ndf_bia.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:06.279989Z","iopub.execute_input":"2024-12-08T20:02:06.280324Z","iopub.status.idle":"2024-12-08T20:02:06.302286Z","shell.execute_reply.started":"2024-12-08T20:02:06.280288Z","shell.execute_reply":"2024-12-08T20:02:06.301407Z"},"id":"m9laSIb-u2UQ"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_dict_bia","metadata":{"id":"QbcyZhMGWyfM","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:06.303300Z","iopub.execute_input":"2024-12-08T20:02:06.303541Z","iopub.status.idle":"2024-12-08T20:02:06.317058Z","shell.execute_reply.started":"2024-12-08T20:02:06.303517Z","shell.execute_reply":"2024-12-08T20:02:06.316173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# General information\n\ngeneral_info(df_bia, name=\"Bio-electric Impedance Analysis\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:06.318042Z","iopub.execute_input":"2024-12-08T20:02:06.318278Z","iopub.status.idle":"2024-12-08T20:02:06.371416Z","shell.execute_reply.started":"2024-12-08T20:02:06.318253Z","shell.execute_reply":"2024-12-08T20:02:06.370584Z"},"id":"x4lMOOSTu2UQ"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Classify variables based on type\n\nqualitative_nominal_bia = ['BIA-Season']\n\nqualitative_ordinal_bia = ['BIA-BIA_Activity_Level_num', 'BIA-BIA_Frame_num']\n\nquantitative_ratio_bia = [\n\n    'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW',\n\n    'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_ICW',\n\n    'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW'\n\n]\n\nquantitative_interval_bia = []","metadata":{"id":"c4fENEMIW_Eo","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:06.372275Z","iopub.execute_input":"2024-12-08T20:02:06.372521Z","iopub.status.idle":"2024-12-08T20:02:06.376930Z","shell.execute_reply.started":"2024-12-08T20:02:06.372497Z","shell.execute_reply":"2024-12-08T20:02:06.375843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Analyze categorical columns\n\ncategorical_columns_bia = qualitative_nominal_bia + qualitative_ordinal_bia\n\nanalyze_categorical(df_bia, categorical_columns_bia)\n\nplot_categorical_distributions(df_bia, categorical_columns_bia)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:06.383219Z","iopub.execute_input":"2024-12-08T20:02:06.383502Z","iopub.status.idle":"2024-12-08T20:02:07.072135Z","shell.execute_reply.started":"2024-12-08T20:02:06.383478Z","shell.execute_reply":"2024-12-08T20:02:07.071137Z"},"id":"o_-HCk_4u2UR"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#### Analyze Numerical columns\n\n\n\nnumerical_columns_bia = quantitative_ratio_bia + quantitative_interval_bia\n\nanalyze_numerical(df_bia, numerical_columns_bia)\n\nplot_numerical_distributions(df_bia, numerical_columns_bia)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:07.073312Z","iopub.execute_input":"2024-12-08T20:02:07.073695Z","iopub.status.idle":"2024-12-08T20:02:11.287251Z","shell.execute_reply.started":"2024-12-08T20:02:07.073656Z","shell.execute_reply":"2024-12-08T20:02:11.286286Z"},"scrolled":true,"id":"mCPKEAy5u2UR"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Insights","metadata":{"id":"Kze25_ewvv2F"}},{"cell_type":"markdown","source":"We have two features that have almost 100% correlation. it is basically the same feature!","metadata":{"id":"kyiuDJSrvzt2"}},{"cell_type":"code","source":"# show columns that have 2% or more difference\n\nbmi_diff = df_train_csv['BIA-BIA_BMI'] - df_train_csv['Physical-BMI']\n\n# round the difference\n\nsum(round(bmi_diff.dropna(), 2))","metadata":{"id":"qBkT9vzSDxfs","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:11.288474Z","iopub.execute_input":"2024-12-08T20:02:11.288832Z","iopub.status.idle":"2024-12-08T20:02:11.296159Z","shell.execute_reply.started":"2024-12-08T20:02:11.288792Z","shell.execute_reply":"2024-12-08T20:02:11.295318Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Calculate BMI using Physical Weight and Height","metadata":{"id":"lwD2v-V0vRWc"}},{"cell_type":"code","source":"\n\n# --- 1. Calculating BMI ---\n\n\n\ndf_train_csv['Calculated-BMI'] = 703 * df_train_csv['Physical-Weight'].dropna() / (df_train_csv['Physical-Height'].dropna() ** 2)\n\n\n\ndf_test_csv['Calculated-BMI'] = 703 * df_test_csv['Physical-Weight'].dropna() / (df_test_csv['Physical-Height'].dropna() ** 2)\n","metadata":{"id":"fkaxE5XuvNIV","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:11.297182Z","iopub.execute_input":"2024-12-08T20:02:11.297437Z","iopub.status.idle":"2024-12-08T20:02:11.314549Z","shell.execute_reply.started":"2024-12-08T20:02:11.297413Z","shell.execute_reply":"2024-12-08T20:02:11.313549Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Visuals","metadata":{"id":"EH4eneh9uhAt"}},{"cell_type":"code","source":"\n\n# --- 2. Calculate Correlation ---\n\nbmi_diff = df_train_csv['BIA-BIA_BMI'] - df_train_csv['Physical-BMI']\n\ncorr = df_train_csv[['Physical-BMI', 'BIA-BIA_BMI']].corr()\n\n\n\n# --- 3. Combined Plot ---\n\nfig, axes = plt.subplots(1, 2, figsize=(16, 6))  # 1x2 grid layout\n\n\n\n# --- Heatmap (Left) ---\n\nsns.heatmap(\n\n    corr, annot=True, cmap='coolwarm', fmt=\".2f\", cbar_kws={'shrink': 0.8}, ax=axes[0]\n\n)\n\naxes[0].set_title('Correlation Between Physical BMI and BIA BMI', fontsize=14, fontweight='bold', pad=10)\n\naxes[0].set_xticklabels(['Physical BMI', 'BIA BMI'], fontsize=10)\n\naxes[0].set_yticklabels(['Physical BMI', 'BIA BMI'], fontsize=10, rotation=0)\n\n\n\n# --- Scatter Plot with Regression Line (Right) ---\n\nsns.scatterplot(\n\n    data=df_train_csv, x='Physical-BMI', y='BIA-BIA_BMI', color='royalblue', s=50, ax=axes[1], label='Measured Data'\n\n)\n\nsns.regplot(\n\n    data=df_train_csv, x='Physical-BMI', y='BIA-BIA_BMI', scatter=False, color='red', ci=None, ax=axes[1], label= 'Fitted Line'\n\n)\n\naxes[1].set_title('Scatter Plot of Physical BMI vs BIA BMI', fontsize=14, fontweight='bold', pad=10)\n\naxes[1].set_xlabel('Physical BMI (Measured)', fontsize=12, fontweight='bold')\n\naxes[1].set_ylabel('BIA BMI (Measured)', fontsize=12, fontweight='bold')\n\naxes[1].legend(loc='upper left', fontsize=10)\n\naxes[1].grid(True, linestyle='--', alpha=0.5)\n\n\n\n# --- Adjust Layout ---\n\nplt.tight_layout()\n\n\n\n# --- Save as PDF ---\n\nplt.savefig(\"BMI_Correlation_and_ScatterPlot.pdf\", dpi=600, bbox_inches='tight')\n\n\n\n# Show the combined plot\n\nplt.show()\n\n\n\n# --- Enhanced BMI Difference Bar Plot ---\n\nplt.figure(figsize=(16, 7))  # Larger figure size for better clarity\n\n\n\n# Create bar plot with reduced width and visible edges\n\nplt.bar(\n\n    df_train_csv.index, bmi_diff,\n\n    width=0.8,  # Narrower bars for spacing\n\n    color='tomato',  # Solid, visible color\n\n    alpha=0.85,  # Slight transparency\n\n    edgecolor='black',  # Add black edges for clarity\n\n    linewidth=2\n\n)\n\n\n\n# Add a horizontal reference line at 0\n\nplt.axhline(0, color='black', linestyle='--', linewidth=1.2)\n\n\n\n# Add gridlines for better readability\n\nplt.grid(axis='y', linestyle='--', alpha=0.5)\n\n\n\n\n\n\n\n# Add titles and labels\n\nplt.title('Difference Between BIA BMI and Physical BMI', fontsize=18, fontweight='bold', pad=20)\n\nplt.xlabel('Sample Index', fontsize=14, fontweight='bold')\n\nplt.ylabel('BMI Difference (BIA - Physical)', fontsize=14, fontweight='bold')\n\n\n\n# Adjust layout for clean display\n\nplt.tight_layout()\n\n\n\n# Show the plot\n\nplt.show()\n","metadata":{"id":"1BxHCLKeFgTL","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:11.315926Z","iopub.execute_input":"2024-12-08T20:02:11.316230Z","iopub.status.idle":"2024-12-08T20:02:18.220734Z","shell.execute_reply.started":"2024-12-08T20:02:11.316205Z","shell.execute_reply":"2024-12-08T20:02:18.219915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Rename columns for clarity in the poster\n\nrenamed_features = {\n\n    'Physical-BMI': 'Measured BMI (Physical)',\n\n    'BIA-BIA_BMI': 'Measured BMI (BIA)',\n\n    'Calculated-BMI': 'Imputed BMI'\n\n}\n\n\n\n# Calculate missing values\n\nmissing_physical_before = df_train_csv[['Physical-BMI']].isna().sum()\n\nmissing_bia_before = df_train_csv[['BIA-BIA_BMI']].isna().sum()\n\nmissing_after = df_train_csv[['Calculated-BMI']].isna().sum()\n\ntotal_values = len(df_train_csv)\n\n\n\n# Create a DataFrame summarizing missing values\n\nmissing_summary = pd.DataFrame({\n\n    'Feature': [renamed_features['Physical-BMI'], renamed_features['BIA-BIA_BMI'], renamed_features['Calculated-BMI']],\n\n    'Missing Values': [missing_physical_before['Physical-BMI'],\n\n                       missing_bia_before['BIA-BIA_BMI'],\n\n                       missing_after['Calculated-BMI']],\n\n    'Percentage Missing': [missing_physical_before['Physical-BMI'] / total_values * 100,\n\n                           missing_bia_before['BIA-BIA_BMI'] / total_values * 100,\n\n                           missing_after['Calculated-BMI'] / total_values * 100]\n\n})\n\n\n\n# Create the bar plot\n\nplt.figure(figsize=(12, 8))\n\nsns.barplot(data=missing_summary, x='Feature', y='Percentage Missing', palette='coolwarm')\n\n\n\n# Title and labels for poster readability\n\nplt.title('Comparison of Missing BMI Values Before and After Imputation', fontsize=16, fontweight='bold', pad=20)\n\nplt.ylabel('Percentage of Missing Values (%)', fontsize=14)\n\nplt.xlabel('BMI Features', fontsize=14)\n\n\n\n# Set Y-axis limits\n\nplt.ylim(0, 100)\n\n\n\n# Annotate each bar with percentage values\n\nfor i, row in missing_summary.iterrows():\n\n    plt.text(i, row['Percentage Missing'] + 2, f\"{row['Percentage Missing']:.1f}%\",\n\n             ha='center', va='bottom', fontsize=12, fontweight='bold', color='black')\n\n\n\n# Add a professional note at the bottom\n\nplt.figtext(0.5, -0.08,\n\n            \"Note: Original 'Measured BMI' features were dropped after imputation to avoid redundancy.\",\n\n            ha='center', fontsize=12, color='red', fontweight='bold')\n\n\n\n# Clean up the layout\n\nplt.tight_layout()\n\n# save as pdf\n\nplt.savefig('bmi_comparison.pdf', dpi=600, bbox_inches='tight')\n\nplt.show()\n","metadata":{"id":"vvy7wcYTIxSE","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:18.222009Z","iopub.execute_input":"2024-12-08T20:02:18.222785Z","iopub.status.idle":"2024-12-08T20:02:18.849166Z","shell.execute_reply.started":"2024-12-08T20:02:18.222742Z","shell.execute_reply":"2024-12-08T20:02:18.848349Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Drop both BMI features and engineer our own","metadata":{"id":"Zp3kKvN_HGRF"}},{"cell_type":"code","source":"df_train_csv.drop(columns=['Physical-BMI', 'BIA-BIA_BMI'], inplace=True)\n\ndf_test_csv.drop(columns=['Physical-BMI', 'BIA-BIA_BMI'], inplace=True)","metadata":{"id":"hsbocm5cvFy3","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:18.850456Z","iopub.execute_input":"2024-12-08T20:02:18.851120Z","iopub.status.idle":"2024-12-08T20:02:18.859379Z","shell.execute_reply.started":"2024-12-08T20:02:18.851078Z","shell.execute_reply":"2024-12-08T20:02:18.858334Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ss","metadata":{"id":"EaGWeRA7vu32"}},{"cell_type":"markdown","source":"#### **Insights for Bio-Electric Impedance Analysis (BIA) Data**\n","metadata":{"id":"9ZBGAiTBvZ1B"}},{"cell_type":"markdown","source":"\n\n**General Observations**\n\n- **Dataset Size:** The dataset contains 3,960 entries and 17 fields.\n\n- **Data Types:**\n\n  - 16 numerical fields (`float64`) primarily measuring physical metrics.\n\n  - 1 categorical field (`BIA-Season`), indicating the season of participation.\n\n- **Missing Data:**\n\n  - The categorical field `BIA-Season` has 45.83% missing values.\n\n  - All numerical fields have approximately 49.72% missing values.\n","metadata":{"id":"rSECDkqdvmYN"}},{"cell_type":"markdown","source":"\n\n**Key Insights**\n\n1. **Missing Values:**\n\n   - Nearly half of the dataset has missing data across all numerical fields. This suggests significant data quality issues that will need to be addressed through imputation or feature elimination.\n\n   - The categorical field `BIA-Season` has slightly fewer missing values compared to the numerical fields, but it still represents a significant proportion of missing data.\n\n\n\n2. **Categorical Field:**\n\n   - `BIA-Season` provides information on the season of data collection, which may introduce seasonal variability in the measurements.\n\n   - This field will need encoding or imputation to be used effectively in modeling.\n\n\n\n3. **Numerical Fields:**\n\n   - Key physiological metrics such as BMI (`BIA-BIA_BMI`), body fat percentage (`BIA-BIA_Fat`), skeletal muscle mass (`BIA-BIA_SMM`), and total body water (`BIA-BIA_TBW`) are captured in this dataset.\n\n   - These metrics are critical for understanding body composition and energy expenditure, making them likely important features for downstream analysis.\n\n\n\n4. **Missingness and Modeling:**\n\n   - High missingness across numerical fields poses challenges for model training. Careful imputation strategies (e.g., KNN, clustering, or mean imputation) will be necessary to address this issue.\n\n   - Alternatively, some fields may need to be dropped if imputation does not yield satisfactory results.\n\n\n\n5. **Potential for Insights:**\n\n   - Despite missing values, the dataset captures comprehensive physiological details. This information can be highly valuable if properly cleaned and preprocessed.\n\n   - Correlation analysis among the numerical fields can reveal relationships between different body composition metrics.\n","metadata":{"id":"djIGbcF9vkdk"}},{"cell_type":"markdown","source":"# Describe: Children's Global Assessment Scale","metadata":{"id":"KQhqz03EXku0"}},{"cell_type":"code","source":"# Apply Filters\n\ninstrument_filter = \"Children's Global Assessment Scale\"\n\ndf_children_ass, df_dict_children_ass = filter_by_instrument(df_train_csv, df_dict_csv, instrument_filter)\n\ndf_children_ass.head()","metadata":{"id":"B0rm16ubXpcO","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:18.860702Z","iopub.execute_input":"2024-12-08T20:02:18.861365Z","iopub.status.idle":"2024-12-08T20:02:18.873773Z","shell.execute_reply.started":"2024-12-08T20:02:18.861337Z","shell.execute_reply":"2024-12-08T20:02:18.872911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_dict_children_ass","metadata":{"id":"W3sqhcYdX4mN","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:18.874868Z","iopub.execute_input":"2024-12-08T20:02:18.875691Z","iopub.status.idle":"2024-12-08T20:02:18.889146Z","shell.execute_reply.started":"2024-12-08T20:02:18.875640Z","shell.execute_reply":"2024-12-08T20:02:18.888274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# General information\n\ngeneral_info(df_children_ass, name=\"Bio-electric Impedance Analysis\")\n\n# Classify variables based on type\n\nqualitative_nominal_children_ass = ['CGAS-Season']\n\nqualitative_ordinal_children_ass = []\n\nquantitative_ratio_children_ass = [\n\n    'CGAS-CGAS_Score'\n\n]\n\nquantitative_interval_children_ass = []\n\n\n\n# Analyze numerical columns\n\nnumerical_columns_children_ass = quantitative_ratio_children_ass + quantitative_interval_children_ass\n\nanalyze_numerical(df_children_ass, numerical_columns_children_ass)\n\nplot_numerical_distributions(df_children_ass, numerical_columns_children_ass)\n\n\n\n# Analyze categorical columns\n\ncategorical_columns_children_ass = qualitative_nominal_children_ass + qualitative_ordinal_children_ass\n\nanalyze_categorical(df_children_ass, categorical_columns_children_ass)\n\nplot_categorical_distributions(df_children_ass, categorical_columns_children_ass)","metadata":{"id":"tqSnZ49GYDhG","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:18.890364Z","iopub.execute_input":"2024-12-08T20:02:18.890735Z","iopub.status.idle":"2024-12-08T20:02:19.449944Z","shell.execute_reply.started":"2024-12-08T20:02:18.890696Z","shell.execute_reply":"2024-12-08T20:02:19.449026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check rows where CGAS-Season is missing\n\nmissing_season = df_children_ass['CGAS-Season'].isnull()\n\n\n\n# Check rows where CGAS-CGAS_Score is missing\n\nmissing_score = df_children_ass['CGAS-CGAS_Score'].isnull()\n\n\n\n# Check if all rows missing 'CGAS-Season' are also missing 'CGAS-CGAS_Score'\n\nmissing_season_and_score = missing_season & missing_score\n\ncount_missing_season = missing_season.sum()\n\ncount_missing_both = missing_season_and_score.sum()\n\n\n\nif count_missing_season == count_missing_both:\n\n    print(f\"All {count_missing_season} rows missing 'CGAS-Season' are also missing 'CGAS-CGAS_Score'.\")\n\nelse:\n\n    print(f\"Not all rows missing 'CGAS-Season' are missing 'CGAS-CGAS_Score'.\")\n\n    print(f\"Rows missing 'CGAS-Season': {count_missing_season}\")\n\n    print(f\"Rows missing both: {count_missing_both}\")","metadata":{"id":"z7RMmU7PaO_l","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:19.451287Z","iopub.execute_input":"2024-12-08T20:02:19.451880Z","iopub.status.idle":"2024-12-08T20:02:19.459770Z","shell.execute_reply.started":"2024-12-08T20:02:19.451839Z","shell.execute_reply":"2024-12-08T20:02:19.458864Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Key Insights\n\n\n\n1. **General Information**:\n\n   - The dataset contains **3960 entries** and **2 columns**: one categorical (`CGAS-Season`) and one numerical (`CGAS-CGAS_Score`).\n\n   - **35.48% of rows completely missing values**.\n\n\n\n2. **Missing Values**:\n\n   - `CGAS-Season`: **35.48% missing**.\n\n   - `CGAS-CGAS_Score`: **38.86% missing**.\n\n   - **ALL the values missing in the season are also missing in the score**\n\n3. **Numerical Column Analysis (`CGAS-CGAS_Score`)**:\n\n   - Total non-null values: **2421**.\n\n   - Mean score: **65.45**, with a standard deviation of **22.34**.\n\n   - Minimum: **25**, Maximum: **999** (suggests a possible outlier).\n\n   - Median (50%): **65**, with an interquartile range (IQR) from **59** (25%) to **75** (75%).\n\n\n\n4. **Categorical Column Analysis (`CGAS-Season`)**:\n\n   - Most common season: **Spring (697 entries)**.\n\n   - Distribution across seasons is relatively balanced, with slightly fewer entries for **Winter (567 entries)**.\n\n\n\n### Key Observations:\n\n- **Missing Data**: A significant portion of data is missing, especially in `CGAS-CGAS_Score`.\n\n- **Outlier in Numerical Data**: The maximum value of **999** in `CGAS-CGAS_Score` appears anomalous and warrants investigation.\n\n- **Seasonal Distribution**: Distribution is relatively balanced, with slightly fewer records in Winter.","metadata":{"id":"OVt8avEgaBBk"}},{"cell_type":"markdown","source":"## Filter the Outlier\n","metadata":{"id":"w3kAHZ-VxgFo"}},{"cell_type":"code","source":"cgas_score_outlier = df_train_csv[df_train_csv['CGAS-CGAS_Score']>100]","metadata":{"id":"-yI7g7B3JgL3","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:19.461191Z","iopub.execute_input":"2024-12-08T20:02:19.461573Z","iopub.status.idle":"2024-12-08T20:02:19.477142Z","shell.execute_reply.started":"2024-12-08T20:02:19.461546Z","shell.execute_reply":"2024-12-08T20:02:19.476301Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# drop the outlier without dropping nans\n\ndf_train_csv.drop(cgas_score_outlier.index, inplace=True)","metadata":{"id":"7LhQyqwZJoMy","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:19.478107Z","iopub.execute_input":"2024-12-08T20:02:19.478353Z","iopub.status.idle":"2024-12-08T20:02:19.488919Z","shell.execute_reply.started":"2024-12-08T20:02:19.478329Z","shell.execute_reply":"2024-12-08T20:02:19.488326Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_csv.groupby('Basic_Demos-Age')['CGAS-CGAS_Score'].count()","metadata":{"id":"6CWHQx-NNvut","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:19.489903Z","iopub.execute_input":"2024-12-08T20:02:19.490203Z","iopub.status.idle":"2024-12-08T20:02:19.500591Z","shell.execute_reply.started":"2024-12-08T20:02:19.490172Z","shell.execute_reply":"2024-12-08T20:02:19.499755Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Average CGAS Score by Age Group and SII","metadata":{"id":"VRljG64swbaa"}},{"cell_type":"code","source":"grouped_data = df_train_csv.groupby(['Age Group', 'Severity Impairment Index (SII)'], as_index=False)['CGAS-CGAS_Score'].mean()\n\n\n\ngrouped_data=grouped_data.rename(columns={\n\n    'CGAS-CGAS_Score': 'Average CGAS Score'\n\n})\n\n\n\nsii_mapping = {0: 'None', 1: 'Mild', 2: 'Moderate', 3: 'Severe'}\n\ngrouped_data['Severity Impairment Index (SII)'] = grouped_data['Severity Impairment Index (SII)'].map(sii_mapping)\n\n# Ensure 'SII Level' is categorical and sorted\n\nsii_order = ['None', 'Mild', 'Moderate', 'Severe']\n\n\n\n# --- 2. Grouped Bar Plot ---\n\nplt.figure(figsize=(14, 8))\n\n\n\n# Create the grouped barplot\n\nsns.barplot(\n\n    data=grouped_data,\n\n    x='Age Group',\n\n    y='Average CGAS Score',\n\n    hue='Severity Impairment Index (SII)',\n\n    palette='coolwarm',  # Use a clean color palette\n\n    errorbar=None\n\n)\n\n\n\n# --- 3. Customizations ---\n\n# Title and axis labels\n\nplt.title(\"Average CGAS Score by Age Group and SII\", fontsize=18, fontweight='bold', pad=20)\n\nplt.xlabel(\"Age Group\", fontsize=16, fontweight='bold')\n\nplt.ylabel(\"Average CGAS Score\", fontsize=16, fontweight='bold')\n\n\n\n# Move legend outside the plot\n\nplt.legend(\n\n    title=\"Severity Impairment Index (SII)\",\n\n    title_fontsize=14,\n\n    fontsize=12,\n\n    loc='upper left',\n\n    bbox_to_anchor=(1, 1)  # Move legend to the right of the plot\n\n)\n\n\n\n# Add values above the bars\n\nfor p in plt.gca().patches:\n\n    height = p.get_height()\n\n    if height > 0:  # Avoid text on empty bars\n\n        plt.text(\n\n            p.get_x() + p.get_width() / 2, height + 0.2,  # Slightly above the bar\n\n            f\"{height:.1f}\",\n\n            ha='center', va='bottom',\n\n            fontsize=11, fontweight='bold', color='black'\n\n        )\n\n\n\n# Add gridlines for better readability\n\nplt.grid(axis='y', linestyle='--', alpha=0.5)\n\n\n\n# Adjust layout to make space for legend\n\nplt.tight_layout(rect=[0, 0, 0.85, 1])  # Shrink plot area to accommodate legend\n\n\n\n# --- 4. Save and Show Plot ---\n\nplt.savefig(\"CGAS_vs_AgeGroup_vs_SII.pdf\", dpi=600, bbox_inches='tight')\n\nplt.show()\n","metadata":{"id":"d1-rozFbIy4S","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:19.501817Z","iopub.execute_input":"2024-12-08T20:02:19.502087Z","iopub.status.idle":"2024-12-08T20:02:20.000580Z","shell.execute_reply.started":"2024-12-08T20:02:19.502062Z","shell.execute_reply":"2024-12-08T20:02:19.999763Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Describe: Demographics","metadata":{"id":"va2tvHpyb5rr"}},{"cell_type":"code","source":"# Apply Filters\n\ninstrument_filter = \"Demographics\"\n\ndf_demo, df_dict_demo = filter_by_instrument(df_train_csv, df_dict_csv, instrument_filter)\n\ndf_demo.head()","metadata":{"id":"l8T1oHWRY1kp","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:20.001824Z","iopub.execute_input":"2024-12-08T20:02:20.002234Z","iopub.status.idle":"2024-12-08T20:02:20.012740Z","shell.execute_reply.started":"2024-12-08T20:02:20.002195Z","shell.execute_reply":"2024-12-08T20:02:20.011919Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_dict_demo","metadata":{"id":"xt8f18kXXxbO","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:20.013721Z","iopub.execute_input":"2024-12-08T20:02:20.013998Z","iopub.status.idle":"2024-12-08T20:02:20.031436Z","shell.execute_reply.started":"2024-12-08T20:02:20.013941Z","shell.execute_reply":"2024-12-08T20:02:20.030653Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# General information\n\ngeneral_info(df_demo, name=\"Demographics\")\n\n# Classify variables based on type\n\nqualitative_nominal_demo = ['Basic_Demos-Enroll_Season','Basic_Demos-Sex']\n\nqualitative_ordinal_demo = []\n\nquantitative_ratio_demo = [\n\n    'Basic_Demos-Age'\n\n]\n\nquantitative_interval_demo = []\n\n\n\n# Analyze numerical columns\n\nnumerical_columns_demo = quantitative_ratio_demo + quantitative_interval_demo\n\nanalyze_numerical(df_demo, numerical_columns_demo)\n\nplot_numerical_distributions(df_demo, numerical_columns_demo)\n\n\n\n# Analyze categorical columns\n\ncategorical_columns_demo = qualitative_nominal_demo + qualitative_ordinal_demo\n\nanalyze_categorical(df_demo, categorical_columns_demo)\n\nplot_categorical_distributions(df_demo, categorical_columns_demo)","metadata":{"id":"EqmX1FUMcT2p","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:20.032556Z","iopub.execute_input":"2024-12-08T20:02:20.033290Z","iopub.status.idle":"2024-12-08T20:02:20.620542Z","shell.execute_reply.started":"2024-12-08T20:02:20.033252Z","shell.execute_reply":"2024-12-08T20:02:20.619681Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Describe: FitnessGram Child\n","metadata":{"id":"U8IUeObXdytH"}},{"cell_type":"code","source":"# Apply Filters\n\ninstrument_filter = \"FitnessGram Child\"\n\ndf_fitness_child, df_dict_fitness_child = filter_by_instrument(df_train_csv, df_dict_csv, instrument_filter)\n\ndf_fitness_child.head()","metadata":{"id":"x86Pv4YepucM","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:20.621597Z","iopub.execute_input":"2024-12-08T20:02:20.621977Z","iopub.status.idle":"2024-12-08T20:02:20.645745Z","shell.execute_reply.started":"2024-12-08T20:02:20.621921Z","shell.execute_reply":"2024-12-08T20:02:20.644765Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_dict_fitness_child","metadata":{"id":"HcT_ISAVp1y9","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:20.646983Z","iopub.execute_input":"2024-12-08T20:02:20.647283Z","iopub.status.idle":"2024-12-08T20:02:20.658699Z","shell.execute_reply.started":"2024-12-08T20:02:20.647253Z","shell.execute_reply":"2024-12-08T20:02:20.657748Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# General information\n\ngeneral_info(df_fitness_child, name=\"FitnessGram Child\")\n\n# Classify variables based on type\n\nqualitative_nominal_fitness_child = ['FGC-Season']\n\nqualitative_ordinal_fitness_child = ['FGC-FGC_CU_Zone', 'FGC-FGC_GSND_Zone',\"FGC-FGC_GSD_Zone\", 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL_Zone']\n\nquantitative_ratio_fitness_child = [\n\n    'FGC-FGC_CU', 'FGC-FGC_GSND', 'FGC-FGC_GSD', 'FGC-FGC_PU', 'FGC-FGC_SRL', 'FGC-FGC_SRR', 'FGC-FGC_TL'\n\n]\n\nquantitative_interval_fitness_child = []\n\n\n\n# Analyze numerical columns\n\nnumerical_columns_fitness_child = quantitative_ratio_fitness_child + quantitative_interval_fitness_child\n\nanalyze_numerical(df_fitness_child, numerical_columns_fitness_child)\n\nplot_numerical_distributions(df_fitness_child, numerical_columns_fitness_child)\n\n\n\n# Analyze categorical columns\n\ncategorical_columns_fitness_child = qualitative_nominal_fitness_child + qualitative_ordinal_fitness_child\n\nanalyze_categorical(df_fitness_child, categorical_columns_fitness_child)\n\nplot_categorical_distributions(df_fitness_child, categorical_columns_fitness_child)","metadata":{"id":"9Jy-eheop6Oh","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:20.660066Z","iopub.execute_input":"2024-12-08T20:02:20.660477Z","iopub.status.idle":"2024-12-08T20:02:23.813653Z","shell.execute_reply.started":"2024-12-08T20:02:20.660438Z","shell.execute_reply":"2024-12-08T20:02:23.812770Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Insights","metadata":{"id":"VbbpuMbDrKZg"}},{"cell_type":"markdown","source":"\n\n\n\n1. **Missing Data**:\n\n   - **15.51% of rows** are completely missing.\n\n   - Highest missing percentages:\n\n     - `FGC-FGC_GSND`: **72.88%**.\n\n     - `FGC-FGC_GSND_Zone`: **73.18%**.\n\n   - Lowest missing percentages:\n\n     - `FGC-Season`: **15.51%**.\n\n     - Other numerical columns: **41.31%–43.73%**.\n\n\n\n2. **Numerical Column Analysis**:\n\n   - **General Ranges**:\n\n     - Most metrics have a minimum of **0**.\n\n     - Maximum values range widely (e.g., `FGC-FGC_CU` max: **115**, `FGC-FGC_GSND` max: **124**).\n\n   - **Descriptive Insights**:\n\n     - `FGC-FGC_CU`: Mean **11.26**, Median **9**, StdDev **11.81**.\n\n     - `FGC-FGC_TL`: Mean **9.25**, Median **10**, StdDev **2.99**.\n\n   - **Outliers**:\n\n     - Significant spread in `FGC-FGC_CU` and `FGC-FGC_GSND` suggests possible outliers.\n\n\n\n4. **Categorical Column Analysis**:\n\n   - **Enroll Season (`FGC-Season`)**:\n\n     - Balanced distribution:\n\n       - Spring: **993 entries** (most common).\n\n       - Winter: **746 entries** (least common).\n\n\n\n5. **Zone Value Counts**:\n\n   - **FGC-FGC_CU_Zone**: Evenly split between `0.0` (**1195**) and `1.0` (**1087**).\n\n   - **FGC-FGC_PU_Zone**: Majority `0.0` (**1521**), fewer `1.0` (**750**).\n\n   - **FGC-FGC_SRL_Zone** and **FGC-FGC_SRR_Zone**:\n\n     - `1.0` (1403–1407 entries) is more common than `0.0` (~860 entries).\n\n   - **FGC-FGC_TL_Zone**:\n\n     - `1.0` dominates (**1795 entries**) compared to `0.0` (**490 entries**).\n\n\n\n---\n\n\n\nKey Observations:\n\n1. **Missing Data**:\n\n   - High missingness in columns like `FGC-FGC_GSND` and `FGC-FGC_GSND_Zone` may impact analyses and requires handling.\n\n   - `FGC-Season` has the lowest missingness among all columns.\n\n\n\n2. **Numerical Data**:\n\n   - `FGC-FGC_CU` and `FGC-FGC_GSND` show large variances, suggesting diverse or outlier data points.\n\n   - Most numerical metrics follow expected distributions, with central tendencies clustered near their medians.\n\n\n\n3. **Categorical Data**:\n\n   - Enroll seasons are fairly balanced, with Spring as the most represented season.\n\n   - Zone classifications are generally skewed towards one category (e.g., `1.0` in `FGC-FGC_SRL_Zone`).\n\n\n\n---\n\n\n\nRecommendations:\n\n- **Outlier Detection**:\n\n  - Investigate high variance in metrics like `FGC-FGC_CU` and `FGC-FGC_GSND`.\n\n- **Missing Data Handling**:\n\n  - Consider imputation strategies for missing numerical and categorical values, especially for columns with >40% missingness.\n\n- **Balanced Analysis**:\n\n  - Utilize categorical zones for comparative analyses as their distributions indicate potential separable patterns.","metadata":{"id":"ZIGmA4VOrRb1"}},{"cell_type":"code","source":"import pandas as pd\n\nimport numpy as np\n\nfrom scipy.stats import f_oneway\n\n\n\ndef compute_correlation(df, numerical_columns, categorical_columns):\n\n    \"\"\"\n\n    Analyze the correlation between numerical features and their associated categorical features.\n\n\n\n    Parameters:\n\n        df (pd.DataFrame): The dataset.\n\n        numerical_columns (list): List of numerical column names.\n\n        categorical_columns (list): List of categorical column names.\n\n\n\n    Returns:\n\n        pd.DataFrame: Correlation results including ANOVA F-statistic and Eta-squared.\n\n    \"\"\"\n\n    results = []\n\n\n\n    for num_col, cat_col in zip(numerical_columns, categorical_columns):\n\n        # Drop missing values\n\n        data = df[[num_col, cat_col]].dropna()\n\n\n\n        # Group statistics\n\n        groups = [data[num_col][data[cat_col] == category] for category in data[cat_col].unique()]\n\n\n\n        # ANOVA F-statistic\n\n        f_stat, p_value = f_oneway(*groups)\n\n\n\n        # Eta-squared\n\n        total_var = np.var(data[num_col], ddof=1)\n\n        between_group_var = sum(len(group) * (np.mean(group) - np.mean(data[num_col]))**2 for group in groups) / len(data)\n\n        eta_squared = between_group_var / total_var\n\n\n\n        results.append({\n\n            'Numerical Feature': num_col,\n\n            'Categorical Feature': cat_col,\n\n            'ANOVA F-statistic': f_stat,\n\n            'p-value': p_value,\n\n            'Eta-squared': eta_squared\n\n        })\n\n\n\n    return pd.DataFrame(results)\n\n\n\n# Example Usage\n\nnumerical_columns = ['FGC-FGC_CU', 'FGC-FGC_GSND', 'FGC-FGC_PU', 'FGC-FGC_SRL', 'FGC-FGC_TL']\n\ncategorical_columns = ['FGC-FGC_CU_Zone', 'FGC-FGC_GSND_Zone', 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL_Zone', 'FGC-FGC_TL_Zone']\n\n\n\ncorrelation_results = compute_correlation(df_fitness_child, numerical_columns, categorical_columns)\n\nprint(correlation_results)\n","metadata":{"id":"6at0dBg8rQCW","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:23.814874Z","iopub.execute_input":"2024-12-08T20:02:23.815181Z","iopub.status.idle":"2024-12-08T20:02:23.847822Z","shell.execute_reply.started":"2024-12-08T20:02:23.815154Z","shell.execute_reply":"2024-12-08T20:02:23.847001Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Describe: FitnessGram Vitals and Treadmill\n","metadata":{"id":"jTwhO08-dyqR"}},{"cell_type":"code","source":"# Apply Filters\n\ninstrument_filter = \"FitnessGram Vitals and Treadmill\"\n\ndf_fitness_vitals, df_dict_fitness_vitals = filter_by_instrument(df_train_csv, df_dict_csv, instrument_filter)\n\ndf_fitness_vitals.head()","metadata":{"id":"5PqH8c5DUjgd","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:23.848984Z","iopub.execute_input":"2024-12-08T20:02:23.849345Z","iopub.status.idle":"2024-12-08T20:02:23.860936Z","shell.execute_reply.started":"2024-12-08T20:02:23.849306Z","shell.execute_reply":"2024-12-08T20:02:23.860134Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_dict_fitness_vitals","metadata":{"id":"4S1dkM2VUp3d","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:23.862036Z","iopub.execute_input":"2024-12-08T20:02:23.862321Z","iopub.status.idle":"2024-12-08T20:02:23.884373Z","shell.execute_reply.started":"2024-12-08T20:02:23.862295Z","shell.execute_reply":"2024-12-08T20:02:23.883462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# General information\n\ngeneral_info(df_fitness_vitals, name=\"FitnessGram Child\")\n\n# Classify variables based on type\n\nqualitative_nominal_fitness_vitals = ['Fitness_Endurance-Season']\n\nqualitative_ordinal_fitness_vitals = []\n\nquantitative_ratio_fitness_vitals = [\n\n    'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n\n]\n\nquantitative_interval_fitness_vitals = []\n\n\n\n# Analyze numerical columns\n\nnumerical_columns_fitness_vitals = quantitative_ratio_fitness_vitals + quantitative_interval_fitness_vitals\n\nanalyze_numerical(df_fitness_vitals, numerical_columns_fitness_vitals)\n\nplot_numerical_distributions(df_fitness_vitals, numerical_columns_fitness_vitals)\n\n\n\n# Analyze categorical columns\n\ncategorical_columns_fitness_vitals = qualitative_nominal_fitness_vitals + qualitative_ordinal_fitness_vitals\n\nanalyze_categorical(df_fitness_vitals, categorical_columns_fitness_vitals)\n\nplot_categorical_distributions(df_fitness_vitals, categorical_columns_fitness_vitals)","metadata":{"id":"wFzZlx9UUwt2","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:23.885568Z","iopub.execute_input":"2024-12-08T20:02:23.885917Z","iopub.status.idle":"2024-12-08T20:02:25.042714Z","shell.execute_reply.started":"2024-12-08T20:02:23.885877Z","shell.execute_reply":"2024-12-08T20:02:25.041872Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# add minutes to seconds\n\ndf_fitness_vitals['Fitness_Endurance-Total'] = df_fitness_vitals['Fitness_Endurance-Time_Mins']*60 + df_fitness_vitals['Fitness_Endurance-Time_Sec']\n\ndf_fitness_vitals.drop(columns=['Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec'], inplace=True)","metadata":{"id":"RY04ccJxP_LC","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:25.043796Z","iopub.execute_input":"2024-12-08T20:02:25.044087Z","iopub.status.idle":"2024-12-08T20:02:25.050717Z","shell.execute_reply.started":"2024-12-08T20:02:25.044060Z","shell.execute_reply":"2024-12-08T20:02:25.049801Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Engineer Total Endurance","metadata":{"id":"IoE7i6uTyJ2G"}},{"cell_type":"code","source":"df_train_csv['Fitness_Endurance-Total'] = df_train_csv['Fitness_Endurance-Time_Mins']*60 + df_train_csv['Fitness_Endurance-Time_Sec']\n\ndf_train_csv.drop(columns=['Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec'], inplace=True)","metadata":{"id":"pP7Zcp5KQJfe","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:25.051829Z","iopub.execute_input":"2024-12-08T20:02:25.052220Z","iopub.status.idle":"2024-12-08T20:02:25.064168Z","shell.execute_reply.started":"2024-12-08T20:02:25.052184Z","shell.execute_reply":"2024-12-08T20:02:25.063284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test_csv['Fitness_Endurance-Total'] = df_test_csv['Fitness_Endurance-Time_Mins']*60 + df_test_csv['Fitness_Endurance-Time_Sec']\n\ndf_test_csv.drop(columns=['Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec'], inplace=True)","metadata":{"id":"0DhoSE03yHKP","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:25.065102Z","iopub.execute_input":"2024-12-08T20:02:25.065345Z","iopub.status.idle":"2024-12-08T20:02:25.079381Z","shell.execute_reply.started":"2024-12-08T20:02:25.065322Z","shell.execute_reply":"2024-12-08T20:02:25.078516Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Plot Max Stage with Total Endurance","metadata":{"id":"hCXfLeA4yTZF"}},{"cell_type":"code","source":"# Set a clean theme\n\nsns.set_theme(style=\"whitegrid\")\n\n\n\n# Create the enhanced line plot\n\nplt.figure(figsize=(10, 6))  # Adjust figure size\n\nsns.lineplot(\n\n    data=df_fitness_vitals,\n\n    x='Fitness_Endurance-Max_Stage',\n\n    y='Fitness_Endurance-Total',\n\n    marker='o',       # Add markers to highlight points\n\n    color='#1f77b4',  # Use a professional blue color\n\n    linewidth=2.5,    # Increase line thickness\n\n    linestyle='-',    # Solid line style\n\n)\n\n\n\n# Add titles and labels\n\nplt.title('Fitness Endurance vs. Max Stage', fontsize=18, fontweight='bold', pad=15)\n\nplt.xlabel('Max Stage Reached', fontsize=14, fontweight='bold')\n\nplt.ylabel('Total Endurance Score', fontsize=14, fontweight='bold')\n\n\n\n# Tweak ticks\n\nplt.xticks(fontsize=12)\n\nplt.yticks(fontsize=12)\n\n\n\n# Add gridlines\n\nplt.grid(True, which='both', linestyle='--', linewidth=0.5, alpha=0.7)\n\n\n\n# Remove top and right spines for a cleaner look\n\nsns.despine()\n\n\n\n# Show the final plot\n\nplt.tight_layout()\n\nplt.show()\n","metadata":{"id":"SDZB3u7oW0Xg","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:25.080451Z","iopub.execute_input":"2024-12-08T20:02:25.080707Z","iopub.status.idle":"2024-12-08T20:02:25.567876Z","shell.execute_reply.started":"2024-12-08T20:02:25.080684Z","shell.execute_reply":"2024-12-08T20:02:25.566998Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# age with max stage\n\ndf_train_csv.groupby('Basic_Demos-Age')['Fitness_Endurance-Max_Stage'].mean()","metadata":{"id":"WraQGGksXFaU","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:25.569004Z","iopub.execute_input":"2024-12-08T20:02:25.569285Z","iopub.status.idle":"2024-12-08T20:02:25.576742Z","shell.execute_reply.started":"2024-12-08T20:02:25.569259Z","shell.execute_reply":"2024-12-08T20:02:25.575878Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## This Feature is ONLY calculated for Children","metadata":{"id":"80Qwb4DAyZP2"}},{"cell_type":"markdown","source":"## Remove Outliers\n","metadata":{"id":"7B6uL5hCypfb"}},{"cell_type":"markdown","source":"Max stage bigger than 12 is clearly outliers","metadata":{"id":"k_uwK3VyX2hP"}},{"cell_type":"code","source":"endurance_outliers = df_fitness_vitals[df_fitness_vitals['Fitness_Endurance-Max_Stage']>12]\n\ndf_train_csv.drop(endurance_outliers.index, inplace=True)","metadata":{"id":"VQ9JMe6gXv52","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:25.578003Z","iopub.execute_input":"2024-12-08T20:02:25.578633Z","iopub.status.idle":"2024-12-08T20:02:25.587909Z","shell.execute_reply.started":"2024-12-08T20:02:25.578591Z","shell.execute_reply":"2024-12-08T20:02:25.587163Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Describe: Internet Use\n","metadata":{"id":"aPh_phcsdyfB"}},{"cell_type":"code","source":"# Apply Filters\n\ninstrument_filter = \"Internet Use\"\n\ndf_internet_use, df_dict_internet_use = filter_by_instrument(df_train_csv, df_dict_csv, instrument_filter)\n\ndf_internet_use.head()","metadata":{"id":"ZLmlEDJ7hxab","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:25.588913Z","iopub.execute_input":"2024-12-08T20:02:25.589210Z","iopub.status.idle":"2024-12-08T20:02:25.602688Z","shell.execute_reply.started":"2024-12-08T20:02:25.589184Z","shell.execute_reply":"2024-12-08T20:02:25.601895Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_dict_internet_use","metadata":{"id":"ha4g8KBKiAzQ","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:25.603734Z","iopub.execute_input":"2024-12-08T20:02:25.604054Z","iopub.status.idle":"2024-12-08T20:02:25.617925Z","shell.execute_reply.started":"2024-12-08T20:02:25.604019Z","shell.execute_reply":"2024-12-08T20:02:25.617131Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# General information\n\ngeneral_info(df_internet_use, name=\"Internet Use\")\n\n# Classify variables based on type\n\nqualitative_nominal_internet_use = ['PreInt_EduHx-Season']\n\nqualitative_ordinal_internet_use = ['PreInt_EduHx-computerinternet_hoursday']\n\nquantitative_ratio_internet_use = [\n\n]\n\nquantitative_interval_internet_use= []\n\n\n\n# Analyze categorical columns\n\ncategorical_columns_internet_use = qualitative_nominal_internet_use+ qualitative_ordinal_internet_use\n\nanalyze_categorical(df_internet_use, categorical_columns_internet_use)\n\nplot_categorical_distributions(df_internet_use, categorical_columns_internet_use)","metadata":{"id":"GBxxtHEZiDCX","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:25.618943Z","iopub.execute_input":"2024-12-08T20:02:25.619275Z","iopub.status.idle":"2024-12-08T20:02:26.064103Z","shell.execute_reply.started":"2024-12-08T20:02:25.619237Z","shell.execute_reply":"2024-12-08T20:02:26.063192Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Describe: Physical Activity Questionnaire (Adolescents)\n","metadata":{"id":"z8-BngQIeKcc"}},{"cell_type":"code","source":"# Apply Filters\n\ninstrument_filter = \"Physical Activity Questionnaire (Adolescents)\"\n\ndf_physical_adol, df_dict_physical_adol= filter_by_instrument(df_train_csv, df_dict_csv, instrument_filter)\n\ndf_physical_adol.describe()","metadata":{"id":"rstXAAPEnKoo","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:26.065668Z","iopub.execute_input":"2024-12-08T20:02:26.066035Z","iopub.status.idle":"2024-12-08T20:02:26.079405Z","shell.execute_reply.started":"2024-12-08T20:02:26.065998Z","shell.execute_reply":"2024-12-08T20:02:26.078515Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# General information\n\ngeneral_info(df_physical_adol, name=\"Physical Activity Questionnaire (Adolescents)\")\n\n# Classify variables based on type\n\nqualitative_nominal_physical_adol = ['PAQ_A-Season']\n\nqualitative_ordinal_physical_adol = []\n\nquantitative_ratio_physical_adol = [\n\n    'PAQ_A-PAQ_A_Total'\n\n]\n\nquantitative_interval_physical_adol = []\n\n\n\n# Analyze numerical columns\n\nnumerical_columns_physical_adol = quantitative_ratio_physical_adol + quantitative_interval_physical_adol\n\nanalyze_numerical(df_physical_adol, numerical_columns_physical_adol)\n\nplot_numerical_distributions(df_physical_adol, numerical_columns_physical_adol)\n\n\n\n# Analyze categorical columns\n\ncategorical_columns_physical_adol = qualitative_nominal_physical_adol + qualitative_ordinal_physical_adol\n\nanalyze_categorical(df_physical_adol, categorical_columns_physical_adol)\n\nplot_categorical_distributions(df_physical_adol, categorical_columns_physical_adol)","metadata":{"id":"BSE36EYBpOF_","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:26.080465Z","iopub.execute_input":"2024-12-08T20:02:26.080709Z","iopub.status.idle":"2024-12-08T20:02:26.735711Z","shell.execute_reply.started":"2024-12-08T20:02:26.080685Z","shell.execute_reply":"2024-12-08T20:02:26.734904Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## As expected, for adolescentes, we have data from 13 to 18 years old.","metadata":{"id":"J2jKR1lt9jk_"}},{"cell_type":"code","source":"# group by age\n\ndf_train_csv.groupby('Basic_Demos-Age')['PAQ_A-PAQ_A_Total'].mean()","metadata":{"id":"SuZSTj70q45e","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:26.736757Z","iopub.execute_input":"2024-12-08T20:02:26.737041Z","iopub.status.idle":"2024-12-08T20:02:26.745124Z","shell.execute_reply.started":"2024-12-08T20:02:26.737014Z","shell.execute_reply":"2024-12-08T20:02:26.744244Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# group by age\n\ndf_train_csv.groupby('Basic_Demos-Age')['PAQ_C-PAQ_C_Total'].mean()","metadata":{"id":"RV1ghZKLEOfZ","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:26.746400Z","iopub.execute_input":"2024-12-08T20:02:26.746720Z","iopub.status.idle":"2024-12-08T20:02:26.756141Z","shell.execute_reply.started":"2024-12-08T20:02:26.746693Z","shell.execute_reply":"2024-12-08T20:02:26.755220Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rows_wanted = df_train_csv[df_train_csv['Basic_Demos-Age']>14][['Basic_Demos-Age','PAQ_C-PAQ_C_Total']].dropna()\n\nprint(rows_wanted)\n\n# get PAQ_A-PAQ_A_Total for these rows\n\ndf_train_csv.loc[rows_wanted.index, 'PAQ_A-PAQ_A_Total']","metadata":{"id":"N3w3YVdY-Jdd","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:26.757474Z","iopub.execute_input":"2024-12-08T20:02:26.757848Z","iopub.status.idle":"2024-12-08T20:02:26.774202Z","shell.execute_reply.started":"2024-12-08T20:02:26.757810Z","shell.execute_reply":"2024-12-08T20:02:26.773255Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"As we can see, we have only one mistake.","metadata":{"id":"QIhQ7rYpFXxl"}},{"cell_type":"markdown","source":"## Remove any rows that have children above 16 with PAQ_C-PAQ_C_Total value","metadata":{"id":"GRkhQczUsCrD"}},{"cell_type":"code","source":"rows_to_drop = df_train_csv[df_train_csv['Basic_Demos-Age']>14][['Basic_Demos-Age','PAQ_C-PAQ_C_Total']].dropna()\n\nprint(f\"Rows to drop: {len(rows_to_drop)}\")\n\ndf_train_csv.drop(rows_to_drop.index, inplace=True)","metadata":{"id":"GZdO5boyrqJ8","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:26.775305Z","iopub.execute_input":"2024-12-08T20:02:26.775617Z","iopub.status.idle":"2024-12-08T20:02:26.785237Z","shell.execute_reply.started":"2024-12-08T20:02:26.775591Z","shell.execute_reply":"2024-12-08T20:02:26.784305Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **HOWEVER** we have one data point for a child -13 years old- that has data in both features","metadata":{"id":"L5uYLND5AM2A"}},{"cell_type":"code","source":"df_train_csv[df_train_csv['PAQ_C-PAQ_C_Total'].notna() & df_train_csv['PAQ_A-PAQ_A_Total'].notna()]","metadata":{"id":"Y8QWjiFJ_2Dl","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:26.786276Z","iopub.execute_input":"2024-12-08T20:02:26.786575Z","iopub.status.idle":"2024-12-08T20:02:26.818769Z","shell.execute_reply.started":"2024-12-08T20:02:26.786538Z","shell.execute_reply":"2024-12-08T20:02:26.818001Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Given that it is on the borderline between children and adolescents, we can choose to put it with either.","metadata":{"id":"yIaBSE-mA5t4"}},{"cell_type":"markdown","source":"## Keep the two features but know that we will need to models.","metadata":{"id":"0cbcAYHc9VsV"}},{"cell_type":"code","source":"# # merge PAQ_A-PAQ_A_Total and PAQ_C-PAQ_C_Total based on one of their values\n\n# df_train_csv['PAQ_Total'] = df_train_csv['PAQ_A-PAQ_A_Total'].fillna(df_train_csv['PAQ_C-PAQ_C_Total'])\n\n# df_train_csv = df_train_csv.drop(columns=['PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total'], errors='ignore')","metadata":{"id":"laTflQU69Vf6","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:26.820064Z","iopub.execute_input":"2024-12-08T20:02:26.820335Z","iopub.status.idle":"2024-12-08T20:02:26.823839Z","shell.execute_reply.started":"2024-12-08T20:02:26.820311Z","shell.execute_reply":"2024-12-08T20:02:26.822986Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Describe: Physical Activity Questionnaire (Children)\n","metadata":{"id":"vpQa6PO0eKZx"}},{"cell_type":"code","source":"# Apply Filters\n\ninstrument_filter = \"Physical Activity Questionnaire (Children)\"\n\ndf_physical_children, df_dict_physical_children= filter_by_instrument(df_train_csv, df_dict_csv, instrument_filter)\n\ndf_physical_children.head()","metadata":{"id":"v8Jr1RjLq2-c","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:26.824862Z","iopub.execute_input":"2024-12-08T20:02:26.825628Z","iopub.status.idle":"2024-12-08T20:02:26.839272Z","shell.execute_reply.started":"2024-12-08T20:02:26.825594Z","shell.execute_reply":"2024-12-08T20:02:26.838468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# General information\n\ngeneral_info(df_physical_children, name=\"Physical Activity Questionnaire (Children)\")\n\n# Classify variables based on type\n\nqualitative_nominal_physical_children= ['PAQ_C-Season']\n\nqualitative_ordinal_physical_children = []\n\nquantitative_ratio_physical_children = [\n\n    'PAQ_C-PAQ_C_Total'\n\n]\n\nquantitative_interval_physical_children = []\n\n\n\n# Analyze numerical columns\n\nnumerical_columns_physical_children= quantitative_ratio_physical_children + quantitative_interval_physical_children\n\nanalyze_numerical(df_physical_children, numerical_columns_physical_children)\n\nplot_numerical_distributions(df_physical_children, numerical_columns_physical_children)\n\n\n\n# Analyze categorical columns\n\ncategorical_columns_physical_children = qualitative_nominal_physical_children + qualitative_ordinal_physical_children\n\nanalyze_categorical(df_physical_children, categorical_columns_physical_children)\n\nplot_categorical_distributions(df_physical_children, categorical_columns_physical_children)","metadata":{"id":"S_QRue-m0rIv","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:26.840474Z","iopub.execute_input":"2024-12-08T20:02:26.840693Z","iopub.status.idle":"2024-12-08T20:02:27.458586Z","shell.execute_reply.started":"2024-12-08T20:02:26.840671Z","shell.execute_reply":"2024-12-08T20:02:27.457796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# select all features with season\n\nseason_columns = df_train_csv.columns[df_train_csv.columns.str.contains('Season')]","metadata":{"id":"6b3d41QoCoDK","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:27.459941Z","iopub.execute_input":"2024-12-08T20:02:27.460358Z","iopub.status.idle":"2024-12-08T20:02:27.465588Z","shell.execute_reply.started":"2024-12-08T20:02:27.460315Z","shell.execute_reply":"2024-12-08T20:02:27.464649Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Describe:Physical Measures\n","metadata":{"id":"cBAT9RM4eKSJ"}},{"cell_type":"markdown","source":"## Drop Height and Weight","metadata":{"id":"qKwplcJ01CzK"}},{"cell_type":"markdown","source":"Given that we have data for Bio-electric impedance analysis, I believe we should drop features like BMI, Height and Weight since they are redundent.\n\n[source](https://pmc.ncbi.nlm.nih.gov/articles/PMC6345442/#:~:text=BMI%20is%20the%20most%20commonly,method%20for%20analyzing%20body%20composition.)","metadata":{"id":"Gam0_8nEG7ml"}},{"cell_type":"code","source":"df_train_csv.drop(columns=['Physical-Height', 'Physical-Weight'], inplace=True)","metadata":{"id":"_zclCivADuIV","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:27.466599Z","iopub.execute_input":"2024-12-08T20:02:27.466861Z","iopub.status.idle":"2024-12-08T20:02:27.477687Z","shell.execute_reply.started":"2024-12-08T20:02:27.466836Z","shell.execute_reply":"2024-12-08T20:02:27.477012Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test_csv.drop(columns=['Physical-Height', 'Physical-Weight'], inplace=True)","metadata":{"id":"ehJRVqrp1Arc","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:27.478578Z","iopub.execute_input":"2024-12-08T20:02:27.478787Z","iopub.status.idle":"2024-12-08T20:02:27.489744Z","shell.execute_reply.started":"2024-12-08T20:02:27.478766Z","shell.execute_reply":"2024-12-08T20:02:27.488936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# drop fro df_dict_csv where field is physical bmi\n\ndf_dict_csv = df_dict_csv[df_dict_csv['Field'] != 'Physical-BMI']\n\ndf_dict_csv = df_dict_csv[df_dict_csv['Field'] != 'Physical-Height']\n\ndf_dict_csv = df_dict_csv[df_dict_csv['Field'] != 'Physical-Weight']","metadata":{"id":"-hCX91emHX7k","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:27.490824Z","iopub.execute_input":"2024-12-08T20:02:27.491159Z","iopub.status.idle":"2024-12-08T20:02:27.501674Z","shell.execute_reply.started":"2024-12-08T20:02:27.491133Z","shell.execute_reply":"2024-12-08T20:02:27.500823Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Apply Filters\n\ninstrument_filter = \"Physical Measures\"\n\ndf_physical_measures, df_dict_physical_measures= filter_by_instrument(df_train_csv, df_dict_csv, instrument_filter)\n\ndf_physical_measures.head()","metadata":{"id":"kGIwUiTuBRpE","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:27.502651Z","iopub.execute_input":"2024-12-08T20:02:27.502875Z","iopub.status.idle":"2024-12-08T20:02:27.519614Z","shell.execute_reply.started":"2024-12-08T20:02:27.502854Z","shell.execute_reply":"2024-12-08T20:02:27.518793Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# General information\n\ngeneral_info(df_physical_measures, name=\"Physical Measures\")\n\n# Classify variables based on type\n\nqualitative_nominal_physical_measures = ['Physical-Season']\n\nqualitative_ordinal_physical_measures = []\n\nquantitative_ratio_physical_measures = ['Physical-Waist_Circumference', 'Physical-Systolic_BP', 'Physical-Diastolic_BP', 'Physical-HeartRate']\n\n\n\nquantitative_interval_physical_measures = []\n\n\n\n# Analyze numerical columns\n\nnumerical_columns_physical_measures = quantitative_ratio_physical_measures + quantitative_interval_physical_measures\n\nanalyze_numerical(df_physical_measures, numerical_columns_physical_measures)\n\nplot_numerical_distributions(df_physical_measures, numerical_columns_physical_measures)\n\n\n\n# Analyze categorical columns\n\ncategorical_columns_physical_measures = qualitative_nominal_physical_measures + qualitative_ordinal_physical_measures\n\nanalyze_categorical(df_physical_measures, categorical_columns_physical_measures)\n\nplot_categorical_distributions(df_physical_measures, categorical_columns_physical_measures)","metadata":{"id":"nsYvBUtjHtMQ","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:27.520655Z","iopub.execute_input":"2024-12-08T20:02:27.520906Z","iopub.status.idle":"2024-12-08T20:02:29.623892Z","shell.execute_reply.started":"2024-12-08T20:02:27.520882Z","shell.execute_reply":"2024-12-08T20:02:29.622971Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Identifying outliers","metadata":{"id":"LKLJ8kl25kt_"}},{"cell_type":"code","source":"# Identifying outliers\n\ndiastolic_outliers = df_physical_measures[df_physical_measures['Physical-Diastolic_BP'] < 40]\n\nsystolic_outliers = df_physical_measures[df_physical_measures['Physical-Systolic_BP'] < 50]\n\nlow_heart_rate = df_physical_measures.loc[df_physical_measures['Physical-HeartRate'] < 50, 'Physical-HeartRate']\n\n\n\n# Plot 1: Distribution of Diastolic BP with Highlighted Outliers\n\nplt.figure(figsize=(8, 5))\n\nsns.histplot(df_physical_measures['Physical-Diastolic_BP'], bins=30, color='blue', label='All Values', kde=True)\n\nplt.axvline(40, color='red', linestyle='--', label='Outlier Threshold (40 mmHg)')\n\nplt.title('Distribution of Physical-Diastolic_BP with Outliers Highlighted')\n\nplt.xlabel('Diastolic Blood Pressure (mmHg)')\n\nplt.ylabel('Frequency')\n\nplt.legend()\n\nplt.show()\n\n\n\n# Plot 2: Distribution of Systolic BP with Highlighted Outliers\n\nplt.figure(figsize=(8, 5))\n\nsns.histplot(df_physical_measures['Physical-Systolic_BP'], bins=30, color='green', label='All Values', kde=True)\n\nplt.axvline(50, color='red', linestyle='--', label='Outlier Threshold (50 mmHg)')\n\nplt.title('Distribution of Physical-Systolic_BP with Outliers Highlighted')\n\nplt.xlabel('Systolic Blood Pressure (mmHg)')\n\nplt.ylabel('Frequency')\n\nplt.legend()\n\nplt.show()\n\n\n\n# Improved Plot 3: Histogram with Highlighted Outliers for Heart Rate\n\n\n\nplt.figure(figsize=(8, 5))\n\nsns.histplot(df_physical_measures['Physical-HeartRate'], bins=30, color='orange', label='All Values', kde=True)\n\nplt.axvline(50, color='red', linestyle='--', label='Outlier Threshold (50 bpm)')\n\nplt.title('Distribution of Physical-HeartRate with Outliers Highlighted')\n\nplt.xlabel('Heart Rate (bpm)')\n\nplt.ylabel('Frequency')\n\nplt.legend()\n\nplt.show()\n\n\n\n\n","metadata":{"id":"Iwrd-fyT51Ta","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:29.625260Z","iopub.execute_input":"2024-12-08T20:02:29.625901Z","iopub.status.idle":"2024-12-08T20:02:30.748870Z","shell.execute_reply.started":"2024-12-08T20:02:29.625860Z","shell.execute_reply":"2024-12-08T20:02:30.748104Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Convert Outliers to NaN","metadata":{"id":"gY5N2Y391npW"}},{"cell_type":"code","source":"\n\ndf_train_csv.loc[df_train_csv['Physical-Diastolic_BP']<50, 'Physical-Diastolic_BP'] = np.nan\n\ndf_train_csv.loc[df_train_csv['Physical-HeartRate']<50, 'Physical-HeartRate'] = np.nan\n\ndf_train_csv.loc[df_train_csv['Physical-Systolic_BP']<50, 'Physical-Systolic_BP'] = np.nan\n","metadata":{"id":"pXhVaSA51VnX","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:30.750117Z","iopub.execute_input":"2024-12-08T20:02:30.750465Z","iopub.status.idle":"2024-12-08T20:02:30.759141Z","shell.execute_reply.started":"2024-12-08T20:02:30.750428Z","shell.execute_reply":"2024-12-08T20:02:30.758016Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Insights","metadata":{"id":"avGwH4vu6-Oq"}},{"cell_type":"markdown","source":"\n\n2. **Physical-Diastolic_BP**:\n\n   - Values like `20.0` and especially `11.0` are **critically low**, even for children. These may be measurement errors or outliers. Context is crucial:\n\n     - During exercise, diastolic pressure may decrease slightly, but it doesn’t reach such low levels in healthy individuals.\n\n     - If you have no way to validate the data, consider setting these values to `NaN`.\n\n\n\n3. **Physical-HeartRate**:\n\n   - Values like `39.0` bpm appear **unrealistic** for an 8–10-year-old during exercise. A normal exercise heart rate for children in this age range is typically **120–160 bpm**, depending on intensity.\n\n   - These extreme values (`39.0`) are likely errors and should be set to `NaN`.\n\n\n\n4. **Physical-Systolic_BP**:\n\n   - Values like `49.0` mmHg are highly unusual and almost certainly erroneous for a child.\n\n   - Children typically have systolic pressures between **90–120 mmHg**, and exercise can increase this slightly. `49.0` likely needs correction or should be set to `NaN`.\n","metadata":{"id":"Jj5gxD5i6_se"}},{"cell_type":"markdown","source":"# Describe: Physical Activity Questionnaire (Sleep Disturbance Scale)\n","metadata":{"id":"8rZ6YUs0edn-"}},{"cell_type":"code","source":"# Apply Filters\n\ninstrument_filter = \"Sleep Disturbance Scale\"\n\ndf_sleep, df_dict_sleep = filter_by_instrument(df_train_csv, df_dict_csv, instrument_filter)\n\ndf_sleep.head()","metadata":{"id":"6vGthjrFzqQa","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:30.760267Z","iopub.execute_input":"2024-12-08T20:02:30.760583Z","iopub.status.idle":"2024-12-08T20:02:30.775131Z","shell.execute_reply.started":"2024-12-08T20:02:30.760549Z","shell.execute_reply":"2024-12-08T20:02:30.774264Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_sleep","metadata":{"id":"vo8ZxjUG0AbX","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:30.776192Z","iopub.execute_input":"2024-12-08T20:02:30.776450Z","iopub.status.idle":"2024-12-08T20:02:30.794457Z","shell.execute_reply.started":"2024-12-08T20:02:30.776426Z","shell.execute_reply":"2024-12-08T20:02:30.793646Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# General information\n\ngeneral_info(df_sleep, name=\"Sleep Disturbance Scale\")\n\n# Classify variables based on type\n\nqualitative_nominal_sleep = ['SDS-Season']\n\nqualitative_ordinal_sleep = []\n\nquantitative_ratio_sleep = [\n\n    'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T'\n\n]\n\nquantitative_interval_sleep = []\n\n\n\n# Analyze numerical columns\n\nnumerical_columns_sleep = quantitative_ratio_sleep + quantitative_interval_sleep\n\nanalyze_numerical(df_sleep, numerical_columns_sleep)\n\nplot_numerical_distributions(df_sleep, numerical_columns_sleep)\n\n\n\n# Analyze categorical columns\n\ncategorical_columns_sleep= qualitative_nominal_sleep + qualitative_ordinal_sleep\n\nanalyze_categorical(df_sleep, categorical_columns_sleep)\n\nplot_categorical_distributions(df_sleep, categorical_columns_sleep)","metadata":{"id":"ESx6ooXY0Clk","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:30.803321Z","iopub.execute_input":"2024-12-08T20:02:30.803556Z","iopub.status.idle":"2024-12-08T20:02:31.640146Z","shell.execute_reply.started":"2024-12-08T20:02:30.803526Z","shell.execute_reply":"2024-12-08T20:02:31.639276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# Set a clean theme\n\nsns.set_theme(style=\"whitegrid\")\n\n\n\n# Create the scatter plot with enhancements\n\nplt.figure(figsize=(10, 6))  # Adjust figure size\n\nsns.scatterplot(\n\n    data=df_sleep,\n\n    x='SDS-SDS_Total_T',\n\n    y='SDS-SDS_Total_Raw',\n\n    color='#1f77b4',  # Professional blue color\n\n    alpha=0.7,        # Transparency for overlapping points\n\n    s=70,             # Marker size\n\n    edgecolor='black' # Add black edge to points for clarity\n\n)\n\n\n\n# Add a regression line for trend\n\nsns.regplot(\n\n    data=df_sleep,\n\n    x='SDS-SDS_Total_T',\n\n    y='SDS-SDS_Total_Raw',\n\n    scatter=False,    # Don't overlap points\n\n    color='red',      # Trend line color\n\n    line_kws={\"linewidth\": 2, \"linestyle\": '--'}  # Line styling\n\n)\n\n\n\n# Add titles and axis labels\n\nplt.title('Relationship Between Sleep Disturbance Scale (SDS) Total and Raw', fontsize=18, fontweight='bold', pad=15)\n\nplt.xlabel('SDS Total Score', fontsize=14, fontweight='bold')\n\nplt.ylabel('SDS Raw Score', fontsize=14, fontweight='bold')\n\n\n\n# Customize ticks\n\nplt.xticks(fontsize=12)\n\nplt.yticks(fontsize=12)\n\n\n\n# Add gridlines\n\nplt.grid(True, linestyle='--', linewidth=0.5, alpha=0.7)\n\n\n\n# Remove top and right spines for a cleaner look\n\nsns.despine()\n\n\n\n# Display the final plot\n\nplt.tight_layout()\n\nplt.show()\n","metadata":{"id":"qfe2vWgW02Gz","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:31.641353Z","iopub.execute_input":"2024-12-08T20:02:31.641725Z","iopub.status.idle":"2024-12-08T20:02:32.148017Z","shell.execute_reply.started":"2024-12-08T20:02:31.641686Z","shell.execute_reply":"2024-12-08T20:02:32.147170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"SDS Total Score Range:\", df_sleep['SDS-SDS_Total_T'].min(), \"-\", df_sleep['SDS-SDS_Total_T'].max())\n\nprint(\"SDS Raw Score Range:\", df_sleep['SDS-SDS_Total_Raw'].min(), \"-\", df_sleep['SDS-SDS_Total_Raw'].max())\n","metadata":{"id":"A-xfh2gD2Xif","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:32.149236Z","iopub.execute_input":"2024-12-08T20:02:32.149889Z","iopub.status.idle":"2024-12-08T20:02:32.156751Z","shell.execute_reply.started":"2024-12-08T20:02:32.149848Z","shell.execute_reply":"2024-12-08T20:02:32.155875Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Drop Raw Score","metadata":{"id":"dmCDnGAt3Ln1"}},{"cell_type":"code","source":"df_train_csv.drop(columns=['SDS-SDS_Total_Raw'], inplace=True)\n\ndf_test_csv.drop(columns=['SDS-SDS_Total_Raw'], inplace=True)","metadata":{"id":"FHGWcokK3OO7","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:32.157870Z","iopub.execute_input":"2024-12-08T20:02:32.158222Z","iopub.status.idle":"2024-12-08T20:02:32.171432Z","shell.execute_reply.started":"2024-12-08T20:02:32.158185Z","shell.execute_reply":"2024-12-08T20:02:32.170778Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Remove Outliers","metadata":{"id":"iQFp5sSFUM9g"}},{"cell_type":"code","source":"\n\ndef remove_outliers_zscore_keep_nan(df, columns, threshold=3):\n\n    \"\"\"\n\n    Removes outliers using the Z-score method, while retaining NaN rows untouched.\n\n\n\n    Parameters:\n\n        df (pd.DataFrame): The input DataFrame.\n\n        columns (list): List of numerical columns to check for outliers.\n\n        threshold (float): Z-score threshold to consider values as outliers.\n\n\n\n    Returns:\n\n        pd.DataFrame: DataFrame without outliers, keeping NaN rows untouched.\n\n    \"\"\"\n\n    df = df.copy()  # Work on a copy to avoid modifying the original DataFrame\n\n\n\n    for col in columns:\n\n        if col in df.columns:\n\n            # Compute Z-scores, ignoring NaN\n\n            z_scores = zscore(df[col], nan_policy='omit')\n\n\n\n            # Create mask for non-outliers and NaN values\n\n            outlier_mask = (z_scores.abs() <= threshold) | df[col].isna()\n\n\n\n            # Apply mask to column (retain NaNs)\n\n            df = df[outlier_mask]\n\n\n\n    return df\n","metadata":{"id":"aFLXK7mqNIKP","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:32.172492Z","iopub.execute_input":"2024-12-08T20:02:32.172883Z","iopub.status.idle":"2024-12-08T20:02:32.180755Z","shell.execute_reply.started":"2024-12-08T20:02:32.172843Z","shell.execute_reply":"2024-12-08T20:02:32.179999Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import pandas as pd\n\n# import numpy as np\n\n# from scipy.stats import ttest_ind, mannwhitneyu, chi2_contingency\n\n# import matplotlib.pyplot as plt\n\n# import seaborn as sns\n\n\n\n\n\n# # Remove outliers from numerical features\n\n# numerical_features = df_train_csv.select_dtypes(include=[np.number]).columns\n\n# df_cleaned = remove_outliers_zscore_keep_nan(df_train_csv, numerical_features)\n\n# # Separate groups\n\n# children = df_cleaned[df_cleaned['age_group'] == 'child']\n\n# teens = df_cleaned[df_cleaned['age_group'] == 'teen']\n\n\n\n# # Numerical Features: Compare statistics\n\n# numerical_features = df_train_csv.select_dtypes(include=[np.number]).columns\n\n# for feature in numerical_features:\n\n#     print(f\"Feature: {feature}\")\n\n#     print(f\"Child Mean: {children[feature].mean()}, Teen Mean: {teens[feature].mean()}\")\n\n\n\n#     # Perform statistical tests\n\n#     _, p_value_ttest = ttest_ind(children[feature].dropna(), teens[feature].dropna(), equal_var=False)\n\n#     try:\n\n#       _, p_value_mannwhitney = mannwhitneyu(children[feature].dropna(), teens[feature].dropna())\n\n\n\n#       print(f\"T-test p-value: {p_value_ttest:.5f}, Mann-Whitney U p-value: {p_value_mannwhitney:.5f}\")\n\n#       print()\n\n#     except:\n\n#       print(f\"T-test p-value: {p_value_ttest:.5f}\")\n\n#       print()\n\n# # Categorical Features: Compare distributions\n\n# categorical_features = df_cleaned.select_dtypes(include=[object, 'category']).columns\n\n# for feature in categorical_features:\n\n#     if feature == 'age_group':  # Skip age_group itself\n\n#         continue\n\n\n\n#     print(f\"Feature: {feature}\")\n\n#     child_counts = children[feature].value_counts(normalize=True)\n\n#     teen_counts = teens[feature].value_counts(normalize=True)\n\n#     print(\"Child Distribution:\\n\", child_counts)\n\n#     print(\"Teen Distribution:\\n\", teen_counts)\n\n\n\n#     # Prepare for Chi-square test\n\n#     contingency_table = pd.crosstab(df_cleaned['age_group'], df_cleaned[feature])\n\n#     _, p_value_chi2, _, _ = chi2_contingency(contingency_table)\n\n#     print(f\"Chi-square p-value: {p_value_chi2:.5f}\")\n\n#     print()\n\n\n\n# # Visualization\n\n# # Boxplot for numerical features\n\n# for feature in numerical_features:\n\n#     plt.figure(figsize=(8, 4))\n\n#     sns.boxplot(data=df_cleaned, x='age_group', y=feature)\n\n#     plt.title(f\"Boxplot of {feature} by Age Group\")\n\n#     plt.show()\n\n\n\n# # Bar plot for categorical features\n\n# for feature in categorical_features:\n\n#     if feature == 'age_group':\n\n#         continue\n\n#     plt.figure(figsize=(8, 4))\n\n#     sns.countplot(data=df_cleaned, x=feature, hue='age_group')\n\n#     plt.title(f\"Distribution of {feature} by Age Group\")\n\n#     plt.show()\n","metadata":{"id":"y331eD2FLSyl","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:32.181666Z","iopub.execute_input":"2024-12-08T20:02:32.181900Z","iopub.status.idle":"2024-12-08T20:02:32.195304Z","shell.execute_reply.started":"2024-12-08T20:02:32.181876Z","shell.execute_reply":"2024-12-08T20:02:32.194569Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# drop all columns that has word season\n","metadata":{"id":"epuqLusZt_4L"}},{"cell_type":"code","source":"df_train_csv = df_train_csv.drop(columns=[col for col in df_train_csv.columns if 'Season' in col])\n\ndf_test_csv = df_test_csv.drop(columns=[col for col in df_test_csv.columns if 'Season' in col])","metadata":{"id":"L_FqQ7Fet_T-","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:32.196315Z","iopub.execute_input":"2024-12-08T20:02:32.196585Z","iopub.status.idle":"2024-12-08T20:02:32.209588Z","shell.execute_reply.started":"2024-12-08T20:02:32.196559Z","shell.execute_reply":"2024-12-08T20:02:32.208972Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Encoding","metadata":{"id":"-pAuzw4o8jwQ"}},{"cell_type":"code","source":"# Combine all features for each data type\n\nratio_features = (\n\n    quantitative_ratio_bia +\n\n    quantitative_ratio_children_ass +\n\n    quantitative_ratio_demo +\n\n    quantitative_ratio_fitness_child +\n\n    quantitative_ratio_fitness_vitals +\n\n    quantitative_ratio_internet_use +\n\n    quantitative_ratio_physical_measures\n\n)\n\n\n\ninterval_features = (\n\n    quantitative_interval_bia +\n\n    quantitative_interval_children_ass +\n\n    quantitative_interval_demo +\n\n    quantitative_interval_fitness_child +\n\n    quantitative_interval_fitness_vitals +\n\n    quantitative_interval_internet_use +\n\n    quantitative_interval_physical_measures\n\n)\n\n\n\nnominal_features = (\n\n    qualitative_nominal_bia +\n\n    qualitative_nominal_children_ass +\n\n    qualitative_nominal_demo +\n\n    qualitative_nominal_fitness_child +\n\n    qualitative_nominal_fitness_vitals +\n\n    qualitative_nominal_internet_use +\n\n    qualitative_nominal_physical_children +\n\n    qualitative_nominal_physical_measures\n\n)\n\n\n\nordinal_features = (\n\n    qualitative_ordinal_bia +\n\n    qualitative_ordinal_children_ass +\n\n    qualitative_ordinal_demo +\n\n    qualitative_ordinal_fitness_child +\n\n    qualitative_ordinal_fitness_vitals +\n\n    qualitative_ordinal_internet_use +\n\n    qualitative_ordinal_physical_adol +\n\n    qualitative_ordinal_physical_children +\n\n    qualitative_ordinal_physical_measures\n\n)\n\n# Features to remove\n\nfeatures_to_remove = ['BIA-BIA_BMI', 'SDS-SDS_Total_Raw', 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec']\n\n\n\n# Features to add\n\nfeatures_to_add = ['Fitness_Endurance-Total', 'Calculated-BMI']\n\n\n\n# Update quantitative_ratio_features\n\nratio_features = [\n\n    feature for feature in ratio_features if feature not in features_to_remove\n\n]\n\n\n\n# Add new features to the list\n\nratio_features.extend(features_to_add)\n\n# Remove any feature containing 'season' from qualitative_nominal_features\n\nnominal_features = [\n\n    feature for feature in nominal_features if 'season' not in feature.lower()\n\n]\n\n# Verify the updated list\n\nprint(\"Updated quantitative_ratio_features:\", ratio_features)\n\n\n\n\n\nnumerical_features = ratio_features + interval_features\n\ncategorical_features = nominal_features + ordinal_features\n\nprint(\"Numerical Features are:\", numerical_features)\n\nprint(\"Categorical Features are:\", categorical_features)","metadata":{"id":"BwmU_QCt4jpA","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:32.210449Z","iopub.execute_input":"2024-12-08T20:02:32.210693Z","iopub.status.idle":"2024-12-08T20:02:32.220862Z","shell.execute_reply.started":"2024-12-08T20:02:32.210669Z","shell.execute_reply":"2024-12-08T20:02:32.219799Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create a copy of the dataframe to avoid modifying the original\n\ndf_encoded = df_train_csv.copy()\n\n\n\nnominal_features = [feature for feature in categorical_features if feature not in ordinal_features]\n\n\n\n# 1. Handle Ordinal Features\n\n# First, impute missing values with the most frequent value\n\nfor feature in ordinal_features:\n\n    # Create an imputer\n\n    imputer = SimpleImputer(strategy='most_frequent')\n\n\n\n    # Reshape the column for imputation\n\n    imputed_column = imputer.fit_transform(df_encoded[[feature]])\n\n\n\n    # Replace the column with imputed values\n\n    df_encoded[feature] = imputed_column\n\n\n\n# Dynamically get categories for each ordinal feature\n\nordinal_categories = [sorted(df_encoded[feature].unique()) for feature in ordinal_features]\n\n\n\n# Now apply OrdinalEncoder with dynamically determined categories\n\nordinal_encoder = OrdinalEncoder(categories=ordinal_categories)\n\ndf_encoded[ordinal_features] = ordinal_encoder.fit_transform(df_encoded[ordinal_features])\n\n\n\n# 2. Handle Nominal Features (One-Hot Encoding)\n\n# Create a OneHotEncoder with handle_unknown='ignore'\n\nonehot_encoder = OneHotEncoder(drop='first',sparse_output=False)\n\n\n\n# Fit and transform nominal features\n\nonehot_encoded = onehot_encoder.fit_transform(df_encoded[nominal_features])\n\n\n\n# Create column names for one-hot encoded features\n\nonehot_columns = onehot_encoder.get_feature_names_out(nominal_features)\n\n\n\n# Convert to DataFrame\n\nonehot_df = pd.DataFrame(\n\n    onehot_encoded,\n\n    columns=onehot_columns,\n\n    index=df_encoded.index\n\n)\n\n\n\n\n\n# Convert specified features to categorical\n\nfor feature in categorical_features:\n\n    df_encoded[feature] = df_encoded[feature].astype('category')\n\n# 3. Combine the encoded features\n\n# Remove original categorical columns\n\ndf_final = df_encoded.drop(columns=categorical_features)\n\n\n\n# Add encoded columns\n\ndf_train_csv = pd.concat([df_final, df_encoded[ordinal_features], onehot_df], axis=1)","metadata":{"id":"IqwgxM9q9tql","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:32.222066Z","iopub.execute_input":"2024-12-08T20:02:32.222831Z","iopub.status.idle":"2024-12-08T20:02:32.285895Z","shell.execute_reply.started":"2024-12-08T20:02:32.222794Z","shell.execute_reply":"2024-12-08T20:02:32.285087Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# apply same on test set\n\n# Create a copy of the dataframe to avoid modifying the original\n\ndf_encoded = df_test_csv.copy()\n\n\n\nnominal_features = [feature for feature in categorical_features if feature not in ordinal_features]\n\n\n\n# 1. Handle Ordinal Features\n\n# First, impute missing values with the most frequent value\n\nfor feature in ordinal_features:\n\n    # Create an imputer\n\n    imputer = SimpleImputer(strategy='most_frequent')\n\n\n\n    # Reshape the column for imputation\n\n    imputed_column = imputer.fit_transform(df_encoded[[feature]])\n\n\n\n    # Replace the column with imputed values\n\n    df_encoded[feature] = imputed_column\n\n\n\n# Dynamically get categories for each ordinal feature\n\nordinal_categories = [sorted(df_encoded[feature].unique()) for feature in ordinal_features]\n\n\n\n# Now apply OrdinalEncoder with dynamically determined categories\n\nordinal_encoder = OrdinalEncoder(categories=ordinal_categories)\n\ndf_encoded[ordinal_features] = ordinal_encoder.fit_transform(df_encoded[ordinal_features])\n\n\n\n# 2. Handle Nominal Features (One-Hot Encoding)\n\n# Create a OneHotEncoder with handle_unknown='ignore'\n\nonehot_encoder = OneHotEncoder(drop='first',sparse_output=False)\n\n\n\n# Fit and transform nominal features\n\nonehot_encoded = onehot_encoder.fit_transform(df_encoded[nominal_features])\n\n\n\n# Create column names for one-hot encoded features\n\nonehot_columns = onehot_encoder.get_feature_names_out(nominal_features)\n\n\n\n# Convert to DataFrame\n\nonehot_df = pd.DataFrame(\n\n    onehot_encoded,\n\n    columns=onehot_columns,\n\n    index=df_encoded.index\n\n)\n\n\n\n\n\n# Convert specified features to categorical\n\nfor feature in categorical_features:\n\n    df_encoded[feature] = df_encoded[feature].astype('category')\n\n# 3. Combine the encoded features\n\n# Remove original categorical columns\n\ndf_final = df_encoded.drop(columns=categorical_features)\n\n\n\n# Add encoded columns\n\ndf_test_csv = pd.concat([df_final, df_encoded[ordinal_features], onehot_df], axis=1)","metadata":{"id":"P9mK8hCsBhMm","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:32.287147Z","iopub.execute_input":"2024-12-08T20:02:32.287425Z","iopub.status.idle":"2024-12-08T20:02:32.338233Z","shell.execute_reply.started":"2024-12-08T20:02:32.287393Z","shell.execute_reply":"2024-12-08T20:02:32.337377Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Split the Dataset into Two","metadata":{"id":"BOgLeFgk3bNG"}},{"cell_type":"code","source":"# split to children and teen datasets\n\ndf_train_children = df_train_csv[df_train_csv['Age Group'] == 'Child']\n\ndf_train_teens = df_train_csv[df_train_csv['Age Group'] == 'Teen']\n\ndf_test_children = df_test_csv[df_test_csv['Age Group'] == 'Child']\n\ndf_test_teens = df_test_csv[df_test_csv['Age Group'] == 'Teen']","metadata":{"id":"yvL7xzXI3dLb","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:32.339292Z","iopub.execute_input":"2024-12-08T20:02:32.340118Z","iopub.status.idle":"2024-12-08T20:02:32.348645Z","shell.execute_reply.started":"2024-12-08T20:02:32.340077Z","shell.execute_reply":"2024-12-08T20:02:32.347660Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Drop Irrelevent Columns","metadata":{"id":"T9ooi0j14UJq"}},{"cell_type":"code","source":"# Step 1: Drop columns with all NaN values\n\n# For Teens\n\nteens_col_drop = df_train_teens.columns[df_train_teens.isna().all()].tolist()\n\nprint(\"Teens columns to drop:\", teens_col_drop)\n\ndf_train_teens.drop(columns=teens_col_drop, inplace=True)\n\ndf_test_teens.drop(columns=teens_col_drop, inplace=True)\n\n\n\n# For Children\n\nchildren_col_drop = df_test_children.columns[df_test_children.isna().all()].tolist()\n\nprint(\"Children columns to drop:\", children_col_drop)\n\ndf_train_children.drop(columns=children_col_drop, inplace=True)\n\ndf_test_children.drop(columns=children_col_drop, inplace=True)","metadata":{"id":"NyvzkDKs3nAC","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:32.349785Z","iopub.execute_input":"2024-12-08T20:02:32.350382Z","iopub.status.idle":"2024-12-08T20:02:32.370714Z","shell.execute_reply.started":"2024-12-08T20:02:32.350337Z","shell.execute_reply":"2024-12-08T20:02:32.369677Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Drop ID and Age Group","metadata":{"id":"XkIEm5ch-E6J"}},{"cell_type":"code","source":"# Step 2: Drop 'id' and 'Age Group' only from the training datasets\n\ndf_train_children.drop(columns=['id', 'Age Group'], inplace=True)\n\ndf_train_teens.drop(columns=['id', 'Age Group'], inplace=True)\n\n\n\n# Retain 'id' in test datasets but drop 'Age Group'\n\ndf_test_children.drop(columns=['Age Group'], inplace=True)\n\ndf_test_teens.drop(columns=['Age Group'], inplace=True)","metadata":{"id":"DmlsiDuf-F83","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:32.372033Z","iopub.execute_input":"2024-12-08T20:02:32.372427Z","iopub.status.idle":"2024-12-08T20:02:32.383603Z","shell.execute_reply.started":"2024-12-08T20:02:32.372382Z","shell.execute_reply":"2024-12-08T20:02:32.382627Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Drop Features with over 90% correlation","metadata":{"id":"0Ary1mauQWz8"}},{"cell_type":"code","source":"# Define a mapping for the columns\n\nname_mapping = {\n\n    \"Physical-Waist_Circumference\": \"Waist Circumference\",\n\n    \"Fitness_Endurance-Max_Stage\": \"Max Endurance Stage\",\n\n    \"FGC-FGC_SRL\": \"Sit & Reach Left\",\n\n    \"BIA-BIA_BMC\": \"Body Mass Content\",\n\n    \"BIA-BIA_BMR\": \"Basal Metabolic Rate\",\n\n    \"BIA-BIA_DEE\": \"Daily Energy Expenditure\",\n\n    \"BIA-BIA_ECW\": \"Extracellular Water\",\n\n    \"BIA-BIA_FFM\": \"Fat-Free Mass\",\n\n    \"BIA-BIA_Fat\": \"Body Fat Percentage\",\n\n    \"BIA-BIA_ICW\": \"Intracellular Water\",\n\n    \"BIA-BIA_LDM\": \"Lean Dry Mass\",\n\n    \"BIA-BIA_LST\": \"Lean Soft Tissue\",\n\n    \"BIA-BIA_SMM\": \"Skeletal Muscle Mass\",\n\n    \"Basic_Demos-Sex_0\": \"Sex (Male)\",\n\n    \"Basic_Demos-Sex_1\": \"Sex (Female)\",\n\n    \"BIA-BIA_TBW\": \"Total Body Water\",\n\n    \"Fitness_Endurance-Total\": \"Total Endurance\",\n\n    \"FGC-FGC_SRR\": \"Sit & Reach total (right side)\",\n\n    \"FGC-FGC_GSD\": \"Grip Strength Total (dominant)\",\n\n    \"FGC-FGC_GSND\": \"Grip Strength Total (non-dominant)\",\n\n    \"Calculated-BMI\": \"BMI\",\n\n    \"BIA-BIA_FFMI\":\"Fat Free Mass Index\"\n\n\n\n}","metadata":{"id":"ESub53OvRahm","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:32.384664Z","iopub.execute_input":"2024-12-08T20:02:32.384975Z","iopub.status.idle":"2024-12-08T20:02:32.395525Z","shell.execute_reply.started":"2024-12-08T20:02:32.384917Z","shell.execute_reply":"2024-12-08T20:02:32.394732Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# Step 2: Function to Filter Correlations > 0.9\n\ndef filter_high_correlation(dataframe):\n\n    correlation_matrix = dataframe.corr()\n\n    high_corr_pairs = correlation_matrix.where(np.triu(np.abs(correlation_matrix) > 0.9, k=1))\n\n    filtered_corr = high_corr_pairs.dropna(how='all').dropna(axis=1, how='all')\n\n    return filtered_corr.rename(columns=name_mapping, index=name_mapping)\n\n\n\n# Step 3: Filter High Correlation for Children and Teens\n\nchildren_corr = filter_high_correlation(df_train_children)\n\nteens_corr = filter_high_correlation(df_train_teens)\n\n\n\n# Step 4: Calculate Unified Color Scale\n\nvmin = min(children_corr.min().min(), teens_corr.min().min())  # Global minimum correlation\n\nvmax = max(children_corr.max().max(), teens_corr.max().max())  # Global maximum correlation\n\n\n\n# Step 5: Plot Heatmaps Side by Side\n\nfig, axes = plt.subplots(1, 2, figsize=(25, 12))\n\n\n\n# Heatmap for Children\n\nsns.heatmap(\n\n    children_corr,\n\n    annot=True,\n\n    cmap=\"coolwarm\",\n\n    vmin=vmin, vmax=vmax,  # Set unified color scale\n\n    fmt=\".2f\",\n\n    linewidths=0.5,\n\n    linecolor=\"black\",\n\n    cbar_kws={\"shrink\": 0.8},\n\n    ax=axes[0],\n\n    square=True\n\n)\n\naxes[0].set_title(\"Features with >90% Correlation (Children)\", fontsize=16, fontweight='bold')\n\naxes[0].tick_params(axis='x', rotation=45)\n\n\n\n# Heatmap for Teens\n\nsns.heatmap(\n\n    teens_corr,\n\n    annot=True,\n\n    cmap=\"coolwarm\",\n\n    vmin=vmin, vmax=vmax,  # Set unified color scale\n\n    fmt=\".2f\",\n\n    linewidths=0.5,\n\n    linecolor=\"black\",\n\n    cbar_kws={\"shrink\": 0.8},\n\n    ax=axes[1],\n\n    square=True\n\n)\n\naxes[1].set_title(\"Features with >90% Correlation (Teens)\", fontsize=16, fontweight='bold')\n\naxes[1].tick_params(axis='x', rotation=45)\n\n\n\n# Adjust Layout and Show Plot\n\nplt.tight_layout()\n\n# save as pdf\n\nplt.savefig('correlation_heatmap.pdf',dpi = 600, format='pdf')\n\nplt.show()\n","metadata":{"id":"YM2A4-RQderP","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:32.396537Z","iopub.execute_input":"2024-12-08T20:02:32.396888Z","iopub.status.idle":"2024-12-08T20:02:36.429818Z","shell.execute_reply.started":"2024-12-08T20:02:32.396849Z","shell.execute_reply":"2024-12-08T20:02:36.428902Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cluster_and_drop_features(dataframe, exclude_columns=None, threshold=0.9):\n\n    \"\"\"\n\n    Perform hierarchical clustering on highly correlated features and drop redundant features,\n\n    while excluding specific columns from the process.\n\n    \"\"\"\n\n    if exclude_columns is None:\n\n        exclude_columns = []\n\n\n\n    # Step 1: Exclude specified columns\n\n    df_to_cluster = dataframe.drop(columns=exclude_columns)\n\n\n\n    # Step 2: Compute the absolute correlation matrix\n\n    corr_matrix = df_to_cluster.corr().abs()\n\n    corr_matrix = corr_matrix.fillna(0)  # Handle NaNs or infinite values\n\n\n\n    # Step 3: Perform hierarchical clustering\n\n    linkage = sch.linkage(corr_matrix, method='ward')\n\n    cluster = AgglomerativeClustering(\n\n        n_clusters=None,\n\n        distance_threshold=1-threshold,\n\n        linkage='ward'\n\n    )\n\n    cluster.fit(corr_matrix)\n\n\n\n    # Step 4: Assign features to clusters\n\n    cluster_labels = cluster.labels_\n\n    clusters = {}\n\n    for feature, label in zip(corr_matrix.columns, cluster_labels):\n\n        clusters.setdefault(label, []).append(feature)\n\n\n\n    # Step 5: Select one representative feature per cluster\n\n    selected_features = [sorted(features)[0] for features in clusters.values()]\n\n    all_features = set(df_to_cluster.columns)\n\n    features_to_drop = list(all_features - set(selected_features))\n\n\n\n    # Step 6: Keep excluded columns intact\n\n    final_features = selected_features + exclude_columns\n\n\n\n    print(\"\\nSelected Features:\", selected_features)\n\n    print(\"Dropped Features:\", features_to_drop)\n\n\n\n    return final_features ,features_to_drop\n\n\n\n# Step 1: Perform Feature Selection on Training Data (Exclude SII)\n\nexclude_columns = [\"Severity Impairment Index (SII)\"]\n\nselected_children_features, dropped_children = cluster_and_drop_features(df_train_children, exclude_columns, threshold=0.9)\n\nselected_teens_features, dropped_teens = cluster_and_drop_features(df_train_teens, exclude_columns, threshold=0.9)\n\n\n\n# Step 2: Drop Redundant Features from Training Data (Keep SII)\n\ndf_train_children_reduced = df_train_children[selected_children_features]\n\ndf_train_teens_reduced = df_train_teens[selected_teens_features]\n\n\n\n# Step 3: Apply the Same Feature Selection to Test Data (Exclude SII)\n\nselected_children_features_no_sii = [col for col in selected_children_features if col != \"Severity Impairment Index (SII)\"]\n\nselected_teens_features_no_sii = [col for col in selected_teens_features if col != \"Severity Impairment Index (SII)\"]\n\n\n\ndf_test_children_reduced = df_test_children[selected_children_features_no_sii]\n\ndf_test_teens_reduced = df_test_teens[selected_teens_features_no_sii]\n\n\n\n# Step 4: Print Results\n\nprint(\"Original Training Children Shape:\", df_train_children.shape)\n\nprint(\"Final Training Children Shape:\", df_train_children_reduced.shape)\n\nprint(\"Original Test Children Shape:\", df_test_children.shape)\n\nprint(\"Final Test Children Shape:\", df_test_children_reduced.shape)\n\n\n\nprint(\"Original Training Teens Shape:\", df_train_teens.shape)\n\nprint(\"Final Training Teens Shape:\", df_train_teens_reduced.shape)\n\nprint(\"Original Test Teens Shape:\", df_test_teens.shape)\n\nprint(\"Final Test Teens Shape:\", df_test_teens_reduced.shape)\n","metadata":{"id":"3OmqMtq2Zk_9","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:36.431170Z","iopub.execute_input":"2024-12-08T20:02:36.431519Z","iopub.status.idle":"2024-12-08T20:02:36.477004Z","shell.execute_reply.started":"2024-12-08T20:02:36.431483Z","shell.execute_reply":"2024-12-08T20:02:36.476204Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Drop Features with over 70% missing","metadata":{"id":"A0Nm14eldjn9"}},{"cell_type":"code","source":"def drop_high_missing_features(df, threshold=0.70):\n\n    \"\"\"\n\n    Drops features with missing values greater than the specified threshold.\n\n\n\n    Parameters:\n\n        df: DataFrame\n\n        threshold: float, proportion of missing values (default is 0.70)\n\n\n\n    Returns:\n\n        df_reduced: DataFrame after dropping features\n\n        dropped_features: List of dropped feature names\n\n    \"\"\"\n\n    # Calculate the percentage of missing values for each column\n\n    missing_percentage = df.isnull().sum() / len(df)\n\n\n\n    # Identify columns to drop\n\n    features_to_drop = missing_percentage[missing_percentage > threshold].index.tolist()\n\n    print(f\"Features with >{threshold*100:.0f}% missing values:\", features_to_drop)\n\n\n\n    # Drop the features\n\n    df_reduced = df.drop(columns=features_to_drop)\n\n\n\n    return df_reduced, features_to_drop\n\n\n\n# Apply to Training and Test Datasets\n\n# For Children\n\ndf_train_children_reduced, dropped_children = drop_high_missing_features(df_train_children_reduced, threshold=0.70)\n\ndf_test_children_reduced = df_test_children_reduced.drop(columns=dropped_children)\n\n\n\n# For Teens\n\ndf_train_teens_reduced, dropped_teens = drop_high_missing_features(df_train_teens_reduced, threshold=0.70)\n\ndf_test_teens_reduced = df_test_teens_reduced.drop(columns=dropped_teens)\n\n\n\n# Print Results\n\nprint(\"\\nDropped Features (Children):\", dropped_children)\n\nprint(\"Reduced Children Dataset Shape (Train):\", df_train_children_reduced.shape)\n\nprint(\"Reduced Children Dataset Shape (Test):\", df_test_children_reduced.shape)\n\n\n\nprint(\"\\nDropped Features (Teens):\", dropped_teens)\n\nprint(\"Reduced Teens Dataset Shape (Train):\", df_train_teens_reduced.shape)\n\nprint(\"Reduced Teens Dataset Shape (Test):\", df_test_teens_reduced.shape)\n","metadata":{"id":"BzLoVViwcv9G","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:36.477923Z","iopub.execute_input":"2024-12-08T20:02:36.478199Z","iopub.status.idle":"2024-12-08T20:02:36.492209Z","shell.execute_reply.started":"2024-12-08T20:02:36.478174Z","shell.execute_reply":"2024-12-08T20:02:36.491388Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Remove Outliers","metadata":{"id":"xy7Sw4wtlfId"}},{"cell_type":"code","source":"def remove_outliers(df, columns=None, iqr_multiplier=1.5, exclude_columns='Severity Impairment Index (SII)'):\n\n    \"\"\"\n\n    Remove outliers from specified columns using IQR method.\n\n\n\n    Parameters:\n\n    - df: Input DataFrame\n\n    - columns: List of columns to check for outliers (if None, process all numeric columns)\n\n    - iqr_multiplier: Multiplier for IQR (default 1.5, standard for outlier detection)\n\n\n\n    Returns:\n\n    - DataFrame with outliers removed\n\n    - Dictionary of removed outliers\n\n    \"\"\"\n\n    # Create a copy of the dataframe\n\n    df_cleaned = df.copy()\n\n\n\n    # If no columns specified, select all numeric columns\n\n    if columns is None:\n\n        columns = df.select_dtypes(include=['float64', 'int64']).columns\n\n\n\n    # Track removed outliers\n\n    removed_outliers = {}\n\n    # exclude columns\n\n    columns = [col for col in columns if col != exclude_columns]\n\n\n\n    for column in columns:\n\n        # Skip if column doesn't exist or is not numeric\n\n        if column not in df.columns or not pd.api.types.is_numeric_dtype(df[column]):\n\n            continue\n\n\n\n        # Calculate Q1, Q3, and IQR, ignoring NaNs\n\n        Q1 = df[column].quantile(0.25)\n\n        Q3 = df[column].quantile(0.75)\n\n        IQR = Q3 - Q1\n\n\n\n        # Define outlier bounds\n\n        lower_bound = Q1 - (iqr_multiplier * IQR)\n\n        upper_bound = Q3 + (iqr_multiplier * IQR)\n\n\n\n        # Identify outliers (excluding NaNs)\n\n        outliers = df[\n\n            (df[column] < lower_bound) |\n\n            (df[column] > upper_bound)\n\n        ]\n\n\n\n        # Remove outliers\n\n        df_cleaned = df_cleaned[\n\n            (df_cleaned[column] >= lower_bound) &\n\n            (df_cleaned[column] <= upper_bound) |\n\n            (df_cleaned[column].isna())\n\n        ]\n\n\n\n        # Store removed outliers\n\n        if len(outliers) > 0:\n\n            removed_outliers[column] = outliers\n\n\n\n        # Print outlier information\n\n        print(f\"\\nColumn: {column}\")\n\n        print(f\"Outlier Bounds: [{lower_bound}, {upper_bound}]\")\n\n        print(f\"Removed {len(outliers)} outliers\")\n\n\n\n    # Overall summary\n\n    print(\"\\nTotal Outliers Removed:\")\n\n    for col, outliers in removed_outliers.items():\n\n        print(f\"{col}: {len(outliers)} outliers\")\n\n\n\n    print(f\"\\nOriginal DataFrame Shape: {df.shape}\")\n\n    print(f\"Cleaned DataFrame Shape: {df_cleaned.shape}\")\n\n\n\n    return df_cleaned, removed_outliers\n\n\n\n# Apply to both datasets\n\ndf_train_children_reduced, _ = remove_outliers(df_train_children_reduced)\n\ndf_train_teens_reduced, _ = remove_outliers(df_train_teens_reduced)","metadata":{"id":"agcD-uXGi_4W","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:36.493296Z","iopub.execute_input":"2024-12-08T20:02:36.493625Z","iopub.status.idle":"2024-12-08T20:02:36.633581Z","shell.execute_reply.started":"2024-12-08T20:02:36.493599Z","shell.execute_reply":"2024-12-08T20:02:36.632862Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#Impute Missing Values for Features","metadata":{"id":"LnLZQxp096Wb"}},{"cell_type":"code","source":"\n\ndef impute_missing_values(df_train, df_test, exclude_columns=None):\n\n    \"\"\"\n\n    Imputes missing values in numerical and categorical columns.\n\n    - Numerical: KNN Imputation\n\n    - Categorical: Most Frequent Imputation (Mode)\n\n\n\n    Parameters:\n\n        df_train: Training DataFrame\n\n        df_test: Testing DataFrame\n\n        exclude_columns: List of columns to exclude from imputation\n\n\n\n    Returns:\n\n        df_train_imputed: Imputed Training DataFrame\n\n        df_test_imputed: Imputed Testing DataFrame\n\n    \"\"\"\n\n    if exclude_columns is None:\n\n        exclude_columns = []\n\n\n\n    # Drop excluded columns temporarily\n\n    train_copy = df_train.drop(columns=exclude_columns, errors='ignore')\n\n    test_copy = df_test.copy()\n\n\n\n    # Separate numerical and categorical columns\n\n    numerical_features = train_copy.select_dtypes(include=['float64', 'int64']).columns\n\n    categorical_features = train_copy.select_dtypes(include=['object', 'category']).columns\n\n\n\n    print(\"Numerical Features:\", numerical_features.tolist())\n\n    print(\"Categorical Features:\", categorical_features.tolist())\n\n\n\n    # Step 1: KNN Imputer for Numerical Features\n\n    knn_imputer = KNNImputer(n_neighbors=5)\n\n    train_copy[numerical_features] = knn_imputer.fit_transform(train_copy[numerical_features])\n\n    test_copy[numerical_features] = knn_imputer.transform(test_copy[numerical_features])\n\n\n\n    # Step 2: Simple Imputer (Mode) for Categorical Features\n\n    mode_imputer = SimpleImputer(strategy='most_frequent')\n\n    train_copy[categorical_features] = mode_imputer.fit_transform(train_copy[categorical_features])\n\n    test_copy[categorical_features] = mode_imputer.transform(test_copy[categorical_features])\n\n\n\n    # Step 3: Add back excluded columns\n\n    for col in exclude_columns:\n\n        train_copy[col] = df_train[col]\n\n\n\n    return train_copy, test_copy\n\n\n\n# Step 1: Apply Imputation on Children Dataset\n\nexclude_columns = [\"Severity Impairment Index (SII)\"]  # Exclude SII from imputation\n\ndf_train_children_imputed, df_test_children_imputed = impute_missing_values(\n\n    df_train_children_reduced, df_test_children_reduced, exclude_columns\n\n)\n\n\n\n# Step 2: Apply Imputation on Teens Dataset\n\ndf_train_teens_imputed, df_test_teens_imputed = impute_missing_values(\n\n    df_train_teens_reduced, df_test_teens_reduced, exclude_columns\n\n)\n\n\n\n# Verify Results\n\nprint(\"\\nImputed Children Dataset Shape (Train):\", df_train_children_imputed.shape)\n\nprint(\"Imputed Children Dataset Shape (Test):\", df_test_children_imputed.shape)\n\n\n\nprint(\"\\nImputed Teens Dataset Shape (Train):\", df_train_teens_imputed.shape)\n\nprint(\"Imputed Teens Dataset Shape (Test):\", df_test_teens_imputed.shape)\n","metadata":{"id":"WBV4ojvgX9TP","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:36.635172Z","iopub.execute_input":"2024-12-08T20:02:36.635679Z","iopub.status.idle":"2024-12-08T20:02:37.875693Z","shell.execute_reply.started":"2024-12-08T20:02:36.635640Z","shell.execute_reply":"2024-12-08T20:02:37.874757Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Impute The Target","metadata":{"id":"KDTKMLYcg0Gr"}},{"cell_type":"markdown","source":"## Advanced Stratified Imputation\n\n\n\n### Key Steps:\n\n1. **Feature Selection**: Use the top 5 most correlated features with `SII`.\n\n2. **Stratified Cross-Validation**: Perform 5-fold stratified sampling to handle class imbalance.\n\n3. **Scaling**: Standardize features using `StandardScaler`.\n\n4. **Model Training**: Train a `RandomForestClassifier` with `class_weight='balanced'`.\n\n5. **Best Model Selection**: Choose the model with the highest validation accuracy.\n\n6. **Imputation**: Predict and fill missing `SII` values using the best model.\n\n\n\n### Benefits:\n\n- Handles class imbalance effectively.  \n\n- Uses top features for accurate predictions.  \n\n- Ensures robust imputation with cross-validation.  \n\n\n\n### Output:\n\nImputed missing `SII` values with distribution summary.","metadata":{"id":"CB1TvFwjilWP"}},{"cell_type":"code","source":"\n\ndef advanced_stratified_imputation(df, target_column):\n\n    \"\"\"\n\n    Advanced imputation strategy with:\n\n    - Stratified sampling\n\n    - Feature selection based on correlations\n\n    - Cross-validation\n\n    - Handling class imbalance\n\n    \"\"\"\n\n    # Identify correlations and select top features\n\n    correlations = df.corr()[target_column].abs().sort_values(ascending=False)\n\n    top_correlated_features = correlations[1:6].index.tolist()\n\n\n\n    # Prepare data\n\n    known_data = df[df[target_column].notna()]\n\n    unknown_data = df[df[target_column].isna()]\n\n\n\n    # Feature preparation\n\n    X = known_data[top_correlated_features]\n\n    y = known_data[target_column]\n\n\n\n    # Stratified Cross-Validation\n\n    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\n\n\n    # Scalers and models for each fold\n\n    scalers = []\n\n    models = []\n\n    fold_scores = []\n\n\n\n    for train_index, val_index in skf.split(X, y):\n\n        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n\n        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n\n\n\n        # Scale features\n\n        scaler = StandardScaler()\n\n        X_train_scaled = scaler.fit_transform(X_train)\n\n        X_val_scaled = scaler.transform(X_val)\n\n        scalers.append(scaler)\n\n\n\n        # Model with class weights\n\n        model = RandomForestClassifier(\n\n            n_estimators=100,\n\n            class_weight='balanced',\n\n            random_state=42\n\n        )\n\n        model.fit(X_train_scaled, y_train)\n\n        models.append(model)\n\n\n\n        # Evaluate\n\n        y_pred = model.predict(X_val_scaled)\n\n        fold_score = accuracy_score(y_val, y_pred)\n\n        fold_scores.append(fold_score)\n\n\n\n        print(f\"Fold Validation Accuracy: {fold_score:.4f}\")\n\n\n\n    # Detailed classification report for best model\n\n    best_model_index = np.argmax(fold_scores)\n\n    best_scaler = scalers[best_model_index]\n\n    best_model = models[best_model_index]\n\n\n\n    # Impute unknown data\n\n    X_unknown = unknown_data[top_correlated_features]\n\n    X_unknown_scaled = best_scaler.transform(X_unknown)\n\n\n\n    imputed_values = best_model.predict(X_unknown_scaled)\n\n\n\n    # Create imputed dataframe\n\n    df_imputed = df.copy()\n\n    df_imputed.loc[df[target_column].isna(), target_column] = imputed_values\n\n\n\n    # Print imputation summary\n\n    print(\"\\nImputation Summary:\")\n\n    print(f\"Total missing values: {len(imputed_values)}\")\n\n    print(\"Imputed Value Distribution:\")\n\n    print(pd.Series(imputed_values).value_counts(normalize=True))\n\n\n\n    return df_imputed\n\n\n\n# Apply to both datasets\n\ndf_train_children_imputed = advanced_stratified_imputation(\n\n    df_train_children_imputed,\n\n    \"Severity Impairment Index (SII)\"\n\n)\n\n\n\ndf_train_teens_imputed = advanced_stratified_imputation(\n\n    df_train_teens_imputed,\n\n    \"Severity Impairment Index (SII)\"\n\n)","metadata":{"id":"LFw1wPQKCdZY","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:37.877506Z","iopub.execute_input":"2024-12-08T20:02:37.877900Z","iopub.status.idle":"2024-12-08T20:02:40.206693Z","shell.execute_reply.started":"2024-12-08T20:02:37.877859Z","shell.execute_reply":"2024-12-08T20:02:40.205850Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ML","metadata":{"id":"0N8THcfGmXID"}},{"cell_type":"code","source":"# Custom Metric: Quadratic Weighted Kappa\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\nqwk_scorer = make_scorer(quadratic_weighted_kappa, greater_is_better=True)\ndef optimize_model(X_train, y_train, X_val, y_val, model_name=\"xgboost\"):\n    def objective(params):\n        # Ensure parameters are mapped correctly\n        params['n_estimators'] = [50, 100, 200, 300, 500][params['n_estimators']]\n        params['max_depth'] = [3, 5, 7, 9, 12][params['max_depth']]\n        \n        # Model selection\n        if model_name == \"xgboost\":\n            model = xgb.XGBClassifier(\n                objective=\"multi:softmax\",\n                num_class=len(np.unique(y_train)),\n                subsample=params.get('subsample', 1.0),\n                colsample_bytree=params.get('colsample_bytree', 1.0),\n                learning_rate=params['learning_rate'],\n                n_estimators=params['n_estimators'],\n                max_depth=params['max_depth'],\n                use_label_encoder=False,\n                n_jobs=-1\n            )\n        else:\n            raise ValueError(\"Unsupported model\")\n\n        # Fit model with early stopping\n        model.fit(\n            X_train, y_train,\n            eval_set=[(X_val, y_val)],\n            early_stopping_rounds=30,\n            verbose=False\n        )\n\n        # Predict and calculate QWK\n        y_pred = model.predict(X_val)\n        score = quadratic_weighted_kappa(y_val, y_pred)\n        return {'loss': -score, 'status': STATUS_OK}\n\n    # Search Space\n    space = {\n        'n_estimators': hp.choice('n_estimators', [0, 1, 2, 3, 4]),  # Indices\n        'max_depth': hp.choice('max_depth', [0, 1, 2, 3, 4]),\n        'learning_rate': hp.uniform('learning_rate', 0.001, 0.1),\n        'subsample': hp.uniform('subsample', 0.5, 1.0),\n        'colsample_bytree': hp.uniform('colsample_bytree', 0.5, 1.0)\n    }\n\n    # Hyperopt Trials\n    trials = Trials()\n    best_params = fmin(fn=objective, space=space, algo=tpe.suggest, max_evals=100, trials=trials)\n\n    # Map indices back to values\n    best_params['n_estimators'] = [50, 100, 200, 300, 500][best_params['n_estimators']]\n    best_params['max_depth'] = [3, 5, 7, 9, 12][best_params['max_depth']]\n    \n    print(\"Best Parameters:\", best_params)\n    return best_params\n\n","metadata":{"id":"4JbH91l6Fs4p","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T22:11:20.258534Z","iopub.execute_input":"2024-12-08T22:11:20.259359Z","iopub.status.idle":"2024-12-08T22:11:20.269230Z","shell.execute_reply.started":"2024-12-08T22:11:20.259321Z","shell.execute_reply":"2024-12-08T22:11:20.268211Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## TPOT","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split, StratifiedShuffleSplit\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T22:51:16.113512Z","iopub.execute_input":"2024-12-08T22:51:16.114350Z","iopub.status.idle":"2024-12-08T22:51:16.118009Z","shell.execute_reply.started":"2024-12-08T22:51:16.114301Z","shell.execute_reply":"2024-12-08T22:51:16.117160Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Custom TPOT Optimization Function\ndef run_tpot(X, y, model_name, generations=5, population_size=20):\n    # Split the dataset into training and validation\n    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n    # Scaling\n    scaler = StandardScaler()\n    X_train_scaled = scaler.fit_transform(X_train)\n    X_val_scaled = scaler.transform(X_val)\n    cv = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)\n\n    # TPOT Classifier\n    print(f\"Running TPOT optimization for {model_name} dataset...\")\n    tpot = TPOTClassifier(\n        generations=generations,\n        population_size=population_size,\n        verbosity=2,\n        scoring='balanced_accuracy',  \n        n_jobs=-1,\n        random_state=42\n    )\n    \n    # Fit TPOT\n    tpot.fit(X_train_scaled, y_train)\n    \n    # Evaluate on validation data\n    print(f\"Validation Score for {model_name}: {tpot.score(X_val_scaled, y_val)}\")\n    \n    # Export the best pipeline\n    pipeline_file = f'best_pipeline_{model_name}.py'\n    tpot.export(pipeline_file)\n    print(f\"Best pipeline for {model_name} exported to '{pipeline_file}'.\")\n    \n    return tpot, scaler","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T22:51:16.883286Z","iopub.execute_input":"2024-12-08T22:51:16.884015Z","iopub.status.idle":"2024-12-08T22:51:16.890227Z","shell.execute_reply.started":"2024-12-08T22:51:16.883976Z","shell.execute_reply":"2024-12-08T22:51:16.889290Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# --- Model for Children ---\n\nX_children = df_train_children_imputed.drop(columns=[ 'Severity Impairment Index (SII)'])\n\ny_children = df_train_children_imputed['Severity Impairment Index (SII)']\n\n\n\nchildren_model, children_scaler = run_tpot(X_children, y_children, model_name=\"children\")\n\n\n\n","metadata":{"id":"6CBfYGixGWFn","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T22:51:16.907693Z","iopub.execute_input":"2024-12-08T22:51:16.908350Z","iopub.status.idle":"2024-12-08T22:55:39.648858Z","shell.execute_reply.started":"2024-12-08T22:51:16.908320Z","shell.execute_reply":"2024-12-08T22:55:39.647678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# --- Model for Teens ---\n\nX_teens = df_train_teens_imputed.drop(columns=[ 'Severity Impairment Index (SII)'])\n\ny_teens = df_train_teens_imputed['Severity Impairment Index (SII)']\n\n\nteens_model, teens_scaler = run_tpot(X_teens, y_teens, model_name=\"teens\")\n\n","metadata":{"id":"TQvmsfjmdEOP","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test_children\n\n# add the id from df_test_children to df_test_children_imputed\n\ndf_test_children_imputed['id'] = df_test_children['id']\n\n# SAME FOR TEENS\n\ndf_test_teens_imputed['id'] = df_test_teens['id']","metadata":{"id":"0ZOBt03wr6Dw","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T22:47:25.260503Z","iopub.execute_input":"2024-12-08T22:47:25.261121Z","iopub.status.idle":"2024-12-08T22:47:25.266044Z","shell.execute_reply.started":"2024-12-08T22:47:25.261084Z","shell.execute_reply":"2024-12-08T22:47:25.265044Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# --- Predictions ---\n\n# Scale Test Data\n\nX_test_children_scaled = children_scaler.transform(df_test_children_imputed.drop(columns=['id']))\n\nX_test_teens_scaled = teens_scaler.transform(df_test_teens_imputed.drop(columns=['id']))\n\nchildren_predictions = children_model.predict(X_test_children_scaled)\n\nteens_predictions = teens_model.predict(X_test_teens_scaled)\n\n\n\n# Combine Predictions\n\nchildren_results = pd.DataFrame({'id': df_test_children_imputed['id'], 'sii': children_predictions})\n\nteens_results = pd.DataFrame({'id': df_test_teens['id'], 'sii': teens_predictions})\n\n\n\nsubmission = pd.concat([children_results, teens_results], axis=0)\n\nsubmission.to_csv('submission.csv', index=False)\n\n\n\nprint(\"Submission file created successfully!\")\n","metadata":{"id":"K9s6R8ezGJTv","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T22:47:25.743057Z","iopub.execute_input":"2024-12-08T22:47:25.743911Z","iopub.status.idle":"2024-12-08T22:47:25.767170Z","shell.execute_reply.started":"2024-12-08T22:47:25.743859Z","shell.execute_reply":"2024-12-08T22:47:25.766266Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Hypothesis Testing","metadata":{"id":"YKF6rLogTsYg"}},{"cell_type":"code","source":"# import scipy.stats as stats\n\n\n\n# # Data preparation: Select relevant columns and drop missing values\n\n# relevant_data = df_train_csv[['sii', 'PreInt_EduHx-computerinternet_hoursday', 'Physical-BMI']].dropna()\n\n\n\n# # Extract variables for hypothesis testing\n\n# ssi = relevant_data['sii']  # Target variable\n\n# internet_hours = relevant_data['PreInt_EduHx-computerinternet_hoursday']  # Predictor 1\n\n# bmi = relevant_data['Physical-BMI']  # Predictor 2\n\n\n\n# # Hypothesis 1: Correlation between SSI and Internet Hours\n\n# corr_internet, p_internet = stats.pearsonr(ssi, internet_hours)\n\n\n\n# # Hypothesis 2: Correlation between SSI and BMI\n\n# corr_bmi, p_bmi = stats.pearsonr(ssi, bmi)\n\n\n\n# # Results summary\n\n# results = {\n\n#     \"Hypothesis\": [\"SSI vs Internet Hours\", \"SSI vs BMI\"],\n\n#     \"Correlation Coefficient\": [corr_internet, corr_bmi],\n\n#     \"P-value\": [p_internet, p_bmi],\n\n# }\n\n# results_df = pd.DataFrame(results)\n\n\n\n# results_df","metadata":{"id":"fRfAb5KGTy9l","trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:02:40.225977Z","iopub.status.idle":"2024-12-08T20:02:40.226408Z","shell.execute_reply.started":"2024-12-08T20:02:40.226175Z","shell.execute_reply":"2024-12-08T20:02:40.226197Z"}},"outputs":[],"execution_count":null}]}