{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score, make_scorer, confusion_matrix\nfrom sklearn.model_selection import StratifiedKFold, KFold, train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\nfrom scipy.optimize import minimize\nfrom scipy import stats\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nimport warnings\nfrom sklearn.linear_model import ElasticNetCV, LassoCV, Lasso, LinearRegression\nfrom sklearn.ensemble import ExtraTreesRegressor, RandomForestRegressor\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nimport optuna\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nimport random\nimport pickle\nfrom sklearn.neural_network import MLPRegressor\n\nwarnings.filterwarnings('ignore')\n\n#A lot of it is copied from and changed from Lennart https://www.kaggle.com/code/lennarthaupts/1st-place-cmi-model-v4-1-1-reduced?scriptVersionId=213769368","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:41.847864Z","iopub.execute_input":"2025-05-20T19:30:41.848404Z","iopub.status.idle":"2025-05-20T19:30:41.856382Z","shell.execute_reply.started":"2025-05-20T19:30:41.848367Z","shell.execute_reply":"2025-05-20T19:30:41.855046Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from lennarthaupts\n# https://www.kaggle.com/code/lennarthaupts/1st-place-cmi-model-v4-1-1-reduced?scriptVersionId=213769368\nimport pandas as pd\n\n\ndef calculate_weights(series):\n    # Create bins for the target variable and assign weights based on frequency\n    bins = pd.cut(series, bins=10, labels=False)\n    weights = bins.value_counts().reset_index()\n    weights.columns = [\"target_bins\", \"count\"]\n    weights[\"count\"] = 1 / weights[\"count\"]\n    weight_map = weights.set_index(\"target_bins\")[\"count\"].to_dict()\n    weights = bins.map(weight_map)\n    return weights / weights.mean()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:41.863312Z","iopub.execute_input":"2025-05-20T19:30:41.863799Z","iopub.status.idle":"2025-05-20T19:30:41.887271Z","shell.execute_reply.started":"2025-05-20T19:30:41.863746Z","shell.execute_reply":"2025-05-20T19:30:41.886007Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def detect_outliers(df, column_name):\n    # drop NaN\n    df_clean = df[column_name].dropna()\n\n    Q1 = df_clean.quantile(0.15)\n    Q3 = df_clean.quantile(0.85)\n    IQR = Q3 - Q1\n\n    lower_bound = Q1 - 1.5 * IQR\n    upper_bound = Q3 + 1.5 * IQR\n\n    # find outliers\n    outliers = df_clean[(df_clean < lower_bound) | (df_clean > upper_bound)]\n\n    return outliers\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:41.888813Z","iopub.execute_input":"2025-05-20T19:30:41.889169Z","iopub.status.idle":"2025-05-20T19:30:41.900247Z","shell.execute_reply.started":"2025-05-20T19:30:41.889141Z","shell.execute_reply":"2025-05-20T19:30:41.899068Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Code for finding optimal thresholds copied from: Michael Semenoff\n# https://www.kaggle.com/code/michaelsemenoff/cmi-actigraphy-feature-engineering-selection\n\nimport numpy as np\nfrom scipy.optimize import minimize\nfrom sklearn.metrics import cohen_kappa_score\n\n\ndef round_with_thresholds(raw_preds, thresholds):\n    return np.where(\n        raw_preds < thresholds[0],\n        int(0),\n        np.where(\n            raw_preds < thresholds[1],\n            int(1),\n            np.where(raw_preds < thresholds[2], int(2), int(3)),\n        ),\n    )\n\n\ndef optimize_thresholds(y_true, raw_preds, start_vals=[0.5, 1.5, 2.5]):\n    def fun(thresholds, y_true, raw_preds):\n        rounded_preds = round_with_thresholds(raw_preds, thresholds)\n        return -cohen_kappa_score(y_true, rounded_preds, weights=\"quadratic\")\n\n    res = minimize(fun, x0=start_vals, args=(y_true, raw_preds), method=\"Powell\")\n    assert res.success\n    return res.x\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:41.985787Z","iopub.execute_input":"2025-05-20T19:30:41.986178Z","iopub.status.idle":"2025-05-20T19:30:41.995897Z","shell.execute_reply.started":"2025-05-20T19:30:41.986146Z","shell.execute_reply":"2025-05-20T19:30:41.993757Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Code for finding optimal thresholds copied from: Michael Semenoff\n# https://www.kaggle.com/code/michaelsemenoff/cmi-actigraphy-feature-engineering-selection\n\nimport numpy as np\nfrom sklearn.metrics import cohen_kappa_score\n\nbase_thresholds = [30, 50, 80]\n\n\ndef cross_validate(\n    model_,\n    data,\n    features,\n    score_col,\n    index_col,\n    cv,\n    sample_weights=False,\n    verbose=False,\n):\n    kappa_scores = []\n    oof_score_predictions = np.zeros(len(data))\n\n    #score_to_index_thresholds = base_thresholds\n    thresholds = []\n    for fold_idx, (train_idx, val_idx) in enumerate(cv.split(data, data[index_col])):\n        X_train, X_val = data[features].iloc[train_idx], data[features].iloc[val_idx]\n        y_train_score = data[score_col].iloc[train_idx]\n        y_train_index = data[index_col].iloc[train_idx]\n        # y_val_score = data[score_col].iloc[val_idx]\n        y_val_index = data[index_col].iloc[val_idx]\n\n        # Train model with sample weights if provided\n        if sample_weights:\n            weights = calculate_weights(y_train_score)\n            model_.fit(X_train, y_train_score, sample_weight=weights)\n        else:\n            model_.fit(X_train, y_train_score)\n\n        y_pred_train_score = model_.predict(X_train)\n        y_pred_val_score = model_.predict(X_val)\n\n        oof_score_predictions[val_idx] = y_pred_val_score\n\n        # Find optimal threshold in sample\n        t_1 = optimize_thresholds(\n            y_train_index, y_pred_train_score, start_vals=base_thresholds\n        )\n        thresholds.append(t_1)\n\n        y_pred_val_index = round_with_thresholds(y_pred_val_score, t_1)\n\n        kappa_score = cohen_kappa_score(\n            y_val_index, y_pred_val_index, weights=\"quadratic\"\n        )\n        kappa_scores.append(kappa_score)\n\n        if verbose:\n            print(f\"Fold {fold_idx}: Optimized Kappa Score = {kappa_score}\")\n\n    if verbose:\n        print(f\"## Mean CV Kappa Score: {np.mean(kappa_scores)} ##\")\n        print(f\"## Std CV: {np.std(kappa_scores)}\")\n\n    return np.mean(kappa_scores), oof_score_predictions, thresholds\n\n\ndef n_cross_validate(\n    model_,\n    data,\n    features,\n    score_col,\n    index_col,\n    cv,\n    seeds,\n    sample_weights=False,\n    verbose=False,\n):\n\n    scores = []\n    for seed in seeds:\n        cv.random_state = seed\n        score, oof, _ = cross_validate(\n            model_,\n            data,\n            features,\n            score_col,\n            index_col,\n            cv,\n            sample_weights=True,\n            verbose=False,\n        )\n        scores.append(score)\n    return score, oof\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:41.998109Z","iopub.execute_input":"2025-05-20T19:30:41.998757Z","iopub.status.idle":"2025-05-20T19:30:42.024582Z","shell.execute_reply.started":"2025-05-20T19:30:41.998516Z","shell.execute_reply":"2025-05-20T19:30:42.023300Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ndf_test = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.027250Z","iopub.execute_input":"2025-05-20T19:30:42.028017Z","iopub.status.idle":"2025-05-20T19:30:42.098957Z","shell.execute_reply.started":"2025-05-20T19:30:42.027929Z","shell.execute_reply":"2025-05-20T19:30:42.097726Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# List of BIA-columns to check for outliers\n# without BIA-BIA_FMI \ncolumns_BIA = [\n    'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', \n    'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num', \n    'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW'\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.100744Z","iopub.execute_input":"2025-05-20T19:30:42.101166Z","iopub.status.idle":"2025-05-20T19:30:42.105867Z","shell.execute_reply.started":"2025-05-20T19:30:42.101132Z","shell.execute_reply":"2025-05-20T19:30:42.104585Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" filtered_df = df[['id', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-Height', 'Physical-Weight'] + columns_BIA]\n\n# Initialize a dictionary for storing outlier data with one row per 'id'\noutlier_data = {}\n\n# Loop through columns to find outliers\nfor column_name in columns_BIA:\n    outliers = detect_outliers(filtered_df, column_name)\n    \n    # Loop through outliers and add them to the dictionary\n    for index, value in outliers.items():\n        if filtered_df.loc[index, 'id'] not in outlier_data:\n            outlier_data[filtered_df.loc[index, 'id']] = {\n                'id': filtered_df.loc[index, 'id'],\n                'age': filtered_df.loc[index, 'Basic_Demos-Age'],\n                'sex': filtered_df.loc[index, 'Basic_Demos-Sex'],\n                'height': filtered_df.loc[index, 'Physical-Height'],\n                'weight': filtered_df.loc[index, 'Physical-Weight']\n            }\n        \n        # Add the outlier value for this column\n        outlier_data[filtered_df.loc[index, 'id']][column_name] = value\n\n# Convert the dictionary to a DataFrame\noutlier_df = pd.DataFrame.from_dict(outlier_data, orient='index')\n\noutlier_df = outlier_df.sort_values(by=\"id\")\noutlier_df.to_csv('outliers_BIA.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.107136Z","iopub.execute_input":"2025-05-20T19:30:42.107613Z","iopub.status.idle":"2025-05-20T19:30:42.179311Z","shell.execute_reply.started":"2025-05-20T19:30:42.107571Z","shell.execute_reply":"2025-05-20T19:30:42.178280Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"filtered_df = df[['id', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-Height', 'Physical-Weight'] + columns_BIA]\n\n# Initialize a dictionary for storing outlier data with one row per 'id'\noutlier_data = {}\n\n# Loop through columns to find outliers\nfor column_name in columns_BIA:\n    outliers = detect_outliers(filtered_df, column_name)\n    \n    # Loop through outliers and add them to the dictionary\n    for index, value in outliers.items():\n        if filtered_df.loc[index, 'id'] not in outlier_data:\n            outlier_data[filtered_df.loc[index, 'id']] = {\n                'id': filtered_df.loc[index, 'id'],\n                #'age': filtered_df.loc[index, 'Basic_Demos-Age'],\n                #'sex': filtered_df.loc[index, 'Basic_Demos-Sex'],\n                'height': filtered_df.loc[index, 'Physical-Height'],\n                'weight': filtered_df.loc[index, 'Physical-Weight']\n            }\n        \n        # Add the outlier value for this column\n        outlier_data[filtered_df.loc[index, 'id']][column_name] = value\n\n# Convert the dictionary to a DataFrame\noutlier_df = pd.DataFrame.from_dict(outlier_data, orient='index')\n\noutlier_df = outlier_df.sort_values(by=\"id\")\noutlier_df.to_csv('outliers_BIA_general.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.180413Z","iopub.execute_input":"2025-05-20T19:30:42.180805Z","iopub.status.idle":"2025-05-20T19:30:42.237693Z","shell.execute_reply.started":"2025-05-20T19:30:42.180768Z","shell.execute_reply":"2025-05-20T19:30:42.236481Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"height_df = df[['id', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-Height']]\nweight_df = df[['id', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-Weight']]\n# bmi_df = df[['id', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-BMI']] without BMI\nwaist_df = df[['id', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-Waist_Circumference']]\ndiastolic_df = df[['id', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-Diastolic_BP']]\nheartrate_df = df[['id', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-HeartRate']]\nsystolic_df = df[['id', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-Systolic_BP']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.238890Z","iopub.execute_input":"2025-05-20T19:30:42.239341Z","iopub.status.idle":"2025-05-20T19:30:42.252394Z","shell.execute_reply.started":"2025-05-20T19:30:42.239299Z","shell.execute_reply":"2025-05-20T19:30:42.251063Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" # replace 0 with NaN\nheight_df.loc[height_df['Physical-Height'] == 0, 'Physical-Height'] = np.nan\nweight_df.loc[weight_df['Physical-Weight'] == 0, 'Physical-Weight'] = np.nan\n# bmi_df.loc[bmi_df['Physical-BMI'] == 0, 'Physical-BMI'] = np.nan\nwaist_df.loc[waist_df['Physical-Waist_Circumference'] == 0, 'Physical-Waist_Circumference'] = np.nan\ndiastolic_df.loc[diastolic_df['Physical-Diastolic_BP'] == 0, 'Physical-Diastolic_BP'] = np.nan\nheartrate_df.loc[heartrate_df['Physical-HeartRate'] == 0, 'Physical-HeartRate'] = np.nan\nsystolic_df.loc[systolic_df['Physical-Systolic_BP'] == 0, 'Physical-Systolic_BP'] = np.nan\n\n# sort → age & sex\nheight_df = height_df.sort_values(by=['Basic_Demos-Age', 'Basic_Demos-Sex'])\nweight_df = weight_df.sort_values(by=['Basic_Demos-Age', 'Basic_Demos-Sex'])\n# bmi_df = bmi_df.sort_values(by=['Basic_Demos-Age', 'Basic_Demos-Sex'])\nwaist_df = waist_df.sort_values(by=['Basic_Demos-Age', 'Basic_Demos-Sex'])\ndiastolic_df = diastolic_df.sort_values(by=['Basic_Demos-Age', 'Basic_Demos-Sex'])\nheartrate_df = heartrate_df.sort_values(by=['Basic_Demos-Age', 'Basic_Demos-Sex'])\nsystolic_df = systolic_df.sort_values(by=['Basic_Demos-Age', 'Basic_Demos-Sex'])\n\n# Dictionary for Outliers \nheight_outliers = []\nweight_outliers = []\n# bmi_outliers = []\nwaist_outliers = []\ndiastolic_outliers = []\nheartrate_outliers = []\nsystolic_outliers = []\n\n# Group data by age and sex and applie the detect_outliers() function separately.\n# outliers for height\nfor (age, sex), group in height_df.groupby(['Basic_Demos-Age', 'Basic_Demos-Sex']):\n    outliers = detect_outliers(group, 'Physical-Height')\n    for index, value in outliers.items():\n        height_outliers.append({\n            'id': group.loc[index, 'id'],\n            'age': age,\n            'sex': sex,\n            'height_outlier': value\n        })\n\n# outliers for weight\nfor (age, sex), group in weight_df.groupby(['Basic_Demos-Age', 'Basic_Demos-Sex']):\n    outliers = detect_outliers(group, 'Physical-Weight')\n    for index, value in outliers.items():\n        weight_outliers.append({\n            'id': group.loc[index, 'id'],\n            'age': age,\n            'sex': sex,\n            'weight_outlier': value\n        })\n\n# outliers for bmi\n# for (age, sex), group in bmi_df.groupby(['Basic_Demos-Age', 'Basic_Demos-Sex']):\n    # outliers = detect_outliers(group, 'Physical-BMI')\n    # for index, value in outliers.items():\n        # bmi_outliers.append({\n            #'id': group.loc[index, 'id'],\n            #'age': age,\n            #'sex': sex,\n            #'bmi_outlier': value\n        #})\n\n# outliers for waist circumference\nfor (age, sex), group in waist_df.groupby(['Basic_Demos-Age', 'Basic_Demos-Sex']):\n    outliers = detect_outliers(group, 'Physical-Waist_Circumference')\n    for index, value in outliers.items():\n        waist_outliers.append({\n            'id': group.loc[index, 'id'],\n            'age': age,\n            'sex': sex,\n            'waist_outlier': value\n        })\n\n# outliers for diastolic BP\nfor (age, sex), group in diastolic_df.groupby(['Basic_Demos-Age', 'Basic_Demos-Sex']):\n    outliers = detect_outliers(group, 'Physical-Diastolic_BP')\n    for index, value in outliers.items():\n        diastolic_outliers.append({\n            'id': group.loc[index, 'id'],\n            'age': age,\n            'sex': sex,\n            'diastolic_outlier': value\n        })\n\n# outliers for heart rate\nfor (age, sex), group in heartrate_df.groupby(['Basic_Demos-Age', 'Basic_Demos-Sex']):\n    outliers = detect_outliers(group, 'Physical-HeartRate')\n    for index, value in outliers.items():\n        heartrate_outliers.append({\n            'id': group.loc[index, 'id'],\n            'age': age,\n            'sex': sex,\n            'heartrate_outlier': value\n        })\n\n# outliers for systolic BP\nfor (age, sex), group in systolic_df.groupby(['Basic_Demos-Age', 'Basic_Demos-Sex']):\n    outliers = detect_outliers(group, 'Physical-Systolic_BP')\n    for index, value in outliers.items():\n        systolic_outliers.append({\n            'id': group.loc[index, 'id'],\n            'age': age,\n            'sex': sex,\n            'systolic_outlier': value\n        })\n\n# create outlier-dataframe\n#height_outliers_df = pd.DataFrame(height_outliers)\n#weight_outliers_df = pd.DataFrame(weight_outliers)\nheight_outliers_df = pd.DataFrame(height_outliers, columns=['id', 'age', 'sex', 'height_outlier'])\nweight_outliers_df = pd.DataFrame(weight_outliers, columns=['id', 'age', 'sex', 'weight_outlier'])\n\n# bmi_outliers_df = pd.DataFrame(bmi_outliers)\n#waist_outliers_df = pd.DataFrame(waist_outliers)\n#diastolic_outliers_df = pd.DataFrame(diastolic_outliers)\n#heartrate_outliers_df = pd.DataFrame(heartrate_outliers)\n#systolic_outliers_df = pd.DataFrame(systolic_outliers)\n\nwaist_outliers_df = pd.DataFrame(waist_outliers, columns=['id', 'age', 'sex', 'waist_outlier'])\ndiastolic_outliers_df = pd.DataFrame(diastolic_outliers, columns=['id', 'age', 'sex', 'diastolic_outlier'])\nheartrate_outliers_df = pd.DataFrame(heartrate_outliers, columns=['id', 'age', 'sex', 'heartrate_outlier'])\nsystolic_outliers_df = pd.DataFrame(systolic_outliers, columns=['id', 'age', 'sex', 'systolic_outlier'])\n\n# Merge outlier dataframes\noutlier_df = pd.merge(height_outliers_df, weight_outliers_df, on=['id', 'age', 'sex'], how='outer')\n# outlier_df = pd.merge(outlier_df, bmi_outliers_df, on=['id', 'age', 'sex'], how='outer')\noutlier_df = pd.merge(outlier_df, waist_outliers_df, on=['id', 'age', 'sex'], how='outer')\noutlier_df = pd.merge(outlier_df, diastolic_outliers_df, on=['id', 'age', 'sex'], how='outer')\noutlier_df = pd.merge(outlier_df, heartrate_outliers_df, on=['id', 'age', 'sex'], how='outer')\noutlier_df = pd.merge(outlier_df, systolic_outliers_df, on=['id', 'age', 'sex'], how='outer')\n\n# id sort\noutlier_df = outlier_df.sort_values(by='id')\noutlier_df.to_csv('outliers_physicals_without_BMI.csv', index=False)\n\n# manually check if the outliers are correct :) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.253891Z","iopub.execute_input":"2025-05-20T19:30:42.254386Z","iopub.status.idle":"2025-05-20T19:30:42.704557Z","shell.execute_reply.started":"2025-05-20T19:30:42.254345Z","shell.execute_reply":"2025-05-20T19:30:42.703199Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.707722Z","iopub.execute_input":"2025-05-20T19:30:42.708049Z","iopub.status.idle":"2025-05-20T19:30:42.713237Z","shell.execute_reply.started":"2025-05-20T19:30:42.708022Z","shell.execute_reply":"2025-05-20T19:30:42.711965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# copy of the dataset for modifications :)\nds_copy = df.copy()\n\nds = df\n\ntest_ds = df_test.copy()\n\n# load outliers_BIA.csv\nbia = pd.read_csv('outliers_BIA.csv')\n\n# load outliers_physicals_wihtout_BMI.csv\nphysicals = pd.read_csv('outliers_physicals_without_BMI.csv')    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.715152Z","iopub.execute_input":"2025-05-20T19:30:42.715476Z","iopub.status.idle":"2025-05-20T19:30:42.744899Z","shell.execute_reply.started":"2025-05-20T19:30:42.715449Z","shell.execute_reply":"2025-05-20T19:30:42.743511Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# List of columns to check and replace 0 values with NaN\ncolumns_physical_outlier = [\n    'Physical-Height', 'Physical-Weight', 'Physical-BMI', \n    'Physical-Waist_Circumference', 'Physical-Diastolic_BP', \n    'Physical-HeartRate', 'Physical-Systolic_BP'\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.746319Z","iopub.execute_input":"2025-05-20T19:30:42.746739Z","iopub.status.idle":"2025-05-20T19:30:42.752373Z","shell.execute_reply.started":"2025-05-20T19:30:42.746700Z","shell.execute_reply":"2025-05-20T19:30:42.751089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# replace 0 with NaN\nds_copy[columns_physical_outlier] = ds_copy[columns_physical_outlier].replace(0, np.nan)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.753610Z","iopub.execute_input":"2025-05-20T19:30:42.753920Z","iopub.status.idle":"2025-05-20T19:30:42.774746Z","shell.execute_reply.started":"2025-05-20T19:30:42.753885Z","shell.execute_reply":"2025-05-20T19:30:42.773749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check for any remaining 0 values\nzero_counts = ds_copy[columns_physical_outlier].eq(0).sum()\n\n# Output the count of remaining zeros\nprint(\"Count of remaining 0 values in each column:\")\nprint(zero_counts)\n\n# check NaN values\nprint(\"\\nSample rows with NaN values after replacement:\")\nprint(ds_copy[columns_physical_outlier].isna().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.775782Z","iopub.execute_input":"2025-05-20T19:30:42.776079Z","iopub.status.idle":"2025-05-20T19:30:42.803701Z","shell.execute_reply.started":"2025-05-20T19:30:42.776054Z","shell.execute_reply":"2025-05-20T19:30:42.802444Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# List of columns to check for outliers in physicals csv\noutlier_columns = ['height_outlier', 'weight_outlier', \n                   'waist_outlier', 'diastolic_outlier', 'heartrate_outlier', \n                   'systolic_outlier']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.804794Z","iopub.execute_input":"2025-05-20T19:30:42.805122Z","iopub.status.idle":"2025-05-20T19:30:42.810072Z","shell.execute_reply.started":"2025-05-20T19:30:42.805095Z","shell.execute_reply":"2025-05-20T19:30:42.809022Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" # copy of ds_copy and set the outliers to NaN to calculate the mean\nds_copy_copy = ds_copy.copy()\n\n# Loop over each outlier column and check the outliers for each id\nfor outlier_col, physical_col in zip(outlier_columns, columns_physical_outlier):\n    # Get the rows where outliers are marked (i.e., not NaN in the outlier column)\n    outlier_ids = physicals[physicals[outlier_col].notna()]['id'].values\n\n    # Replace the corresponding physical measurement values with NaN in ds_copy_copy\n    ds_copy_copy.loc[ds_copy_copy['id'].isin(outlier_ids), physical_col] = np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.811174Z","iopub.execute_input":"2025-05-20T19:30:42.811547Z","iopub.status.idle":"2025-05-20T19:30:42.843419Z","shell.execute_reply.started":"2025-05-20T19:30:42.811512Z","shell.execute_reply":"2025-05-20T19:30:42.841970Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Filter the outliers_physicals dataset to check outliers for a specific ID\nds_copy_copy_filtered = ds_copy_copy[ds_copy_copy['id'] == '00e6167c']\nphysicals_filtered = physicals[physicals['id'] == '00e6167c']\n\n# Check if the outliers for this ID are NaN\nprint(\"outliers replaced with NaN\")\nprint(ds_copy_copy_filtered[['id', 'Physical-Height', 'Physical-Weight', \n                              'Physical-Waist_Circumference', 'Physical-Diastolic_BP', \n                              'Physical-HeartRate', 'Physical-Systolic_BP']])\n\nprint(\"outliers from outliers_phyiscals\")\nprint(physicals_filtered)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.844741Z","iopub.execute_input":"2025-05-20T19:30:42.845148Z","iopub.status.idle":"2025-05-20T19:30:42.860195Z","shell.execute_reply.started":"2025-05-20T19:30:42.845110Z","shell.execute_reply":"2025-05-20T19:30:42.858803Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate mean for each column grouped by age & sex\nmean_values = ds_copy_copy.groupby(['Basic_Demos-Age', 'Basic_Demos-Sex'])[columns_physical_outlier].mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.861725Z","iopub.execute_input":"2025-05-20T19:30:42.862139Z","iopub.status.idle":"2025-05-20T19:30:42.891885Z","shell.execute_reply.started":"2025-05-20T19:30:42.862090Z","shell.execute_reply":"2025-05-20T19:30:42.890642Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Loop over the outlier columns and corresponding physical columns\nfor outlier_col, physical_col in zip(outlier_columns, columns_physical_outlier):\n    # Get the rows where outliers are marked (i.e., not NaN in the outlier column)\n    outlier_ids = physicals[physicals[outlier_col].notna()]['id'].values\n    \n    # Iterate through each id and replace the outlier values in ds_copy at the cell level\n    for id_value in outlier_ids:\n        # Find the corresponding mean value for this id's age and sex\n        age = ds_copy.loc[ds_copy['id'] == id_value, 'Basic_Demos-Age'].values[0]\n        sex = ds_copy.loc[ds_copy['id'] == id_value, 'Basic_Demos-Sex'].values[0]\n        \n        # Get the mean value for this age and sex group\n        mean_value = mean_values.loc[(age, sex), physical_col]\n        \n        # Replace the outlier value in the specific cell of ds_copy with the mean value\n        ds_copy.loc[(ds_copy['id'] == id_value), physical_col] = ds_copy.loc[\n            (ds_copy['id'] == id_value) & \n            pd.notna(ds_copy[physical_col]), physical_col\n        ].apply(lambda x: mean_value if pd.notna(x) else x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:42.893537Z","iopub.execute_input":"2025-05-20T19:30:42.893967Z","iopub.status.idle":"2025-05-20T19:30:43.383790Z","shell.execute_reply.started":"2025-05-20T19:30:42.893911Z","shell.execute_reply":"2025-05-20T19:30:43.382567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Filter ds_copy to check the cell value for ID 00e6167c in Physical-Height\nds_copy_filtered = ds_copy[ds_copy['id'] == '00e6167c']\n\n# Print the filtered cell to check the updated data\nprint(ds_copy_filtered[['id', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-Height', 'Physical-Weight',  \n                              'Physical-Waist_Circumference', 'Physical-Diastolic_BP', \n                              'Physical-HeartRate', 'Physical-Systolic_BP']])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:43.384815Z","iopub.execute_input":"2025-05-20T19:30:43.385147Z","iopub.status.idle":"2025-05-20T19:30:43.397530Z","shell.execute_reply.started":"2025-05-20T19:30:43.385118Z","shell.execute_reply":"2025-05-20T19:30:43.396080Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# calculate BMI for the original dataset\n\n# Caclulate BMI (Umrechnung der Einheiten)\nds['BMI_Calculated'] = (ds['Physical-Weight'] * 0.453592) / ((ds['Physical-Height'] * 0.0254) ** 2)\n\n# compare with existing values\nds['BMI_Correct'] = ds['BMI_Calculated'].round(2) == ds['Physical-BMI'].round(2)\n\n# Round both columns to 2 decimal places and compare\nincorrect_bmi = ds[ds['Physical-BMI'].notna() & ds['BMI_Calculated'].notna() &\n                   (round(ds['BMI_Calculated'], 0) != round(ds['Physical-BMI'], 0))]\n\n# Display the rows where the BMI is incorrect\nincorrect_bmi[['Physical-Weight', 'Physical-Height', 'Physical-BMI', 'BMI_Calculated']]\n\n# seems to be ok :) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:43.398791Z","iopub.execute_input":"2025-05-20T19:30:43.399254Z","iopub.status.idle":"2025-05-20T19:30:43.437404Z","shell.execute_reply.started":"2025-05-20T19:30:43.399207Z","shell.execute_reply":"2025-05-20T19:30:43.435923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# calculate BMI for ds_copy\n\n# Caclulate BMI (Umrechnung der Einheiten)\nds_copy['BMI_Calculated'] = (ds_copy['Physical-Weight'] * 0.453592) / ((ds_copy['Physical-Height'] * 0.0254) ** 2)\n\n# compare with existing values\nds_copy['BMI_Correct'] = ds_copy['BMI_Calculated'].round(4) == ds_copy['Physical-BMI'].round(4)\n\n# Round both columns to 2 decimal places and compare\nincorrect_bmi = ds_copy[ds_copy['Physical-BMI'].notna() & ds_copy['BMI_Calculated'].notna() &\n                   (round(ds_copy['BMI_Calculated'], 0) != round(ds['Physical-BMI'], 0))]\n\n# Display the rows where the BMI is incorrect\nincorrect_bmi[['id', 'Physical-Weight', 'Physical-Height', 'Physical-BMI', 'BMI_Calculated']]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:43.438565Z","iopub.execute_input":"2025-05-20T19:30:43.439072Z","iopub.status.idle":"2025-05-20T19:30:43.464652Z","shell.execute_reply.started":"2025-05-20T19:30:43.439035Z","shell.execute_reply":"2025-05-20T19:30:43.463531Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" # For these rows, overwrite Physical-BMi with BMI_Calculated\n\n# Iterate through the rows in ds_copy to replace incorrect Physical-BMI values with BMI_Calculated\nfor index, row in incorrect_bmi.iterrows():\n    id_value = row['id']\n    calculated_bmi = row['BMI_Calculated']\n    \n    # Replace the incorrect Physical-BMI value with the calculated BMI value\n    ds_copy.loc[ds_copy['id'] == id_value, 'Physical-BMI'] = calculated_bmi\n    \n\n# Recheck the replaced values and filter again for any remaining incorrect BMI values\nds_copy['BMI_Correct'] = ds_copy['BMI_Calculated'].round(2) == ds_copy['Physical-BMI'].round(2)\n\n# Filter rows where the BMI is still incorrect\nincorrect_bmi_after = ds_copy[ds_copy['Physical-BMI'].notna() & ds_copy['BMI_Calculated'].notna() & \n                              (round(ds_copy['BMI_Calculated'], 0) != round(ds_copy['Physical-BMI'], 0))]\n\n# Display the rows where the BMI is still incorrect after replacement\nincorrect_bmi_after[['id', 'Physical-Weight', 'Physical-Height', 'Physical-BMI', 'BMI_Calculated']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:43.465787Z","iopub.execute_input":"2025-05-20T19:30:43.466185Z","iopub.status.idle":"2025-05-20T19:30:43.539024Z","shell.execute_reply.started":"2025-05-20T19:30:43.466148Z","shell.execute_reply":"2025-05-20T19:30:43.537914Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for index, row in incorrect_bmi_after.iterrows():\n    id_value = row['id']\n    calculated_bmi = row['BMI_Calculated']\n    \n    # Replace the incorrect Physical-BMI value with the calculated BMI value\n    ds_copy.loc[ds_copy['id'] == id_value, 'Physical-BMI'] = calculated_bmi\n    \n\n# Recheck the replaced values and filter again for any remaining incorrect BMI values\nds_copy['BMI_Correct'] = ds_copy['BMI_Calculated'].round(2) == ds_copy['Physical-BMI'].round(2)\n\n# Filter rows where the BMI is still incorrect\nincorrect_bmi_after2= ds_copy[ds_copy['Physical-BMI'].notna() & ds_copy['BMI_Calculated'].notna() & \n                              (round(ds_copy['BMI_Calculated'], 0) != round(ds_copy['Physical-BMI'], 0))]\n\n# Display the rows where the BMI is still incorrect after replacement\nincorrect_bmi_after2[['id', 'Physical-Weight', 'Physical-Height', 'Physical-BMI', 'BMI_Calculated']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:43.540644Z","iopub.execute_input":"2025-05-20T19:30:43.541086Z","iopub.status.idle":"2025-05-20T19:30:43.562105Z","shell.execute_reply.started":"2025-05-20T19:30:43.541044Z","shell.execute_reply":"2025-05-20T19:30:43.561080Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# The file outliers_BIA.csv was not accurate enough, so we are manually identifying and handling the outliers in the BIA columns. :)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:43.563220Z","iopub.execute_input":"2025-05-20T19:30:43.563501Z","iopub.status.idle":"2025-05-20T19:30:43.581774Z","shell.execute_reply.started":"2025-05-20T19:30:43.563478Z","shell.execute_reply":"2025-05-20T19:30:43.580582Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" # remove rows with id cedf96c5 and e252dcb6\nds_copy = ds_copy[~ds_copy['id'].isin(['cedf96c5', 'e252dcb6'])]\n\n# BIA-BIA_BMC: If greater than 25 or less than 0.1, set NaN\nds_copy['BIA-BIA_BMC'] = ds_copy['BIA-BIA_BMC'].apply(lambda x: np.nan if x > 25 or x < 0.1 else x)\n\n# BIA-BIA_BMI: If greater than 45 or less than 10, set NaN\nds_copy['BIA-BIA_BMI'] = ds_copy['BIA-BIA_BMI'].apply(lambda x: np.nan if x > 45 or x < 10 else x)\n\n# BIA-BIA_BMR: If greater than 4000, set NaN\nds_copy['BIA-BIA_BMR'] = ds_copy['BIA-BIA_BMR'].apply(lambda x: np.nan if x > 4000 else x)\n\n# BIA-BIA_DEE: If greater than 5000, set NaN\nds_copy['BIA-BIA_DEE'] = ds_copy['BIA-BIA_DEE'].apply(lambda x: np.nan if x > 5000 else x)\n\n# BIA-BIA_ECW: If greater than 60 or less than 3, set NaN\nds_copy['BIA-BIA_ECW'] = ds_copy['BIA-BIA_ECW'].apply(lambda x: np.nan if x > 60 or x < 3 else x)\n\n# BIA-BIA_FFMI: If greater than 40 or less than 10, set NaN\nds_copy['BIA-BIA_FFMI'] = ds_copy['BIA-BIA_FFMI'].apply(lambda x: np.nan if x > 40 or x < 10 else x)\n\n# BIA-BIA_FMI: If less than 0, set NaN, if greater than weight, replace with weight\nds_copy['BIA-BIA_FMI'] = ds_copy.apply(lambda row: np.nan if row['BIA-BIA_FMI'] < 0 else (row['Physical-Weight'] if row['BIA-BIA_FMI'] > row['Physical-Weight'] else row['BIA-BIA_FMI']), axis=1)\n\n# BIA-BIA_Fat: If less than 0, set NaN\nds_copy['BIA-BIA_Fat'] = ds_copy['BIA-BIA_Fat'].apply(lambda x: np.nan if x < 0 else x)\n\n# BIA-BIA_ICW: If greater than 100, set NaN\nds_copy['BIA-BIA_ICW'] = ds_copy['BIA-BIA_ICW'].apply(lambda x: np.nan if x > 100 else x)\n\n# BIA-BIA_LDM: If greater than 100, set NaN\nds_copy['BIA-BIA_LDM'] = ds_copy['BIA-BIA_LDM'].apply(lambda x: np.nan if x > 100 else x)\n\n# BIA-BIA_LST: If greater than weight, replace with weight\nds_copy['BIA-BIA_LST'] = ds_copy.apply(lambda row: row['Physical-Weight'] if row['BIA-BIA_LST'] > row['Physical-Weight'] else row['BIA-BIA_LST'], axis=1)\n\n# BIA-BIA_SMM: If less than 10 or greater than weight, set NaN\nds_copy['BIA-BIA_SMM'] = ds_copy.apply(lambda row: np.nan if row['BIA-BIA_SMM'] < 10 or row['BIA-BIA_SMM'] > row['Physical-Weight'] else row['BIA-BIA_SMM'], axis=1)\n\n# BIA-BIA_TBW: If greater than weight, set NaN\nds_copy['BIA-BIA_TBW'] = ds_copy.apply(lambda row: np.nan if row['BIA-BIA_TBW'] > row['Physical-Weight'] else row['BIA-BIA_TBW'], axis=1)\n\n# BIA-BIA_FFM: If greater than weight, replace with weight\nds_copy['BIA-BIA_FFM'] = ds_copy.apply(lambda row: row['Physical-Weight'] if row['BIA-BIA_FFM'] > row['Physical-Weight'] else row['BIA-BIA_FFM'], axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:43.583067Z","iopub.execute_input":"2025-05-20T19:30:43.583395Z","iopub.status.idle":"2025-05-20T19:30:43.929915Z","shell.execute_reply.started":"2025-05-20T19:30:43.583367Z","shell.execute_reply":"2025-05-20T19:30:43.928261Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\ndef clean_tbw_ffm(df):\n\n    df = df.copy()  # Work on a copy to avoid modifying the original\n    \n    # Check for required columns\n    required_cols = ['BIA-BIA_TBW', 'BIA-BIA_FFM', 'Physical-Weight']\n    if not all(col in df.columns for col in required_cols):\n        missing = [col for col in required_cols if col not in df.columns]\n        print(f\"Warning: Missing required columns: {missing}\")\n        return df\n    \n    # First pass: mark any impossible values\n    tbw_impossible = (df['BIA-BIA_TBW'] > df['Physical-Weight']) & df['BIA-BIA_TBW'].notna() & df['Physical-Weight'].notna()\n    ffm_impossible = (df['BIA-BIA_FFM'] > df['Physical-Weight']) & df['BIA-BIA_FFM'].notna() & df['Physical-Weight'].notna()\n    \n    # Case 1: TBW is possible but FFM is impossible - recalculate FFM from TBW\n    recalc_ffm = (~tbw_impossible) & ffm_impossible & df['BIA-BIA_TBW'].notna()\n    if recalc_ffm.sum() > 0:\n        print(f\"Case 1: Recalculating {recalc_ffm.sum()} FFM values from valid TBW values\")\n        # Print details of each recalculation\n        for idx in df[recalc_ffm].index:\n            original_ffm = df.loc[idx, 'BIA-BIA_FFM']\n            weight = df.loc[idx, 'Physical-Weight']\n            tbw = df.loc[idx, 'BIA-BIA_TBW']\n            new_ffm_raw = tbw / 0.73\n            \n            # Check if new FFM also exceeds weight\n            if new_ffm_raw > weight:\n                print(f\"  Row {idx}: FFM {original_ffm:.2f} > Weight {weight:.2f}, recalculated FFM {new_ffm_raw:.2f} still > Weight, capping at {weight:.2f}\")\n                new_ffm = weight\n            else:\n                print(f\"  Row {idx}: FFM {original_ffm:.2f} > Weight {weight:.2f}, recalculating from TBW {tbw:.2f} → New FFM: {new_ffm_raw:.2f}\")\n                new_ffm = new_ffm_raw\n                \n            df.loc[idx, 'BIA-BIA_FFM'] = new_ffm\n    \n    # Case 2: FFM is possible but TBW is impossible - recalculate TBW from FFM\n    recalc_tbw = tbw_impossible & (~ffm_impossible) & df['BIA-BIA_FFM'].notna()\n    if recalc_tbw.sum() > 0:\n        print(f\"Case 2: Recalculating {recalc_tbw.sum()} TBW values from valid FFM values\")\n        # Print details of each recalculation\n        for idx in df[recalc_tbw].index:\n            original_tbw = df.loc[idx, 'BIA-BIA_TBW']\n            weight = df.loc[idx, 'Physical-Weight']\n            ffm = df.loc[idx, 'BIA-BIA_FFM']\n            new_tbw_raw = ffm * 0.73\n            \n            # Check if new TBW also exceeds weight\n            if new_tbw_raw > weight:\n                print(f\"  Row {idx}: TBW {original_tbw:.2f} > Weight {weight:.2f}, recalculated TBW {new_tbw_raw:.2f} still > Weight, setting to NaN\")\n                new_tbw = np.nan\n            else:\n                print(f\"  Row {idx}: TBW {original_tbw:.2f} > Weight {weight:.2f}, recalculating from FFM {ffm:.2f} → New TBW: {new_tbw_raw:.2f}\")\n                new_tbw = new_tbw_raw\n                \n            df.loc[idx, 'BIA-BIA_TBW'] = new_tbw\n    \n    # Case 3: Both are impossible - set TBW to NaN and FFM to weight\n    both_impossible = tbw_impossible & ffm_impossible\n    if both_impossible.sum() > 0:\n        print(f\"Case 3: Found {both_impossible.sum()} rows where both TBW and FFM exceed weight\")\n        # Print details of each case\n        for idx in df[both_impossible].index:\n            tbw = df.loc[idx, 'BIA-BIA_TBW']\n            ffm = df.loc[idx, 'BIA-BIA_FFM']\n            weight = df.loc[idx, 'Physical-Weight']\n            print(f\"  Row {idx}: TBW {tbw:.2f} > Weight {weight:.2f}, FFM {ffm:.2f} > Weight {weight:.2f}\")\n            print(f\"      Setting TBW to NaN and FFM to {weight:.2f}\")\n        df.loc[both_impossible, 'BIA-BIA_TBW'] = np.nan\n        df.loc[both_impossible, 'BIA-BIA_FFM'] = df.loc[both_impossible, 'Physical-Weight']\n    \n    # Final pass: ensure any missing checks are caught\n    remaining_tbw_impossible = (df['BIA-BIA_TBW'] > df['Physical-Weight']) & df['BIA-BIA_TBW'].notna() & df['Physical-Weight'].notna()\n    remaining_ffm_impossible = (df['BIA-BIA_FFM'] > df['Physical-Weight']) & df['BIA-BIA_FFM'].notna() & df['Physical-Weight'].notna()\n    \n    if remaining_tbw_impossible.sum() > 0:\n        print(f\"Final check: Found {remaining_tbw_impossible.sum()} remaining rows with TBW > Weight, setting to NaN\")\n        df.loc[remaining_tbw_impossible, 'BIA-BIA_TBW'] = np.nan\n        \n    if remaining_ffm_impossible.sum() > 0:\n        print(f\"Final check: Found {remaining_ffm_impossible.sum()} remaining rows with FFM > Weight, capping at Weight\")\n        df.loc[remaining_ffm_impossible, 'BIA-BIA_FFM'] = df.loc[remaining_ffm_impossible, 'Physical-Weight']\n    \n    return df\n\n# Example usage\n#df_cleaned = clean_tbw_ffm(df)\n#ds_copy = clean_tbw_ffm(ds_copy)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:43.931240Z","iopub.execute_input":"2025-05-20T19:30:43.931650Z","iopub.status.idle":"2025-05-20T19:30:43.947640Z","shell.execute_reply.started":"2025-05-20T19:30:43.931605Z","shell.execute_reply":"2025-05-20T19:30:43.946159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# check one column\nsorted_bmc = ds_copy.sort_values(by='BIA-BIA_BMC', ascending=True)\n\n# show 10 first rows\nprint(sorted_bmc[['id', 'BIA-BIA_BMC']].head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:43.954384Z","iopub.execute_input":"2025-05-20T19:30:43.954777Z","iopub.status.idle":"2025-05-20T19:30:43.984535Z","shell.execute_reply.started":"2025-05-20T19:30:43.954747Z","shell.execute_reply":"2025-05-20T19:30:43.983483Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_columns = set(ds_copy.columns)\ntest_columns = set(test_ds.columns)\n\nonly_in_train = sorted(train_columns - test_columns)\nonly_in_test = sorted(test_columns - train_columns)\n\nmax_len = max(len(only_in_train), len(only_in_test))\nonly_in_train += [''] * (max_len - len(only_in_train))\nonly_in_test += [''] * (max_len - len(only_in_test))\n\n\ndf_diff = pd.DataFrame({'Only in Train': only_in_train, \n                        ' ': [' ']*max_len,  # empty column for space\n                        'Only in Test': only_in_test})\n\nprint(df_diff.to_string(index=False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:43.986842Z","iopub.execute_input":"2025-05-20T19:30:43.987357Z","iopub.status.idle":"2025-05-20T19:30:44.007090Z","shell.execute_reply.started":"2025-05-20T19:30:43.987322Z","shell.execute_reply":"2025-05-20T19:30:44.005743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def check_sii_range(row):\n    if pd.notna(row['PCIAT-PCIAT_Total']) and pd.notna(row['sii']):\n        if row['sii'] == 0 and (0 <= row['PCIAT-PCIAT_Total'] < 31):\n            return True\n        elif row['sii'] == 1 and (31 <= row['PCIAT-PCIAT_Total'] < 50):\n            return True\n        elif row['sii'] == 2 and (50 <= row['PCIAT-PCIAT_Total'] < 80):\n            return True\n        elif row['sii'] == 3 and (80 <= row['PCIAT-PCIAT_Total'] <= 100):\n            return True\n    return False\n\nds_copy['Correct_sii'] = ds_copy.apply(check_sii_range, axis=1)\n\n# Zeige die Zeilen, bei denen die Übereinstimmung nicht korrekt ist und keine NaN-Werte vorliegen\nincorrect_rows = ds_copy[~ds_copy['Correct_sii'] & pd.notna(ds_copy['PCIAT-PCIAT_Total']) & pd.notna(ds_copy['sii'])]\n\n# Gib die falschen Zeilen aus\nprint(incorrect_rows[['id', 'PCIAT-PCIAT_Total', 'sii']])\n\n# Optional: Ausgabe der Anzahl der falschen Übereinstimmungen\nincorrect_count = len(incorrect_rows)\nprint(f\"Falsche Übereinstimmungen: {incorrect_count}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:44.008139Z","iopub.execute_input":"2025-05-20T19:30:44.008486Z","iopub.status.idle":"2025-05-20T19:30:44.110231Z","shell.execute_reply.started":"2025-05-20T19:30:44.008459Z","shell.execute_reply":"2025-05-20T19:30:44.108757Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CGAs if over 100 to Nan\nds_copy['CGAS-CGAS_Score'] = ds_copy['CGAS-CGAS_Score'].mask(ds_copy['CGAS-CGAS_Score'] > 100)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:44.111178Z","iopub.execute_input":"2025-05-20T19:30:44.111488Z","iopub.status.idle":"2025-05-20T19:30:44.118414Z","shell.execute_reply.started":"2025-05-20T19:30:44.111461Z","shell.execute_reply":"2025-05-20T19:30:44.117095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" # from lennarthaupts \n# https://www.kaggle.com/code/lennarthaupts/1st-place-cmi-model-v4-1-1-reduced?scriptVersionId=213769368\ndef perform_pca(train, test, n_components=None, random_state=42):\n    \n    pca = PCA(n_components=n_components, random_state=random_state)\n    train_pca = pca.fit_transform(train)\n    test_pca = pca.transform(test)\n    \n    explained_variance_ratio = pca.explained_variance_ratio_\n    print(f\"Explained variance ratio of the components:\\n {explained_variance_ratio}\")\n    print(np.sum(explained_variance_ratio))\n    \n    train_pca_df = pd.DataFrame(train_pca, columns=[f'PC_{i+1}' for i in range(train_pca.shape[1])])\n    test_pca_df = pd.DataFrame(test_pca, columns=[f'PC_{i+1}' for i in range(test_pca.shape[1])])\n    \n    return train_pca_df, test_pca_df, pca","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:44.119576Z","iopub.execute_input":"2025-05-20T19:30:44.119890Z","iopub.status.idle":"2025-05-20T19:30:44.138789Z","shell.execute_reply.started":"2025-05-20T19:30:44.119864Z","shell.execute_reply":"2025-05-20T19:30:44.137454Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from lennarthaupts \n# https://www.kaggle.com/code/lennarthaupts/1st-place-cmi-model-v4-1-1-reduced?scriptVersionId=213769368\ndef time_features(df):\n    # Convert time_of_day to hours\n    df[\"hours\"] = df[\"time_of_day\"] // (3_600 * 1_000_000_000)\n    # Basic features \n    features = [\n        df[\"non-wear_flag\"].mean(),\n        df[\"enmo\"][df[\"enmo\"] >= 0.05].sum(),\n    ]\n    \n    # Define conditions for night, day, and no mask (full data)\n    night = ((df[\"hours\"] >= 22) | (df[\"hours\"] <= 5))\n    day = ((df[\"hours\"] <= 20) & (df[\"hours\"] >= 7))\n    no_mask = np.ones(len(df), dtype=bool)\n    \n    # List of columns of interest and masks\n    keys = [\"enmo\", \"anglez\", \"light\", \"battery_voltage\"]\n    masks = [no_mask, night, day]\n    \n    # Helper function for feature extraction\n    def extract_stats(data):\n        return [\n            data.mean(), \n            data.std(), \n            data.max(), \n            data.min(), \n            data.diff().mean(), \n            data.diff().std()\n        ]\n    \n    # Iterate over keys and masks to generate the statistics\n    for key in keys:\n        for mask in masks:\n            filtered_data = df.loc[mask, key]\n            features.extend(extract_stats(filtered_data))\n\n    return features\n\n# Code for parallelized computation of time series data from: Sheikh Muhammad Abdullah \n# https://www.kaggle.com/code/abdmental01/cmi-best-single-model\ndef process_file(filename, dirname):\n    # Process file and extract time features\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return time_features(df), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    # Load time series from directory in parallel\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:44.140022Z","iopub.execute_input":"2025-05-20T19:30:44.140400Z","iopub.status.idle":"2025-05-20T19:30:44.164531Z","shell.execute_reply.started":"2025-05-20T19:30:44.140370Z","shell.execute_reply":"2025-05-20T19:30:44.163329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# load time series\ntrain_ts = load_time_series('/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet')\n\ntest_ts = load_time_series('/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:30:44.165781Z","iopub.execute_input":"2025-05-20T19:30:44.166127Z","iopub.status.idle":"2025-05-20T19:31:56.308060Z","shell.execute_reply.started":"2025-05-20T19:30:44.166099Z","shell.execute_reply":"2025-05-20T19:31:56.306862Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from lennarthaupts \n# https://www.kaggle.com/code/lennarthaupts/1st-place-cmi-model-v4-1-1-reduced?scriptVersionId=213769368\n\n# drop id\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\n# scale with standardscaler\nscaler = StandardScaler()\ndf_train = pd.DataFrame(scaler.fit_transform(df_train), columns=df_train.columns)\ndf_test = pd.DataFrame(scaler.transform(df_test), columns=df_test.columns)\n\n# replace missing values with mean\nfor c in df_train.columns:\n    m = np.mean(df_train[c])\n    df_train[c] = df_train[c].fillna(m)\n    df_test[c] = df_test[c].fillna(m)\n\nprint(df_train.shape)\n\nSEED = 42\ndf_train_pca, df_test_pca, pca = perform_pca(df_train, df_test, n_components=15, random_state=SEED)\n\ndf_train_pca['id'] = train_ts['id']\ndf_test_pca['id'] = test_ts['id']\n\ntrain = pd.merge(ds_copy, df_train_pca, how=\"left\", on='id')\ntest = pd.merge(test_ds, df_test_pca, how=\"left\", on='id')\ntrain.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:56.309270Z","iopub.execute_input":"2025-05-20T19:31:56.309625Z","iopub.status.idle":"2025-05-20T19:31:56.412161Z","shell.execute_reply.started":"2025-05-20T19:31:56.309599Z","shell.execute_reply":"2025-05-20T19:31:56.410802Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n# --- Gender Code Definitions (Confirmed: Male=0, Female=1) ---\nMALE_CODE = 0\nFEMALE_CODE = 1\n\n# --- Helper function for Safe Normalization (Unchanged) ---\ndef safe_normalize(value, group, sex, map_male, map_female,\n                   male_code=MALE_CODE, female_code=FEMALE_CODE):\n    \"\"\"\n    Safely normalizes a value using age group and sex-specific maps.\n    Handles NaN groups, NaN values, unexpected sex codes, missing map keys, and division by zero.\n    \"\"\"\n    if pd.isna(group) or pd.isna(value) or pd.isna(sex):\n        return np.nan\n    try:\n        group_int = int(group) # Ensure group is int for lookup\n        if sex == male_code:\n            norm_value = map_male[group_int]\n        elif sex == female_code:\n            norm_value = map_female[group_int]\n        else:\n            return np.nan\n        if norm_value == 0 or pd.isna(norm_value):\n            return np.nan if value != 0 else 0\n        return value / norm_value\n    except KeyError:\n        # print(f\"Warning: Group {group_int} not found in normalization maps for sex code {sex}.\") # Optional\n        return np.nan\n    except Exception as e:\n        # print(f\"An error occurred during normalization (Group: {group}, Sex: {sex}): {e}\") # Optional\n        return np.nan\n\ndef feature_engineering(df):\n    \"\"\"\n    Performs feature engineering on the input DataFrame.\n    - Creates age groups.\n    - Normalizes various metrics (BMI, Grip, Fitness Tests, BIA, Physical) by age group and sex.\n    - Uses updated, more plausible normalization values where available.\n    - Creates aggregate features.\n    - Drops original and intermediate columns.\n    \"\"\"\n    df = df.copy() # Work on a copy\n\n    # Drop season columns if they exist\n    season_cols = [col for col in df.columns if 'Season' in col]\n    if season_cols:\n        df = df.drop(columns=season_cols, axis=1)\n\n    # --- Age Grouping (Unchanged) ---\n    def assign_group(age):\n        if pd.isna(age) or not isinstance(age, (int, float)):\n             return np.nan\n        thresholds = [5, 6, 7, 8, 10, 12, 14, 17, 22]\n        for i, j in enumerate(thresholds):\n            if age <= j:\n                return i\n        return np.nan\n    df[\"group\"] = df['Basic_Demos-Age'].apply(assign_group)\n\n    # --- Normalization Maps (Using plausible suggestions - VALIDATE WITH SOURCE DATA & UNITS!) ---\n\n    # Physical Measurements - ** ADDED - VALIDATE UNITS/VALUES **\n    # Height (cm?)\n    Height_map_male =   {0: 110, 1: 116, 2: 122, 3: 128, 4: 138, 5: 150, 6: 163, 7: 175, 8: 177} # Suggestion\n    Height_map_female = {0: 109, 1: 115, 2: 121, 3: 127, 4: 137, 5: 151, 6: 160, 7: 163, 8: 164} # Suggestion\n    # Weight (kg?)\n    Weight_map_male =   {0: 18.5, 1: 20.5, 2: 23.0, 3: 26.0, 4: 32.0, 5: 40.0, 6: 52.0, 7: 65.0, 8: 70.0} # Suggestion\n    Weight_map_female = {0: 18.0, 1: 20.0, 2: 22.5, 3: 25.0, 4: 31.0, 5: 41.0, 6: 50.0, 7: 55.0, 8: 58.0} # Suggestion\n    # Waist Circumference (cm?)\n    Waist_map_male =   {0: 52, 1: 54, 2: 56, 3: 58, 4: 62, 5: 67, 6: 73, 7: 78, 8: 82} # Suggestion\n    Waist_map_female = {0: 51, 1: 53, 2: 55, 3: 57, 4: 61, 5: 66, 6: 70, 7: 74, 8: 76} # Suggestion\n\n    # --- Other Maps (Existing - still need validation) ---\n    # BMI (kg/m^2 ?)\n    BMI_map_male =   {0: 16.3, 1: 15.9, 2: 16.1, 3: 16.8, 4: 17.3, 5: 19.2, 6: 20.2, 7: 22.3, 8: 23.6}\n    BMI_map_female = {0: 15.8, 1: 15.5, 2: 15.8, 3: 16.4, 4: 17.1, 5: 18.8, 6: 19.8, 7: 21.5, 8: 22.9}\n    # Grip Strength (GSD) (kg?)\n    GSD_max_map_male =   {0: 7.0,  1: 9.0,  2: 11.0, 3: 13.0, 4: 18.0, 5: 24.0, 6: 32.0, 7: 38.0, 8: 42.0}\n    GSD_min_map_male =   {0: 6.0,  1: 8.0,  2: 10.0, 3: 12.0, 4: 16.0, 5: 22.0, 6: 29.0, 7: 34.0, 8: 37.0}\n    GSD_max_map_female = {0: 6.0,  1: 8.0,  2: 10.0, 3: 12.0, 4: 16.0, 5: 20.0, 6: 25.0, 7: 28.0, 8: 30.0}\n    GSD_min_map_female = {0: 5.0,  1: 7.0,  2: 9.0,  3: 11.0, 4: 14.0, 5: 18.0, 6: 22.0, 7: 25.0, 8: 27.0}\n    # Curl-ups (CU) (Reps/min?)\n    cu_map_male =   {0: 2.0, 1: 4.0, 2: 7.0, 3: 10.0, 4: 14.0, 5: 18.0, 6: 22.0, 7: 24.0, 8: 25.0}\n    cu_map_female = {0: 2.0, 1: 3.0, 2: 6.0, 3: 9.0,  4: 13.0, 5: 16.0, 6: 19.0, 7: 21.0, 8: 22.0}\n    # Push-ups (PU) (Standard Reps?)\n    pu_map_male =   {0: 1.0, 1: 2.0, 2: 3.0, 3: 5.0, 4: 7.0,  5: 10.0, 6: 15.0, 7: 20.0, 8: 25.0}\n    pu_map_female = {0: 1.0, 1: 2.0, 2: 3.0, 3: 4.0, 4: 5.0,  5: 7.0,  6: 8.0,  7: 9.0,  8: 10.0}\n    # Trunk-lifts (TL) (Inches/cm?)\n    tl_map_male =   {0: 7.0, 1: 8.0, 2: 8.0, 3: 9.0, 4: 9.0,  5: 10.0, 6: 11.0, 7: 12.0, 8: 12.0}\n    tl_map_female = {0: 7.0, 1: 7.0, 2: 8.0, 3: 8.0, 4: 9.0,  5: 9.0,  6: 10.0, 7: 11.0, 8: 11.0}\n    # BMR (kcal/day?)\n    bmr_map_male =   {0: 934.0, 1: 941.0, 2: 999.0, 3: 1048.0, 4: 1283.0, 5: 1350.0, 6: 1481.0, 7: 1519.0, 8: 1650.0}\n    bmr_map_female = {0: 865.0, 1: 875.0, 2: 924.0, 3: 972.0,  4: 1203.0, 5: 1250.0, 6: 1385.0, 7: 1430.0, 8: 1560.0}\n    # DEE (kcal/day?)\n    dee_map_male =   {0: 1471.0, 1: 1508.0, 2: 1640.0, 3: 1735.0, 4: 2132.0, 5: 2250.0, 6: 2528.0, 7: 2566.0, 8: 2793.0}\n    dee_map_female = {0: 1400.0, 1: 1450.0, 2: 1570.0, 3: 1650.0, 4: 2000.0, 5: 2100.0, 6: 2300.0, 7: 2400.0, 8: 2600.0}\n    # FFM (kg or lbs?) - ** VERIFY UNITS/VALUES **\n    ffm_map_male   = {0: 42.0, 1: 43.0, 2: 49.0, 3: 54.0, 4: 60.0, 5: 76.0, 6: 94.0, 7: 104.0, 8: 111.0}\n    ffm_map_female = {0: 40.0, 1: 41.0, 2: 46.0, 3: 51.0, 4: 57.0, 5: 72.0, 6: 88.0, 7: 98.0, 8: 105.0}\n    # ECW (L?)\n    ecw_map_male   = {0: 9.0, 1: 9.5, 2: 10.0, 3: 10.5, 4: 12.0, 5: 15.0, 6: 18.0, 7: 20.0, 8: 22.0}\n    ecw_map_female = {0: 8.5, 1: 9.0, 2: 9.5, 3: 10.0, 4: 11.5, 5: 14.0, 6: 16.5, 7: 18.5, 8: 20.0}\n    # ICW (L?)\n    icw_map_male   = {0: 15.0, 1: 16.0, 2: 17.5, 3: 18.5, 4: 20.0, 5: 22.5, 6: 25.0, 7: 27.0, 8: 29.0}\n    icw_map_female = {0: 14.0, 1: 15.0, 2: 16.5, 3: 17.5, 4: 19.0, 5: 21.0, 6: 23.0, 7: 25.0, 8: 27.0}\n    # BMC (kg?)\n    BMC_map_male =   {0: 0.8, 1: 0.9, 2: 1.1, 3: 1.3, 4: 1.6, 5: 2.0, 6: 2.5, 7: 3.0, 8: 3.2}\n    BMC_map_female = {0: 0.7, 1: 0.8, 2: 1.0, 3: 1.2, 4: 1.5, 5: 1.8, 6: 2.2, 7: 2.5, 8: 2.6}\n    # Fat (%)\n    Fat_map_male =   {0: 16.0, 1: 15.0, 2: 15.5, 3: 16.5, 4: 17.0, 5: 16.0, 6: 15.0, 7: 14.5, 8: 15.0}\n    Fat_map_female = {0: 18.0, 1: 17.5, 2: 18.0, 3: 19.0, 4: 21.0, 5: 23.0, 6: 25.0, 7: 26.0, 8: 27.0}\n    # SMM (kg?)\n    SMM_map_male =   {0: 10.0, 1: 11.0, 2: 12.5, 3: 14.5, 4: 17.0, 5: 21.0, 6: 26.0, 7: 30.0, 8: 31.0}\n    SMM_map_female = {0: 9.5,  1: 10.0, 2: 11.5, 3: 12.5, 4: 14.5, 5: 17.0, 6: 19.5, 7: 21.5, 8: 22.5}\n    # TBW (L? or kg?)\n    TBW_map_male =   {0: 15.0, 1: 16.5, 2: 18.5, 3: 21.0, 4: 24.0, 5: 29.5, 6: 35.0, 7: 40.0, 8: 42.0}\n    TBW_map_female = {0: 14.0, 1: 15.0, 2: 17.0, 3: 19.0, 4: 21.5, 5: 25.5, 6: 30.0, 7: 33.5, 8: 35.0}\n    # FFMI (kg/m^2?)\n    FFMI_map_male =   {0: 12.5, 1: 12.8, 2: 13.2, 3: 13.8, 4: 14.5, 5: 16.0, 6: 18.0, 7: 20.0, 8: 21.0}\n    FFMI_map_female = {0: 12.0, 1: 12.3, 2: 12.7, 3: 13.3, 4: 14.0, 5: 15.0, 6: 16.5, 7: 17.5, 8: 18.0}\n    # FMI (kg/m^2?)\n    FMI_map_male =   {0: 2.5, 1: 2.2, 2: 2.4, 3: 2.8, 4: 3.0, 5: 3.2, 6: 3.5, 7: 3.8, 8: 4.0}\n    FMI_map_female = {0: 2.8, 1: 2.6, 2: 2.8, 3: 3.2, 4: 3.8, 5: 4.5, 6: 5.5, 7: 6.0, 8: 6.5}\n\n\n    # --- Feature Creation / Normalization ---\n\n    # --- ADDED: Physical Measurement Normalization ---\n    df[\"Height_norm\"] = df.apply(lambda x: safe_normalize(x.get(\"Physical-Height\"), x['group'], x['Basic_Demos-Sex'], Height_map_male, Height_map_female), axis=1)\n    df[\"Weight_norm\"] = df.apply(lambda x: safe_normalize(x.get(\"Physical-Weight\"), x['group'], x['Basic_Demos-Sex'], Weight_map_male, Weight_map_female), axis=1)\n    df[\"Waist_norm\"] = df.apply(lambda x: safe_normalize(x.get(\"Physical-Waist_Circumference\"), x['group'], x['Basic_Demos-Sex'], Waist_map_male, Waist_map_female), axis=1)\n\n    # --- Existing Normalizations ---\n    # BMI\n    bmi_cols = ['Physical-BMI', 'BIA-BIA_BMI']\n    existing_bmi_cols = [col for col in bmi_cols if col in df.columns]\n    if len(existing_bmi_cols) > 0:\n        df['BMI_mean'] = df[existing_bmi_cols].mean(axis=1, skipna=True)\n        df['BMI_norm'] = df.apply(lambda x: safe_normalize(x.get('BMI_mean'), x['group'], x['Basic_Demos-Sex'], BMI_map_male, BMI_map_female), axis=1)\n    else:\n        df['BMI_norm'] = np.nan\n\n    # FGC Zones Aggregate\n    zones = ['FGC-FGC_CU_Zone', 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD_Zone',\n             'FGC-FGC_PU_Zone', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR_Zone',\n             'FGC-FGC_TL_Zone']\n    existing_zones = [zone for zone in zones if zone in df.columns]\n    if existing_zones:\n        df['FGC_Zones_mean'] = df[existing_zones].mean(axis=1, skipna=True)\n        df['FGC_Zones_min'] = df[existing_zones].min(axis=1, skipna=True)\n        df['FGC_Zones_max'] = df[existing_zones].max(axis=1, skipna=True)\n    else:\n        df['FGC_Zones_mean'], df['FGC_Zones_min'], df['FGC_Zones_max'] = np.nan, np.nan, np.nan\n\n    # Grip Strength (Gender Specific)\n    grip_cols = ['FGC-FGC_GSND', 'FGC-FGC_GSD']\n    existing_grip_cols = [col for col in grip_cols if col in df.columns]\n    if len(existing_grip_cols) > 0:\n        df['GS_raw_max'] = df[existing_grip_cols].max(axis=1, skipna=True)\n        df['GS_raw_min'] = df[existing_grip_cols].min(axis=1, skipna=True)\n        df['GS_max_norm'] = df.apply(lambda x: safe_normalize(x.get('GS_raw_max'), x['group'], x['Basic_Demos-Sex'], GSD_max_map_male, GSD_max_map_female), axis=1)\n        df['GS_min_norm'] = df.apply(lambda x: safe_normalize(x.get('GS_raw_min'), x['group'], x['Basic_Demos-Sex'], GSD_min_map_male, GSD_min_map_female), axis=1)\n    else:\n        df['GS_max_norm'], df['GS_min_norm'] = np.nan, np.nan\n\n    # Fitness Tests (CU, PU, TL)\n    df['CU_norm'] = df.apply(lambda x: safe_normalize(x.get('FGC-FGC_CU'), x['group'], x['Basic_Demos-Sex'], cu_map_male, cu_map_female), axis=1)\n    df['PU_norm'] = df.apply(lambda x: safe_normalize(x.get('FGC-FGC_PU'), x['group'], x['Basic_Demos-Sex'], pu_map_male, pu_map_female), axis=1)\n    df['TL_norm'] = df.apply(lambda x: safe_normalize(x.get('FGC-FGC_TL'), x['group'], x['Basic_Demos-Sex'], tl_map_male, tl_map_female), axis=1)\n\n    # Reach (Min/Max)\n    reach_cols = ['FGC-FGC_SRL', 'FGC-FGC_SRR']\n    existing_reach_cols = [col for col in reach_cols if col in df.columns]\n    if len(existing_reach_cols) > 0:\n        df[\"SR_min\"] = df[existing_reach_cols].min(axis=1, skipna=True)\n        df[\"SR_max\"] = df[existing_reach_cols].max(axis=1, skipna=True)\n    else:\n        df[\"SR_min\"], df[\"SR_max\"] = np.nan, np.nan\n\n    # BIA Metrics Normalization\n    df[\"BMR_norm\"] = df.apply(lambda x: safe_normalize(x.get(\"BIA-BIA_BMR\"), x['group'], x['Basic_Demos-Sex'], bmr_map_male, bmr_map_female), axis=1)\n    df[\"DEE_norm\"] = df.apply(lambda x: safe_normalize(x.get(\"BIA-BIA_DEE\"), x['group'], x['Basic_Demos-Sex'], dee_map_male, dee_map_female), axis=1)\n    if \"BIA-BIA_DEE\" in df.columns and \"BIA-BIA_BMR\" in df.columns:\n        df[\"DEE_BMR_diff\"] = df[\"BIA-BIA_DEE\"] - df[\"BIA-BIA_BMR\"]\n    else:\n        df[\"DEE_BMR_diff\"] = np.nan\n    df[\"FFM_norm\"] = df.apply(lambda x: safe_normalize(x.get(\"BIA-BIA_FFM\"), x['group'], x['Basic_Demos-Sex'], ffm_map_male, ffm_map_female), axis=1)\n    df[\"ECW_norm\"] = df.apply(lambda x: safe_normalize(x.get(\"BIA-BIA_ECW\"), x['group'], x['Basic_Demos-Sex'], ecw_map_male, ecw_map_female), axis=1)\n    df[\"ICW_norm\"] = df.apply(lambda x: safe_normalize(x.get(\"BIA-BIA_ICW\"), x['group'], x['Basic_Demos-Sex'], icw_map_male, icw_map_female), axis=1)\n    df[\"ECW_ICW_norm_ratio\"] = np.where(\n        (df[\"ICW_norm\"].isna()) | (df[\"ICW_norm\"] == 0) | (df[\"ECW_norm\"].isna()),\n        np.nan, df[\"ECW_norm\"] / df[\"ICW_norm\"]\n    )\n    df[\"BMC_norm\"] = df.apply(lambda x: safe_normalize(x.get(\"BIA-BIA_BMC\"), x['group'], x['Basic_Demos-Sex'], BMC_map_male, BMC_map_female), axis=1)\n    df[\"Fat_norm\"] = df.apply(lambda x: safe_normalize(x.get(\"BIA-BIA_Fat\"), x['group'], x['Basic_Demos-Sex'], Fat_map_male, Fat_map_female), axis=1)\n    df[\"SMM_norm\"] = df.apply(lambda x: safe_normalize(x.get(\"BIA-BIA_SMM\"), x['group'], x['Basic_Demos-Sex'], SMM_map_male, SMM_map_female), axis=1)\n    df[\"TBW_norm\"] = df.apply(lambda x: safe_normalize(x.get(\"BIA-BIA_TBW\"), x['group'], x['Basic_Demos-Sex'], TBW_map_male, TBW_map_female), axis=1)\n    df[\"FFMI_norm\"] = df.apply(lambda x: safe_normalize(x.get(\"BIA-BIA_FFMI\"), x['group'], x['Basic_Demos-Sex'], FFMI_map_male, FFMI_map_female), axis=1)\n    df[\"FMI_norm\"] = df.apply(lambda x: safe_normalize(x.get(\"BIA-BIA_FMI\"), x['group'], x['Basic_Demos-Sex'], FMI_map_male, FMI_map_female), axis=1)\n\n\n    # --- Feature Dropping ---\n    # Added new raw Physical columns to the drop list\n    drop_feats_candidates = [\n        # Original Physical Measures + BMI mean\n        'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumenference', # Added\n        'Physical-BMI', 'BIA-BIA_BMI', 'BMI_mean',\n        # Original Grip + raw min/max\n        'FGC-FGC_GSND', 'FGC-FGC_GSD', 'GS_raw_max', 'GS_raw_min',\n        # Original Fitness tests\n        'FGC-FGC_CU', 'FGC-FGC_PU', 'FGC-FGC_TL',\n        # Original Reach tests\n        'FGC-FGC_SRL', 'FGC-FGC_SRR',\n        # Original BIA - Consider keeping BMR/DEE if DEE_BMR_diff is used\n        #'BIA-BIA_BMR', 'BIA-BIA_DEE',\n        'BIA-BIA_FFM', 'BIA-BIA_ECW', 'BIA-BIA_ICW',\n        'BIA-BIA_BMC', 'BIA-BIA_Fat', 'BIA-BIA_SMM', 'BIA-BIA_TBW',\n        'BIA-BIA_FFMI', 'BIA-BIA_FMI', # Add these if they exist in input\n        # Original Zone scores\n        'FGC-FGC_CU_Zone', 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD_Zone',\n        'FGC-FGC_PU_Zone', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR_Zone',\n        'FGC-FGC_TL_Zone',\n        # Intermediate normalized water values (if only ratio is needed)\n        'ECW_norm', 'ICW_norm'\n    ]\n\n    cols_to_drop = [col for col in drop_feats_candidates if col in df.columns]\n    df = df.drop(columns=cols_to_drop, axis=1)\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:56.413403Z","iopub.execute_input":"2025-05-20T19:31:56.413808Z","iopub.status.idle":"2025-05-20T19:31:56.467186Z","shell.execute_reply.started":"2025-05-20T19:31:56.413770Z","shell.execute_reply":"2025-05-20T19:31:56.465996Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = feature_engineering(train)\ntest = feature_engineering(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:56.468435Z","iopub.execute_input":"2025-05-20T19:31:56.469143Z","iopub.status.idle":"2025-05-20T19:31:57.968200Z","shell.execute_reply.started":"2025-05-20T19:31:56.469072Z","shell.execute_reply":"2025-05-20T19:31:57.965833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# remove columns \ncolumns_to_remove = [\n    \"BMI_Calculated\", \"BMI_Correct\", \"Correct_sii\"\n]\n\ntrain = train.drop(columns=columns_to_remove, errors=\"ignore\")\ntest = test.drop(columns=columns_to_remove, errors=\"ignore\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:57.969550Z","iopub.execute_input":"2025-05-20T19:31:57.970063Z","iopub.status.idle":"2025-05-20T19:31:57.982121Z","shell.execute_reply.started":"2025-05-20T19:31:57.970019Z","shell.execute_reply":"2025-05-20T19:31:57.980828Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.to_csv(\"train_after_FE.csv\", index=False)\ntest.to_csv(\"test_after_FE.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:57.982919Z","iopub.execute_input":"2025-05-20T19:31:57.983225Z","iopub.status.idle":"2025-05-20T19:31:58.246108Z","shell.execute_reply.started":"2025-05-20T19:31:57.983201Z","shell.execute_reply":"2025-05-20T19:31:58.245130Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from lennarthaupts \n# https://www.kaggle.com/code/lennarthaupts/1st-place-cmi-model-v4-1-1-reduced?scriptVersionId=213769368\n\ndef bin_data(train, test, columns, n_bins=10):\n    import pandas as pd\n    import numpy as np\n    \n    # Combine train and test for consistent bin edges\n    combined = pd.concat([train, test], axis=0)\n\n    bin_edges = {}\n\n    for col in columns:\n        # Compute quantile bin edges\n        try:\n            edges = pd.qcut(combined[col], n_bins, retbins=True, duplicates=\"drop\")[1]\n            bin_edges[col] = edges\n        except ValueError:\n            print(f\"Skipping column {col}: not enough unique values to create {n_bins} bins.\")\n\n    # Apply the same bin edges to both train and test\n    for col, edges in bin_edges.items():\n        num_bins = len(edges) - 1  \n        train[col] = pd.cut(\n            train[col], bins=edges, labels=range(num_bins), include_lowest=True\n        ).astype(float)\n        test[col] = pd.cut(\n            test[col], bins=edges, labels=range(num_bins), include_lowest=True\n        ).astype(float)\n\n    return train, test\n\n\ncolumns_to_bin_updated = [\n    \"PAQ_A-PAQ_A_Total\", # Assuming this raw feature is kept/available\n    \"DEE_BMR_diff\",      # Correct name for the difference feature\n    \"ECW_ICW_norm_ratio\" # Correct name for the ratio feature\n    # Potentially add other non-normalized features if they exist and need binning\n]\n\n# Filter the list to only include columns that actually exist in the dataframe\n# (Good practice in case some features weren't generated)\nfinal_columns_to_bin = [col for col in columns_to_bin_updated if col in train.columns]\n\n\n# Apply binning only to the selected columns\nif final_columns_to_bin: # Proceed only if there are columns to bin\n    train, test = bin_data(train, test, final_columns_to_bin, n_bins=10)\nelse:\n    print(\"No valid columns found for binning. Skipping binning step.\")\n    train, test = train, test # Use the unbinned featured data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:58.247229Z","iopub.execute_input":"2025-05-20T19:31:58.247537Z","iopub.status.idle":"2025-05-20T19:31:58.282318Z","shell.execute_reply.started":"2025-05-20T19:31:58.247510Z","shell.execute_reply":"2025-05-20T19:31:58.281103Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Features to exclude, because they're not in test\nexclude = ['PCIAT-Season', 'PCIAT-PCIAT_01', 'PCIAT-PCIAT_02', 'PCIAT-PCIAT_03',\n           'PCIAT-PCIAT_04', 'PCIAT-PCIAT_05', 'PCIAT-PCIAT_06', 'PCIAT-PCIAT_07',\n           'PCIAT-PCIAT_08', 'PCIAT-PCIAT_09', 'PCIAT-PCIAT_10', 'PCIAT-PCIAT_11',\n           'PCIAT-PCIAT_12', 'PCIAT-PCIAT_13', 'PCIAT-PCIAT_14', 'PCIAT-PCIAT_15',\n           'PCIAT-PCIAT_16', 'PCIAT-PCIAT_17', 'PCIAT-PCIAT_18', 'PCIAT-PCIAT_19',\n           'PCIAT-PCIAT_20', 'PCIAT-PCIAT_Total', 'sii', 'id']\n\ny_model = \"PCIAT-PCIAT_Total\" # Score, target for the model\ny_comp = \"sii\" # Index, target of the competition\nfeatures = [f for f in train.columns if f not in exclude]\n\n# Categorical features\ncat_c = []\n\n# Mapping of categorical features (already dropped in feature engineering)\nfor col in cat_c:\n    a_map = {}\n    all_unique = set(train[col].unique()) | set(test[col].unique())\n    for i, value in enumerate(all_unique):\n        a_map[value] = i\n\n    train[col] = train[col].map(a_map)\n    test[col] = test[col].map(a_map)\n    \ntrain = train[train[\"sii\"].notna()] # Keep rows where target is available\ntrain.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:58.283571Z","iopub.execute_input":"2025-05-20T19:31:58.283921Z","iopub.status.idle":"2025-05-20T19:31:58.298417Z","shell.execute_reply.started":"2025-05-20T19:31:58.283892Z","shell.execute_reply":"2025-05-20T19:31:58.297011Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport matplotlib.ticker as ticker\n\ndef plot_bins(train, columns, n_bins=10):\n    # Berechne die Anzahl der Reihen, basierend auf der Anzahl der Spalten\n    num_cols = len(columns)\n    rows = (num_cols // 3) + (num_cols % 3 > 0)  # 3 Plots pro Reihe\n    \n    fig, axes = plt.subplots(rows, 3, figsize=(15, rows * 5))  # 3 Spalten pro Reihe\n    axes = axes.flatten()  \n\n    for i, col in enumerate(columns):\n        if col in train:\n            # Bin-Grenzen berechnen (direkt aus den Daten)\n            min_val, max_val = train[col].min(), train[col].max()\n            bins = np.linspace(min_val, max_val, n_bins + 1)  # n_bins gleichmäßige Bins\n\n            counts, bins, patches = axes[i].hist(train[col], bins=bins, color='#003F5C', edgecolor='white')\n\n            axes[i].set_xlabel(col)\n            axes[i].set_ylabel('Count')\n            axes[i].set_title(f'Bin-Verteilung für {col}')\n\n            bin_centers = (bins[:-1] + bins[1:]) / 2\n            axes[i].set_xticks(bin_centers)\n\n            axes[i].set_xticklabels([f\"{round(b, 2)}\" for b in bin_centers], rotation=0, ha='center')\n\n            axes[i].yaxis.set_major_locator(ticker.MaxNLocator(integer=True))\n\n            axes[i].grid(axis='y', linestyle='-', alpha=0.7)\n            axes[i].grid(axis='x', linestyle='')\n\n    # Falls weniger als 3*rows Plots da sind, leere Subplots ausblenden\n    for j in range(i + 1, len(axes)):\n        fig.delaxes(axes[j])\n\n# Aufrufen mit den gebinnten Spalten\nplot_bins(train, final_columns_to_bin, n_bins=10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:58.299414Z","iopub.execute_input":"2025-05-20T19:31:58.299740Z","iopub.status.idle":"2025-05-20T19:31:59.316373Z","shell.execute_reply.started":"2025-05-20T19:31:58.299713Z","shell.execute_reply":"2025-05-20T19:31:59.314917Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.to_csv(\"train_after_bin.csv\", index=False)\ntest.to_csv(\"test_after_bin.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:59.317386Z","iopub.execute_input":"2025-05-20T19:31:59.317701Z","iopub.status.idle":"2025-05-20T19:31:59.514980Z","shell.execute_reply.started":"2025-05-20T19:31:59.317672Z","shell.execute_reply":"2025-05-20T19:31:59.513776Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.base import clone\nfrom sklearn.linear_model import LassoCV","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:59.516136Z","iopub.execute_input":"2025-05-20T19:31:59.516446Z","iopub.status.idle":"2025-05-20T19:31:59.521498Z","shell.execute_reply.started":"2025-05-20T19:31:59.516414Z","shell.execute_reply":"2025-05-20T19:31:59.520372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# load dataset after binning \ntrain = pd.read_csv('train_after_bin.csv')\ntest = pd.read_csv('test_after_bin.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:59.522538Z","iopub.execute_input":"2025-05-20T19:31:59.522841Z","iopub.status.idle":"2025-05-20T19:31:59.576077Z","shell.execute_reply.started":"2025-05-20T19:31:59.522817Z","shell.execute_reply":"2025-05-20T19:31:59.575066Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from lennarthaupts \n# https://www.kaggle.com/code/lennarthaupts/1st-place-cmi-model-v4-1-1-reduced?scriptVersionId=213769368\n\n# Features to exclude, because they're not in test\nexclude = ['PCIAT-Season', 'PCIAT-PCIAT_01', 'PCIAT-PCIAT_02', 'PCIAT-PCIAT_03',\n           'PCIAT-PCIAT_04', 'PCIAT-PCIAT_05', 'PCIAT-PCIAT_06', 'PCIAT-PCIAT_07',\n           'PCIAT-PCIAT_08', 'PCIAT-PCIAT_09', 'PCIAT-PCIAT_10', 'PCIAT-PCIAT_11',\n           'PCIAT-PCIAT_12', 'PCIAT-PCIAT_13', 'PCIAT-PCIAT_14', 'PCIAT-PCIAT_15',\n           'PCIAT-PCIAT_16', 'PCIAT-PCIAT_17', 'PCIAT-PCIAT_18', 'PCIAT-PCIAT_19',\n           'PCIAT-PCIAT_20', 'PCIAT-PCIAT_Total', 'sii', 'id']\n\ny_model = \"PCIAT-PCIAT_Total\" # Score, target for the model\ny_comp = \"sii\" # Index, target of the competition\nfeatures = [f for f in train.columns if f not in exclude]\n\ncat_c = []\n\nfor col in cat_c:\n    a_map = {}\n    all_unique = set(train[col].unique()) | set(test[col].unique())\n    for i, value in enumerate(all_unique):\n        a_map[value] = i\n\n    train[col] = train[col].map(a_map)\n    test[col] = test[col].map(a_map)\n    \ntrain = train[train[\"sii\"].notna()] # Keep rows where target is available\ntrain.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:59.577545Z","iopub.execute_input":"2025-05-20T19:31:59.578000Z","iopub.status.idle":"2025-05-20T19:31:59.589927Z","shell.execute_reply.started":"2025-05-20T19:31:59.577934Z","shell.execute_reply":"2025-05-20T19:31:59.588762Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from lennarthaupts \n# https://www.kaggle.com/code/lennarthaupts/1st-place-cmi-model-v4-1-1-reduced?scriptVersionId=213769368\n\n# Plot distribution of total scores which determine the sii\n# Note the excess zeros -> consider other objective functions\nsns.set_theme(style=\"whitegrid\")\nplt.hist(train['PCIAT-PCIAT_Total'], bins=50, color=\"darkorange\")\nplt.title('Score Distribution')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:59.591027Z","iopub.execute_input":"2025-05-20T19:31:59.591432Z","iopub.status.idle":"2025-05-20T19:31:59.934494Z","shell.execute_reply.started":"2025-05-20T19:31:59.591390Z","shell.execute_reply":"2025-05-20T19:31:59.933347Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from lennarthaupts \n# https://www.kaggle.com/code/lennarthaupts/1st-place-cmi-model-v4-1-1-reduced?scriptVersionId=213769368\n\n\nclass Impute_With_Model:\n    \n    def __init__(self, na_frac=0.5, min_samples=0):\n        self.model_dict = {}\n        self.mean_dict = {}\n        self.features = None\n        self.na_frac = na_frac\n        self.min_samples = min_samples\n        \n    def find_features(self, data, feature, tmp_features):\n        missing_rows = data[feature].isna()\n        na_fraction = data[missing_rows][tmp_features].isna().mean(axis=0)\n        valid_features = np.array(tmp_features)[na_fraction <= self.na_frac]\n        return valid_features\n\n    def fit_models(self, model, data, features):\n        self.features = features\n        n_data = data.shape[0]\n        for feature in features:\n            self.mean_dict[feature] = np.mean(data[feature])\n        for feature in tqdm(features):\n            if data[feature].isna().sum() > 0:\n                model_clone = clone(model)\n                X = data[data[feature].notna()].copy()\n                tmp_features = [f for f in features if f != feature]\n                tmp_features = self.find_features(data, feature, tmp_features)\n                if len(tmp_features) >= 1 and X.shape[0] > self.min_samples:\n                    for f in tmp_features:\n                        X[f] = X[f].fillna(self.mean_dict[f])\n                    model_clone.fit(X[tmp_features], X[feature])\n                    self.model_dict[feature] = (model_clone, tmp_features.copy())\n                else:\n                    self.model_dict[feature] = (\"mean\", np.mean(data[feature]))\n            \n    def impute(self, data):\n        imputed_data = data.copy()\n        for feature, model in self.model_dict.items():\n            missing_rows = imputed_data[feature].isna()\n            if missing_rows.any():\n                if model[0] == \"mean\":\n                    imputed_data[feature].fillna(model[1], inplace=True)\n                else:\n                    tmp_features = [f for f in self.features if f != feature]\n                    X_missing = data.loc[missing_rows, tmp_features].copy()\n                    for f in tmp_features:\n                        X_missing[f] = X_missing[f].fillna(self.mean_dict[f])\n                    imputed_data.loc[missing_rows, feature] = model[0].predict(X_missing[model[1]])\n        return imputed_data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:59.935584Z","iopub.execute_input":"2025-05-20T19:31:59.935926Z","iopub.status.idle":"2025-05-20T19:31:59.948277Z","shell.execute_reply.started":"2025-05-20T19:31:59.935897Z","shell.execute_reply":"2025-05-20T19:31:59.946700Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from lennarthaupts \n# https://www.kaggle.com/code/lennarthaupts/1st-place-cmi-model-v4-1-1-reduced?scriptVersionId=213769368\n\n# values with more the 30% missing\n\nmissing = pd.DataFrame(train.isna().sum() / len(train))\nmissing[missing[0] > 0.3][:60]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:59.949474Z","iopub.execute_input":"2025-05-20T19:31:59.949790Z","iopub.status.idle":"2025-05-20T19:31:59.987706Z","shell.execute_reply.started":"2025-05-20T19:31:59.949757Z","shell.execute_reply":"2025-05-20T19:31:59.986518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Predict Missing Values with Lasso\nSEED = 9365\nmodel = LassoCV(cv=5, random_state=SEED)\nimputer = Impute_With_Model(na_frac=0.4) \n# na_frac is the maximum fraction of missing values until which a feature is imputed with the model\n# if there are more missing values than for example 40% then we revert to mean imputation\nimputer.fit_models(model, train, features)\ntrain = imputer.impute(train)\ntest = imputer.impute(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:31:59.989097Z","iopub.execute_input":"2025-05-20T19:31:59.989523Z","iopub.status.idle":"2025-05-20T19:32:09.878714Z","shell.execute_reply.started":"2025-05-20T19:31:59.989485Z","shell.execute_reply":"2025-05-20T19:32:09.877573Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.to_csv(\"train_cleaned.csv\", index=False)\ntest.to_csv(\"test_cleaned.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:32:09.880051Z","iopub.execute_input":"2025-05-20T19:32:09.880436Z","iopub.status.idle":"2025-05-20T19:32:10.164690Z","shell.execute_reply.started":"2025-05-20T19:32:09.880406Z","shell.execute_reply":"2025-05-20T19:32:10.163482Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# load dataset\ntrain = pd.read_csv('train_cleaned.csv')\ntest = pd.read_csv('test_cleaned.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:32:10.166279Z","iopub.execute_input":"2025-05-20T19:32:10.166804Z","iopub.status.idle":"2025-05-20T19:32:10.221256Z","shell.execute_reply.started":"2025-05-20T19:32:10.166772Z","shell.execute_reply":"2025-05-20T19:32:10.220253Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 643\nn_splits = 10\noptimize_params = False\nn_trials = 25 # n_trials for optuna \nvoting = True\nbase_thresholds = [30, 50, 80]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:32:10.222423Z","iopub.execute_input":"2025-05-20T19:32:10.222894Z","iopub.status.idle":"2025-05-20T19:32:10.228275Z","shell.execute_reply.started":"2025-05-20T19:32:10.222848Z","shell.execute_reply":"2025-05-20T19:32:10.227071Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective(trial, X, features, score_col, index_col, cv, sample_weights=False):\n    params = {\n        'loss_function': trial.suggest_categorical('loss_function', ['Tweedie:variance_power=1.5', 'Poisson', 'RMSE']),\n        'random_state': SEED,\n        'iterations': trial.suggest_int('iterations', 100, 300),\n        'depth': trial.suggest_int('depth', 2, 4),\n        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.05, log=True), \n        'l2_leaf_reg': trial.suggest_float('l2_leaf_reg', 1e-3, 1e-1, log=True),  \n        'subsample': trial.suggest_float('subsample', 0.5, 0.7),\n        'bagging_temperature': trial.suggest_float('bagging_temperature', 0.0, 1.0),\n        'random_strength': trial.suggest_float('random_strength', 1e-3, 10.0),\n        'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 20, 60),\n    }\n    model = CatBoostRegressor(**params, verbose=0)\n    \n    seeds = [random.randint(1, 10000) for _ in range(20)]\n    score, _ = n_cross_validate(model, X, features, score_col, index_col, cv, seeds, sample_weights=True, verbose=True)\n    \n    return score\n    \ndef run_optimization(X, features, score_col, index_col, n_trials=30, cv=None, sample_weights=False):\n    study = optuna.create_study(direction=\"maximize\")\n    study.optimize(lambda trial: objective(trial, X, features, score_col, index_col, cv, sample_weights), \n                   n_trials=n_trials)\n    \n    print(\"Best params for CatBoost:\", study.best_params)\n    print(\"Best score:\", study.best_value)\n    return study.best_params","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:32:10.229548Z","iopub.execute_input":"2025-05-20T19:32:10.229973Z","iopub.status.idle":"2025-05-20T19:32:10.249911Z","shell.execute_reply.started":"2025-05-20T19:32:10.229912Z","shell.execute_reply":"2025-05-20T19:32:10.248771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# delete rows without \"sii\"\ntrain = train[train[\"sii\"].notna()]  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:32:10.251170Z","iopub.execute_input":"2025-05-20T19:32:10.251602Z","iopub.status.idle":"2025-05-20T19:32:10.279519Z","shell.execute_reply.started":"2025-05-20T19:32:10.251561Z","shell.execute_reply":"2025-05-20T19:32:10.278409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:32:10.280591Z","iopub.execute_input":"2025-05-20T19:32:10.280971Z","iopub.status.idle":"2025-05-20T19:32:10.294314Z","shell.execute_reply.started":"2025-05-20T19:32:10.280919Z","shell.execute_reply":"2025-05-20T19:32:10.293015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Features to exclude, because they're not in test\n\nexclude = [\n    'PCIAT-Season', 'PCIAT-PCIAT_01', 'PCIAT-PCIAT_02', 'PCIAT-PCIAT_03',\n    'PCIAT-PCIAT_04', 'PCIAT-PCIAT_05', 'PCIAT-PCIAT_06', 'PCIAT-PCIAT_07',\n    'PCIAT-PCIAT_08', 'PCIAT-PCIAT_09', 'PCIAT-PCIAT_10', 'PCIAT-PCIAT_11',\n    'PCIAT-PCIAT_12', 'PCIAT-PCIAT_13', 'PCIAT-PCIAT_14', 'PCIAT-PCIAT_15',\n    'PCIAT-PCIAT_16', 'PCIAT-PCIAT_17', 'PCIAT-PCIAT_18', 'PCIAT-PCIAT_19',\n    'PCIAT-PCIAT_20', 'PCIAT-PCIAT_Total', 'sii', 'id'\n]\nfeatures = [f for f in train.columns if f not in exclude]\n\n\nif optimize_params:\n    cat_params = run_optimization(train, features, 'PCIAT-PCIAT_Total', 'sii', n_trials=n_trials, cv=kf, sample_weights=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:32:10.295443Z","iopub.execute_input":"2025-05-20T19:32:10.295893Z","iopub.status.idle":"2025-05-20T19:32:10.319780Z","shell.execute_reply.started":"2025-05-20T19:32:10.295860Z","shell.execute_reply":"2025-05-20T19:32:10.318395Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.neural_network import MLPRegressor\n\n\ncat_params = {\n    'objective': 'RMSE', \n    'iterations': 238, \n    'depth': 4, \n    'learning_rate': 0.044523361750173816, \n    'l2_leaf_reg': 0.09301285673435761, \n    'subsample': 0.6902492783438681, \n    'bagging_temperature': 0.3007304771330199, \n    'random_strength': 3.562201626987314, \n    'min_data_in_leaf': 60\n}\n\n# Parameters for LGBM, XGB and CatBoost\nlgb_params = {\n    'objective': 'poisson', \n    'n_estimators': 295, \n    'max_depth': 4, \n    'learning_rate': 0.04505693066482616, \n    'subsample': 0.6042489155604022, \n    'colsample_bytree': 0.5021876720502726, \n    'min_data_in_leaf': 100\n}\n\nxgb_params = {'objective': 'reg:tweedie', 'num_parallel_tree': 12, 'n_estimators': 236, 'max_depth': 3, 'learning_rate': 0.04223740904479563, 'subsample': 0.7157264603586825, 'colsample_bytree': 0.7897918901977528, 'reg_alpha': 0.005335705058190553, 'reg_lambda': 0.0001897435318347022, 'tweedie_variance_power': 1.1393958601390142}\n\nxgb_params_2 = {\n    'objective': 'reg:tweedie', \n    'num_parallel_tree': 18, \n    'n_estimators': 175, \n    'max_depth': 3, \n    'learning_rate': 0.032620453423049305, \n    'subsample': 0.6155579670568023, \n    'colsample_bytree': 0.5988773292417443, \n    'reg_alpha': 0.0028895066837627205, \n    'reg_lambda': 0.002232531512636924, \n    'tweedie_variance_power': 1.1708678482038286\n}\n\nxtrees_params = {\n    'n_estimators': 500, \n    'max_depth': 15, \n    'min_samples_leaf': 20, \n    'bootstrap': False\n}\n\nmlp_params = {\n    'hidden_layer_sizes': (200, 150, 100, 50),  # Wider and deeper network\n    'activation': 'relu',  # 'tanh' or 'logistic' could be alternatives\n    'solver': 'adam',\n    'alpha': 0.001,  # L2 regularization strength\n    'learning_rate': 'adaptive',\n    'max_iter': 1000,  # Increase iterations\n    'early_stopping': True,\n    'validation_fraction': 0.2,  # For early stopping\n    'n_iter_no_change': 20,  # Patience for early stopping\n    'batch_size': 32  # Try mini-batch training\n}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:32:10.321119Z","iopub.execute_input":"2025-05-20T19:32:10.321442Z","iopub.status.idle":"2025-05-20T19:32:10.349490Z","shell.execute_reply.started":"2025-05-20T19:32:10.321414Z","shell.execute_reply":"2025-05-20T19:32:10.348278Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Add polynomial features\nfrom sklearn.preprocessing import PolynomialFeatures\npoly = PolynomialFeatures(degree=2, interaction_only=True)\n#train = poly.fit_transform(train[features])\n#test = poly.transform(test[features])\n\n# 2. Feature selection - keep only the most important features\n#from sklearn.feature_selection import SelectKBest, f_regression\n#selector = SelectKBest(f_regression, k=min(20, len(features)))\n#train_selected = selector.fit_transform(train_scaled[features], train['PCIAT-PCIAT_Total'])\n#test_selected = selector.transform(test_scaled[features])\n\n# 3. Add more advanced scaling - try normalization or robust scaling\n#from sklearn.preprocessing import RobustScaler\n#robust_scaler = RobustScaler()\n#train_robust = robust_scaler.fit_transform(train[features])\n#test_robust = robust_scaler.transform(test[features])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:32:10.350599Z","iopub.execute_input":"2025-05-20T19:32:10.351086Z","iopub.status.idle":"2025-05-20T19:32:10.377365Z","shell.execute_reply.started":"2025-05-20T19:32:10.351042Z","shell.execute_reply":"2025-05-20T19:32:10.375766Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MLPRegressorWrapper(MLPRegressor):\n    def fit(self, X, y, sample_weight=None, **kwargs):\n        # Ignore sample_weight and call parent's fit\n        return super().fit(X, y, **kwargs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:34:10.106666Z","iopub.execute_input":"2025-05-20T19:34:10.107100Z","iopub.status.idle":"2025-05-20T19:34:10.112892Z","shell.execute_reply.started":"2025-05-20T19:34:10.107069Z","shell.execute_reply":"2025-05-20T19:34:10.111195Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create new DataFrames with polynomial features\nX_train_poly = poly.fit_transform(train[features])\npoly_feature_names = poly.get_feature_names_out(features)\n\n# Convert back to pandas DataFrames\ntrain_poly = pd.DataFrame(X_train_poly, columns=poly_feature_names, index=train.index)\ntrain_poly['PCIAT-PCIAT_Total'] = train['PCIAT-PCIAT_Total']  # Add target variable\ntrain_poly['sii'] = train['sii']  # Add the 'sii' column\n\nX_test_poly = poly.transform(test[features])\ntest_poly = pd.DataFrame(X_test_poly, columns=poly_feature_names, index=test.index)\n\n# Now scale\nscaler = StandardScaler()\ntrain_poly_scaled = train_poly.copy()\ntest_poly_scaled = test_poly.copy()\ntrain_poly_scaled[poly_feature_names] = scaler.fit_transform(train_poly[poly_feature_names])\ntest_poly_scaled[poly_feature_names] = scaler.transform(test_poly[poly_feature_names])\n\n# Create and train MLP model\nmlp_params = {\n    'hidden_layer_sizes': (200, 150, 100, 50),  # Wider and deeper network\n    'activation': 'relu',  # 'tanh' or 'logistic' could be alternatives\n    'solver': 'adam',\n    'alpha': 0.001,  # L2 regularization strength\n    'learning_rate': 'adaptive',\n    'max_iter': 1000,  # Increase iterations\n    'early_stopping': True,\n    'validation_fraction': 0.2,  # For early stopping\n    'n_iter_no_change': 20,  # Patience for early stopping\n    'batch_size': 32  # Try mini-batch training\n}\n\nmlp_model = MLPRegressorWrapper(**mlp_params, random_state=SEED)\n\n# Cross-validate with the new features\nscore_mlp, oof_mlp, mlp_thresholds = cross_validate(\n    mlp_model, train_poly_scaled, poly_feature_names, 'PCIAT-PCIAT_Total', 'sii', kf, \n    verbose=True, sample_weights=True\n)\n\n# For the final fit - use the polynomial features consistently\nmlp_model.fit(train_poly_scaled[poly_feature_names], train_poly_scaled['PCIAT-PCIAT_Total'])\ntest_mlp = mlp_model.predict(test_poly_scaled[poly_feature_names])\n\n# Calculate ensemble thresholds\nmlp_thresholds_ens = mlp_thresholds[0]  # Or use whatever logic you have for ensemble thresholds\n\n# Apply thresholds\ntest_mlp_rounded = round_with_thresholds(test_mlp, mlp_thresholds_ens)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:34:12.631162Z","iopub.execute_input":"2025-05-20T19:34:12.631531Z","iopub.status.idle":"2025-05-20T19:37:28.200725Z","shell.execute_reply.started":"2025-05-20T19:34:12.631505Z","shell.execute_reply":"2025-05-20T19:37:28.199600Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create submission\nsubmission = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv\")\nsubmission['sii'] = test_mlp_rounded\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T19:40:58.035850Z","iopub.execute_input":"2025-05-20T19:40:58.036291Z","iopub.status.idle":"2025-05-20T19:40:58.045807Z","shell.execute_reply.started":"2025-05-20T19:40:58.036261Z","shell.execute_reply":"2025-05-20T19:40:58.044866Z"}},"outputs":[],"execution_count":null}]}