{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries Loading and Memory Reset","metadata":{}},{"cell_type":"code","source":"%reset -f","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:13:18.333710Z","iopub.execute_input":"2024-12-23T09:13:18.334138Z","iopub.status.idle":"2024-12-23T09:13:18.471578Z","shell.execute_reply.started":"2024-12-23T09:13:18.334097Z","shell.execute_reply":"2024-12-23T09:13:18.470448Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nfrom tqdm import tqdm\n\nimport numpy as np\nimport pandas as pd\nimport dask.dataframe as dd\npd.set_option('display.max_columns', None)\nfrom IPython.display import display\n\n# Graph\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom lightgbm import plot_importance\n\n# ML\nfrom sklearn.model_selection import train_test_split#, GridSearchCV\nfrom sklearn.linear_model import LinearRegression, Lasso\nfrom sklearn.metrics import cohen_kappa_score, make_scorer\nimport lightgbm as lgb","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:13:18.473742Z","iopub.execute_input":"2024-12-23T09:13:18.474094Z","iopub.status.idle":"2024-12-23T09:13:21.171754Z","shell.execute_reply.started":"2024-12-23T09:13:18.474061Z","shell.execute_reply":"2024-12-23T09:13:21.170633Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# List of parameters","metadata":{}},{"cell_type":"code","source":"flag_dask=False\n\nparquet_features = ['X', 'Y', 'Z', 'light', 'enmo' , 'non-wear_flag','anglez'\n                    , 'battery_voltage'\n                    # ,'relative_date_PCIAT'\n                    ]\n\n\ntarget_type='numeric'#numeric or categorical\nscoring_method='distribution'#distribution, threshold - only if target_type='numeric'. 'threshold' means that we are spliting the numeric target following the definition of sii and 'distribution' means that we are using the distribution of sii in train to split our numeric prediction\n\nclean_target = True # If true, we remove the occurences of sii or PCIAT_Total where there are 'too many' PCIAT question not answered\nflag_clean_overlap_train_test = True # also target related\n\nratio_tr_te = 0.2# % of dataset allocated to test\n\ncategorical_na = 'missing'# If 'missing' then we impute the categorical features with a new category  \nratio_na = 0.3 # We impute missing values of features for which the ratio of NAs is below ration_na\n\nstandardization_flag = True\n\noverfitting_th = 0.05# max overfitting allowed in the lightgbm, ignored for linear models\nmodel_type='lightgbm'# can be 'lasso', 'lightgbm', 'linear\n\nfinal_features ='all'#'all', 'flat' - when 'flat', I am not using the parquet data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:13:21.173008Z","iopub.execute_input":"2024-12-23T09:13:21.173583Z","iopub.status.idle":"2024-12-23T09:13:21.183556Z","shell.execute_reply.started":"2024-12-23T09:13:21.173549Z","shell.execute_reply":"2024-12-23T09:13:21.182068Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Flat Data Loading","metadata":{}},{"cell_type":"code","source":"flat_train_df = pd.read_csv('../input/child-mind-institute-problematic-internet-use/train.csv')\nprint(f'''We have {flat_train_df.shape[0]} rows and {flat_train_df.shape[1]} columns in the flat train dataset''')\ndisplay(flat_train_df.sample(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:13:21.184829Z","iopub.execute_input":"2024-12-23T09:13:21.185274Z","iopub.status.idle":"2024-12-23T09:13:21.371514Z","shell.execute_reply.started":"2024-12-23T09:13:21.185214Z","shell.execute_reply":"2024-12-23T09:13:21.370490Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get target distribution\ntarget_distr = flat_train_df['sii'].value_counts(normalize=True)\ntarget_distr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:13:21.374053Z","iopub.execute_input":"2024-12-23T09:13:21.374430Z","iopub.status.idle":"2024-12-23T09:13:21.391545Z","shell.execute_reply.started":"2024-12-23T09:13:21.374394Z","shell.execute_reply":"2024-12-23T09:13:21.390305Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"flat_test_df = pd.read_csv('../input/child-mind-institute-problematic-internet-use/test.csv')\nprint(f'''We have {flat_test_df.shape[0]} rows and {flat_test_df.shape[1]} columns in the flat test dataset''')\ndisplay(flat_test_df.sample(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:13:21.393293Z","iopub.execute_input":"2024-12-23T09:13:21.393773Z","iopub.status.idle":"2024-12-23T09:13:21.457024Z","shell.execute_reply.started":"2024-12-23T09:13:21.393724Z","shell.execute_reply":"2024-12-23T09:13:21.455886Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dictionary\ndico_df = pd.read_csv('../input/child-mind-institute-problematic-internet-use/data_dictionary.csv')\ndisplay(dico_df.sample(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:13:21.458444Z","iopub.execute_input":"2024-12-23T09:13:21.458788Z","iopub.status.idle":"2024-12-23T09:13:21.476930Z","shell.execute_reply.started":"2024-12-23T09:13:21.458734Z","shell.execute_reply":"2024-12-23T09:13:21.475833Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Parquet Data Load and Aggregation","metadata":{}},{"cell_type":"code","source":"# TRAIN\ntrain_file_path = '../input/child-mind-institute-problematic-internet-use/series_train.parquet'\nid_paths = [os.path.join(train_file_path, d) for d in os.listdir(train_file_path) if d.startswith('id=')]\n\n\n# Prepare an empty list to store results\naggregated_data = []\n\n# Read the parquet file in chunks (filter by id if needed)\nunique_ids = pd.read_parquet(train_file_path, columns=['id']).id.unique()\n\nfor id_path in tqdm(id_paths, desc=\"Processing IDs\", unit=\"ID\"):\n    # Extract the id from the folder name (e.g., 'id=ffed1dd5' -> 'ffed1dd5')\n    unique_id = id_path.split('=')[-1]\n      \n    # Load the data for the current id\n    df_id = pd.read_parquet(os.path.join(id_path, 'part-0.parquet'))\n\n       \n    # Calculate aggregation for the current id\n    aggregated_row = {}\n    for feature in parquet_features:\n        # aggregated_row[f\"min_{feature}\"] = df_id[feature].min()\n        # aggregated_row[f\"max_{feature}\"] = df_id[feature].max()\n        aggregated_row[f\"mean_{feature}\"] = df_id[feature].mean()\n        # aggregated_row[f\"std_{feature}\"] = df_id[feature].std()\n        # aggregated_row[f\"p2_{feature}\"] = df_id[feature].quantile(0.02)  # 2nd percentile\n        # aggregated_row[f\"p98_{feature}\"] = df_id[feature].quantile(0.98)  # 98th percentile\n        aggregated_row[f\"p1_{feature}\"] = df_id[feature].quantile(0.01)  # 1st percentile\n        aggregated_row[f\"p99_{feature}\"] = df_id[feature].quantile(0.99)  # 99th percentile\n        aggregated_row[f\"p25_{feature}\"] = df_id[feature].quantile(0.25)  # Q1\n        aggregated_row[f\"p75_{feature}\"] = df_id[feature].quantile(0.75)  # Q3\n        aggregated_row[f\"median_{feature}\"] = df_id[feature].median()  # Median\n        mad_value = np.median(np.abs(df_id[feature] - df_id[feature].median()))\n        aggregated_row[f\"mad_{feature}\"] = mad_value    \n        \n    aggregated_row['id'] = unique_id  # Add the id to the row\n    aggregated_row['num_rows'] = df_id.shape[0]\n    \n    # Append the result to the list\n    aggregated_data.append(aggregated_row)\n\n# Combine all rows into a single DataFrame\nparquet_tr_macro_df = pd.DataFrame(aggregated_data)\ndel aggregated_data\ngc.collect()\n\nprint(f'''We have {parquet_tr_macro_df.shape[0]} rows, {parquet_tr_macro_df.shape[1]} columns and {parquet_tr_macro_df['id'].nunique()} unique IDs in the train parquet dataset''')\ndisplay(parquet_tr_macro_df.sample(5))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:13:21.478237Z","iopub.execute_input":"2024-12-23T09:13:21.478597Z","iopub.status.idle":"2024-12-23T09:18:47.725409Z","shell.execute_reply.started":"2024-12-23T09:13:21.478555Z","shell.execute_reply":"2024-12-23T09:18:47.724176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# TEST\ntest_file_path = '../input/child-mind-institute-problematic-internet-use/series_test.parquet'\nid_paths = [os.path.join(test_file_path, d) for d in os.listdir(test_file_path) if d.startswith('id=')]\n\n\n# Prepare an empty list to store results\naggregated_data = []\n\n# Read the parquet file in chunks (filter by id if needed)\nunique_ids = pd.read_parquet(test_file_path, columns=['id']).id.unique()\n\nfor id_path in tqdm(id_paths, desc=\"Processing IDs\", unit=\"ID\"):\n    # Extract the id from the folder name (e.g., 'id=ffed1dd5' -> 'ffed1dd5')\n    unique_id = id_path.split('=')[-1]\n      \n    # Load the data for the current id\n    df_id = pd.read_parquet(os.path.join(id_path, 'part-0.parquet'))\n\n       \n    # Calculate aggregation for the current id\n    aggregated_row = {}\n    for feature in parquet_features:\n        # aggregated_row[f\"min_{feature}\"] = df_id[feature].min()\n        # aggregated_row[f\"max_{feature}\"] = df_id[feature].max()\n        aggregated_row[f\"mean_{feature}\"] = df_id[feature].mean()\n        # aggregated_row[f\"std_{feature}\"] = df_id[feature].std()\n        # aggregated_row[f\"p2_{feature}\"] = df_id[feature].quantile(0.02)  # 2nd percentile\n        # aggregated_row[f\"p98_{feature}\"] = df_id[feature].quantile(0.98)  # 98th percentile\n        aggregated_row[f\"p1_{feature}\"] = df_id[feature].quantile(0.01)  # 1st percentile\n        aggregated_row[f\"p99_{feature}\"] = df_id[feature].quantile(0.99)  # 99th percentile\n        aggregated_row[f\"p25_{feature}\"] = df_id[feature].quantile(0.25)  # Q1\n        aggregated_row[f\"p75_{feature}\"] = df_id[feature].quantile(0.75)  # Q3\n        aggregated_row[f\"median_{feature}\"] = df_id[feature].median()  # Median\n        mad_value = np.median(np.abs(df_id[feature] - df_id[feature].median()))\n        aggregated_row[f\"mad_{feature}\"] = mad_value    \n        \n    aggregated_row['id'] = unique_id  # Add the id to the row\n    aggregated_row['num_rows'] = df_id.shape[0]\n    \n    # Append the result to the list\n    aggregated_data.append(aggregated_row)\n\n# Combine all rows into a single DataFrame\nparquet_realte_macro_df = pd.DataFrame(aggregated_data)\ndel aggregated_data\ngc.collect()\n\nprint(f'''We have {parquet_realte_macro_df.shape[0]} rows, {parquet_realte_macro_df.shape[1]} columns and {parquet_realte_macro_df['id'].nunique()} unique IDs in the train parquet dataset''')\ndisplay(parquet_realte_macro_df.sample(2))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:47.727445Z","iopub.execute_input":"2024-12-23T09:18:47.727831Z","iopub.status.idle":"2024-12-23T09:18:48.424271Z","shell.execute_reply.started":"2024-12-23T09:18:47.727794Z","shell.execute_reply":"2024-12-23T09:18:48.423182Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Target Definition and Dataset Perimeter\n - We consider IDs with a non null target\n - Among those, we remove those where at least 2 of the PCIAT questions is not answered\n - When there is one question not answered excatly, we keep the IDs when the sii is not impacted (not at the edges)","metadata":{}},{"cell_type":"code","source":"if clean_target:\n\n    # Select relevant features\n    sub_features = [f'PCIAT-PCIAT_{str(i).zfill(2)}' for i in range(1, 21)]\n\n    # Filter rows where PCIAT-PCIAT_Total is not null\n    not_null_mask = flat_train_df['PCIAT-PCIAT_Total'].notnull()\n\n    # Condition 1: PCIAT-PCIAT_Total is not null and there is more than 1 null in sub-features\n    more_than_one_null_in_sub_features = flat_train_df[sub_features].isnull().sum(axis=1) > 1\n\n    # Condition 2: PCIAT-PCIAT_Total is not null and exactly 1 null in sub-features\n    one_null_in_sub_features = flat_train_df[sub_features].isnull().sum(axis=1) == 1\n\n    # Condition 3: PCIAT-PCIAT_Total score is between the excluded ranges\n    score_exclude_condition = (flat_train_df['PCIAT-PCIAT_Total'] >= 25) & (flat_train_df['PCIAT-PCIAT_Total'] < 30) | \\\n                            (flat_train_df['PCIAT-PCIAT_Total'] >= 45) & (flat_train_df['PCIAT-PCIAT_Total'] < 49) | \\\n                            (flat_train_df['PCIAT-PCIAT_Total'] >= 74) & (flat_train_df['PCIAT-PCIAT_Total'] < 79)\n\n    # Combine the conditions\n    condition = not_null_mask & ~(more_than_one_null_in_sub_features | (one_null_in_sub_features & score_exclude_condition))\n\n    # Extract the IDs that satisfy the final condition\n    ids_to_keep = flat_train_df.loc[condition, 'id']\n\n    # Filter Train Dataframe on these rows\n    flat_train_df = flat_train_df[flat_train_df['id'].isin(ids_to_keep)]\n\nelse : #We only select the rows where the target is not null\n    flat_train_df = flat_train_df[~flat_train_df['sii'].isna()]\n\nif flag_clean_overlap_train_test:\n    # We noticed that some IDs from test are in train, let's remove them althought we don't know what is the real test\n    flat_train_df = flat_train_df[~flat_train_df['id'].isin(flat_test_df['id'])]\n    print(f'''We have {flat_train_df.shape[0]} rows and {flat_train_df.shape[1]} columns in the flat train dataset''')\nelse:\n    pass\n\n\n\nprint(f'''We have {flat_train_df.shape[0]} rows and {flat_train_df.shape[1]} columns in the flat train dataset''')\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:48.425671Z","iopub.execute_input":"2024-12-23T09:18:48.426011Z","iopub.status.idle":"2024-12-23T09:18:48.564205Z","shell.execute_reply.started":"2024-12-23T09:18:48.425977Z","shell.execute_reply":"2024-12-23T09:18:48.562973Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Apply same selection on the train parquet file\nparquet_tr_macro_df = parquet_tr_macro_df[parquet_tr_macro_df['id'].isin(set(flat_train_df['id']))]\nprint(f'''We have {parquet_tr_macro_df.shape[0]} rows, {parquet_tr_macro_df.shape[1]} columns and {parquet_tr_macro_df['id'].nunique()} unique IDs in the train parquet dataset''')\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:48.566112Z","iopub.execute_input":"2024-12-23T09:18:48.566513Z","iopub.status.idle":"2024-12-23T09:18:48.685777Z","shell.execute_reply.started":"2024-12-23T09:18:48.566477Z","shell.execute_reply":"2024-12-23T09:18:48.684763Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Combine Features","metadata":{}},{"cell_type":"markdown","source":"## BMI","metadata":{}},{"cell_type":"code","source":"# The two features can be very different\nflat_train_df[~(flat_train_df['Physical-BMI'].isna()) & (~flat_train_df['BIA-BIA_BMI'].isna()) & (round(flat_train_df['Physical-BMI'],1)>round(flat_train_df['BIA-BIA_BMI'],1))][['id','Basic_Demos-Age','Physical-BMI','BIA-BIA_BMI']].sample(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:48.687323Z","iopub.execute_input":"2024-12-23T09:18:48.687800Z","iopub.status.idle":"2024-12-23T09:18:48.706286Z","shell.execute_reply.started":"2024-12-23T09:18:48.687751Z","shell.execute_reply":"2024-12-23T09:18:48.705074Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'''Number of missing values in Physical-BMI : {flat_train_df[flat_train_df['Physical-BMI'].isna()].shape[0]}''')\nprint(f'''Number of missing values in BIA-BIA_BMI : {flat_train_df[flat_train_df['BIA-BIA_BMI'].isna()].shape[0]}''')\n\nprint(f'''Number of cases where Physical-BMI is missing but not BIA-BMI : {flat_train_df[(flat_train_df['Physical-BMI'].isna()) & ~(flat_train_df['BIA-BIA_BMI'].isna())].shape[0]}''')\nprint(f'''Number of cases where Physical-BMI is not missing but BIA-BMI is : {flat_train_df[~(flat_train_df['Physical-BMI'].isna()) & (flat_train_df['BIA-BIA_BMI'].isna())].shape[0]}''')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:48.708016Z","iopub.execute_input":"2024-12-23T09:18:48.708503Z","iopub.status.idle":"2024-12-23T09:18:48.721281Z","shell.execute_reply.started":"2024-12-23T09:18:48.708452Z","shell.execute_reply":"2024-12-23T09:18:48.720034Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualization of differences\nflat_train_df['Gap_to_delete'] = flat_train_df['Physical-BMI'] - flat_train_df['BIA-BIA_BMI']\nsns.scatterplot(data=flat_train_df, x='Physical-BMI', y='BIA-BIA_BMI', hue='Gap_to_delete', palette='coolwarm')\nplt.title('Scatter Plot of Physical-BMI vs BIA-BIA_BMI with Gap')\nplt.show()\ndel flat_train_df['Gap_to_delete']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:48.726133Z","iopub.execute_input":"2024-12-23T09:18:48.726543Z","iopub.status.idle":"2024-12-23T09:18:49.305572Z","shell.execute_reply.started":"2024-12-23T09:18:48.726506Z","shell.execute_reply":"2024-12-23T09:18:49.304313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use Physical-BMI, and when NA use BIA-BIA_BMI\nflat_train_df['Physical-BMI'] = flat_train_df['Physical-BMI'].fillna(flat_train_df['BIA-BIA_BMI'])\nflat_test_df['Physical-BMI'] = flat_test_df['Physical-BMI'].fillna(flat_test_df['BIA-BIA_BMI'])\ndel flat_train_df['BIA-BIA_BMI'], flat_test_df['BIA-BIA_BMI']\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:49.306920Z","iopub.execute_input":"2024-12-23T09:18:49.307234Z","iopub.status.idle":"2024-12-23T09:18:49.432371Z","shell.execute_reply.started":"2024-12-23T09:18:49.307205Z","shell.execute_reply":"2024-12-23T09:18:49.430901Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Activity Summary Score\nHere the 'problem' is different : there are two variables that are calculated exactly in the same way, but one is for teenagers and the other one for children","metadata":{}},{"cell_type":"code","source":"display(flat_train_df[['Basic_Demos-Age','PAQ_A-PAQ_A_Total','PAQ_C-PAQ_C_Total']].sample(5))\nprint(f'''The min Age for the Adolescents variable is {min(flat_train_df[~flat_train_df['PAQ_A-PAQ_A_Total'].isna()]['Basic_Demos-Age'])} and the max is {max(flat_train_df[~flat_train_df['PAQ_A-PAQ_A_Total'].isna()]['Basic_Demos-Age'])}''')\nprint(f'''The min Age for the Children variable is {min(flat_train_df[~flat_train_df['PAQ_C-PAQ_C_Total'].isna()]['Basic_Demos-Age'])} and the max is {max(flat_train_df[~flat_train_df['PAQ_C-PAQ_C_Total'].isna()]['Basic_Demos-Age'])}''')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:49.433966Z","iopub.execute_input":"2024-12-23T09:18:49.434310Z","iopub.status.idle":"2024-12-23T09:18:49.464095Z","shell.execute_reply.started":"2024-12-23T09:18:49.434279Z","shell.execute_reply":"2024-12-23T09:18:49.462651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'''Total number of missing values in PAQ A: {flat_train_df[flat_train_df['PAQ_A-PAQ_A_Total'].isna()].shape[0]}''')\nprint(f'''Total number of missing values in PAQ C: {flat_train_df[flat_train_df['PAQ_C-PAQ_C_Total'].isna()].shape[0]}''')\n\nprint(f'''Number of missing values in PAQ A when Age>=14 : {flat_train_df[(flat_train_df['PAQ_A-PAQ_A_Total'].isna()) & (flat_train_df['Basic_Demos-Age']>=14)].shape[0]}''')\nprint(f'''Number of missing values in PAQ C when Age<14 : {flat_train_df[(flat_train_df['PAQ_C-PAQ_C_Total'].isna()) & (flat_train_df['Basic_Demos-Age']<14)].shape[0]}''')\n\n\nprint(f'''Number of missing values in PAQ C when Age>=14 : {flat_train_df[(flat_train_df['PAQ_C-PAQ_C_Total'].isna()) & (flat_train_df['Basic_Demos-Age']>=14)].shape[0]}''')\nprint(f'''Number of missing values in PAQ A when Age<14 : {flat_train_df[(flat_train_df['PAQ_A-PAQ_A_Total'].isna()) & (flat_train_df['Basic_Demos-Age']<14)].shape[0]}''')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:49.465637Z","iopub.execute_input":"2024-12-23T09:18:49.466135Z","iopub.status.idle":"2024-12-23T09:18:49.491373Z","shell.execute_reply.started":"2024-12-23T09:18:49.466083Z","shell.execute_reply":"2024-12-23T09:18:49.489889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# We give the adolescent name to the final feature\nflat_train_df['PAQ_A-PAQ_A_Total'] = np.where(\n    flat_train_df['Basic_Demos-Age'] >= 14, \n    # For age >= 14, use PAQ_A-PAQ_A_Total, but fall back to PAQ_C-PAQ_C_Total if null\n    np.where(flat_train_df['PAQ_A-PAQ_A_Total'].notnull(), flat_train_df['PAQ_A-PAQ_A_Total'], flat_train_df['PAQ_C-PAQ_C_Total']),\n    # For age < 14, use PAQ_C-PAQ_C_Total, but fall back to PAQ_A-PAQ_A_Total if null\n    np.where(flat_train_df['PAQ_C-PAQ_C_Total'].notnull(), flat_train_df['PAQ_C-PAQ_C_Total'], flat_train_df['PAQ_A-PAQ_A_Total'])\n)\n\nflat_test_df['PAQ_A-PAQ_A_Total'] = np.where(\n    flat_test_df['Basic_Demos-Age'] >= 14, \n    # For age >= 14, use PAQ_A-PAQ_A_Total, but fall back to PAQ_C-PAQ_C_Total if null\n    np.where(flat_test_df['PAQ_A-PAQ_A_Total'].notnull(), flat_test_df['PAQ_A-PAQ_A_Total'], flat_test_df['PAQ_C-PAQ_C_Total']),\n    # For age < 14, use PAQ_C-PAQ_C_Total, but fall back to PAQ_A-PAQ_A_Total if null\n    np.where(flat_test_df['PAQ_C-PAQ_C_Total'].notnull(), flat_test_df['PAQ_C-PAQ_C_Total'], flat_test_df['PAQ_A-PAQ_A_Total']))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:49.492879Z","iopub.execute_input":"2024-12-23T09:18:49.493241Z","iopub.status.idle":"2024-12-23T09:18:49.503237Z","shell.execute_reply.started":"2024-12-23T09:18:49.493207Z","shell.execute_reply":"2024-12-23T09:18:49.501776Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# It only helps kids >=14\nprint(f'''Total number of missing values in PAQ A: {flat_train_df[flat_train_df['PAQ_A-PAQ_A_Total'].isna()].shape[0]}''')\nprint(f'''Number of missing values in PAQ A when Age>=14 : {flat_train_df[(flat_train_df['PAQ_A-PAQ_A_Total'].isna()) & (flat_train_df['Basic_Demos-Age']>=14)].shape[0]}''')\nprint(f'''Number of missing values in PAQ A when Age<14 : {flat_train_df[(flat_train_df['PAQ_A-PAQ_A_Total'].isna()) & (flat_train_df['Basic_Demos-Age']<14)].shape[0]}''')\ndel flat_train_df['PAQ_C-PAQ_C_Total'], flat_test_df['PAQ_C-PAQ_C_Total']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:49.505247Z","iopub.execute_input":"2024-12-23T09:18:49.505687Z","iopub.status.idle":"2024-12-23T09:18:49.539735Z","shell.execute_reply.started":"2024-12-23T09:18:49.505651Z","shell.execute_reply":"2024-12-23T09:18:49.538315Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Sleep Disturbance Scale ","metadata":{}},{"cell_type":"code","source":"print(f'''We have {flat_train_df[~(flat_train_df['SDS-SDS_Total_Raw'].isna()) & (flat_train_df['SDS-SDS_Total_T'].isna())].shape[0]} cases where SDS_Total_T is not null but SDS_Total_Raw is''')\nprint(f'''We have {flat_train_df[~(flat_train_df['SDS-SDS_Total_T'].isna()) & (flat_train_df['SDS-SDS_Total_Raw'].isna())].shape[0]} cases where SDS_Total_Raw is not null but SDS_Total_T is''')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:49.541155Z","iopub.execute_input":"2024-12-23T09:18:49.541570Z","iopub.status.idle":"2024-12-23T09:18:49.558933Z","shell.execute_reply.started":"2024-12-23T09:18:49.541532Z","shell.execute_reply":"2024-12-23T09:18:49.557439Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualization of differences\nflat_train_df['Gap_to_delete'] = flat_train_df['SDS-SDS_Total_Raw'] - flat_train_df['SDS-SDS_Total_T']\nsns.scatterplot(data=flat_train_df, x='SDS-SDS_Total_Raw', y='SDS-SDS_Total_T', hue='Gap_to_delete', palette='coolwarm')\nplt.title('Scatter Plot of SDS-SDS_Total_Raw vs SDS-SDS_Total_T with Gap')\nplt.show()\n# del train_df['Gap_to_delete']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:49.560150Z","iopub.execute_input":"2024-12-23T09:18:49.560508Z","iopub.status.idle":"2024-12-23T09:18:50.123777Z","shell.execute_reply.started":"2024-12-23T09:18:49.560475Z","shell.execute_reply":"2024-12-23T09:18:50.122558Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Try to rebuild the Total version\nflat_train_df['new'] = flat_train_df['SDS-SDS_Total_T']*(100/flat_train_df['SDS-SDS_Total_Raw'].max())\nflat_train_df[['new','SDS-SDS_Total_Raw','SDS-SDS_Total_T']].sample(5)\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:50.125300Z","iopub.execute_input":"2024-12-23T09:18:50.125771Z","iopub.status.idle":"2024-12-23T09:18:50.254903Z","shell.execute_reply.started":"2024-12-23T09:18:50.125721Z","shell.execute_reply":"2024-12-23T09:18:50.253671Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Split train - test","metadata":{}},{"cell_type":"code","source":"train_df, test_df = train_test_split(flat_train_df, \n                                     test_size=ratio_tr_te,  \n                                     stratify=flat_train_df['sii'],  \n                                     random_state=14) \n\nprint(f'''We have {train_df.shape[0]} rows and {train_df.shape[1]} columns in the train dataset''')\nprint(f'''We have {test_df.shape[0]} rows and {test_df.shape[1]} columns in the test dataset''')\ndel flat_train_df\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:50.256199Z","iopub.execute_input":"2024-12-23T09:18:50.256651Z","iopub.status.idle":"2024-12-23T09:18:50.388955Z","shell.execute_reply.started":"2024-12-23T09:18:50.256615Z","shell.execute_reply":"2024-12-23T09:18:50.387684Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Same split for parquet data\nparquet_te_macro_df = parquet_tr_macro_df[parquet_tr_macro_df['id'].isin(set(test_df['id']))]\nparquet_tr_macro_df = parquet_tr_macro_df[parquet_tr_macro_df['id'].isin(set(train_df['id']))]\n\n\nprint(f'''We have {parquet_tr_macro_df.shape[0]} rows, {parquet_tr_macro_df.shape[1]} columns and {parquet_tr_macro_df['id'].nunique()} unique IDs in the train parquet dataset''')\nprint(f'''We have {parquet_te_macro_df.shape[0]} rows, {parquet_te_macro_df.shape[1]} columns and {parquet_te_macro_df['id'].nunique()} unique IDs in the 'fake test' parquet dataset''')\n\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:50.390272Z","iopub.execute_input":"2024-12-23T09:18:50.390657Z","iopub.status.idle":"2024-12-23T09:18:50.520988Z","shell.execute_reply.started":"2024-12-23T09:18:50.390622Z","shell.execute_reply":"2024-12-23T09:18:50.519730Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Pseudo Features Selection","metadata":{}},{"cell_type":"markdown","source":"## Features present in both train and test","metadata":{}},{"cell_type":"code","source":"flat_predictors = set(train_df.columns) - (set(train_df.columns)-set(flat_test_df.columns)) - set(['id'])\nprint(f'''We have {len(flat_predictors)} flat predictors''')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:50.522415Z","iopub.execute_input":"2024-12-23T09:18:50.522740Z","iopub.status.idle":"2024-12-23T09:18:50.529243Z","shell.execute_reply.started":"2024-12-23T09:18:50.522710Z","shell.execute_reply":"2024-12-23T09:18:50.527912Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"type_df = dico_df[['Field', 'Type']]\ntype_df = type_df[type_df['Field'].isin(flat_predictors)]\ntype_df['Type'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:50.530626Z","iopub.execute_input":"2024-12-23T09:18:50.530959Z","iopub.status.idle":"2024-12-23T09:18:50.553659Z","shell.execute_reply.started":"2024-12-23T09:18:50.530929Z","shell.execute_reply":"2024-12-23T09:18:50.552458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Numerical features\nflat_predictors_num = list(type_df[type_df['Type'].isin(['float','int'])]['Field'])\n\n# Categorical features\nflat_predictors_cat = list(set(list(type_df['Field'])) - set(flat_predictors_num))\nflat_predictors_cat = list(set(flat_predictors_cat) - set(['id']))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:50.554985Z","iopub.execute_input":"2024-12-23T09:18:50.555371Z","iopub.status.idle":"2024-12-23T09:18:50.573005Z","shell.execute_reply.started":"2024-12-23T09:18:50.555310Z","shell.execute_reply":"2024-12-23T09:18:50.571718Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(flat_predictors_num)+len(flat_predictors_cat) == len(flat_predictors)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:50.574457Z","iopub.execute_input":"2024-12-23T09:18:50.574884Z","iopub.status.idle":"2024-12-23T09:18:50.592268Z","shell.execute_reply.started":"2024-12-23T09:18:50.574809Z","shell.execute_reply":"2024-12-23T09:18:50.590904Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Remove the Seasons Features\nWe can think of using them later on","metadata":{}},{"cell_type":"code","source":"flat_predictors_cat = [s for s in flat_predictors_cat if not s.endswith(\"Season\")]\nflat_predictors_num = [s for s in flat_predictors_num if not s.endswith(\"Season\")]\nlen(flat_predictors_cat)+len(flat_predictors_num)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:50.593772Z","iopub.execute_input":"2024-12-23T09:18:50.594254Z","iopub.status.idle":"2024-12-23T09:18:50.609368Z","shell.execute_reply.started":"2024-12-23T09:18:50.594203Z","shell.execute_reply":"2024-12-23T09:18:50.607892Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Filter out huge NA features\nI checked the distribution of NA in test","metadata":{}},{"cell_type":"code","source":"# Missing values for the categorical features\nna_cat = train_df[flat_predictors_cat].isna().sum(axis=0)/len(train_df)\nflat_predictors_cat = list(na_cat[na_cat<ratio_na].index)\ndisplay(na_cat[na_cat<ratio_na])# ratio at the top of the NB","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:50.611584Z","iopub.execute_input":"2024-12-23T09:18:50.611989Z","iopub.status.idle":"2024-12-23T09:18:50.635005Z","shell.execute_reply.started":"2024-12-23T09:18:50.611939Z","shell.execute_reply":"2024-12-23T09:18:50.633515Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Missing values for the numeric features\nna_num = train_df[flat_predictors_num].isna().sum(axis=0)/len(train_df)\nflat_predictors_num = list(na_num[na_num<ratio_na].index)\ndisplay(na_num[na_num<ratio_na])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:50.636533Z","iopub.execute_input":"2024-12-23T09:18:50.636909Z","iopub.status.idle":"2024-12-23T09:18:50.652195Z","shell.execute_reply.started":"2024-12-23T09:18:50.636872Z","shell.execute_reply":"2024-12-23T09:18:50.650911Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Missing Imputation","metadata":{}},{"cell_type":"markdown","source":"## Create the Age Group Variable","metadata":{}},{"cell_type":"code","source":"# Train\nsns.countplot(data=train_df, x='Basic_Demos-Age')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:50.653814Z","iopub.execute_input":"2024-12-23T09:18:50.654185Z","iopub.status.idle":"2024-12-23T09:18:51.011962Z","shell.execute_reply.started":"2024-12-23T09:18:50.654151Z","shell.execute_reply":"2024-12-23T09:18:51.010674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Test\nsns.countplot(data=test_df, x='Basic_Demos-Age')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:51.013538Z","iopub.execute_input":"2024-12-23T09:18:51.013910Z","iopub.status.idle":"2024-12-23T09:18:51.429067Z","shell.execute_reply.started":"2024-12-23T09:18:51.013876Z","shell.execute_reply":"2024-12-23T09:18:51.427943Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Actual Test\nsns.countplot(data=flat_test_df, x='Basic_Demos-Age')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:51.430553Z","iopub.execute_input":"2024-12-23T09:18:51.430921Z","iopub.status.idle":"2024-12-23T09:18:51.713962Z","shell.execute_reply.started":"2024-12-23T09:18:51.430880Z","shell.execute_reply":"2024-12-23T09:18:51.712663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def age_group(age, th1=8, th2=12, th3=15):\n    out = 3\n    if age <= th1:\n        out = 0\n    elif age <= th2:\n        out = 1\n    elif age <= th3:\n        out = 2\n    else : \n        pass\n    return out\n\ntrain_df['Age-Group'] = train_df['Basic_Demos-Age'].apply(lambda x : age_group(x))\ntest_df['Age-Group'] = test_df['Basic_Demos-Age'].apply(lambda x : age_group(x))\nflat_test_df['Age-Group'] = flat_test_df['Basic_Demos-Age'].apply(lambda x : age_group(x))\n\nprint(train_df['Age-Group'].value_counts(normalize=True).sort_values(ascending=False))\nprint(test_df['Age-Group'].value_counts(normalize=True).sort_values(ascending=False))\nprint(flat_test_df['Age-Group'].value_counts(normalize=True).sort_values(ascending=False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:51.715498Z","iopub.execute_input":"2024-12-23T09:18:51.715898Z","iopub.status.idle":"2024-12-23T09:18:51.735498Z","shell.execute_reply.started":"2024-12-23T09:18:51.715862Z","shell.execute_reply":"2024-12-23T09:18:51.734280Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Join Flat DFs and Macro Parquet","metadata":{}},{"cell_type":"code","source":"# Train\ntrain_cl_df = pd.merge(train_df[['id','sii','PCIAT-PCIAT_Total','Age-Group']+flat_predictors_cat+flat_predictors_num], parquet_tr_macro_df, on='id', how='left')\nprint(f'''We have {train_cl_df.shape[0]} rows, {train_cl_df.shape[1]} columns and {train_cl_df['id'].nunique()} unique IDs in the pre-imputation train dataset''')\ndisplay(train_cl_df.sample(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:51.737109Z","iopub.execute_input":"2024-12-23T09:18:51.737553Z","iopub.status.idle":"2024-12-23T09:18:51.822747Z","shell.execute_reply.started":"2024-12-23T09:18:51.737506Z","shell.execute_reply":"2024-12-23T09:18:51.821550Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Test\ntest_cl_df = pd.merge(test_df[['id','sii','PCIAT-PCIAT_Total','Age-Group']+flat_predictors_cat+flat_predictors_num], parquet_te_macro_df, on='id', how='left')\nprint(f'''We have {test_cl_df.shape[0]} rows, {test_cl_df.shape[1]} columns and {test_cl_df['id'].nunique()} unique IDs in the pre-imputation test dataset''')\ndisplay(test_cl_df.sample(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:51.824241Z","iopub.execute_input":"2024-12-23T09:18:51.824692Z","iopub.status.idle":"2024-12-23T09:18:51.896881Z","shell.execute_reply.started":"2024-12-23T09:18:51.824646Z","shell.execute_reply":"2024-12-23T09:18:51.895734Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Real Test\nrealtest_cl_df = pd.merge(flat_test_df[['id','Age-Group']+flat_predictors_cat+flat_predictors_num], parquet_realte_macro_df, on='id', how='left')\nprint(f'''We have {realtest_cl_df.shape[0]} rows, {realtest_cl_df.shape[1]} columns and {realtest_cl_df['id'].nunique()} unique IDs in the pre-imputation real test dataset''')\ndisplay(realtest_cl_df.sample(5))\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:51.898520Z","iopub.execute_input":"2024-12-23T09:18:51.898986Z","iopub.status.idle":"2024-12-23T09:18:52.078489Z","shell.execute_reply.started":"2024-12-23T09:18:51.898934Z","shell.execute_reply":"2024-12-23T09:18:52.077307Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Specific NA Imputation","metadata":{}},{"cell_type":"code","source":"# REWORK ON THIS\n\ntrain_cl_df['num_rows'] = train_cl_df['num_rows'].fillna(0) \ntrain_cl_df['median_non-wear_flag'] = train_cl_df['median_non-wear_flag'].fillna(1) \n\ntest_cl_df['num_rows'] = test_cl_df['num_rows'].fillna(0) \ntest_cl_df['median_non-wear_flag'] = test_cl_df['median_non-wear_flag'].fillna(1) \n\nrealtest_cl_df['num_rows'] = realtest_cl_df['num_rows'].fillna(0) \nrealtest_cl_df['median_non-wear_flag'] = realtest_cl_df['median_non-wear_flag'].fillna(1) \n\nif 'mean_non-wear_flag' in train_cl_df.columns : \n    train_cl_df['mean_non-wear_flag'] = train_cl_df['mean_non-wear_flag'].fillna(1) \n    test_cl_df['mean_non-wear_flag'] = test_cl_df['mean_non-wear_flag'].fillna(1) \n    realtest_cl_df['mean_non-wear_flag'] = realtest_cl_df['mean_non-wear_flag'].fillna(1) \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:52.079801Z","iopub.execute_input":"2024-12-23T09:18:52.080150Z","iopub.status.idle":"2024-12-23T09:18:52.094576Z","shell.execute_reply.started":"2024-12-23T09:18:52.080116Z","shell.execute_reply":"2024-12-23T09:18:52.093389Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Build Logic on Age and Gender","metadata":{}},{"cell_type":"markdown","source":"### Numeric features","metadata":{}},{"cell_type":"code","source":"train_num_df = train_cl_df[flat_predictors_num+['Age-Group','Basic_Demos-Sex']+list(parquet_tr_macro_df.drop(columns=['id']).columns)]\ngrouped_medians = train_num_df.groupby(['Age-Group', 'Basic_Demos-Sex']).median()\ngrouped_medians = grouped_medians.apply(lambda col: col.fillna(col.median()), axis=0)# there are some NA, deal better with that later\ndisplay(grouped_medians)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:52.096045Z","iopub.execute_input":"2024-12-23T09:18:52.096464Z","iopub.status.idle":"2024-12-23T09:18:52.204207Z","shell.execute_reply.started":"2024-12-23T09:18:52.096384Z","shell.execute_reply":"2024-12-23T09:18:52.203123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Imputation function\ndef fill_na_with_group_median(row):\n    for col in flat_predictors_num+list(parquet_tr_macro_df.drop(columns=['id']).columns):\n        if pd.isna(row[col]):\n            # Get the median for the respective group\n            median_value = grouped_medians.loc[(row['Age-Group'], row['Basic_Demos-Sex']), col]\n            row[col] = median_value\n    return row\n\n# Apply the function to each row in the DataFrame\ntrain_num_df = train_num_df.apply(fill_na_with_group_median, axis=1)\ntrain_num_df.drop(columns = ['Age-Group','Basic_Demos-Sex'], inplace=True)\ntrain_num_df.sample(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:18:52.210658Z","iopub.execute_input":"2024-12-23T09:18:52.211071Z","iopub.status.idle":"2024-12-23T09:19:02.501011Z","shell.execute_reply.started":"2024-12-23T09:18:52.211024Z","shell.execute_reply":"2024-12-23T09:19:02.499923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Apply to test dataset\ntest_num_df = test_cl_df[flat_predictors_num+['Age-Group','Basic_Demos-Sex']+list(parquet_tr_macro_df.drop(columns=['id']).columns)]\n\ntest_num_df = test_num_df.apply(fill_na_with_group_median, axis=1)\ntest_num_df.drop(columns = ['Age-Group','Basic_Demos-Sex'], inplace=True)\ntest_num_df.sample(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:02.502306Z","iopub.execute_input":"2024-12-23T09:19:02.502671Z","iopub.status.idle":"2024-12-23T09:19:05.081565Z","shell.execute_reply.started":"2024-12-23T09:19:02.502637Z","shell.execute_reply":"2024-12-23T09:19:05.080391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"realtest_cl_df[list(parquet_tr_macro_df.drop(columns=['id']).columns)].head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:05.083217Z","iopub.execute_input":"2024-12-23T09:19:05.083707Z","iopub.status.idle":"2024-12-23T09:19:05.134659Z","shell.execute_reply.started":"2024-12-23T09:19:05.083653Z","shell.execute_reply":"2024-12-23T09:19:05.133624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Apply to actual test dataset\nflat_test_num_df = realtest_cl_df[flat_predictors_num+['Age-Group','Basic_Demos-Sex']+list(parquet_tr_macro_df.drop(columns=['id']).columns)]\n\nflat_test_num_df = flat_test_num_df.apply(fill_na_with_group_median, axis=1)\nflat_test_num_df.drop(columns = ['Age-Group','Basic_Demos-Sex'], inplace=True)\nflat_test_num_df.sample(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:05.136374Z","iopub.execute_input":"2024-12-23T09:19:05.137169Z","iopub.status.idle":"2024-12-23T09:19:05.342392Z","shell.execute_reply.started":"2024-12-23T09:19:05.137117Z","shell.execute_reply":"2024-12-23T09:19:05.341172Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Categorical Features\n - We get the distribution of each feature per Age X Sex\n - We use these distributions to generate a value when it's missing","metadata":{}},{"cell_type":"code","source":"# Add the Age-Group Variable\ntrain_cat_df = train_cl_df[flat_predictors_cat+['Age-Group']]\ntest_cat_df = test_cl_df[flat_predictors_cat+['Age-Group']]\nflat_test_cat_df = realtest_cl_df[flat_predictors_cat+['Age-Group']]\n\nprint('Pre-Imputation')\ndisplay(train_cat_df.sample(5))\n\n\nif categorical_na=='age-sex' : \n\n    # Get the distribution of each feature per group (age x sex)\n    grouped_distributions = {}\n\n    for col in list(set(train_cat_df.columns)-set(['Age-Group','Basic_Demos-Sex'])):\n        grouped_distributions[col] = train_cat_df.groupby(['Age-Group', 'Basic_Demos-Sex'])[col].value_counts(normalize=True)\n\n    def fill_na_with_category(row):\n        for col in list(set(train_cat_df.columns)-set(['Age-Group','Basic_Demos-Sex'])):\n            if pd.isna(row[col]):\n\n                group = (row['Age-Group'], row['Basic_Demos-Sex'])\n                distribution = grouped_distributions[col].loc[group]\n                \n                sampled_category = np.random.choice(distribution.index, p=distribution.values)\n                \n                # Replace NA with the sampled category\n                row[col] = sampled_category\n        return row\n\n    # Application to train\n    train_cat_df = train_cat_df.apply(fill_na_with_category, axis=1)\n    train_cat_df.drop(columns = ['Age-Group'], inplace=True)\n\n    # Apply to test dataset\n    test_cat_df = test_cat_df.apply(fill_na_with_category, axis=1)\n    test_cat_df.drop(columns = ['Age-Group'], inplace=True)\n\n    # Apply to actual test dataset\n    flat_test_cat_df = flat_test_cat_df.apply(fill_na_with_category, axis=1)\n    flat_test_cat_df.drop(columns = ['Age-Group'], inplace=True)\n\n\nelif categorical_na=='missing' :\n\n    train_cat_df[flat_predictors_cat] = train_cat_df[flat_predictors_cat].fillna(-1)\n    test_cat_df[flat_predictors_cat] = test_cat_df[flat_predictors_cat].fillna(-1)\n    flat_test_cat_df[flat_predictors_cat] = flat_test_cat_df[flat_predictors_cat].fillna(-1)\n\n\nelif categorical_na=='ohe' :   \n\n    ## Make an exception for PreInt_EduHx-computerinternet_hoursday\n    train_cat_df['PreInt_EduHx-computerinternet_hoursday'] = train_cat_df['PreInt_EduHx-computerinternet_hoursday'].fillna(-1)\n    test_cat_df['PreInt_EduHx-computerinternet_hoursday'] = test_cat_df['PreInt_EduHx-computerinternet_hoursday'].fillna(-1)\n    flat_test_cat_df['PreInt_EduHx-computerinternet_hoursday'] = flat_test_cat_df['PreInt_EduHx-computerinternet_hoursday'].fillna(-1)# THEN TRY EXP\n\n    train_cat_df = pd.get_dummies(train_cat_df, columns=list(set(flat_predictors_cat)-set(['PreInt_EduHx-computerinternet_hoursday'])))\n    train_cat_df.drop(columns = ['Age-Group'], inplace=True)\n    train_cat_df = train_cat_df.astype(int)#get 1 and 0, not True and False\n\n    test_cat_df = pd.get_dummies(test_cat_df, columns=list(set(flat_predictors_cat)-set(['PreInt_EduHx-computerinternet_hoursday'])))\n    test_cat_df.drop(columns = ['Age-Group'], inplace=True)\n    test_cat_df = test_cat_df.astype(int)\n\n    flat_test_cat_df = pd.get_dummies(flat_test_cat_df, columns=list(set(flat_predictors_cat)-set(['PreInt_EduHx-computerinternet_hoursday'])))\n    flat_test_cat_df.drop(columns = ['Age-Group'], inplace=True)\n    flat_test_cat_df = flat_test_cat_df.astype(int)\n\n    # train_cat_df[list(set(flat_predictors_cat)-set(['PreInt_EduHx-computerinternet_hoursday']))] =  train_cat_df[list(set(flat_predictors_cat)-set(['PreInt_EduHx-computerinternet_hoursday']))].fillna(-1)\n    # test_cat_df[list(set(flat_predictors_cat)-set(['PreInt_EduHx-computerinternet_hoursday']))] =  test_cat_df[list(set(flat_predictors_cat)-set(['PreInt_EduHx-computerinternet_hoursday']))].fillna(-1)\n    # flat_test_cat_df[list(set(flat_predictors_cat)-set(['PreInt_EduHx-computerinternet_hoursday']))] =  flat_test_cat_df[list(set(flat_predictors_cat)-set(['PreInt_EduHx-computerinternet_hoursday']))].fillna(-1)\n\n\nelse:\n    pass\n\n    \nprint('Post-Imputation')\ndisplay(train_cat_df.sample(5)) \ngc.collect()  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:05.344384Z","iopub.execute_input":"2024-12-23T09:19:05.344734Z","iopub.status.idle":"2024-12-23T09:19:05.510547Z","shell.execute_reply.started":"2024-12-23T09:19:05.344700Z","shell.execute_reply":"2024-12-23T09:19:05.509411Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Standardization of continous features","metadata":{}},{"cell_type":"code","source":"if standardization_flag:\n\n    # Display pre standardization train df\n    print('Pre-Standardization dataframe')\n    display(train_num_df.sample(5))\n\n    # Get mean and sd per column from train_df\n    mean_std = {\n    col: {\n        'mean': train_num_df[col].mean(),\n        'std': train_num_df[col].std()\n    }\n    for col in train_num_df.columns\n    }\n\n    # Apply the standardization to the 3 datasets\n    for col in train_num_df.columns:\n        mean = mean_std[col]['mean']\n        std = mean_std[col]['std']\n\n        if std==0:#can happen with some min and max features\n            del train_num_df[col], test_num_df[col], flat_test_num_df[col]\n            \n        else:\n            train_num_df[col] = (train_num_df[col] - mean) / std\n            test_num_df[col] = (test_num_df[col] - mean) / std\n            flat_test_num_df[col] = (flat_test_num_df[col] - mean) / std\n    \n    # Display post standardization train df\n    print('Post-Standardization dataframe')\n    display(train_num_df.sample(5))\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:05.512162Z","iopub.execute_input":"2024-12-23T09:19:05.512651Z","iopub.status.idle":"2024-12-23T09:19:05.703464Z","shell.execute_reply.started":"2024-12-23T09:19:05.512602Z","shell.execute_reply":"2024-12-23T09:19:05.702324Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Put together Num and Cat DFs","metadata":{}},{"cell_type":"code","source":"# Train\ntrain_final_df  = pd.concat([train_cat_df, train_num_df[list(set(train_num_df.columns)-set(train_cat_df.columns))],train_cl_df[['id','PCIAT-PCIAT_Total','sii']]], axis=1)\nprint(f'''We have {train_final_df.shape[0]} rows and {train_final_df.shape[1]} columns in the final train dataset''')\ndel train_num_df, train_cl_df\ndisplay(train_final_df.sample(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:05.704838Z","iopub.execute_input":"2024-12-23T09:19:05.705267Z","iopub.status.idle":"2024-12-23T09:19:05.770144Z","shell.execute_reply.started":"2024-12-23T09:19:05.705218Z","shell.execute_reply":"2024-12-23T09:19:05.768997Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Test\ntest_final_df  = pd.concat([test_cat_df, test_num_df[list(set(test_num_df.columns)-set(test_cat_df.columns))],test_cl_df[['id','PCIAT-PCIAT_Total','sii']]], axis=1)\nprint(f'''We have {test_final_df.shape[0]} rows and {test_final_df.shape[1]} columns in the final test dataset''')\ndel test_num_df, test_cl_df\ndisplay(test_final_df.sample(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:05.772053Z","iopub.execute_input":"2024-12-23T09:19:05.772522Z","iopub.status.idle":"2024-12-23T09:19:05.838374Z","shell.execute_reply.started":"2024-12-23T09:19:05.772476Z","shell.execute_reply":"2024-12-23T09:19:05.837179Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Actual Test\nreal_test_final_df  = pd.concat([flat_test_cat_df, flat_test_num_df[list(set(flat_test_num_df.columns)-set(flat_test_cat_df.columns))],flat_test_df[['id']]], axis=1)\nprint(f'''We have {real_test_final_df.shape[0]} rows and {real_test_final_df.shape[1]} columns in the final test dataset''')\ndisplay(real_test_final_df.sample(5))\ndel flat_test_num_df, flat_test_cat_df\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:05.839617Z","iopub.execute_input":"2024-12-23T09:19:05.839936Z","iopub.status.idle":"2024-12-23T09:19:06.028973Z","shell.execute_reply.started":"2024-12-23T09:19:05.839906Z","shell.execute_reply":"2024-12-23T09:19:06.027732Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Features Selection","metadata":{}},{"cell_type":"code","source":"if final_features=='flat':\n    selected_features = flat_predictors_cat + flat_predictors_num + ['id','PCIAT-PCIAT_Total','sii']\nelse:\n    selected_features = train_final_df.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:06.030471Z","iopub.execute_input":"2024-12-23T09:19:06.030852Z","iopub.status.idle":"2024-12-23T09:19:06.036225Z","shell.execute_reply.started":"2024-12-23T09:19:06.030807Z","shell.execute_reply":"2024-12-23T09:19:06.034990Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ML Model","metadata":{}},{"cell_type":"markdown","source":"## Fit and Pred","metadata":{}},{"cell_type":"code","source":"# Fit\n\nif target_type=='numeric':\n    \n    if model_type=='lasso':\n        print(\"Lasso\")\n        model = Lasso()\n        model.fit(train_final_df[selected_features].drop(columns=['id','PCIAT-PCIAT_Total','sii']), train_final_df['PCIAT-PCIAT_Total'])\n\n\n    elif model_type=='linear':\n        print(\"Linear Regression\")\n        model = LinearRegression()\n        model.fit(train_final_df[selected_features].drop(columns=['id','PCIAT-PCIAT_Total','sii']), train_final_df['PCIAT-PCIAT_Total'])\n\n\n    elif model_type=='lightgbm':\n        print(\"Light GBM\")\n\n        # def cohen_kappa_scorer(y_true, y_pred):\n        #     return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n        # kappa_scorer = make_scorer(cohen_kappa_scorer)\n\n        # Manual Grid\n        l_max_depth = [2,3]  # -1 means no limit\n        l_learning_rate = [0.00825, 0.0085, 0.00875]\n        l_n_estimators = [390, 400, 410]\n        #, 'num_leaves': [15, 31, 63]\n        l_feature_fraction = [0.5]\n\n        model_counter = 1\n        model_hyperparameters = []\n        l_tr_metric = []\n        l_te_metric = []\n        l_models = []\n\n\n\n        for md in range(0,len(l_max_depth)):\n            for lr in range(0,len(l_learning_rate)):\n                for tree in range(0,len(l_n_estimators)):\n                    for fra in range(0,len(l_feature_fraction)):\n\n                        gbm = lgb.LGBMRegressor(random_state=14, verbose=-1\n                                    , max_depth = l_max_depth[md]\n                                    , learning_rate = l_learning_rate[lr]\n                                    , n_estimators = l_n_estimators[tree]\n                                    , feature_fraction = l_feature_fraction[fra]                                                                \n                                    )\n                                        \n                        print(f'''\\t Iteration {model_counter}/{len(l_max_depth)*len(l_learning_rate)*len(l_n_estimators)*len(l_feature_fraction)}...''')\n\n                        \n                        # Fit Model\n                        gbm.fit(train_final_df[selected_features].drop(columns=['id','PCIAT-PCIAT_Total','sii']), train_final_df['PCIAT-PCIAT_Total'])\n\n                        # Predictions\n                        y_train_pred = gbm.predict(train_final_df[selected_features].drop(columns=['id','PCIAT-PCIAT_Total','sii']))\n                        y_test_pred = gbm.predict(test_final_df[selected_features].drop(columns=['id','PCIAT-PCIAT_Total','sii']))\n\n                        # Recompute categorical target\n                        y_train_true = train_final_df['sii'].copy()\n                        y_test_true = test_final_df['sii'].copy()\n                        # print('pre-sort')\n                        # print(y_train_true)\n\n                        if scoring_method=='threshold':\n\n                            ## Train\n                            conditions_tr = [\n                            (y_train_pred <= 30),    \n                            (y_train_pred <= 49),     \n                            (y_train_pred <= 79),    \n                            (y_train_pred >= 80)     \n                            ]\n                            values = [0, 1, 2, 3]\n                            sii_train_pred = pd.Series(np.select(conditions_tr, values, default=-1))\n\n                            ## Test\n                            \n                            conditions_te = [\n                            (y_test_pred <= 30),    \n                            (y_test_pred <= 49),     \n                            (y_test_pred <= 79),    \n                            (y_test_pred >= 80)     \n                            ]\n                            sii_test_pred = pd.Series(np.select(conditions_te, values, default=-1))\n\n\n                        else:#distribution\n                            ## Train\n                            sorted_indices_tr = np.argsort(y_train_pred)\n                            y_train_pred = y_train_pred[sorted_indices_tr]\n                            y_train_true = y_train_true[sorted_indices_tr]\n                            cut_offs_tr = (np.cumsum(target_distr) * len(y_train_pred)).astype(int)\n\n                            sii_train_pred = np.zeros_like(y_train_pred, dtype=int)\n                            previous_cutoff = 0\n                            for value, cutoff in zip(target_distr.index, cut_offs_tr):\n                                sii_train_pred[previous_cutoff:cutoff] = value\n                                previous_cutoff = cutoff\n                            sii_train_pred = pd.Series(sii_train_pred)\n\n                            ## Test\n                            sorted_indices_te = np.argsort(y_test_pred)\n                            y_test_pred = y_test_pred[sorted_indices_te]\n                            y_test_true = y_test_true[sorted_indices_te]\n                            cut_offs_te = (np.cumsum(target_distr) * len(y_test_pred)).astype(int)\n\n                            sii_test_pred = np.zeros_like(y_test_pred, dtype=int)\n                            previous_cutoff = 0\n                            for value, cutoff in zip(target_distr.index, cut_offs_te):\n                                sii_test_pred[previous_cutoff:cutoff] = value\n                                previous_cutoff = cutoff\n                            sii_test_pred = pd.Series(sii_test_pred)\n                            \n\n                        # Metric\n                        # print('post-sort')\n                        # print(y_train_true)\n                        qwk_tr = cohen_kappa_score(y_train_true, sii_train_pred, weights='quadratic')\n                        qwk_te = cohen_kappa_score(y_test_true, sii_test_pred, weights='quadratic')\n\n                        # print(f'''/t /t Train Quadratic Weighted Kappa : {round(qwk_tr,3)}''')\n                        # print(f'''/t /t Test Quadratic Weighted Kappa : {round(qwk_te,3)}''')\n                        l_tr_metric.append(qwk_tr)\n                        l_te_metric.append(qwk_te)\n                        l_models.append(gbm)\n\n                        # Save iteration\n                        model_hyperparameters.append({\n                        'model_number': model_counter,\n                        'max_depth': l_max_depth[md],\n                        'learning_rate': l_learning_rate[lr],\n                        'n_estimators': l_n_estimators[tree],\n                        'feature_fraction': l_feature_fraction[fra]\n                        })\n                        model_counter=model_counter+1\n        \n        # Best gbm\n        best_index = -1\n        max_te_metric = -float('inf')\n\n        # Iterate through both lists\n        for i in range(len(l_tr_metric)):\n            if abs(l_tr_metric[i] - l_te_metric[i]) < overfitting_th:\n                if l_te_metric[i] > max_te_metric:\n                    max_te_metric = l_te_metric[i]\n                    best_index = i\n\n        # Model picking\n        model = l_models[best_index]\n        print(f'''\\n \\t Train metric {round(l_tr_metric[best_index],3)}''')\n        print(f'''\\t Test metric {round(l_te_metric[best_index],3)}''')\n        print(f'''\\t For these hyperparameters {model_hyperparameters[best_index]}''')\n\n                    \n                    \n\n    else : \n        print(\"No model selected\")\n\nelse:#categorical target\n\n    if model_type=='lightgbm':\n        print(\"Light GBM\")\n\n        # Manual Grid\n        l_max_depth = [2, 3, 4]  # -1 means no limit\n        l_learning_rate = [0.05, 0.075, 0.1, 0.125]\n        l_n_estimators = [75, 100, 125, 150, 200, 300]\n        #, 'num_leaves': [15, 31, 63]\n        l_feature_fraction = [0.2, 0.33,0.5, 0.75]\n\n        model_counter = 1\n        model_hyperparameters = []\n        l_tr_metric = []\n        l_te_metric = []\n        l_models = []\n\n\n\n        for md in range(0,len(l_max_depth)):\n            for lr in range(0,len(l_learning_rate)):\n                for tree in range(0,len(l_n_estimators)):\n                    for fra in range(0,len(l_feature_fraction)):\n\n                        gbm = lgb.LGBMClassifier(random_state=13, verbose=-1\n                                    , max_depth = l_max_depth[md]\n                                    , learning_rate = l_learning_rate[lr]\n                                    , n_estimators = l_n_estimators[tree]\n                                    , feature_fraction = l_feature_fraction[fra]                                                                \n                                    )\n                                        \n                        print(f'''\\t Iteration {model_counter}/{len(l_max_depth)*len(l_learning_rate)*len(l_n_estimators)*len(l_feature_fraction)}...''')\n\n                        \n                        # Fit Model\n                        gbm.fit(train_final_df[selected_features].drop(columns=['id','PCIAT-PCIAT_Total','sii']), train_final_df['sii'].astype(int))\n\n                        # Predictions\n                        sii_train_pred = gbm.predict(train_final_df[selected_features].drop(columns=['id','PCIAT-PCIAT_Total','sii']))\n                        sii_test_pred = gbm.predict(test_final_df[selected_features].drop(columns=['id','PCIAT-PCIAT_Total','sii']))\n\n                        # Metric\n                        qwk_tr = cohen_kappa_score(train_final_df['sii'].astype(int), sii_train_pred, weights='quadratic')\n                        qwk_te = cohen_kappa_score(test_final_df['sii'].astype(int), sii_test_pred, weights='quadratic')\n\n                        l_tr_metric.append(qwk_tr)\n                        l_te_metric.append(qwk_te)\n                        l_models.append(gbm)\n\n                        # Save iteration\n                        model_hyperparameters.append({\n                        'model_number': model_counter,\n                        'max_depth': l_max_depth[md],\n                        'learning_rate': l_learning_rate[lr],\n                        'n_estimators': l_n_estimators[tree],\n                        'feature_fraction': l_feature_fraction[fra]\n                        })\n                        model_counter=model_counter+1\n        \n        # Best gbm\n        best_index = -1\n        max_te_metric = -float('inf')\n\n        # Iterate through both lists\n        for i in range(len(l_tr_metric)):\n            if abs(l_tr_metric[i] - l_te_metric[i]) < overfitting_th:\n                if l_te_metric[i] > max_te_metric:\n                    max_te_metric = l_te_metric[i]\n                    best_index = i\n\n        # Model picking\n        model = l_models[best_index]\n        print(f'''\\n \\t Train metric {round(l_tr_metric[best_index],3)}''')\n        print(f'''\\t Test metric {round(l_te_metric[best_index],3)}''')\n        print(f'''\\t For these hyperparameters {model_hyperparameters[best_index]}''')\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:06.038330Z","iopub.execute_input":"2024-12-23T09:19:06.038764Z","iopub.status.idle":"2024-12-23T09:19:12.827975Z","shell.execute_reply.started":"2024-12-23T09:19:06.038729Z","shell.execute_reply":"2024-12-23T09:19:12.826836Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:12.829651Z","iopub.execute_input":"2024-12-23T09:19:12.830157Z","iopub.status.idle":"2024-12-23T09:19:12.845052Z","shell.execute_reply.started":"2024-12-23T09:19:12.830107Z","shell.execute_reply":"2024-12-23T09:19:12.843671Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Predict\nif target_type=='numeric':\n    y_train_pred = model.predict(train_final_df[selected_features].drop(columns=['id','PCIAT-PCIAT_Total','sii']))\n    y_test_pred = model.predict(test_final_df[selected_features].drop(columns=['id','PCIAT-PCIAT_Total','sii']))\n    y_actual_test_pred = model.predict(real_test_final_df[train_final_df[selected_features].drop(columns=['id','PCIAT-PCIAT_Total','sii']).columns])\nelse :\n    sii_train_pred = model.predict(train_final_df[selected_features].drop(columns=['id','PCIAT-PCIAT_Total','sii']))\n    sii_test_pred = model.predict(test_final_df[selected_features].drop(columns=['id','PCIAT-PCIAT_Total','sii']))\n    sii_ac_test_pred = model.predict(real_test_final_df[train_final_df[selected_features].drop(columns=['id','PCIAT-PCIAT_Total','sii']).columns])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:12.846904Z","iopub.execute_input":"2024-12-23T09:19:12.847400Z","iopub.status.idle":"2024-12-23T09:19:12.896056Z","shell.execute_reply.started":"2024-12-23T09:19:12.847328Z","shell.execute_reply":"2024-12-23T09:19:12.894756Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Mean Absolute Error","metadata":{}},{"cell_type":"code","source":"if target_type=='numeric':\n    y_train_true = train_final_df['PCIAT-PCIAT_Total']\n    y_test_true = test_final_df['PCIAT-PCIAT_Total']\n\n    MAE_tr = (np.abs(y_train_pred-y_train_true)).mean()\n    MAE_te = (np.abs(y_test_pred-y_test_true)).mean()\n\n    print(f'''Train Mean Absolute Error (MAE) : {round(MAE_tr,1)}''')\n    print(f'''Test Mean Absolute Error (MAE) : {round(MAE_te,1)}''')\nelse : \n    y_train_true = train_final_df['sii'].astype(int)\n    y_test_true = test_final_df['sii'].astype(int)\n    print(\"No MAE for Categorical Target\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:12.897435Z","iopub.execute_input":"2024-12-23T09:19:12.897779Z","iopub.status.idle":"2024-12-23T09:19:12.906794Z","shell.execute_reply.started":"2024-12-23T09:19:12.897748Z","shell.execute_reply":"2024-12-23T09:19:12.905433Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Quadratic Weighted Kappa","metadata":{}},{"cell_type":"code","source":"if target_type=='numeric':\n    y_true_train = train_final_df['sii'].copy()\n\n    if scoring_method=='threshold':\n        conditions_tr = [\n            (y_train_pred <= 30),    \n            (y_train_pred <= 49),     \n            (y_train_pred <= 79),    \n            (y_train_pred >= 80)     \n        ]\n\n        values = [0, 1, 2, 3]\n\n        sii_train_pred = pd.Series(np.select(conditions_tr, values, default=-1))\n        \n        \n    else:#distribution\n        sorted_indices_tr = np.argsort(y_train_pred)\n        y_train_pred = y_train_pred[sorted_indices_tr]\n        y_true_train = y_true_train[sorted_indices_tr]\n        cut_offs_tr = (np.cumsum(target_distr) * len(y_train_pred)).astype(int)\n\n        sii_train_pred = np.zeros_like(y_train_pred, dtype=int)\n        previous_cutoff = 0\n        for value, cutoff in zip(target_distr.index, cut_offs_tr):\n            sii_train_pred[previous_cutoff:cutoff] = value\n            previous_cutoff = cutoff\n        sii_train_pred = pd.Series(sii_train_pred)\n\n    \n    print(sii_train_pred.value_counts())\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:12.908272Z","iopub.execute_input":"2024-12-23T09:19:12.908710Z","iopub.status.idle":"2024-12-23T09:19:12.928810Z","shell.execute_reply.started":"2024-12-23T09:19:12.908672Z","shell.execute_reply":"2024-12-23T09:19:12.927602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if target_type=='numeric':\n\n    y_true_test = test_final_df['sii'].copy()\n\n    if scoring_method=='threshold':\n\n        conditions_te = [\n        (y_test_pred <= 30),    \n        (y_test_pred <= 49),     \n        (y_test_pred <= 79),    \n        (y_test_pred >= 80)     \n        ]\n\n        sii_test_pred = pd.Series(np.select(conditions_te, values, default=-1))\n\n    else:#distribution\n        sorted_indices_te = np.argsort(y_test_pred)\n        y_test_pred = y_test_pred[sorted_indices_te]\n        y_true_test = y_true_test[sorted_indices_te]\n        cut_offs_te = (np.cumsum(target_distr) * len(y_test_pred)).astype(int)\n\n        sii_test_pred = np.zeros_like(y_test_pred, dtype=int)\n        previous_cutoff = 0\n        for value, cutoff in zip(target_distr.index, cut_offs_te):\n            sii_test_pred[previous_cutoff:cutoff] = value\n            previous_cutoff = cutoff\n        sii_test_pred = pd.Series(sii_test_pred)\n\n\n    print(sii_test_pred.value_counts())\n    \n        ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:12.930606Z","iopub.execute_input":"2024-12-23T09:19:12.930988Z","iopub.status.idle":"2024-12-23T09:19:12.942907Z","shell.execute_reply.started":"2024-12-23T09:19:12.930955Z","shell.execute_reply":"2024-12-23T09:19:12.941670Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scoring_df = real_test_final_df[['id']].copy()\ndisplay(scoring_df)\n\nif target_type=='numeric':\n\n    if scoring_method=='threshold':\n        conditions_ac_te = [\n        (y_actual_test_pred <= 30),    \n        (y_actual_test_pred <= 49),     \n        (y_actual_test_pred <= 79),    \n        (y_actual_test_pred >= 80)     \n        ]\n\n        sii_ac_test_pred = pd.Series(np.select(conditions_ac_te, values, default=-1))\n\n    else:#distribution\n        sorted_indices_acte = np.argsort(y_actual_test_pred)\n        y_actual_test_pred = y_actual_test_pred[sorted_indices_acte]\n        cut_offs_acte = (np.cumsum(target_distr) * len(y_actual_test_pred)).astype(int)\n\n        sii_ac_test_pred = np.zeros_like(y_actual_test_pred, dtype=int)\n        previous_cutoff = 0\n        for value, cutoff in zip(target_distr.index, cut_offs_acte):\n            sii_ac_test_pred[previous_cutoff:cutoff] = value\n            previous_cutoff = cutoff\n        sii_ac_test_pred = pd.Series(sii_ac_test_pred)\n        scoring_df = scoring_df.iloc[sorted_indices_acte].reset_index(drop=True)\n\n\n    print(sii_ac_test_pred.value_counts())\n\n\n\n# Final DF\nscoring_df['sii'] = sii_ac_test_pred\ndisplay(scoring_df)\n\n# Export csv\nscoring_df.to_csv('submission.csv', index=False)\n# del real_test_final_df['sii']\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:12.944388Z","iopub.execute_input":"2024-12-23T09:19:12.944949Z","iopub.status.idle":"2024-12-23T09:19:12.980302Z","shell.execute_reply.started":"2024-12-23T09:19:12.944883Z","shell.execute_reply":"2024-12-23T09:19:12.979149Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if target_type=='numeric':\n    qwk_tr = cohen_kappa_score(y_true_train, sii_train_pred, weights='quadratic')# check again if it's really the same metrics as in kaggleqwk_tr = cohen_kappa_score(train_final_df['sii'], sii_train_pred, weights='quadratic')# check again if it's really the same metrics as in kaggle\n    qwk_te = cohen_kappa_score(y_true_test, sii_test_pred, weights='quadratic')# check again if it's really the same metrics as in kaggle\n    \nelse:\n    qwk_tr = cohen_kappa_score(y_train_true, sii_train_pred, weights='quadratic')# check again if it's really the same metrics as in kaggleqwk_tr = cohen_kappa_score(train_final_df['sii'], sii_train_pred, weights='quadratic')# check again if it's really the same metrics as in kaggle\n    qwk_te = cohen_kappa_score(y_test_true, sii_test_pred, weights='quadratic')\n\nprint(f'''Train Quadratic Weighted Kappa : {round(qwk_tr,3)}''')\nprint(f'''Test Quadratic Weighted Kappa : {round(qwk_te,3)}''')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:12.981738Z","iopub.execute_input":"2024-12-23T09:19:12.982087Z","iopub.status.idle":"2024-12-23T09:19:12.996242Z","shell.execute_reply.started":"2024-12-23T09:19:12.982053Z","shell.execute_reply":"2024-12-23T09:19:12.995161Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Features Importance","metadata":{}},{"cell_type":"code","source":"if model_type=='lightgbm':\n    importances = model.feature_importances_\n    feature_names = model.feature_name_\n\n    feature_importance_df = pd.DataFrame({\n        'Feature': feature_names,\n        'Importance': importances\n    }).sort_values(by='Importance', ascending=False)\n\n    # print(feature_importance_df)\n\n    # Plot feature importance using LightGBM's built-in function\n    plot_importance(model, max_num_features=25, importance_type='split')  # Use 'split' or 'gain'\n    plt.show()\n\nelse : \n\n    # Assume model is your trained Lasso/LinearRegression model\n    # and X_train is your training data with feature names\n\n    # Get feature importance (coefficients)\n    coefficients = model.coef_\n    features = model.feature_names_in_  # Replace with feature names\n\n    # Sort coefficients by absolute value\n    sorted_indices = np.argsort(np.abs(coefficients))[::-1][:20]\n    sorted_features = features[sorted_indices]\n    sorted_coefficients = coefficients[sorted_indices]\n\n    # Plot\n    plt.figure(figsize=(10, 6))\n    plt.barh(sorted_features, sorted_coefficients, color='skyblue')\n    plt.xlabel(\"Coefficient Value\")\n    plt.ylabel(\"Feature\")\n    plt.title(\"Feature Importance (Lasso/Linear Regression)\")\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T09:19:12.997621Z","iopub.execute_input":"2024-12-23T09:19:12.997957Z","iopub.status.idle":"2024-12-23T09:19:13.466181Z","shell.execute_reply.started":"2024-12-23T09:19:12.997926Z","shell.execute_reply":"2024-12-23T09:19:13.464860Z"}},"outputs":[],"execution_count":null}]}