{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30776,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Description of Imported Libraries\n\n- **NumPy (`np`)**: Used for efficient numerical operations, including linear algebra and array manipulation.\n- **Pandas (`pd`)**: Provides data structures like DataFrames for handling structured data, essential for data preprocessing.\n- **Polars (`pl`)**: A faster alternative to pandas for DataFrame operations, particularly useful for large datasets.\n- **Matplotlib & Seaborn (`plt`, `sns`)**: Visualization libraries. Matplotlib is used for basic plots, while Seaborn builds on it to create more advanced statistical visualizations.\n- **LightGBM, XGBoost, CatBoost**: Machine learning libraries used for gradient boosting, which is efficient for both regression and classification tasks.\n- **Colorama**: Enhances console output with colored text, making it easier to highlight important results or warnings.\n- **SciPy (`minimize`)**: Provides optimization routines, such as adjusting thresholds to maximize performance metrics like kappa scores.\n- **OS**: Used for file path manipulations and system-related functions.\n- **Scikit-learn (`sklearn`)**: A powerful machine learning library, providing utilities for cross-validation, metrics, and model cloning.\n- **YDF**: A specialized library for machine learning tasks, likely including decision forests.\n- **ThreadPoolExecutor & TQDM**: Tools for parallelizing tasks and displaying progress bars for long-running loops, improving efficiency and usability.\n- **Warnings**: Filters out unwanted warnings to keep the output clean, useful when dealing with noisy outputs from multiple libraries.\n- **IPython display (`clear_output`)**: A utility for clearing the Jupyter notebook output, often used to avoid clutter in long-running scripts.\n","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:14.495618Z","iopub.execute_input":"2024-11-14T08:16:14.496549Z","iopub.status.idle":"2024-11-14T08:16:14.501366Z","shell.execute_reply.started":"2024-11-14T08:16:14.496502Z","shell.execute_reply":"2024-11-14T08:16:14.500252Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport polars.selectors as cs\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\nimport seaborn as sns\nimport lightgbm as lgb\nfrom colorama import Fore, Style\nfrom scipy.optimize import minimize\nimport os","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:14.503133Z","iopub.execute_input":"2024-11-14T08:16:14.503456Z","iopub.status.idle":"2024-11-14T08:16:14.513651Z","shell.execute_reply.started":"2024-11-14T08:16:14.503423Z","shell.execute_reply":"2024-11-14T08:16:14.512700Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import cohen_kappa_score, ConfusionMatrixDisplay, confusion_matrix\n\nimport re\nfrom sklearn.base import clone\nimport ydf\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\nfrom keras.models import Model\nfrom keras.layers import Input, Dense, Dropout\nfrom keras.optimizers import Adam\nfrom sklearn.preprocessing import StandardScaler\n\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:14.536032Z","iopub.execute_input":"2024-11-14T08:16:14.536441Z","iopub.status.idle":"2024-11-14T08:16:14.544914Z","shell.execute_reply.started":"2024-11-14T08:16:14.536398Z","shell.execute_reply":"2024-11-14T08:16:14.543779Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_labels = ['None', 'Mild', 'Moderate', 'Severe']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:14.546742Z","iopub.execute_input":"2024-11-14T08:16:14.547092Z","iopub.status.idle":"2024-11-14T08:16:14.553588Z","shell.execute_reply.started":"2024-11-14T08:16:14.547058Z","shell.execute_reply":"2024-11-14T08:16:14.552702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"season_dtype = pl.Enum(['Spring', 'Summer', 'Fall', 'Winter'])\n\ntrain = (\n    pl.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\n    .with_columns(pl.col('^.*Season$').cast(season_dtype))\n)\n\ntest = (\n    pl.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n    .with_columns(pl.col('^.*Season$').cast(season_dtype))\n)\n\ntrain\ntest","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:14.554855Z","iopub.execute_input":"2024-11-14T08:16:14.555199Z","iopub.status.idle":"2024-11-14T08:16:14.616152Z","shell.execute_reply.started":"2024-11-14T08:16:14.555164Z","shell.execute_reply":"2024-11-14T08:16:14.615066Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"For a supervised learning, we need the target value, but some (sii) are missing. So we only use the part with valid target value(sii).","metadata":{}},{"cell_type":"code","source":"supervised_usable = (\n    train\n    .filter(pl.col('sii').is_not_null())\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\nplt.figure(figsize=(6, 15))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:14.618185Z","iopub.execute_input":"2024-11-14T08:16:14.618529Z","iopub.status.idle":"2024-11-14T08:16:15.854458Z","shell.execute_reply.started":"2024-11-14T08:16:14.618494Z","shell.execute_reply":"2024-11-14T08:16:15.853461Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train.select(pl.col('PCIAT-PCIAT_Total').is_null() == pl.col('sii').is_null()).to_series().mean())\n\n(train\n .select(pl.col('PCIAT-PCIAT_Total'))\n .group_by(train.get_column('sii'))\n .agg(pl.col('PCIAT-PCIAT_Total').min().alias('PCIAT-PCIAT_Total min'),\n      pl.col('PCIAT-PCIAT_Total').max().alias('PCIAT-PCIAT_Total max'),\n      pl.col('PCIAT-PCIAT_Total').len().alias('count'))\n .sort('sii')\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:15.855842Z","iopub.execute_input":"2024-11-14T08:16:15.856228Z","iopub.status.idle":"2024-11-14T08:16:15.870069Z","shell.execute_reply.started":"2024-11-14T08:16:15.856187Z","shell.execute_reply":"2024-11-14T08:16:15.869064Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Insight:**\n\nThis dataset is imbalanced. Half of the samples are in class 0, while very few in class 3.\n","metadata":{}},{"cell_type":"code","source":"print('Columns missing in test:')\nprint([f for f in train.columns if f not in test.columns])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:15.871797Z","iopub.execute_input":"2024-11-14T08:16:15.872665Z","iopub.status.idle":"2024-11-14T08:16:15.878763Z","shell.execute_reply.started":"2024-11-14T08:16:15.872598Z","shell.execute_reply":"2024-11-14T08:16:15.877743Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Demographics\n","metadata":{}},{"cell_type":"markdown","source":"Now we look at some basic demographics.","metadata":{}},{"cell_type":"code","source":"vc = train.get_column('Basic_Demos-Enroll_Season').value_counts()\nplt.pie(vc.get_column('count'), labels=vc.get_column('Basic_Demos-Enroll_Season'))\nplt.title('Season of enrollment')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:15.880103Z","iopub.execute_input":"2024-11-14T08:16:15.880406Z","iopub.status.idle":"2024-11-14T08:16:16.011071Z","shell.execute_reply.started":"2024-11-14T08:16:15.880374Z","shell.execute_reply":"2024-11-14T08:16:16.009638Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vc = train.get_column('Basic_Demos-Sex').value_counts()\nplt.pie(vc.get_column('count'), labels=['boys', 'girls'])\nplt.title('Sex of participant')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:16.016295Z","iopub.execute_input":"2024-11-14T08:16:16.016827Z","iopub.status.idle":"2024-11-14T08:16:16.164306Z","shell.execute_reply.started":"2024-11-14T08:16:16.016754Z","shell.execute_reply":"2024-11-14T08:16:16.162531Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_, axs = plt.subplots(2, 1, sharex=True)\nfor sex in range(2):\n    ax = axs.ravel()[sex]\n    vc = train.filter(pl.col('Basic_Demos-Sex') == sex).get_column('Basic_Demos-Age').value_counts()\n    ax.bar(vc.get_column('Basic_Demos-Age'),\n           vc.get_column('count'),\n           color=['lightblue', 'coral'][sex],\n           label=['boys', 'girls'][sex])\n    ax.xaxis.set_major_locator(MaxNLocator(integer=True))\n    ax.set_ylabel('count')\n    ax.legend()\nplt.suptitle('Age distribution')\naxs.ravel()[1].set_xlabel('years')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:16.166931Z","iopub.execute_input":"2024-11-14T08:16:16.167530Z","iopub.status.idle":"2024-11-14T08:16:16.614789Z","shell.execute_reply.started":"2024-11-14T08:16:16.167464Z","shell.execute_reply":"2024-11-14T08:16:16.613853Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_, axs = plt.subplots(2, 1, sharex=True, sharey=True)\nfor sex in range(2):\n    ax = axs.ravel()[sex]\n    vc = train.filter(pl.col('Basic_Demos-Sex') == sex).get_column('sii').value_counts()\n    ax.bar(vc.get_column('sii'),\n           vc.get_column('count') / vc.get_column('count').sum(),\n           color=['lightblue', 'coral'][sex],\n           label=['boys', 'girls'][sex])\n    ax.set_xticks(np.arange(4), target_labels)\n    ax.yaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\n    ax.set_ylabel('count')\n    ax.legend()\nplt.suptitle('Target distribution')\naxs.ravel()[1].set_xlabel('Severity Impairment Index (sii)')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:16.616303Z","iopub.execute_input":"2024-11-14T08:16:16.616996Z","iopub.status.idle":"2024-11-14T08:16:16.903582Z","shell.execute_reply.started":"2024-11-14T08:16:16.616946Z","shell.execute_reply":"2024-11-14T08:16:16.902506Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Now we look at correlations","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(14, 12))\ncorr_matrix = supervised_usable.select([\n    'PCIAT-PCIAT_Total', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-BMI', \n    'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n    'Physical-Diastolic_BP', 'Physical-Systolic_BP', 'Physical-HeartRate',\n    'PreInt_EduHx-computerinternet_hoursday', 'SDS-SDS_Total_T', 'PAQ_A-PAQ_A_Total',\n    'PAQ_C-PAQ_C_Total', 'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins','Fitness_Endurance-Time_Sec',\n    'FGC-FGC_CU', 'FGC-FGC_GSND','FGC-FGC_GSD','FGC-FGC_PU','FGC-FGC_SRL','FGC-FGC_SRR','FGC-FGC_TL','BIA-BIA_Activity_Level_num', \n    'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n    'BIA-BIA_FFMI','BIA-BIA_FMI', 'BIA-BIA_Fat','BIA-BIA_Frame_num','BIA-BIA_ICW','BIA-BIA_LDM','BIA-BIA_LST',\n    'BIA-BIA_SMM','BIA-BIA_TBW'\n    # Add other relevant columns\n]).to_pandas().corr()\n\nsii_corr = corr_matrix['PCIAT-PCIAT_Total'].drop('PCIAT-PCIAT_Total')\nfiltered_corr = sii_corr[(sii_corr > 0.1) | (sii_corr < -0.1)]\n\nprint(filtered_corr)\n\nplt.figure(figsize=(8, 6))\nfiltered_corr.sort_values().plot(kind='barh', color='coral')\nplt.title('Features with Correlation > 0.1 or < -0.1 with PCIAT-PCIAT_Total')\nplt.xlabel('Correlation coefficient')\nplt.ylabel('Features')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:16.904732Z","iopub.execute_input":"2024-11-14T08:16:16.905054Z","iopub.status.idle":"2024-11-14T08:16:17.290576Z","shell.execute_reply.started":"2024-11-14T08:16:16.905022Z","shell.execute_reply":"2024-11-14T08:16:17.289524Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Actig\nraphy (time series)","metadata":{}},{"cell_type":"code","source":"actigraphy = pl.read_parquet('/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id=0417c91e/part-0.parquet')\nactigraphy","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:17.291834Z","iopub.execute_input":"2024-11-14T08:16:17.292147Z","iopub.status.idle":"2024-11-14T08:16:17.315098Z","shell.execute_reply.started":"2024-11-14T08:16:17.292115Z","shell.execute_reply":"2024-11-14T08:16:17.314048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_actigraphy(id, only_one_week=False, small=False):\n    actigraphy = pl.read_parquet(f'/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id={id}/part-0.parquet')\n    day = actigraphy.get_column('relative_date_PCIAT') + actigraphy.get_column('time_of_day') / 86400e9\n    sample = train.filter(pl.col('id') == id)\n    age = sample.get_column('Basic_Demos-Age').item()\n    sex = ['boy', 'girl'][sample.get_column('Basic_Demos-Sex').item()]\n    actigraphy = (\n        actigraphy\n        .with_columns(\n            (day.diff() * 86400).alias('diff_seconds'),\n            (np.sqrt(np.square(pl.col('X')) + np.square(pl.col('Y')) + np.square(pl.col('Z'))).alias('norm'))\n        )\n    )\n\n    if only_one_week:\n        start = np.ceil(day.min())\n        mask = (start <= day.to_numpy()) & (day.to_numpy() <= start + 7*3)\n        mask &= ~ actigraphy.get_column('non-wear_flag').cast(bool).to_numpy()\n    else:\n        mask = np.full(len(day), True)\n        \n    if small:\n        timelines = [\n            ('enmo', 'forestgreen'),\n            ('light', 'orange'),\n        ]\n    else:\n        timelines = [\n            ('X', 'm'),\n            ('Y', 'm'),\n            ('Z', 'm'),\n#             ('norm', 'c'),\n            ('enmo', 'forestgreen'),\n            ('anglez', 'lightblue'),\n            ('light', 'orange'),\n            ('non-wear_flag', 'chocolate')\n    #         ('diff_seconds', 'k'),\n        ]\n        \n    _, axs = plt.subplots(len(timelines), 1, sharex=True, figsize=(12, len(timelines) * 1.1 + 0.5))\n    for ax, (feature, color) in zip(axs, timelines):\n        ax.set_facecolor('#eeeeee')\n        ax.scatter(day.to_numpy()[mask],\n                   actigraphy.get_column(feature).to_numpy()[mask],\n                   color=color, label=feature, s=1)\n        ax.legend(loc='upper left', facecolor='#eeeeee')\n        if feature == 'diff_seconds':\n            ax.set_ylim(-0.5, 20.5)\n    axs[-1].set_xlabel('day')\n    axs[-1].xaxis.set_major_locator(MaxNLocator(integer=True))\n    plt.tight_layout()\n    axs[0].set_title(f'id={id}, {sex}, age={age}')\n    plt.show()\n\nanalyze_actigraphy('0417c91e', only_one_week=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:17.316353Z","iopub.execute_input":"2024-11-14T08:16:17.316664Z","iopub.status.idle":"2024-11-14T08:16:20.205796Z","shell.execute_reply.started":"2024-11-14T08:16:17.316631Z","shell.execute_reply":"2024-11-14T08:16:20.204824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42\nn_splits = 5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:20.207364Z","iopub.execute_input":"2024-11-14T08:16:20.208332Z","iopub.status.idle":"2024-11-14T08:16:20.212029Z","shell.execute_reply.started":"2024-11-14T08:16:20.208278Z","shell.execute_reply":"2024-11-14T08:16:20.211141Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering\n\n- **Feature Selection**: The dataset contains features related to physical characteristics (e.g., BMI, Height, Weight), behavioral aspects (e.g., internet usage), and fitness data (e.g., endurance time). \n- **Categorical Feature Encoding**: Categorical features are mapped to numerical values using custom mappings for each unique category within the dataset. This ensures compatibility with machine learning algorithms that require numerical input.\n- **Time Series Aggregation**: Time series statistics (e.g., mean, standard deviation) from the actigraphy data are computed and merged into the main dataset to create additional features for model training.\n","metadata":{}},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    stats, indexes = zip(*results)\n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n\n\ndef build_autoencoder(input_dim, encoding_dim):\n    input_layer = Input(shape=(input_dim,))\n    encoded = Dense(encoding_dim, activation='relu')(input_layer)\n    decoded = Dense(input_dim, activation='sigmoid')(encoded)\n    autoencoder = Model(inputs=input_layer, outputs=decoded)\n    encoder     = Model(inputs=input_layer, outputs=encoded)\n    autoencoder.compile(optimizer=Adam(), loss='mse')   \n    #legacy_h5_format.save_model_to_hdf5(autoencoder, 'autoencoder_model', overwrite, include_optimizer)\n    return autoencoder, encoder\n\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    input_dim = df_scaled.shape[1]\n    autoencoder, encoder = build_autoencoder(input_dim, encoding_dim)\n    autoencoder.fit(df_scaled, df_scaled, epochs=epochs, batch_size=batch_size, shuffle=True, verbose=1)\n    encoded_data = encoder.predict(df_scaled)\n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i+1}' for i in range(encoded_data.shape[1])])\n    return df_encoded","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:20.213204Z","iopub.execute_input":"2024-11-14T08:16:20.213500Z","iopub.status.idle":"2024-11-14T08:16:20.226576Z","shell.execute_reply.started":"2024-11-14T08:16:20.213467Z","shell.execute_reply":"2024-11-14T08:16:20.225743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"featuresCols_1 = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\n# ---------------------------------------------------------------------------------\n\nfeaturesCols_2 = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\n# ---------------------------------------------------------------------------------\n\nfeaturesCols_3 = [\n    'Basic_Demos-Enroll_Season',\n    'Basic_Demos-Age', \n    'Basic_Demos-Sex',\n    \n    'CGAS-Season',\n    'CGAS-CGAS_Score',\n    \n    'Physical-Season',\n    'Physical-BMI',\n    'Physical-Height',\n    'Physical-Weight',\n    'Physical-Waist_Circumference',\n    'Physical-Diastolic_BP',\n    'Physical-HeartRate',\n    'Physical-Systolic_BP',\n    \n    'Fitness_Endurance-Season', \n    'Fitness_Endurance-Max_Stage',\n    'Fitness_Endurance-Time_Mins',\n    'Fitness_Endurance-Time_Sec',\n    \n    'FGC-Season',\n    \n    'FGC-FGC_CU', \n    'FGC-FGC_CU_Zone',\n    \n    'FGC-FGC_GSND',\n    'FGC-FGC_GSND_Zone',\n    \n    'FGC-FGC_GSD', \n    'FGC-FGC_GSD_Zone', \n    \n    'FGC-FGC_PU',\n    'FGC-FGC_PU_Zone',\n    \n    'FGC-FGC_SRL',\n    'FGC-FGC_SRL_Zone',\n    'FGC-FGC_SRR',\n    'FGC-FGC_SRR_Zone',\n    \n    'FGC-FGC_TL',\n    'FGC-FGC_TL_Zone',\n    \n    'BIA-Season',\n    'BIA-BIA_Activity_Level_num',\n    'BIA-BIA_BMC',\n    'BIA-BIA_BMI',\n    'BIA-BIA_BMR',\n    'BIA-BIA_DEE',\n    'BIA-BIA_ECW', \n    'BIA-BIA_FFM',\n    'BIA-BIA_FFMI',\n    'BIA-BIA_FMI',\n    'BIA-BIA_Fat',\n    'BIA-BIA_Frame_num',    \n    'BIA-BIA_ICW',\n    'BIA-BIA_LDM',\n    'BIA-BIA_LST',\n    'BIA-BIA_SMM',\n    'BIA-BIA_TBW',\n    \n    'PAQ_A-Season',\n    'PAQ_A-PAQ_A_Total',\n    \n    'PAQ_C-Season',\n    'PAQ_C-PAQ_C_Total',\n    \n    'SDS-Season',\n    'SDS-SDS_Total_Raw',\n    'SDS-SDS_Total_T',\n    \n    'PreInt_EduHx-Season',\n    'PreInt_EduHx-computerinternet_hoursday',\n    \n    'sii'\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:20.227849Z","iopub.execute_input":"2024-11-14T08:16:20.228131Z","iopub.status.idle":"2024-11-14T08:16:20.242715Z","shell.execute_reply.started":"2024-11-14T08:16:20.228100Z","shell.execute_reply":"2024-11-14T08:16:20.241730Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def merge_bmi_columns(df):\n    \"\"\"\n    Merge 'Physical-BMI' and 'BIA-BIA_BMI' columns into a single 'merged_BMI' column.\n    Ensures only one 'merged_BMI' column exists in the dataframe.\n    \n    Parameters:\n    df (pd.DataFrame): Input dataframe containing BMI columns.\n\n    Returns:\n    pd.DataFrame: DataFrame with a single 'merged_BMI' column.\n    \"\"\"\n    # Ensure only one 'merged_BMI' column by dropping duplicates\n    if 'merged_BMI' in df.columns:\n        df = df.drop(columns=['merged_BMI'])\n\n    # Check if the necessary columns exist\n    bmi_columns = ['Physical-BMI', 'BIA-BIA_BMI']\n    existing_columns = [col for col in bmi_columns if col in df.columns]\n\n    if not existing_columns:\n        print(\"No BMI columns to merge.\")\n        return df\n\n    # Merge the BMI columns\n    if 'Physical-BMI' in existing_columns and 'BIA-BIA_BMI' in existing_columns:\n        df['merged_BMI'] = df[['Physical-BMI', 'BIA-BIA_BMI']].mean(axis=1)\n    elif 'Physical-BMI' in existing_columns:\n        df['merged_BMI'] = df['Physical-BMI']\n    elif 'BIA-BIA_BMI' in existing_columns:\n        df['merged_BMI'] = df['BIA-BIA_BMI']\n\n    # Drop the original BMI columns\n    df = df.drop(columns=bmi_columns, errors='ignore')\n\n    return df\n    \n# Generate featuresCols_1, featuresCols_2, and featuresCols_3 with the new 'merged_BMI' column\nfeaturesCols_1 = [\n    col if col not in ['Physical-BMI', 'BIA-BIA_BMI'] else 'merged_BMI'\n    for col in featuresCols_1\n]\nfeaturesCols_2 = [\n    col if col not in ['Physical-BMI', 'BIA-BIA_BMI'] else 'merged_BMI'\n    for col in featuresCols_2\n]\nfeaturesCols_3 = [\n    col if col not in ['Physical-BMI', 'BIA-BIA_BMI'] else 'merged_BMI'\n    for col in featuresCols_3\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:20.245982Z","iopub.execute_input":"2024-11-14T08:16:20.246404Z","iopub.status.idle":"2024-11-14T08:16:20.257167Z","shell.execute_reply.started":"2024-11-14T08:16:20.246368Z","shell.execute_reply":"2024-11-14T08:16:20.256216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\ndisplay(train_ts.head(5))\n\n# Autoencoder\ntrain_ts_encoded = perform_autoencoder(df_train, encoding_dim=32, epochs=100, batch_size=32)\ntest_ts_encoded = perform_autoencoder(df_test, encoding_dim=32, epochs=100, batch_size=32)\n\ndisplay(train_ts_encoded.head(5))\n\ntime_series_cols = train_ts_encoded.columns.tolist()\ntrain_ts_encoded['id'] = train_ts['id']\n\ntrain = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\ntest = pd.merge(test, train_ts_encoded, how=\"left\", on='id')\ntest_ = pd.merge(test, test_ts, how='left', on='id')\n\n# Apply the merge_bmi_columns function\ntrain = merge_bmi_columns(train)\ntest = merge_bmi_columns(test)\n\n# Drop duplicate columns\ntrain = train.loc[:, ~train.columns.duplicated()]\ntest = test.loc[:, ~test.columns.duplicated()]\n\n# Drop 'id' column\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n\nfeaturesCols = featuresCols_1 + time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\n# Verify columns\nfor column in train.columns:\n    print(column)","metadata":{"trusted":true,"scrolled":true,"execution":{"iopub.status.busy":"2024-11-14T08:16:20.258549Z","iopub.execute_input":"2024-11-14T08:16:20.259307Z","iopub.status.idle":"2024-11-14T08:17:55.517974Z","shell.execute_reply.started":"2024-11-14T08:16:20.259273Z","shell.execute_reply":"2024-11-14T08:17:55.516863Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ensure no duplicate 'merged_BMI' column exists\nif train.columns.duplicated().any():\n    train = train.loc[:, ~train.columns.duplicated()]\nif test.columns.duplicated().any():\n    test = test.loc[:, ~test.columns.duplicated()]\n\n# Verify the columns after removing duplicates\nfor column in train.columns:\n    print(column)","metadata":{"trusted":true,"scrolled":true,"execution":{"iopub.status.busy":"2024-11-14T08:17:55.519433Z","iopub.execute_input":"2024-11-14T08:17:55.519890Z","iopub.status.idle":"2024-11-14T08:17:55.530884Z","shell.execute_reply.started":"2024-11-14T08:17:55.519844Z","shell.execute_reply":"2024-11-14T08:17:55.529868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Verify the columns after removing duplicates\nfor column in test.columns:\n    print(column)","metadata":{"trusted":true,"scrolled":true,"execution":{"iopub.status.busy":"2024-11-14T08:17:55.538530Z","iopub.execute_input":"2024-11-14T08:17:55.538901Z","iopub.status.idle":"2024-11-14T08:17:55.545505Z","shell.execute_reply.started":"2024-11-14T08:17:55.538864Z","shell.execute_reply":"2024-11-14T08:17:55.544371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Identify columns common to both train and test\ncommon_columns = [col for col in train.columns if col in test.columns]\n\n# Reorder the common columns in test to match train\ntest_reordered = test[common_columns]\n\n# Append the columns in test that are not in train\nadditional_columns = [col for col in test.columns if col not in train.columns]\n\n# Concatenate reordered and additional columns\ntest = pd.concat([test_reordered, test[additional_columns]], axis=1)\n\n# Verify the columns after removing duplicates\nfor column in train.columns:\n    print(column)\n\nfor column in test.columns:\n    print(column)","metadata":{"trusted":true,"scrolled":true,"execution":{"iopub.status.busy":"2024-11-14T08:17:55.547089Z","iopub.execute_input":"2024-11-14T08:17:55.547542Z","iopub.status.idle":"2024-11-14T08:17:55.560466Z","shell.execute_reply.started":"2024-11-14T08:17:55.547498Z","shell.execute_reply":"2024-11-14T08:17:55.559340Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n        \ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:17:55.564224Z","iopub.execute_input":"2024-11-14T08:17:55.564585Z","iopub.status.idle":"2024-11-14T08:17:55.635872Z","shell.execute_reply.started":"2024-11-14T08:17:55.564544Z","shell.execute_reply":"2024-11-14T08:17:55.634849Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Training and Evaluation\n\n- **Model Types**: Various models are used, including:\n  - **LightGBM**: A gradient-boosting framework known for its speed and efficiency with large datasets.\n  - **XGBoost**: Another powerful gradient-boosting model used for structured data.\n  - **CatBoost**: Optimized for categorical features without the need for extensive preprocessing.\n  - **Voting Regressor**: An ensemble model that combines the predictions of LightGBM, XGBoost, and CatBoost for better accuracy.\n- **Cross-Validation**: Stratified K-Folds cross-validation is employed to split the data into training and validation sets, ensuring balanced class distribution in each fold.\n- **Quadratic Weighted Kappa (QWK)**: The performance of the models is evaluated using QWK, which measures the agreement between predicted and actual values, taking into account the ordinal nature of the target variable.\n- **Threshold Optimization**: The `minimize` function from `scipy.optimize` is used to fine-tune decision thresholds that map continuous predictions to discrete categories (None, Mild, Moderate, Severe).\n","metadata":{}},{"cell_type":"code","source":"def TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n    trained_models = []  # Store models for each fold\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n        trained_models.append(model)  # Save the trained model\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission, trained_models  # Return both submission and models","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:17:55.637408Z","iopub.execute_input":"2024-11-14T08:17:55.637844Z","iopub.status.idle":"2024-11-14T08:17:55.655993Z","shell.execute_reply.started":"2024-11-14T08:17:55.637773Z","shell.execute_reply":"2024-11-14T08:17:55.654778Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n# Hyperparameter Tuning\n\n- **LightGBM Parameters**: Hyperparameters such as `learning_rate`, `max_depth`, `num_leaves`, and `feature_fraction` are tuned to improve the performance of the LightGBM model. These parameters control the complexity of the model and its ability to generalize to new data.\n- **XGBoost and CatBoost Parameters**: Similar tuning is applied for XGBoost and CatBoost, adjusting parameters such as `n_estimators`, `max_depth`, `learning_rate`, `subsample`, and `regularization` terms (`reg_alpha`, `reg_lambda`). These help in controlling overfitting and ensuring the model's robustness.","metadata":{}},{"cell_type":"code","source":"\n# Model parameters for LightGBM\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01,  # Increased from 2.68e-06\n    'device': 'gpu'\n\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED,\n    'tree_method': 'gpu_hist',\n\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'cat_features': cat_c,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n    'task_type': 'GPU'\n\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:17:55.657908Z","iopub.execute_input":"2024-11-14T08:17:55.658455Z","iopub.status.idle":"2024-11-14T08:17:55.670785Z","shell.execute_reply.started":"2024-11-14T08:17:55.658404Z","shell.execute_reply":"2024-11-14T08:17:55.669871Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Plot the feature importance","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\ndef plot_feature_importance(importance_df, model_name, top_n=50):\n    \"\"\"\n    Plot the feature importance for a given model.\n\n    Parameters:\n    - importance_df: DataFrame containing feature names and their importances\n    - model_name: Name of the model (for the plot title)\n    - top_n: Number of top features to display\n    \"\"\"\n    plt.figure(figsize=(12, 8))\n    sns.barplot(\n        x='Importance',\n        y='Feature',\n        data=importance_df.head(top_n),\n        palette='viridis'\n    )\n    plt.title(f\"Top {top_n} Feature Importance - {model_name}\")\n    plt.xlabel(\"Importance\")\n    plt.ylabel(\"Feature\")\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:17:55.672098Z","iopub.execute_input":"2024-11-14T08:17:55.672420Z","iopub.status.idle":"2024-11-14T08:17:55.686093Z","shell.execute_reply.started":"2024-11-14T08:17:55.672386Z","shell.execute_reply":"2024-11-14T08:17:55.685105Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ensemble Learning and Submission Preparation\n\n- **Ensemble Learning**: The model uses a **Voting Regressor**, which combines the predictions from LightGBM, XGBoost, and CatBoost. This approach is beneficial as it leverages the strengths of multiple models, reducing overfitting and improving overall model performance.\n- **Out-of-Fold (OOF) Predictions**: During cross-validation, out-of-fold predictions are generated for the training set, which helps in model evaluation without data leakage.\n- **Kappa Optimizer**: The Kappa Optimizer ensures that the predicted values are as close to the actual values as possible by adjusting the thresholds used to convert raw model outputs into class labels.\n- **Test Set Predictions**: After the model is trained and thresholds are optimized, the test dataset is processed, and predictions are generated using the ensemble model. These predictions are converted into the appropriate format for submission.\n- **Submission File Creation**: The predictions are saved in a CSV file following the required format for submission (e.g., for a Kaggle competition), which includes columns like `id` and `sii` (Severity Impairment Index).","metadata":{}},{"cell_type":"markdown","source":"# Final Results and Performance Metrics\n\n- **Train and Validation Scores**: After training across multiple folds, the mean Quadratic Weighted Kappa (QWK) score is calculated for both the training and validation datasets, providing an indicator of model performance. \n- **Optimized QWK Score**: The final optimized QWK score after threshold tuning is displayed, showcasing the model's ability to predict the severity levels effectively.\n- **Test Predictions**: The test set predictions are evaluated, and a breakdown of the predicted severity levels (None, Mild, Moderate, Severe) is shown, along with their respective counts.","metadata":{}},{"cell_type":"markdown","source":"### featuresCols_1","metadata":{}},{"cell_type":"code","source":"# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\n# Define feature names\nfeature_names = train.drop(['sii'], axis=1).columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:17:55.687339Z","iopub.execute_input":"2024-11-14T08:17:55.687659Z","iopub.status.idle":"2024-11-14T08:17:55.699922Z","shell.execute_reply.started":"2024-11-14T08:17:55.687622Z","shell.execute_reply":"2024-11-14T08:17:55.699004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Identify duplicate columns in train and test datasets\nduplicate_columns = [col for col in train.columns if train.columns.tolist().count(col) > 1]\n\nif duplicate_columns:\n    print(f\"Duplicate columns detected: {duplicate_columns}\")\nelse:\n    print(\"No duplicate columns detected.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:17:55.701167Z","iopub.execute_input":"2024-11-14T08:17:55.701531Z","iopub.status.idle":"2024-11-14T08:17:55.710499Z","shell.execute_reply.started":"2024-11-14T08:17:55.701483Z","shell.execute_reply":"2024-11-14T08:17:55.709407Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train LightGBM Model\nprint(\"\\nTraining LightGBM Model...\")\nlgb_submission, lgb_models = TrainML(Light, test)\n\n# Extract Feature Importance from Trained Models\nlgb_importance = pd.DataFrame({\n    'Feature': feature_names,\n    'Importance': np.mean([model.feature_importances_ for model in lgb_models], axis=0)\n}).sort_values(by='Importance', ascending=False)\n\n# Plot LightGBM Feature Importance\nplot_feature_importance(lgb_importance, \"LightGBM\")\n\n# Print Top 5 Features\nprint(\"\\nTop 5 Features for LightGBM:\")\nprint(lgb_importance.head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:17:55.711737Z","iopub.execute_input":"2024-11-14T08:17:55.712089Z","iopub.status.idle":"2024-11-14T08:18:07.061789Z","shell.execute_reply.started":"2024-11-14T08:17:55.712055Z","shell.execute_reply":"2024-11-14T08:18:07.060883Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train XGBoost Model\nprint(\"\\nTraining XGBoost Model...\")\nxgb_submission, xgb_models = TrainML(XGB_Model, test)\n\n# Extract Feature Importance from Trained Models\nxgb_importance = pd.DataFrame({\n    'Feature': feature_names,\n    'Importance': np.mean([model.feature_importances_ for model in xgb_models], axis=0)\n}).sort_values(by='Importance', ascending=False)\n\n# Plot XGBoost Feature Importance\nplot_feature_importance(xgb_importance, \"XGBoost\")\n\n# Print Top 5 Features\nprint(\"\\nTop 5 Features for XGBoost:\")\nprint(xgb_importance.head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:18:07.063139Z","iopub.execute_input":"2024-11-14T08:18:07.063489Z","iopub.status.idle":"2024-11-14T08:18:12.811056Z","shell.execute_reply.started":"2024-11-14T08:18:07.063452Z","shell.execute_reply":"2024-11-14T08:18:12.810056Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train CatBoost Model\nprint(\"\\nTraining CatBoost Model...\")\ncat_submission, cat_models = TrainML(CatBoost_Model, test)\n\n# Extract Feature Importance from Trained Models\ncat_importance = pd.DataFrame({\n    'Feature': feature_names,\n    'Importance': np.mean([model.get_feature_importance() for model in cat_models], axis=0)\n}).sort_values(by='Importance', ascending=False)\n\n# Plot CatBoost Feature Importance\nplot_feature_importance(cat_importance, \"CatBoost\")\n\n# Print Top 5 Features\nprint(\"\\nTop 5 Features for CatBoost:\")\nprint(cat_importance.head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:18:12.812384Z","iopub.execute_input":"2024-11-14T08:18:12.812705Z","iopub.status.idle":"2024-11-14T08:18:31.902861Z","shell.execute_reply.started":"2024-11-14T08:18:12.812672Z","shell.execute_reply":"2024-11-14T08:18:31.901884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save individual submissions\nlgb_submission.to_csv(\"LightGBM_submission.csv\", index=False)\nxgb_submission.to_csv(\"XGBoost_submission.csv\", index=False)\ncat_submission.to_csv(\"submission.csv\", index=False)\n\nprint(\"Individual model training and submissions of feature_1 completed!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:18:31.904267Z","iopub.execute_input":"2024-11-14T08:18:31.904618Z","iopub.status.idle":"2024-11-14T08:18:31.914456Z","shell.execute_reply.started":"2024-11-14T08:18:31.904583Z","shell.execute_reply":"2024-11-14T08:18:31.913408Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Voting Regresor","metadata":{}},{"cell_type":"code","source":"# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model)\n])\n\n\n\n# Train the ensemble model\nSubmission1, trained_models = TrainML(voting_model, test)\n\nSubmission1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:18:31.915617Z","iopub.execute_input":"2024-11-14T08:18:31.915951Z","iopub.status.idle":"2024-11-14T08:19:06.529543Z","shell.execute_reply.started":"2024-11-14T08:18:31.915903Z","shell.execute_reply":"2024-11-14T08:19:06.528642Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### FeaturesCols_2","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\n# Apply the merge_bmi_columns function\ntrain = merge_bmi_columns(train)\ntest = merge_bmi_columns(test)\n\n# Drop duplicate columns\ntrain = train.loc[:, ~train.columns.duplicated()]\ntest = test.loc[:, ~test.columns.duplicated()]\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)  \n\nfeaturesCols = featuresCols_2 + time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\n# Ensure no duplicate 'merged_BMI' column exists\nif train.columns.duplicated().any():\n    train = train.loc[:, ~train.columns.duplicated()]\nif test.columns.duplicated().any():\n    test = test.loc[:, ~test.columns.duplicated()]\n\n# Identify columns common to both train and test\ncommon_columns = [col for col in train.columns if col in test.columns]\n\n# Reorder the common columns in test to match train\ntest_reordered = test[common_columns]\n\n# Append the columns in test that are not in train\nadditional_columns = [col for col in test.columns if col not in train.columns]\n\n# Concatenate reordered and additional columns\ntest = pd.concat([test_reordered, test[additional_columns]], axis=1)\n\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n        \ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n    trained_models = []  # Store models for each fold\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n        trained_models.append(model)  # Save the trained model\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission, trained_models  # Return both submission and models\n    \nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01,  # Increased from 2.68e-06\n    'device': 'gpu'\n}\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED,\n    'tree_method': 'gpu_hist'\n}\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'cat_features': cat_c,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n    'task_type': 'GPU'\n}\n\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\n# Define feature names\nfeature_names = train.drop(['sii'], axis=1).columns\n\n# voting_model = VotingRegressor(estimators=[\n#     ('lightgbm', Light),\n#     ('xgboost', XGB_Model),\n#     ('catboost', CatBoost_Model)\n# ])\n\n# Submission2 = TrainML(voting_model, test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:19:06.531222Z","iopub.execute_input":"2024-11-14T08:19:06.531892Z","iopub.status.idle":"2024-11-14T08:20:28.912306Z","shell.execute_reply.started":"2024-11-14T08:19:06.531847Z","shell.execute_reply":"2024-11-14T08:20:28.911180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train LightGBM Model\nprint(\"\\nTraining LightGBM Model...\")\nlgb_submission, lgb_models = TrainML(Light, test)\n\n# Extract Feature Importance from Trained Models\nlgb_importance = pd.DataFrame({\n    'Feature': feature_names,\n    'Importance': np.mean([model.feature_importances_ for model in lgb_models], axis=0)\n}).sort_values(by='Importance', ascending=False)\n\n# Plot LightGBM Feature Importance\nplot_feature_importance(lgb_importance, \"LightGBM\")\n\n# Print Top 5 Features\nprint(\"\\nTop 5 Features for LightGBM:\")\nprint(lgb_importance.head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:20:28.913925Z","iopub.execute_input":"2024-11-14T08:20:28.914347Z","iopub.status.idle":"2024-11-14T08:20:42.471049Z","shell.execute_reply.started":"2024-11-14T08:20:28.914295Z","shell.execute_reply":"2024-11-14T08:20:42.470027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train XGBoost Model\nprint(\"\\nTraining XGBoost Model...\")\nxgb_submission, xgb_models = TrainML(XGB_Model, test)\n\n# Extract Feature Importance from Trained Models\nxgb_importance = pd.DataFrame({\n    'Feature': feature_names,\n    'Importance': np.mean([model.feature_importances_ for model in xgb_models], axis=0)\n}).sort_values(by='Importance', ascending=False)\n\n# Plot XGBoost Feature Importance\nplot_feature_importance(xgb_importance, \"XGBoost\")\n\n# Print Top 5 Features\nprint(\"\\nTop 5 Features for XGBoost:\")\nprint(xgb_importance.head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:20:42.472384Z","iopub.execute_input":"2024-11-14T08:20:42.472706Z","iopub.status.idle":"2024-11-14T08:20:49.322478Z","shell.execute_reply.started":"2024-11-14T08:20:42.472671Z","shell.execute_reply":"2024-11-14T08:20:49.321310Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train CatBoost Model\nprint(\"\\nTraining CatBoost Model...\")\ncat_submission, cat_models = TrainML(CatBoost_Model, test)\n\n# Extract Feature Importance from Trained Models\ncat_importance = pd.DataFrame({\n    'Feature': feature_names,\n    'Importance': np.mean([model.get_feature_importance() for model in cat_models], axis=0)\n}).sort_values(by='Importance', ascending=False)\n\n# Plot CatBoost Feature Importance\nplot_feature_importance(cat_importance, \"CatBoost\")\n\n# Print Top 5 Features\nprint(\"\\nTop 5 Features for CatBoost:\")\nprint(cat_importance.head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:20:49.323843Z","iopub.execute_input":"2024-11-14T08:20:49.324212Z","iopub.status.idle":"2024-11-14T08:21:09.436002Z","shell.execute_reply.started":"2024-11-14T08:20:49.324176Z","shell.execute_reply":"2024-11-14T08:21:09.435025Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Voting Regresor","metadata":{}},{"cell_type":"code","source":"# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model)\n])\n\n# Train the ensemble model\nSubmission2, trained_models = TrainML(voting_model, test)\n\nSubmission2","metadata":{"trusted":true,"scrolled":true,"execution":{"iopub.status.busy":"2024-11-14T08:21:09.437319Z","iopub.execute_input":"2024-11-14T08:21:09.437645Z","iopub.status.idle":"2024-11-14T08:21:47.249856Z","shell.execute_reply.started":"2024-11-14T08:21:09.437611Z","shell.execute_reply":"2024-11-14T08:21:47.248788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save individual submissions\nlgb_submission.to_csv(\"LightGBM_submission_2.csv\", index=False)\nxgb_submission.to_csv(\"XGBoost_submission_2.csv\", index=False)\ncat_submission.to_csv(\"CatBoost_submission_2.csv\", index=False)\n\nprint(\"Individual model training and submissions of feature_2 completed!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:21:47.251174Z","iopub.execute_input":"2024-11-14T08:21:47.251483Z","iopub.status.idle":"2024-11-14T08:21:47.261163Z","shell.execute_reply.started":"2024-11-14T08:21:47.251452Z","shell.execute_reply":"2024-11-14T08:21:47.260093Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### FeaturesCols_3","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\n#train_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\n#test_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n\n# Apply the merge_bmi_columns function\ntrain = merge_bmi_columns(train)\ntest = merge_bmi_columns(test)\n\n# Drop duplicate columns\ntrain = train.loc[:, ~train.columns.duplicated()]\ntest = test.loc[:, ~test.columns.duplicated()]\n\nfeaturesCols = featuresCols_3 + time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\n# Ensure no duplicate 'merged_BMI' column exists\nif train.columns.duplicated().any():\n    train = train.loc[:, ~train.columns.duplicated()]\nif test.columns.duplicated().any():\n    test = test.loc[:, ~test.columns.duplicated()]\n\n# Identify columns common to both train and test\ncommon_columns = [col for col in train.columns if col in test.columns]\n\n# Reorder the common columns in test to match train\ntest_reordered = test[common_columns]\n\n# Append the columns in test that are not in train\nadditional_columns = [col for col in test.columns if col not in train.columns]\n\n# Concatenate reordered and additional columns\ntest = pd.concat([test_reordered, test[additional_columns]], axis=1)\n\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n\ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n    trained_models = []  # Store models for each fold\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n        trained_models.append(model)  # Save the trained model\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tp_rounded = threshold_Rounder(tpm, KappaOPtimizer.x)\n\n    return tp_rounded\n\nimputer = SimpleImputer(strategy='median')\n\nensemble = VotingRegressor(estimators=[\n    ('lgb', Pipeline(steps=[('imputer', imputer), ('regressor', LGBMRegressor(random_state=SEED))])),\n    ('xgb', Pipeline(steps=[('imputer', imputer), ('regressor', XGBRegressor(random_state=SEED))])),\n    ('cat', Pipeline(steps=[('imputer', imputer), ('regressor', CatBoostRegressor(random_state=SEED, silent=True))])),\n    ('rf', Pipeline(steps=[('imputer', imputer), ('regressor', RandomForestRegressor(random_state=SEED))])),\n    ('gb', Pipeline(steps=[('imputer', imputer), ('regressor', GradientBoostingRegressor(random_state=SEED))]))\n])\n\nSubmission3  = TrainML(ensemble, test)\nsample['sii'] = Submission3\nSubmission3 = sample","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:21:47.262423Z","iopub.execute_input":"2024-11-14T08:21:47.262753Z","iopub.status.idle":"2024-11-14T08:23:52.845007Z","shell.execute_reply.started":"2024-11-14T08:21:47.262719Z","shell.execute_reply":"2024-11-14T08:23:52.843993Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n    trained_models = []  # Store models for each fold\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n        trained_models.append(model)  # Save the trained model\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission, trained_models  # Return both submission and models","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:23:52.846272Z","iopub.execute_input":"2024-11-14T08:23:52.846597Z","iopub.status.idle":"2024-11-14T08:23:52.860894Z","shell.execute_reply.started":"2024-11-14T08:23:52.846564Z","shell.execute_reply":"2024-11-14T08:23:52.859985Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define Models and Parameters\nmodels = {\n    \"LightGBM\": LGBMRegressor(**Params),\n    \"XGBoost\": XGBRegressor(**XGB_Params),\n    \"CatBoost\": CatBoostRegressor(**CatBoost_Params),\n    \"RandomForest\": RandomForestRegressor(n_estimators=300, random_state=SEED),\n    \"GradientBoosting\": GradientBoostingRegressor(n_estimators=300, random_state=SEED)\n}\n\n# Feature Names for Importance\nfeature_names = train.drop(['sii'], axis=1).columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:23:52.862238Z","iopub.execute_input":"2024-11-14T08:23:52.862785Z","iopub.status.idle":"2024-11-14T08:23:52.877285Z","shell.execute_reply.started":"2024-11-14T08:23:52.862751Z","shell.execute_reply":"2024-11-14T08:23:52.876402Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train LightGBM Model\nprint(\"\\nTraining LightGBM Model...\")\nlgb_submission, lgb_models = TrainML(Light, test)\n\n# Extract Feature Importance from Trained Models\nlgb_importance = pd.DataFrame({\n    'Feature': feature_names,\n    'Importance': np.mean([model.feature_importances_ for model in lgb_models], axis=0)\n}).sort_values(by='Importance', ascending=False)\n\n# Plot LightGBM Feature Importance\nplot_feature_importance(lgb_importance, \"LightGBM\")\n\n# Print Top 5 Features\nprint(\"\\nTop 5 Features for LightGBM:\")\nprint(lgb_importance.head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:23:52.878563Z","iopub.execute_input":"2024-11-14T08:23:52.878917Z","iopub.status.idle":"2024-11-14T08:24:05.763207Z","shell.execute_reply.started":"2024-11-14T08:23:52.878883Z","shell.execute_reply":"2024-11-14T08:24:05.762180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train XGBoost Model\nprint(\"\\nTraining XGBoost Model...\")\nxgb_submission, xgb_models = TrainML(XGB_Model, test)\n\n# Extract Feature Importance from Trained Models\nxgb_importance = pd.DataFrame({\n    'Feature': feature_names,\n    'Importance': np.mean([model.feature_importances_ for model in xgb_models], axis=0)\n}).sort_values(by='Importance', ascending=False)\n\n# Plot XGBoost Feature Importance\nplot_feature_importance(xgb_importance, \"XGBoost\")\n\n# Print Top 5 Features\nprint(\"\\nTop 5 Features for XGBoost:\")\nprint(xgb_importance.head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:24:05.764425Z","iopub.execute_input":"2024-11-14T08:24:05.764723Z","iopub.status.idle":"2024-11-14T08:24:12.577685Z","shell.execute_reply.started":"2024-11-14T08:24:05.764690Z","shell.execute_reply":"2024-11-14T08:24:12.576661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train CatBoost Model\nprint(\"\\nTraining CatBoost Model...\")\ncat_submission, cat_models = TrainML(CatBoost_Model, test)\n\n# Extract Feature Importance from Trained Models\ncat_importance = pd.DataFrame({\n    'Feature': feature_names,\n    'Importance': np.mean([model.get_feature_importance() for model in cat_models], axis=0)\n}).sort_values(by='Importance', ascending=False)\n\n# Plot CatBoost Feature Importance\nplot_feature_importance(cat_importance, \"CatBoost\")\n\n# Print Top 5 Features\nprint(\"\\nTop 5 Features for CatBoost:\")\nprint(cat_importance.head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:24:12.578966Z","iopub.execute_input":"2024-11-14T08:24:12.579299Z","iopub.status.idle":"2024-11-14T08:24:32.199698Z","shell.execute_reply.started":"2024-11-14T08:24:12.579264Z","shell.execute_reply":"2024-11-14T08:24:32.198721Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define an imputer for missing values\nimputer = SimpleImputer(strategy=\"median\")  # Use \"mean\" or \"most_frequent\" if preferred\n\n# Impute missing values in training and testing data\nX_train = train.drop(['sii'], axis=1)\ny_train = train['sii']\nX_test = test.copy()\n\nX_train_imputed = imputer.fit_transform(X_train)\nX_test_imputed = imputer.transform(X_test)\n\n# Train Random Forest Model with Imputed Data\nprint(\"\\nTraining Random Forest Model...\")\nrf_model = RandomForestRegressor(n_estimators=300, random_state=SEED)\nrf_model.fit(X_train_imputed, y_train)\n\n# Predict on Test Data\nrf_predictions = rf_model.predict(X_test_imputed)\n\n# Feature Importance\nrf_importance = pd.DataFrame({\n    'Feature': feature_names,\n    'Importance': rf_model.feature_importances_\n}).sort_values(by='Importance', ascending=False)\n\n# Plot Feature Importance\nplot_feature_importance(rf_importance, \"Random Forest\")\n\n# Print Top 5 Features\nprint(\"\\nTop 5 Features for Random Forest:\")\nprint(rf_importance.head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:24:32.200990Z","iopub.execute_input":"2024-11-14T08:24:32.201307Z","iopub.status.idle":"2024-11-14T08:25:05.031828Z","shell.execute_reply.started":"2024-11-14T08:24:32.201273Z","shell.execute_reply":"2024-11-14T08:25:05.030855Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer\nfrom sklearn.ensemble import GradientBoostingRegressor\n\n# Define an imputer for missing values\nimputer = SimpleImputer(strategy=\"median\")  # You can also use \"mean\" or \"most_frequent\"\n\n# Prepare training and testing datasets\nX_train = train.drop(['sii'], axis=1)\ny_train = train['sii']\nX_test = test.copy()\n\n# Impute missing values\nX_train_imputed = imputer.fit_transform(X_train)\nX_test_imputed = imputer.transform(X_test)\n\n# Train Gradient Boosting Model\nprint(\"\\nTraining Gradient Boosting Model...\")\ngb_model = GradientBoostingRegressor(n_estimators=300, random_state=SEED)\ngb_model.fit(X_train_imputed, y_train)\n\n# Predict on test data\ngb_predictions = gb_model.predict(X_test_imputed)\n\n# Feature Importance\ngb_importance = pd.DataFrame({\n    'Feature': feature_names,\n    'Importance': gb_model.feature_importances_\n}).sort_values(by='Importance', ascending=False)\n\n# Plot Feature Importance\nplot_feature_importance(gb_importance, \"Gradient Boosting\")\n\n# Print Top 5 Features\nprint(\"\\nTop 5 Features for Gradient Boosting:\")\nprint(gb_importance.head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:25:05.033064Z","iopub.execute_input":"2024-11-14T08:25:05.033378Z","iopub.status.idle":"2024-11-14T08:25:17.521176Z","shell.execute_reply.started":"2024-11-14T08:25:05.033344Z","shell.execute_reply":"2024-11-14T08:25:17.519858Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save individual submissions\nlgb_submission.to_csv(\"LightGBM_submission_3.csv\", index=False)\nxgb_submission.to_csv(\"XGBoost_submission_3.csv\", index=False)\ncat_submission.to_csv(\"CatBoost_submission_3.csv\", index=False)\n\nprint(\"Individual model training and submissions of feature_3 completed!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:25:17.522676Z","iopub.execute_input":"2024-11-14T08:25:17.523060Z","iopub.status.idle":"2024-11-14T08:25:17.532875Z","shell.execute_reply.started":"2024-11-14T08:25:17.523020Z","shell.execute_reply":"2024-11-14T08:25:17.531790Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Conbination","metadata":{}},{"cell_type":"code","source":"sub1 = Submission1\nsub2 = Submission2\nsub3 = Submission3\n\nsub1 = sub1.sort_values(by='id').reset_index(drop=True)\nsub2 = sub2.sort_values(by='id').reset_index(drop=True)\nsub3 = sub3.sort_values(by='id').reset_index(drop=True)\n\nsub1 = sub1.rename(columns={'sii': 'sii_1'})\nsub2 = sub2.rename(columns={'sii': 'sii_2'})\nsub3 = sub3.rename(columns={'sii': 'sii_3'})\nsubs = pd.merge(sub1,sub2,on=['id'])\nsubs = pd.merge(subs,sub3,on=['id'])\n\n#subs['sii_s'] = subs['sii_1'] *0.555 + 0.001* subs['sii_2'] + 0.444* subs['sii_3']\n#subs['sii_s'] = subs['sii_1'] *0.450 + 0.450* subs['sii_2'] + 0.100* subs['sii_3']\n#subs['sii_s'] = subs['sii_1'] *0.525 + 0.050* subs['sii_2'] + 0.425* subs['sii_3']\n#subs['sii_s'] = subs['sii_1'] *0.500 + 0.100* subs['sii_2'] + 0.400* subs['sii_3']\n\nsubs['sii_s'] = np.round(subs['sii_1'] *0.85 + 0.10* subs['sii_2'] + 0.05* subs['sii_3'])\n\nsubs['sii_s'] = subs['sii_s'].astype(int)\n\ncombined = pd.DataFrame({\n    \n    'id'   : sub1['id'],\n    \n    'sii_1': sub1['sii_1'],\n    'sii_2': sub2['sii_2'],\n    'sii_3': sub3['sii_3'],\n    \n    'sii_s': subs['sii_s'],\n})\n\ndisplay(combined)\n\ndef majority_vote(row):\n    return row.mode()[0]\n                                                         \ncombined['final_sii'] = combined[['sii_1', 'sii_2', 'sii_3', 'sii_s']].apply(majority_vote, axis=1)\n\nfinal_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\n\nfinal_submission.to_csv('final_submission.csv', index=False)\n\nprint(\"Majority voting completed and saved to 'final_submission.csv'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:25:17.534161Z","iopub.execute_input":"2024-11-14T08:25:17.534494Z","iopub.status.idle":"2024-11-14T08:25:17.573599Z","shell.execute_reply.started":"2024-11-14T08:25:17.534460Z","shell.execute_reply":"2024-11-14T08:25:17.572350Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Save submission\n#Submission.to_csv('submission.csv', index=False)\n#print(Submission['sii'].value_counts())\n#print(Submission)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:25:17.574832Z","iopub.execute_input":"2024-11-14T08:25:17.575165Z","iopub.status.idle":"2024-11-14T08:25:17.579465Z","shell.execute_reply.started":"2024-11-14T08:25:17.575130Z","shell.execute_reply":"2024-11-14T08:25:17.578499Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T08:25:17.580708Z","iopub.execute_input":"2024-11-14T08:25:17.581111Z","iopub.status.idle":"2024-11-14T08:25:17.593837Z","shell.execute_reply.started":"2024-11-14T08:25:17.581076Z","shell.execute_reply":"2024-11-14T08:25:17.592929Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}