{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30775,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom sklearn.metrics import cohen_kappa_score\n\n# As a simple approach:\n# - Only use labeled tabular data for now.\n# - Ignore parquet data.\n# - Ignore PCIAT columns which don't exist in test set (these are directly used to calculate sii, so we could consider them as intermediate targets to predict)\n# - One-hot encode strings.\n# - Impute missing numbers as mean of that feature. This includes string one-hot encodings for now.\n# - Avoid further prep by using XGBoost as model\n# - Use simple set-aside test set for local evaluation. Do not tune hyperparameters yet.\n\ndf_train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ndf_test = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\nstudies_to_drop = ['PCIAT', 'BIA', 'Season']\nfor study in studies_to_drop:\n    df_train = df_train.drop(columns=df_train.filter(like=study).columns)\n    df_test = df_test.drop(columns=df_test.filter(like=study).columns)\n\ndf_train = df_train.drop(columns=['id'], axis=1)\ndf_train_unlabeled = df_train.query('sii!=sii') # TODO - determine what to do with this. Clustering?\ndf_train_labeled = df_train.query('sii==sii')\nX_train_labeled, Y_train_labeled = df_train_labeled.drop('sii', axis=1), df_train_labeled['sii']\n\ndf_test, index_test = df_test.drop(columns=['id'], axis=1), df_test['id']","metadata":{"execution":{"iopub.status.busy":"2024-10-29T21:43:49.511456Z","iopub.execute_input":"2024-10-29T21:43:49.511914Z","iopub.status.idle":"2024-10-29T21:43:49.606058Z","shell.execute_reply.started":"2024-10-29T21:43:49.511865Z","shell.execute_reply":"2024-10-29T21:43:49.605183Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"run_cv = False          # set this to True if you want to run k-fold CV, set to False if not\nrun_output = False      # set this to True if you want to run the model on the test data to generate an output","metadata":{"execution":{"iopub.status.busy":"2024-10-29T21:43:49.608708Z","iopub.execute_input":"2024-10-29T21:43:49.609519Z","iopub.status.idle":"2024-10-29T21:43:49.613916Z","shell.execute_reply.started":"2024-10-29T21:43:49.609454Z","shell.execute_reply":"2024-10-29T21:43:49.612905Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nimport keras\n\ndef apply_triang_nn_model(trainX, trainY, testX, learning_rate=0.007, n_epochs=5, verbose=False):\n    # create an 80/20 test split to train the MLP\n    tX, vX, tY, vY = train_test_split(trainX, trainY, test_size=0.2, random_state=0)\n    \n    Init = keras.initializers.RandomNormal(seed=0)\n\n    # creates an MLP with 3 hidden layers and an output to calculate between 0-1, which then gets scaled up to 0-5\n    m = keras.models.Sequential()\n    m.add(keras.layers.Flatten(input_shape=[tX.shape[1]], name=\"input\"))      # creates input layer with n_features perceptrons\n    m.add(keras.layers.Dense(16, activation=\"relu\", kernel_initializer=Init, name='hidden_1'))    # creates hidden layer with 16 perceptrons\n    m.add(keras.layers.Dense(8, activation=\"relu\", kernel_initializer=Init, name='hidden_2'))    # creates hidden layer with 18 perceptrons\n    m.add(keras.layers.Dense(4, activation=\"relu\", kernel_initializer=Init, name='hidden_3'))    # creates hidden layer with 4 perceptrons\n    m.add(keras.layers.Dense(1, activation='sigmoid', kernel_initializer=Init, name='output'))  # create output layer with 1 perceptron (regressor with bins)\n    if verbose: m.summary()   # displays all model alyers\n    m.compile(loss='binary_crossentropy', optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate))\n\n    if verbose:\n        hist = m.fit(tX, tY/5, epochs=n_epochs, validation_data=(vX, vY/5))\n    else:\n        hist = m.fit(tX, tY/5, epochs=n_epochs, validation_data=(vX, vY/5), verbose=0)\n\n    pred_out = m.predict(testX) * 5\n    Y_pred = pred_out.copy()\n    Y_pred = np.round(Y_pred)\n\n    return Y_pred, hist","metadata":{"execution":{"iopub.status.busy":"2024-10-29T21:43:49.615433Z","iopub.execute_input":"2024-10-29T21:43:49.615890Z","iopub.status.idle":"2024-10-29T21:43:49.630656Z","shell.execute_reply.started":"2024-10-29T21:43:49.615838Z","shell.execute_reply":"2024-10-29T21:43:49.629713Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.impute import KNNImputer\n\ndef remove_outliers(input_X, features, std_threshold=3, impute=False):\n    # replaces any outliers outside the std threshold\n    X = input_X.copy()\n\n    for feature in features:\n        std = np.std(X[feature])\n        mean = np.mean(X[feature])\n        X_missing = (X[feature] < mean - std_threshold * std) | (X[feature] > mean + std_threshold * std)\n        X.loc[X_missing, feature] = np.NaN\n      \n    if impute:\n        # imputes any NaN values using kNN imputer\n        imputer = KNNImputer(n_neighbors = 5, missing_values=-1)\n        X_subset = X.loc[X[features] > 0]\n        X_subset = X_subset[features]\n        X_new_subset = pd.DataFrame(imputer.fit_transform(X_subset), columns=features)\n        X[features] = X_new_subset\n    return X\n\ndef calculate_FGC_PU_Zone(input_X):\n    for i, row in input_X.iterrows():\n        PU = row['FGC-FGC_PU']\n        sex = row['Basic_Demos-Sex']\n        age = row['Basic_Demos-Age']\n        # calculates the FGC PU Zone based on # pushups, sex, and age\n        if sex == 1: # female\n            table = {5: 3,\n                    6: 3,\n                    7: 4,\n                    8: 5,\n                    9: 6}\n            if age in table.keys():\n                if PU >= table[age]:\n                    input_X.at[i,'FGC-FGC_PU_Zone'] = 1\n                else:\n                    input_X.at[i,'FGC-FGC_PU_Zone'] = 0\n            else:\n                if PU >= 7:\n                    input_X.at[i,'FGC-FGC_PU_Zone'] = 1\n                else:\n                    input_X.at[i,'FGC-FGC_PU_Zone'] = 0\n        elif sex == 0: # male\n            table = {5: 3,\n                    6: 3,\n                    7: 4,\n                    8: 5,\n                    9: 6,\n                    10: 7,\n                    11: 8,\n                    12: 10,\n                    13: 12,\n                    14: 14,\n                    15: 16}\n            if age in table.keys():\n                if PU >= table[age]:\n                    input_X.at[i,'FGC-FGC_PU_Zone'] = 1\n                else:\n                    input_X.at[i,'FGC-FGC_PU_Zone'] = 0\n            else:\n                if PU >= 18:\n                    input_X.at[i,'FGC-FGC_PU_Zone'] = 1\n                else:\n                    input_X.at[i,'FGC-FGC_PU_Zone'] = 0\n        else:\n            input_X.at[i,'FGC-FGC_PU_Zone'] = np.NaN\n    return input_X\n\ndef calculate_FGC_SR_Zone(input_X, side='L'):\n    if side=='L':\n        col = 'FGC-FGC_SRL'\n        zone = 'FGC-FGC_SRL_Zone'\n    elif side=='R':\n        col = 'FGC-FGC_SRR'\n        zone = 'FGC-FGC_SRR_Zone'\n    else:\n        return input_X\n    \n    for i, row in input_X.iterrows():\n        SR = row[col]\n        sex = row['Basic_Demos-Sex']\n        age = row['Basic_Demos-Age']\n        # calculates the FGC SRL/R Zone based on SRL/SRR, sex, and age\n        if sex == 1: # female\n            table = {11:10,\n                    12:10,\n                    13:10,\n                    14:10,\n                    15:12,\n                    16:12,\n                    17:12,\n                    18:12}\n            if age in table.keys():\n                if SR >= table[age]:\n                    input_X.at[i, zone] = 1\n                else:\n                    input_X.at[i, zone] = 0\n            else:\n                if SR >= 9:\n                    input_X.at[i, zone] = 1\n                else:\n                    input_X.at[i, zone] = 0\n        elif sex == 0: # male\n            if SR >= 8:\n                input_X.at[i, zone] = 1\n            else:\n                input_X.at[i, zone] = 0\n        else:\n            input_X.at[i, zone] = np.NaN\n    return input_X\n\ndef calculate_FGC_TL_Zone(input_X):\n    for i, row in input_X.iterrows():\n        TL = row['FGC-FGC_TL']\n        age = row['Basic_Demos-Age']\n        # calculates the FGC TL Zone based on TL and age\n        if age <= 9:\n            if TL >= 6:\n                input_X.at[i,'FGC-FGC_TL_Zone'] = 1\n            else:\n                input_X.at[i,'FGC-FGC_TL_Zone'] = 0\n        else:\n            if TL >= 9:\n                input_X.at[i,'FGC-FGC_TL_Zone'] = 1\n            else:\n                input_X.at[i,'FGC-FGC_TL_Zone'] = 0\n        \n    return input_X\n\ndef correct_errors(input_X):\n    X = input_X.copy()\n    \n    # physical exam\n    features_to_impute = ['Physical-Diastolic_BP', 'Physical-Systolic_BP', 'Physical-HeartRate']\n    X.loc[X['Physical-Weight'] == 0, 'Physical-BMI'] = None # maybe need to look into imputing bmi and weight?\n    X.loc[X['Physical-Weight'] == 0, 'Physical-Weight'] = None\n    #X = remove_outliers(X, features_to_impute) # remove_outliers, need to look into better method for this\n    \n    # fitnessgram vitals and treadmill have no observable errors\n    \n    # fitnessgram child\n    # any grip strength reading of 0 lbs is likely an error\n    X.loc[X['FGC-FGC_GSND'] == 0, 'FGC-FGC_GSND'] = None\n    X.loc[X['FGC-FGC_GSND'] == 0, 'FGC-FGC_GSND_Zone'] = None\n    \n    X.loc[X['FGC-FGC_GSD'] == 0, 'FGC-FGC_GSD'] = None\n    X.loc[X['FGC-FGC_GSD'] == 0, 'FGC-FGC_GSD_Zone'] = None\n    \n    \n    # if PU_Zone is blank but PU is not blank, compute PU_Zone\n    mask = (X['FGC-FGC_PU_Zone'] != X['FGC-FGC_PU_Zone']) & (X['FGC-FGC_PU'] == X['FGC-FGC_PU'])\n    X_mask = X[mask]\n    X_mask = calculate_FGC_PU_Zone(X_mask)\n    X.loc[X_mask.index, 'FGC-FGC_PU_Zone'] = X_mask['FGC-FGC_PU_Zone']\n    \n    X.loc[X['FGC-FGC_SRL'] == 0, 'FGC-FGC_SRL_Zone'] = 0  # if SRL is zero, the zone should also be zero\n    # if SRL_Zone is blank but X['FGC-FGC_SRL'] is not blank, compute SRL_Zone\n    mask = (X['FGC-FGC_SRL_Zone'] != X['FGC-FGC_SRL_Zone']) & (X['FGC-FGC_SRL'] == X['FGC-FGC_SRL'])\n    X_mask = X[mask]\n    X_mask = calculate_FGC_SR_Zone(X_mask, side='L')\n    X.loc[X_mask.index, 'FGC-FGC_SRL_Zone'] = X_mask['FGC-FGC_SRL_Zone']\n    \n    \n    X.loc[X['FGC-FGC_SRR'] == 0, 'FGC-FGC_SRR_Zone'] = 0  # if SRR is zero, the zone should also be zero\n    # if SRR_Zone is blank but X['FGC-FGC_SRR'] is not blank, compute SRR_Zone\n    mask = (X['FGC-FGC_SRR_Zone'] != X['FGC-FGC_SRR_Zone']) & (X['FGC-FGC_SRR'] == X['FGC-FGC_SRR'])\n    X_mask = X[mask]\n    X_mask = calculate_FGC_SR_Zone(X_mask, side='R')\n    X.loc[X_mask.index, 'FGC-FGC_SRR_Zone'] = X_mask['FGC-FGC_SRR_Zone']\n    \n    X.loc[X['FGC-FGC_TL'] == 0, 'FGC-FGC_TL_Zone'] = 0  # if TL is zero, the zone should also be zero\n    # if TL_Zone is blank but X['FGC-FGC_TL'] is not blank, compute TL_Zone\n    mask = (X['FGC-FGC_TL_Zone'] != X['FGC-FGC_TL_Zone']) & (X['FGC-FGC_TL'] == X['FGC-FGC_TL'])\n    X_mask = X[mask]\n    X_mask = calculate_FGC_TL_Zone(X_mask)\n    X.loc[X_mask.index, 'FGC-FGC_TL_Zone'] = X_mask['FGC-FGC_TL_Zone']\n    \n    # bio-electric impedance analysis: throw this out?\n    \n    # physical activity questionnaire, PAQ-C vs PAQ-A? no visible measurement errors\n    \n    # sleep disturbance scale, no visible measurement errors\n    \n    return X","metadata":{"execution":{"iopub.status.busy":"2024-10-29T21:43:49.632080Z","iopub.execute_input":"2024-10-29T21:43:49.632430Z","iopub.status.idle":"2024-10-29T21:43:49.667832Z","shell.execute_reply.started":"2024-10-29T21:43:49.632396Z","shell.execute_reply":"2024-10-29T21:43:49.666614Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def impute_numeric_cols(input_X):\n    X = input_X.copy()\n    \n    imputer = KNNImputer(n_neighbors=5)\n    numeric_cols = X.select_dtypes(include=['float64', 'int64']).columns\n    imputed_data = imputer.fit_transform(X[numeric_cols])\n    X_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\n    for col in X.columns:\n        if col not in numeric_cols:\n            X_imputed[col] = X[col]\n            \n    return X","metadata":{"execution":{"iopub.status.busy":"2024-10-29T21:43:49.671041Z","iopub.execute_input":"2024-10-29T21:43:49.671591Z","iopub.status.idle":"2024-10-29T21:43:49.683021Z","shell.execute_reply.started":"2024-10-29T21:43:49.671541Z","shell.execute_reply":"2024-10-29T21:43:49.682179Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# starting code for ensemble model and params from https://www.kaggle.com/code/honganzhu/cmi-piu-competition\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom scipy.optimize import minimize\nfrom sklearn.model_selection import StratifiedKFold\nfrom tqdm import tqdm\nfrom sklearn.ensemble import VotingRegressor\nfrom IPython.display import clear_output\nfrom colorama import Fore, Style\n\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01,  # Increased from 2.68e-06\n    'device': 'gpu'\n\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': 0,\n    'tree_method': 'gpu_hist',\n\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': 0,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n    'task_type': 'GPU'\n\n}\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef apply_voting_model(trainX, trainY, testX, verbose=False, seed=0):\n    np.random.seed(seed)\n    \n    X = trainX.copy()\n    Y = trainY.copy()\n    \n    SKF = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed)\n    \n    train_scores = []\n    test_scores = []\n    \n    oof_non_rounded = np.zeros(len(Y), dtype=float)\n    test_preds = np.zeros((len(testX), 5))\n    \n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, Y), desc=\"Training Folds\", total=5)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        Y_train, Y_val = Y.iloc[train_idx], Y.iloc[test_idx]\n        \n        lgbm_model = LGBMRegressor(**Params, random_state=seed, verbose=-1, n_estimators=300)\n        xgb_model = XGBRegressor(**XGB_Params)\n        catboost_model = CatBoostRegressor(**CatBoost_Params)\n\n        model = VotingRegressor(estimators=[\n            ('lightgbm', lgbm_model),\n            ('xgboost', xgb_model),\n            ('catboost', catboost_model)\n        ])\n        model.fit(X_train, Y_train)\n        \n        Y_train_pred = model.predict(X_train)\n        Y_val_pred = model.predict(X_val)\n        \n        oof_non_rounded[test_idx] = Y_val_pred\n        Y_val_pred_rounded = Y_val_pred.round(0).astype(int)\n        \n        train_kappa = quadratic_weighted_kappa(Y_train, Y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(Y_val, Y_val_pred_rounded)\n        \n        train_scores.append(train_kappa)\n        test_scores.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(testX)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n        \n    print(f\"Mean Train QWK --> {np.mean(train_scores):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_scores):.4f}\")\n    \n    kappa_optimizer = minimize(evaluate_predictions, x0=[0.5, 1.5, 2.5], args=(Y, oof_non_rounded), method='Nelder-Mead')\n    assert kappa_optimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_rounder(oof_non_rounded, kappa_optimizer.x)\n    t_kappa = quadratic_weighted_kappa(Y, oof_tuned)\n    \n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {t_kappa:.3f}{Style.RESET_ALL}\")\n    \n    tpm = test_preds.mean(axis=1)\n    tp_tuned = threshold_rounder(tpm, kappa_optimizer.x)\n    \n    submission = pd.DataFrame({'id': sample['id'],\n                              'sii': tp_tuned})\n    return submission\n\nX_train, Y_train = df_train_labeled.drop('sii', axis=1), df_train_labeled['sii']\nX_test = df_test.copy()\n\nX_train = correct_errors(X_train)\nX_test = correct_errors(X_test)\n\nX_train = impute_numeric_cols(X_train)\n#X_test = impute_numeric_cols(X_test)\nsubmission = apply_voting_model(X_train, Y_train, X_test, verbose=False, seed=0)\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T21:43:49.726127Z","iopub.execute_input":"2024-10-29T21:43:49.726834Z","iopub.status.idle":"2024-10-29T21:44:12.095135Z","shell.execute_reply.started":"2024-10-29T21:43:49.726786Z","shell.execute_reply":"2024-10-29T21:44:12.094086Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(submission)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T21:44:12.099660Z","iopub.execute_input":"2024-10-29T21:44:12.100693Z","iopub.status.idle":"2024-10-29T21:44:12.112702Z","shell.execute_reply.started":"2024-10-29T21:44:12.100646Z","shell.execute_reply":"2024-10-29T21:44:12.111620Z"},"trusted":true},"outputs":[],"execution_count":null}]}