{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"b4ba8d68-ef45-4c7b-8939-81daa0dc85f1","cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nfrom sklearn.impute import KNNImputer\npd.set_option('display.max_columns', None)\nworking_dir = '/kaggle/input/child-mind-institute-problematic-internet-use/'\noutput_dir = '/kaggle/working/'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:44:52.759299Z","iopub.execute_input":"2024-12-19T09:44:52.759738Z","iopub.status.idle":"2024-12-19T09:44:55.379544Z","shell.execute_reply.started":"2024-12-19T09:44:52.759693Z","shell.execute_reply":"2024-12-19T09:44:55.378378Z"}},"outputs":[],"execution_count":null},{"id":"0b6dd8d5-05a6-45f8-8439-287e73e245bf","cell_type":"code","source":"df = pd.read_csv(working_dir + 'train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:44:55.380751Z","iopub.execute_input":"2024-12-19T09:44:55.381300Z","iopub.status.idle":"2024-12-19T09:44:55.466990Z","shell.execute_reply.started":"2024-12-19T09:44:55.381257Z","shell.execute_reply":"2024-12-19T09:44:55.465934Z"}},"outputs":[],"execution_count":null},{"id":"8c4c4751-877e-4ff2-ab09-b9d6496a8dc6","cell_type":"code","source":"def plot_nas(df: pd.DataFrame):\n    # Calculate the missing and available data ratios\n    na_df = (df.isnull().sum() / len(df)) * 100  # Missing ratio in percentage\n    na_df = na_df.sort_values(ascending=False)  # Sort values\n    \n    available_df = 100 - na_df  # Available ratio in percentage\n    \n    # Create a horizontal stacked bar chart\n    plot_width, plot_height = (16, 18)\n    plt.rcParams['figure.figsize'] = (plot_width, plot_height)\n    \n    fig, ax = plt.subplots()\n    bar_height = 0.4  # Set bar height to make them thinner\n    y_pos = np.arange(len(na_df))  # Positions for bars\n    \n    ax.barh(y_pos, na_df, color='salmon', label='Missing Ratio (%)', height=bar_height)\n    ax.barh(y_pos, available_df, left=na_df, color='lightgreen', label='Available Ratio (%)', height=bar_height)\n    \n    ax.set_yticks(y_pos)\n    ax.set_yticklabels(na_df.index)\n    ax.set_xlabel('Percentage')\n    ax.set_title('Missing and Available Data Ratios')\n    ax.legend()\n    plt.tight_layout()\n    plt.show()\n\nplt.rcParams['figure.figsize'] = (3,3)\nplot_nas(df)","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:44:55.468202Z","iopub.execute_input":"2024-12-19T09:44:55.468601Z","iopub.status.idle":"2024-12-19T09:44:56.904007Z","shell.execute_reply.started":"2024-12-19T09:44:55.468560Z","shell.execute_reply":"2024-12-19T09:44:56.902796Z"}},"outputs":[],"execution_count":null},{"id":"f97152f4-d14e-4d98-bd7f-306606c73672","cell_type":"code","source":"# Dealing with season data\n# Method 1: Dropping\n# df = df.loc[: ,~df.columns.str.contains('season', case=False)]\n\n# Method 2: Encoding\ndisplay(df.head().select_dtypes(exclude='number'))\n\nseason_cols = df.select_dtypes(exclude='number').columns\nseason_cols = [col for col in season_cols if 'id' not in col]\n\n# Encode\nseason_mapping = {\n    'Spring' : 1,\n    'Sumemr' : 2,\n    'Fall'   : 3,\n    'Winter' : 4\n}\n\nfor col in season_cols:\n    df[col] = df[col].apply(lambda x : np.uint8(season_mapping.get(x, 0))).astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:44:56.906598Z","iopub.execute_input":"2024-12-19T09:44:56.906912Z","iopub.status.idle":"2024-12-19T09:44:57.265138Z","shell.execute_reply.started":"2024-12-19T09:44:56.906883Z","shell.execute_reply":"2024-12-19T09:44:57.264032Z"}},"outputs":[],"execution_count":null},{"id":"b2c31233-364e-4648-8f21-4dd395ace627","cell_type":"code","source":"# Trim rows by sii and pciat\ndf = df.drop(columns=['id'])\npciat_columns = [col for col in df.columns if col.split(sep='-')[0] == 'PCIAT']\npciat_columns.append('sii')\ndf = df.dropna(subset=pciat_columns)\ndf = df.dropna(subset='sii')\ndf.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:44:57.266866Z","iopub.execute_input":"2024-12-19T09:44:57.267252Z","iopub.status.idle":"2024-12-19T09:44:57.349700Z","shell.execute_reply.started":"2024-12-19T09:44:57.267213Z","shell.execute_reply":"2024-12-19T09:44:57.348624Z"}},"outputs":[],"execution_count":null},{"id":"1c011ad2-bd78-4611-9d85-6a2476abd33e","cell_type":"code","source":"# Dropping all data with > 40% missing ratio\ndropped_cols = []\ncolumns = df.columns\nfor col in columns:\n    missing_ratio = df[col].isna().sum() / len(df)\n    if missing_ratio > 0.56:\n        df = df.drop(columns=col)\n        dropped_cols.append(col)\n\nprint('Dropped: ')\nprint(dropped_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:44:57.350606Z","iopub.execute_input":"2024-12-19T09:44:57.350924Z","iopub.status.idle":"2024-12-19T09:44:57.378522Z","shell.execute_reply.started":"2024-12-19T09:44:57.350897Z","shell.execute_reply":"2024-12-19T09:44:57.377384Z"}},"outputs":[],"execution_count":null},{"id":"2c5749e3-786c-4c2c-8189-605282926dd1","cell_type":"code","source":"df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:44:57.379706Z","iopub.execute_input":"2024-12-19T09:44:57.380092Z","iopub.status.idle":"2024-12-19T09:44:57.556920Z","shell.execute_reply.started":"2024-12-19T09:44:57.380041Z","shell.execute_reply":"2024-12-19T09:44:57.555858Z"}},"outputs":[],"execution_count":null},{"id":"ee4a39b8-84cd-4ae4-a5b7-21af569735de","cell_type":"code","source":"num_columns = [col for col in df.columns if \n              (col.split(sep='-')[0] != 'PCIAT' and col.split(sep='-')[0] != 'sii'\n                  and col != 'id'\n              )\n          ]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:44:57.557755Z","iopub.execute_input":"2024-12-19T09:44:57.558003Z","iopub.status.idle":"2024-12-19T09:44:57.562780Z","shell.execute_reply.started":"2024-12-19T09:44:57.557981Z","shell.execute_reply":"2024-12-19T09:44:57.561607Z"}},"outputs":[],"execution_count":null},{"id":"29d4f632-f511-4ed3-a342-83715b852556","cell_type":"code","source":"# General quantile capping\nfeature_cap = {}\nfor col in num_columns:\n    if col == 'id':\n        continue\n    feature_cap[col] = {\n        'lower': df[col].quantile(0.01),\n        'upper': df[col].quantile(0.99)\n    }\n    df[col] = df[col].clip(df[col].quantile(0.01), df[col].quantile(0.99))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:44:57.563840Z","iopub.execute_input":"2024-12-19T09:44:57.564111Z","iopub.status.idle":"2024-12-19T09:44:57.753191Z","shell.execute_reply.started":"2024-12-19T09:44:57.564078Z","shell.execute_reply":"2024-12-19T09:44:57.752102Z"}},"outputs":[],"execution_count":null},{"id":"e9b690d5-738f-4fd0-a883-2f6536903d95","cell_type":"code","source":"# Considering all Physical and BIA data that are <= less than or equal to 0 are errors\n# we will replace it with NaN then impute (Age too)\nphysical_columns = [col for col in df.columns if col.split(sep='-')[0] == 'Physical']\nBIA_columns = [col for col in df.columns if col.split(sep='-')[0] == 'BIA']\ndf[df['Basic_Demos-Age'] == 0] = np.nan\ndf[df[physical_columns + BIA_columns] <= 0] = np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:44:57.753943Z","iopub.execute_input":"2024-12-19T09:44:57.754223Z","iopub.status.idle":"2024-12-19T09:44:57.803398Z","shell.execute_reply.started":"2024-12-19T09:44:57.754198Z","shell.execute_reply":"2024-12-19T09:44:57.802087Z"}},"outputs":[],"execution_count":null},{"id":"7360f69f-7cf1-49a1-9780-6967b84cf111","cell_type":"code","source":"# Method 1 Using KNN imputer\nfrom sklearn.impute import KNNImputer\nfrom sklearn.impute import SimpleImputer\n\n# Divide into 5 cluster corresponding to 5 groups of PCIAT\n\n# 5 is an intuitive guess, need to find optimal number of k-neighbours later.\nimputer = KNNImputer(\n    n_neighbors=5\n)\n\nfeature_columns = [col for col in df.columns if \n                      col not in pciat_columns and\n                      col != 'sii' and\n                      col != 'id'\n                 ]\n\nX = np.array(df[feature_columns])\nX_imputed = imputer.fit_transform(X)\ndf[feature_columns] = X_imputed","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:44:57.804639Z","iopub.execute_input":"2024-12-19T09:44:57.805002Z","iopub.status.idle":"2024-12-19T09:44:59.705528Z","shell.execute_reply.started":"2024-12-19T09:44:57.804964Z","shell.execute_reply":"2024-12-19T09:44:59.704607Z"}},"outputs":[],"execution_count":null},{"id":"e44bbdfc-edb3-4026-bb50-9380d145eed7","cell_type":"code","source":"df['sii'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:44:59.706457Z","iopub.execute_input":"2024-12-19T09:44:59.706828Z","iopub.status.idle":"2024-12-19T09:44:59.718777Z","shell.execute_reply.started":"2024-12-19T09:44:59.706794Z","shell.execute_reply":"2024-12-19T09:44:59.717745Z"}},"outputs":[],"execution_count":null},{"id":"931d38ad-06f8-4c5f-aa97-f91a6805bd6f","cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import make_scorer\nfrom sklearn.model_selection import GridSearchCV\nfrom xgboost import XGBClassifier","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:44:59.722067Z","iopub.execute_input":"2024-12-19T09:44:59.722388Z","iopub.status.idle":"2024-12-19T09:45:00.178676Z","shell.execute_reply.started":"2024-12-19T09:44:59.722360Z","shell.execute_reply":"2024-12-19T09:45:00.177729Z"}},"outputs":[],"execution_count":null},{"id":"fe150400-6dc4-49f3-8e68-163784522439","cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom xgboost import XGBRegressor\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': 42\n}\n\nxgb_reg = XGBRegressor(**XGB_Params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:45:00.180032Z","iopub.execute_input":"2024-12-19T09:45:00.180361Z","iopub.status.idle":"2024-12-19T09:45:00.185819Z","shell.execute_reply.started":"2024-12-19T09:45:00.180331Z","shell.execute_reply":"2024-12-19T09:45:00.184649Z"}},"outputs":[],"execution_count":null},{"id":"3394e6e0-387d-4979-a607-d6a3a1c446c5","cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom tqdm import tqdm\nfrom scipy.optimize import minimize\nfrom sklearn.base import clone\nfrom IPython.display import clear_output\n\n\ndef get_qwk(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -get_qwk(y_true, rounded_p)\n\n\ndef trainModel(model_class, train_data, test_data): \n    X = train_data.drop(['sii'], axis=1)\n    Y = train_data['sii']\n\n    print(Y.value_counts())\n\n    folds = 5\n    cross_val = StratifiedKFold(n_splits=folds, shuffle=True, random_state=42)\n\n    train_score = []\n    val_score = []\n\n    oof_raw = np.zeros(len(Y), dtype=float)\n    oof_rounded = np.zeros(len(Y), dtype=int)\n    test_preds = np.zeros((len(test_data), folds))\n\n    for fold, (train_idx, val_idx) in enumerate(tqdm(cross_val.split(X, Y), desc=\"Training Folds\", total=folds)):\n        x_train, x_val = X.iloc[train_idx], X.iloc[val_idx]\n        y_train, y_val = Y.iloc[train_idx], Y.iloc[val_idx]\n\n        model = clone(model_class)\n        model.fit(x_train, y_train)\n\n        y_train_pred = model.predict(x_train)\n        y_val_pred = model.predict(x_val)\n\n        oof_raw[val_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[val_idx] = y_val_pred_rounded\n\n        train_kappa = get_qwk(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = get_qwk(y_val, y_val_pred_rounded)\n\n        train_score.append(train_kappa)\n        val_score.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n\n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_score):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(val_score):.4f}\")\n\n    kappaOptimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(Y, oof_raw), \n                              method='Nelder-Mead')\n    \n    assert kappaOptimizer.success, \"Optimizer did not converge\"\n    thresholds = kappaOptimizer.x\n\n    oof_tuned = threshold_Rounder(oof_raw, thresholds)\n    tKappa = get_qwk(Y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {tKappa:.3f}\")\n\n    tpm = test_preds.mean(axis=1)\n    tp_rounded = threshold_Rounder(tpm, thresholds)\n\n    print(tp_rounded)\n    return tp_rounded\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:45:00.186798Z","iopub.execute_input":"2024-12-19T09:45:00.187089Z","iopub.status.idle":"2024-12-19T09:45:00.212531Z","shell.execute_reply.started":"2024-12-19T09:45:00.187062Z","shell.execute_reply":"2024-12-19T09:45:00.211330Z"}},"outputs":[],"execution_count":null},{"id":"d96f4a01-1c33-4a1e-b9f3-35bdf07b42f8","cell_type":"code","source":"def getResult(model_for_validation, X_train, Y_train, X_val, Y_val):\n    fig, ax = plt.subplots(1, 2, figsize=(12,4))\n    \n    sns.heatmap(confusion_matrix(Y_train, model_for_validation.predict(X_train)), annot=True, cmap='viridis', ax=ax[0])\n    print(f\"Train: {cohen_kappa_score(Y_train, model_for_validation.predict(X_train), weights='quadratic')}\")\n    \n    # Problem class 3 always get wrong prediction\n    # Class 2 get misclassified alot with class \n    \n    # Class accuracy improved with balanced weight but same problem still persist\n    Y_pred = model_for_validation.predict(X_val)\n    sns.heatmap(confusion_matrix(Y_val, Y_pred), annot=True, cmap='viridis', ax=ax[1])\n    print(f\"Test: {cohen_kappa_score(Y_val, Y_pred, weights='quadratic')}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:45:00.213813Z","iopub.execute_input":"2024-12-19T09:45:00.214160Z","iopub.status.idle":"2024-12-19T09:45:00.223162Z","shell.execute_reply.started":"2024-12-19T09:45:00.214121Z","shell.execute_reply":"2024-12-19T09:45:00.222170Z"}},"outputs":[],"execution_count":null},{"id":"7171e8c3-68c6-4cae-9236-7143f03b4f80","cell_type":"markdown","source":"<h1>Submission</h1>","metadata":{}},{"id":"76638ecd-424e-4a26-98e6-5d6e37ebcba3","cell_type":"code","source":"# Model 1 pipeline\n# Load data\ntest_df = pd.read_csv(working_dir + 'test.csv')\n                 \ndef transform(test_df):\n    test_df.drop(columns=['id'], inplace=True)\n    # Drop text features\n    test_df = test_df.drop(dropped_cols, axis=1)\n    # test_df = test_df.loc[: ,~test_df.columns.str.contains('season', case=False)]\n\n\n    for col in season_cols:\n        if col in dropped_cols or col == 'PCIAT-Season':\n            continue\n        test_df[col] = test_df[col].apply(lambda x : np.uint8(season_mapping.get(x, 0)).astype(int))\n\n    # Get numeric columns\n    num_columns = [col for col in test_df.columns if \n                      (col.split(sep='-')[0] != 'PCIAT' and col.split(sep='-')[0] != 'sii' and col != 'id')\n                  ]\n\n    # Value capping, use specific cap values acquired from training quantile capping\n    for col in num_columns:\n        if col == 'id':\n            continue\n        test_df[col] = test_df[col].clip(feature_cap[col]['lower'], feature_cap[col]['upper'])\n        \n    # Considering all Physical and BIA data that are <= less than or equal to 0 are errors\n    # we will replace it with NaN then impute (Age too)\n    physical_columns = [col for col in test_df.columns if col.split(sep='-')[0] == 'Physical']\n    BIA_columns = [col for col in test_df.columns if col.split(sep='-')[0] == 'BIA']\n    test_df[test_df['Basic_Demos-Age'] == 0] = np.nan\n    test_df[test_df[physical_columns + BIA_columns] <= 0] = np.nan\n\n\n    imputer = KNNImputer(\n        n_neighbors=5\n    )\n    \n    feature_columns = [col for col in test_df.columns if col != 'id']\n    \n    X = np.array(test_df[feature_columns])\n    X_imputed = imputer.fit_transform(X)\n    test_df[feature_columns] = X_imputed\n\n    return test_df\n\n# test_df = transform(test_df)\n# X_test = test_df.drop(columns=['id'])\n# pred = forest.predict(X_test)\n\nid_col = test_df['id']\ntest_df = transform(test_df)\ntrain_data = df[feature_columns + ['sii']]\nresult = trainModel(xgb_reg, train_data, test_df)","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:45:00.224373Z","iopub.execute_input":"2024-12-19T09:45:00.224715Z","iopub.status.idle":"2024-12-19T09:45:06.874615Z","shell.execute_reply.started":"2024-12-19T09:45:00.224686Z","shell.execute_reply":"2024-12-19T09:45:06.873412Z"}},"outputs":[],"execution_count":null},{"id":"3658e7b7-792c-4639-bcc8-00891c33073f","cell_type":"code","source":"output = pd.DataFrame({\n    'id' : id_col,\n    'sii': result\n})\n\noutput.to_csv(output_dir + 'submission.csv', sep=',', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T09:45:06.875670Z","iopub.execute_input":"2024-12-19T09:45:06.875949Z","iopub.status.idle":"2024-12-19T09:45:06.884668Z","shell.execute_reply.started":"2024-12-19T09:45:06.875923Z","shell.execute_reply":"2024-12-19T09:45:06.883617Z"}},"outputs":[],"execution_count":null}]}