{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\npd.set_option('display.max_columns', None)\nworking_dir = '/kaggle/input/child-mind-institute-problematic-internet-use/'\noutput_dir = '/kaggle/working/'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:08:42.532244Z","iopub.execute_input":"2024-12-06T06:08:42.532793Z","iopub.status.idle":"2024-12-06T06:08:42.539858Z","shell.execute_reply.started":"2024-12-06T06:08:42.532737Z","shell.execute_reply":"2024-12-06T06:08:42.538592Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(working_dir + 'train.csv')\ndef load_act_id(act_id):\n    return pd.read_parquet(f\"series_train.parquet/id={act_id}\")\n\nactigraphy_list = os.listdir(working_dir + 'series_train.parquet/')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:08:42.542514Z","iopub.execute_input":"2024-12-06T06:08:42.543030Z","iopub.status.idle":"2024-12-06T06:08:42.625272Z","shell.execute_reply.started":"2024-12-06T06:08:42.542978Z","shell.execute_reply":"2024-12-06T06:08:42.624061Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Trim column by sii and pciat\ndf = df.loc[: ,~df.columns.str.contains('season', case=False)]\npciat_columns = [col for col in df.columns if col.split(sep='-')[0] == 'PCIAT']\npciat_columns.append('sii')\ndf = df.dropna(subset=pciat_columns)\ndf = df.dropna(subset='sii')\ndf.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:08:44.194753Z","iopub.execute_input":"2024-12-06T06:08:44.195080Z","iopub.status.idle":"2024-12-06T06:08:44.279747Z","shell.execute_reply.started":"2024-12-06T06:08:44.195048Z","shell.execute_reply":"2024-12-06T06:08:44.278619Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dropping all data with > 40% missing ratio\ndropped_cols = []\ncolumns = df.columns\nfor col in columns:\n    missing_ratio = df[col].isna().sum() / len(df)\n    if missing_ratio > 0.4:\n        df = df.drop(columns=col)\n        dropped_cols.append(col)\n\nprint('Dropped: ')\nprint(dropped_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:08:44.282702Z","iopub.execute_input":"2024-12-06T06:08:44.283068Z","iopub.status.idle":"2024-12-06T06:08:44.313925Z","shell.execute_reply.started":"2024-12-06T06:08:44.283035Z","shell.execute_reply":"2024-12-06T06:08:44.312779Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_columns = [col for col in df.columns if \n              (col.split(sep='-')[0] != 'PCIAT' and col.split(sep='-')[0] != 'sii'\n                  and col != 'id'\n              )\n          ]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:08:44.471659Z","iopub.execute_input":"2024-12-06T06:08:44.472009Z","iopub.status.idle":"2024-12-06T06:08:44.477710Z","shell.execute_reply.started":"2024-12-06T06:08:44.471973Z","shell.execute_reply":"2024-12-06T06:08:44.476583Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# General quantile capping\nfeature_cap = {}\nfor col in num_columns:\n    if col == 'id':\n        continue\n    feature_cap[col] = {\n        'lower': df[col].quantile(0.01),\n        'upper': df[col].quantile(0.99)\n    }\n    df[col] = df[col].clip(df[col].quantile(0.01), df[col].quantile(0.99))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:09:45.983933Z","iopub.execute_input":"2024-12-06T06:09:45.984416Z","iopub.status.idle":"2024-12-06T06:09:46.141951Z","shell.execute_reply.started":"2024-12-06T06:09:45.984366Z","shell.execute_reply":"2024-12-06T06:09:46.140662Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Considering all Physical and BIA data that are <= less than or equal to 0 are errors\n# we will replace it with NaN then impute (Age too)\nphysical_columns = [col for col in df.columns if col.split(sep='-')[0] == 'Physical']\nBIA_columns = [col for col in df.columns if col.split(sep='-')[0] == 'BIA']\ndf[df['Basic_Demos-Age'] == 0] = np.nan\ndf[df[physical_columns + BIA_columns] <= 0] = np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:09:58.339895Z","iopub.execute_input":"2024-12-06T06:09:58.340249Z","iopub.status.idle":"2024-12-06T06:09:58.381571Z","shell.execute_reply.started":"2024-12-06T06:09:58.340215Z","shell.execute_reply":"2024-12-06T06:09:58.380270Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Method 1 Using KNN imputer\nfrom sklearn.impute import KNNImputer\n\n# Divide into 5 cluster corresponding to 5 groups of PCIAT\nimputer = KNNImputer(\n    n_neighbors=5\n    \n)\n\nfeature_columns = [col for col in df.columns if \n                      col not in pciat_columns and\n                      col != 'sii' and\n                      col != 'id'\n                 ]\n\nX = np.array(df[feature_columns])\nX_imputed = imputer.fit_transform(X)\ndf[feature_columns] = X_imputed","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:09:58.413339Z","iopub.execute_input":"2024-12-06T06:09:58.413685Z","iopub.status.idle":"2024-12-06T06:10:00.260863Z","shell.execute_reply.started":"2024-12-06T06:09:58.413653Z","shell.execute_reply":"2024-12-06T06:10:00.259498Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\n\ndef quadradic_kappa(y_true, y_pred):\n    N = 4\n    O = confusion_matrix(y_true, y_pred, labels=[0,1,2,3])\n    \n    W = np.zeros((N, N))\n    for i in range(N):\n        for j in range(N):\n            W[i, j] = ((i - j) ** 2) / ((N - 1) ** 2)\n\n    # Expected matrix E\n    # Normalize the true and predicted histograms\n    actual_hist = np.sum(O, axis=1)\n    predicted_hist = np.sum(O, axis=0)\n    expected = np.outer(actual_hist, predicted_hist) / np.sum(O)\n    \n    # Quadratic Weighted Kappa\n    numerator = np.sum(W * O)\n    denominator = np.sum(W * expected)\n    qwk = 1 - (numerator / denominator)\n\n    return qwk","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:10:00.262729Z","iopub.execute_input":"2024-12-06T06:10:00.263210Z","iopub.status.idle":"2024-12-06T06:10:00.381740Z","shell.execute_reply.started":"2024-12-06T06:10:00.263161Z","shell.execute_reply":"2024-12-06T06:10:00.380453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model 1: Fitting with Random Forest Regressor using PCIAT to infer SII\n# Step 1: Split train/test data\ntest_quantity = 100\nX = df[feature_columns]\nY = df[pciat_columns]\n\nX_train = X[:-test_quantity]\nX_test = X[-test_quantity:]\nY_train = Y[:-test_quantity]\nY_test = Y[-test_quantity:]\n\n# Step 2: Build an ensemble tree for each PCIAT question\npciat_forest = {} # 20 forest for 20 questions, each forest contain 50 to 100 trees\n\nfor question in pciat_columns:\n    if question == 'PCIAT-PCIAT_Total' or question == 'sii':\n        continue\n    \n    forest = RandomForestClassifier(\n        n_estimators=100,\n        criterion='entropy',\n        max_depth=10,\n        random_state=35, # 35 F1 = 0.9 kappa = 89\n        bootstrap=True\n    )\n\n    forest.fit(np.array(X),np.array(np.array(Y[question])))\n    pciat_forest[question] = forest\n\n# Step 3: Testing\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import f1_score\npciat = []\npciat_total = []\nsii = []\ntrue_pciat_total = np.array(Y_test['PCIAT-PCIAT_Total'])\ntrue_sii = np.array(Y_test['sii'])\n\nfor col in pciat_columns:\n    if col == 'PCIAT-PCIAT_Total' or col == 'sii':\n        continue\n    pred = pciat_forest[col].predict(np.array(X_test))\n    pciat.append(pred)\n    actual = np.array(Y_test[col])\n    \npciat_total = np.sum(pciat, axis=0)\nfor value in pciat_total: \n    if value <= 30: \n        sii.append(0)\n    elif 31 <= value <= 49:\n        sii.append(1)\n    elif 50 <= value <= 79:\n        sii.append(2)\n    else:\n        sii.append(3)\n\nprint(f1_score(true_sii, sii, average='macro'))\nplt.figure(figsize=(3,3))\nsns.heatmap(confusion_matrix(true_sii, sii), annot=True, cmap='viridis')","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:10:00.383312Z","iopub.execute_input":"2024-12-06T06:10:00.383710Z","iopub.status.idle":"2024-12-06T06:10:33.742358Z","shell.execute_reply.started":"2024-12-06T06:10:00.383674Z","shell.execute_reply":"2024-12-06T06:10:33.741124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"quadradic_kappa(true_sii, sii)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:10:33.744119Z","iopub.execute_input":"2024-12-06T06:10:33.744619Z","iopub.status.idle":"2024-12-06T06:10:33.756034Z","shell.execute_reply.started":"2024-12-06T06:10:33.744567Z","shell.execute_reply":"2024-12-06T06:10:33.754815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model 1 pipeline\n\n# Load data\ntest_df = pd.read_csv(working_dir + 'test.csv')\npciat_questions = [\n 'PCIAT-PCIAT_01',\n 'PCIAT-PCIAT_02',\n 'PCIAT-PCIAT_03',\n 'PCIAT-PCIAT_04',\n 'PCIAT-PCIAT_05',\n 'PCIAT-PCIAT_06',\n 'PCIAT-PCIAT_07',\n 'PCIAT-PCIAT_08',\n 'PCIAT-PCIAT_09',\n 'PCIAT-PCIAT_10',\n 'PCIAT-PCIAT_11',\n 'PCIAT-PCIAT_12',\n 'PCIAT-PCIAT_13',\n 'PCIAT-PCIAT_14',\n 'PCIAT-PCIAT_15',\n 'PCIAT-PCIAT_16',\n 'PCIAT-PCIAT_17',\n 'PCIAT-PCIAT_18',\n 'PCIAT-PCIAT_19',\n 'PCIAT-PCIAT_20']\n                 \ndef transform(test_df):\n    # Drop text features\n    test_df = test_df.drop(dropped_cols, axis=1)\n    test_df = test_df.loc[: ,~test_df.columns.str.contains('season', case=False)]\n\n    # Get numeric columns\n    num_columns = [col for col in test_df.columns if \n                      (col.split(sep='-')[0] != 'PCIAT' and col.split(sep='-')[0] != 'sii' and col != 'id')\n                  ]\n\n    # Value capping, use specific cap values acquired from training quantile capping\n    for col in num_columns:\n        if col == 'id':\n            continue\n        test_df[col] = test_df[col].clip(feature_cap[col]['lower'], feature_cap[col]['upper'])\n        \n    # Considering all Physical and BIA data that are <= less than or equal to 0 are errors\n    # we will replace it with NaN then impute (Age too)\n    physical_columns = [col for col in test_df.columns if col.split(sep='-')[0] == 'Physical']\n    BIA_columns = [col for col in test_df.columns if col.split(sep='-')[0] == 'BIA']\n    test_df[test_df['Basic_Demos-Age'] == 0] = np.nan\n    test_df[test_df[physical_columns + BIA_columns] <= 0] = np.nan\n\n\n    imputer = KNNImputer(\n        n_neighbors=5\n    )\n    \n    feature_columns = [col for col in test_df.columns if col != 'id']\n    \n    X = np.array(test_df[feature_columns])\n    X_imputed = imputer.fit_transform(X)\n    test_df[feature_columns] = X_imputed\n\n    return test_df\n\ndef predict(test_df):\n    feature_columns = [col for col in test_df.columns if col != 'id']\n\n    X = test_df[feature_columns]\n\n    pciat = []\n    pciat_total = []\n    sii = []\n    \n    for col in pciat_questions:\n        pred = pciat_forest[col].predict(np.array(X))\n        pciat.append(pred)\n        \n    pciat_total = np.sum(pciat, axis=0)\n    for value in pciat_total: \n        if value <= 30: \n            sii.append(0)\n        elif 31 <= value <= 49:\n            sii.append(1)\n        elif 50 <= value <= 79:\n            sii.append(2)\n        else:\n            sii.append(3)\n    return sii\n\ntest_df = transform(test_df)\npred = predict(test_df)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:10:33.758074Z","iopub.execute_input":"2024-12-06T06:10:33.758482Z","iopub.status.idle":"2024-12-06T06:10:33.986185Z","shell.execute_reply.started":"2024-12-06T06:10:33.758445Z","shell.execute_reply":"2024-12-06T06:10:33.984443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output = pd.DataFrame({'id': test_df.id, 'sii': pred})\noutput.to_csv(output_dir + 'submission.csv', sep=',', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:10:33.987775Z","iopub.execute_input":"2024-12-06T06:10:33.988276Z","iopub.status.idle":"2024-12-06T06:10:33.998403Z","shell.execute_reply.started":"2024-12-06T06:10:33.988227Z","shell.execute_reply":"2024-12-06T06:10:33.996945Z"}},"outputs":[],"execution_count":null}]}