{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:46.476159Z","iopub.execute_input":"2024-12-05T11:13:46.476553Z","iopub.status.idle":"2024-12-05T11:13:47.010261Z","shell.execute_reply.started":"2024-12-05T11:13:46.47652Z","shell.execute_reply":"2024-12-05T11:13:47.008921Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\")\n\n\ntrain.isna().sum().to_frame().sort_values(0, ascending=False) / len(train) * 100\nprint(train.columns)  # Cek nama kolom yang ada\npciat_cols = ['PCIAT_Total', 'sii']  # Pastikan nama kolom sesuai yang ada\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.012725Z","iopub.execute_input":"2024-12-05T11:13:47.013196Z","iopub.status.idle":"2024-12-05T11:13:47.068187Z","shell.execute_reply.started":"2024-12-05T11:13:47.013148Z","shell.execute_reply":"2024-12-05T11:13:47.066879Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temp_things = [train.columns, \n               train[\"sii\"].isna().sum() / len(train)    \n    ]\n\nfor thing in temp_things:\n    print(thing)\n    print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.069711Z","iopub.execute_input":"2024-12-05T11:13:47.070206Z","iopub.status.idle":"2024-12-05T11:13:47.079176Z","shell.execute_reply.started":"2024-12-05T11:13:47.070158Z","shell.execute_reply":"2024-12-05T11:13:47.077923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt  # Pastikan ini sudah ada di awal notebook jika belum ada\n\n# Periksa nama kolom yang ada di DataFrame 'train'\nprint(train.columns)  # Menampilkan kolom yang ada di DataFrame\n\n# Sesuaikan dengan nama kolom yang benar\npciat_cols = ['PCIAT', 'sii']  # Sesuaikan dengan kolom yang benar\n\n# Pastikan kolom PCIAT dan sii ada di dalam DataFrame 'train'\nif all(col in train.columns for col in pciat_cols):\n    # Pilih kolom dari DataFrame 'train' menggunakan pciat_cols\n    pciat_train = train[pciat_cols]\n    \n    # Hapus baris dengan nilai kosong pada kolom PCIAT dan sii\n    pciat_train_notna = pciat_train.dropna(subset=pciat_cols)\n    \n    # Plot\n    fig, ax = plt.subplots()\n    ax.scatter(pciat_train_notna['PCIAT'], pciat_train_notna['sii'])\n    \n    ax.set_xlabel('PCIAT')\n    ax.set_ylabel('sii')\n    ax.set_title('Plot PCIAT vs sii')\n    \n    plt.show()\nelse:\n    print(\"Kolom 'PCIAT' dan 'sii' tidak ditemukan di DataFrame.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.080585Z","iopub.execute_input":"2024-12-05T11:13:47.080974Z","iopub.status.idle":"2024-12-05T11:13:47.097671Z","shell.execute_reply.started":"2024-12-05T11:13:47.08094Z","shell.execute_reply":"2024-12-05T11:13:47.096027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt  # Pastikan ini sudah ada di awal notebook jika belum ada\n\n# Periksa nama kolom yang ada di DataFrame 'train'\nprint(train.columns)  # Menampilkan kolom yang ada di DataFrame\n\n# Sesuaikan dengan nama kolom yang benar\npciat_cols = ['PCIAT', 'sii']  # Sesuaikan dengan kolom yang benar\n\n# Pastikan kolom PCIAT dan sii ada di dalam DataFrame 'train'\nif all(col in train.columns for col in pciat_cols):\n    # Pilih kolom dari DataFrame 'train' menggunakan pciat_cols\n    pciat_train = train[pciat_cols]\n    \n    # Hapus baris dengan nilai kosong pada kolom PCIAT dan sii\n    pciat_train_notna = pciat_train.dropna(subset=pciat_cols)\n    \n    # Plot\n    fig, ax = plt.subplots()\n    ax.scatter(pciat_train_notna['PCIAT'], pciat_train_notna['sii'])\n    \n    ax.set_xlabel('PCIAT')\n    ax.set_ylabel('sii')\n    ax.set_title('Plot PCIAT vs sii')\n    \n    plt.show()\n    \n    # Operasi groupby dan max\n    grouped_data = pciat_train_notna.groupby(\"sii\").max()\n    print(grouped_data)\nelse:\n    print(\"Kolom 'PCIAT' dan 'sii' tidak ditemukan di DataFrame.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.101811Z","iopub.execute_input":"2024-12-05T11:13:47.102336Z","iopub.status.idle":"2024-12-05T11:13:47.118645Z","shell.execute_reply.started":"2024-12-05T11:13:47.102283Z","shell.execute_reply":"2024-12-05T11:13:47.117376Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Pastikan nama kolom yang relevan ada\npciat_cols = ['PCIAT', 'sii', 'PCIAT-PCIAT_Total']  # Sesuaikan dengan kolom yang benar\n\n# Periksa apakah kolom yang dibutuhkan ada dalam DataFrame 'train'\nif all(col in train.columns for col in pciat_cols):\n    # Pilih kolom yang diperlukan\n    pciat_train = train[pciat_cols]\n    \n    # Menangani NaN values: misalnya, dengan mengganti NaN dengan 0 atau nilai lain yang sesuai\n    pciat_train_filled = pciat_train.fillna(0)  # Mengganti NaN dengan 0, bisa disesuaikan dengan kebutuhan\n    \n    # Pastikan skor PCIAT dijumlahkan dengan benar\n    recalculated_pciat = pciat_train_filled.drop(columns=[\"PCIAT-PCIAT_Total\", \"sii\"]).sum(axis=1)\n    \n    # Hitung selisih antara total PCIAT dan skor yang dihitung ulang\n    diff = pciat_train_filled[\"PCIAT-PCIAT_Total\"] - recalculated_pciat\n    print(f\"Sum of differences (should be 0 if correct): {diff.sum()}\")\n    \nelse:\n    print(\"Kolom yang dibutuhkan tidak ditemukan di DataFrame.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.120487Z","iopub.execute_input":"2024-12-05T11:13:47.121Z","iopub.status.idle":"2024-12-05T11:13:47.134979Z","shell.execute_reply.started":"2024-12-05T11:13:47.12095Z","shell.execute_reply":"2024-12-05T11:13:47.13368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# see how many rows of all NaN values\nlen(pciat_train[pciat_train.isna().all(axis=1)]) / len(pciat_train)\n\n# 1224 rows, 30% of data, that's a lot! likely needs an unsupervised learning # TODO","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.13623Z","iopub.execute_input":"2024-12-05T11:13:47.136562Z","iopub.status.idle":"2024-12-05T11:13:47.155615Z","shell.execute_reply.started":"2024-12-05T11:13:47.136532Z","shell.execute_reply":"2024-12-05T11:13:47.154188Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pciat_train[~pciat_train.isna().all(axis=1)].isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.15719Z","iopub.execute_input":"2024-12-05T11:13:47.157574Z","iopub.status.idle":"2024-12-05T11:13:47.171552Z","shell.execute_reply.started":"2024-12-05T11:13:47.157527Z","shell.execute_reply":"2024-12-05T11:13:47.170362Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Menghapus baris yang seluruh kolomnya NaN\nnon_missing_rows = pciat_train.loc[~pciat_train.isna().all(axis=1)]\n\n# Menyaring baris yang memiliki setidaknya satu kolom NaN\ntemp = non_missing_rows.loc[non_missing_rows.isna().any(axis=1)]\n\n# Menampilkan hasil\nprint(temp)\n\n# Jika temp masih kosong, tambahkan pemeriksaan berikut\nif temp.empty:\n    print(\"Tidak ada baris yang memenuhi kedua kondisi (tidak semua NaN dan ada beberapa NaN).\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.173377Z","iopub.execute_input":"2024-12-05T11:13:47.173784Z","iopub.status.idle":"2024-12-05T11:13:47.201078Z","shell.execute_reply.started":"2024-12-05T11:13:47.173748Z","shell.execute_reply":"2024-12-05T11:13:47.199535Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.reset_option(\"display.max_rows\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.203063Z","iopub.execute_input":"2024-12-05T11:13:47.203583Z","iopub.status.idle":"2024-12-05T11:13:47.209623Z","shell.execute_reply.started":"2024-12-05T11:13:47.203513Z","shell.execute_reply":"2024-12-05T11:13:47.208367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# consider two extremes:\n# for a given row with some partial NaN values, if we replace the NaN with the scores of 5 and add them all\n# if the total score now is still within the predefined thresholds, it means the sii still holds\n# if it's larger, then we now just don't know and can't assume anything, it must be NaN for sii score\n# we will deal with NaN sii scores later \n\n# I learned this from https://www.kaggle.com/code/antoninadolgorukova/cmi-piu-features-eda?scriptVersionId=206130660&cellId=32\n\n# Jika temp adalah subset dari DataFrame asli, buat salinan untuk mencegah peringatan\ntemp = temp.copy()\n\n# Tambahkan kolom recalc_sii menggunakan fungsi apply\ntemp['recalc_sii'] = temp.apply(recalculate_sii, axis=1)\n\n# Periksa hasil\nprint(temp)\n\ndef recalculate_sii(row):\n    responses20_cols = [col for col in train.columns if col.startswith(\"PCIAT-PCIAT\") and col != \"PCIAT-PCIAT_Total\"]\n    \n    if pd.isna(row['PCIAT-PCIAT_Total']):\n        return np.nan\n    max_possible = row['PCIAT-PCIAT_Total'] + row[responses20_cols].isna().sum() * 5\n    if row['PCIAT-PCIAT_Total'] <= 30 and max_possible <= 30:\n        return 0\n    elif 31 <= row['PCIAT-PCIAT_Total'] <= 49 and max_possible <= 49:\n        return 1\n    elif 50 <= row['PCIAT-PCIAT_Total'] <= 79 and max_possible <= 79:\n        return 2\n    elif row['PCIAT-PCIAT_Total'] >= 80 and max_possible >= 80:\n        return 3\n    return np.nan\n\ntemp['recalc_sii'] = temp.apply(recalculate_sii, axis=1)\n\nlen(temp), temp['recalc_sii'].isna().sum()\n\n# only 17 rows out of 65 that are now having missing sii scores (not including initially NaN sii rows)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.211509Z","iopub.execute_input":"2024-12-05T11:13:47.212036Z","iopub.status.idle":"2024-12-05T11:13:47.331015Z","shell.execute_reply.started":"2024-12-05T11:13:47.211973Z","shell.execute_reply":"2024-12-05T11:13:47.329716Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.reset_option(\"display.max_rows\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.332197Z","iopub.execute_input":"2024-12-05T11:13:47.33249Z","iopub.status.idle":"2024-12-05T11:13:47.337648Z","shell.execute_reply.started":"2024-12-05T11:13:47.332462Z","shell.execute_reply":"2024-12-05T11:13:47.336572Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[\"sii\"].isna().sum() / len(train) # after preprocessing ssi, there is a bit more NaN","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.339004Z","iopub.execute_input":"2024-12-05T11:13:47.339391Z","iopub.status.idle":"2024-12-05T11:13:47.352111Z","shell.execute_reply.started":"2024-12-05T11:13:47.339359Z","shell.execute_reply":"2024-12-05T11:13:47.350904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.356382Z","iopub.execute_input":"2024-12-05T11:13:47.356731Z","iopub.status.idle":"2024-12-05T11:13:47.395797Z","shell.execute_reply.started":"2024-12-05T11:13:47.356697Z","shell.execute_reply":"2024-12-05T11:13:47.394606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# labels distribution\nfig, ax = plt.subplots()\ncounts = train[\"sii\"].value_counts(dropna=False).sort_index()\nax.bar(counts.index.astype(str), counts.values)\ncounts\n\n# mostly no severity, but still lots of missing sii","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.397119Z","iopub.execute_input":"2024-12-05T11:13:47.397446Z","iopub.status.idle":"2024-12-05T11:13:47.583881Z","shell.execute_reply.started":"2024-12-05T11:13:47.397415Z","shell.execute_reply":"2024-12-05T11:13:47.582567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def transform_tabular(data):\n    # drop the PCIAT-PCIAT columns to avoid potential data leakage\n    # for train mostly\n    responses20_cols = [col for col in train.columns if col.startswith(\"PCIAT\")]\n    data = data.drop(columns=responses20_cols)\n\n    return data\n\ntrain = transform_tabular(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.585462Z","iopub.execute_input":"2024-12-05T11:13:47.585784Z","iopub.status.idle":"2024-12-05T11:13:47.594163Z","shell.execute_reply.started":"2024-12-05T11:13:47.585754Z","shell.execute_reply":"2024-12-05T11:13:47.592746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"###\n# check column types\ncats = [col for col in train.columns if train[col].dtype == \"object\"]\ncons = [col for col in train.columns if train[col].dtype != \"object\"]\ncats, cons","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.595744Z","iopub.execute_input":"2024-12-05T11:13:47.596223Z","iopub.status.idle":"2024-12-05T11:13:47.61854Z","shell.execute_reply.started":"2024-12-05T11:13:47.596177Z","shell.execute_reply":"2024-12-05T11:13:47.617267Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# imputation\n\n# first, let's focus on continuous vars\n\n# these columns seem to have no missing data, we can use them as group matching for imputing continuous vars? \n# hmm let's try\ntrain[[\"Basic_Demos-Enroll_Season\", \"Basic_Demos-Age\", \"Basic_Demos-Sex\"]].isna().sum()\n# but I don't use \"Basic_Demos-Enroll_Season\" because I expect every season would observe some \n# similar people within a certain group of age and sex, not including it would help me have a \n# more broader groups  by age and sex","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.620229Z","iopub.execute_input":"2024-12-05T11:13:47.620605Z","iopub.status.idle":"2024-12-05T11:13:47.632423Z","shell.execute_reply.started":"2024-12-05T11:13:47.620569Z","shell.execute_reply":"2024-12-05T11:13:47.63124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# try on Physical-Height\ncols_no_missing = [\"Basic_Demos-Age\", \"Basic_Demos-Sex\"]\ngroup_means_draft = train.groupby(cols_no_missing)[\"Physical-Height\"].agg(\"mean\")\ngroup_means_draft\n\n# it looks like group of 22-yo and male only has one person and that person's data is NaN so I replace\n# it with data from male person but 21-yo (mean)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.633946Z","iopub.execute_input":"2024-12-05T11:13:47.634393Z","iopub.status.idle":"2024-12-05T11:13:47.649238Z","shell.execute_reply.started":"2024-12-05T11:13:47.634344Z","shell.execute_reply":"2024-12-05T11:13:47.648004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_group_means(data, cols_no_missing, cons):\n    \"\"\"\n    Calculate group means for continuous variables based on given grouping columns.\n    \"\"\"\n    group_means_dict = {}\n    too_many_missing = []\n    \n    for col in cons:\n        if col not in cols_no_missing:\n            group_means = data.groupby(cols_no_missing)[col].mean()\n            group_means_dict[col] = group_means\n\n            data[col] = data.groupby(cols_no_missing)[col].transform(\"mean\")\n            \n            if data[col].isna().sum() > 10:\n                too_many_missing.append(col)\n    \n    return group_means_dict, too_many_missing\n\n\ndef impute_remaining_values(data, group_means_dict, selected_cons):\n    \"\"\"\n    Impute remaining missing values in the group means dictionary.\n    \"\"\"\n    for col, group_means in group_means_dict.items():\n        if col in selected_cons:\n            overall_mean = data[col].mean()\n            group_means.fillna(overall_mean, inplace=True)\n\n    return data\n\n\n# calculate group means and identify columns with too many missing values\ncols_no_missing = [\"Basic_Demos-Age\", \"Basic_Demos-Sex\"]\ntransformed_cons_train = train.copy()\ngroup_means_dict, too_many_missing = calculate_group_means(transformed_cons_train, cols_no_missing, cons)\n\n# filter selected continuous variables\nselected_cons = [col for col in cons if col not in too_many_missing]\n\n# impute missing values in the group means dictionary\nimpute_remaining_values(transformed_cons_train, group_means_dict, selected_cons)\n\n\nfor col, group_means in group_means_dict.items():\n    transformed_cons_train[col] = transformed_cons_train.apply(\n        lambda row: group_means.get((row[\"Basic_Demos-Age\"], row[\"Basic_Demos-Sex\"]), row[col])\n        if pd.isna(row[col]) else row[col],\n        axis=1\n    )\n\ntransformed_cons_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:47.650887Z","iopub.execute_input":"2024-12-05T11:13:47.651289Z","iopub.status.idle":"2024-12-05T11:13:50.113493Z","shell.execute_reply.started":"2024-12-05T11:13:47.651245Z","shell.execute_reply":"2024-12-05T11:13:50.112252Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transformed_cons_train[transformed_cons_train[selected_cons].isna().any(axis=1)]\n# this one record is all NaN => but can't drop, because in the hidden test set might have it","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.114921Z","iopub.execute_input":"2024-12-05T11:13:50.115247Z","iopub.status.idle":"2024-12-05T11:13:50.142802Z","shell.execute_reply.started":"2024-12-05T11:13:50.115216Z","shell.execute_reply":"2024-12-05T11:13:50.141622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# transformed_cons_train = transformed_cons_train[transformed_cons_train[\"id\"] != \"3cb2c4da\"]\n\ntransformed_cons_train[\"sii\"] = transformed_cons_train[\"sii\"].apply(lambda x: round(x))\n\ntransformed_cons_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.144613Z","iopub.execute_input":"2024-12-05T11:13:50.145263Z","iopub.status.idle":"2024-12-05T11:13:50.157229Z","shell.execute_reply.started":"2024-12-05T11:13:50.145211Z","shell.execute_reply":"2024-12-05T11:13:50.155904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"only_transformed_cons = [col for col in selected_cons if transformed_cons_train[col].dtype != \"object\"]\nonly_transformed_cons_train = transformed_cons_train[only_transformed_cons]\n\nonly_transformed_cons_train.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.159161Z","iopub.execute_input":"2024-12-05T11:13:50.159634Z","iopub.status.idle":"2024-12-05T11:13:50.181308Z","shell.execute_reply.started":"2024-12-05T11:13:50.159594Z","shell.execute_reply":"2024-12-05T11:13:50.180136Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = only_transformed_cons_train.drop(columns=[\"sii\"])\ny = only_transformed_cons_train[\"sii\"]\nX.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.183142Z","iopub.execute_input":"2024-12-05T11:13:50.183641Z","iopub.status.idle":"2024-12-05T11:13:50.196101Z","shell.execute_reply.started":"2024-12-05T11:13:50.18359Z","shell.execute_reply":"2024-12-05T11:13:50.194625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.decomposition import PCA  # Tambahkan impor untuk PCA\n\n# Normalisasi data\nX_scaled = pd.DataFrame(preprocessing.scale(X), columns=X.columns)\n\n# PCA\npca = PCA(n_components=3)\n\n# normalize data\nX_scaled = pd.DataFrame(preprocessing.scale(X),columns = X.columns) \n\n# PCA\npca = PCA(n_components=3)\npca.fit_transform(X_scaled)\n\nPCS = ['PC1','PC2', 'PC3'] #, 'PC4', 'PC5']\n\npc_data = pd.DataFrame(pca.components_,columns=X_scaled.columns,\n             index = PCS)\npc_data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.197507Z","iopub.execute_input":"2024-12-05T11:13:50.197998Z","iopub.status.idle":"2024-12-05T11:13:50.265031Z","shell.execute_reply.started":"2024-12-05T11:13:50.197946Z","shell.execute_reply":"2024-12-05T11:13:50.26394Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pc_data = pc_data.T.reset_index().rename(columns={\"index\": \"Field\"})\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.26619Z","iopub.execute_input":"2024-12-05T11:13:50.266574Z","iopub.status.idle":"2024-12-05T11:13:50.276206Z","shell.execute_reply.started":"2024-12-05T11:13:50.266534Z","shell.execute_reply":"2024-12-05T11:13:50.273212Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Memeriksa kolom yang ada dalam DataFrame\nprint(pc_data_info.columns)\n\n# Memastikan kolom PCA ada sebelum menerapkan fungsi abs\npca_columns = ['PC1', 'PC2', 'PC3']  # Kolom PCA yang diinginkan\nexisting_pca_columns = [col for col in pca_columns if col in pc_data_info.columns]\n\n# Terapkan abs hanya pada kolom yang ada\nif existing_pca_columns:\n    pc_data_info[existing_pca_columns] = pc_data_info[existing_pca_columns].apply(lambda x: abs(x))\nelse:\n    print(f\"Kolom PCA berikut tidak ditemukan: {set(pca_columns) - set(pc_data_info.columns)}\")\n\n# Tampilkan DataFrame untuk memeriksa hasil\nprint(pc_data_info.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.277313Z","iopub.execute_input":"2024-12-05T11:13:50.277681Z","iopub.status.idle":"2024-12-05T11:13:50.306068Z","shell.execute_reply.started":"2024-12-05T11:13:50.277642Z","shell.execute_reply":"2024-12-05T11:13:50.304531Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Memeriksa kolom yang ada dalam pc_data_info\nprint(pc_data_info.columns)\n\n# Pastikan kolom 'PC1' ada, jika ada, lanjutkan untuk menampilkan top 10\nif 'PC1' in pc_data_info.columns:\n    top_10_pc1 = pc_data_info[[\"Field\", \"PC1\"]].sort_values(\"PC1\", ascending=False).head(10)\n    print(top_10_pc1)\nelse:\n    print(\"Kolom 'PC1' tidak ditemukan dalam data\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.307344Z","iopub.execute_input":"2024-12-05T11:13:50.308797Z","iopub.status.idle":"2024-12-05T11:13:50.324551Z","shell.execute_reply.started":"2024-12-05T11:13:50.308745Z","shell.execute_reply":"2024-12-05T11:13:50.321139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Memeriksa kolom yang ada dalam pc_data_info\nprint(pc_data_info.columns)\n\n# Pastikan kolom 'PC2' ada, jika ada, lanjutkan untuk menampilkan top 10\nif 'PC2' in pc_data_info.columns:\n    # Menghapus baris dengan nilai NaN pada PC2\n    pc_data_info_cleaned = pc_data_info.dropna(subset=[\"PC2\"])\n    \n    # Sorting dan mengambil 10 besar nilai PC2\n    top_10_pc2 = pc_data_info_cleaned[[\"Field\", \"PC2\"]].sort_values(\"PC2\", ascending=False).head(10)\n    \n    print(top_10_pc2)\nelse:\n    print(\"Kolom 'PC2' tidak ditemukan dalam data\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.325909Z","iopub.execute_input":"2024-12-05T11:13:50.326322Z","iopub.status.idle":"2024-12-05T11:13:50.347446Z","shell.execute_reply.started":"2024-12-05T11:13:50.326282Z","shell.execute_reply":"2024-12-05T11:13:50.3439Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Memeriksa kolom yang ada dalam pc_data_info\nprint(pc_data_info.columns)\n\n# Pastikan kolom 'PC3' ada, jika ada, lanjutkan untuk menampilkan top 10\nif 'PC3' in pc_data_info.columns:\n    # Menghapus baris dengan nilai NaN pada PC3\n    pc_data_info_cleaned = pc_data_info.dropna(subset=[\"PC3\"])\n    \n    # Sorting dan mengambil 10 besar nilai PC3\n    top_10_pc3 = pc_data_info_cleaned[[\"Field\", \"PC3\"]].sort_values(\"PC3\", ascending=False).head(10)\n    \n    print(top_10_pc3)\nelse:\n    print(\"Kolom 'PC3' tidak ditemukan dalam data\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.348646Z","iopub.execute_input":"2024-12-05T11:13:50.349081Z","iopub.status.idle":"2024-12-05T11:13:50.362263Z","shell.execute_reply.started":"2024-12-05T11:13:50.349034Z","shell.execute_reply":"2024-12-05T11:13:50.360912Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"picked_cons = [\"BIA-BIA_LST\", \n               \"Physical-Weight\", \"BIA-BIA_BMI\", \"Physical-Waist_Circumference\", \n               \"SDS-SDS_Total_T\", \"SDS-SDS_Total_Raw\"\n              ]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.363969Z","iopub.execute_input":"2024-12-05T11:13:50.3643Z","iopub.status.idle":"2024-12-05T11:13:50.372686Z","shell.execute_reply.started":"2024-12-05T11:13:50.364268Z","shell.execute_reply":"2024-12-05T11:13:50.371303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"filtered_group_means_dict = {key: value for key, value in group_means_dict.items() if key in picked_cons}\nfiltered_group_means_dict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.374616Z","iopub.execute_input":"2024-12-05T11:13:50.37501Z","iopub.status.idle":"2024-12-05T11:13:50.398226Z","shell.execute_reply.started":"2024-12-05T11:13:50.374974Z","shell.execute_reply":"2024-12-05T11:13:50.397116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = only_transformed_cons_train[picked_cons]\ny = y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.400068Z","iopub.execute_input":"2024-12-05T11:13:50.400423Z","iopub.status.idle":"2024-12-05T11:13:50.411176Z","shell.execute_reply.started":"2024-12-05T11:13:50.400391Z","shell.execute_reply":"2024-12-05T11:13:50.409465Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#  modeling/training\nX","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.412907Z","iopub.execute_input":"2024-12-05T11:13:50.414494Z","iopub.status.idle":"2024-12-05T11:13:50.438775Z","shell.execute_reply.started":"2024-12-05T11:13:50.414437Z","shell.execute_reply":"2024-12-05T11:13:50.437341Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.metrics import classification_report\n\n# Misalkan X dan y sudah didefinisikan sebagai fitur dan target variabel Anda\n# X = ... (fitur)\n# y = ... (target)\n\n# Split data menjadi training dan validation\nX_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=123)\n\n# Membuat model Decision Tree\ndt_model = DecisionTreeClassifier(max_depth=3, random_state=123) \n\n# Melatih model dengan data training\ndt_model.fit(X_train, y_train)\n\n# Melakukan prediksi pada data validasi\ny_pred = dt_model.predict(X_valid)\n\n# Menghitung dan menampilkan classification report\nreport = classification_report(y_valid, y_pred, output_dict=True)\nprint(classification_report(y_valid, y_pred))\n\n# Untuk akses lebih lanjut ke nilai metrik\nprecision = report['0']['precision'], report['1']['precision']\nrecall = report['0']['recall'], report['1']['recall']\nf1_score = report['0']['f1-score'], report['1']['f1-score']\nsupport = report['0']['support'], report['1']['support']\n\n# Menampilkan precision, recall, f1-score, dan support untuk setiap kelas\nprint(f\"Precision untuk kelas 0: {precision[0]}\")\nprint(f\"Precision untuk kelas 1: {precision[1]}\")\nprint(f\"Recall untuk kelas 0: {recall[0]}\")\nprint(f\"Recall untuk kelas 1: {recall[1]}\")\nprint(f\"F1-Score untuk kelas 0: {f1_score[0]}\")\nprint(f\"F1-Score untuk kelas 1: {f1_score[1]}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.440659Z","iopub.execute_input":"2024-12-05T11:13:50.441084Z","iopub.status.idle":"2024-12-05T11:13:50.478805Z","shell.execute_reply.started":"2024-12-05T11:13:50.441046Z","shell.execute_reply":"2024-12-05T11:13:50.477531Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\")\n# there are some rows that have almost no data at all!\n\n\nfor col in picked_cons:\n    if col != \"sii\" and col != \"Basic_Demos-Age\" and col != \"Basic_Demos-Sex\":\n        test[col] = test.apply(\n            lambda row: filtered_group_means_dict[col].get((row[\"Basic_Demos-Age\"], row[\"Basic_Demos-Sex\"]), np.nan)\n            if pd.isna(row[col]) else row[col],\n            axis=1\n        )\n\nX_test = test[picked_cons]\nX_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.480067Z","iopub.execute_input":"2024-12-05T11:13:50.480372Z","iopub.status.idle":"2024-12-05T11:13:50.515007Z","shell.execute_reply.started":"2024-12-05T11:13:50.480343Z","shell.execute_reply":"2024-12-05T11:13:50.513678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_model.predict(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.516378Z","iopub.execute_input":"2024-12-05T11:13:50.516726Z","iopub.status.idle":"2024-12-05T11:13:50.526122Z","shell.execute_reply.started":"2024-12-05T11:13:50.516681Z","shell.execute_reply":"2024-12-05T11:13:50.524674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"id\": test[\"id\"],\n    \"sii\": dt_model.predict(X_test)\n})\nsubmission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.527577Z","iopub.execute_input":"2024-12-05T11:13:50.52796Z","iopub.status.idle":"2024-12-05T11:13:50.547328Z","shell.execute_reply.started":"2024-12-05T11:13:50.527913Z","shell.execute_reply":"2024-12-05T11:13:50.546089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.54864Z","iopub.execute_input":"2024-12-05T11:13:50.549017Z","iopub.status.idle":"2024-12-05T11:13:50.564167Z","shell.execute_reply.started":"2024-12-05T11:13:50.548977Z","shell.execute_reply":"2024-12-05T11:13:50.562919Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mine = pd.read_csv(\"/kaggle/working/submission.csv\")\nmine","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.565768Z","iopub.execute_input":"2024-12-05T11:13:50.566245Z","iopub.status.idle":"2024-12-05T11:13:50.581794Z","shell.execute_reply.started":"2024-12-05T11:13:50.566197Z","shell.execute_reply":"2024-12-05T11:13:50.580643Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in filtered_group_means_dict.values():\n    print(type(i))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T11:13:50.587646Z","iopub.execute_input":"2024-12-05T11:13:50.588081Z","iopub.status.idle":"2024-12-05T11:13:50.593909Z","shell.execute_reply.started":"2024-12-05T11:13:50.588042Z","shell.execute_reply":"2024-12-05T11:13:50.592779Z"}},"outputs":[],"execution_count":null}]}