{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:39:27.314605Z","iopub.execute_input":"2024-12-05T09:39:27.315979Z","iopub.status.idle":"2024-12-05T09:39:29.019928Z","shell.execute_reply.started":"2024-12-05T09:39:27.315929Z","shell.execute_reply":"2024-12-05T09:39:29.018833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np  # Untuk operasi numerik dan manipulasi array\nimport pandas as pd  # Untuk pengolahan data berbasis DataFrame\nimport matplotlib.pyplot as plt  # Untuk membuat grafik dan visualisasi\nimport os  # Untuk berinteraksi dengan sistem file dan direktori\nimport pyarrow.parquet as pq  # Untuk membaca dan menulis file dengan format Parquet\nfrom sklearn.cluster import KMeans  # Algoritma clustering K-Means\nfrom sklearn.preprocessing import MinMaxScaler  # Untuk normalisasi skala data\nfrom sklearn.metrics import accuracy_score  # Untuk menghitung akurasi model\nimport lightgbm as lgb  # Framework untuk algoritma gradient boosting\nfrom sklearn.model_selection import train_test_split  # Untuk membagi dataset menjadi data latih dan uji\ndef preprocessing(name):\n    df = pd.read_csv(name)\n    # Your preprocessing logic here\n    return df\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:39:29.022039Z","iopub.execute_input":"2024-12-05T09:39:29.023111Z","iopub.status.idle":"2024-12-05T09:39:29.028945Z","shell.execute_reply.started":"2024-12-05T09:39:29.023060Z","shell.execute_reply":"2024-12-05T09:39:29.027800Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_dataset(filepath):\n    # Membaca file CSV ke dalam DataFrame\n    data = pd.read_csv(filepath)\n    \n    # Menghapus kolom yang diawali dengan \"PCIAT\"\n    pciat_cols = [col for col in data.columns if col.startswith('PCIAT')]\n    data.drop(columns=pciat_cols, inplace=True)\n    \n    # Menangani label 'sii' jika file adalah dataset pelatihan\n    if filepath.endswith('train.csv'):\n        labels = data.pop('sii')\n    else:\n        labels = None  # Tidak ada label untuk dataset pengujian\n    \n    # Menyimpan kolom 'id' ke DataFrame terpisah dan menghapus dari dataset utama\n    ids = pd.DataFrame({'id': data.pop('id')})\n    \n    # One-hot encoding untuk kolom kategorikal tertentu\n    data.drop(columns=['Basic_Demos-Enroll_Season', 'PAQ_C-Season', 'PAQ_A-Season'], inplace=True)\n    categorical_features = data.select_dtypes(include=['object']).columns\n    data = pd.get_dummies(data, columns=categorical_features)\n    \n    # Mengelompokkan data berdasarkan umur dan jenis kelamin\n    group_by_age_sex = data.groupby(['Basic_Demos-Age', 'Basic_Demos-Sex'])\n    \n    # Mengisi nilai kosong dengan nilai median dari kelompok\n    columns_to_fill_median = [\n        'Physical-Diastolic_BP', 'Physical-Systolic_BP', \n        'Physical-Height', 'Physical-Weight', \n        'Physical-Waist_Circumference', 'Physical-HeartRate'\n    ]\n    for col in columns_to_fill_median:\n        data[col].fillna(group_by_age_sex[col].transform('median'), inplace=True)\n    \n    # Menghitung nilai BMI jika kosong\n    data['Physical-BMI'] = data['Physical-BMI'].fillna(703 * data['Physical-Weight'] / data['Physical-Height'] ** 2)\n    \n    # Mengisi nilai kosong untuk kolom tambahan berdasarkan median kelompok\n    additional_columns = [\n        'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n        'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE',\n        'BIA-BIA_ECW', 'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI',\n        'BIA-BIA_Fat', 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST',\n        'BIA-BIA_SMM', 'BIA-BIA_TBW'\n    ]\n    for col in additional_columns:\n        data[col].fillna(group_by_age_sex[col].transform('median'), inplace=True)\n    \n    # Membuat kolom 'Fitness_Endurance-Time' sebagai gabungan menit dan detik\n    data['Fitness_Endurance-Time'] = data['Fitness_Endurance-Time_Mins'] * 60 + data['Fitness_Endurance-Time_Sec']\n    \n    # Menggabungkan nilai PAQ_A dan PAQ_C menjadi kolom 'PAQ_Total'\n    data['PAQ_Total'] = data['PAQ_A-PAQ_A_Total'].fillna(data['PAQ_C-PAQ_C_Total'])\n    data['PAQ_Total'].fillna(group_by_age_sex['PAQ_Total'].transform('median'), inplace=True)\n    data.drop(columns=['PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total'], inplace=True)\n    \n    # Mengisi nilai kosong pada kolom lain berdasarkan mode atau median\n    data['PreInt_EduHx-computerinternet_hoursday'].fillna(\n        data['PreInt_EduHx-computerinternet_hoursday'].mode()[0], inplace=True\n    )\n    data['CGAS-CGAS_Score'].fillna(group_by_age_sex['CGAS-CGAS_Score'].transform('median'), inplace=True)\n    \n    # Menghapus kolom yang tidak relevan\n    columns_to_remove = [\n        'Physical-BMI', 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n        'FGC-FGC_CU_Zone', 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD_Zone',\n        'FGC-FGC_PU_Zone', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL_Zone'\n    ]\n    data.drop(columns=columns_to_remove, inplace=True)\n    \n    return data, ids, labels\n\n\"\"\"\nDeskripsi: Fungsi ini bertujuan untuk membersihkan dan memproses dataset. \nProses meliputi penghapusan kolom tertentu, pengisian nilai kosong, \none-hot encoding, dan pembuatan kolom baru.\n\nInput:\n    filepath: Lokasi file dataset (string).\n    \nOutput:\n    data: Dataset setelah diproses.\n    ids: DataFrame berisi kolom 'id'.\n    labels: Label data jika tersedia (untuk dataset pelatihan).\n\"\"\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:39:29.030332Z","iopub.execute_input":"2024-12-05T09:39:29.030721Z","iopub.status.idle":"2024-12-05T09:39:29.049741Z","shell.execute_reply.started":"2024-12-05T09:39:29.030690Z","shell.execute_reply":"2024-12-05T09:39:29.048770Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def semi_supervised_model(data, labels, num_leaves=61, learning_rate=0.01):\n    # Bobot kelas untuk menangani ketidakseimbangan data\n    class_weights = [1, 12, 24, 300]\n    \n    # Langkah 1: Melatih model menggunakan subset data yang berlabel (80% untuk pelatihan, 20% untuk validasi)\n    labeled_data = data[labels.notnull()]\n    labeled_labels = labels[labels.notnull()]\n    X_train, X_val, y_train, y_val = train_test_split(labeled_data, labeled_labels, test_size=0.2, random_state=42)\n    \n    train_data = lgb.Dataset(\n        X_train, \n        label=y_train, \n        params={'use_missing': True}, \n        weight=np.array([class_weights[int(y)] for y in y_train])\n    )\n    val_data = lgb.Dataset(\n        X_val, \n        label=y_val, \n        weight=np.array([class_weights[int(y)] for y in y_val]), \n        reference=train_data\n    )\n    \n    # Parameter untuk model LightGBM\n    params = {\n        'objective': 'multiclass',\n        'num_class': 4,\n        'metric': 'multi_logloss',\n        'boosting_type': 'goss',\n        'num_leaves': num_leaves,\n        'learning_rate': learning_rate,\n        'feature_fraction': 0.9,\n    }\n    \n    num_rounds = 200\n    initial_model = lgb.train(params, train_data, num_rounds, valid_sets=[val_data])\n    \n    # Langkah 2: Membuat prediksi untuk data tanpa label\n    unlabeled_data = data[labels.isnull()]\n    predictions = initial_model.predict(unlabeled_data)\n    \n    # Menentukan kelas prediksi berdasarkan probabilitas maksimum\n    predicted_labels = [np.argmax(pred) for pred in predictions]\n    labels[labels.isnull()] = predicted_labels\n    \n    # Langkah 3: Melatih ulang model menggunakan seluruh data (dengan label awal dan hasil prediksi)\n    full_X_train, full_X_val, full_y_train, full_y_val = train_test_split(data, labels, test_size=0.2, random_state=42)\n    \n    full_train_data = lgb.Dataset(\n        full_X_train, \n        label=full_y_train, \n        params={'use_missing': True}, \n        weight=np.array([class_weights[int(y)] for y in full_y_train])\n    )\n    full_val_data = lgb.Dataset(\n        full_X_val, \n        label=full_y_val, \n        weight=np.array([class_weights[int(y)] for y in full_y_val]), \n        reference=full_train_data\n    )\n    \n    final_model = lgb.train(params, full_train_data, num_rounds, valid_sets=[full_val_data])\n    \n    return final_model\n\n\"\"\"\nDeskripsi: Fungsi ini melakukan pembelajaran semi-supervised untuk mengatasi data yang sebagian tidak berlabel. \nProses ini melibatkan pelatihan awal, prediksi untuk data tanpa label, dan pelatihan ulang menggunakan semua data.\n\nInput:\n    data: DataFrame setelah preprocessing.\n    labels: Label awal (dengan beberapa nilai kosong untuk data tanpa label).\n    num_leaves: Jumlah daun pada setiap pohon (default 61).\n    learning_rate: Kecepatan pembelajaran (default 0.01).\n    \nOutput:\n    final_model: Model LightGBM hasil pelatihan ulang.\n\"\"\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:39:29.051931Z","iopub.execute_input":"2024-12-05T09:39:29.052290Z","iopub.status.idle":"2024-12-05T09:39:29.073678Z","shell.execute_reply.started":"2024-12-05T09:39:29.052257Z","shell.execute_reply":"2024-12-05T09:39:29.072171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_prediction(test_file, trained_model):\n    # Preprocessing dataset uji\n    df_encoded_filled, df_result, _ = preprocessing(test_file)\n    \n    # Menyesuaikan format kolom dataset uji dengan dataset pelatihan\n    required_columns = [\n        'Basic_Demos-Age', 'Basic_Demos-Sex', 'CGAS-CGAS_Score',\n        'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n        'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n        'Fitness_Endurance-Max_Stage', 'FGC-FGC_CU', 'FGC-FGC_GSND',\n        'FGC-FGC_GSD', 'FGC-FGC_PU', 'FGC-FGC_SRL', 'FGC-FGC_SRR', 'FGC-FGC_TL',\n        'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE',\n        'BIA-BIA_ECW', 'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI',\n        'BIA-BIA_Fat', 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST',\n        'BIA-BIA_SMM', 'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total',\n        'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T',\n        'PreInt_EduHx-computerinternet_hoursday', 'CGAS-Season_Fall',\n        'CGAS-Season_Spring', 'CGAS-Season_Summer', 'CGAS-Season_Winter',\n        'Physical-Season_Fall', 'Physical-Season_Spring',\n        'Physical-Season_Summer', 'Physical-Season_Winter',\n        'Fitness_Endurance-Season_Fall', 'Fitness_Endurance-Season_Spring',\n        'Fitness_Endurance-Season_Summer', 'Fitness_Endurance-Season_Winter',\n        'FGC-Season_Fall', 'FGC-Season_Spring', 'FGC-Season_Summer',\n        'FGC-Season_Winter', 'BIA-Season_Fall', 'BIA-Season_Spring',\n        'BIA-Season_Summer', 'BIA-Season_Winter', 'SDS-Season_Fall',\n        'SDS-Season_Spring', 'SDS-Season_Summer', 'SDS-Season_Winter',\n        'PreInt_EduHx-Season_Fall', 'PreInt_EduHx-Season_Spring',\n        'PreInt_EduHx-Season_Summer', 'PreInt_EduHx-Season_Winter',\n        'BIA-BIA_Activity_Level_num_1.0', 'BIA-BIA_Activity_Level_num_2.0',\n        'BIA-BIA_Activity_Level_num_3.0', 'BIA-BIA_Activity_Level_num_4.0',\n        'BIA-BIA_Activity_Level_num_5.0', 'BIA-BIA_Frame_num_1.0',\n        'BIA-BIA_Frame_num_2.0', 'BIA-BIA_Frame_num_3.0',\n        'Fitness_Endurance-Time', 'PAQ_Total'\n    ]\n    \n    # Menambahkan kolom yang hilang dengan nilai NaN\n    for column in required_columns:\n        if column not in df_encoded_filled.columns:\n            df_encoded_filled[column] = np.nan\n    \n    # Menyusun ulang urutan kolom agar sesuai dengan format yang dibutuhkan\n    df_encoded_filled = df_encoded_filled.reindex(columns=required_columns)\n    \n    # Membuat prediksi menggunakan model terlatih\n    predictions = trained_model.predict(df_encoded_filled)\n    predicted_labels = [np.argmax(pred) for pred in predictions]\n    \n    # Menyimpan hasil prediksi dalam DataFrame hasil\n    df_result['sii'] = predicted_labels\n    return df_result\n\n\"\"\"\nDeskripsi: Fungsi ini membuat prediksi untuk dataset uji menggunakan model yang telah dilatih.\n\nInput:\n    test_file: Nama file dataset uji.\n    trained_model: Model hasil pembelajaran semi-supervised.\n    \nOutput:\n    df_result: DataFrame dengan prediksi yang sesuai dengan format pengumpulan/submisi.\n\"\"\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:39:29.075179Z","iopub.execute_input":"2024-12-05T09:39:29.075658Z","iopub.status.idle":"2024-12-05T09:39:29.094994Z","shell.execute_reply.started":"2024-12-05T09:39:29.075610Z","shell.execute_reply":"2024-12-05T09:39:29.093950Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ndef preprocessing(name):\n    # Read the CSV file\n    df = pd.read_csv(name)\n    \n    # Your preprocessing logic goes here\n    # For example, dropping some columns and creating labels\n    columns_to_delete = [col for col in df.columns if col.startswith('PCIAT')]\n    df = df.drop(columns=columns_to_delete)\n    \n    # If it's a training dataset, extract the labels\n    if name.endswith('train.csv'):\n        labels = df['sii']\n        df = df.drop(columns='sii')\n    else:\n        labels = None  # No labels for test set\n\n    # Returning processed data, labels, and any other needed data\n    return df, df[['id']], labels  # Adjust based on your logic\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:39:29.096459Z","iopub.execute_input":"2024-12-05T09:39:29.096883Z","iopub.status.idle":"2024-12-05T09:39:29.110684Z","shell.execute_reply.started":"2024-12-05T09:39:29.096838Z","shell.execute_reply":"2024-12-05T09:39:29.109553Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Ensure that preprocessing function is defined\ndef preprocessing(name):\n    # Read the CSV file\n    df = pd.read_csv(name)\n    \n    # Example preprocessing steps (you can add your actual logic here)\n    # Remove columns starting with 'PCIAT'\n    columns_to_delete = [col for col in df.columns if col.startswith('PCIAT')]\n    df = df.drop(columns=columns_to_delete)\n    \n    # If it's a training dataset, extract the labels\n    if name.endswith('train.csv'):\n        labels = df['sii']  # Extracting labels from the 'sii' column\n        df = df.drop(columns='sii')  # Drop 'sii' from the data\n    else:\n        labels = None  # No labels for the test set\n\n    # Create the train_result (ID column)\n    train_result = df[['id']]  # Assuming 'id' column exists in the dataset\n    \n    # Remove 'id' column from df since it's just used for results\n    df = df.drop(columns='id')\n    \n    # Example: Perform encoding or other preprocessing steps here\n    # Example: One-hot encode categorical columns (if any)\n    df_encoded = pd.get_dummies(df)\n    \n    # Return the processed data, IDs, and labels (if available)\n    return df_encoded, train_result, labels\n\n\n# Ensure that run_model (main function) is defined\ndef run_model(train_file, test_file):\n    # Call preprocessing to get the processed data, ID column, and labels\n    df_train_encoded, train_result, labels = preprocessing(train_file)\n    \n    # Continue with the rest of your model code (e.g., train the model, make predictions, etc.)\n    \n    # For this example, return the processed data (you can modify as per your logic)\n    return df_train_encoded\n\n\n# Run the model with the provided train and test dataset files\ndf_result = run_model('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv', '/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n\n# Output the resulting processed data (df_result is the processed training data here)\nprint(df_result.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:39:29.111877Z","iopub.execute_input":"2024-12-05T09:39:29.112219Z","iopub.status.idle":"2024-12-05T09:39:29.199943Z","shell.execute_reply.started":"2024-12-05T09:39:29.112184Z","shell.execute_reply":"2024-12-05T09:39:29.198806Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_model(train_file, test_file):\n    # Melakukan preprocessing pada dataset pelatihan\n    score = []\n    df_result_list = []\n    \n    # Melatih model untuk berbagai kombinasi hyperparameter\n    for num_leaves in [61]:\n        for learning_rate in [0.04]:\n            # Proses preprocessing pada data pelatihan\n            df_train_encoded, train_result, labels = preprocessing(train_file)\n            trained_model = model(df_train_encoded, labels, num_leaves, learning_rate)\n            \n            # Membuat prediksi pada dataset uji\n            df_prediction_result = prediction(test_file, trained_model)\n            \"\"\"\n            predi = df_prediction_result['sii']\n            print(f\"F1: {f1_score(ans, predi, average='weighted')}\")\n            score.append(f1_score(ans, predi, average='weighted'))\n            df_result_list.append(df_prediction_result['sii'])\n            \"\"\"\n    \n    return df_prediction_result\n\n\"\"\"\nDeskripsi: Fungsi utama untuk menjalankan seluruh proses program\n\nInput:\n    train_file: Nama file dataset pelatihan.\n    test_file: Nama file dataset uji.\n    \nOutput:\n    df_prediction_result: DataFrame hasil prediksi yang sesuai dengan format pengumpulan/submisi.\n\"\"\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:39:29.201421Z","iopub.execute_input":"2024-12-05T09:39:29.201836Z","iopub.status.idle":"2024-12-05T09:39:29.211284Z","shell.execute_reply.started":"2024-12-05T09:39:29.201791Z","shell.execute_reply":"2024-12-05T09:39:29.210135Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_result.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:39:29.213223Z","iopub.execute_input":"2024-12-05T09:39:29.213623Z","iopub.status.idle":"2024-12-05T09:39:29.395634Z","shell.execute_reply.started":"2024-12-05T09:39:29.213589Z","shell.execute_reply":"2024-12-05T09:39:29.394438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df_result.head())  # Menampilkan 5 baris pertama dari dataframe\ndf_result","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:39:29.398153Z","iopub.execute_input":"2024-12-05T09:39:29.398590Z","iopub.status.idle":"2024-12-05T09:39:29.442987Z","shell.execute_reply.started":"2024-12-05T09:39:29.398533Z","shell.execute_reply":"2024-12-05T09:39:29.441900Z"}},"outputs":[],"execution_count":null}]}