{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-05T12:42:52.472944Z","iopub.execute_input":"2024-12-05T12:42:52.473477Z","iopub.status.idle":"2024-12-05T12:42:57.546852Z","shell.execute_reply.started":"2024-12-05T12:42:52.473424Z","shell.execute_reply":"2024-12-05T12:42:57.545542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport polars as pl\nimport pandas as pd\nfrom sklearn.base import clone\nfrom copy import deepcopy\nimport optuna\nfrom scipy.optimize import minimize\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport re\nfrom colorama import Fore, Style\n\nfrom tqdm import tqdm\nfrom IPython.display import clear_output\nfrom concurrent.futures import ThreadPoolExecutor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor, CatBoostClassifier\nfrom xgboost import XGBRegressor\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\nfrom sklearn.preprocessing import StandardScaler\n\nSEED = 42\nn_splits = 5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T12:54:02.463884Z","iopub.execute_input":"2024-12-05T12:54:02.464362Z","iopub.status.idle":"2024-12-05T12:54:02.474467Z","shell.execute_reply.started":"2024-12-05T12:54:02.464323Z","shell.execute_reply":"2024-12-05T12:54:02.473161Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(\n            executor.map(lambda fname: process_file(fname, dirname), ids),\n            total=len(ids))\n        )\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"Stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    \n    return df\n    \n# Reading data files\ndata_path = '/kaggle/input/child-mind-institute-problematic-internet-use'\ntrain = pd.read_csv(f'{data_path}/train.csv')\ntest = pd.read_csv(f'{data_path}/test.csv')\nsample = pd.read_csv(f'{data_path}/sample_submission.csv')\n\ntrain_ts = load_time_series(f'{data_path}/series_train.parquet')\ntest_ts = load_time_series(f'{data_path}/series_test.parquet')\n\ntrain_orig = pd.merge(train, train_ts, how=\"left\", on='id')\ntest_orig = pd.merge(test, test_ts, how=\"left\", on='id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T12:43:01.352557Z","iopub.execute_input":"2024-12-05T12:43:01.353194Z","iopub.status.idle":"2024-12-05T12:44:34.997753Z","shell.execute_reply.started":"2024-12-05T12:43:01.353151Z","shell.execute_reply":"2024-12-05T12:44:34.996569Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# save original data\ntrain = train_orig.copy()\ntest = test_orig.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T12:47:11.574358Z","iopub.execute_input":"2024-12-05T12:47:11.574895Z","iopub.status.idle":"2024-12-05T12:47:11.595630Z","shell.execute_reply.started":"2024-12-05T12:47:11.574844Z","shell.execute_reply":"2024-12-05T12:47:11.593811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from keras import models, layers\n\ndef create_nn_model(input_dim):\n    model = models.Sequential()\n    model.add(layers.InputLayer(shape=(input_dim,)))  # Menggunakan 'shape' dan tambahkan tuple\n    model.add(layers.Dense(128, activation='relu'))\n    model.add(layers.Dropout(0.3))\n    model.add(layers.Dense(1, activation='linear'))  # Sesuaikan output sesuai kebutuhan\n    model.compile(optimizer='adam', loss='mse', metrics=['mae'])\n    return model\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T13:08:38.856767Z","iopub.execute_input":"2024-12-05T13:08:38.857300Z","iopub.status.idle":"2024-12-05T13:08:38.866200Z","shell.execute_reply.started":"2024-12-05T13:08:38.857256Z","shell.execute_reply":"2024-12-05T13:08:38.864807Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Drop PCIAT Cols from the training data (they don't exist in the test data)\n\npciat_Cols = [col for col in train.columns if 'PCIAT' in col]\ntrain = train.drop(pciat_Cols, axis=1)\n\ntrain.shape, test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T12:47:35.061645Z","iopub.execute_input":"2024-12-05T12:47:35.062277Z","iopub.status.idle":"2024-12-05T12:47:35.081612Z","shell.execute_reply.started":"2024-12-05T12:47:35.062230Z","shell.execute_reply":"2024-12-05T12:47:35.080110Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare feature values\n\ntrain = train.dropna(subset='sii')\n\ncat_Cols = [col for col in train.columns if 'Season' in col]\n\ndef update(df):\n    for c in cat_Cols:\n        if df[c].dtype.name == 'category':\n            # Add 'Missing' to the categories if it's not already present\n            if 'Missing' not in df[c].cat.categories:\n                df[c] = df[c].cat.add_categories('Missing')\n\n        # Fill missing values with 'Missing'\n        df[c] = df[c].fillna('Missing')\n\n        # Ensure the column is of 'category' dtype\n        df[c] = df[c].astype('category')\n    return df\n\n\ntrain = update(train)\ntest = update(test)\n\n\"\"\"\n    This Mapping Works Fine For me, I also \n    check each values in train and test using \n    logic. There no Data Lekage.\n\"\"\"\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n    \nfor col in cat_Cols:\n    mapping_train = create_mapping(col, train)\n    mapping_test = create_mapping(col, test)\n\n    train[col] = train[col].replace(mapping_train).astype(int)\n    test[col] = test[col].replace(mapping_test).astype(int)\n\nprint(f'Train Shape : {train.shape} || Test Shape : {test.shape}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T12:48:00.765087Z","iopub.execute_input":"2024-12-05T12:48:00.766052Z","iopub.status.idle":"2024-12-05T12:48:00.859289Z","shell.execute_reply.started":"2024-12-05T12:48:00.765990Z","shell.execute_reply":"2024-12-05T12:48:00.857902Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.drop('id', axis=1)\ntest_id = test['id'].copy()\ntest = test.drop('id', axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T12:48:12.799414Z","iopub.execute_input":"2024-12-05T12:48:12.799899Z","iopub.status.idle":"2024-12-05T12:48:12.811091Z","shell.execute_reply.started":"2024-12-05T12:48:12.799860Z","shell.execute_reply":"2024-12-05T12:48:12.809351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n# Functions for training the evaluating the selected model \n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    \n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead') # Nelder-Mead | # Powell\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': test_id,     #sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission, model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T12:58:27.181870Z","iopub.execute_input":"2024-12-05T12:58:27.183061Z","iopub.status.idle":"2024-12-05T12:58:27.198820Z","shell.execute_reply.started":"2024-12-05T12:58:27.183010Z","shell.execute_reply":"2024-12-05T12:58:27.197400Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# TrainML function with added Neural Network option\ndef TrainMLWithNN(model_class, test_data, use_nn=False):\n    \n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T12:58:52.172795Z","iopub.execute_input":"2024-12-05T12:58:52.173226Z","iopub.status.idle":"2024-12-05T12:58:52.181718Z","shell.execute_reply.started":"2024-12-05T12:58:52.173188Z","shell.execute_reply":"2024-12-05T12:58:52.180116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nfrom sklearn.base import clone\nfrom scipy.optimize import minimize\nfrom sklearn.model_selection import KFold\nimport numpy as np\nfrom keras import models, layers\nfrom IPython.display import clear_output\n\n# Fungsi untuk membuat model NN\ndef create_nn_model(input_dim):\n    model = models.Sequential()\n    model.add(layers.InputLayer(input_shape=(input_dim,)))\n    model.add(layers.Dense(128, activation='relu'))\n    model.add(layers.Dropout(0.3))\n    model.add(layers.Dense(64, activation='relu'))\n    model.add(layers.Dropout(0.2))\n    model.add(layers.Dense(1, activation='linear'))\n    model.compile(optimizer='adam', loss='mean_squared_error')\n    return model\n\n# Dummy data untuk testing (ganti dengan data sebenarnya)\nX = np.random.rand(100, 10)  # Contoh data fitur dengan 100 sampel, 10 fitur\ny = np.random.randint(0, 4, 100)  # Target kategori dengan nilai antara 0 dan 3\n\n# Inisialisasi variabel\nn_splits = 5  # Jumlah fold untuk cross-validation\nkf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n\noof_non_rounded = np.zeros(len(y))  # Prediksi kontinu pada data validasi\noof_rounded = np.zeros(len(y), dtype=int)  # Prediksi kategori pada data validasi\ntest_preds = np.zeros((len(X), n_splits))  # Inisialisasi berdasarkan data X (sementara)\n\ntrain_S = []  # Skor QWK pada data train di setiap fold\ntest_S = []   # Skor QWK pada data validasi di setiap fold\n\nuse_nn = True  # Pilih apakah menggunakan model neural network atau tidak\nmodel_class = None  # Ganti dengan model lain jika tidak menggunakan NN\n\n# Loop cross-validation\nfor fold, (train_idx, test_idx) in enumerate(kf.split(X)):\n    X_train, X_val = X[train_idx], X[test_idx]\n    y_train, y_val = y[train_idx], y[test_idx]\n    \n    # Standardisasi fitur\n    scaler = StandardScaler()\n    X_train_scaled = scaler.fit_transform(X_train)\n    X_val_scaled = scaler.transform(X_val)\n    \n    if use_nn:\n        model = create_nn_model(X_train.shape[1])\n        model.fit(X_train_scaled, y_train, epochs=50, batch_size=128, verbose=0)\n        y_train_pred = model.predict(X_train_scaled).flatten()\n        y_val_pred = model.predict(X_val_scaled).flatten()\n    else:\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n    \n    oof_non_rounded[test_idx] = y_val_pred\n    y_val_pred_rounded = y_val_pred.round(0).astype(int)\n    oof_rounded[test_idx] = y_val_pred_rounded\n\n    # Evaluasi dengan quadratic weighted kappa\n    train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n    val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n    \n    train_S.append(train_kappa)\n    test_S.append(val_kappa)\n    \n    # Prediksi pada data test\n    test_preds[:, fold] = model.predict(scaler.transform(X)).flatten() if use_nn else model.predict(X)\n\n    print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n    clear_output(wait=True)\n\nprint(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\nprint(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n# Optimasi QWK\ndef evaluate_predictions(thresholds, y_true, preds):\n    preds_rounded = threshold_Rounder(preds, thresholds)\n    return -quadratic_weighted_kappa(y_true, preds_rounded)\n\nKappaOptimizer = minimize(evaluate_predictions,\n                          x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                          method='Nelder-Mead')\nassert KappaOptimizer.success, \"Optimization did not converge.\"\n\noof_tuned = threshold_Rounder(oof_non_rounded, KappaOptimizer.x)\ntKappa = quadratic_weighted_kappa(y, oof_tuned)\n\nprint(f\"----> || Optimized QWK SCORE :: {tKappa:.3f}\")\n\ntpm = test_preds.mean(axis=1)\ntpTuned = threshold_Rounder(tpm, KappaOptimizer.x)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T13:20:51.820894Z","iopub.execute_input":"2024-12-05T13:20:51.821697Z","iopub.status.idle":"2024-12-05T13:21:16.485487Z","shell.execute_reply.started":"2024-12-05T13:20:51.821614Z","shell.execute_reply":"2024-12-05T13:21:16.484097Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\n#Train and predict sii for test data \n\nLGB_Params = {\n    'learning_rate': 0.07, \n    'random_state': SEED, \n    'n_estimators': 200,\n    'max_depth': 8, \n    'num_leaves': 300, \n    'min_data_in_leaf': 17,\n    'feature_fraction': 0.7689, \n    'bagging_fraction': 0.6879, \n    'bagging_freq': 2, \n    'lambda_l1': 4.74, \n    'lambda_l2': 4.743e-06,\n    'verbose': -1,\n    # CV : 0.4094 | LB : 0.471\n}\n\nModel = lgb.LGBMRegressor(**LGB_Params)\n\nSubmission, model = TrainML(Model,test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T12:48:38.635952Z","iopub.execute_input":"2024-12-05T12:48:38.636489Z","iopub.status.idle":"2024-12-05T12:48:51.275042Z","shell.execute_reply.started":"2024-12-05T12:48:38.636446Z","shell.execute_reply":"2024-12-05T12:48:51.273782Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\nSubmission.to_csv('submission.csv', index=False)\nprint(Submission['sii'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T12:49:21.089428Z","iopub.execute_input":"2024-12-05T12:49:21.089912Z","iopub.status.idle":"2024-12-05T12:49:21.108953Z","shell.execute_reply.started":"2024-12-05T12:49:21.089872Z","shell.execute_reply":"2024-12-05T12:49:21.107325Z"}},"outputs":[],"execution_count":null}]}