{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-05T02:49:52.590389Z","iopub.execute_input":"2024-12-05T02:49:52.590879Z","iopub.status.idle":"2024-12-05T02:49:53.663726Z","shell.execute_reply.started":"2024-12-05T02:49:52.590833Z","shell.execute_reply":"2024-12-05T02:49:53.662059Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Neural Network","metadata":{}},{"cell_type":"markdown","source":"#  MLP (Multilayer Perceptron)","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import accuracy_score\nimport numpy as np\n\n# Simulasi dataset (ganti dengan dataset Anda)\nnp.random.seed(42)\ntorch.manual_seed(42)\nnum_samples = 1000\nnum_features = 10\nX = np.random.rand(num_samples, num_features)\ny = np.random.randint(0, 2, size=num_samples)  # Binary target [0, 1]\n\n# Split dataset menjadi training dan validation\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)\n\n# Standarisasi fitur\nscaler = StandardScaler()\nX_train = scaler.fit_transform(X_train)\nX_val = scaler.transform(X_val)\n\n# Convert ke tensor\nX_train = torch.tensor(X_train, dtype=torch.float32)\ny_train = torch.tensor(y_train, dtype=torch.long)\nX_val = torch.tensor(X_val, dtype=torch.float32)\ny_val = torch.tensor(y_val, dtype=torch.long)\n\n# Definisikan MLP\nclass MLPModel(nn.Module):\n    def __init__(self, input_dim):\n        super(MLPModel, self).__init__()\n        self.network = nn.Sequential(\n            nn.Linear(input_dim, 128),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(128, 64),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(64, 32),\n            nn.ReLU(),\n            nn.Linear(32, 2)  # Output sesuai jumlah kelas (binary: 2 kelas)\n        )\n    \n    def forward(self, x):\n        return self.network(x)\n\n# Inisialisasi model, loss, dan optimizer\nmodel = MLPModel(input_dim=X_train.shape[1])\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Training loop\nnum_epochs = 20\nfor epoch in range(num_epochs):\n    # Training\n    model.train()\n    optimizer.zero_grad()\n    outputs = model(X_train)\n    loss = criterion(outputs, y_train)\n    loss.backward()\n    optimizer.step()\n\n    # Validation\n    model.eval()\n    with torch.no_grad():\n        val_outputs = model(X_val)\n        val_loss = criterion(val_outputs, y_val)\n        val_preds = torch.argmax(val_outputs, dim=1)\n        val_accuracy = accuracy_score(y_val.numpy(), val_preds.numpy())\n\n    print(f\"Epoch {epoch + 1}/{num_epochs}, Loss: {loss.item():.4f}, Val Loss: {val_loss.item():.4f}, Val Acc: {val_accuracy:.4f}\")\n\n# Evaluasi model\nmodel.eval()\nwith torch.no_grad():\n    y_pred = model(X_val)\n    y_pred = torch.argmax(y_pred, dim=1).numpy()\nprint(\"Validation Accuracy:\", accuracy_score(y_val.numpy(), y_pred))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T02:49:53.665773Z","iopub.execute_input":"2024-12-05T02:49:53.666152Z","iopub.status.idle":"2024-12-05T02:49:53.782171Z","shell.execute_reply.started":"2024-12-05T02:49:53.666099Z","shell.execute_reply":"2024-12-05T02:49:53.781035Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# RNN","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import accuracy_score\nimport numpy as np\n\n# Simulasi dataset (ganti dengan dataset Anda)\nnp.random.seed(42)\ntorch.manual_seed(42)\nnum_samples = 1000\nnum_features = 10\nX = np.random.rand(num_samples, num_features)\ny = np.random.randint(0, 2, size=num_samples)  # Binary target [0, 1]\n\n# Split dataset menjadi training dan validation\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)\n\n# Standarisasi fitur\nscaler = StandardScaler()\nX_train = scaler.fit_transform(X_train)\nX_val = scaler.transform(X_val)\n\n# Convert ke tensor\nX_train = torch.tensor(X_train, dtype=torch.float32)\ny_train = torch.tensor(y_train, dtype=torch.long)\nX_val = torch.tensor(X_val, dtype=torch.float32)\ny_val = torch.tensor(y_val, dtype=torch.long)\n\n# RNN Model Definition\nclass RNNModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(RNNModel, self).__init__()\n        self.hidden_dim = hidden_dim\n        \n        # Define the RNN layer (input_dim: features, hidden_dim: number of hidden units)\n        self.rnn = nn.RNN(input_dim, hidden_dim, batch_first=True)\n        \n        # Fully connected layer to output the predictions\n        self.fc = nn.Linear(hidden_dim, output_dim)\n    \n    def forward(self, x):\n        # Add an extra dimension to x to make it compatible with RNN (batch_size, seq_len, input_dim)\n        x = x.unsqueeze(1)  # Make each feature a sequence (length = 1)\n        \n        # Get the output of the RNN layer\n        rnn_out, _ = self.rnn(x)\n        \n        # Use the output from the last time step (last element of the sequence)\n        final_hidden_state = rnn_out[:, -1, :]\n        \n        # Output predictions\n        out = self.fc(final_hidden_state)\n        return out\n\n# Inisialisasi model, loss, dan optimizer\ninput_dim = X_train.shape[1]  # Number of features\nhidden_dim = 64  # Number of hidden units in RNN\noutput_dim = 2  # Number of classes (binary classification)\n\nmodel = RNNModel(input_dim, hidden_dim, output_dim)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Training loop\nnum_epochs = 20\nfor epoch in range(num_epochs):\n    # Training phase\n    model.train()\n    optimizer.zero_grad()\n    outputs = model(X_train)\n    loss = criterion(outputs, y_train)\n    loss.backward()\n    optimizer.step()\n\n    # Validation phase\n    model.eval()\n    with torch.no_grad():\n        val_outputs = model(X_val)\n        val_loss = criterion(val_outputs, y_val)\n        val_preds = torch.argmax(val_outputs, dim=1)\n        val_accuracy = accuracy_score(y_val.numpy(), val_preds.numpy())\n\n    print(f\"Epoch {epoch + 1}/{num_epochs}, Loss: {loss.item():.4f}, Val Loss: {val_loss.item():.4f}, Val Acc: {val_accuracy:.4f}\")\n\n# Evaluasi model setelah training\nmodel.eval()\nwith torch.no_grad():\n    y_pred = model(X_val)\n    y_pred = torch.argmax(y_pred, dim=1).numpy()\n\nprint(\"Final Validation Accuracy:\", accuracy_score(y_val.numpy(), y_pred))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T02:49:53.783901Z","iopub.execute_input":"2024-12-05T02:49:53.784283Z","iopub.status.idle":"2024-12-05T02:49:53.886704Z","shell.execute_reply.started":"2024-12-05T02:49:53.784249Z","shell.execute_reply":"2024-12-05T02:49:53.885436Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## LightGBM, XGBoost, dan CatBoost (boosting algorithm)","metadata":{}},{"cell_type":"code","source":"import os\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.ensemble import VotingClassifier\n\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom imblearn.pipeline import Pipeline\nfrom imblearn.over_sampling import RandomOverSampler\n\n\nimport lightgbm as lgb\nimport catboost as cb\nimport xgboost as xgb\n\nimport optuna\nfrom optuna.samplers import TPESampler\n\n#downsampling techniques\n# they took long time, so we use RandomUnderSampler\nfrom imblearn.under_sampling import NearMiss\nfrom imblearn.under_sampling import ClusterCentroids\nfrom imblearn.under_sampling import TomekLinks\nfrom imblearn.under_sampling import EditedNearestNeighbours\nfrom imblearn.pipeline import Pipeline as ImbPipeline\nfrom imblearn.under_sampling import NearMiss, TomekLinks\nfrom sklearn.impute import SimpleImputer\n\nimport time\nfrom sklearn.feature_selection import SelectKBest, chi2, mutual_info_classif, VarianceThreshold\nimport numpy as np\nimport polars as pl\nimport pandas as pd\nfrom sklearn.base import clone\nfrom copy import deepcopy\nimport optuna\nfrom scipy.optimize import minimize\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport re\nfrom colorama import Fore, Style\n\nfrom tqdm import tqdm\nfrom IPython.display import clear_output\nfrom concurrent.futures import ThreadPoolExecutor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor, CatBoostClassifier\nfrom xgboost import XGBRegressor\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\nseed = 42\nn_splits = 5\nDO_TUNING = False\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"Stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    \n    return df\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday']\n\nfeaturesCols += time_series_cols\nsii=train.sii\ntrain=train.drop('sii', axis=1, inplace=False)\ntrain = train[featuresCols]\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', \n          'FGC-Season', 'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ndef update(df):\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n        \ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\n\"\"\"This Mapping Works Fine For me I also Check Each Values in Train and test Using Logic. There no Data Lekage.\"\"\"\n\nfor col in cat_c:\n    mapping_train = create_mapping(col, train)\n    mapping_test = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping_train).astype(int)\n    test[col] = test[col].replace(mapping_test).astype(int)\ntrain['sii'] = sii\nun_lable = train[train['sii'].isnull()]\ntrain = train.dropna(subset='sii')\ndef quadratic_weighted_kappa(estimator, X, y_true):\n    y_pred=estimator.predict(X).astype(y_true.dtype)\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n# Output statistik deskriptif dari time series\nprint(\"Train Time Series Statistics:\")\nprint(train_ts.head())\n\nprint(\"\\nTest Time Series Statistics:\")\nprint(test_ts.head())\n\n# Distribusi Target\nplt.figure(figsize=(8, 6))\nsns.countplot(x=sii)\nplt.title(\"Distribution of Target Variable (sii)\")\nplt.xlabel(\"Sii\")\nplt.ylabel(\"Count\")\nplt.show()\n\n\n# Feature importance dari LightGBM\nmodel = lgb.LGBMClassifier(random_state=seed)\nmodel.fit(train.drop('sii', axis=1), train['sii'])\n\n# Plot Feature Importance\nlgb.plot_importance(model, max_num_features=20, importance_type='gain', figsize=(12, 8))\nplt.title(\"Top 20 Feature Importances\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T02:49:53.888967Z","iopub.execute_input":"2024-12-05T02:49:53.889402Z","iopub.status.idle":"2024-12-05T02:51:44.177803Z","shell.execute_reply.started":"2024-12-05T02:49:53.889365Z","shell.execute_reply":"2024-12-05T02:51:44.175922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\nsii=train.sii\ndef TrainML(model_class, test_data):\n    \n    \n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)\n    \n    train_S = []\n    test_S = []\n\n\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n       \n        model = clone(model_class)\n        model.fit(X_train, y_train)\n         \n   \n        \n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission\ntrain['sii'].value_counts()\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01  # Increased from 2.68e-06\n}\n\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': seed,\n    'tree_method': 'exact'\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': seed,\n    'verbose': 0,\n    'l2_leaf_reg': 10  # Increase this value\n}\n\n# Create model instances\nLight = LGBMRegressor(**Params, random_state=seed, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model)\n])\n\n# Train the ensemble model\nSubmission = TrainML(voting_model, test)\nSubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T02:51:44.180922Z","iopub.execute_input":"2024-12-05T02:51:44.181503Z","iopub.status.idle":"2024-12-05T02:52:30.235381Z","shell.execute_reply.started":"2024-12-05T02:51:44.181448Z","shell.execute_reply":"2024-12-05T02:52:30.234068Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\ndef TrainMLWithVisualization(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    # Visualize target distribution\n    plt.figure(figsize=(8, 4))\n    sns.countplot(x=y, palette=\"viridis\")\n    plt.title(\"Target Distribution (sii)\")\n    plt.xlabel(\"Target Value\")\n    plt.ylabel(\"Count\")\n    plt.show()\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)\n    \n    train_S = []\n    test_S = []\n\n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n       \n        model = clone(model_class)\n        model.fit(X_train, y_train)\n         \n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n\n    # Plot QWK per fold\n    plt.figure(figsize=(10, 6))\n    plt.plot(range(1, n_splits + 1), train_S, label=\"Train QWK\", marker=\"o\")\n    plt.plot(range(1, n_splits + 1), test_S, label=\"Validation QWK\", marker=\"o\")\n    plt.axhline(np.mean(test_S), color=\"red\", linestyle=\"--\", label=\"Mean Validation QWK\")\n    plt.title(\"Quadratic Weighted Kappa (QWK) Per Fold\")\n    plt.xlabel(\"Fold\")\n    plt.ylabel(\"QWK Score\")\n    plt.legend()\n    plt.grid()\n    plt.show()\n\n    KappaOptimizer = minimize(evaluate_predictions,\n                               x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                               method='Nelder-Mead')\n    assert KappaOptimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOptimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOptimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    # Visualize predicted distribution\n    plt.figure(figsize=(8, 4))\n    sns.histplot(oof_non_rounded, bins=50, kde=True, label=\"Predicted\")\n    sns.histplot(y, color='orange', bins=50, kde=True, label=\"Actual\", alpha=0.5)\n    plt.title(\"Predicted vs Actual Distribution\")\n    plt.xlabel(\"Value\")\n    plt.ylabel(\"Frequency\")\n    plt.legend()\n    plt.show()\n\n    return submission\n\n# Train and visualize\nSubmission = TrainMLWithVisualization(voting_model, test)\nSubmission.to_csv('submission.csv', index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T02:52:30.237028Z","iopub.execute_input":"2024-12-05T02:52:30.237469Z","iopub.status.idle":"2024-12-05T02:53:17.950621Z","shell.execute_reply.started":"2024-12-05T02:52:30.237431Z","shell.execute_reply":"2024-12-05T02:53:17.949207Z"}},"outputs":[],"execution_count":null}]}