{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nimport polars as pl\nimport polars.selectors as cs\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\nimport seaborn as sns\n\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nimport lightgbm as lgb\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.impute import KNNImputer\n\nfrom scipy.optimize import minimize\nimport optuna\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import StratifiedKFold, cross_val_score\nfrom sklearn.preprocessing import StandardScaler\n\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-10T22:58:59.760484Z","iopub.execute_input":"2024-11-10T22:58:59.760985Z","iopub.status.idle":"2024-11-10T22:58:59.773587Z","shell.execute_reply.started":"2024-11-10T22:58:59.760944Z","shell.execute_reply":"2024-11-10T22:58:59.772481Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_engineering(df,numerical_columns):\n    # Mevcut özellik oluşturma işlemleri\n    \n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    \n\n    \"\"\"df['FFMI_FMI_Avg'] = (df['FFMI_BFP'] + df['FMI_BFP']) / 2\n    df['BFP_BMR_BFP_DEE_Sum'] = df['BFP_BMR'] + df['BFP_DEE']\"\"\"\n    \n\n\n    \n    return df\n\n\n\n\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n\n\nclass AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n        \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled)\n    \n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n    \n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n                 \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n        \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:58:59.775961Z","iopub.execute_input":"2024-11-10T22:58:59.776481Z","iopub.status.idle":"2024-11-10T22:58:59.797290Z","shell.execute_reply.started":"2024-11-10T22:58:59.776430Z","shell.execute_reply":"2024-11-10T22:58:59.795966Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_col  = \"sii\"","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:58:59.799239Z","iopub.execute_input":"2024-11-10T22:58:59.799724Z","iopub.status.idle":"2024-11-10T22:58:59.812436Z","shell.execute_reply.started":"2024-11-10T22:58:59.799671Z","shell.execute_reply":"2024-11-10T22:58:59.811212Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n\"\"\"\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\ntrain_ts_encoded = perform_autoencoder(df_train, encoding_dim=50, epochs=100, batch_size=32)\ntest_ts_encoded = perform_autoencoder(df_test, encoding_dim=50, epochs=100, batch_size=32)\n\ntime_series_cols = train_ts_encoded.columns.tolist()\n\ntrain_ts_encoded[\"id\"]=train_ts[\"id\"]\ntest_ts_encoded['id']=test_ts[\"id\"]\n\ntrain_ts_encoded = pd.merge(train_ts_encoded, train[['id', 'sii']], on='id', how='left')\ntest_ts_encoded = pd.merge(test_ts_encoded, test[['id']], on='id', how='left')\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:58:59.815412Z","iopub.execute_input":"2024-11-10T22:58:59.815918Z","iopub.status.idle":"2024-11-10T22:58:59.888127Z","shell.execute_reply.started":"2024-11-10T22:58:59.815869Z","shell.execute_reply":"2024-11-10T22:58:59.887050Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)   \n\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\n#featuresCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset=[\"sii\"]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:58:59.889903Z","iopub.execute_input":"2024-11-10T22:58:59.890254Z","iopub.status.idle":"2024-11-10T22:58:59.912015Z","shell.execute_reply.started":"2024-11-10T22:58:59.890216Z","shell.execute_reply":"2024-11-10T22:58:59.910491Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_cols = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:58:59.913528Z","iopub.execute_input":"2024-11-10T22:58:59.914003Z","iopub.status.idle":"2024-11-10T22:58:59.919899Z","shell.execute_reply.started":"2024-11-10T22:58:59.913952Z","shell.execute_reply":"2024-11-10T22:58:59.918631Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def updateCats(df):\n    global cat_cols\n    for c in cat_cols: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n        ","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:58:59.923072Z","iopub.execute_input":"2024-11-10T22:58:59.923489Z","iopub.status.idle":"2024-11-10T22:58:59.930224Z","shell.execute_reply.started":"2024-11-10T22:58:59.923444Z","shell.execute_reply":"2024-11-10T22:58:59.929091Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = updateCats(train)\ntest  = updateCats(test) \n\nnumerical_columns = train.select_dtypes(include=['int64', 'float64']).columns\nnumerical_columns = numerical_columns.drop('sii') \n\nscaler = StandardScaler()\n\ntrain[numerical_columns] = scaler.fit_transform(train[numerical_columns])\ntest[numerical_columns] = scaler.transform(test[numerical_columns])\n","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:58:59.931839Z","iopub.execute_input":"2024-11-10T22:58:59.932255Z","iopub.status.idle":"2024-11-10T22:58:59.990856Z","shell.execute_reply.started":"2024-11-10T22:58:59.932211Z","shell.execute_reply":"2024-11-10T22:58:59.989777Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"encoder = OrdinalEncoder(\n    dtype=np.int32,\n    handle_unknown='use_encoded_value',\n    unknown_value=-1,\n    encoded_missing_value=-2,\n)\n\ntrain[cat_cols] = encoder.fit_transform(train[cat_cols])\ntrain[cat_cols] = train[cat_cols].astype('category')\n\ntest[cat_cols] = encoder.transform(test[cat_cols])\ntest[cat_cols] = test[cat_cols].astype('category')\n\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_cols:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:58:59.992478Z","iopub.execute_input":"2024-11-10T22:58:59.992961Z","iopub.status.idle":"2024-11-10T22:59:00.077234Z","shell.execute_reply.started":"2024-11-10T22:58:59.992910Z","shell.execute_reply":"2024-11-10T22:59:00.076062Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = feature_engineering(train,numerical_columns)\ntest  = feature_engineering(test,numerical_columns)\n","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.078629Z","iopub.execute_input":"2024-11-10T22:59:00.078978Z","iopub.status.idle":"2024-11-10T22:59:00.108500Z","shell.execute_reply.started":"2024-11-10T22:59:00.078942Z","shell.execute_reply":"2024-11-10T22:59:00.107305Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.impute import SimpleImputer\n\ndef simple_impute(d_f, strategy='mean'):\n    non_empty_columns = d_f.columns[d_f.isnull().mean() < 1.0]\n    df_reduced = d_f[non_empty_columns].copy()\n\n    imputer = SimpleImputer(strategy=strategy)\n\n\n    df_imputed = pd.DataFrame(imputer.fit_transform(df_reduced), columns=df_reduced.columns)\n\n\n    for col in df_reduced.select_dtypes(include='object').columns:\n        if col in df_imputed.columns:\n \n            df_imputed[col] = df_imputed[col].astype(int).map(\n                dict(enumerate(d_f[col].astype('category').cat.categories))\n            )\n\n    for col in d_f.columns:\n        if col not in df_imputed.columns:\n            df_imputed[col] = np.nan\n\n   \n    df_imputed = df_imputed[d_f.columns]\n\n    return df_imputed\n\ntrain = simple_impute(train, strategy='mean')  \ntest = simple_impute(test, strategy='mean')","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.110237Z","iopub.execute_input":"2024-11-10T22:59:00.110735Z","iopub.status.idle":"2024-11-10T22:59:00.160299Z","shell.execute_reply.started":"2024-11-10T22:59:00.110684Z","shell.execute_reply":"2024-11-10T22:59:00.159200Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler\nimport numpy as np\nimport pandas as pd\n\ndef add_outlier_flag(train, test, target_col='sii', outlier_threshold=3):\n\n    train_target = train[target_col].reset_index(drop=True)  \n    train = train.drop(columns=[target_col])  \n    \n\n    pca = PCA(n_components=2)\n    train_pca = pca.fit_transform(train)\n    test_pca = pca.transform(test)\n    \n\n    train_pca_df = pd.DataFrame(train_pca, columns=['PC1', 'PC2'])\n    test_pca_df = pd.DataFrame(test_pca, columns=['PC1', 'PC2'])\n    \n \n    train_z_scores = np.abs((train_pca_df - train_pca_df.mean()) / train_pca_df.std())\n    test_z_scores = np.abs((test_pca_df - train_pca_df.mean()) / train_pca_df.std())  # Test için train ortalama ve std kullanılır\n\n    train_outliers = (train_z_scores['PC1'] > outlier_threshold) | (train_z_scores['PC2'] > outlier_threshold)\n    test_outliers = (test_z_scores['PC1'] > outlier_threshold) | (test_z_scores['PC2'] > outlier_threshold)\n    \n\n    train['is_outlier'] = train_outliers.astype(int)\n    test['is_outlier'] = test_outliers.astype(int)\n    \n\n    train[target_col] = train_target \n    \n\n    train = train[[col for col in train.columns if col != target_col] + [target_col]]\n    \n    return train, test\n\ntrain, test = add_outlier_flag(train, test)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.161704Z","iopub.execute_input":"2024-11-10T22:59:00.162134Z","iopub.status.idle":"2024-11-10T22:59:00.233333Z","shell.execute_reply.started":"2024-11-10T22:59:00.162089Z","shell.execute_reply":"2024-11-10T22:59:00.231944Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntrain_pca = train.drop(['sii'], axis = 1)\n","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.239140Z","iopub.execute_input":"2024-11-10T22:59:00.239656Z","iopub.status.idle":"2024-11-10T22:59:00.255710Z","shell.execute_reply.started":"2024-11-10T22:59:00.239557Z","shell.execute_reply":"2024-11-10T22:59:00.252047Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler\n\ndef plot_cumulative_variance(data):\n    scaler = StandardScaler()\n    data_scaled = scaler.fit_transform(data)\n    \n    # PCA uygulama (tüm bileşenleri dahil ederek)\n    pca = PCA()\n    pca.fit(data_scaled)\n    \n\n    cumulative_variance = np.cumsum(pca.explained_variance_ratio_)\n    \n\n    plt.figure(figsize=(8, 5))\n    plt.plot(range(1, len(cumulative_variance) + 1), cumulative_variance, marker='o')\n    plt.xlabel('PCA Bileşen Sayısı')\n    plt.ylabel('Kümülatif Varyans Oranı')\n    plt.title('PCA Kümülatif Varyans Grafiği')\n    plt.grid(True)\n    plt.show()\n\nplot_cumulative_variance(train_pca)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.257795Z","iopub.execute_input":"2024-11-10T22:59:00.258280Z","iopub.status.idle":"2024-11-10T22:59:00.718497Z","shell.execute_reply.started":"2024-11-10T22:59:00.258231Z","shell.execute_reply":"2024-11-10T22:59:00.717165Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.720509Z","iopub.execute_input":"2024-11-10T22:59:00.720996Z","iopub.status.idle":"2024-11-10T22:59:00.726386Z","shell.execute_reply.started":"2024-11-10T22:59:00.720946Z","shell.execute_reply":"2024-11-10T22:59:00.725064Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def quadratic_weighted_kappa(estimator, X, y_true):\n    y_pred = estimator.predict(X).round()\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.728118Z","iopub.execute_input":"2024-11-10T22:59:00.729121Z","iopub.status.idle":"2024-11-10T22:59:00.737568Z","shell.execute_reply.started":"2024-11-10T22:59:00.729068Z","shell.execute_reply":"2024-11-10T22:59:00.736397Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def threshold_rounder(y_pred, thresholds):\n    return np.where(y_pred < thresholds[0], 0,\n                    np.where(y_pred < thresholds[1], 1,\n                             np.where(y_pred < thresholds[2], 2, 3)))","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.739260Z","iopub.execute_input":"2024-11-10T22:59:00.739712Z","iopub.status.idle":"2024-11-10T22:59:00.748663Z","shell.execute_reply.started":"2024-11-10T22:59:00.739663Z","shell.execute_reply":"2024-11-10T22:59:00.747670Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def eval_preds(thresholds, y_true, y_pred):\n    y_pred = threshold_rounder(y_pred, thresholds)\n    score = cohen_kappa_score(y_true, y_pred, weights='quadratic')\n    return -score","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.750117Z","iopub.execute_input":"2024-11-10T22:59:00.750484Z","iopub.status.idle":"2024-11-10T22:59:00.758222Z","shell.execute_reply.started":"2024-11-10T22:59:00.750447Z","shell.execute_reply":"2024-11-10T22:59:00.757145Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import make_scorer\nKAPPA_SCORER = make_scorer(\n    cohen_kappa_score, \n    greater_is_better=True, \n    weights='quadratic',\n)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.759535Z","iopub.execute_input":"2024-11-10T22:59:00.760545Z","iopub.status.idle":"2024-11-10T22:59:00.769191Z","shell.execute_reply.started":"2024-11-10T22:59:00.760493Z","shell.execute_reply":"2024-11-10T22:59:00.768078Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CustomLGBMRegressor(lgb.LGBMRegressor):\n    '''\n    Custom LightGBM Regressor\n    \n    It optimizes threshold values during fitting.\n    Main goal is preventing overfit on validation data.\n    '''\n    def fit(self, X, y, **kwargs):\n        super().fit(X, y, **kwargs)\n        y_pred = super().predict(X, **kwargs)\n        \n        self.optimizer = minimize(\n            eval_preds, \n            x0=[0.5, 1.5, 2.5], \n            args=(y, y_pred), \n            method='Nelder-Mead',\n        )\n        \n    def predict(self, X, **kwargs):\n        y_pred = super().predict(X, **kwargs)\n        y_pred = threshold_rounder(y_pred, self.optimizer.x)\n        return y_pred","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.770791Z","iopub.execute_input":"2024-11-10T22:59:00.771183Z","iopub.status.idle":"2024-11-10T22:59:00.779823Z","shell.execute_reply.started":"2024-11-10T22:59:00.771147Z","shell.execute_reply":"2024-11-10T22:59:00.778661Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def lgb_objective(trial):\n    # Hyperparameters to be tuned\n    params = {\n        'objective':         'l2',\n        'verbosity':         -1,\n        'random_state':      SEED,\n        'boosting_type':     'gbdt',\n        'lambda_l1':         trial.suggest_float('lambda_l1', 1e-3, 10.0, log=True),\n        'lambda_l2':         trial.suggest_float('lambda_l2', 1e-3, 10.0, log=True),\n        'learning_rate':     trial.suggest_float('learning_rate', 1e-2, 1e-1, log=True),\n        'max_depth':         trial.suggest_int('max_depth', 4, 8),\n        'num_leaves':        trial.suggest_int('num_leaves', 16, 256),\n        'colsample_bytree':  trial.suggest_float('colsample_bytree', 0.4, 1.0),\n        'colsample_bynode':  trial.suggest_float('colsample_bynode', 0.4, 1.0),\n        'bagging_fraction':  trial.suggest_float('bagging_fraction', 0.4, 1.0),\n        'bagging_freq':      trial.suggest_int('bagging_freq', 1, 7),\n        'min_data_in_leaf':  trial.suggest_int('min_data_in_leaf', 5, 100),\n    }\n\n\n    n_iter = trial.suggest_int('n_iter', 50, 500)\n    \n\n    X = train.drop([\"sii\"], axis=1)\n    y = train[target_col]\n\n\n    estimator = CustomLGBMRegressor(**params, n_estimators=n_iter)\n    \n\n    cv = StratifiedKFold(5, shuffle=True, random_state=SEED)\n    val_scores = cross_val_score(\n        estimator=estimator, \n        X=X, y=y, \n        cv=cv, \n        scoring=KAPPA_SCORER,\n    )\n\n    return np.mean(val_scores)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.781676Z","iopub.execute_input":"2024-11-10T22:59:00.782104Z","iopub.status.idle":"2024-11-10T22:59:00.794871Z","shell.execute_reply.started":"2024-11-10T22:59:00.782055Z","shell.execute_reply":"2024-11-10T22:59:00.793599Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"do_tuning = False","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.796252Z","iopub.execute_input":"2024-11-10T22:59:00.796674Z","iopub.status.idle":"2024-11-10T22:59:00.809848Z","shell.execute_reply.started":"2024-11-10T22:59:00.796597Z","shell.execute_reply":"2024-11-10T22:59:00.808602Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if do_tuning:  \n    study = optuna.create_study(direction='maximize', study_name='Regressor')\n    study.optimize(lgb_objective, n_trials=150, show_progress_bar=True)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.811188Z","iopub.execute_input":"2024-11-10T22:59:00.811539Z","iopub.status.idle":"2024-11-10T22:59:00.820895Z","shell.execute_reply.started":"2024-11-10T22:59:00.811502Z","shell.execute_reply":"2024-11-10T22:59:00.819805Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nbest_params = {\n     'objective'       : 'l2',\n    'verbosity'    : -1,\n    'lambda_l1': 0.009635121762776935,\n    'lambda_l2': 0.005945245990700756,\n    'learning_rate': 0.011284203461464767,\n    'max_depth': 6,\n    'num_leaves': 35,\n    'colsample_bytree': 0.6688741529014656,\n    'colsample_bynode': 0.8420377613912466,\n    'bagging_fraction': 0.5424370397601171,\n    'bagging_freq': 5,\n    'min_data_in_leaf': 58,\n    'n_iter': 297\n}\n\nmodel = CustomLGBMRegressor(**best_params, random_state=SEED)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.822512Z","iopub.execute_input":"2024-11-10T22:59:00.822939Z","iopub.status.idle":"2024-11-10T22:59:00.831860Z","shell.execute_reply.started":"2024-11-10T22:59:00.822901Z","shell.execute_reply":"2024-11-10T22:59:00.830752Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = train.drop([\"sii\"], axis = 1)\ny = train[target_col]","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.835154Z","iopub.execute_input":"2024-11-10T22:59:00.835674Z","iopub.status.idle":"2024-11-10T22:59:00.847162Z","shell.execute_reply.started":"2024-11-10T22:59:00.835599Z","shell.execute_reply":"2024-11-10T22:59:00.846150Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ncv = StratifiedKFold(5, shuffle=True, random_state=SEED)\n\nval_scores = cross_val_score(\n    model, X, y, cv=cv, \n    scoring=KAPPA_SCORER,\n)\n\nprint(f'kappa score: {np.mean(val_scores):.4f}')","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:00.848709Z","iopub.execute_input":"2024-11-10T22:59:00.849165Z","iopub.status.idle":"2024-11-10T22:59:04.383440Z","shell.execute_reply.started":"2024-11-10T22:59:00.849127Z","shell.execute_reply":"2024-11-10T22:59:04.382176Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = VotingRegressor([\n    ('lgb_0', CustomLGBMRegressor(**best_params, random_state=12)),\n    ('lgb_1', CustomLGBMRegressor(**best_params, random_state=22)),\n    ('lgb_2', CustomLGBMRegressor(**best_params, random_state=32)),\n    ('lgb_3', CustomLGBMRegressor(**best_params, random_state=42)),\n    ('lgb_4', CustomLGBMRegressor(**best_params, random_state=52)),\n    ('lgb_5', CustomLGBMRegressor(**best_params, random_state=62)),\n    ('lgb_6', CustomLGBMRegressor(**best_params, random_state=72)),\n    ('lgb_7', CustomLGBMRegressor(**best_params, random_state=82)),\n    ('lgb_8', CustomLGBMRegressor(**best_params, random_state=92)),\n    ('lgb_9', CustomLGBMRegressor(**best_params, random_state=102))\n])","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:04.384914Z","iopub.execute_input":"2024-11-10T22:59:04.385301Z","iopub.status.idle":"2024-11-10T22:59:04.392030Z","shell.execute_reply.started":"2024-11-10T22:59:04.385263Z","shell.execute_reply":"2024-11-10T22:59:04.391007Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#model = CustomLGBMRegressor(**best_params, random_state=SEED)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:04.393247Z","iopub.execute_input":"2024-11-10T22:59:04.393596Z","iopub.status.idle":"2024-11-10T22:59:04.401435Z","shell.execute_reply.started":"2024-11-10T22:59:04.393560Z","shell.execute_reply":"2024-11-10T22:59:04.400579Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.fit(X, y)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:04.402749Z","iopub.execute_input":"2024-11-10T22:59:04.403423Z","iopub.status.idle":"2024-11-10T22:59:12.589102Z","shell.execute_reply.started":"2024-11-10T22:59:04.403376Z","shell.execute_reply":"2024-11-10T22:59:12.587915Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample[target_col] = model.predict(test)\nsample[target_col] = sample[target_col].round()\nsubmission_df = sample.reset_index(drop=True)\n\nprint(submission_df.head(20))\nsubmission_df[\"sii\"] = submission_df[\"sii\"].astype(int)\nsubmission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T22:59:12.590241Z","iopub.execute_input":"2024-11-10T22:59:12.590545Z","iopub.status.idle":"2024-11-10T22:59:12.619528Z","shell.execute_reply.started":"2024-11-10T22:59:12.590513Z","shell.execute_reply":"2024-11-10T22:59:12.618516Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}