{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30762,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n\n# Load the data\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample_submission = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:30.035515Z","iopub.execute_input":"2025-02-08T02:39:30.035849Z","iopub.status.idle":"2025-02-08T02:39:30.492238Z","shell.execute_reply.started":"2025-02-08T02:39:30.035817Z","shell.execute_reply":"2025-02-08T02:39:30.491465Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inspect the first few rows of each DataFrame\nprint(\"Train Data:\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:30.493881Z","iopub.execute_input":"2025-02-08T02:39:30.494180Z","iopub.status.idle":"2025-02-08T02:39:30.535820Z","shell.execute_reply.started":"2025-02-08T02:39:30.494151Z","shell.execute_reply":"2025-02-08T02:39:30.534839Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_values = train[test.columns].isnull().sum()\nmissing_values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-08T02:39:30.537110Z","iopub.execute_input":"2025-02-08T02:39:30.537443Z","iopub.status.idle":"2025-02-08T02:39:30.550766Z","shell.execute_reply.started":"2025-02-08T02:39:30.537412Z","shell.execute_reply":"2025-02-08T02:39:30.550033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[test.columns].dtypes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-08T02:39:30.551742Z","iopub.execute_input":"2025-02-08T02:39:30.551995Z","iopub.status.idle":"2025-02-08T02:39:30.561671Z","shell.execute_reply.started":"2025-02-08T02:39:30.551969Z","shell.execute_reply":"2025-02-08T02:39:30.560656Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Test Data:\")\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:30.563806Z","iopub.execute_input":"2025-02-08T02:39:30.564093Z","iopub.status.idle":"2025-02-08T02:39:30.592051Z","shell.execute_reply.started":"2025-02-08T02:39:30.564063Z","shell.execute_reply":"2025-02-08T02:39:30.591034Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Sample Submission:\")\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:30.593299Z","iopub.execute_input":"2025-02-08T02:39:30.593655Z","iopub.status.idle":"2025-02-08T02:39:30.607941Z","shell.execute_reply.started":"2025-02-08T02:39:30.593623Z","shell.execute_reply":"2025-02-08T02:39:30.606798Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_columns = test.select_dtypes(include=['object', 'category']).columns[1:]\nnumerical_columns = test.select_dtypes(include=['number']).columns","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:30.609436Z","iopub.execute_input":"2025-02-08T02:39:30.610099Z","iopub.status.idle":"2025-02-08T02:39:30.617340Z","shell.execute_reply.started":"2025-02-08T02:39:30.610054Z","shell.execute_reply":"2025-02-08T02:39:30.616338Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Handle missing values for categorical columns\n\nfor col in categorical_columns:\n # Option 1: Fill with 'Missing'\n test[col].fillna('Missing', inplace=True)\n train[col].fillna('Missing', inplace=True)\n #means = train.groupby(col)['sii'].mean()\n #if col != \"id\" and col != \"sii\":\n # means = train.groupby(col)['sii'].mean()\n # # Replace categories with their mean\n # train[col] =train[col].map(means)\n # test[col] =test[col].map(means)\n\n# Handle missing values for categorical columns\n#for col in categorical_columns:\n # Find the most frequent category (mode) in the training set\n# most_frequent = train[col].mode()[0]\n \n # Fill missing values in both train and test sets with the most frequent category\n# train[col].fillna(most_frequent, inplace=True)\n# test[col].fillna(most_frequent, inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:30.618522Z","iopub.execute_input":"2025-02-08T02:39:30.618882Z","iopub.status.idle":"2025-02-08T02:39:30.638914Z","shell.execute_reply.started":"2025-02-08T02:39:30.618853Z","shell.execute_reply":"2025-02-08T02:39:30.637723Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_columns = test.select_dtypes(include=['object', 'category']).columns[1:]\nnumerical_columns = test.select_dtypes(include=['number']).columns","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:30.640163Z","iopub.execute_input":"2025-02-08T02:39:30.641064Z","iopub.status.idle":"2025-02-08T02:39:30.648362Z","shell.execute_reply.started":"2025-02-08T02:39:30.641023Z","shell.execute_reply":"2025-02-08T02:39:30.647561Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# handing ordenal features \nimport pandas as pd\n\n# Define the mapping for ordinal encoding of seasons\nseason_mapping = {\n 'Spring': 0,\n 'Summer': 1,\n 'Fall': 2,\n 'Winter': 3\n}\n\n# List of season columns to transform\nseason_columns = [\n 'Basic_Demos-Enroll_Season',\n 'CGAS-Season',\n 'Physical-Season',\n 'PAQ_A-Season',\n 'PAQ_C-Season',\n 'SDS-Season',\n 'PreInt_EduHx-Season',\n 'BIA-Season',\n 'FGC-Season',\n 'Fitness_Endurance-Season'\n]\n\n# Apply the mapping to each season column\nfor col in season_columns:\n train[col] = train[col].map(season_mapping)\n test[col] = test[col].map(season_mapping)\n\n# Verify the result\ntrain.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-08T02:39:30.649535Z","iopub.execute_input":"2025-02-08T02:39:30.649843Z","iopub.status.idle":"2025-02-08T02:39:30.695835Z","shell.execute_reply.started":"2025-02-08T02:39:30.649815Z","shell.execute_reply":"2025-02-08T02:39:30.694880Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_columns = test.select_dtypes(include=['object', 'category']).columns[1:]\nnumerical_columns = test.select_dtypes(include=['number']).columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-08T02:39:30.696896Z","iopub.execute_input":"2025-02-08T02:39:30.697159Z","iopub.status.idle":"2025-02-08T02:39:30.703985Z","shell.execute_reply.started":"2025-02-08T02:39:30.697132Z","shell.execute_reply":"2025-02-08T02:39:30.702885Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-08T02:39:30.705222Z","iopub.execute_input":"2025-02-08T02:39:30.706283Z","iopub.status.idle":"2025-02-08T02:39:30.714734Z","shell.execute_reply.started":"2025-02-08T02:39:30.706237Z","shell.execute_reply":"2025-02-08T02:39:30.713917Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder\n\n# One-Hot Encoding\n# Use OneHotEncoder for nominal categorical variables\nonehot_encoder = OneHotEncoder(sparse=False, handle_unknown='ignore')\n\n# Fit and transform the training data\ntrain_encoded = onehot_encoder.fit_transform(train[categorical_columns])\n\n# Transform the test data\ntest_encoded = onehot_encoder.transform(test[categorical_columns])\n\n# Convert the encoded features back to DataFrame\ntrain_encoded_df = pd.DataFrame(train_encoded, columns=onehot_encoder.get_feature_names_out(categorical_columns))\ntest_encoded_df = pd.DataFrame(test_encoded, columns=onehot_encoder.get_feature_names_out(categorical_columns))\n\n# Drop original categorical columns and concatenate the encoded columns\ntrain = train.drop(categorical_columns, axis=1)\ntest = test.drop(categorical_columns, axis=1)\n\ntrain = pd.concat([train, train_encoded_df], axis=1)\ntest = pd.concat([test, test_encoded_df], axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:30.715987Z","iopub.execute_input":"2025-02-08T02:39:30.716737Z","iopub.status.idle":"2025-02-08T02:39:31.214807Z","shell.execute_reply.started":"2025-02-08T02:39:30.716686Z","shell.execute_reply":"2025-02-08T02:39:31.213681Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Handle missing values for numerical columns\n#for col in numerical_columns:\n # Option 1: Fill with mean\n# test[col].fillna(train[col].median(), inplace=True)\n# train[col].fillna(train[col].median(), inplace=True)\n#numerical_indices = {col: test.drop(['id'], axis=1).columns.get_loc(col) for col in numerical_columns}\n#median_dict = {numerical_indices[col]: train[col].median() for col in numerical_columns}","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:31.218795Z","iopub.execute_input":"2025-02-08T02:39:31.219205Z","iopub.status.idle":"2025-02-08T02:39:31.223525Z","shell.execute_reply.started":"2025-02-08T02:39:31.219176Z","shell.execute_reply":"2025-02-08T02:39:31.222374Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.base import BaseEstimator, TransformerMixin\n\nclass PresetMedianImputer(BaseEstimator, TransformerMixin):\n \"\"\"\n Imputer that uses pre-calculated medians with column indices\n \n Parameters\n ----------\n medians_dict : dict\n Dictionary mapping column indices to their median values\n \"\"\"\n def __init__(self, medians_dict):\n self.medians_dict = medians_dict\n \n def fit(self, X, y=None):\n \"\"\"No fitting necessary as we use preset medians\"\"\"\n return self\n \n def transform(self, X):\n \"\"\"\n Transform X by filling missing values with preset medians\n \n Parameters\n ----------\n X : array-like of shape (n_samples, n_features)\n Input data to impute\n \n Returns\n -------\n X_imputed : ndarray of shape (n_samples, n_features)\n Imputed data\n \"\"\"\n # Convert to numpy array if DataFrame\n if isinstance(X, pd.DataFrame):\n X = X.values\n \n # Make a copy to avoid modifying original data\n X_imputed = X.copy()\n \n # Apply imputation for each column index in medians_dict\n for col_idx, median_value in self.medians_dict.items():\n mask = np.isnan(X_imputed[:, col_idx])\n X_imputed[mask, col_idx] = median_value\n \n return X_imputed","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-08T02:39:31.224744Z","iopub.execute_input":"2025-02-08T02:39:31.225090Z","iopub.status.idle":"2025-02-08T02:39:31.238619Z","shell.execute_reply.started":"2025-02-08T02:39:31.225042Z","shell.execute_reply":"2025-02-08T02:39:31.237638Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_col = test.columns.to_list() + [\"sii\"]\ntest_col = test.columns.to_list()","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:31.239882Z","iopub.execute_input":"2025-02-08T02:39:31.240156Z","iopub.status.idle":"2025-02-08T02:39:31.250093Z","shell.execute_reply.started":"2025-02-08T02:39:31.240130Z","shell.execute_reply":"2025-02-08T02:39:31.249258Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\ntrain_extra = train[train['sii'].isna().values]\ntrain_extra.reset_index(inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-08T02:39:31.251331Z","iopub.execute_input":"2025-02-08T02:39:31.251690Z","iopub.status.idle":"2025-02-08T02:39:31.265198Z","shell.execute_reply.started":"2025-02-08T02:39:31.251650Z","shell.execute_reply":"2025-02-08T02:39:31.264119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.dropna(subset=['sii'])\nprint(\"Missing Values After Imputation:\")\nprint(train[train_col].isnull().sum().sum()) # Should output 0 if all missing values are handled","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:31.266438Z","iopub.execute_input":"2025-02-08T02:39:31.266833Z","iopub.status.idle":"2025-02-08T02:39:31.280090Z","shell.execute_reply.started":"2025-02-08T02:39:31.266790Z","shell.execute_reply":"2025-02-08T02:39:31.279024Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Missing Values After Imputation:\")\nprint(test[test_col].isnull().sum().sum()) # Should output 0 if all missing values are handled","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:31.281343Z","iopub.execute_input":"2025-02-08T02:39:31.281672Z","iopub.status.idle":"2025-02-08T02:39:31.294521Z","shell.execute_reply.started":"2025-02-08T02:39:31.281641Z","shell.execute_reply":"2025-02-08T02:39:31.293503Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.feature_selection import SelectFromModel\nfrom sklearn.preprocessing import PolynomialFeatures","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:31.295802Z","iopub.execute_input":"2025-02-08T02:39:31.296111Z","iopub.status.idle":"2025-02-08T02:39:31.734241Z","shell.execute_reply.started":"2025-02-08T02:39:31.296081Z","shell.execute_reply":"2025-02-08T02:39:31.733127Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from lightgbm import LGBMClassifier\nfrom sklearn.inspection import permutation_importance\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import make_scorer, cohen_kappa_score\nimport matplotlib.pyplot as plt\n# Drop the target column for feature selection\nX = train[train_col].drop(['sii'], axis=1)\nX_extra = train_extra[train_col].drop(['sii'], axis=1)\n\nX_test = test[test_col]\ny = train['sii']\n","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:31.735759Z","iopub.execute_input":"2025-02-08T02:39:31.736181Z","iopub.status.idle":"2025-02-08T02:39:34.962399Z","shell.execute_reply.started":"2025-02-08T02:39:31.736137Z","shell.execute_reply":"2025-02-08T02:39:34.961453Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class_counts = y.value_counts()\nclass_counts","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:34.963627Z","iopub.execute_input":"2025-02-08T02:39:34.964211Z","iopub.status.idle":"2025-02-08T02:39:34.977250Z","shell.execute_reply.started":"2025-02-08T02:39:34.964177Z","shell.execute_reply":"2025-02-08T02:39:34.976192Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optuna\nimport numpy as np\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.feature_selection import SelectKBest, f_classif\n\nimport optuna\nimport numpy as np\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score\nfrom xgboost import XGBClassifier\nfrom sklearn.utils.class_weight import compute_class_weight","metadata":{"execution":{"iopub.status.busy":"2025-03-26T14:02:13.287701Z","iopub.execute_input":"2025-03-26T14:02:13.288048Z","iopub.status.idle":"2025-03-26T14:02:14.938119Z","shell.execute_reply.started":"2025-03-26T14:02:13.288009Z","shell.execute_reply":"2025-03-26T14:02:14.937159Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.utils.validation import check_is_fitted, check_array\nimport random\nimport os\nimport warnings\n\nclass SimpleTransformerBlock(nn.Module):\n # [Previous SimpleTransformerBlock implementation remains the same]\n def __init__(self, n_features, n_heads=4, dropout=0.1):\n super().__init__()\n self.n_features = n_features\n self.n_heads = n_heads\n self.head_dim = n_features // n_heads\n self.adjusted_n_features = self.head_dim * n_heads\n \n self.dropout = nn.Dropout(dropout)\n self.attention = nn.MultiheadAttention(self.adjusted_n_features, n_heads, dropout=dropout)\n self.norm1 = nn.LayerNorm(self.adjusted_n_features)\n self.norm2 = nn.LayerNorm(self.adjusted_n_features)\n \n self.ffn = nn.Sequential(\n nn.Linear(self.adjusted_n_features, self.adjusted_n_features * 4),\n nn.GELU(),\n nn.Dropout(dropout),\n nn.Linear(self.adjusted_n_features * 4, self.adjusted_n_features),\n nn.Dropout(dropout)\n )\n \n self.proj_in = nn.Linear(n_features, self.adjusted_n_features)\n self.proj_out = nn.Linear(self.adjusted_n_features, n_features)\n \n def forward(self, x):\n x = self.proj_in(x)\n attended, _ = self.attention(x, x, x)\n attended = self.dropout(attended)\n x = self.norm1(x + attended)\n ffn_out = self.ffn(x)\n x = self.norm2(x + ffn_out)\n x = self.proj_out(x)\n return x\n\nclass TransformerImputer(BaseEstimator, TransformerMixin):\n def __init__(\n self,\n max_iter=10,\n batch_size=30,\n n_heads=4,\n learning_rate=0.01,\n strategy='median',\n fill_value=None,\n add_indicator=False,\n random_state=None,\n verbose=0\n ):\n self.max_iter = max_iter\n self.batch_size = batch_size\n self.n_heads = n_heads\n self.learning_rate = learning_rate\n self.strategy = strategy\n self.fill_value = fill_value\n self.add_indicator = add_indicator\n self.random_state = random_state\n self.verbose = verbose\n\n def _validate_input(self, X):\n if not isinstance(X, np.ndarray):\n X = check_array(X, force_all_finite='allow-nan', dtype=np.float32)\n \n if self.strategy not in ['mean', 'median', 'most_frequent', 'constant']:\n raise ValueError(f\"Can't handle strategy {self.strategy}\")\n \n return X\n\n def _set_seed(self):\n if self.random_state is not None:\n torch.manual_seed(self.random_state)\n torch.cuda.manual_seed_all(self.random_state)\n np.random.seed(self.random_state)\n random.seed(self.random_state)\n if torch.cuda.is_available():\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = False\n\n def _initial_imputation(self, X):\n if not hasattr(self, 'statistics_'):\n self.statistics_ = np.zeros(X.shape[1], dtype=X.dtype)\n for i in range(X.shape[1]):\n if self.strategy == 'mean':\n self.statistics_[i] = np.nanmean(X[:, i])\n elif self.strategy == 'median':\n self.statistics_[i] = np.nanmedian(X[:, i])\n elif self.strategy == 'most_frequent':\n self.statistics_[i] = np.nanmode(X[:, i])[0]\n else: # constant\n self.statistics_[i] = self.fill_value if self.fill_value is not None else 0\n\n if np.any(np.isnan(self.statistics_)):\n warnings.warn(\"Some features have all NaN values. Using 0 for imputation.\")\n self.statistics_[np.isnan(self.statistics_)] = 0\n\n X_imputed = X.copy()\n mask = np.isnan(X)\n indices = np.where(mask)\n X_imputed[indices] = np.take(self.statistics_, indices[1])\n \n return X_imputed, mask\n\n def fit(self, X, y=None):\n X = self._validate_input(X)\n self._set_seed()\n \n # Set device\n self.device_ = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n \n # Initial imputation\n X_imputed, mask = self._initial_imputation(X)\n \n # Save missing indicator if requested\n if self.add_indicator:\n self.indicator_ = ~mask\n \n # Create dataset with CPU tensors\n dataset = TensorDataset(\n torch.FloatTensor(X_imputed),\n torch.FloatTensor(~mask)\n )\n \n # Create DataLoader with pin_memory=True for GPU training\n dataloader = DataLoader(\n dataset,\n batch_size=self.batch_size,\n shuffle=True,\n pin_memory=True if torch.cuda.is_available() else False\n )\n \n # Initialize model\n self._torch_model = SimpleTransformerBlock(\n n_features=X.shape[1],\n n_heads=self.n_heads,\n dropout=0.1\n ).to(self.device_)\n \n # Training setup\n optimizer = torch.optim.AdamW(self._torch_model.parameters(), \n lr=self.learning_rate)\n criterion = nn.MSELoss()\n \n # Training loop\n self._torch_model.train()\n for epoch in range(self.max_iter):\n total_loss = 0\n for batch_X, batch_mask in dataloader:\n # Move batch to device\n batch_X = batch_X.to(self.device_)\n batch_mask = batch_mask.to(self.device_)\n \n outputs = self._torch_model(batch_X.unsqueeze(0)).squeeze(0)\n loss = criterion(outputs[batch_mask == 1], \n batch_X[batch_mask == 1])\n \n optimizer.zero_grad()\n loss.backward()\n torch.nn.utils.clip_grad_norm_(\n self._torch_model.parameters(), \n max_norm=1.0\n )\n optimizer.step()\n \n total_loss += loss.item()\n \n if self.verbose > 0 and (epoch + 1) % 10 == 0:\n print(f'Epoch [{epoch+1}/{self.max_iter}], '\n f'Loss: {total_loss/len(dataloader):.4f}')\n \n return self\n\n def transform(self, X):\n check_is_fitted(self, ['statistics_', '_torch_model'])\n X = self._validate_input(X)\n \n # Initial imputation\n X_imputed, mask = self._initial_imputation(X)\n \n # Transform using the trained model\n self._torch_model.eval()\n with torch.no_grad():\n result = X_imputed.copy()\n \n # Process in batches\n batch_size = 1024\n for i in range(0, len(X), batch_size):\n batch = torch.FloatTensor(\n X_imputed[i:i+batch_size]\n ).to(self.device_)\n imputed = self._torch_model(\n batch.unsqueeze(0)\n ).squeeze(0).cpu().numpy()\n result[i:i+batch_size][mask[i:i+batch_size]] = \\\n imputed[mask[i:i+batch_size]]\n \n if self.add_indicator:\n return np.column_stack([result, self.indicator_])\n \n return result\n\n def fit_transform(self, X, y=None):\n return self.fit(X).transform(X)\n\n def get_feature_names_out(self, feature_names=None):\n check_is_fitted(self, ['statistics_'])\n \n if feature_names is None:\n feature_names = [f'x{i}' for i in range(len(self.statistics_))]\n \n feature_names_out = list(feature_names)\n if self.add_indicator:\n feature_names_out.extend([f'missing_indicator_{name}' \n for name in feature_names])\n \n return np.array(feature_names_out)\n\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, TensorDataset, random_split\nfrom sklearn.base import BaseEstimator, ClassifierMixin\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, classification_report\nfrom sklearn.utils.validation import check_X_y, check_array, check_is_fitted\n\nclass TransformerClassifier(BaseEstimator, ClassifierMixin):\n def __init__(\n self, \n n_features=None,\n n_classes=None,\n max_iter=100,\n batch_size=32,\n n_heads=4,\n learning_rate=0.001,\n dropout=0.1,\n validation_split=0.2,\n random_state=42\n ):\n self.n_features = n_features\n self.n_classes = n_classes\n self.max_iter = max_iter\n self.batch_size = batch_size\n self.n_heads = n_heads\n self.learning_rate = learning_rate\n self.dropout = dropout\n self.validation_split = validation_split\n self.random_state = random_state\n\n class _TransformerClassificationModel(nn.Module):\n def __init__(self, n_features, n_classes=4, n_heads=4, dropout=0.1):\n super().__init__()\n self.transformer_block = SimpleTransformerBlock(\n n_features=n_features, \n n_heads=n_heads, \n dropout=dropout\n )\n \n self.classifier = nn.Sequential(\n nn.Linear(n_features, n_features * 2),\n nn.BatchNorm1d(n_features * 2),\n nn.ReLU(),\n nn.Dropout(dropout),\n nn.Linear(n_features * 2, n_features),\n nn.BatchNorm1d(n_features),\n nn.ReLU(),\n nn.Dropout(dropout),\n nn.Linear(n_features, n_classes)\n )\n\n def forward(self, x):\n # Apply transformer block\n x = self.transformer_block(x)\n \n # Classification layer\n return self.classifier(x)\n\n def _validate_and_preprocess_input(self, X, y=None):\n # Validate input\n if y is not None:\n X, y = check_X_y(X, y)\n # Encode labels if they are not numeric\n self.label_encoder_ = LabelEncoder()\n y = self.label_encoder_.fit_transform(y)\n self.n_classes_ = len(np.unique(y))\n else:\n X = check_array(X)\n \n # Set n_features if not provided\n if self.n_features is None:\n self.n_features_ = X.shape[1]\n else:\n self.n_features_ = self.n_features\n\n # Convert to float tensor\n X = X.astype(np.float32)\n \n return X, y\n\n def _set_random_seed(self):\n torch.manual_seed(self.random_state)\n np.random.seed(self.random_state)\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = False\n\n def fit(self, X, y):\n # Validate and preprocess input\n X, y = self._validate_and_preprocess_input(X, y)\n \n # Set random seed\n self._set_random_seed()\n \n # Determine device\n self.device_ = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n \n # Convert to tensors\n X_tensor = torch.FloatTensor(X)\n y_tensor = torch.LongTensor(y)\n \n # Create dataset and split into train/validation\n full_dataset = TensorDataset(X_tensor, y_tensor)\n \n # Split into train and validation sets\n val_size = int(len(full_dataset) * self.validation_split)\n train_size = len(full_dataset) - val_size\n train_dataset, val_dataset = random_split(\n full_dataset, \n [train_size, val_size],\n generator=torch.Generator().manual_seed(self.random_state)\n )\n \n # Create dataloaders\n train_loader = DataLoader(\n train_dataset, \n batch_size=self.batch_size, \n shuffle=True\n )\n val_loader = DataLoader(\n val_dataset, \n batch_size=self.batch_size, \n shuffle=False\n )\n \n # Initialize model\n self.model_ = self._TransformerClassificationModel(\n n_features=self.n_features_, \n n_classes=self.n_classes_,\n n_heads=self.n_heads,\n dropout=self.dropout\n ).to(self.device_)\n \n # Loss and optimizer\n criterion = nn.CrossEntropyLoss()\n optimizer = torch.optim.AdamW(\n self.model_.parameters(), \n lr=self.learning_rate\n )\n \n # Training loop\n best_val_accuracy = 0\n for epoch in range(self.max_iter):\n self.model_.train()\n train_loss = 0\n \n for batch_X, batch_y in train_loader:\n batch_X = batch_X.to(self.device_)\n batch_y = batch_y.to(self.device_)\n \n # Forward pass\n outputs = self.model_(batch_X)\n loss = criterion(outputs, batch_y)\n \n # Backward pass and optimize\n optimizer.zero_grad()\n loss.backward()\n torch.nn.utils.clip_grad_norm_(self.model_.parameters(), max_norm=1.0)\n optimizer.step()\n \n train_loss += loss.item()\n \n # Validation\n self.model_.eval()\n val_loss = 0\n val_correct = 0\n val_total = 0\n \n with torch.no_grad():\n for batch_X, batch_y in val_loader:\n batch_X = batch_X.to(self.device_)\n batch_y = batch_y.to(self.device_)\n \n outputs = self.model_(batch_X)\n loss = criterion(outputs, batch_y)\n val_loss += loss.item()\n \n _, predicted = torch.max(outputs.data, 1)\n val_total += batch_y.size(0)\n val_correct += (predicted == batch_y).sum().item()\n \n val_accuracy = val_correct / val_total\n \n # Save best model\n if val_accuracy > best_val_accuracy:\n best_val_accuracy = val_accuracy\n torch.save(self.model_.state_dict(), 'best_model.pth')\n \n # Optional: print training progress\n if epoch % 10 == 0:\n print(f'Epoch [{epoch+1}/{self.max_iter}], '\n f'Train Loss: {train_loss/len(train_loader):.4f}, '\n f'Val Loss: {val_loss/len(val_loader):.4f}, '\n f'Val Accuracy: {val_accuracy:.4f}')\n \n # Load best model\n self.model_.load_state_dict(torch.load('best_model.pth'))\n \n return self\n\n def predict(self, X):\n # Validate input\n X, _ = self._validate_and_preprocess_input(X)\n \n # Ensure model is fitted\n check_is_fitted(self, ['model_', 'label_encoder_'])\n \n # Convert to tensor and move to device\n X_tensor = torch.FloatTensor(X).to(self.device_)\n \n # Set model to evaluation mode\n self.model_.eval()\n \n # Predict\n with torch.no_grad():\n outputs = self.model_(X_tensor)\n _, predicted = torch.max(outputs.data, 1)\n \n # Convert back to original labels\n return self.label_encoder_.inverse_transform(\n predicted.cpu().numpy()\n )\n\n def predict_proba(self, X):\n # Validate input\n X, _ = self._validate_and_preprocess_input(X)\n \n # Ensure model is fitted\n check_is_fitted(self, ['model_', 'label_encoder_'])\n \n # Convert to tensor and move to device\n X_tensor = torch.FloatTensor(X).to(self.device_)\n \n # Set model to evaluation mode\n self.model_.eval()\n \n # Predict probabilities\n with torch.no_grad():\n outputs = self.model_(X_tensor)\n probas = torch.softmax(outputs, dim=1)\n \n return probas.cpu().numpy()\n\n def score(self, X, y):\n # Validate input\n X, y = self._validate_and_preprocess_input(X, y)\n \n # Predict\n y_pred = self.predict(X)\n \n # Calculate accuracy\n return accuracy_score(y, self.label_encoder_.transform(y_pred))\n\n# Example usage\ndef test_transformer_classifier():\n from sklearn.datasets import load_iris\n from sklearn.model_selection import train_test_split\n\n # Load dataset\n X, y = load_iris(return_X_y=True)\n \n # Split the data\n X_train, X_test, y_train, y_test = train_test_split(\n X, y, test_size=0.2, random_state=42\n )\n \n # Initialize and train the classifier\n clf = TransformerClassifier(\n max_iter=100,\n batch_size=16,\n learning_rate=0.001,\n random_state=42\n )\n \n clf.fit(X_train, y_train)\n \n # Make predictions\n y_pred = clf.predict(X_test)\n \n # Print classification report\n print(classification_report(y_test, y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-08T02:39:35.396666Z","iopub.execute_input":"2025-02-08T02:39:35.397070Z","iopub.status.idle":"2025-02-08T02:39:39.386878Z","shell.execute_reply.started":"2025-02-08T02:39:35.397025Z","shell.execute_reply":"2025-02-08T02:39:39.385679Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optunity\nprint(optunity.solvers.__all__)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-08T02:39:53.596753Z","iopub.status.idle":"2025-02-08T02:39:53.597402Z","shell.execute_reply.started":"2025-02-08T02:39:53.597034Z","shell.execute_reply":"2025-02-08T02:39:53.597061Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"probabilities = np.zeros((X.shape[0], len(np.unique(y)))) # To store probabilities for each class\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.ensemble import VotingClassifier\nfrom catboost import CatBoostClassifier\nfrom lightgbm import LGBMClassifier\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.decomposition import PCA\nfrom imblearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import MinMaxScaler, PolynomialFeatures\n\n# Assuming X and y are already defined\nn_splits = 5\n\n# Calculate class weights based on the number of samples for each class\nclass_weights = compute_class_weight('balanced', classes=np.unique(y), y=y)\nprobabilities_train = np.zeros((X.shape[0], len(np.unique(y)))) # To store probabilities for each class\n\n# Best parameters for LightGBM\nbest_parameters_lgbm = {\n 'class_weight': 'balanced',\n 'verbosity': -1,\n 'num_leaves': 43,\n 'max_depth': 6,\n 'learning_rate': 0.023041087955220526,\n 'n_estimators': 184,\n 'min_child_samples': 5,\n 'subsample': 0.702785794333845,\n 'colsample_bytree': 0.735399472263,\n 'reg_alpha': 5.166181985577914,\n 'reg_lambda': 0.014957632475494106\n}\n\n# Best parameters for CatBoost\nbest_parameters_cat = {\n 'depth': 8,\n 'class_weights': class_weights.tolist(),\n 'learning_rate': 0.01549897905168682,\n 'iterations': 214,\n 'l2_leaf_reg': 0.028903423453919733,\n 'early_stopping_rounds': 20,\n 'task_type': 'GPU',\n 'devices': '0',\n 'random_seed': 42,\n 'verbose': 0,\n 'loss_function': 'MultiClass'\n}\n\nbest_parameters_xgb = {'max_depth': 14, 'learning_rate': 0.07575964240607065, 'n_estimators': 475, 'min_child_weight': 4, 'subsample': 0.7005498757000181, 'colsample_bytree': 0.9544170496730132,\n 'gamma': 3.240589984983939e-08, 'scale_pos_weight': 0.13702336163101592,\n 'tree_method': 'gpu_hist', # Use GPU acceleration\n 'gpu_id': 0, 'class_weight': 'balanced',\n} # Specify the GPU device ID (usually 0 if you have one GPU)}\n\nbest_paramters_rf = {'n_estimators': 250, 'max_depth': 10, \n 'min_samples_split': 4, 'min_samples_leaf': 1, \n 'max_features': 'auto',\n 'class_weight': 'balanced',\n 'random_state': 42,}\n\n# Define hyperparameters for Decision Tree\nbest_param_dic = {'max_depth': 15, 'min_samples_split': 7, 'min_samples_leaf': 1, 'max_features': 'auto'}\n \n# Initialize StratifiedKFold\nskf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n\n# Cross-validation\nkappa_scores = []\n\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.neural_network import BernoulliRBM\nfrom sklearn.svm import SVC\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.manifold import TSNE\nfrom sklearn.gaussian_process import GaussianProcessClassifier\nfrom sklearn.gaussian_process.kernels import RBF\nfrom sklearn.feature_selection import SelectKBest, f_classif, SelectFromModel\n\n# Scale X_train and X_val between 0 and 1\n\n# Cross-validation\nfor fold ,(train_index, val_index) in enumerate(skf.split(X, y)):\n X_train, X_val = X.iloc[train_index,1:], X.iloc[val_index,1:]\n y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n\n # Initialize models\n\n model_lgbm = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('model_lgbm', LGBMClassifier(**best_parameters_lgbm))])\n \n model_cat = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('model_cat', CatBoostClassifier(**best_parameters_cat))])\n \n model_rf = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('model_rf', RandomForestClassifier(**best_paramters_rf))])\n model_dic = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('model_dic', DecisionTreeClassifier(**best_param_dic, random_state=42))])\n \n model_xgb = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('model_xgb', XGBClassifier(**best_parameters_xgb, random_state=42))])\n \n\n\n # Create SVC pipeline\n svc_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('scale', MinMaxScaler()),\n\n ('rbm', BernoulliRBM(n_components=2,n_iter=300, random_state=42)),\n\n ('classifier', SVC(C=0.1, kernel='rbf', gamma=0.001, \n probability=True, random_state=42))\n ])\n\n\n knn_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n\n ('scale', MinMaxScaler()),\n ('rbm', BernoulliRBM(n_components=20, n_iter=3, random_state=42)),\n #('pca', PCA(n_components=0.95)),\n\n ('classifier', KNeighborsClassifier(\n n_neighbors=20,\n weights='distance',\n algorithm='auto'\n ))])\n\n gb_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('scale', MinMaxScaler()),\n ('rbm', BernoulliRBM(n_components=20, n_iter=10, random_state=42)),\n #('pca', PCA(n_components=0.95)),\n\n ('classifier', GradientBoostingClassifier(\n n_estimators=10,\n learning_rate=0.02,\n max_depth=6,\n random_state=42\n ))\n ]) \n\n from sklearn.linear_model import LogisticRegression\n\n lr_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('scale', MinMaxScaler()),\n ('rbm', BernoulliRBM(n_components=10, n_iter=50, random_state=42)),\n #('pca', PCA(n_components=0.95)),\n ('classifier', LogisticRegression(\n C=2,\n max_iter=10,\n multi_class='multinomial',\n solver='lbfgs',\n random_state=42\n ))\n ])\n from sklearn.neural_network import MLPClassifier\n nn_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('poly', PolynomialFeatures(degree=3, include_bias=False)), # Generate polynomial features\n\n ('smote', SMOTE(random_state=42)),\n ('anova', SelectKBest(score_func=f_classif, k=19)), # ANOVA for feature selection\n ('scaler', MinMaxScaler()),\n ('classifier', MLPClassifier(\n hidden_layer_sizes=(41,10),\n activation='relu',\n solver='adam',\n alpha=0.00011395040226719397,\n learning_rate='adaptive',\n learning_rate_init= 0.044740260552943,\n power_t=0.40175197089329534,\n momentum=0.9123843613121819,\n nesterovs_momentum=True,\n early_stopping=True,\n validation_fraction= 0.1001433219599319,\n beta_1=0.9,\n beta_2=0.999,\n epsilon=1e-8,\n max_iter=39,\n random_state=1\n )),\n ])\n \n\n gp_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('poly', PolynomialFeatures(degree=2, include_bias=False)), # Generate polynomial features\n\n ('scale', MinMaxScaler()),\n ('rbm', BernoulliRBM(n_components=2, n_iter=10, random_state=42)),\n #('pca', PCA(n_components=0.95)),\n ('classifier', GaussianProcessClassifier(\n kernel=RBF(length_scale=1.0),\n random_state=42\n ))\n])\n\n fast_rf_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('poly', PolynomialFeatures(degree=2, include_bias=False)), # Generate polynomial features\n ('scale', MinMaxScaler()),\n ('rbm', BernoulliRBM(n_components=2, n_iter=20, random_state=42)),\n #('pca', PCA(n_components=0.95)),\n ('classifier', RandomForestClassifier(\n n_estimators=10,\n max_depth=10,\n min_samples_split=5,\n n_jobs=-1, # parallel processing\n random_state=42,\n class_weight=\"balanced\"\n ))\n])\n cat_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('poly', PolynomialFeatures(degree=3, include_bias=False)), # Generate polynomial features\n\n ('scale', MinMaxScaler()),\n #('pca', PCA(n_components=0.95)),\n ('rbm', BernoulliRBM(n_components=100, n_iter=100, random_state=42)),\n ('classifier', model_cat)\n])\n from sklearn.ensemble import StackingClassifier\n estimators=[\n ('nn', nn_pipeline),\n ('svc', svc_pipeline),\n ('gb',model_lgbm),\n]\n model_rf_ens = Pipeline([\n ('poly', PolynomialFeatures(degree=2, include_bias=False)), # Generate polynomial features\n ('model_rf', RandomForestClassifier(**best_paramters_rf))])\n # Create the StackingClassifier\n s1 = StackingClassifier(\n estimators=estimators,\n final_estimator= model_rf_ens,\n cv=3, # Cross-validation for stacking\n passthrough=False, # Include original features in meta-model training\n)\n estimators=[\n ('rf', fast_rf_pipeline),\n ('lr', lr_pipeline),\n ('knn', knn_pipeline),\n ('gb',model_lgbm),\n]\n\n # Create the StackingClassifier\n s2 = StackingClassifier(\n estimators=estimators,\n final_estimator= model_rf_ens,\n cv=3, # Cross-validation for stacking\n passthrough=False, # Include original features in meta-model training\n)\n estimators=[\n ('model_lgbm', model_lgbm),\n ('model_cat', model_cat),\n ('model_rf', model_rf),\n]\n svc_ens = Pipeline([\n ('scale', MinMaxScaler()),\n ('poly', PolynomialFeatures(degree=2, include_bias=False)), # Generate polynomial features\n\n ('rbm', BernoulliRBM(n_components=2,n_iter=300, random_state=42)),\n\n ('classifier', SVC(C=0.1, kernel='rbf', gamma=0.001, \n probability=True, random_state=42))\n ])\n\n # Create the StackingClassifier\n s3 = StackingClassifier(\n estimators=estimators,\n final_estimator= svc_ens,\n cv=3, # Cross-validation for stacking\n passthrough=False, # Include original features in meta-model training\n)\n\n from sklearn.preprocessing import StandardScaler\n\n\n\n\n \n # Hyperparameters for MLP\n hidden_layer_sizes = (\n 87,\n 60\n )\n activation = 'relu'\n solver = 'adam'\n alpha = 0.006940228915990604\n learning_rate_init =0.00022039953967804564\n\n # Hyperparameters for XGBoost\n xgb_max_depth = 3\n xgb_learning_rate = 0.137087890107909\n xgb_n_estimators = 70\n\n # Hyperparameters for PCA\n pca_components = 0.9262728004165126\n\n # Final Estimator (Blender) Hyperparameters\n rf_n_estimators = 52\n rf_max_depth = 17\n\n # Define base estimators\n model_mlp = Pipeline([\n ('scale', StandardScaler()),\n ('TransformerImputer', TransformerImputer(strategy='median')),\n ('pca', PCA(n_components=pca_components)),\n ('classifier', MLPClassifier(\n hidden_layer_sizes=hidden_layer_sizes,\n activation=activation,\n solver=solver,\n alpha=alpha,\n learning_rate_init=learning_rate_init,\n max_iter=200,\n random_state=42\n ))\n ])\n model_xgb = Pipeline([\n ('scale', MinMaxScaler()),\n ('TransformerImputer', TransformerImputer(strategy='median')),\n ('pca', PCA(n_components=pca_components)),\n ('classifier', XGBClassifier(\n max_depth=xgb_max_depth,\n learning_rate=xgb_learning_rate,\n n_estimators=xgb_n_estimators,\n use_label_encoder=False,\n eval_metric='mlogloss',\n random_state=42\n ))\n ])\n model_rf_ens = Pipeline([\n ('TransformerImputer', TransformerImputer(strategy='median')),\n ('model_rf', RandomForestClassifier(**best_paramters_rf))])\n # Stacking Classifier\n estimators = [\n ('mlp', model_mlp),\n ('xgb', model_xgb),\n ]\n\n s4 = StackingClassifier(\n estimators=estimators,\n final_estimator=model_rf_ens,\n cv=3,\n passthrough=True\n )\n\n # transformer imputer \n # Create a Voting Classifier\n voting_clf = VotingClassifier(\n estimators=[\n ('lgbm', model_lgbm),\n ('catboost', model_cat),\n ('rf', model_rf), # Adding Random Forest to the ensemble\n ('dic', model_dic), # Adding Random Forest to the ensemble\n ('svc', svc_pipeline), # Adding Random Forest to the ensemble\n ('knn',knn_pipeline),\n ('gb',gb_pipeline),\n #('cat_pipe',cat_pipeline),\n ('nn',nn_pipeline),\n ('s1',s1),\n ('s2',s2),\n ('s3',s3),\n # ('s4',s4)\n ],\n voting='soft', # Use soft voting\n weights=[1.41,1.8,0.26,0.05,0.51,0.04,0.021,1.3,1.5,0.5,0.5] \n )\n\n \n #voting_clf = s1 \n # Train the Voting Classifier\n voting_clf.fit(X_train, y_train)\n #y_extra = voting_clf.predict(X_extra.iloc[:,1:])\n #X_train_with_extra = pd.concat([X_train, X_extra.iloc[:,1:]], ignore_index=True) # `ignore_index=True` resets the index\n #y_train_with_extra = np.concatenate([y_train, y_extra])\n #voting_clf.fit(X_train_with_extra, y_train_with_extra)\n\n # Predict on the validation set\n y_pred = voting_clf.predict(X_val)\n\n prob = voting_clf.predict_proba(X_val)\n \n # Calculate quadratic weighted kappa\n kappa = cohen_kappa_score(y_val,y_pred, weights='quadratic')\n kappa_scores.append(kappa)\n print(f\"Fold Kappa: {kappa}\")\n if fold==0: \n predictions = voting_clf.predict_proba(X_test.iloc[:,1:]) / 5\n else:\n predictions += voting_clf.predict_proba(X_test.iloc[:,1:]) / 5\n \n# Average kappa score across all folds\naverage_kappa = np.mean(kappa_scores)\nprint(f\"Average Quadratic Weighted Kappa: {average_kappa}\")","metadata":{"execution":{"iopub.status.busy":"2025-02-08T02:39:53.599009Z","iopub.status.idle":"2025-02-08T02:39:53.599556Z","shell.execute_reply.started":"2025-02-08T02:39:53.599261Z","shell.execute_reply":"2025-02-08T02:39:53.599288Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"probabilities = np.zeros((X.shape[0], len(np.unique(y)))) # To store probabilities for each class\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.ensemble import VotingClassifier\nfrom catboost import CatBoostClassifier\nfrom lightgbm import LGBMClassifier\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.decomposition import PCA\nfrom imblearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\n\n# Assuming X and y are already defined\nn_splits = 5\n\n# Calculate class weights based on the number of samples for each class\nclass_weights = compute_class_weight('balanced', classes=np.unique(y), y=y)\nprobabilities_train = np.zeros((X.shape[0], len(np.unique(y)))) # To store probabilities for each class\n\n# Best parameters for LightGBM\nbest_parameters_lgbm = {\n 'class_weight': 'balanced',\n 'verbosity': -1,\n 'num_leaves': 43,\n 'max_depth': 6,\n 'learning_rate': 0.023041087955220526,\n 'n_estimators': 184,\n 'min_child_samples': 5,\n 'subsample': 0.702785794333845,\n 'colsample_bytree': 0.735399472263,\n 'reg_alpha': 5.166181985577914,\n 'reg_lambda': 0.014957632475494106\n}\n\n# Best parameters for CatBoost\nbest_parameters_cat = {\n 'depth': 8,\n 'class_weights': class_weights.tolist(),\n 'learning_rate': 0.01549897905168682,\n 'iterations': 214,\n 'l2_leaf_reg': 0.028903423453919733,\n 'early_stopping_rounds': 20,\n 'task_type': 'GPU',\n 'devices': '0',\n 'random_seed': 42,\n 'verbose': 0,\n 'loss_function': 'MultiClass'\n}\n\nbest_parameters_xgb = {'max_depth': 14, 'learning_rate': 0.07575964240607065, 'n_estimators': 475, 'min_child_weight': 4, 'subsample': 0.7005498757000181, 'colsample_bytree': 0.9544170496730132,\n 'gamma': 3.240589984983939e-08, 'scale_pos_weight': 0.13702336163101592,\n 'tree_method': 'gpu_hist', # Use GPU acceleration\n 'gpu_id': 0, 'class_weight': 'balanced',\n} # Specify the GPU device ID (usually 0 if you have one GPU)}\n\nbest_paramters_rf = {'n_estimators': 250, 'max_depth': 10, \n 'min_samples_split': 4, 'min_samples_leaf': 1, \n 'max_features': 'auto',\n 'class_weight': 'balanced',\n 'random_state': 42,}\n\n# Define hyperparameters for Decision Tree\nbest_param_dic = {'max_depth': 15, 'min_samples_split': 7, 'min_samples_leaf': 1, 'max_features': 'auto'}\n \n# Initialize StratifiedKFold\nskf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n\n# Cross-validation\nkappa_scores = []\n\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.neural_network import BernoulliRBM\nfrom sklearn.svm import SVC\nfrom sklearn.ensemble import GradientBoostingClassifier\n \nfrom sklearn.gaussian_process import GaussianProcessClassifier\nfrom sklearn.gaussian_process.kernels import RBF\nfrom sklearn.feature_selection import SelectKBest, f_classif, SelectFromModel\n\n# Scale X_train and X_val between 0 and 1\n\n# Cross-validation\nfor fold ,(train_index, val_index) in enumerate(skf.split(X, y)):\n X_train, X_val = X.iloc[train_index,1:], X.iloc[val_index,1:]\n y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n\n # Initialize models\n\n model_lgbm = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('model_lgbm', LGBMClassifier(**best_parameters_lgbm))])\n \n model_cat = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('model_cat', CatBoostClassifier(**best_parameters_cat))])\n \n model_rf = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('model_rf', RandomForestClassifier(**best_paramters_rf))])\n model_dic = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('model_dic', DecisionTreeClassifier(**best_param_dic, random_state=42))])\n \n model_xgb = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('model_xgb', XGBClassifier(**best_parameters_xgb, random_state=42))])\n \n\n\n # Create SVC pipeline\n svc_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('scale', MinMaxScaler()),\n\n ('rbm', BernoulliRBM(n_components=2,n_iter=300, random_state=42)),\n\n ('classifier', SVC(C=0.1, kernel='rbf', gamma=0.001, \n probability=True, random_state=42))\n ])\n\n\n knn_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n\n ('scale', MinMaxScaler()),\n ('rbm', BernoulliRBM(n_components=20, n_iter=3, random_state=42)),\n #('pca', PCA(n_components=0.95)),\n\n ('classifier', KNeighborsClassifier(\n n_neighbors=20,\n weights='distance',\n algorithm='auto'\n ))])\n\n gb_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('scale', MinMaxScaler()),\n ('rbm', BernoulliRBM(n_components=20, n_iter=10, random_state=42)),\n #('pca', PCA(n_components=0.95)),\n\n ('classifier', GradientBoostingClassifier(\n n_estimators=10,\n learning_rate=0.02,\n max_depth=6,\n random_state=42\n ))\n ]) \n\n from sklearn.linear_model import LogisticRegression\n\n lr_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('scale', MinMaxScaler()),\n ('rbm', BernoulliRBM(n_components=10, n_iter=50, random_state=42)),\n #('pca', PCA(n_components=0.95)),\n ('classifier', LogisticRegression(\n C=2,\n max_iter=10,\n multi_class='multinomial',\n solver='lbfgs',\n random_state=42\n ))\n ])\n from sklearn.neural_network import MLPClassifier\n nn_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('smote', SMOTE(random_state=42)),\n ('anova', SelectKBest(score_func=f_classif, k=19)), # ANOVA for feature selection\n ('scaler', MinMaxScaler()),\n ('classifier', MLPClassifier(\n hidden_layer_sizes=(41,10),\n activation='relu',\n solver='adam',\n alpha=0.00011395040226719397,\n learning_rate='adaptive',\n learning_rate_init= 0.044740260552943,\n power_t=0.40175197089329534,\n momentum=0.9123843613121819,\n nesterovs_momentum=True,\n early_stopping=True,\n validation_fraction= 0.1001433219599319,\n beta_1=0.9,\n beta_2=0.999,\n epsilon=1e-8,\n max_iter=39,\n random_state=1\n )),\n ])\n \n\n gp_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('scale', MinMaxScaler()),\n ('rbm', BernoulliRBM(n_components=2, n_iter=10, random_state=42)),\n #('pca', PCA(n_components=0.95)),\n ('classifier', GaussianProcessClassifier(\n kernel=RBF(length_scale=1.0),\n random_state=42\n ))\n])\n\n fast_rf_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n ('scale', MinMaxScaler()),\n ('rbm', BernoulliRBM(n_components=2, n_iter=20, random_state=42)),\n #('pca', PCA(n_components=0.95)),\n ('classifier', RandomForestClassifier(\n n_estimators=10,\n max_depth=10,\n min_samples_split=5,\n n_jobs=-1, # parallel processing\n random_state=42,\n class_weight=\"balanced\"\n ))\n])\n cat_pipeline = Pipeline([\n ('SimpleImputer', SimpleImputer(strategy='median')),\n\n ('scale', MinMaxScaler()),\n #('pca', PCA(n_components=0.95)),\n ('rbm', BernoulliRBM(n_components=100, n_iter=100, random_state=42)),\n ('classifier', model_cat)\n])\n from sklearn.ensemble import StackingClassifier\n estimators=[\n ('nn', nn_pipeline),\n ('svc', svc_pipeline),\n ('gb',model_lgbm),\n]\n model_rf_ens = Pipeline([\n ('model_rf', RandomForestClassifier(**best_paramters_rf))])\n # Create the StackingClassifier\n s1 = StackingClassifier(\n estimators=estimators,\n final_estimator= model_rf_ens,\n cv=3, # Cross-validation for stacking\n passthrough=False, # Include original features in meta-model training\n)\n estimators=[\n ('rf', fast_rf_pipeline),\n ('lr', lr_pipeline),\n ('knn', knn_pipeline),\n ('gb',model_lgbm),\n]\n\n # Create the StackingClassifier\n s2 = StackingClassifier(\n estimators=estimators,\n final_estimator= model_rf_ens,\n cv=3, # Cross-validation for stacking\n passthrough=False, # Include original features in meta-model training\n)\n estimators=[\n ('model_lgbm', model_lgbm),\n ('model_cat', model_cat),\n ('model_rf', model_rf),\n]\n svc_ens = Pipeline([\n ('scale', MinMaxScaler()),\n\n ('rbm', BernoulliRBM(n_components=2,n_iter=300, random_state=42)),\n\n ('classifier', SVC(C=0.1, kernel='rbf', gamma=0.001, \n probability=True, random_state=42))\n ])\n\n # Create the StackingClassifier\n s3 = StackingClassifier(\n estimators=estimators,\n final_estimator= svc_ens,\n cv=3, # Cross-validation for stacking\n passthrough=False, # Include original features in meta-model training\n)\n model_lgbm_with_transformer = Pipeline([\n ('scale', MinMaxScaler()), \n ('TransformerImputer', TransformerImputer()),\n ('model_lgbm', LGBMClassifier(**best_parameters_lgbm))])\n \n model_lr_with_transformer = Pipeline([\n ('scale', MinMaxScaler()), \n ('TransformerImputer', TransformerImputer()),\n ('pca', PCA(n_components=0.95)),\n ('classifier', LogisticRegression(\n C=2,\n max_iter=10,\n multi_class='multinomial',\n solver='lbfgs',\n random_state=42\n ))])\n model_mlp_with_transformer = Pipeline([\n ('scale', MinMaxScaler()), \n ('TransformerImputer', TransformerImputer(strategy='median')),\n ('smote', SMOTE(random_state=42)),\n ('anova', SelectKBest(score_func=f_classif, k=8)), # ANOVA for feature selection\n ('classifier', MLPClassifier(\n hidden_layer_sizes=(41,10),\n activation='relu',\n solver='adam',\n alpha=0.00011395040226719397,\n learning_rate='adaptive',\n learning_rate_init= 0.044740260552943,\n power_t=0.40175197089329534,\n momentum=0.9123843613121819,\n nesterovs_momentum=True,\n early_stopping=True,\n validation_fraction= 0.1001433219599319,\n beta_1=0.9,\n beta_2=0.999,\n epsilon=1e-8,\n max_iter=10,\n random_state=1\n ))]) \n s3 = StackingClassifier(\n estimators=estimators,\n final_estimator= svc_ens,\n cv=3, # Cross-validation for stacking\n passthrough=False, # Include original features in meta-model training\n) \n estimators=[ \n #('lgbm', model_lgbm),\n ('mlp', nn_pipeline),\n ('gp', gp_pipeline),\n ('mlp_with_tf',model_mlp_with_transformer),\n ]\n s4 = StackingClassifier(\n estimators=estimators,\n final_estimator= model_rf_ens,\n cv=2, # Cross-validation for stacking\n passthrough=False, # Include original features in meta-model training\n) \n \n estimators=[ \n ('s1',s1),\n ('s2',s2),\n ('s3',s3)\n ]\n \n s5 = StackingClassifier(\n estimators=estimators,\n final_estimator= model_rf_ens,\n cv=3, # Cross-validation for stacking\n passthrough=False, # Include original features in meta-model training\n) \n # transformer imputer \n # Create a Voting Classifier\n voting_clf = VotingClassifier(\n estimators=[\n ('lgbm', model_lgbm),\n ('catboost', model_cat),\n ('rf', model_rf), # Adding Random Forest to the ensemble\n ('dic', model_dic), # Adding Random Forest to the ensemble\n ('svc', svc_pipeline), # Adding Random Forest to the ensemble\n ('knn',knn_pipeline),\n ('gb',gb_pipeline),\n #('cat_pipe',cat_pipeline),\n ('nn',nn_pipeline),\n ('s1',s1),\n ('s2',s2),\n ('s3',s3),\n \n ],\n voting='soft', # Use soft voting\n weights=[1.41,1.8,0.26,0.05,0.51,0.04,0.021,1.3,1.5,0.5,0.5] \n )\n\n \n #voting_clf = s1 \n # Train the Voting Classifier\n voting_clf.fit(X_train, y_train)\n #y_extra = voting_clf.predict(X_extra.iloc[:,1:])\n #X_train_with_extra = pd.concat([X_train, X_extra.iloc[:,1:]], ignore_index=True) # `ignore_index=True` resets the index\n #y_train_with_extra = np.concatenate([y_train, y_extra])\n #voting_clf.fit(X_train_with_extra, y_train_with_extra)\n\n # Predict on the validation set\n y_pred = voting_clf.predict(X_val)\n\n prob = voting_clf.predict_proba(X_val)\n \n # Calculate quadratic weighted kappa\n kappa = cohen_kappa_score(y_val,y_pred, weights='quadratic')\n kappa_scores.append(kappa)\n print(f\"Fold Kappa: {kappa}\")\n if fold==0: \n predictions = voting_clf.predict_proba(X_test.iloc[:,1:]) / 5\n else:\n predictions += voting_clf.predict_proba(X_test.iloc[:,1:]) / 5\n \n# Average kappa score across all folds\naverage_kappa = np.mean(kappa_scores)\nprint(f\"Average Quadratic Weighted Kappa: {average_kappa}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-08T02:39:53.601517Z","iopub.status.idle":"2025-02-08T02:39:53.602033Z","shell.execute_reply.started":"2025-02-08T02:39:53.601773Z","shell.execute_reply":"2025-02-08T02:39:53.601802Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"highest_probs = np.argmax(predictions, axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-08T02:39:53.603364Z","iopub.status.idle":"2025-02-08T02:39:53.603861Z","shell.execute_reply.started":"2025-02-08T02:39:53.603595Z","shell.execute_reply":"2025-02-08T02:39:53.603621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_classes = y.unique().shape[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-08T02:39:53.605198Z","iopub.status.idle":"2025-02-08T02:39:53.605709Z","shell.execute_reply.started":"2025-02-08T02:39:53.605457Z","shell.execute_reply":"2025-02-08T02:39:53.605483Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n\nsubmission = pd.DataFrame({'id': test['id'], 'sii': highest_probs})\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Submission file created.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-08T02:39:53.607011Z","iopub.status.idle":"2025-02-08T02:39:53.607530Z","shell.execute_reply.started":"2025-02-08T02:39:53.607246Z","shell.execute_reply":"2025-02-08T02:39:53.607271Z"}},"outputs":[],"execution_count":null}]}