1/ Introduction
\n","metadata":{}},{"cell_type":"markdown","source":"3/ Import data
","metadata":{}},{"cell_type":"code","source":"path = '../input/child-mind-institute-problematic-internet-use/'\n\ntrain = pd.read_csv(path + 'train.csv', index_col = 'id')\nprint(\"The train data has the shape: \",train.shape)\ntest = pd.read_csv(path + 'test.csv', index_col = 'id')\nprint(\"The test data has the shape: \",test.shape)\nprint(\"\")\nprint(\"Total number of missing training values: \", train.isna().sum().sum())\n\npath = '../input/piu-bp-children/'\nBP_boys_table = pd.read_csv (path + 'BP_boys_table.csv', header=None, names=['age', 'description', \\\n 'value_1', 'value_2', 'value_3', 'value_4', 'value_5', 'value_6', 'value_7'])\nBP_girls_table = pd.read_csv (path + 'BP_girls_table.csv', header=None, names=['age', 'description', \\\n 'value_1', 'value_2', 'value_3', 'value_4', 'value_5', 'value_6', 'value_7'])\nBP_girls_table.head(5)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import blood pressure tables into a dictionary\ndef import_BP_table (df):\n # First fill the first column with the corresponding age\n for index, row in df.iterrows():\n if pd.notna(row['age']):\n last_age_seen = row['age']\n else:\n df.loc[index,'age'] = last_age_seen\n ages = (df['age'].drop_duplicates().sort_values().to_list())\n value_cols_names = ['value_' + str(i) for i in range (1,8)]\n filters = ['Prehypertension', 'Stage_1_HTN', 'Stage_2_HTN']\n BP_table = {}\n for age in ages:\n Selected_age_df = df[df['age'] == age]\n Length_df = Selected_age_df [Selected_age_df ['description'] == 'Length']\n Lengths_array = Length_df [value_cols_names].to_numpy()\n BP_values_df = Selected_age_df [Selected_age_df ['description'].isin (filters)]\n BP_values_array = BP_values_df [value_cols_names].to_numpy()\n BP_table [age] = (Lengths_array, BP_values_array)\n return BP_table\n \nBP_tables = {'boys': import_BP_table (BP_boys_table), \\\n 'girls': import_BP_table (BP_girls_table)}\nprint (BP_tables ['girls'])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#4/ Exploratory data analysis","metadata":{}},{"cell_type":"markdown","source":"
4/ Feature engineering
","metadata":{}},{"cell_type":"markdown","source":"6/ Feature selection
","metadata":{}},{"cell_type":"code","source":"# List all available features to select from\nprint (train.columns)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_features = ['Basic_Demos-Age', 'Basic_Demos-Sex', 'CGAS-CGAS_Score', 'Physical-BMI', \\\n 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference', 'Physical-Diastolic_BP', \\\n 'Physical-HeartRate', 'Physical-Systolic_BP', 'Fitness_Endurance-Max_Stage', \\\n 'Fitness_Endurance-Time_Sec', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND', 'FGC-FGC_GSND_Zone', \\\n 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU', 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', \\\n 'FGC-FGC_SRR', 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-BIA_Activity_Level_num', \\\n 'BIA-BIA_BMC', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM', 'BIA-BIA_FFMI', \\\n 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num', 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', \\\n 'BIA-BIA_SMM', 'BIA-BIA_TBW', 'PAQ_Total', 'SDS-SDS_Total_T']\nengineered_features = ['Basic_Demos-Age', 'Basic_Demos-Sex', 'CGAS-CGAS_Score', 'Physical-BMI', \\\n 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference', \\\n 'Physical-HeartRate', 'Fitness_Endurance-Max_Stage', \\\n 'Fitness_Endurance-Time_Sec', 'SDS-SDS_Total_T', 'PreInt_EduHx-computerinternet_hoursday', \\\n 'PAQ_Total', 'PreInt_EduHx-AvgTemp', 'PreInt_EduHx-AvgSuns', 'BIA-BIA_Activity_Level_num', \\\n 'FGC_total', 'Physical-BP_category', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours', \\\n 'BMR_Weight', 'DEE_Weight', 'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status']","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#6/ Ensemble regression models
","metadata":{}},{"cell_type":"markdown","source":"6/ Neural network models: Tabnet","metadata":{}},{"cell_type":"code","source":"# New: TabNet \n\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom pytorch_tabnet.callbacks import Callback\nimport os\nimport torch\nfrom pytorch_tabnet.callbacks import Callback\n\nclass TabNetWrapper(BaseEstimator, RegressorMixin):\n def __init__(self, **kwargs):\n self.model = TabNetRegressor(**kwargs)\n self.kwargs = kwargs\n self.imputer = SimpleImputer(strategy='median')\n self.best_model_path = 'best_tabnet_model.pt'\n \n def fit(self, X, y):\n # Handle missing values\n X_imputed = self.imputer.fit_transform(X)\n \n if hasattr(y, 'values'):\n y = y.values\n \n # Create internal validation set\n X_train, X_valid, y_train, y_valid = train_test_split(\n X_imputed, \n y, \n test_size=0.2,\n random_state=42\n )\n \n # Train TabNet model\n history = self.model.fit(\n X_train=X_train,\n y_train=y_train.reshape(-1, 1),\n eval_set=[(X_valid, y_valid.reshape(-1, 1))],\n eval_name=['valid'],\n eval_metric=['mse'],\n max_epochs=500,\n patience=50,\n batch_size=1024,\n virtual_batch_size=128,\n num_workers=0,\n drop_last=False,\n callbacks=[\n TabNetPretrainedModelCheckpoint(\n filepath=self.best_model_path,\n monitor='valid_mse',\n mode='min',\n save_best_only=True,\n verbose=True\n )\n ]\n )\n \n # Load the best model\n if os.path.exists(self.best_model_path):\n self.model.load_model(self.best_model_path)\n os.remove(self.best_model_path) # Remove temporary file\n \n return self\n \n def predict(self, X):\n X_imputed = self.imputer.transform(X)\n return self.model.predict(X_imputed).flatten()\n \n def __deepcopy__(self, memo):\n # Add deepcopy support for scikit-learn\n cls = self.__class__\n result = cls.__new__(cls)\n memo[id(self)] = result\n for k, v in self.__dict__.items():\n setattr(result, k, deepcopy(v, memo))\n return result\n\n# TabNet hyperparameters\nTabNet_Params = {\n 'n_d': 64, # Width of the decision prediction layer\n 'n_a': 64, # Width of the attention embedding for each step\n 'n_steps': 5, # Number of steps in the architecture\n 'gamma': 1.5, # Coefficient for feature selection regularization\n 'n_independent': 2, # Number of independent GLU layer in each GLU block\n 'n_shared': 2, # Number of shared GLU layer in each GLU block\n 'lambda_sparse': 1e-4, # Sparsity regularization\n 'optimizer_fn': torch.optim.Adam,\n 'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n 'mask_type': 'entmax',\n 'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n 'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n 'verbose': 1,\n 'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\nclass TabNetPretrainedModelCheckpoint(Callback):\n def __init__(self, filepath, monitor='val_loss', mode='min', \n save_best_only=True, verbose=1):\n super().__init__() # Initialize parent class\n self.filepath = filepath\n self.monitor = monitor\n self.mode = mode\n self.save_best_only = save_best_only\n self.verbose = verbose\n self.best = float('inf') if mode == 'min' else -float('inf')\n \n def on_train_begin(self, logs=None):\n self.model = self.trainer # Use trainer itself as model\n \n def on_epoch_end(self, epoch, logs=None):\n logs = logs or {}\n current = logs.get(self.monitor)\n if current is None:\n return\n \n # Check if current metric is better than best\n if (self.mode == 'min' and current < self.best) or \\\n (self.mode == 'max' and current > self.best):\n if self.verbose:\n print(f'\\nEpoch {epoch}: {self.monitor} improved from {self.best:.4f} to {current:.4f}')\n self.best = current\n if self.save_best_only:\n self.model.save_model(self.filepath) # Save the entire model","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# TabNet hyperparameters\nTabNet_Params = {\n 'n_d': 64, # Width of the decision prediction layer\n 'n_a': 64, # Width of the attention embedding for each step\n 'n_steps': 5, # Number of steps in the architecture\n 'gamma': 1.5, # Coefficient for feature selection regularization\n 'n_independent': 2, # Number of independent GLU layer in each GLU block\n 'n_shared': 2, # Number of shared GLU layer in each GLU block\n 'lambda_sparse': 1e-4, # Sparsity regularization\n 'optimizer_fn': torch.optim.Adam,\n 'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n 'mask_type': 'entmax',\n 'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n 'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n 'verbose': 0,\n 'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n# Create model instances\nTabNet_Model = TabNetWrapper(**TabNet_Params) # New\n\nscores = cross_val_score(TabNet_Model, X, y, cv=skf, scoring=kappa_scorer)\nprint(\"Quadratic Cohen's Kappa Scores:\", scores)\nprint(\"Mean Quadratic Cohen's Kappa:\", np.mean(scores))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#
7/ Neural network models: Custom neural network
","metadata":{}},{"cell_type":"code","source":"use_total = True\nif use_total:\n output_cols = ['PCIAT-PCIAT_Total']\nelse:\n output_cols = list (train.filter (like='PCIAT_').columns.tolist())\n output_cols.remove('PCIAT-PCIAT_Total')\nprint (output_cols)\nselection = engineered_features\nX = train[selection]\nN_inputs = len (selection)\ny = train[output_cols]\nN_outputs = len (output_cols)\nprint (f\"{N_inputs} input features and {N_outputs} output features\")\n# Impute column median for missing values\nimputer = SimpleImputer(strategy=\"median\")\nct = ColumnTransformer(\n [(\"imputer\",imputer, selection)],\n remainder=\"passthrough\"\n ) .set_output(transform=\"pandas\")\nX = ct.fit_transform(X)\n\n# Splitting \ntrain_X, val_X, train_y, val_y = train_test_split(X, y, \n test_size = 0.15, random_state = 123)\nprint (X[X.isna().any(axis=1)])\nprint (y[y.isna().any(axis=1)])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CustomNN(nn.Module):\n def __init__(self):\n super().__init__()\n self.layers = nn.Sequential(\n nn.Linear(N_inputs, 24),\n nn.ReLU(),\n nn.Linear(24, 24),\n nn.ReLU(),\n nn.Linear(24, N_outputs)\n )\n\n def forward(self, x):\n return self.layers(x)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CustomNeuralNetRegressor (NeuralNetRegressor):\n def __init__(\n self,\n module,\n *args,\n criterion=torch.nn.MSELoss,\n **kwargs\n ):\n super(CustomNeuralNetRegressor, self).__init__(\n module,\n *args,\n criterion=criterion,\n **kwargs\n )\n \n def predict(self, X):\n y_pred = super().predict (X)\n y_pred_total = np.sum (y_pred, axis=1)\n return y_pred_total\n \n def fit (self, X, y):\n # nX, ny = X.to_numpy(np.float32), y.to_numpy(np.float32)\n nX, ny = X, y\n return super().fit (nX, ny) ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# create the skorch wrapper\nnn_model = CustomNeuralNetRegressor (\n CustomNN, \n criterion=nn.MSELoss(),\n optimizer=optim.Adam,\n lr=0.02,\n max_epochs=20,\n batch_size=1,\n train_split=skorch.dataset.ValidSplit(0.15)\n)\nnX, ny = X.to_numpy(np.float32), y.to_numpy(np.float32) \nnn_model.fit(nX, ny)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scores = cross_val_score(nn_model, nX, ny, cv=skf, scoring=kappa_scorer)\nprint(\"Quadratic Cohen's Kappa Scores:\", scores)\nprint(\"Mean Quadratic Cohen's Kappa:\", np.mean(scores))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#6/ Submission
","metadata":{}},{"cell_type":"markdown","source":"