{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":7453542,"sourceType":"datasetVersion","datasetId":921302}],"dockerImageVersionId":30823,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":2042.749083,"end_time":"2024-12-19T12:57:24.551319","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-12-19T12:23:21.802236","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip -q install /kaggle/input/pytorchtabnet/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:23:24.170277Z","iopub.status.busy":"2024-12-19T12:23:24.169746Z","iopub.status.idle":"2024-12-19T12:24:05.291485Z","shell.execute_reply":"2024-12-19T12:24:05.290543Z"},"papermill":{"duration":41.138365,"end_time":"2024-12-19T12:24:05.293524","exception":false,"start_time":"2024-12-19T12:23:24.155159","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone, BaseEstimator, RegressorMixin\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score, mean_squared_error\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.decomposition import PCA\nfrom sklearn.datasets import make_classification\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nfrom sklearn.linear_model import Lasso\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom keras import models, layers, optimizers\nModel = models.Model\nInput = layers.Input\nDense = layers.Dense\nAdam = optimizers.Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom pytorch_tabnet.tab_model import TabNetRegressor\nfrom pytorch_tabnet.callbacks import Callback\n\nfrom sklearn.svm import LinearSVR\nfrom sklearn.linear_model import ElasticNet\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor, StackingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\n\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nfrom copy import deepcopy\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None # type: ignore\n\nimport os\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\n\nSEED = 42\nn_splits = 5","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2024-12-19T12:24:05.310551Z","iopub.status.busy":"2024-12-19T12:24:05.310262Z","iopub.status.idle":"2024-12-19T12:24:22.586288Z","shell.execute_reply":"2024-12-19T12:24:22.585355Z"},"papermill":{"duration":17.286475,"end_time":"2024-12-19T12:24:22.588758","exception":false,"start_time":"2024-12-19T12:24:05.302283","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\ndef seed_everything(seed):\n random.seed(seed)\n os.environ['PYTHONHASHSEED'] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = True\nseed_everything(5204)","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:24:22.606954Z","iopub.status.busy":"2024-12-19T12:24:22.606292Z","iopub.status.idle":"2024-12-19T12:24:22.615011Z","shell.execute_reply":"2024-12-19T12:24:22.614418Z"},"papermill":{"duration":0.019287,"end_time":"2024-12-19T12:24:22.616651","exception":false,"start_time":"2024-12-19T12:24:22.597364","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_file(filename, dirname):\n df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n df.drop('step', axis=1, inplace=True)\n return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n ids = os.listdir(dirname)\n \n with ThreadPoolExecutor() as executor:\n results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n \n stats, indexes = zip(*results)\n \n df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n df['id'] = indexes\n return df\n\nclass AutoEncoder(nn.Module):\n def __init__(self, input_dim, encoding_dim):\n super(AutoEncoder, self).__init__()\n self.encoder = nn.Sequential(\n nn.Linear(input_dim, encoding_dim*3),\n nn.ReLU(),\n nn.Linear(encoding_dim*3, encoding_dim*2),\n nn.ReLU(),\n nn.Linear(encoding_dim*2, encoding_dim),\n nn.ReLU()\n )\n self.decoder = nn.Sequential(\n nn.Linear(encoding_dim, encoding_dim*2),\n nn.ReLU(),\n nn.Linear(encoding_dim*2, encoding_dim*3),\n nn.ReLU(),\n nn.Linear(encoding_dim*3, input_dim),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n encoded = self.encoder(x)\n decoded = self.decoder(encoded)\n return decoded\n \ndef perform_autoencoder(train_data, test_data, encoding_dim=50, epochs=50, batch_size=32):\n scaler = StandardScaler()\n train_scaled = scaler.fit_transform(train_data)\n test_scaled = scaler.transform(test_data)\n \n train_tensor = torch.FloatTensor(train_scaled)\n test_tensor = torch.FloatTensor(test_scaled)\n \n input_dim = train_tensor.shape[1]\n autoencoder = AutoEncoder(input_dim, encoding_dim)\n \n criterion = nn.MSELoss()\n optimizer = optim.Adam(autoencoder.parameters())\n \n for epoch in range(epochs):\n for i in range(0, len(train_tensor), batch_size):\n batch = train_tensor[i : i + batch_size]\n optimizer.zero_grad()\n reconstructed = autoencoder(batch)\n loss = criterion(reconstructed, batch)\n loss.backward()\n optimizer.step()\n \n if (epoch + 1) % 10 == 0:\n print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n \n with torch.no_grad():\n # Thêm cái này để đánh giá auto encoder, mặc dù mình không chắc có thực sự đánh giá này hợp lí không\n reconstructed_train = autoencoder(train_tensor)\n reconstructed_test = autoencoder(test_tensor)\n \n train_loss = criterion(reconstructed_train, train_tensor).item()\n test_loss = criterion(reconstructed_test, test_tensor).item()\n \n print(f'Train loss: {train_loss:.4f}')\n print(f'Test loss: {test_loss:.4f}')\n train_encoded = autoencoder.encoder(train_tensor).numpy()\n test_encoded = autoencoder.encoder(test_tensor).numpy()\n train_df_encoded = pd.DataFrame(train_encoded, columns=[f'Enc_{i + 1}' for i in range(train_encoded.shape[1])])\n test_df_encoded = pd.DataFrame(test_encoded, columns=[f'Enc_{i + 1}' for i in range(test_encoded.shape[1])])\n \n return train_df_encoded, test_df_encoded\n\ndef feature_engineering(df):\n season_cols = [col for col in df.columns if 'Season' in col]\n df = df.drop(season_cols, axis=1) \n df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n df['BMI_PHR'] = df['Physical-BMI'] * df['Physical-HeartRate']\n \n return df\n\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:24:22.632213Z","iopub.status.busy":"2024-12-19T12:24:22.631985Z","iopub.status.idle":"2024-12-19T12:24:22.715514Z","shell.execute_reply":"2024-12-19T12:24:22.714739Z"},"papermill":{"duration":0.094031,"end_time":"2024-12-19T12:24:22.717491","exception":false,"start_time":"2024-12-19T12:24:22.623460","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PATH_PREFIX = \"/kaggle/input/child-mind-institute-problematic-internet-use\"\n\n# Ignore the annoying FutureWarning from pandas (it is caused by sklearn anyway!)\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.impute import KNNImputer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\n\nfrom typing import Literal, TypeVar, Any\nFloatOrSeries = TypeVar('FloatOrSeries', float, pd.Series)\n\n# pd.set_option('display.max_rows', None)\n# pd.set_option('display.max_columns', None)\n# pd.set_option('display.width', 1000)\n","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:24:22.734364Z","iopub.status.busy":"2024-12-19T12:24:22.733925Z","iopub.status.idle":"2024-12-19T12:24:22.738462Z","shell.execute_reply":"2024-12-19T12:24:22.737763Z"},"papermill":{"duration":0.014104,"end_time":"2024-12-19T12:24:22.740071","exception":false,"start_time":"2024-12-19T12:24:22.725967","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_worn_data(id: str):\n file_path = f'{PATH_PREFIX}/series_test.parquet/id={id}/part-0.parquet'\n data = pd.read_parquet(file_path)\n worn_data = data[data['non-wear_flag'] == 0]\n return worn_data\n\n# test = pd.read_csv(f\"{PATH_PREFIX}/test.csv\")\ndef read_tabular_data(id: str):\n return test[test['id'] == id]\n\ndef convert_period_to_hms(period: int, granularity_in_hours: float):\n time_of_day_in_hours = period * granularity_in_hours\n h = int(time_of_day_in_hours)\n m = int((time_of_day_in_hours - h) * 60)\n s = int((time_of_day_in_hours - h - m / 60) * 3600)\n return f\"{h:02}:{m:02}:{s:02}\"\n\nclass SkillfulImputer:\n \"\"\"Uses an sklearn imputer. Takes into account the categorical columns. Leaves NaN columns intact.\n (NaN columns are columns that contain only NaN values.)\n \"\"\"\n\n def __init__(self, imputer: Any):\n \"\"\"\n imputer - Must be an instance of an sklearn imputer, e.g.\n KNNImputer(n_neighbors=2)\n \"\"\"\n self.imputer = imputer\n\n def fit_transform(self, df: pd.DataFrame, categorical_columns: list[str], **kwargs):\n df = df.copy()\n categorical_columns = list(categorical_columns)\n\n # A. Identify NaN columns\n nan_columns = df.columns[df.isna().all()].to_list()\n \n # B. Convert categorical to numerical (temporarily)\n label_encoders = {}\n for col in categorical_columns:\n le = LabelEncoder()\n df[col] = df[col].astype(str) # convert to string to handle NaN\n df[col] = le.fit_transform(df[col])\n label_encoders[col] = le # Save the encoder to decode later!\n # C. Impute!\n imputer = self.imputer\n imputed_data = imputer.fit_transform(df)\n imputed_df = pd.DataFrame(imputed_data, columns=df.columns, **kwargs)\n # D. Convert the categorical columns back\n for col in categorical_columns:\n imputed_df[col] = imputed_df[col].round(0).astype(int)\n imputed_df[col] = label_encoders[col].inverse_transform(imputed_df[col])\n # E. Restore the NaN values\n for col in nan_columns:\n imputed_df[col] = np.nan\n return imputed_df\n","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:24:22.754917Z","iopub.status.busy":"2024-12-19T12:24:22.754264Z","iopub.status.idle":"2024-12-19T12:24:22.762633Z","shell.execute_reply":"2024-12-19T12:24:22.761877Z"},"papermill":{"duration":0.017394,"end_time":"2024-12-19T12:24:22.764176","exception":false,"start_time":"2024-12-19T12:24:22.746782","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BaseDataPreprocessor:\n def __init__(self, granularity_in_hours: float = 1.0):\n self.granularity_in_hours = granularity_in_hours\n self.period_range = lambda: range(0, int(24 / self.granularity_in_hours))\n self.num_periods = len(self.period_range())\n","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:24:22.778614Z","iopub.status.busy":"2024-12-19T12:24:22.778372Z","iopub.status.idle":"2024-12-19T12:24:22.782375Z","shell.execute_reply":"2024-12-19T12:24:22.781605Z"},"papermill":{"duration":0.01322,"end_time":"2024-12-19T12:24:22.784122","exception":false,"start_time":"2024-12-19T12:24:22.770902","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CalculateDailyPeriodicActivityRelatedValues(BaseDataPreprocessor):\n def __init__(self, worn_data: pd.DataFrame):\n super().__init__()\n self.worn_data = worn_data.copy()\n self.available_days = self.worn_data['relative_date_PCIAT'].unique()\n\n def _preprocess_periodically_aggregated_values(self) -> pd.DataFrame:\n def convert_daily_periodic_series_to_list(series: pd.Series) -> list[Any]:\n period_to_value_dict = {}\n for period in series.index:\n period_to_value_dict[period] = series[period]\n for period in self.period_range():\n period_to_value_dict[period] = period_to_value_dict.get(period, np.nan)\n values = list(period_to_value_dict.items())\n values.sort(key=lambda elem: elem[0])\n values = [ e[1] for e in values ]\n values = np.array([x if x is not None else np.nan for x in values])\n # a list of which the nth value is the actual value at the nth period of the day.\n # e.g. [4, 6, 2, 7, 5, 8] => 4 is the value at period 0, 8 is the value at period 5.\n return list(values)\n\n all_days_data = {\n \"day\": [],\n \"daily_periodic_df\": [],\n }\n \n for day in self.available_days:\n base_data = self.worn_data[self.worn_data['relative_date_PCIAT'] == day].copy()\n base_data['daily_period'] = base_data['time_of_day'] / 1e9 / (self.granularity_in_hours * 3600)\n periodic_group = base_data.groupby(\n base_data['daily_period'].astype(int)\n )\n\n periodic_enmo = periodic_group['enmo'].agg(['mean', 'max', 'std'])\n periodic_light = periodic_group['light'].agg(['mean', 'max'])\n\n periodic_enmo_mean = periodic_enmo['mean'] # pd.Series\n periodic_enmo_max = periodic_enmo['max'] # pd.Series\n periodic_enmo_std = periodic_enmo['std'] # pd.Series\n periodic_anglez_std = periodic_group['anglez'].agg(['std'])['std'] # pd.Series\n periodic_anglez_sum_of_changes = periodic_group['anglez'].apply(lambda x: np.sum(np.abs(np.diff(x)))) # pd.Series\n periodic_light_mean = periodic_light['mean']\n periodic_light_max = periodic_light['max']\n\n daily_periodic_data = {\n \"period\": np.array(list(self.period_range())),\n \"periodic_enmo_mean\": convert_daily_periodic_series_to_list(periodic_enmo_mean),\n \"periodic_enmo_max\": convert_daily_periodic_series_to_list(periodic_enmo_max),\n \"periodic_enmo_std\": convert_daily_periodic_series_to_list(periodic_enmo_std),\n \"periodic_anglez_std\": convert_daily_periodic_series_to_list(periodic_anglez_std),\n \"periodic_anglez_sum_of_changes\": convert_daily_periodic_series_to_list(periodic_anglez_sum_of_changes),\n \"periodic_light_mean\": convert_daily_periodic_series_to_list(periodic_light_mean),\n \"periodic_light_max\": convert_daily_periodic_series_to_list(periodic_light_max),\n }\n \n daily_periodic_df = pd.DataFrame(daily_periodic_data)\n # daily_periodic_df.set_index('period', inplace=True)\n \n all_days_data[\"day\"].append(day)\n all_days_data[\"daily_periodic_df\"].append(daily_periodic_df)\n\n all_days_df = pd.DataFrame(all_days_data)\n # all_days_df.set_index(\"day\", inplace=True)\n return all_days_df\n\n def _impute_periodically_aggregated_values(self, all_days_df: pd.DataFrame) -> pd.DataFrame:\n if len(all_days_df.index) == 0:\n return all_days_df\n\n data_requiring_imputation = {\n \"period\": [],\n }\n reassembly_info = []\n daily_periodic_columns = set()\n\n for day in all_days_df['day']:\n daily_periodic_df = all_days_df[all_days_df['day'] == day]['daily_periodic_df'].iloc[0]\n for col in daily_periodic_df.columns:\n data_requiring_imputation[col] = data_requiring_imputation.get(col, [])\n daily_periodic_columns.add(col)\n for period in daily_periodic_df.index:\n values_at_period = daily_periodic_df[daily_periodic_df.index == period]\n for col in values_at_period.columns:\n data_requiring_imputation[col].append(values_at_period[col])\n reassembly_info.append((day, period))\n\n df_requiring_imputation = pd.DataFrame(data_requiring_imputation)\n imputer = SkillfulImputer(KNNImputer(n_neighbors=2))\n imputed_df = imputer.fit_transform(df_requiring_imputation, categorical_columns=[])\n\n new_all_days_data = {\n \"day\": [],\n \"daily_periodic_df\": [],\n }\n\n # for _i, imputed_entry in imputed_df.iterrows():\n # day, period = reassembly_info.pop(0)\n # day = int(day)\n # period = int(period)\n # while True:\n # try:\n # i = new_all_days_data['day'].index(day)\n # except ValueError: # day is not added yet\n # print('check check 1')\n # new_all_days_data['day'].append(day)\n # new_all_days_data['daily_periodic_df'].append(\n # dict((col, []) for col in daily_periodic_columns)\n # )\n # continue\n # else:\n # daily_periodic_df = new_all_days_data['daily_periodic_df'][i]\n # for col in daily_periodic_columns:\n # daily_periodic_df[col].append(imputed_entry[col])\n # break\n\n for _i, imputed_entry in imputed_df.iterrows():\n day, period = reassembly_info.pop(0)\n day = int(day)\n period = int(period)\n \n # Kiểm tra xem 'day' đã có trong 'new_all_days_data['day']' hay chưa\n if day not in new_all_days_data['day']:\n new_all_days_data['day'].append(day)\n new_all_days_data['daily_periodic_df'].append(\n dict((col, []) for col in daily_periodic_columns)\n )\n \n # Sau khi chắc chắn 'day' đã có trong danh sách, tiếp tục xử lý\n i = new_all_days_data['day'].index(day)\n daily_periodic_df = new_all_days_data['daily_periodic_df'][i]\n for col in daily_periodic_columns:\n daily_periodic_df[col].append(imputed_entry[col])\n \n new_daily_periodic_dfs = []\n for daily_periodic_data in new_all_days_data['daily_periodic_df']:\n daily_periodic_df = pd.DataFrame(daily_periodic_data)\n daily_periodic_df.set_index('period', inplace=True)\n new_daily_periodic_dfs.append(daily_periodic_df)\n\n new_all_days_data['daily_periodic_df'] = new_daily_periodic_dfs\n imputed_all_days_df = pd.DataFrame(new_all_days_data)\n imputed_all_days_df.set_index('day', inplace=True)\n return imputed_all_days_df\n\n def process(self):\n df = self._preprocess_periodically_aggregated_values()\n df = self._impute_periodically_aggregated_values(df)\n return df\n","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:24:22.799101Z","iopub.status.busy":"2024-12-19T12:24:22.798630Z","iopub.status.idle":"2024-12-19T12:24:22.815704Z","shell.execute_reply":"2024-12-19T12:24:22.814922Z"},"papermill":{"duration":0.026405,"end_time":"2024-12-19T12:24:22.817384","exception":false,"start_time":"2024-12-19T12:24:22.790979","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CalculateDailyPeriodicActivityLevels(BaseDataPreprocessor):\n def __init__(self, all_days_df: pd.DataFrame):\n super().__init__()\n self.all_days_df = all_days_df\n\n def process(self) -> tuple[pd.DataFrame, pd.DataFrame]:\n data = {\n \"day\": [],\n \"daily_periodic_activity_levels\": [],\n }\n\n all_days_df = self.all_days_df\n\n for day in self.all_days_df.index:\n data[\"day\"].append(day)\n\n daily_periodic_df = all_days_df[all_days_df.index == day]['daily_periodic_df'].iloc[0].copy()\n\n daily_periodic_df = daily_periodic_df.reset_index()\n daily_periodic_df.drop(columns=['period'], inplace=True)\n\n scaler = StandardScaler()\n standardized_data = scaler.fit_transform(daily_periodic_df)\n\n pca = PCA()\n pca.fit_transform(standardized_data)\n pc1_weight = pca.components_[0]\n\n periodic_activities = np.dot(standardized_data, pc1_weight)\n periodic_activities_series = pd.Series(periodic_activities, index=self.period_range())\n data[\"daily_periodic_activity_levels\"].append(periodic_activities_series)\n\n df = pd.DataFrame(data)\n df.set_index('day', inplace=True)\n return all_days_df, df\n","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:24:22.831997Z","iopub.status.busy":"2024-12-19T12:24:22.831303Z","iopub.status.idle":"2024-12-19T12:24:22.838086Z","shell.execute_reply":"2024-12-19T12:24:22.837346Z"},"papermill":{"duration":0.015687,"end_time":"2024-12-19T12:24:22.839754","exception":false,"start_time":"2024-12-19T12:24:22.824067","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class EngineerFeaturesFromPeriodicActivityLevels(BaseDataPreprocessor):\n def __init__(self, dfs: tuple[pd.DataFrame, pd.DataFrame]):\n \"\"\"\n dfs = (df, aldf)\n\n df: all_days_df as returned by\n CalculateDailyPeriodicActivityRelatedValues.process()\n \n aldf: activity-level-per-days dataframe : {\n \"day\": [],\n \"daily_periodic_activity_levels\": [],\n }\n \"\"\"\n super().__init__()\n self.all_days_df, self.aldf = dfs\n\n epsilon = 0.1\n SLEEP_AL = -1.0\n MODERATE_AL = 0\n VIGOROUS_AL = 1.5\n NIGHTTIME_SPT_WINDOW_TOLERANCE = 3\n\n def _detect_nighttime_spt_window(self, periodic_activity_levels: np.ndarray):\n look_left = look_right = True\n # Because w_start decreases and w_end increases over iterations\n # we set w_start to hour 24 initially, so that when there is an overlap,\n # we have w_start <= w_end.\n w_start = int(24 / self.granularity_in_hours) # 0\n w_end = int(4 / self.granularity_in_hours)\n\n # if w_start changes then change this one too!\n sleep_periods = np.arange(w_end+1)\n # if w_start or w_end changes then change this one too!\n d = np.sum(np.array(periodic_activity_levels[0:w_end+1]) >= self.SLEEP_AL)\n\n N = len(periodic_activity_levels)\n\n def get_max_k_right():\n k = self.NIGHTTIME_SPT_WINDOW_TOLERANCE\n points = []\n indices = []\n for i in range(w_end+1, w_end+k+1):\n points.append( periodic_activity_levels[i % N] )\n indices.append(i % N)\n for i in range(len(points) - 1, -1, -1):\n if points[i] + self.epsilon < self.SLEEP_AL:\n break\n return np.array(points[:i+1]), np.array(indices[:i+1])\n\n def get_max_k_left():\n k = self.NIGHTTIME_SPT_WINDOW_TOLERANCE\n points = []\n indices = []\n for i in range(w_start-k, w_start):\n points.append( periodic_activity_levels[i % N] )\n indices.append(i % N)\n for i in range(len(points)):\n if points[i] + self.epsilon < self.SLEEP_AL:\n break\n return np.array(points[i:]), np.array(indices[i:])\n\n while (look_left or look_right) and w_start > w_end:\n if look_left:\n points, indices = get_max_k_left()\n if len(points) == 0:\n look_left = False\n else:\n num_sleep_points = np.sum(points + self.epsilon < self.SLEEP_AL)\n if num_sleep_points / len(points) < 0.5:\n look_left = False\n else:\n w_start -= len(points)\n d += len(points) - num_sleep_points\n sleep_periods = np.concatenate((indices, sleep_periods))\n if look_right:\n points, indices = get_max_k_right()\n if len(points) == 0:\n look_right = False\n else:\n num_sleep_points = np.sum(points + self.epsilon < self.SLEEP_AL)\n if num_sleep_points / len(points) < 0.5:\n look_right = False\n else:\n w_end += len(points)\n d += len(points) - num_sleep_points\n sleep_periods = np.concatenate((sleep_periods, indices))\n \n if w_start <= w_end:\n return np.nan, np.nan, np.array([]), np.nan, np.nan\n\n w_duration = w_end % N + N - w_start % N\n\n return w_start % N, w_end % N, sleep_periods, w_duration, d\n \n def _percentage_of_moderate_and_vigorous_physical_movements(self, periodic_activity_levels: np.ndarray):\n \"\"\"\n Calculates the percentage of moderate and vigorous physical movements in the given activity levels.\n Returns a tuple of (percentage of moderate movements, percentage of vigorous movements).\n \"\"\"\n arr = periodic_activity_levels\n N_percent = 100 / len(arr)\n moderate = np.sum(arr + self.epsilon > self.MODERATE_AL)\n vigorous = np.sum(arr + self.epsilon > self.VIGOROUS_AL)\n moderate -= vigorous # type: ignore\n return moderate * N_percent, vigorous * N_percent\n\n def process(self) -> pd.DataFrame:\n data = {\n \"day\": [],\n }\n\n for day in self.aldf.index:\n data['day'].append(day)\n \n periodic_activity_levels = np.array(list(self.aldf[self.aldf.index == day]['daily_periodic_activity_levels'].iloc[0]))\n w_start, w_end, sleep_periods, w_duration, d = self._detect_nighttime_spt_window(periodic_activity_levels)\n sleep_start_hour = w_start * self.granularity_in_hours\n sleep_end_hour = w_end * self.granularity_in_hours\n # In case there is too little data on that day, and the data points are all sleep points,\n # then it is totally fine to have a sleep duration of 24 hours.\n # Hope the models can spot it out and learn from it.\n sleep_duration_in_hours = w_duration * self.granularity_in_hours\n if not np.isnan(sleep_duration_in_hours):\n sleep_duration_in_hours = min(24, sleep_duration_in_hours)\n \n sleep_start_earliness = (\n np.nan if np.isnan(sleep_start_hour)\n else -1 if sleep_start_hour < 12 and sleep_start_hour >= 7\n else 10 if sleep_start_hour <= 21\n else 9 if sleep_start_hour <= 22\n else 7 if sleep_start_hour <= 23\n else 5 if sleep_start_hour <= 24\n else 2\n )\n \n sleep_end_earliness = (\n np.nan if np.isnan(sleep_end_hour)\n else -1 if sleep_end_hour <= 3 or sleep_end_hour >= 13\n else 10 if sleep_end_hour <= 8\n else 9 if sleep_end_hour <= 9\n else 7 if sleep_end_hour <= 10\n else 5 if sleep_end_hour <= 11\n else 2\n )\n\n def aggregate_light(sleep_periods):\n day_df = self.all_days_df[self.all_days_df.index == day]['daily_periodic_df'].iloc[0].copy()\n day_lights = list(day_df['periodic_light_mean'])\n wake_lights = []\n sleep_lights = []\n for i in self.period_range():\n if i not in sleep_periods:\n wake_lights.append(day_lights[i])\n else:\n sleep_lights.append(day_lights[i])\n wake_lights_aggregates = pd.Series(wake_lights).agg(['mean', 'max'])\n sleep_lights_aggregates = pd.Series(sleep_lights).agg(['mean', 'max'])\n light_mean_diff = wake_lights_aggregates['mean'] - sleep_lights_aggregates['mean']\n light_max_diff = wake_lights_aggregates['max'] - sleep_lights_aggregates['max']\n return light_mean_diff, light_max_diff\n\n if np.isnan(w_start) or np.isnan(w_end):\n percent_moderate_activities_waketime = percent_vigorous_activities_waketime = np.nan\n # Defaulting sleep time to be 0h - 5h\n light_mean_diff, light_max_diff = aggregate_light(np.arange(0, 5 / self.granularity_in_hours + 1))\n else:\n waketime_activity_levels = []\n for i in self.period_range():\n if i not in sleep_periods:\n waketime_activity_levels.append(periodic_activity_levels[i])\n waketime_activity_levels = np.array(waketime_activity_levels)\n percent_moderate_activities_waketime, percent_vigorous_activities_waketime = self._percentage_of_moderate_and_vigorous_physical_movements(waketime_activity_levels)\n \n day_df = self.all_days_df[self.all_days_df.index == day]['daily_periodic_df'].iloc[0].copy()\n day_lights = list(day_df['periodic_light_mean'])\n wake_lights = []\n sleep_lights = []\n for i in self.period_range():\n if i not in sleep_periods:\n wake_lights.append(day_lights[i])\n else:\n sleep_lights.append(day_lights[i])\n wake_lights_aggregates = pd.Series(wake_lights).agg(['mean', 'max'])\n sleep_lights_aggregates = pd.Series(sleep_lights).agg(['mean', 'max'])\n light_mean_diff = wake_lights_aggregates['mean'] - sleep_lights_aggregates['mean']\n light_max_diff = wake_lights_aggregates['max'] - sleep_lights_aggregates['max']\n \n for col, value in {\n 'percent_moderate_activities_waketime': percent_moderate_activities_waketime,\n 'percent_vigorous_activities_waketime': percent_vigorous_activities_waketime,\n 'sleep_duration_in_hours': sleep_duration_in_hours,\n 'sleep_start_earliness': sleep_start_earliness,\n 'sleep_end_earliness': sleep_end_earliness,\n 'sleep_disturbance_duration_in_hours': d * self.granularity_in_hours,\n 'light_mean_diff': light_mean_diff,\n 'light_max_diff': light_max_diff,\n }.items():\n data[col] = data.get(col, [])\n data[col].append(value)\n\n df = pd.DataFrame(data)\n df.set_index('day', inplace=True)\n return df\n\n @classmethod\n def get_features(cls):\n return [\n 'percent_moderate_activities_waketime',\n 'percent_vigorous_activities_waketime',\n 'sleep_duration_in_hours',\n 'sleep_start_earliness',\n 'sleep_end_earliness',\n 'sleep_disturbance_duration_in_hours',\n 'light_mean_diff',\n 'light_max_diff'\n ]\n","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:24:22.854195Z","iopub.status.busy":"2024-12-19T12:24:22.853950Z","iopub.status.idle":"2024-12-19T12:24:22.877376Z","shell.execute_reply":"2024-12-19T12:24:22.876613Z"},"papermill":{"duration":0.032614,"end_time":"2024-12-19T12:24:22.879012","exception":false,"start_time":"2024-12-19T12:24:22.846398","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AggregateFeaturesPerParticipant:\n def __init__(self, efdf: pd.DataFrame):\n self.efdf = efdf.copy()\n\n AGGREGATE_NAMES = ['mean', 'min', 'max', 'std']\n \n def process(self):\n df = self.efdf\n\n features = []\n values = []\n\n for col in df.columns:\n aggregates = df[col].agg(self.AGGREGATE_NAMES) # type: ignore\n for aggregate_name in self.AGGREGATE_NAMES:\n features.append( f\"{col}_{aggregate_name.upper()}\" )\n values.append( aggregates[aggregate_name] )\n\n return pd.Series(values, index=features)\n\n @classmethod\n def get_features(cls):\n features: list[str] = []\n for col in EngineerFeaturesFromPeriodicActivityLevels.get_features():\n for aggregate_name in cls.AGGREGATE_NAMES:\n features.append( f\"{col}_{aggregate_name.upper()}\" )\n return features\n","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:24:22.893405Z","iopub.status.busy":"2024-12-19T12:24:22.893158Z","iopub.status.idle":"2024-12-19T12:24:22.898730Z","shell.execute_reply":"2024-12-19T12:24:22.897972Z"},"papermill":{"duration":0.014461,"end_time":"2024-12-19T12:24:22.900275","exception":false,"start_time":"2024-12-19T12:24:22.885814","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_worn_data(participant_id: str, dataset: Literal['train', 'test']) -> pd.DataFrame:\n data = pd.read_parquet(f'{PATH_PREFIX}/series_{dataset}.parquet/id={participant_id}/part-0.parquet')\n worn_data = data[data['non-wear_flag'] == 0]\n return worn_data\n\ndef timeseries_feature_engineering(participant_id: str|None, dataset: Literal['train', 'test']):\n try:\n if not participant_id or pd.isna(participant_id) or pd.isnull(participant_id):\n raise ValueError(\"Invalid participant ID\")\n data = read_worn_data(participant_id, dataset=dataset)\n except Exception as e:\n print(f\"Error while engineering features from timeseries data: {e}\")\n return pd.Series(dict(\n (feature, np.nan) for feature in AggregateFeaturesPerParticipant.get_features()\n ))\n \n data = CalculateDailyPeriodicActivityRelatedValues(data).process()\n data = CalculateDailyPeriodicActivityLevels(data).process()\n data = EngineerFeaturesFromPeriodicActivityLevels(data).process()\n data = AggregateFeaturesPerParticipant(data).process()\n return data # pd.Series\n\ndef load_and_engineer_data(PATH_PREFIX: str, dataset: Literal['train', 'test']) -> pd.DataFrame:\n tabular = pd.read_csv(f\"{PATH_PREFIX}/{dataset}.csv\")\n\n data: dict[str, list[Any]] = {}\n\n for _i, row in tabular.iterrows():\n for col in tabular.columns:\n data[col] = data.get(col, [])\n data[col].append(row[col])\n \n if \"id\" in row.index:\n participant_id = row[\"id\"]\n else:\n participant_id = None\n series = timeseries_feature_engineering(participant_id, dataset=dataset)\n for col in series.index:\n data[col] = data.get(col, [])\n data[col].append(series[col])\n\n df = pd.DataFrame(data)\n df.set_index('id')\n return df\n","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:24:22.914492Z","iopub.status.busy":"2024-12-19T12:24:22.914283Z","iopub.status.idle":"2024-12-19T12:24:22.922152Z","shell.execute_reply":"2024-12-19T12:24:22.921539Z"},"papermill":{"duration":0.016582,"end_time":"2024-12-19T12:24:22.923642","exception":false,"start_time":"2024-12-19T12:24:22.907060","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = load_and_engineer_data(PATH_PREFIX, dataset='test')","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:24:22.937803Z","iopub.status.busy":"2024-12-19T12:24:22.937532Z","iopub.status.idle":"2024-12-19T12:24:25.870797Z","shell.execute_reply":"2024-12-19T12:24:25.869882Z"},"papermill":{"duration":2.942304,"end_time":"2024-12-19T12:24:25.872583","exception":false,"start_time":"2024-12-19T12:24:22.930279","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = load_and_engineer_data(PATH_PREFIX, dataset='train')","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:24:25.889078Z","iopub.status.busy":"2024-12-19T12:24:25.888724Z","iopub.status.idle":"2024-12-19T12:42:24.127661Z","shell.execute_reply":"2024-12-19T12:42:24.126730Z"},"papermill":{"duration":1078.249062,"end_time":"2024-12-19T12:42:24.129474","exception":false,"start_time":"2024-12-19T12:24:25.880412","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fe_cols = AggregateFeaturesPerParticipant.get_features()","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:42:24.254523Z","iopub.status.busy":"2024-12-19T12:42:24.254196Z","iopub.status.idle":"2024-12-19T12:42:24.258077Z","shell.execute_reply":"2024-12-19T12:42:24.257464Z"},"papermill":{"duration":0.085673,"end_time":"2024-12-19T12:42:24.259568","exception":false,"start_time":"2024-12-19T12:42:24.173895","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\ntrain_ts_encoded, test_ts_encoded = perform_autoencoder(df_train, df_test, encoding_dim=60, epochs=100, batch_size=32)\n\ntime_series_cols = train_ts_encoded.columns.tolist()\ntrain_ts_encoded[\"id\"]=train_ts[\"id\"]\ntest_ts_encoded['id']=test_ts[\"id\"]\n\ntrain = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\ntest = pd.merge(test, test_ts_encoded, how=\"left\", on='id')","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:42:24.346824Z","iopub.status.busy":"2024-12-19T12:42:24.346557Z","iopub.status.idle":"2024-12-19T12:43:42.942710Z","shell.execute_reply":"2024-12-19T12:43:42.941741Z"},"papermill":{"duration":78.642139,"end_time":"2024-12-19T12:43:42.944716","exception":false,"start_time":"2024-12-19T12:42:24.302577","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:43:43.072960Z","iopub.status.busy":"2024-12-19T12:43:43.071880Z","iopub.status.idle":"2024-12-19T12:43:43.209617Z","shell.execute_reply":"2024-12-19T12:43:43.208870Z"},"papermill":{"duration":0.202409,"end_time":"2024-12-19T12:43:43.212626","exception":false,"start_time":"2024-12-19T12:43:43.010217","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imputer = KNNImputer(n_neighbors=5)\nnumeric_cols = train.select_dtypes(include=['int32', 'int64', 'float64', 'int64']).columns\nimputed_data = imputer.fit_transform(train[numeric_cols])\ntrain_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\ntrain_imputed['sii'] = train_imputed['sii'].round().astype(int)\nfor col in train.columns:\n if col not in numeric_cols:\n train_imputed[col] = train[col]\n \ntrain = train_imputed\n\ntrain = feature_engineering(train)\ntrain = train.dropna(thresh=10, axis=0)\ntest = feature_engineering(test)","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:43:43.339283Z","iopub.status.busy":"2024-12-19T12:43:43.339000Z","iopub.status.idle":"2024-12-19T12:43:43.361313Z","shell.execute_reply":"2024-12-19T12:43:43.360724Z"},"papermill":{"duration":0.087044,"end_time":"2024-12-19T12:43:43.362898","exception":false,"start_time":"2024-12-19T12:43:43.275854","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.drop('id', axis=1)\ntrain","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:43:43.488519Z","iopub.status.busy":"2024-12-19T12:43:43.488192Z","iopub.status.idle":"2024-12-19T12:43:43.616938Z","shell.execute_reply":"2024-12-19T12:43:43.616127Z"},"papermill":{"duration":0.194414,"end_time":"2024-12-19T12:43:43.619215","exception":false,"start_time":"2024-12-19T12:43:43.424801","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.drop('id', axis=1)\ntest","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:43:43.752055Z","iopub.status.busy":"2024-12-19T12:43:43.751773Z","iopub.status.idle":"2024-12-19T12:43:43.902520Z","shell.execute_reply":"2024-12-19T12:43:43.901610Z"},"papermill":{"duration":0.219215,"end_time":"2024-12-19T12:43:43.905510","exception":false,"start_time":"2024-12-19T12:43:43.686295","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"featuresCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n 'CGAS-CGAS_Score', 'Physical-BMI',\n 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n 'Fitness_Endurance-Max_Stage',\n 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n 'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n 'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n 'SDS-SDS_Total_T',\n 'PreInt_EduHx-computerinternet_hoursday', 'sii', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n 'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n 'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW', 'BMI_PHR']\n\nfeaturesCols += time_series_cols\nfeaturesCols += fe_cols\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n 'CGAS-CGAS_Score', 'Physical-BMI',\n 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n 'Fitness_Endurance-Max_Stage',\n 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n 'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n 'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n 'SDS-SDS_Total_T',\n 'PreInt_EduHx-computerinternet_hoursday', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n 'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n 'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW', 'BMI_PHR']\n\nfeaturesCols += time_series_cols\nfeaturesCols += fe_cols\ntest = test[featuresCols]","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:43:44.046963Z","iopub.status.busy":"2024-12-19T12:43:44.046642Z","iopub.status.idle":"2024-12-19T12:43:44.058291Z","shell.execute_reply":"2024-12-19T12:43:44.057735Z"},"papermill":{"duration":0.083251,"end_time":"2024-12-19T12:43:44.059744","exception":false,"start_time":"2024-12-19T12:43:43.976493","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:43:44.195323Z","iopub.status.busy":"2024-12-19T12:43:44.195052Z","iopub.status.idle":"2024-12-19T12:43:44.284530Z","shell.execute_reply":"2024-12-19T12:43:44.283742Z"},"papermill":{"duration":0.159421,"end_time":"2024-12-19T12:43:44.286705","exception":false,"start_time":"2024-12-19T12:43:44.127284","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:43:44.441328Z","iopub.status.busy":"2024-12-19T12:43:44.440971Z","iopub.status.idle":"2024-12-19T12:43:44.566228Z","shell.execute_reply":"2024-12-19T12:43:44.565425Z"},"papermill":{"duration":0.205913,"end_time":"2024-12-19T12:43:44.569230","exception":false,"start_time":"2024-12-19T12:43:44.363317","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if np.any(np.isinf(train)):\n train = train.replace([np.inf, -np.inf], np.nan)","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:43:44.721250Z","iopub.status.busy":"2024-12-19T12:43:44.720485Z","iopub.status.idle":"2024-12-19T12:43:44.726261Z","shell.execute_reply":"2024-12-19T12:43:44.725434Z"},"papermill":{"duration":0.082964,"end_time":"2024-12-19T12:43:44.728007","exception":false,"start_time":"2024-12-19T12:43:44.645043","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def quadratic_weighted_kappa(y_true, y_pred):\n return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n return np.where(oof_non_rounded < thresholds[0], 0,\n np.where(oof_non_rounded < thresholds[1], 1,\n np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n X = train.drop(['sii'], axis=1)\n y = train['sii']\n\n SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n \n train_S = []\n test_S = []\n \n oof_non_rounded = np.zeros(len(y), dtype=float) \n oof_rounded = np.zeros(len(y), dtype=int) \n test_preds = np.zeros((len(test_data), n_splits))\n\n for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n model = clone(model_class)\n model.fit(X_train, y_train)\n\n y_train_pred = model.predict(X_train)\n y_val_pred = model.predict(X_val)\n\n oof_non_rounded[test_idx] = y_val_pred\n y_val_pred_rounded = y_val_pred.round(0).astype(int)\n oof_rounded[test_idx] = y_val_pred_rounded\n\n train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n train_S.append(train_kappa)\n test_S.append(val_kappa)\n \n test_preds[:, fold] = model.predict(test_data)\n \n print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n clear_output(wait=True)\n\n print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n KappaOPtimizer = minimize(evaluate_predictions,\n x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n method='Nelder-Mead')\n assert KappaOPtimizer.success, \"Optimization did not converge.\"\n \n oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n tpm = test_preds.mean(axis=1)\n tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n \n submission = pd.DataFrame({\n 'id': sample['id'],\n 'sii': tpTuned\n })\n\n return submission","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:43:44.876314Z","iopub.status.busy":"2024-12-19T12:43:44.876021Z","iopub.status.idle":"2024-12-19T12:43:44.886485Z","shell.execute_reply":"2024-12-19T12:43:44.885749Z"},"papermill":{"duration":0.086864,"end_time":"2024-12-19T12:43:44.888003","exception":false,"start_time":"2024-12-19T12:43:44.801139","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TabNetWrapper(BaseEstimator, RegressorMixin):\n def __init__(self, **kwargs):\n self.model = TabNetRegressor(**kwargs)\n self.kwargs = kwargs\n self.imputer = SimpleImputer(strategy='median')\n self.best_model_path = 'best_tabnet_model.pt'\n \n def fit(self, X, y):\n X_imputed = self.imputer.fit_transform(X)\n \n if hasattr(y, 'values'):\n y = y.values\n\n X_train, X_valid, y_train, y_valid = train_test_split(X_imputed, y, test_size=0.2, random_state=SEED)\n\n history = self.model.fit(\n X_train = X_train,\n y_train = y_train.reshape(-1, 1),\n eval_set = [(X_valid, y_valid.reshape(-1, 1))],\n eval_name = ['valid'],\n eval_metric = ['mse'],\n max_epochs = 200,\n patience = 20,\n batch_size = 1024,\n virtual_batch_size = 128,\n num_workers = 0,\n drop_last = False,\n callbacks = [\n TabNetPretrainedModelCheckpoint(\n filepath = self.best_model_path,\n monitor = 'valid_mse',\n mode = 'min',\n save_best_only = True,\n verbose = True\n )\n ]\n )\n\n if os.path.exists(self.best_model_path):\n self.model.load_model(self.best_model_path)\n os.remove(self.best_model_path)\n\n return self\n\n def predict(self, X):\n X_imputed = self.imputer.transform(X)\n return self.model.predict(X_imputed).flatten()\n\n def __deepcopy__(self, memo):\n cls = self.__class__\n result = self.__new__(cls)\n memo[id(self)] = result\n for k, v in self.__dict__.items():\n setattr(result, k, deepcopy(v, memo))\n\n return result\n\nTabNet_Params = {\n 'n_d': 64, # Width of the decision prediction layer\n 'n_a': 64, # Width of the attention embedding for each step\n 'n_steps': 5, # Number of steps in the architecture\n 'gamma': 1.5, # Coefficient for feature selection regularization\n 'n_independent': 2, # Number of independent GLU layer in each GLU block\n 'n_shared': 2, # Number of shared GLU layer in each GLU block\n 'lambda_sparse': 1e-4, # Sparsity regularization\n 'optimizer_fn': torch.optim.Adam,\n 'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n 'mask_type': 'entmax',\n 'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n 'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n 'verbose': 1,\n 'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\nclass TabNetPretrainedModelCheckpoint(Callback):\n def __init__(self, filepath, monitor='val_loss', mode='min', save_best_only=True, verbose=1):\n super().__init__()\n self.filepath = filepath\n self.monitor = monitor\n self.mode = mode\n self.save_best_only = save_best_only\n self.verbose = verbose\n self.best = float('inf') if mode == 'min' else -float('inf')\n \n def on_train_begin(self, logs=None):\n self.model = self.trainer\n \n def on_epoch_end(self, epoch, logs=None):\n logs = logs or {}\n current = logs.get(self.monitor)\n if current is None:\n return\n \n if (self.mode == 'min' and current < self.best) or (self.mode == 'max' and current > self.best):\n if self.verbose:\n print(f'\\nEpoch {epoch}: {self.monitor} improved from {self.best:.4f} to {current:.4f}')\n self.best = current\n if self.save_best_only:\n self.model.save_model(self.filepath)","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:43:45.033745Z","iopub.status.busy":"2024-12-19T12:43:45.033418Z","iopub.status.idle":"2024-12-19T12:43:45.047475Z","shell.execute_reply":"2024-12-19T12:43:45.046797Z"},"papermill":{"duration":0.089019,"end_time":"2024-12-19T12:43:45.048983","exception":false,"start_time":"2024-12-19T12:43:44.959964","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Params = {\n 'learning_rate': 0.046,\n 'max_depth': 15,\n 'num_leaves': 400,\n 'min_data_in_leaf': 8,\n 'feature_fraction': 0.893,\n 'bagging_fraction': 0.784,\n 'bagging_freq': 3,\n 'lambda_l1': 7, # Increased from 6.59\n 'lambda_l2': 0.05, # Increased from 2.68e-06\n 'device': 'cpu',\n}\n\n\nXGB_Params = {\n 'learning_rate': 0.05,\n 'max_depth': 10,\n 'n_estimators': 200,\n 'subsample': 0.9,\n 'colsample_bytree': 0.9,\n 'reg_alpha': 5, # Increased from 0.1\n 'reg_lambda': 5, # Increased from 1\n 'random_state': SEED,\n 'tree_method': 'gpu_hist'\n}\n\n\nCatBoost_Params = {\n 'learning_rate': 0.05,\n 'depth': 6,\n 'iterations': 200,\n 'random_seed': SEED,\n 'verbose': 0,\n 'l2_leaf_reg': 10 # Increase this value\n}\n\n# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\nTabNet_Model = TabNetWrapper(**TabNet_Params)\n#ODT_Model = ObliqueDecisionTreeRegressor(**ODT_Params)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n ('lightgbm', Light),\n ('xgboost', XGB_Model),\n ('catboost', CatBoost_Model),\n ('tabnet', TabNet_Model),\n #('odt', ODT_Model),\n])","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:43:45.196260Z","iopub.status.busy":"2024-12-19T12:43:45.195581Z","iopub.status.idle":"2024-12-19T12:43:45.207978Z","shell.execute_reply":"2024-12-19T12:43:45.207265Z"},"papermill":{"duration":0.087887,"end_time":"2024-12-19T12:43:45.209436","exception":false,"start_time":"2024-12-19T12:43:45.121549","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission1 = TrainML(voting_model, test)\n\n# Save submission\n# Submission1.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:43:45.359298Z","iopub.status.busy":"2024-12-19T12:43:45.359023Z","iopub.status.idle":"2024-12-19T12:44:15.166307Z","shell.execute_reply":"2024-12-19T12:44:15.165274Z"},"papermill":{"duration":29.88585,"end_time":"2024-12-19T12:44:15.167816","exception":false,"start_time":"2024-12-19T12:43:45.281966","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission1","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:44:15.315731Z","iopub.status.busy":"2024-12-19T12:44:15.315396Z","iopub.status.idle":"2024-12-19T12:44:15.323681Z","shell.execute_reply":"2024-12-19T12:44:15.322845Z"},"papermill":{"duration":0.083442,"end_time":"2024-12-19T12:44:15.325445","exception":false,"start_time":"2024-12-19T12:44:15.242003","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SVR_Model = LinearSVR(random_state = SEED)\nElastic_Model = ElasticNet(random_state = SEED)\nLasso_Model = Lasso(random_state = SEED)\n#imputer = IterativeImputer(random_state=SEED, n_nearest_features = 5)\nimputer = SimpleImputer(strategy='median')\nvoting_model = StackingRegressor(estimators=[\n ('lightgbm', Light),\n ('xgboost', XGB_Model),\n ('catboost', CatBoost_Model),\n ('svr', Pipeline(steps=[('imputer', imputer), ('regressor', SVR_Model)])),\n ('els', Pipeline(steps=[('imputer', imputer), ('regressor', Elastic_Model)]))\n])\n\nSubmission4 = TrainML(voting_model, test)","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:44:15.511770Z","iopub.status.busy":"2024-12-19T12:44:15.511411Z","iopub.status.idle":"2024-12-19T12:47:13.049620Z","shell.execute_reply":"2024-12-19T12:47:13.048752Z"},"papermill":{"duration":177.612877,"end_time":"2024-12-19T12:47:13.051459","exception":false,"start_time":"2024-12-19T12:44:15.438582","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission4","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:47:13.201289Z","iopub.status.busy":"2024-12-19T12:47:13.200992Z","iopub.status.idle":"2024-12-19T12:47:13.209213Z","shell.execute_reply":"2024-12-19T12:47:13.208410Z"},"papermill":{"duration":0.08405,"end_time":"2024-12-19T12:47:13.210860","exception":false,"start_time":"2024-12-19T12:47:13.126810","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ndef process_file(filename, dirname):\n df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n df.drop('step', axis=1, inplace=True)\n return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n ids = os.listdir(dirname)\n \n with ThreadPoolExecutor() as executor:\n results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n \n stats, indexes = zip(*results)\n \n df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n df['id'] = indexes\n return df\n \ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1) \n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n 'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n 'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n 'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n 'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n 'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n 'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n 'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n 'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ndef update(df):\n global cat_c\n for c in cat_c: \n df[c] = df[c].fillna('Missing')\n df[c] = df[c].astype('category')\n return df\n \ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n unique_values = dataset[column].unique()\n return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n mapping = create_mapping(col, train)\n mappingTe = create_mapping(col, test)\n \n train[col] = train[col].replace(mapping).astype(int)\n test[col] = test[col].replace(mappingTe).astype(int)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n return np.where(oof_non_rounded < thresholds[0], 0,\n np.where(oof_non_rounded < thresholds[1], 1,\n np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n X = train.drop(['sii'], axis=1)\n y = train['sii']\n\n SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n \n train_S = []\n test_S = []\n \n oof_non_rounded = np.zeros(len(y), dtype=float) \n oof_rounded = np.zeros(len(y), dtype=int) \n test_preds = np.zeros((len(test_data), n_splits))\n\n for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n model = clone(model_class)\n model.fit(X_train, y_train)\n\n y_train_pred = model.predict(X_train)\n y_val_pred = model.predict(X_val)\n\n oof_non_rounded[test_idx] = y_val_pred\n y_val_pred_rounded = y_val_pred.round(0).astype(int)\n oof_rounded[test_idx] = y_val_pred_rounded\n\n train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n train_S.append(train_kappa)\n test_S.append(val_kappa)\n \n test_preds[:, fold] = model.predict(test_data)\n \n print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n clear_output(wait=True)\n\n print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n KappaOPtimizer = minimize(evaluate_predictions,\n x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n method='Nelder-Mead')\n assert KappaOPtimizer.success, \"Optimization did not converge.\"\n \n oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n tpm = test_preds.mean(axis=1)\n tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n \n submission = pd.DataFrame({\n 'id': sample['id'],\n 'sii': tpTuned\n })\n\n return submission\n\n# Model parameters for LightGBM\nParams = {\n 'learning_rate': 0.046,\n 'max_depth': 12,\n 'num_leaves': 478,\n 'min_data_in_leaf': 13,\n 'feature_fraction': 0.893,\n 'bagging_fraction': 0.784,\n 'bagging_freq': 4,\n 'lambda_l1': 10, # Increased from 6.59\n 'lambda_l2': 0.01 # Increased from 2.68e-06\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n 'learning_rate': 0.05,\n 'max_depth': 6,\n 'n_estimators': 200,\n 'subsample': 0.8,\n 'colsample_bytree': 0.8,\n 'reg_alpha': 1, # Increased from 0.1\n 'reg_lambda': 5, # Increased from 1\n 'random_state': SEED\n}\n\n\nCatBoost_Params = {\n 'learning_rate': 0.05,\n 'depth': 6,\n 'iterations': 200,\n 'random_seed': SEED,\n 'cat_features': cat_c,\n 'verbose': 0,\n 'l2_leaf_reg': 10 # Increase this value\n}\n\n# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\nTabNet_Model = TabNetWrapper(**TabNet_Params)\n#ODT_Model = ObliqueDecisionTreeRegressor(**ODT_Params)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n ('lightgbm', Light),\n ('xgboost', XGB_Model),\n ('catboost', CatBoost_Model),\n #('tabnet', TabNet_Model),\n #('odt', ODT_Model),\n])\n\n# Train the ensemble model\nSubmission2 = TrainML(voting_model, test)","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:47:13.357555Z","iopub.status.busy":"2024-12-19T12:47:13.357244Z","iopub.status.idle":"2024-12-19T12:49:13.267708Z","shell.execute_reply":"2024-12-19T12:49:13.266868Z"},"papermill":{"duration":119.985896,"end_time":"2024-12-19T12:49:13.269363","exception":false,"start_time":"2024-12-19T12:47:13.283467","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission2","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:49:13.419945Z","iopub.status.busy":"2024-12-19T12:49:13.419132Z","iopub.status.idle":"2024-12-19T12:49:13.427443Z","shell.execute_reply":"2024-12-19T12:49:13.426668Z"},"papermill":{"duration":0.084806,"end_time":"2024-12-19T12:49:13.429334","exception":false,"start_time":"2024-12-19T12:49:13.344528","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SVR_Model = LinearSVR(random_state = SEED)\nElastic_Model = ElasticNet(random_state = SEED)\nLasso_Model = Lasso(random_state = SEED)\n#imputer = IterativeImputer(random_state=SEED, n_nearest_features = 5)\nimputer = SimpleImputer(strategy='median')\nvoting_model = StackingRegressor(estimators=[\n ('lightgbm', Light),\n ('xgboost', XGB_Model),\n ('catboost', CatBoost_Model),\n ('svr', Pipeline(steps=[('imputer', imputer), ('regressor', SVR_Model)])),\n ('els', Pipeline(steps=[('imputer', imputer), ('regressor', Elastic_Model)]))\n])\n\n\n\n# Train the ensemble model\nSubmission5 = TrainML(voting_model, test)","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:49:13.580034Z","iopub.status.busy":"2024-12-19T12:49:13.579195Z","iopub.status.idle":"2024-12-19T12:54:14.485120Z","shell.execute_reply":"2024-12-19T12:54:14.484270Z"},"papermill":{"duration":300.981828,"end_time":"2024-12-19T12:54:14.486829","exception":false,"start_time":"2024-12-19T12:49:13.505001","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n 'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n 'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n 'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n 'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n 'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n 'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n 'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n 'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ndef update(df):\n global cat_c\n for c in cat_c: \n df[c] = df[c].fillna('Missing')\n df[c] = df[c].astype('category')\n return df\n\ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n unique_values = dataset[column].unique()\n return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n mapping = create_mapping(col, train)\n mappingTe = create_mapping(col, test)\n \n train[col] = train[col].replace(mapping).astype(int)\n test[col] = test[col].replace(mappingTe).astype(int)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n return np.where(oof_non_rounded < thresholds[0], 0,\n np.where(oof_non_rounded < thresholds[1], 1,\n np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n X = train.drop(['sii'], axis=1)\n y = train['sii']\n\n SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n \n train_S = []\n test_S = []\n \n oof_non_rounded = np.zeros(len(y), dtype=float) \n oof_rounded = np.zeros(len(y), dtype=int) \n test_preds = np.zeros((len(test_data), n_splits))\n\n for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n model = clone(model_class)\n model.fit(X_train, y_train)\n\n y_train_pred = model.predict(X_train)\n y_val_pred = model.predict(X_val)\n\n oof_non_rounded[test_idx] = y_val_pred\n y_val_pred_rounded = y_val_pred.round(0).astype(int)\n oof_rounded[test_idx] = y_val_pred_rounded\n\n train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n train_S.append(train_kappa)\n test_S.append(val_kappa)\n \n test_preds[:, fold] = model.predict(test_data)\n \n print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n clear_output(wait=True)\n\n print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n KappaOPtimizer = minimize(evaluate_predictions,\n x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n method='Nelder-Mead')\n assert KappaOPtimizer.success, \"Optimization did not converge.\"\n \n oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n tpm = test_preds.mean(axis=1)\n tp_rounded = threshold_Rounder(tpm, KappaOPtimizer.x)\n\n return tp_rounded\n\nimputer = SimpleImputer(strategy='median')\n\nensemble = VotingRegressor(estimators=[\n ('lgb', Pipeline(steps=[('imputer', imputer), ('regressor', LGBMRegressor(random_state=SEED))])),\n ('xgb', Pipeline(steps=[('imputer', imputer), ('regressor', XGBRegressor(random_state=SEED))])),\n ('cat', Pipeline(steps=[('imputer', imputer), ('regressor', CatBoostRegressor(random_state=SEED, silent=True))])),\n ('rf', Pipeline(steps=[('imputer', imputer), ('regressor', RandomForestRegressor(random_state=SEED))])),\n ('gb', Pipeline(steps=[('imputer', imputer), ('regressor', GradientBoostingRegressor(random_state=SEED))])),\n #('tabnet', Pipeline(steps=[('imputer', imputer), ('regressor', TabNetWrapper(**TabNet_Params))])),\n #('odt', Pipeline(steps=[('imputer', imputer), ('regressor', ObliqueDecisionTreeRegressor(**ODT_Params))])),\n])\n\nSubmission3 = TrainML(ensemble, test)","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:54:14.638190Z","iopub.status.busy":"2024-12-19T12:54:14.637838Z","iopub.status.idle":"2024-12-19T12:57:20.827645Z","shell.execute_reply":"2024-12-19T12:57:20.826775Z"},"papermill":{"duration":186.265628,"end_time":"2024-12-19T12:57:20.829476","exception":false,"start_time":"2024-12-19T12:54:14.563848","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission3 = pd.DataFrame({\n 'id': sample['id'],\n 'sii': Submission3\n})\n\nSubmission3","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:57:20.979351Z","iopub.status.busy":"2024-12-19T12:57:20.979056Z","iopub.status.idle":"2024-12-19T12:57:20.987846Z","shell.execute_reply":"2024-12-19T12:57:20.986991Z"},"papermill":{"duration":0.084436,"end_time":"2024-12-19T12:57:20.989532","exception":false,"start_time":"2024-12-19T12:57:20.905096","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub1 = Submission1\nsub2 = Submission2\nsub3 = Submission3\nsub4 = Submission4\nsub5 = Submission5\nsub1 = sub1.sort_values(by='id').reset_index(drop=True)\nsub2 = sub2.sort_values(by='id').reset_index(drop=True)\nsub3 = sub3.sort_values(by='id').reset_index(drop=True)\nsub4 = sub4.sort_values(by='id').reset_index(drop=True)\nsub5 = sub5.sort_values(by='id').reset_index(drop=True)\ncombined = pd.DataFrame({\n 'id': sub1['id'],\n 'sii_1': sub1['sii'],\n 'sii_2': sub2['sii'],\n 'sii_3': sub3['sii'],\n 'sii_4': sub4['sii'],\n 'sii_5': sub5['sii']\n})\n\ndef majority_vote(row):\n return row.mode()[0]\n\ncombined['final_sii'] = combined[['sii_1', 'sii_2', 'sii_3', 'sii_4', 'sii_5']].apply(majority_vote, axis=1)\n\nfinal_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\n\nfinal_submission.to_csv('submission.csv', index=False)\n\nprint(\"Majority voting completed and saved to 'Final_Submission.csv'\")","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:57:21.140767Z","iopub.status.busy":"2024-12-19T12:57:21.140423Z","iopub.status.idle":"2024-12-19T12:57:21.160738Z","shell.execute_reply":"2024-12-19T12:57:21.159858Z"},"papermill":{"duration":0.097428,"end_time":"2024-12-19T12:57:21.162218","exception":false,"start_time":"2024-12-19T12:57:21.064790","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_submission","metadata":{"execution":{"iopub.execute_input":"2024-12-19T12:57:21.352743Z","iopub.status.busy":"2024-12-19T12:57:21.352390Z","iopub.status.idle":"2024-12-19T12:57:21.360575Z","shell.execute_reply":"2024-12-19T12:57:21.359847Z"},"papermill":{"duration":0.12378,"end_time":"2024-12-19T12:57:21.362268","exception":false,"start_time":"2024-12-19T12:57:21.238488","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.073363,"end_time":"2024-12-19T12:57:21.511090","exception":false,"start_time":"2024-12-19T12:57:21.437727","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}