{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":10144188,"sourceType":"datasetVersion","datasetId":6261488},{"sourceId":10161833,"sourceType":"datasetVersion","datasetId":6274967},{"sourceId":10200802,"sourceType":"datasetVersion","datasetId":6303553},{"sourceId":10203245,"sourceType":"datasetVersion","datasetId":6305398},{"sourceId":10208164,"sourceType":"datasetVersion","datasetId":6308872},{"sourceId":198747,"sourceType":"modelInstanceVersion","modelInstanceId":169524,"modelId":191870}],"dockerImageVersionId":30805,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## PARQUET PROCESSING","metadata":{}},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nimport os\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nimport torch\nimport torch.nn as nn\nfrom sklearn.preprocessing import StandardScaler\nimport torch.optim as optim","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:15.737406Z","iopub.execute_input":"2024-12-19T21:50:15.737827Z","iopub.status.idle":"2024-12-19T21:50:19.979746Z","shell.execute_reply.started":"2024-12-19T21:50:15.737779Z","shell.execute_reply":"2024-12-19T21:50:19.979025Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## TABULAR NORMAL DATA PROCESSING","metadata":{}},{"cell_type":"markdown","source":"#### Drop any samplers which only missed any values in PCIAT test","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\")\ntest_data = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:19.981002Z","iopub.execute_input":"2024-12-19T21:50:19.981543Z","iopub.status.idle":"2024-12-19T21:50:20.050370Z","shell.execute_reply.started":"2024-12-19T21:50:19.981513Z","shell.execute_reply":"2024-12-19T21:50:20.049732Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"columns_not_in_test = ['PCIAT-PCIAT_01', 'PCIAT-PCIAT_02', 'PCIAT-PCIAT_03', 'PCIAT-PCIAT_04', 'PCIAT-PCIAT_05', 'PCIAT-PCIAT_06', 'PCIAT-PCIAT_07', 'PCIAT-PCIAT_08', 'PCIAT-PCIAT_09', 'PCIAT-PCIAT_10', 'PCIAT-PCIAT_11', 'PCIAT-PCIAT_12', 'PCIAT-PCIAT_13', 'PCIAT-PCIAT_14', 'PCIAT-PCIAT_15', 'PCIAT-PCIAT_16', 'PCIAT-PCIAT_17', 'PCIAT-PCIAT_18', 'PCIAT-PCIAT_19', 'PCIAT-PCIAT_20', 'PCIAT-PCIAT_Total', 'PCIAT-Season', 'sii']\ntrain_data = train_data.dropna(subset=columns_not_in_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:20.051455Z","iopub.execute_input":"2024-12-19T21:50:20.051839Z","iopub.status.idle":"2024-12-19T21:50:20.069572Z","shell.execute_reply.started":"2024-12-19T21:50:20.051797Z","shell.execute_reply":"2024-12-19T21:50:20.068860Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Drop season data","metadata":{"execution":{"iopub.status.busy":"2024-12-16T07:14:27.674415Z","iopub.execute_input":"2024-12-16T07:14:27.674761Z","iopub.status.idle":"2024-12-16T07:14:27.678811Z","shell.execute_reply.started":"2024-12-16T07:14:27.674731Z","shell.execute_reply":"2024-12-16T07:14:27.677871Z"}}},{"cell_type":"code","source":"train_seasonal_columns = [col for col in train_data.columns if 'Season' in col]\ntest_seasonal_columns = [col for col in test_data.columns if 'Season' in col]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:20.071553Z","iopub.execute_input":"2024-12-19T21:50:20.071833Z","iopub.status.idle":"2024-12-19T21:50:20.076062Z","shell.execute_reply.started":"2024-12-19T21:50:20.071807Z","shell.execute_reply":"2024-12-19T21:50:20.075256Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data_wo_season = train_data.drop(train_seasonal_columns, axis = 1)\ntest_data_wo_season = test_data.drop(test_seasonal_columns, axis = 1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:20.077047Z","iopub.execute_input":"2024-12-19T21:50:20.077332Z","iopub.status.idle":"2024-12-19T21:50:20.089863Z","shell.execute_reply.started":"2024-12-19T21:50:20.077307Z","shell.execute_reply":"2024-12-19T21:50:20.089059Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### So, we got a quire reliable labels here, the next step would be create X and y","metadata":{}},{"cell_type":"code","source":"label_related_features = ['PCIAT-PCIAT_01', 'PCIAT-PCIAT_02', 'PCIAT-PCIAT_03', 'PCIAT-PCIAT_04', 'PCIAT-PCIAT_05', 'PCIAT-PCIAT_06', 'PCIAT-PCIAT_07', 'PCIAT-PCIAT_08', 'PCIAT-PCIAT_09', 'PCIAT-PCIAT_10', 'PCIAT-PCIAT_11', 'PCIAT-PCIAT_12', 'PCIAT-PCIAT_13', 'PCIAT-PCIAT_14', 'PCIAT-PCIAT_15', 'PCIAT-PCIAT_16', 'PCIAT-PCIAT_17', 'PCIAT-PCIAT_18', 'PCIAT-PCIAT_19', 'PCIAT-PCIAT_20', 'PCIAT-PCIAT_Total', 'sii']\nlabel = ['sii']\nX = train_data_wo_season.drop(label_related_features, axis = 1)\nnew_y = train_data_wo_season[label]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:20.090986Z","iopub.execute_input":"2024-12-19T21:50:20.091619Z","iopub.status.idle":"2024-12-19T21:50:20.103408Z","shell.execute_reply.started":"2024-12-19T21:50:20.091576Z","shell.execute_reply":"2024-12-19T21:50:20.102814Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_X = X.drop(['id'], axis = 1)\nnew_test = test_data_wo_season.drop(['id'], axis = 1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:20.104311Z","iopub.execute_input":"2024-12-19T21:50:20.104594Z","iopub.status.idle":"2024-12-19T21:50:20.112965Z","shell.execute_reply.started":"2024-12-19T21:50:20.104568Z","shell.execute_reply":"2024-12-19T21:50:20.112135Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_engineering(df):\n    #Age\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['Physical-Waist_Age'] = df['Basic_Demos-Age'] * df['Physical-Waist_Circumference']\n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Physical-Height_Age'] = df['Basic_Demos-Age'] * df['Physical-Height']\n    df['SDS_InternetHours'] = df['SDS-SDS_Total_T'] * df['PreInt_EduHx-computerinternet_hoursday']\n\n    #SDS\n    df['SDS_BMI'] = df['BIA-BIA_BMI'] * df['SDS-SDS_Total_T']\n    df['CGAS_SDS'] = df['CGAS-CGAS_Score'] * df['SDS-SDS_Total_T']\n    df['CGAS_Endurance_Mins'] = df['CGAS-CGAS_Score'] * df['Fitness_Endurance-Time_Mins']\n    df['SDS_Activity'] = df['BIA-BIA_Activity_Level_num'] * df['SDS-SDS_Total_T']\n\n    df['BMI_Systolic_BP'] = df['BIA-BIA_BMI'] * df['Physical-Systolic_BP']\n    df['Age_Systolic_BP'] = df['Basic_Demos-Age'] * df['Physical-Systolic_BP']\n    df['PreInt_Systolic_BP'] = df['Physical-Systolic_BP'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['PAQ_A_Activity'] = df['BIA-BIA_Activity_Level_num'] * df['PAQ_A-PAQ_A_Total']\n    df['Activity_CU_PU'] = df['BIA-BIA_Activity_Level_num'] * df['FGC-FGC_CU'] * df['FGC-FGC_PU']\n\n    #FGC\n    df['FGC_CU_PU'] = df['FGC-FGC_CU'] * df['FGC-FGC_PU']\n    df['FGC_CU_PU_Age'] = df['FGC-FGC_CU'] * df['FGC-FGC_PU'] * df['Basic_Demos-Age']\n    df['FGC_GSND_GSD'] = df['FGC-FGC_GSND'] * df['FGC-FGC_GSD']\n    df['FGC_GSND_GSD_Age'] = df['FGC-FGC_GSND'] * df['FGC-FGC_GSD'] * df['Basic_Demos-Age']\n    df['CGAS_CU_PU'] = df['CGAS-CGAS_Score'] * df['FGC-FGC_CU'] * df['FGC-FGC_PU']\n    df['PreInt_FGC_CU_PU'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['FGC-FGC_CU'] * df['FGC-FGC_PU']\n    df['Endurance_CU_PU'] = df['Fitness_Endurance-Time_Mins'] * df['FGC-FGC_CU'] * df['FGC-FGC_PU']\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:20.114150Z","iopub.execute_input":"2024-12-19T21:50:20.114475Z","iopub.status.idle":"2024-12-19T21:50:20.126565Z","shell.execute_reply.started":"2024-12-19T21:50:20.114438Z","shell.execute_reply":"2024-12-19T21:50:20.125819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_X = feature_engineering(new_X)\nnew_test = feature_engineering(new_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:20.127507Z","iopub.execute_input":"2024-12-19T21:50:20.127832Z","iopub.status.idle":"2024-12-19T21:50:20.155727Z","shell.execute_reply.started":"2024-12-19T21:50:20.127794Z","shell.execute_reply":"2024-12-19T21:50:20.155130Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_test.shape\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:20.158306Z","iopub.execute_input":"2024-12-19T21:50:20.158559Z","iopub.status.idle":"2024-12-19T21:50:20.164186Z","shell.execute_reply.started":"2024-12-19T21:50:20.158534Z","shell.execute_reply":"2024-12-19T21:50:20.163427Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_X.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:20.165190Z","iopub.execute_input":"2024-12-19T21:50:20.165432Z","iopub.status.idle":"2024-12-19T21:50:20.175184Z","shell.execute_reply.started":"2024-12-19T21:50:20.165408Z","shell.execute_reply":"2024-12-19T21:50:20.174361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nstandard_scaler = StandardScaler()\n\nnew_X = pd.DataFrame(standard_scaler.fit_transform(new_X), columns=new_X.columns)\nnew_test = pd.DataFrame(standard_scaler.fit_transform(new_test), columns=new_test.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:20.176090Z","iopub.execute_input":"2024-12-19T21:50:20.176317Z","iopub.status.idle":"2024-12-19T21:50:20.205815Z","shell.execute_reply.started":"2024-12-19T21:50:20.176293Z","shell.execute_reply":"2024-12-19T21:50:20.205000Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_X = new_X.drop('PAQ_A_Activity', axis=1)\nnew_test = new_test.drop('PAQ_A_Activity', axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:20.206670Z","iopub.execute_input":"2024-12-19T21:50:20.206894Z","iopub.status.idle":"2024-12-19T21:50:20.211293Z","shell.execute_reply.started":"2024-12-19T21:50:20.206871Z","shell.execute_reply":"2024-12-19T21:50:20.210515Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## IMPUTATION IN NORMAL TABULAR DATA","metadata":{}},{"cell_type":"code","source":"from sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nimport numpy as np","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:20.212386Z","iopub.execute_input":"2024-12-19T21:50:20.212678Z","iopub.status.idle":"2024-12-19T21:50:20.441760Z","shell.execute_reply.started":"2024-12-19T21:50:20.212653Z","shell.execute_reply":"2024-12-19T21:50:20.441056Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def fill_na_with_MICE(df):\n    df_copy = df.copy()\n    missing_mask = df_copy.isna()\n    original_columns = df_copy.columns.tolist()\n    imputer = IterativeImputer(max_iter=50, random_state=0)\n    imputed_values = imputer.fit_transform(df_copy)\n    imputed_df = pd.DataFrame(\n        imputed_values,\n        columns=original_columns,\n        index=df_copy.index\n    )\n    df_copy[missing_mask] = imputed_df[missing_mask]\n    return df_copy\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:20.442743Z","iopub.execute_input":"2024-12-19T21:50:20.443001Z","iopub.status.idle":"2024-12-19T21:50:20.448203Z","shell.execute_reply.started":"2024-12-19T21:50:20.442975Z","shell.execute_reply":"2024-12-19T21:50:20.447262Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_X = fill_na_with_MICE(new_X)\nnew_test = fill_na_with_MICE(new_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:20.449092Z","iopub.execute_input":"2024-12-19T21:50:20.449365Z","iopub.status.idle":"2024-12-19T21:50:59.466458Z","shell.execute_reply.started":"2024-12-19T21:50:20.449338Z","shell.execute_reply":"2024-12-19T21:50:59.465499Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_X","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:59.467522Z","iopub.execute_input":"2024-12-19T21:50:59.467794Z","iopub.status.idle":"2024-12-19T21:50:59.492674Z","shell.execute_reply.started":"2024-12-19T21:50:59.467767Z","shell.execute_reply":"2024-12-19T21:50:59.491891Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:59.493616Z","iopub.execute_input":"2024-12-19T21:50:59.493940Z","iopub.status.idle":"2024-12-19T21:50:59.517124Z","shell.execute_reply.started":"2024-12-19T21:50:59.493900Z","shell.execute_reply":"2024-12-19T21:50:59.516340Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Training","metadata":{}},{"cell_type":"code","source":"def get_info_for_ftt(df):\n    number_of_cat = 0\n    cat_ranges = []\n    all_features = df.columns.tolist()\n    cat_idx = []\n    for i, feature in enumerate(all_features):\n        if (df[feature].nunique() <= 2):\n            number_of_cat = number_of_cat + 1\n            cat_ranges.append(df[feature].nunique())\n            cat_idx.append(i)\n    \n    num_continuous = df.shape[-1] - number_of_cat\n    return cat_ranges, num_continuous, cat_idx\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:59.518288Z","iopub.execute_input":"2024-12-19T21:50:59.518607Z","iopub.status.idle":"2024-12-19T21:50:59.526101Z","shell.execute_reply.started":"2024-12-19T21:50:59.518570Z","shell.execute_reply":"2024-12-19T21:50:59.525217Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset\n\nclass MyDataset(Dataset):\n    def __init__(self, samples, labels, cat_idx):\n        if isinstance(labels, pd.DataFrame) or isinstance(labels, pd.Series):\n            labels = labels.to_numpy()\n        self.samples = samples\n        self.labels = labels\n        self.cat_idx = cat_idx\n        \n    def __len__(self):\n        return len(self.labels)\n        \n    def divide_cat_num(self, row_data):\n        mask = np.zeros(len(row_data), dtype=bool)\n        mask[self.cat_idx] = True\n        cat_elements = row_data[mask]\n        remaining_elements = row_data[~mask]\n        return cat_elements, remaining_elements\n\n    def __getitem__(self, idx):\n        row_value = self.samples[idx]\n        tensor_row_value = torch.tensor(row_value)\n        cat_values, num_values = self.divide_cat_num(row_value)\n        cat_values = torch.tensor(cat_values, dtype=torch.int32)\n        num_values = torch.tensor(num_values, dtype=torch.float32)\n        label = self.labels[idx]\n        tensor_label = torch.tensor(label, dtype=torch.float32) \n        return cat_values, num_values, tensor_label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:59.527154Z","iopub.execute_input":"2024-12-19T21:50:59.527518Z","iopub.status.idle":"2024-12-19T21:50:59.535895Z","shell.execute_reply.started":"2024-12-19T21:50:59.527489Z","shell.execute_reply":"2024-12-19T21:50:59.535122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.utils.data import DataLoader","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:59.536904Z","iopub.execute_input":"2024-12-19T21:50:59.537171Z","iopub.status.idle":"2024-12-19T21:50:59.549657Z","shell.execute_reply.started":"2024-12-19T21:50:59.537145Z","shell.execute_reply":"2024-12-19T21:50:59.548888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install /kaggle/input/fttransformer/einops-0.8.0-py3-none-any.whl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:50:59.550657Z","iopub.execute_input":"2024-12-19T21:50:59.550893Z","iopub.status.idle":"2024-12-19T21:51:40.513751Z","shell.execute_reply.started":"2024-12-19T21:50:59.550869Z","shell.execute_reply":"2024-12-19T21:51:40.512631Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch.nn as nn\nimport torch.optim as optim","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:40.515212Z","iopub.execute_input":"2024-12-19T21:51:40.515525Z","iopub.status.idle":"2024-12-19T21:51:40.520051Z","shell.execute_reply.started":"2024-12-19T21:51:40.515495Z","shell.execute_reply":"2024-12-19T21:51:40.519146Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_ranges, num_continuous, cat_idx = get_info_for_ftt(new_X)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:40.521273Z","iopub.execute_input":"2024-12-19T21:51:40.521630Z","iopub.status.idle":"2024-12-19T21:51:40.546519Z","shell.execute_reply.started":"2024-12-19T21:51:40.521592Z","shell.execute_reply":"2024-12-19T21:51:40.545708Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Making wrapper for FTTransformer","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn.functional as F\nfrom torch import nn, einsum\n\nfrom einops import rearrange, repeat\n\n# feedforward and attention\n\nclass GEGLU(nn.Module):\n    def forward(self, x):\n        x, gates = x.chunk(2, dim = -1)\n        return x * F.gelu(gates)\n\ndef FeedForward(dim, mult = 4, dropout = 0.):\n    return nn.Sequential(\n        nn.LayerNorm(dim),\n        nn.Linear(dim, dim * mult * 2),\n        GEGLU(),\n        nn.Dropout(dropout),\n        nn.Linear(dim * mult, dim)\n    )\n\nclass Attention(nn.Module):\n    def __init__(\n        self,\n        dim,\n        heads = 8,\n        dim_head = 64,\n        dropout = 0.\n    ):\n        super().__init__()\n        inner_dim = dim_head * heads\n        self.heads = heads\n        self.scale = dim_head ** -0.5\n\n        self.norm = nn.LayerNorm(dim)\n        \n        self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)\n        self.to_out = nn.Linear(inner_dim, dim, bias = False)\n\n        self.dropout = nn.Dropout(dropout)\n\n    def forward(self, x):\n        h = self.heads\n\n        x = self.norm(x)\n\n        q, k, v = self.to_qkv(x).chunk(3, dim = -1)\n        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = h), (q, k, v))\n        q = q * self.scale\n\n        sim = einsum('b h i d, b h j d -> b h i j', q, k)\n\n        attn = sim.softmax(dim = -1)\n        dropped_attn = self.dropout(attn)\n\n        out = einsum('b h i j, b h j d -> b h i d', dropped_attn, v)\n        out = rearrange(out, 'b h n d -> b n (h d)', h = h)\n        out = self.to_out(out)\n\n        return out, attn\n\n# transformer\n\nclass Transformer(nn.Module):\n    def __init__(\n        self,\n        dim,\n        depth,\n        heads,\n        dim_head,\n        attn_dropout,\n        ff_dropout\n    ):\n        super().__init__()\n        self.layers = nn.ModuleList([])\n\n        for _ in range(depth):\n            self.layers.append(nn.ModuleList([\n                Attention(dim, heads = heads, dim_head = dim_head, dropout = attn_dropout),\n                FeedForward(dim, dropout = ff_dropout),\n            ]))\n\n    def forward(self, x, return_attn = False):\n        post_softmax_attns = []\n\n        for attn, ff in self.layers:\n            attn_out, post_softmax_attn = attn(x)\n            post_softmax_attns.append(post_softmax_attn)\n\n            x = attn_out + x\n            x = ff(x) + x\n\n        if not return_attn:\n            return x\n\n        return x, torch.stack(post_softmax_attns)\n        \n# batch norm\nclass BatchNormSequence(nn.Module):\n    def __init__(self, dim):\n        super().__init__()\n        self.bn = nn.BatchNorm1d(dim)\n        \n    def forward(self, x):\n        # x: (batch, sequence, features)\n        x = x.transpose(1, 2)\n        x = self.bn(x)\n        x = x.transpose(1, 2) \n        return x\n        \n# numerical embedder\nclass NumericalEmbedder(nn.Module):\n    def __init__(self, dim, num_numerical_types):\n        super().__init__()\n        self.weights = nn.Parameter(torch.randn(num_numerical_types, dim))\n        self.biases = nn.Parameter(torch.randn(num_numerical_types, dim))\n\n    def forward(self, x):\n        x = rearrange(x, 'b n -> b n 1')\n        return x * self.weights + self.biases\n\n# main class\n\nclass FTTransformer(nn.Module):\n    def __init__(\n        self,\n        *,\n        categories,\n        num_continuous,\n        dim,\n        depth,\n        heads,\n        dim_head = 16,\n        dim_out = 1,\n        num_special_tokens = 2,\n        attn_dropout = 0.,\n        ff_dropout = 0.\n    ):\n        super().__init__()\n        assert all(map(lambda n: n > 0, categories)), 'number of each category must be positive'\n        assert len(categories) + num_continuous > 0, 'input shape must not be null'\n\n        # categories related calculations\n\n        self.num_categories = len(categories)\n        self.num_unique_categories = sum(categories)\n\n        # create category embeddings table\n\n        self.num_special_tokens = num_special_tokens\n        total_tokens = self.num_unique_categories + num_special_tokens\n\n        # for automatically offsetting unique category ids to the correct position in the categories embedding table\n\n        if self.num_unique_categories > 0:\n            categories_offset = F.pad(torch.tensor(list(categories)), (1, 0), value = num_special_tokens)\n            categories_offset = categories_offset.cumsum(dim = -1)[:-1]\n            self.register_buffer('categories_offset', categories_offset)\n\n            # categorical embedding\n\n            self.categorical_embeds = nn.Embedding(total_tokens, dim)\n            self.categ_bn = BatchNormSequence(dim)\n\n\n        # continuous\n\n        self.num_continuous = num_continuous\n\n        if self.num_continuous > 0:\n            self.numerical_embedder = NumericalEmbedder(dim, self.num_continuous)\n            self.numer_bn = BatchNormSequence(dim)\n\n\n        # cls token\n\n        self.cls_token = nn.Parameter(torch.randn(1, 1, dim))\n        self.pre_transformer_bn = BatchNormSequence(dim)\n\n        # transformer\n\n        self.transformer = Transformer(\n            dim = dim,\n            depth = depth,\n            heads = heads,\n            dim_head = dim_head,\n            attn_dropout = attn_dropout,\n            ff_dropout = ff_dropout\n        )\n\n        # to logits\n\n        self.to_logits = nn.Sequential(\n            nn.LayerNorm(dim),\n            nn.ReLU(),\n            nn.Linear(dim, dim_out)\n        )\n\n    def forward(self, x_categ, x_numer, return_attn = False):\n        assert x_categ.shape[-1] == self.num_categories, f'you must pass in {self.num_categories} values for your categories input'\n\n        xs = []\n        if self.num_unique_categories > 0:\n            x_categ = x_categ + self.categories_offset\n\n            x_categ = self.categorical_embeds(x_categ)\n            x_categ = self.categ_bn(x_categ)\n            xs.append(x_categ)\n\n        # add numerically embedded tokens\n        if self.num_continuous > 0:\n            x_numer = self.numerical_embedder(x_numer)\n            x_numer = self.numer_bn(x_numer)\n            xs.append(x_numer)\n\n        # concat categorical and numerical\n\n        x = torch.cat(xs, dim = 1)\n        x = self.pre_transformer_bn(x)\n\n\n        # append cls tokens\n        b = x.shape[0]\n        cls_tokens = repeat(self.cls_token, '1 1 d -> b 1 d', b = b)\n        x = torch.cat((cls_tokens, x), dim = 1)\n        \n        # attend\n\n        x, attns = self.transformer(x, return_attn = True)\n\n        # get cls token\n\n        x = x[:, 0]\n\n        # out in the paper is linear(relu(ln(cls)))\n\n        logits = self.to_logits(x)\n\n        if not return_attn:\n            return logits\n\n        return logits, attns\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:40.547821Z","iopub.execute_input":"2024-12-19T21:51:40.548091Z","iopub.status.idle":"2024-12-19T21:51:40.706180Z","shell.execute_reply.started":"2024-12-19T21:51:40.548052Z","shell.execute_reply":"2024-12-19T21:51:40.705373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.base import BaseEstimator, RegressorMixin\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nimport os\nimport torch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:40.707231Z","iopub.execute_input":"2024-12-19T21:51:40.707553Z","iopub.status.idle":"2024-12-19T21:51:40.716737Z","shell.execute_reply.started":"2024-12-19T21:51:40.707526Z","shell.execute_reply":"2024-12-19T21:51:40.715977Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import cohen_kappa_score\n\ndef qwk(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:40.717784Z","iopub.execute_input":"2024-12-19T21:51:40.718120Z","iopub.status.idle":"2024-12-19T21:51:40.729785Z","shell.execute_reply.started":"2024-12-19T21:51:40.718061Z","shell.execute_reply":"2024-12-19T21:51:40.728939Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class FTTransformerWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, categories, num_continuous, dim, dim_out, depth, heads, attn_dropout, \n                 ff_dropout, batch_size, num_epochs, learning_rate, cat_ranges, cat_idx):\n        self.categories = categories\n        self.num_continuous = num_continuous\n        self.dim = dim\n        self.dim_out = dim_out\n        self.depth = depth\n        self.heads = heads\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.batch_size = batch_size\n        self.num_epochs = num_epochs\n        self.learning_rate = learning_rate\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        self.cat_ranges = cat_ranges\n        self.cat_idx = cat_idx\n        self.num_epochs = num_epochs\n        \n    def _init_model(self):\n        self.model = FTTransformer(\n            categories=self.categories,\n            num_continuous=self.num_continuous,\n            dim=self.dim,\n            dim_out=self.dim_out,\n            depth=self.depth,\n            heads=self.heads,\n            attn_dropout=self.attn_dropout,\n            ff_dropout=self.ff_dropout\n        ).to(self.device)\n        self.criterion = torch.nn.MSELoss()\n        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.learning_rate)\n\n    def fit(self, X, y):\n        X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n        from sklearn.preprocessing import StandardScaler\n        self.scaler = StandardScaler()\n        scaled_X_train = self.scaler.fit_transform(X_train)\n        scaled_X_val = self.scaler.transform(X_val)\n        \n        train_dataset = MyDataset(scaled_X_train, y_train, self.cat_idx)\n        val_dataset = MyDataset(scaled_X_val, y_val, self.cat_idx)\n        train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n        val_loader = DataLoader(val_dataset, batch_size=32, shuffle=True)\n\n        self._init_model()\n        best_val_loss = float('inf')\n\n        save_path='model_checkpoints'\n        os.makedirs(save_path, exist_ok=True)\n\n        for epoch in range(self.num_epochs):\n            self.model.train()\n            train_loss = 0\n            for x_cat, x_num, y in train_loader:\n                x_cat = x_cat.to(self.device) if x_cat is not None else None\n                x_num = x_num.to(self.device)\n                y = y.to(self.device)\n\n                self.optimizer.zero_grad()\n                output = self.model(x_cat, x_num).squeeze(1)\n                \n                loss = self.criterion(output, y)\n                loss.backward()\n                self.optimizer.step()\n                train_loss += loss.item()\n            \n            self.model.eval()\n            val_loss = 0\n            with torch.no_grad():\n                for x_cat, x_num, y in val_loader:\n                    x_cat = x_cat.to(self.device) if x_cat is not None else None\n                    x_num = x_num.to(self.device)\n                    y = y.to(self.device)\n                    \n                    output = self.model(x_cat, x_num).squeeze(1)\n\n                    val_loss += self.criterion(output, y).item()\n\n            avg_train_loss = train_loss / len(train_loader)\n            avg_val_loss = val_loss / len(val_loader)\n            print(f'Epoch {epoch+1}, Train Loss: {avg_train_loss:.4f}, Val Loss: {avg_val_loss:.4f}')\n\n            if avg_val_loss < best_val_loss:\n                best_val_loss = avg_val_loss\n                checkpoint = {\n                    'epoch': epoch,\n                    'model_state_dict': self.model.state_dict(),\n                    'optimizer_state_dict': self.optimizer.state_dict(),\n                    'train_loss': avg_train_loss,\n                    'val_loss': avg_val_loss,\n                    'best_val_loss': best_val_loss\n                }\n                torch.save(checkpoint, os.path.join(save_path, f'best_model.pth'))\n                print(f'Saved best model with validation loss: {best_val_loss:.4f}')\n\n    def load_model(self, model, checkpoint_path):\n        checkpoint = torch.load(checkpoint_path)\n        model.load_state_dict(checkpoint['model_state_dict'])\n        return model\n\n    def load_model_checkpoint(self, checkpoint_path):\n        try:\n            checkpoint = torch.load(checkpoint_path, map_location=self.device)\n            self.model.load_state_dict(checkpoint['model_state_dict'])\n            self.model.to(self.device)\n            self.model.eval()\n            print(f\"Model loaded from {checkpoint_path}\")\n        except Exception as e:\n            print(f\"Error loading the model: {e}\")\n    \n    def predict(self, X):\n        self.model = self.load_model(self.model, \"/kaggle/working/model_checkpoints/best_model.pth\")\n\n        def divide_cat_num(data, cat_idx):\n            mask = np.zeros(data.shape[1], dtype=bool)\n            mask[cat_idx] = True\n            cat_elements = data[:, mask]\n            remaining_elements = data[:, ~mask]\n            return cat_elements, remaining_elements\n\n        self.model.eval()\n        new_test_values = X.values\n        new_test_values = self.scaler.transform(new_test_values)\n        num_samples = len(new_test_values)\n        predictions = []\n\n        for i in range(0, num_samples, self.batch_size):\n            batch_data = new_test_values[i:i + self.batch_size]\n            cat, num = divide_cat_num(batch_data, self.cat_idx)\n            cat = torch.tensor(cat, dtype=torch.int32).to(self.device)\n            num = torch.tensor(num, dtype=torch.float32).to(self.device)\n\n            with torch.no_grad():\n                output = self.model(cat, num)\n                output = output.squeeze(1)\n                predictions.extend(output.cpu().numpy())\n\n            del cat\n            del num\n            del output\n            torch.cuda.empty_cache()\n\n        return np.array(predictions)\n\n        for epoch in range(self.num_epochs):\n            self.model.train()\n            train_loss = 0\n            for x_cat, x_num, y in train_loader:\n                x_cat = x_cat.to(self.device) if x_cat is not None else None\n                x_num = x_num.to(self.device)\n                y = y.to(self.device)\n\n                self.optimizer.zero_grad()\n                output = self.model(x_cat, x_num).squeeze(1)\n                \n                loss = self.criterion(output, y)\n                loss.backward()\n                self.optimizer.step()\n                train_loss += loss.item()\n            \n            self.model.eval()\n            val_loss = 0\n            with torch.no_grad():\n                for x_cat, x_num, y in val_loader:\n                    x_cat = x_cat.to(self.device) if x_cat is not None else None\n                    x_num = x_num.to(self.device)\n                    y = y.to(self.device)\n                    \n                    output = self.model(x_cat, x_num).squeeze(1)\n\n                    val_loss += self.criterion(output, y).item()\n\n            avg_train_loss = train_loss / len(train_loader)\n            avg_val_loss = val_loss / len(val_loader)\n            print(f'Epoch {epoch+1}, Train Loss: {avg_train_loss:.4f}, Val Loss: {avg_val_loss:.4f}')\n\n            if avg_val_loss < best_val_loss:\n                best_val_loss = avg_val_loss\n                checkpoint = {\n                    'epoch': epoch,\n                    'model_state_dict': self.model.state_dict(),\n                    'optimizer_state_dict': self.optimizer.state_dict(),\n                    'train_loss': avg_train_loss,\n                    'val_loss': avg_val_loss,\n                    'best_val_loss': best_val_loss\n                }\n                torch.save(checkpoint, os.path.join(save_path, f'best_model.pth'))\n                print(f'Saved best model with validation loss: {best_val_loss:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:40.734513Z","iopub.execute_input":"2024-12-19T21:51:40.734836Z","iopub.status.idle":"2024-12-19T21:51:40.757106Z","shell.execute_reply.started":"2024-12-19T21:51:40.734809Z","shell.execute_reply":"2024-12-19T21:51:40.756408Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Try with only tabnet","metadata":{}},{"cell_type":"markdown","source":"## Submission preparation","metadata":{}},{"cell_type":"code","source":"def thresh_rounder_opt(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef eval_preds(thresholds, y_true, oof_non_rounded):\n    rounded_p = thresh_rounder_opt(oof_non_rounded, thresholds)\n    return -qwk(y_true, rounded_p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:40.757981Z","iopub.execute_input":"2024-12-19T21:51:40.758232Z","iopub.status.idle":"2024-12-19T21:51:40.770099Z","shell.execute_reply.started":"2024-12-19T21:51:40.758188Z","shell.execute_reply":"2024-12-19T21:51:40.769275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom sklearn.base import clone","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:40.771066Z","iopub.execute_input":"2024-12-19T21:51:40.771408Z","iopub.status.idle":"2024-12-19T21:51:40.788803Z","shell.execute_reply.started":"2024-12-19T21:51:40.771367Z","shell.execute_reply":"2024-12-19T21:51:40.788122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:40.789728Z","iopub.execute_input":"2024-12-19T21:51:40.790040Z","iopub.status.idle":"2024-12-19T21:51:40.805250Z","shell.execute_reply.started":"2024-12-19T21:51:40.789998Z","shell.execute_reply":"2024-12-19T21:51:40.804587Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_splits = 5\nSEED = 42","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:40.806129Z","iopub.execute_input":"2024-12-19T21:51:40.806356Z","iopub.status.idle":"2024-12-19T21:51:40.810356Z","shell.execute_reply.started":"2024-12-19T21:51:40.806331Z","shell.execute_reply":"2024-12-19T21:51:40.809453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def thresh_rounder(oof_non_rounded):\n    return np.where(oof_non_rounded <= 30, 0,\n                    np.where(oof_non_rounded <= 49, 1,\n                             np.where(oof_non_rounded <= 79, 2, 3)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:40.811284Z","iopub.execute_input":"2024-12-19T21:51:40.811533Z","iopub.status.idle":"2024-12-19T21:51:40.822643Z","shell.execute_reply.started":"2024-12-19T21:51:40.811508Z","shell.execute_reply":"2024-12-19T21:51:40.821821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prepare_submission(model_class, test_data):\n    X = new_X\n    y = new_y['sii']\n    \n    KF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(KF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        oof_rounded[test_idx] = y_val_pred.round(0).astype(int)\n\n        train_kappa = qwk(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = qwk(y_val, y_val_pred.round(0).astype(int))\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n\n    qwkOptimizer = minimize(eval_preds, x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), method='Nelder-Mead')\n    tpm = test_preds.mean(axis=1)\n    tp_tuned = thresh_rounder_opt(tpm, qwkOptimizer.x)\n    print(tpm)\n\n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tp_tuned\n    })\n\n    return submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:40.823602Z","iopub.execute_input":"2024-12-19T21:51:40.823846Z","iopub.status.idle":"2024-12-19T21:51:40.834342Z","shell.execute_reply.started":"2024-12-19T21:51:40.823821Z","shell.execute_reply":"2024-12-19T21:51:40.833581Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Ensemble learning","metadata":{}},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:40.835483Z","iopub.execute_input":"2024-12-19T21:51:40.835751Z","iopub.status.idle":"2024-12-19T21:51:43.528454Z","shell.execute_reply.started":"2024-12-19T21:51:40.835701Z","shell.execute_reply":"2024-12-19T21:51:43.527548Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Submission 1","metadata":{}},{"cell_type":"code","source":"lgbm_params = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,\n    'lambda_l2': 0.01,\n    'device': 'cpu'\n}\n\nxgb_params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,\n    'reg_lambda': 5,\n    'random_state': 42,\n    'device': \"cuda\"\n}\n\n\ncatboost_params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': 42,\n    'verbose': 0,\n    'l2_leaf_reg': 10,\n    'task_type': 'GPU',\n}\n\nftt_params = {\n    'categories': tuple(cat_ranges), \n    'num_continuous': num_continuous,    \n    'dim': 16,             \n    'dim_out': 1,          \n    'depth': 2,            \n    'heads': 3,            \n    'attn_dropout': 0.1,   \n    'ff_dropout': 0.2,\n    'batch_size': 32,\n    'num_epochs': 70,\n    'learning_rate': 0.001,\n    'cat_ranges': cat_ranges,\n    'cat_idx': cat_idx \n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:43.529700Z","iopub.execute_input":"2024-12-19T21:51:43.530398Z","iopub.status.idle":"2024-12-19T21:51:43.536261Z","shell.execute_reply.started":"2024-12-19T21:51:43.530367Z","shell.execute_reply":"2024-12-19T21:51:43.535358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"FTT_Model = FTTransformerWrapper(**ftt_params)\nLGBM_Model = LGBMRegressor(**lgbm_params, random_state=42, n_estimators=300, verbose=-1)\nXGB_Model = XGBRegressor(**xgb_params)\nCatBoost_Model = CatBoostRegressor(**catboost_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:43.537335Z","iopub.execute_input":"2024-12-19T21:51:43.537624Z","iopub.status.idle":"2024-12-19T21:51:43.609917Z","shell.execute_reply.started":"2024-12-19T21:51:43.537597Z","shell.execute_reply":"2024-12-19T21:51:43.609136Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_splits = 5\nSEED = 42\nX = new_X\ny = new_y['sii']\n\nKF = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n\nfor fold, (train_idx, test_idx) in enumerate(tqdm(KF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n    X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n    y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n    \n    # Train LightGBM\n    LGBM_Model.fit(\n        X_train, y_train,\n        eval_set=[(X_val, y_val)],\n        eval_metric='mse',\n    )\n    \n    # Train XGBoost\n    XGB_Model.fit(\n        X_train, y_train,\n        eval_set=[(X_val, y_val)],\n        eval_metric='rmse',\n        early_stopping_rounds=30,\n        verbose=True\n    )\n    \n    # Train CatBoost\n    CatBoost_Model.fit(\n        X_train, y_train,\n        eval_set=(X_val, y_val),\n        use_best_model=True,\n        verbose=10\n    )\n    \n    # Train FTT_Model\n    FTT_Model.fit(\n        X_train, y_train,\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:51:43.610981Z","iopub.execute_input":"2024-12-19T21:51:43.611265Z","iopub.status.idle":"2024-12-19T21:54:53.880047Z","shell.execute_reply.started":"2024-12-19T21:51:43.611239Z","shell.execute_reply":"2024-12-19T21:54:53.879156Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"voting_model = VotingRegressor(estimators=[\n    ('lightgbm', LGBM_Model),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model),\n    ('ftt', FTT_Model),\n],weights=[4.0,4.0,5.0, 3.0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:54:53.881478Z","iopub.execute_input":"2024-12-19T21:54:53.882177Z","iopub.status.idle":"2024-12-19T21:54:53.887035Z","shell.execute_reply.started":"2024-12-19T21:54:53.882131Z","shell.execute_reply":"2024-12-19T21:54:53.886273Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# X_train, X_test, y_train, y_test = train_test_split(\n#     new_X,              \n#     new_y,               \n#     test_size=0.2,       \n#     random_state=42,     \n#     shuffle=True         \n# )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:54:53.888245Z","iopub.execute_input":"2024-12-19T21:54:53.888672Z","iopub.status.idle":"2024-12-19T21:54:53.898789Z","shell.execute_reply.started":"2024-12-19T21:54:53.888632Z","shell.execute_reply":"2024-12-19T21:54:53.898018Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:54:53.899636Z","iopub.execute_input":"2024-12-19T21:54:53.899913Z","iopub.status.idle":"2024-12-19T21:54:53.910840Z","shell.execute_reply.started":"2024-12-19T21:54:53.899872Z","shell.execute_reply":"2024-12-19T21:54:53.909995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission1 = prepare_submission(voting_model, new_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:54:53.911979Z","iopub.execute_input":"2024-12-19T21:54:53.912806Z","iopub.status.idle":"2024-12-19T21:57:50.300971Z","shell.execute_reply.started":"2024-12-19T21:54:53.912764Z","shell.execute_reply":"2024-12-19T21:57:50.300094Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:57:50.302296Z","iopub.execute_input":"2024-12-19T21:57:50.303298Z","iopub.status.idle":"2024-12-19T21:57:50.311984Z","shell.execute_reply.started":"2024-12-19T21:57:50.303233Z","shell.execute_reply":"2024-12-19T21:57:50.311130Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Submission 2","metadata":{}},{"cell_type":"code","source":"!pip -q install /kaggle/input/pytorchtabnet/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:57:50.312988Z","iopub.execute_input":"2024-12-19T21:57:50.313575Z","iopub.status.idle":"2024-12-19T21:58:30.664550Z","shell.execute_reply.started":"2024-12-19T21:57:50.313545Z","shell.execute_reply":"2024-12-19T21:58:30.663249Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pytorch_tabnet.tab_model import TabNetRegressor\nfrom pytorch_tabnet.callbacks import Callback\nimport pytorch_tabnet\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.linear_model import Ridge\nfrom sklearn.svm import SVR\nfrom sklearn.impute import KNNImputer","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:00:44.872271Z","iopub.execute_input":"2024-12-19T22:00:44.872724Z","iopub.status.idle":"2024-12-19T22:00:44.878297Z","shell.execute_reply.started":"2024-12-19T22:00:44.872686Z","shell.execute_reply":"2024-12-19T22:00:44.877298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TabNetWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, **kwargs):\n        self.model = TabNetRegressor(**kwargs)\n        self.kwargs = kwargs\n        self.imputer = KNNImputer(n_neighbors=5)\n        #self.imputer = SimpleImputer(strategy='median')\n        self.best_model_path = 'best_tabnet_model.pt'\n\n    def fit(self, X, y):\n        X_imputed = self.imputer.fit_transform(X)\n\n        if hasattr(y, 'values'):\n            y = y.values\n\n        X_train, X_valid, y_train, y_valid = train_test_split(\n            X_imputed,\n            y,\n            test_size=0.2,\n            random_state=42\n        )\n\n        # Train TabNet model\n        history = self.model.fit(\n            X_train=X_train,\n            y_train=y_train.reshape(-1, 1),\n            eval_set=[(X_valid, y_valid.reshape(-1, 1))],\n            eval_name=['valid'],\n            eval_metric=['mse', 'mae', 'rmse'],\n            max_epochs=500,\n            patience=50,\n            batch_size=1024,\n            virtual_batch_size=128,\n            num_workers=0,\n            drop_last=False,\n            callbacks=[\n                TabNetPretrainedModelCheckpoint(\n                    filepath=self.best_model_path,\n                    monitor='valid_mse',\n                    mode='min',\n                    save_best_only=True,\n                    verbose=True\n                )\n            ]\n        )\n\n        # Load the best model\n        if os.path.exists(self.best_model_path):\n            self.model.load_model(self.best_model_path)\n            os.remove(self.best_model_path)  # Remove temporary file\n\n        return self\n\n    def predict(self, X):\n        X_imputed = self.imputer.transform(X)\n        return self.model.predict(X_imputed).flatten()\n\n    def __deepcopy__(self, memo):\n        cls = self.__class__\n        result = cls.__new__(cls)\n        memo[id(self)] = result\n        for k, v in self.__dict__.items():\n            setattr(result, k, deepcopy(v, memo))\n        return result\n        \nTabNet_Params = {\n    'n_d': 64,\n    'n_a': 64,\n    'n_steps': 5,\n    'gamma': 1.5,\n    'n_independent': 2,\n    'n_shared': 2,\n    'lambda_sparse': 1e-4,\n    'optimizer_fn': torch.optim.Adam,\n    'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n    'mask_type': 'entmax',\n    'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n    'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n    'verbose': 1,\n    'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\n\nclass TabNetPretrainedModelCheckpoint(Callback):\n    def __init__(self, filepath, monitor='val_loss', mode='min',\n                 save_best_only=True, verbose=1):\n        super().__init__()\n        self.filepath = filepath\n        self.monitor = monitor\n        self.mode = mode\n        self.save_best_only = save_best_only\n        self.verbose = verbose\n        self.best = float('inf') if mode == 'min' else -float('inf')\n\n    def on_train_begin(self, logs=None):\n        self.model = self.trainer\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is None:\n            return\n\n        if (self.mode == 'min' and current < self.best) or \\\n           (self.mode == 'max' and current > self.best):\n            if self.verbose:\n                print(f'\\nEpoch {epoch}: {self.monitor} improved from {self.best:.4f} to {current:.4f}')\n            self.best = current\n            if self.save_best_only:\n                self.model.save_model(self.filepath)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:00:46.051807Z","iopub.execute_input":"2024-12-19T22:00:46.052759Z","iopub.status.idle":"2024-12-19T22:00:46.067349Z","shell.execute_reply.started":"2024-12-19T22:00:46.052717Z","shell.execute_reply":"2024-12-19T22:00:46.066459Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgbm_params = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,\n    'lambda_l2': 0.01,\n    'device': 'cpu'\n}\nCatBoost_Params = {\n        'learning_rate': 0.05,\n        'depth': 6,\n        'iterations': 200,\n        'random_seed': SEED,\n        'verbose': 0,\n        'l2_leaf_reg': 10,\n        'task_type': 'CPU',\n        'use_best_model': True\n}\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  \n    'reg_lambda': 5,  \n    'random_state': SEED,\n    'tree_method': 'gpu_hist',\n}\n\nftt_params = {\n    'categories': tuple(cat_ranges), \n    'num_continuous': num_continuous,    \n    'dim': 16,             \n    'dim_out': 1,          \n    'depth': 2,            \n    'heads': 3,            \n    'attn_dropout': 0.1,   \n    'ff_dropout': 0.3,\n    'batch_size': 64 ,\n    'num_epochs': 70,\n    'learning_rate': 0.001,\n    'cat_ranges': cat_ranges,\n    'cat_idx': cat_idx \n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:58:30.712995Z","iopub.execute_input":"2024-12-19T21:58:30.713307Z","iopub.status.idle":"2024-12-19T21:58:30.726210Z","shell.execute_reply.started":"2024-12-19T21:58:30.713280Z","shell.execute_reply":"2024-12-19T21:58:30.725384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TabNet_Model = TabNetWrapper(**TabNet_Params) \nFTT_Model = FTTransformerWrapper(**ftt_params)\nXGB_Model = XGBRegressor(**xgb_params)\nCatBoost_Model = CatBoostRegressor(**catboost_params)\nLGBM_Model = LGBMRegressor(**lgbm_params, random_state=42, n_estimators=300, verbose=-1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:00:51.779961Z","iopub.execute_input":"2024-12-19T22:00:51.780697Z","iopub.status.idle":"2024-12-19T22:00:51.786499Z","shell.execute_reply.started":"2024-12-19T22:00:51.780659Z","shell.execute_reply":"2024-12-19T22:00:51.785795Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_splits = 5\nSEED = 42\nX = new_X\ny = new_y['sii']\n\nKF = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n\nfor fold, (train_idx, test_idx) in enumerate(tqdm(KF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n    X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n    y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n    \n    # Train LightGBM\n    LGBM_Model.fit(\n        X_train, y_train,\n        eval_set=[(X_val, y_val)],\n        eval_metric='mse',\n    )\n    \n    # Train XGBoost\n    XGB_Model.fit(\n        X_train, y_train,\n        eval_set=[(X_val, y_val)],\n        eval_metric='rmse',\n        early_stopping_rounds=30,\n        verbose=True\n    )\n    \n    # Train CatBoost\n    CatBoost_Model.fit(\n        X_train, y_train,\n        eval_set=(X_val, y_val),\n        use_best_model=True,\n        verbose=10\n    )\n    \n    # Train FTT_Model\n    FTT_Model.fit(\n        X_train, y_train,\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:00:53.339530Z","iopub.execute_input":"2024-12-19T22:00:53.340175Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"voting_model = VotingRegressor(estimators=[\n    ('lightgbm', LGBM_Model),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model),\n    ('ftt', FTT_Model),\n    ('tabnet', TabNet_Model)\n],weights=[4.0,4.0,4.0, 3.0,5.0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:58:31.126759Z","iopub.status.idle":"2024-12-19T21:58:31.127179Z","shell.execute_reply.started":"2024-12-19T21:58:31.126956Z","shell.execute_reply":"2024-12-19T21:58:31.126979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission2= prepare_submission(voting_model, new_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:58:31.128630Z","iopub.status.idle":"2024-12-19T21:58:31.128949Z","shell.execute_reply.started":"2024-12-19T21:58:31.128800Z","shell.execute_reply":"2024-12-19T21:58:31.128818Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:58:31.130430Z","iopub.status.idle":"2024-12-19T21:58:31.130752Z","shell.execute_reply.started":"2024-12-19T21:58:31.130596Z","shell.execute_reply":"2024-12-19T21:58:31.130613Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Submission 3","metadata":{}},{"cell_type":"code","source":"TabNet_Model = TabNetWrapper(**TabNet_Params) \nXGB_Model = XGBRegressor(**xgb_params)\nCatBoost_Model = CatBoostRegressor(**catboost_params)\nLGBM_Model = LGBMRegressor(**lgbm_params, random_state=42, n_estimators=300, verbose=-1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:58:31.131805Z","iopub.status.idle":"2024-12-19T21:58:31.132137Z","shell.execute_reply.started":"2024-12-19T21:58:31.131959Z","shell.execute_reply":"2024-12-19T21:58:31.131975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_splits = 5\nSEED = 42\nX = new_X\ny = new_y['sii']\n\nKF = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n\nfor fold, (train_idx, test_idx) in enumerate(tqdm(KF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n    X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n    y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n    \n    # Train LightGBM\n    LGBM_Model.fit(\n        X_train, y_train,\n        eval_set=[(X_val, y_val)],\n        eval_metric='mse',\n    )\n    \n    # Train XGBoost\n    XGB_Model.fit(\n        X_train, y_train,\n        eval_set=[(X_val, y_val)],\n        eval_metric='rmse',\n        early_stopping_rounds=30,\n        verbose=True\n    )\n    \n    # Train CatBoost\n    CatBoost_Model.fit(\n        X_train, y_train,\n        eval_set=(X_val, y_val),\n        use_best_model=True,\n        verbose=10\n    )\n    \n    # Train FTT_Model\n    FTT_Model.fit(\n        X_train, y_train,\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:58:31.133499Z","iopub.status.idle":"2024-12-19T21:58:31.134045Z","shell.execute_reply.started":"2024-12-19T21:58:31.133658Z","shell.execute_reply":"2024-12-19T21:58:31.133676Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"voting_model = VotingRegressor(estimators=[\n    ('lightgbm', LGBM_Model),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model),\n    ('tabnet', TabNet_Model)\n],weights=[4.0,3.0,5.0, 4.0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:58:31.135004Z","iopub.status.idle":"2024-12-19T21:58:31.135346Z","shell.execute_reply.started":"2024-12-19T21:58:31.135192Z","shell.execute_reply":"2024-12-19T21:58:31.135215Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission3= prepare_submission(voting_model, new_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:58:31.136466Z","iopub.status.idle":"2024-12-19T21:58:31.136787Z","shell.execute_reply.started":"2024-12-19T21:58:31.136627Z","shell.execute_reply":"2024-12-19T21:58:31.136644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission3","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:58:31.137915Z","iopub.status.idle":"2024-12-19T21:58:31.138249Z","shell.execute_reply.started":"2024-12-19T21:58:31.138086Z","shell.execute_reply":"2024-12-19T21:58:31.138114Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub1 = submission1\nsub2 = submission2\nsub3 = submission3\n\nsub1 = sub1.sort_values(by='id').reset_index(drop=True)\nsub2 = sub2.sort_values(by='id').reset_index(drop=True)\nsub3 = sub3.sort_values(by='id').reset_index(drop=True)\n\ncombined = pd.DataFrame({\n    'id': sub1['id'],\n    'sii_1': sub1['sii'],\n    'sii_2': sub2['sii'],\n    'sii_3': sub3['sii']\n})\n\ndef majority_vote(row):\n    return row.mode()[0]\n\ncombined['final_sii'] = combined[['sii_1', 'sii_2', 'sii_3']].apply(majority_vote, axis=1)\n\nsum_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\nsum_submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:58:31.140127Z","iopub.status.idle":"2024-12-19T21:58:31.140437Z","shell.execute_reply.started":"2024-12-19T21:58:31.140292Z","shell.execute_reply":"2024-12-19T21:58:31.140309Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sum_submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T21:58:31.141912Z","iopub.status.idle":"2024-12-19T21:58:31.142251Z","shell.execute_reply.started":"2024-12-19T21:58:31.142087Z","shell.execute_reply":"2024-12-19T21:58:31.142115Z"}},"outputs":[],"execution_count":null}]}