{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"- handle time-series data => make it id-level (using a python package to select features other than max, min...)\n- rmb: only impute the 3000 rows (which I did) and when left-join the 1000 timeseries => dont impute these, leave them as NaN (rationale later)\n- perform AE on EACH of the 3000 rows and then 1000 rows (since each of these two tables has no NaN values before joining)\n- now left join 3000 rows with 1000 rows (ignore NaN in the 1000 rows)\n- if there is time, try using the quadratic weighted kappa loss (https://www.kaggle.com/code/cchangyyy/0-494-notebook?scriptVersionId=210383140&cellId=13)\n- after this, let's do some modelling + hyperpams \n\nwhy can ignore?\n\nModels that cannot handle NaN in X include:\n- Linear Models: LinearRegression, LogisticRegression, Ridge, Lasso.\n- Support Vector Machines (SVMs): SVC, SVR.\n- Neural Networks: Most implementations (e.g., MLPClassifier, MLPRegressor).\n- k-Nearest Neighbors (KNN): KNeighborsClassifier, KNeighborsRegressor.\n- Naive Bayes: GaussianNB, MultinomialNB.\n- Ensemble Models: RandomForest, GradientBoostingClassifier, unless preprocessed.\n\nModels that can handle NaN in X natively:\n- Tree-based Models:\n- XGBoost\n- LightGBM\n- CatBoost","metadata":{}},{"cell_type":"code","source":"# ! pip install tsfresh","metadata":{"trusted":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-12-07T03:20:52.503239Z","iopub.execute_input":"2024-12-07T03:20:52.503694Z","iopub.status.idle":"2024-12-07T03:20:52.527992Z","shell.execute_reply.started":"2024-12-07T03:20:52.503622Z","shell.execute_reply":"2024-12-07T03:20:52.526748Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sn\n\nimport os\n\nfrom tsfresh.feature_extraction import MinimalFCParameters\nfrom tsfresh import extract_features\n# https://tsfresh.readthedocs.io/en/latest/api/tsfresh.feature_extraction.html#tsfresh.feature_extraction.settings.MinimalFCParameters\n\nfrom concurrent.futures import ThreadPoolExecutor\n\nfrom tqdm import tqdm\n\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.datasets import make_classification\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:53:24.211677Z","iopub.execute_input":"2024-12-07T03:53:24.213170Z","iopub.status.idle":"2024-12-07T03:53:24.220253Z","shell.execute_reply.started":"2024-12-07T03:53:24.213126Z","shell.execute_reply":"2024-12-07T03:53:24.218984Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Time-series data","metadata":{}},{"cell_type":"code","source":"train_series_dir = \"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\"\ntest_series_dir = \"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:21:00.430261Z","iopub.execute_input":"2024-12-07T03:21:00.430990Z","iopub.status.idle":"2024-12-07T03:21:00.436580Z","shell.execute_reply.started":"2024-12-07T03:21:00.430950Z","shell.execute_reply":"2024-12-07T03:21:00.435384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:21:00.438329Z","iopub.execute_input":"2024-12-07T03:21:00.439374Z","iopub.status.idle":"2024-12-07T03:21:00.449036Z","shell.execute_reply.started":"2024-12-07T03:21:00.439322Z","shell.execute_reply":"2024-12-07T03:21:00.447817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nprint(\"extract fetaures for train series df\")\ntrain_series = load_time_series(train_series_dir)","metadata":{"trusted":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-12-07T03:21:00.452230Z","iopub.execute_input":"2024-12-07T03:21:00.452687Z","iopub.status.idle":"2024-12-07T03:22:37.637170Z","shell.execute_reply.started":"2024-12-07T03:21:00.452604Z","shell.execute_reply":"2024-12-07T03:22:37.635113Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_series","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:22:37.640543Z","iopub.execute_input":"2024-12-07T03:22:37.640984Z","iopub.status.idle":"2024-12-07T03:22:37.685412Z","shell.execute_reply.started":"2024-12-07T03:22:37.640905Z","shell.execute_reply":"2024-12-07T03:22:37.684094Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"extract features for test series df\")\ntest_series = load_time_series(test_series_dir)\n\ntest_series","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:22:37.687047Z","iopub.execute_input":"2024-12-07T03:22:37.687465Z","iopub.status.idle":"2024-12-07T03:22:38.016431Z","shell.execute_reply.started":"2024-12-07T03:22:37.687419Z","shell.execute_reply":"2024-12-07T03:22:38.014997Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# PCA on train => pick pc => apply same transformation to test/future\n\n# assune linearity\nfrom sklearn.decomposition import PCA\nfrom numpy.testing import assert_array_almost_equal\n\n# Assumptions:\n# data relationships are linear.\n# Mean-Centered Data: features are centered at zero.\n# Large Variance Signals: variance reflects important structure.\n# Independent Components: principal components are uncorrelated.\n\n# why normalize\n# https://stats.stackexchange.com/questions/69157/why-do-we-need-to-normalize-data-before-principal-component-analysis-pca","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:22:38.017840Z","iopub.execute_input":"2024-12-07T03:22:38.018189Z","iopub.status.idle":"2024-12-07T03:22:38.023894Z","shell.execute_reply.started":"2024-12-07T03:22:38.018158Z","shell.execute_reply":"2024-12-07T03:22:38.022330Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# normalize train based on mean normalization (standardization) but \n# we are not sure about the normal distribution => MinMaxScaler\n\ntrain_ids = train_series[\"id\"]\ntrain_series_no_id = train_series.drop([\"id\"], axis=1)\n\n# mean imputting\n# imputer = SimpleImputer(missing_values=np.nan, strategy='mean')\n# imputer.fit(train_series_no_id)\n\n# KNN imputting\nimputer = KNNImputer(n_neighbors=4)\nimputer.fit(train_series_no_id)\n\ntrain_series_no_id_imputed = pd.DataFrame(\n    imputer.transform(train_series_no_id),\n    columns=train_series_no_id.columns,\n    index=train_series_no_id.index\n)\n\n\nmin_vals = train_series_no_id_imputed.min()\nmax_vals = train_series_no_id_imputed.max()\n\n# avoid division by zero by adding a small epsilon or setting to 0 for constant columns\ndenominator = max_vals - min_vals\ndenominator[denominator == 0] = 1 \n\nnormalized_train_series = (train_series_no_id_imputed - min_vals) / denominator","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:53:31.565861Z","iopub.execute_input":"2024-12-07T03:53:31.566323Z","iopub.status.idle":"2024-12-07T03:53:31.590625Z","shell.execute_reply.started":"2024-12-07T03:53:31.566288Z","shell.execute_reply":"2024-12-07T03:53:31.589401Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# https://mikulskibartosz.name/pca-how-to-choose-the-number-of-components#how-to-select-the-number-of-components\n# how to select no. components\n\npca = PCA().fit(normalized_train_series)\n\nplt.rcParams[\"figure.figsize\"] = (12,6)\n\nfig, ax = plt.subplots()\nxi = np.arange(1, len(pca.explained_variance_ratio_) + 1, step=1)  # possible n_components\ny = np.cumsum(pca.explained_variance_ratio_)\n\nplt.ylim(0.0,1.1)\nplt.plot(xi, y, marker='o', linestyle='--')\n\nplt.xlabel('n_components')\nplt.ylabel('Cumulative variance (%)')\nplt.title('The number of components needed to explain variance')\n\nplt.axhline(y=0.95, color='red', linestyle='-')\nplt.text(0.5, 0.85, '95% cut-off threshold', color = 'red', fontsize=14)\n\nax.grid(axis='x')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:53:33.982862Z","iopub.execute_input":"2024-12-07T03:53:33.983304Z","iopub.status.idle":"2024-12-07T03:53:34.332094Z","shell.execute_reply.started":"2024-12-07T03:53:33.983270Z","shell.execute_reply":"2024-12-07T03:53:34.330621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca = PCA(n_components=20)\npca.fit(normalized_train_series)\n\nreduced_train_series_pca = pca.fit_transform(normalized_train_series)\n\nreduced_train_series = pd.DataFrame(\n    reduced_train_series_pca, \n    columns=[f'PC{i+1}' for i in range(20)],  \n    index=normalized_train_series.index     \n)\n\nreduced_train_series[\"id\"] = train_ids\nreduced_train_series","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:53:36.340189Z","iopub.execute_input":"2024-12-07T03:53:36.340639Z","iopub.status.idle":"2024-12-07T03:53:36.650104Z","shell.execute_reply.started":"2024-12-07T03:53:36.340603Z","shell.execute_reply":"2024-12-07T03:53:36.647749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Any new data that is fed to the model in the future when making predictions, \n# such as test datasets and new datasets, must also be projected using the same technique.\n\ntest_ids = test_series[\"id\"]\ntest_series_no_id = test_series.drop([\"id\"], axis=1)\n\ntest_series_no_id_imputed = pd.DataFrame(\n    imputer.transform(test_series_no_id),\n    columns=test_series_no_id.columns,\n    index=test_series_no_id.index\n)\n\nmin_vals = test_series_no_id_imputed.min()\nmax_vals = test_series_no_id_imputed.max()\n\n# avoid division by zero by adding a small epsilon or setting to 0 for constant columns\ndenominator = max_vals - min_vals\ndenominator[denominator == 0] = 1 \n\nnormalized_test_series = (test_series_no_id_imputed - min_vals) / denominator\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:53:39.552578Z","iopub.execute_input":"2024-12-07T03:53:39.553090Z","iopub.status.idle":"2024-12-07T03:53:39.568840Z","shell.execute_reply.started":"2024-12-07T03:53:39.553051Z","shell.execute_reply":"2024-12-07T03:53:39.567307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"reduced_test_series_pc = pca.transform(normalized_test_series)\n\nreduced_test_series = pd.DataFrame(\n    reduced_test_series_pc, \n    columns=[f'PC{i+1}' for i in range(20)],  \n    index=normalized_test_series.index       \n)\n\nreduced_test_series[\"id\"] = test_ids\nreduced_test_series","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:53:42.117888Z","iopub.execute_input":"2024-12-07T03:53:42.118269Z","iopub.status.idle":"2024-12-07T03:53:42.147617Z","shell.execute_reply.started":"2024-12-07T03:53:42.118237Z","shell.execute_reply":"2024-12-07T03:53:42.146101Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Tabular data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\")\ntrain_id = train[\"id\"]\ntest_id = test[\"id\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:53:44.078084Z","iopub.execute_input":"2024-12-07T03:53:44.078581Z","iopub.status.idle":"2024-12-07T03:53:44.134254Z","shell.execute_reply.started":"2024-12-07T03:53:44.078543Z","shell.execute_reply":"2024-12-07T03:53:44.133019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# tabular data processing\n\n# for train only: (deal with PCAIT and SII missing)\n\ndef train_recalculate_sii(row):\n    responses20_cols = [col for col in train.columns if col.startswith(\"PCIAT-PCIAT\") and col != \"PCIAT-PCIAT_Total\"]\n    \n    if pd.isna(row['PCIAT-PCIAT_Total']):\n        return np.nan\n    max_possible = row['PCIAT-PCIAT_Total'] + row[responses20_cols].isna().sum() * 5\n    if row['PCIAT-PCIAT_Total'] <= 30 and max_possible <= 30:\n        return 0\n    elif 31 <= row['PCIAT-PCIAT_Total'] <= 49 and max_possible <= 49:\n        return 1\n    elif 50 <= row['PCIAT-PCIAT_Total'] <= 79 and max_possible <= 79:\n        return 2\n    elif row['PCIAT-PCIAT_Total'] >= 80 and max_possible >= 80:\n        return 3\n    return np.nan\n\ntrain['sii'] = train.apply(train_recalculate_sii, axis=1)\npciat_cols = [col for col in train.columns if col.startswith(\"PCIAT\")]\ntrain = train.drop(columns=pciat_cols)\n\n# deal with missing data in SII scores by median per group of age and sex","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:53:45.933485Z","iopub.execute_input":"2024-12-07T03:53:45.934768Z","iopub.status.idle":"2024-12-07T03:53:47.499288Z","shell.execute_reply.started":"2024-12-07T03:53:45.934724Z","shell.execute_reply":"2024-12-07T03:53:47.497778Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# general data/feature engineering applied to both train and test\n\n# for categorical variables:\n# replace misisng obs in categorical variables with \"Missing\" category because\n# I don't know how to deal with them (for now, I assume that these missing values\n# are not at random and missing is meaningful)\n\n\n# handle continuous variable missing data\n# FOR AND FROM TRAIN\ngroup_bases = [\"Basic_Demos-Age\", \"Basic_Demos-Sex\"] # no missing data\ncons = [col for col in train.columns if train[col].dtype != \"object\" and col not in group_bases]\n\nrange_age = range(min(train[group_bases[0]]), max(train[group_bases[0]]) + 1)\n\ngroups_median_cons = pd.DataFrame([(a, s) for a in range_age for s in (0, 1)], columns=group_bases)\n\nfor con in cons:\n    right = (train[[*group_bases, con]].groupby(group_bases).median().reset_index())\n    groups_median_cons = pd.merge(groups_median_cons, right, on=group_bases, how=\"left\")\n\nover_10_missing_medians = []\nfor idx, s in enumerate(groups_median_cons.isna().sum()):\n    if idx > 1 and s > 10:\n        over_10_missing_medians.append(cons[idx-2])\ngroups_median_cons = groups_median_cons.drop(columns=over_10_missing_medians)\n\n# the rest, for now, I just fill the NaN values with the column's median, not very rigorous\ngroups_median_cons = groups_median_cons.apply(lambda x: x.fillna(x.median()), axis=0)\ngroups_median_cons[\"sii\"] = groups_median_cons[\"sii\"].apply(lambda x: round(x))\n\n# convert into dict\ngroup_median_dict = groups_median_cons.set_index(group_bases).to_dict(orient=\"index\")\n\n\ndef handle_missing(df):\n\n    df = df.drop(columns=[\"id\"])\n    \n    group_bases = [\"Basic_Demos-Age\", \"Basic_Demos-Sex\"]\n    cats = [col for col in df.columns if df[col].dtype == \"object\"]\n    cons = [col for col in df.columns if df[col].dtype != \"object\" and col not in group_bases]\n\n    # handle missing dat\n    for cat in cats:\n        df[cat] = df[cat].fillna(\"Missing\")\n    # think about categorical later, for now, remove them :(\n    df = df.drop(columns=cats)\n\n    # handle continuous\n    df = df.drop(columns=over_10_missing_medians)\n    new_cons = [col for col in cons if col not in over_10_missing_medians]\n    # fill na with groups_median_cons\n    for con in new_cons:\n        df[con] = df.apply(\n            lambda row: group_median_dict[(row[group_bases[0]], row[group_bases[1]])][con]\n            if pd.isna(row[con]) else row[con],\n            axis=1\n        )\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:53:47.501590Z","iopub.execute_input":"2024-12-07T03:53:47.501993Z","iopub.status.idle":"2024-12-07T03:53:47.763991Z","shell.execute_reply.started":"2024-12-07T03:53:47.501958Z","shell.execute_reply":"2024-12-07T03:53:47.762562Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"groups_median_cons.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:53:50.173856Z","iopub.execute_input":"2024-12-07T03:53:50.174274Z","iopub.status.idle":"2024-12-07T03:53:50.203970Z","shell.execute_reply.started":"2024-12-07T03:53:50.174238Z","shell.execute_reply":"2024-12-07T03:53:50.202573Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transformed_train = handle_missing(train)\nX = transformed_train.drop(columns=\"sii\")\ny = transformed_train[\"sii\"]\n\n# scaler = MinMaxScaler().fit(X)\nscaler = StandardScaler().fit(X)\n\nX_scaled = scaler.transform(X)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:53:52.163710Z","iopub.execute_input":"2024-12-07T03:53:52.164309Z","iopub.status.idle":"2024-12-07T03:53:53.815825Z","shell.execute_reply.started":"2024-12-07T03:53:52.164268Z","shell.execute_reply":"2024-12-07T03:53:53.814591Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class Autoencoder(nn.Module):\n#     # https://stackoverflow.com/questions/74141355/dimensionality-reduction-autoencoder-pytorch#74141903\n\n#     def __init__(self, input_dim, hidden_dim):\n#         super(Autoencoder, self).__init__()\n        \n#         self.encoder = nn.Sequential(\n#             nn.Linear(input_dim, hidden_dim),\n#             nn.Sigmoid()\n#         )\n#         self.decoder = nn.Sequential(\n#             nn.Linear(hidden_dim, input_dim),\n#             nn.Sigmoid()\n#         )\n\n#     def forward(self, x):\n#         encoded = self.encoder(x)\n#         decoded = self.decoder(encoded)\n#         return encoded, decoded\n\n\nclass Autoencoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(Autoencoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n        \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n\n\ninput_dim = X.shape[1]\nhidden_dim = 100   # tuning\nautoencoder = Autoencoder(input_dim, hidden_dim)\n\n# X_train_tensor = torch.FloatTensor(X_train)\n# X_valid_tensor = torch.FloatTensor(X_valid)\n\nX_tensor = torch.FloatTensor(X_scaled)\n\ntrain_dataset = TensorDataset(X_tensor)\ntrain_loader = DataLoader(train_dataset, \n                          batch_size=32,  # tuning\n                            ) \n\n# training epochs\nepochs = 100 # tuning\ncriterion = nn.MSELoss()\noptimizer = optim.Adam(autoencoder.parameters(), lr=0.001) # tuning\n\nfor epoch in range(epochs):\n    for batch in train_loader:\n        inputs = batch[0]\n        optimizer.zero_grad()\n        reconstructed = autoencoder(inputs)\n        # reconstructed = autoencoder(inputs)\n        loss = criterion(reconstructed, inputs)\n        loss.backward()\n        optimizer.step()\n    if epoch % 10 == 0:\n        print(f\"epoch [{epoch + 1}/{epochs}], loss: {loss.item():.4f}\")\n\nwith torch.no_grad():\n    # X_train_encoded = autoencoder.encoder(X_train_tensor).numpy()\n    # X_valid_encoded = autoencoder.encoder(X_valid_tensor).numpy()\n\n    X_encoded = autoencoder.encoder(X_tensor).numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:53:56.260908Z","iopub.execute_input":"2024-12-07T03:53:56.261319Z","iopub.status.idle":"2024-12-07T03:54:47.361219Z","shell.execute_reply.started":"2024-12-07T03:53:56.261284Z","shell.execute_reply":"2024-12-07T03:54:47.359639Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_encoded_df = pd.DataFrame(\n    X_encoded, \n    columns=[f'F{i+1}' for i in range(hidden_dim)],  \n)\n\nX_encoded_df[\"id\"] = train_id\nX_encoded_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:54:53.340981Z","iopub.execute_input":"2024-12-07T03:54:53.341412Z","iopub.status.idle":"2024-12-07T03:54:53.374554Z","shell.execute_reply.started":"2024-12-07T03:54:53.341375Z","shell.execute_reply":"2024-12-07T03:54:53.373302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# now, merge X (scaled train) and reduced_train_series\nreduced_train_series","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:54:56.023762Z","iopub.execute_input":"2024-12-07T03:54:56.024165Z","iopub.status.idle":"2024-12-07T03:54:56.052691Z","shell.execute_reply.started":"2024-12-07T03:54:56.024131Z","shell.execute_reply":"2024-12-07T03:54:56.051196Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_final = pd.merge(X_encoded_df, reduced_train_series, how=\"left\", on=\"id\")\nX_final = X_final.drop('id', axis=1)\nX_final","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:54:57.603038Z","iopub.execute_input":"2024-12-07T03:54:57.604045Z","iopub.status.idle":"2024-12-07T03:54:57.645276Z","shell.execute_reply.started":"2024-12-07T03:54:57.603982Z","shell.execute_reply":"2024-12-07T03:54:57.643762Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train 1:\n\nfrom lightgbm import LGBMClassifier, LGBMRegressor\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error, auc, recall_score, precision_score, f1_score, roc_auc_score, confusion_matrix, classification_report\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\nfrom sklearn.model_selection import KFold, StratifiedKFold, GridSearchCV\n\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\n\n# X_train, X_valid, y_train, y_valid = train_test_split(X_final, y, test_size=0.3, random_state=123)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:55:00.489825Z","iopub.execute_input":"2024-12-07T03:55:00.490261Z","iopub.status.idle":"2024-12-07T03:55:00.497788Z","shell.execute_reply.started":"2024-12-07T03:55:00.490226Z","shell.execute_reply":"2024-12-07T03:55:00.496089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# after a quick grid search on cv=3, check the following hyperparams\nmodel_lightgbm = LGBMRegressor(\n    learning_rate = 0.046,\n    ax_depth = 12,\n    num_leaves = 478,\n    min_data_in_leaf = 13,\n    feature_fraction = 0.893,\n    bagging_fraction = 0.784,\n    bagging_freq = 4,\n    lambda_l1 = 10,  \n    lambda_l2 = 0.01,\n    verbosity=-1\n)\n\nX_train, X_valid, y_train, y_valid = train_test_split(X_final, y, test_size=0.3, random_state=123)\nmodel_lightgbm.fit(X_train, y_train)\ny_pred = model_lightgbm.predict(X_valid)\n\n\ndef round_predictions(predictions, thresholds):\n    rounded_preds = np.zeros_like(predictions)\n    for i, pred in enumerate(predictions):\n        if pred < thresholds[0]:\n            rounded_preds[i] = 0\n        elif pred < thresholds[1]:\n            rounded_preds[i] = 1\n        elif pred < thresholds[2]:\n            rounded_preds[i] = 2\n        else:\n            rounded_preds[i] = 3  # Adjust based on your target's class range\n    return rounded_preds\nx0 = [0.5, 1.5, 2.5]\n\ny_pred_rounded = round_predictions(y_pred, x0)\n\n\nrmse = np.sqrt(mean_squared_error(y_valid, y_pred))\nmae = mean_absolute_error(y_valid, y_pred)\nr2 = r2_score(y_valid, y_pred)\n\nprint(\"RMSE:\", rmse)\nprint(\"MAE:\", mae)\nprint(\"R^2:\", r2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:55:02.346397Z","iopub.execute_input":"2024-12-07T03:55:02.346898Z","iopub.status.idle":"2024-12-07T03:55:03.108610Z","shell.execute_reply.started":"2024-12-07T03:55:02.346857Z","shell.execute_reply":"2024-12-07T03:55:03.107429Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"param_grid = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01,  # Increased from 2.68e-06\n    'device': 'cpu',\n    'verbosity': -1\n}\n# https://www.kaggle.com/code/cchangyyy/0-494-notebook","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:41:42.225901Z","iopub.execute_input":"2024-12-07T03:41:42.226286Z","iopub.status.idle":"2024-12-07T03:41:42.232897Z","shell.execute_reply.started":"2024-12-07T03:41:42.226254Z","shell.execute_reply":"2024-12-07T03:41:42.231519Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# kf = StratifiedKFold(n_splits=3, shuffle=True, random_state=123)\n# rmse_scores, mae_scores, r2_scores = [], [], []\n\n# thresholds = [0.5, 1.5, 2.5]\n\n# def round_predictions(predictions, thresholds):\n#     rounded_preds = np.zeros_like(predictions)\n#     for i, pred in enumerate(predictions):\n#         if pred < thresholds[0]:\n#             rounded_preds[i] = 0\n#         elif pred < thresholds[1]:\n#             rounded_preds[i] = 1\n#         elif pred < thresholds[2]:\n#             rounded_preds[i] = 2\n#         else:\n#             rounded_preds[i] = 3\n#     return rounded_preds\n\n# for train_index, valid_index in kf.split(X_final, y):\n#     X_train, X_valid = X_final.iloc[train_index], X_final.iloc[valid_index] # numpy array\n#     y_train, y_valid = y[train_index], y[valid_index]\n\n#     model_lightgbm = LGBMRegressor(**param_grid)\n#     model_lightgbm.fit(X_train, y_train)\n#     y_pred = model_lightgbm.predict(X_valid)\n#     y_pred_rounded = round_predictions(y_pred, thresholds)\n    \n#     rmse = np.sqrt(mean_squared_error(y_valid, y_pred))\n#     mae = mean_absolute_error(y_valid, y_pred)\n#     r2 = r2_score(y_valid, y_pred)\n\n#     rmse_scores.append(rmse)\n#     mae_scores.append(mae)\n#     r2_scores.append(r2)\n\n# avg_rmse = np.mean(rmse_scores)\n# avg_mae = np.mean(mae_scores)\n# avg_r2 = np.mean(r2_scores)\n\n# avg_rmse, avg_mae, avg_r2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:23:18.319325Z","iopub.execute_input":"2024-12-07T03:23:18.319722Z","iopub.status.idle":"2024-12-07T03:23:28.512979Z","shell.execute_reply.started":"2024-12-07T03:23:18.319647Z","shell.execute_reply":"2024-12-07T03:23:28.511649Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"my_param_grid = {\n    'learning_rate': [0.1, 0.2, 0.3],\n    'max_depth': [8, 12, 16],\n    'num_leaves': [200, 300, 450],\n    'min_data_in_leaf': [10, 13, 20],\n    'feature_fraction': [0.7, 0.8, 0.9],\n    'bagging_fraction': [0.7, 0.8, 0.9],\n    'lambda_l1': [1, 5, 10],\n    'lambda_l2': [0.01, 0.1, 1]\n}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:23:28.514712Z","iopub.execute_input":"2024-12-07T03:23:28.515071Z","iopub.status.idle":"2024-12-07T03:23:28.522121Z","shell.execute_reply.started":"2024-12-07T03:23:28.515037Z","shell.execute_reply":"2024-12-07T03:23:28.520750Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def rmse_scorer(estimator, X, y):\n#     preds = estimator.predict(X)\n#     rounded_preds = np.digitize(preds, thresholds)\n#     return np.sqrt(mean_squared_error(y, rounded_preds))\n\n# # Cross-validation setup\n# kf = StratifiedKFold(n_splits=3, shuffle=True, random_state=123)\n\n# # Initialize GridSearchCV\n# grid_search = GridSearchCV(\n#     estimator=LGBMRegressor(verbosity=-1),\n#     param_grid=my_param_grid,\n#     scoring=rmse_scorer,\n#     cv=kf,\n#     verbose=1,\n#     n_jobs=-1\n# )\n\n# # Fit the grid search\n# grid_search.fit(X_final, y)\n\n# # Output the best parameters and RMSE\n# best_params = grid_search.best_params_\n# best_rmse = grid_search.best_score_\n\n# print(\"Best Parameters:\", best_params)\n# print(\"Best RMSE:\", best_rmse)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:23:28.523803Z","iopub.execute_input":"2024-12-07T03:23:28.524145Z","iopub.status.idle":"2024-12-07T03:24:25.768370Z","shell.execute_reply.started":"2024-12-07T03:23:28.524113Z","shell.execute_reply":"2024-12-07T03:24:25.766040Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# best_params, best_rmse","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:24:25.769578Z","iopub.status.idle":"2024-12-07T03:24:25.770033Z","shell.execute_reply.started":"2024-12-07T03:24:25.769838Z","shell.execute_reply":"2024-12-07T03:24:25.769862Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Test","metadata":{}},{"cell_type":"code","source":"# tabular data for test\n\n# scale\nX_test = handle_missing(test)\nX_test_scaled = scaler.transform(X_test)\nX_test_scaled\n\n# autoencoder\nX_test_tensor = torch.FloatTensor(X_test_scaled)\nwith torch.no_grad():\n    X_test_encoded = autoencoder.encoder(X_test_tensor).numpy()\n\n# prep for merge with test_series\nX_test_encoded_df = pd.DataFrame(\n    X_test_encoded, \n    columns=[f'F{i+1}' for i in range(hidden_dim)],  \n)\nX_test_encoded_df[\"id\"] = test_id\n\nX_test_final = pd.merge(X_test_encoded_df, reduced_test_series, how=\"left\", on=\"id\")\nX_test_final = X_test_final.drop('id', axis=1)\n\nX_test_final","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:55:19.246825Z","iopub.execute_input":"2024-12-07T03:55:19.247260Z","iopub.status.idle":"2024-12-07T03:55:19.337069Z","shell.execute_reply.started":"2024-12-07T03:55:19.247224Z","shell.execute_reply":"2024-12-07T03:55:19.335646Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_test_pred = model_lightgbm.predict(X_test_final)\ny_test_pred_rounded = round_predictions(y_test_pred, x0)\ny_test_pred_rounded","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:55:21.516059Z","iopub.execute_input":"2024-12-07T03:55:21.517134Z","iopub.status.idle":"2024-12-07T03:55:21.528339Z","shell.execute_reply.started":"2024-12-07T03:55:21.517092Z","shell.execute_reply":"2024-12-07T03:55:21.527087Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub1 = pd.DataFrame({\"id\": test_id, \"sii\": y_test_pred_rounded})\nsub1.to_csv(\"submission.csv\", index=False)\npd.read_csv(\"/kaggle/working/submission.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T03:41:50.757557Z","iopub.execute_input":"2024-12-07T03:41:50.758026Z","iopub.status.idle":"2024-12-07T03:41:50.780756Z","shell.execute_reply.started":"2024-12-07T03:41:50.757991Z","shell.execute_reply":"2024-12-07T03:41:50.779583Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}