{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.datasets import make_classification\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.preprocessing import MinMaxScaler\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:44:55.801875Z","iopub.execute_input":"2024-11-29T21:44:55.802289Z","iopub.status.idle":"2024-11-29T21:44:55.809186Z","shell.execute_reply.started":"2024-11-29T21:44:55.802253Z","shell.execute_reply":"2024-11-29T21:44:55.807918Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Deal with missing data + Try AE on continous data and KNN (k=5)","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\")\ntest_id = test[\"id\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:44:55.811301Z","iopub.execute_input":"2024-11-29T21:44:55.812218Z","iopub.status.idle":"2024-11-29T21:44:55.873910Z","shell.execute_reply.started":"2024-11-29T21:44:55.812179Z","shell.execute_reply":"2024-11-29T21:44:55.872644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# tabular data processing\n\n# for train only: (deal with PCAIT and SII missing)\n\ndef train_recalculate_sii(row):\n    responses20_cols = [col for col in train.columns if col.startswith(\"PCIAT-PCIAT\") and col != \"PCIAT-PCIAT_Total\"]\n    \n    if pd.isna(row['PCIAT-PCIAT_Total']):\n        return np.nan\n    max_possible = row['PCIAT-PCIAT_Total'] + row[responses20_cols].isna().sum() * 5\n    if row['PCIAT-PCIAT_Total'] <= 30 and max_possible <= 30:\n        return 0\n    elif 31 <= row['PCIAT-PCIAT_Total'] <= 49 and max_possible <= 49:\n        return 1\n    elif 50 <= row['PCIAT-PCIAT_Total'] <= 79 and max_possible <= 79:\n        return 2\n    elif row['PCIAT-PCIAT_Total'] >= 80 and max_possible >= 80:\n        return 3\n    return np.nan\n\ntrain['sii'] = train.apply(train_recalculate_sii, axis=1)\npciat_cols = [col for col in train.columns if col.startswith(\"PCIAT\")]\ntrain = train.drop(columns=pciat_cols)\n\n# deal with missing data in SII scores by median per group of age and sex","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:44:55.876222Z","iopub.execute_input":"2024-11-29T21:44:55.877070Z","iopub.status.idle":"2024-11-29T21:44:57.414807Z","shell.execute_reply.started":"2024-11-29T21:44:55.876993Z","shell.execute_reply":"2024-11-29T21:44:57.413732Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# general data/feature engineering applied to both train and test\n\n# for categorical variables:\n# replace misisng obs in categorical variables with \"Missing\" category because\n# I don't know how to deal with them (for now, I assume that these missing values\n# are not at random and missing is meaningful)\n\n\n# handle continuous variable missing data\n# FOR AND FROM TRAIN\ngroup_bases = [\"Basic_Demos-Age\", \"Basic_Demos-Sex\"] # no missing data\ncons = [col for col in train.columns if train[col].dtype != \"object\" and col not in group_bases]\n\nrange_age = range(min(train[group_bases[0]]), max(train[group_bases[0]]) + 1)\n\ngroups_median_cons = pd.DataFrame([(a, s) for a in range_age for s in (0, 1)], columns=group_bases)\n\nfor con in cons:\n    right = (train[[*group_bases, con]].groupby(group_bases).median().reset_index())\n    groups_median_cons = pd.merge(groups_median_cons, right, on=group_bases, how=\"left\")\n\nover_10_missing_medians = []\nfor idx, s in enumerate(groups_median_cons.isna().sum()):\n    if idx > 1 and s > 10:\n        over_10_missing_medians.append(cons[idx-2])\ngroups_median_cons = groups_median_cons.drop(columns=over_10_missing_medians)\n\n# the rest, for now, I just fill the NaN values with the column's median, not very rigorous\ngroups_median_cons = groups_median_cons.apply(lambda x: x.fillna(x.median()), axis=0)\ngroups_median_cons[\"sii\"] = groups_median_cons[\"sii\"].apply(lambda x: round(x))\n\n# convert into dict\ngroup_median_dict = groups_median_cons.set_index(group_bases).to_dict(orient=\"index\")\n\n\ndef handle_missing(df):\n\n    df = df.drop(columns=[\"id\"])\n    \n    group_bases = [\"Basic_Demos-Age\", \"Basic_Demos-Sex\"]\n    cats = [col for col in df.columns if df[col].dtype == \"object\"]\n    cons = [col for col in df.columns if df[col].dtype != \"object\" and col not in group_bases]\n\n    # handle missing dat\n    for cat in cats:\n        df[cat] = df[cat].fillna(\"Missing\")\n    # think about categorical later, for now, remove them :(\n    df = df.drop(columns=cats)\n\n    # handle continuous\n    df = df.drop(columns=over_10_missing_medians)\n    new_cons = [col for col in cons if col not in over_10_missing_medians]\n    # fill na with groups_median_cons\n    for con in new_cons:\n        df[con] = df.apply(\n            lambda row: group_median_dict[(row[group_bases[0]], row[group_bases[1]])][con]\n            if pd.isna(row[con]) else row[con],\n            axis=1\n        )\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:44:57.416204Z","iopub.execute_input":"2024-11-29T21:44:57.416716Z","iopub.status.idle":"2024-11-29T21:44:57.684785Z","shell.execute_reply.started":"2024-11-29T21:44:57.416663Z","shell.execute_reply":"2024-11-29T21:44:57.683485Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"groups_median_cons.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:44:57.687186Z","iopub.execute_input":"2024-11-29T21:44:57.687572Z","iopub.status.idle":"2024-11-29T21:44:57.714007Z","shell.execute_reply.started":"2024-11-29T21:44:57.687535Z","shell.execute_reply":"2024-11-29T21:44:57.712795Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"handle_missing(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:44:57.715641Z","iopub.execute_input":"2024-11-29T21:44:57.716162Z","iopub.status.idle":"2024-11-29T21:44:59.413156Z","shell.execute_reply.started":"2024-11-29T21:44:57.716118Z","shell.execute_reply":"2024-11-29T21:44:59.411945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transformed_train = handle_missing(train)\nX = transformed_train.drop(columns=\"sii\")\ny = transformed_train[\"sii\"]\n\nX_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=123)\n\nscaler = MinMaxScaler().fit(X_train)\nX_train = scaler.transform(X_train)\nX_valid = scaler.transform(X_valid)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:44:59.414612Z","iopub.execute_input":"2024-11-29T21:44:59.414956Z","iopub.status.idle":"2024-11-29T21:45:01.071971Z","shell.execute_reply.started":"2024-11-29T21:44:59.414922Z","shell.execute_reply":"2024-11-29T21:45:01.070768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Autoencoder(nn.Module):\n    # https://stackoverflow.com/questions/74141355/dimensionality-reduction-autoencoder-pytorch#74141903\n\n    def __init__(self, input_dim, hidden_dim):\n        super(Autoencoder, self).__init__()\n        \n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, hidden_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(hidden_dim, input_dim),\n            nn.ReLU()\n        )\n\n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return encoded, decoded\n\n\n\ninput_dim = X_train.shape[1]\nhidden_dim = 8 # tuning\nautoencoder = Autoencoder(input_dim, hidden_dim)\n\nX_train_tensor = torch.FloatTensor(X_train)\nX_valid_tensor = torch.FloatTensor(X_valid)\n\n\ntrain_dataset = TensorDataset(X_train_tensor)\ntrain_loader = DataLoader(train_dataset, batch_size=16,  # tuning\n                          shuffle=True) \n\n\n# training epochs\nepochs = 100 # tuning\ncriterion = nn.MSELoss()\noptimizer = optim.Adam(autoencoder.parameters(), lr=0.001) # tuning\n\nfor epoch in range(epochs):\n    for batch in train_loader:\n        inputs = batch[0]\n        optimizer.zero_grad()\n        _, reconstructed = autoencoder(inputs)\n        loss = criterion(reconstructed, inputs)\n        loss.backward()\n        optimizer.step()\n    if epoch % 10 == 0:\n        print(f\"epoch [{epoch + 1}/{epochs}], loss: {loss.item():.4f}\")\n\nwith torch.no_grad():\n    X_train_encoded = autoencoder.encoder(X_train_tensor).numpy()\n    X_valid_encoded = autoencoder.encoder(X_valid_tensor).numpy()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:45:01.073452Z","iopub.execute_input":"2024-11-29T21:45:01.073899Z","iopub.status.idle":"2024-11-29T21:45:21.975656Z","shell.execute_reply.started":"2024-11-29T21:45:01.073859Z","shell.execute_reply":"2024-11-29T21:45:21.974341Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # KNN\n\n# knn = KNeighborsClassifier(n_neighbors=6)\n# knn.fit(X_train_encoded, y_train)\n\n# y_pred = knn.predict(X_valid_encoded)\n# accuracy = accuracy_score(y_valid, y_pred)\n# print(accuracy)\n\n# X_test = handle_missing(test)\n\n# X_test = scaler.transform(X_test)\n# X_test_tensor = torch.FloatTensor(X_test)\n\n# with torch.no_grad():\n#     X_test_encoded = autoencoder.encoder(X_test_tensor).numpy()\n\n# print(X_test_encoded.shape)\n\n# y_test_pred = knn.predict(X_test_encoded)\n# print(y_test_pred)\n\n# sub1 = pd.DataFrame({\"id\": test_id, \"sii\": y_test_pred})\n# sub1.to_csv(\"submission.csv\", index=False)\n# pd.read_csv(\"/kaggle/working/submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:45:21.977645Z","iopub.execute_input":"2024-11-29T21:45:21.978141Z","iopub.status.idle":"2024-11-29T21:45:21.984164Z","shell.execute_reply.started":"2024-11-29T21:45:21.978087Z","shell.execute_reply":"2024-11-29T21:45:21.982825Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from sklearn.tree import DecisionTreeClassifier\n\n# dt_clf = DecisionTreeClassifier()\n# dt_clf.fit(X_train_encoded, y_train)\n# y_pred = dt_clf.predict(X_valid_encoded)\n\n# X_test = handle_missing(test)\n# X_test = scaler.transform(X_test)\n# X_test_tensor = torch.FloatTensor(X_test)\n\n# with torch.no_grad():\n#     X_test_encoded = autoencoder.encoder(X_test_tensor).numpy()\n\n# print(X_test_encoded.shape)\n\n# y_test_pred = dt_clf.predict(X_test_encoded)\n\n# sub1 = pd.DataFrame({\"id\": test_id, \"sii\": y_test_pred})\n# sub1.to_csv(\"submission.csv\", index=False)\n# pd.read_csv(\"/kaggle/working/submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:45:21.985980Z","iopub.execute_input":"2024-11-29T21:45:21.986817Z","iopub.status.idle":"2024-11-29T21:45:22.006618Z","shell.execute_reply.started":"2024-11-29T21:45:21.986750Z","shell.execute_reply":"2024-11-29T21:45:22.005298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\nclf = LogisticRegression(multi_class=\"ovr\")\nclf.fit(X_train_encoded, y_train)\ny_pred = clf.predict(X_valid_encoded)\n\n\nX_test = handle_missing(test)\nX_test = scaler.transform(X_test)\nX_test_tensor = torch.FloatTensor(X_test)\n\nwith torch.no_grad():\n    X_test_encoded = autoencoder.encoder(X_test_tensor).numpy()\n\nprint(X_test_encoded.shape)\n\ny_test_pred = clf.predict(X_test_encoded)\n\nsub1 = pd.DataFrame({\"id\": test_id, \"sii\": y_test_pred})\nsub1.to_csv(\"submission.csv\", index=False)\npd.read_csv(\"/kaggle/working/submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:45:22.010003Z","iopub.execute_input":"2024-11-29T21:45:22.010542Z","iopub.status.idle":"2024-11-29T21:45:22.156339Z","shell.execute_reply.started":"2024-11-29T21:45:22.010469Z","shell.execute_reply":"2024-11-29T21:45:22.154192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}