{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#import and general settings\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n# settings so that pandas shows all columns and rows, because per default, it \n# shows only some if the dataset is large\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:31:26.447315Z","iopub.execute_input":"2025-01-11T14:31:26.447680Z","iopub.status.idle":"2025-01-11T14:31:27.830545Z","shell.execute_reply.started":"2025-01-11T14:31:26.447634Z","shell.execute_reply":"2025-01-11T14:31:27.829563Z"}},"outputs":[],"execution_count":null},{"cell_type":"raw","source":"","metadata":{"execution":{"iopub.status.busy":"2025-01-10T21:30:59.883919Z","iopub.execute_input":"2025-01-10T21:30:59.884416Z","iopub.status.idle":"2025-01-10T21:30:59.963102Z","shell.execute_reply.started":"2025-01-10T21:30:59.884385Z","shell.execute_reply":"2025-01-10T21:30:59.961809Z"}}},{"cell_type":"code","source":"df = pd.read_csv('../input/child-mind-institute-problematic-internet-use/train.csv')\ntest_data = pd.read_csv('../input/child-mind-institute-problematic-internet-use/test.csv')\n\n# drop the PCAT-columns from the dataset, because the target value (sii) is calculated\n# with them and in the test dataset, we won't have them to predict sii\n\ncolumns_to_drop = [\n    'PCIAT-Season',\n    'PCIAT-PCIAT_01', 'PCIAT-PCIAT_02', 'PCIAT-PCIAT_03', \n    'PCIAT-PCIAT_04', 'PCIAT-PCIAT_05', 'PCIAT-PCIAT_06', \n    'PCIAT-PCIAT_07', 'PCIAT-PCIAT_08', 'PCIAT-PCIAT_09', \n    'PCIAT-PCIAT_10', 'PCIAT-PCIAT_11', 'PCIAT-PCIAT_12',\n    'PCIAT-PCIAT_13', 'PCIAT-PCIAT_14', 'PCIAT-PCIAT_15', \n    'PCIAT-PCIAT_16', 'PCIAT-PCIAT_17', 'PCIAT-PCIAT_18', \n    'PCIAT-PCIAT_19', 'PCIAT-PCIAT_20', 'PCIAT-PCIAT_Total'\n]\ndf = df.drop(columns=columns_to_drop)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:31:27.831824Z","iopub.execute_input":"2025-01-11T14:31:27.832400Z","iopub.status.idle":"2025-01-11T14:31:27.935354Z","shell.execute_reply.started":"2025-01-11T14:31:27.832360Z","shell.execute_reply":"2025-01-11T14:31:27.934134Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# merge the columns PAQ_A-PAQ_A_Total and PAQ_C-PAQ_C_Total into one column called PAQ_Total\ndef calculate_paq_total(row):\n    a = row['PAQ_A-PAQ_A_Total']\n    c = row['PAQ_C-PAQ_C_Total']\n    \n    if pd.notnull(a) and pd.notnull(c): \n        return (a + c) / 2 # if both values are there, calculate average\n    elif pd.notnull(a):\n        return a\n    elif pd.notnull(c):\n        return c\n    else: \n        return None\n\n# create a new column and fill it with help of the created function\ndf['PAQ_Total'] = df.apply(calculate_paq_total, axis=1)\n# drop the old two columns\ndf = df.drop(columns=['PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:31:27.936477Z","iopub.execute_input":"2025-01-11T14:31:27.936790Z","iopub.status.idle":"2025-01-11T14:31:28.005252Z","shell.execute_reply.started":"2025-01-11T14:31:27.936763Z","shell.execute_reply":"2025-01-11T14:31:28.004111Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# merge the columns Fitness_Endurance-Time_Mins and Fitness_Endurance-Time_Sec, because\n# seconds alone have no meaning\n\n# create new column: Fitness_Endurance-Time\ndf['Fitness_Endurance-Time'] = df['Fitness_Endurance-Time_Mins'] + (df['Fitness_Endurance-Time_Sec'] / 60)\n\n# drop old columns\ndf = df.drop(columns=['Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec', 'BIA-Season', 'Fitness_Endurance-Season', 'PAQ_A-Season'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:31:28.007408Z","iopub.execute_input":"2025-01-11T14:31:28.007742Z","iopub.status.idle":"2025-01-11T14:31:28.015546Z","shell.execute_reply.started":"2025-01-11T14:31:28.007714Z","shell.execute_reply":"2025-01-11T14:31:28.014421Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Different types of data\n* numerical discrete data: **numerical_discrete_columns**\n* numerical continuous data: **numerical_continuous_columns**\n* categorical data (but expressed in numbers): **categorical_columns_expressedInNumbers**\n* categorical data (expressed in string): **categorical_columns_expressedInText**\n* ordinal data","metadata":{}},{"cell_type":"code","source":"categorical_columns_expressedInNumbers = ['Basic_Demos-Sex', 'PreInt_EduHx-computerinternet_hoursday', 'FGC-FGC_TL_Zone', 'FGC-FGC_CU_Zone', \n                               'FGC-FGC_PU_Zone', 'FGC-FGC_SRR_Zone', 'FGC-FGC_SRL_Zone', 'BIA-BIA_Activity_Level_num', \n                              'BIA-BIA_Frame_num', 'FGC-FGC_GSD_Zone','FGC-FGC_GSND_Zone']\n\n# Konvertiere die Spalten in den Typ 'category'\nfor col in categorical_columns_expressedInNumbers:\n    df[col] = df[col].astype('category')\n\ncategorical_columns_expressedInText = df.select_dtypes(include=\"object\").columns\n\nnumerical_discrete_columns = ['Basic_Demos-Age', 'CGAS-CGAS_Score', 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                     'Fitness_Endurance-Max_Stage', 'FGC-FGC_CU', 'FGC-FGC_PU',\n                                'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T']\nnumerical_continuous_columns = ['Physical-BMI', 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                       'FGC-FGC_GSND', 'FGC-FGC_GSD', 'FGC-FGC_SRL', 'FGC-FGC_SRR', 'FGC-FGC_TL',\n                       'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW',\n                       'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_ICW', 'BIA-BIA_LDM',\n                       'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW', 'Fitness_Endurance-Time',\n                       'PAQ_Total']\n\ncategorical_columns_expressedInText = list(categorical_columns_expressedInText)\ncategorical_columns_expressedInNumbers = list(categorical_columns_expressedInNumbers)\n\nif \"id\" in categorical_columns_expressedInText:\n    categorical_columns_expressedInText.remove(\"id\")\n\n# Combine categorical columns\nall_categorical_columns = categorical_columns_expressedInNumbers + categorical_columns_expressedInText\n\nnumerical_discrete_columns = list(numerical_discrete_columns)\nnumerical_continuous_columns = list(numerical_continuous_columns)\n\nall_numerical_columns = numerical_discrete_columns + numerical_continuous_columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:31:28.017283Z","iopub.execute_input":"2025-01-11T14:31:28.017618Z","iopub.status.idle":"2025-01-11T14:31:28.056804Z","shell.execute_reply.started":"2025-01-11T14:31:28.017586Z","shell.execute_reply":"2025-01-11T14:31:28.055651Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Negative value treatment","metadata":{}},{"cell_type":"code","source":"def whisker(col):\n    q1,q3=np.percentile(col, [25, 75])\n    iqr=q3-q1\n    lw=q1-1.5*iqr\n    uw=q3+1.5*iqr\n    return lw, uw\n\n\ndef replace_negatives_with_median(df, columns):\n    for col in columns:\n        median_value = df[col].median() \n        df[col] = np.where(df[col] <= 0, median_value, df[col])\n    return df\n\n\n# def replace_negatives_with_lower_whisker(df, columns):\n#     for col in columns:\n#         lw = whisker(df[col])[0] \n#         df[col] = np.where(df[col] <= 0, lw, df[col])\n#     return df\n\n\ncolumns_with_negative_values = ['Physical-BMI', 'Physical-Weight', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                               'BIA-BIA_BMR', 'BIA-BIA_FMI', 'BIA-BIA_Fat']\ndf = replace_negatives_with_median(df, columns_with_negative_values)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:31:28.057845Z","iopub.execute_input":"2025-01-11T14:31:28.058156Z","iopub.status.idle":"2025-01-11T14:31:28.078642Z","shell.execute_reply.started":"2025-01-11T14:31:28.058130Z","shell.execute_reply":"2025-01-11T14:31:28.077553Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Drop rows that make no sense","metadata":{}},{"cell_type":"code","source":"def dropRows(df):\n    df = df.drop(df[df['BIA-BIA_FFM'] > 6000].index)\n    df = df.drop(df[df['BIA-BIA_BMC'] > 100].index)\n    #reset index if you want to remain it consecutive\n    df = df.reset_index(drop=True)\n    return df\n\ndf = dropRows(df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:31:28.079872Z","iopub.execute_input":"2025-01-11T14:31:28.080236Z","iopub.status.idle":"2025-01-11T14:31:28.100010Z","shell.execute_reply.started":"2025-01-11T14:31:28.080191Z","shell.execute_reply":"2025-01-11T14:31:28.098696Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Missing value treatment","metadata":{}},{"cell_type":"code","source":"# Missing Value Treatment Attempt 1: Impute with median and mode\ndf_i_w_m_a_m = df.copy()\n# Für numerische Spalten\nfor i in all_numerical_columns:\n    df_i_w_m_a_m[i] = df_i_w_m_a_m[i].fillna(df_i_w_m_a_m[i].median()) \n\n# Für kategorische Spalten, Werte mit dem Modus auffüllen\nfor i in all_categorical_columns:\n    df_i_w_m_a_m[i] = df_i_w_m_a_m[i].fillna(df_i_w_m_a_m[i].mode()[0])\n\n# Check if values were filled\ndf_i_w_m_a_m.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:31:28.101647Z","iopub.execute_input":"2025-01-11T14:31:28.102016Z","iopub.status.idle":"2025-01-11T14:31:28.194783Z","shell.execute_reply.started":"2025-01-11T14:31:28.101946Z","shell.execute_reply":"2025-01-11T14:31:28.193805Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Outlier treatment\n- no outlier treatment","metadata":{}},{"cell_type":"markdown","source":"# One-Hot Encoding","metadata":{}},{"cell_type":"code","source":"df_i_w_m_a_m = pd.get_dummies(df_i_w_m_a_m, columns=['Basic_Demos-Sex', 'Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'FGC-Season','PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season'], drop_first=True)\ndf_i_w_m_a_m.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:31:28.195770Z","iopub.execute_input":"2025-01-11T14:31:28.196051Z","iopub.status.idle":"2025-01-11T14:31:28.273669Z","shell.execute_reply.started":"2025-01-11T14:31:28.196025Z","shell.execute_reply":"2025-01-11T14:31:28.272529Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Semi-supervised learning","metadata":{}},{"cell_type":"code","source":"from xgboost import XGBClassifier\nfrom sklearn.semi_supervised import SelfTrainingClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport pandas as pd\n\n# Gelabelte Daten (sii ist bekannt)\nlabeled_data = df_i_w_m_a_m[df_i_w_m_a_m['sii'].notna()]\n\n# Unlabelte Daten (sii ist unbekannt)\nunlabeled_data = df_i_w_m_a_m[df_i_w_m_a_m['sii'].isna()]\n\n# Gelabelte Daten\nids_labeled = labeled_data['id']  # IDs der gelabelten Daten\nX_labeled = labeled_data.drop(['sii', 'id'], axis=1) \ny_labeled = labeled_data['sii']  # Zielvariable\n\n# Ungelabelte Daten\nids_unlabeled = unlabeled_data['id']  # IDs der ungelabelten Daten\nX_unlabeled = unlabeled_data.drop(['sii', 'id'], axis=1)\ny_unlabeled = pd.Series([-1] * len(X_unlabeled), index=X_unlabeled.index)  # Zielvariable als -1\n\n# Aufteilen der gelabelten Daten in Training und Test\nX_train, X_test, y_train, y_test = train_test_split(\n    X_labeled, y_labeled, test_size=0.2, random_state=42, stratify=y_labeled\n)\n\n# Berechnung von Klassen-Gewichtungen\nclass_counts = y_train.value_counts()\ntotal_samples = len(y_train)\nclass_weights = {cls: total_samples / (len(class_counts) * count) for cls, count in class_counts.items()}\n\nprint(f\"Klassen-Gewichtungen: {class_weights}\")\n\n# XGBClassifier als Basis-Klassifikator\nbase_model = XGBClassifier(\n    random_state=42,\n    use_label_encoder=False,\n    eval_metric=\"mlogloss\"\n)\n\n# SelfTrainingClassifier mit XGBoost als Basis\nself_training_model = SelfTrainingClassifier(base_model)\n\n# Kombinieren der gelabelten und ungelabelten Daten\nX_combined = pd.concat([X_train, X_unlabeled])\ny_combined = pd.concat([y_train, y_unlabeled])\n\n# Training des SelfTrainingClassifiers\nself_training_model.fit(\n    X_combined,\n    y_combined\n)\n\n# Initialisiere eine Variable, um die Anzahl der neuen hochsicheren Labels zu verfolgen\nnew_high_conf_count = 1  # Startwert > 0, um die Schleife zu starten\n\n# Schleife: Wiederholen, solange neue hochsichere Labels gefunden werden\nwhile new_high_conf_count > 0:\n    # Vorhersagen und Wahrscheinlichkeiten für ungelabelte Daten\n    predictions = self_training_model.predict(X_unlabeled)\n    probabilities = self_training_model.predict_proba(X_unlabeled)\n\n    # Sicherheitsschwelle festlegen (z. B. 0.9)\n    confidence_threshold = 0.9\n    high_confidence_indices = (probabilities.max(axis=1) >= confidence_threshold)\n\n    # Hochsichere Vorhersagen und ihre Features extrahieren\n    X_high_conf = X_unlabeled[high_confidence_indices]\n    y_high_conf = predictions[high_confidence_indices]\n\n    # Zähle die Anzahl der neuen hochsicheren Labels\n    new_high_conf_count = len(X_high_conf)\n    print(f\"Neue hochsichere Vorhersagen in dieser Iteration: {new_high_conf_count}\")\n    print(f\"Anzahl der nicht hoch sicheren Vorhersagen: {len(probabilities) - new_high_conf_count}\")\n\n    # Wenn keine neuen hochsicheren Labels mehr vorhanden sind, abbrechen\n    if new_high_conf_count == 0:\n        break\n\n    # Hochsichere Daten zu gelabelten Daten hinzufügen\n    X_train = pd.concat([X_train, X_high_conf])\n    y_train = pd.concat([y_train, pd.Series(y_high_conf, index=X_high_conf.index)])\n\n    # Entfernen der hochsicheren Daten aus den ungelabelten Daten\n    X_unlabeled = X_unlabeled.drop(index=X_high_conf.index)\n\n    # Zielvariable für verbleibende ungelabelte Daten aktualisieren (-1 bleibt für diese erhalten)\n    y_unlabeled = pd.Series([-1] * len(X_unlabeled), index=X_unlabeled.index)\n\n    # Kombinieren der gelabelten und verbleibenden ungelabelten Daten\n    X_combined = pd.concat([X_train, X_unlabeled])\n    y_combined = pd.concat([y_train, y_unlabeled])\n\n    # Modell erneut trainieren\n    self_training_model.fit(X_combined, y_combined)\n\nprint(\"Training abgeschlossen. Keine weiteren hochsicheren Vorhersagen verfügbar.\")\n\n# Vorhersagen auf den Testdaten\ny_pred = self_training_model.predict(X_test)\n\n# Confusion Matrix\nconf_matrix = confusion_matrix(y_test, y_pred)\nprint(\"\\nConfusion Matrix:\")\nprint(conf_matrix)\n\n# Classification Report\nprint(\"\\nClassification Report:\")\nprint(classification_report(y_test, y_pred))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:50:16.050722Z","iopub.execute_input":"2025-01-11T14:50:16.051157Z","iopub.status.idle":"2025-01-11T14:50:43.712677Z","shell.execute_reply.started":"2025-01-11T14:50:16.051122Z","shell.execute_reply":"2025-01-11T14:50:43.711830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:31:28.274768Z","iopub.execute_input":"2025-01-11T14:31:28.275178Z","iopub.status.idle":"2025-01-11T14:31:29.030429Z","shell.execute_reply.started":"2025-01-11T14:31:28.275139Z","shell.execute_reply":"2025-01-11T14:31:29.029403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:32:47.868768Z","iopub.execute_input":"2025-01-11T14:32:47.869329Z","iopub.status.idle":"2025-01-11T14:32:48.228457Z","shell.execute_reply.started":"2025-01-11T14:32:47.869192Z","shell.execute_reply":"2025-01-11T14:32:48.227279Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:32:48.229701Z","iopub.execute_input":"2025-01-11T14:32:48.230024Z","iopub.status.idle":"2025-01-11T14:32:57.163711Z","shell.execute_reply.started":"2025-01-11T14:32:48.229988Z","shell.execute_reply":"2025-01-11T14:32:57.162657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:32:57.165455Z","iopub.execute_input":"2025-01-11T14:32:57.165741Z","iopub.status.idle":"2025-01-11T14:33:20.295123Z","shell.execute_reply.started":"2025-01-11T14:32:57.165717Z","shell.execute_reply":"2025-01-11T14:33:20.293727Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:33:20.298456Z","iopub.execute_input":"2025-01-11T14:33:20.298746Z","iopub.status.idle":"2025-01-11T14:33:20.327099Z","shell.execute_reply.started":"2025-01-11T14:33:20.298721Z","shell.execute_reply":"2025-01-11T14:33:20.326139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Vorhersagen und Wahrscheinlichkeiten für ungelabelte Daten\n# predictions = self_training_model.predict(X_unlabeled)\n# probabilities = self_training_model.predict_proba(X_unlabeled)\n\n# # Sicherheitsschwelle festlegen (z. B. 0.9)\n# confidence_threshold = 0.9\n# high_confidence_indices = (probabilities.max(axis=1) >= confidence_threshold)\n# num_high_confidence = high_confidence_indices.sum()\n# print(f\"Anzahl der hochsicheren Vorhersagen: {num_high_confidence}\")\n# print(f\"Anzahl der nicht hoch sicheren Vorhersagen: {len(probabilities)}\")\n\n# # Hochsichere Vorhersagen und ihre Features extrahieren\n# X_high_conf = X_unlabeled[high_confidence_indices]\n# y_high_conf = predictions[high_confidence_indices]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:33:20.329123Z","iopub.execute_input":"2025-01-11T14:33:20.329449Z","iopub.status.idle":"2025-01-11T14:33:20.333777Z","shell.execute_reply.started":"2025-01-11T14:33:20.329417Z","shell.execute_reply":"2025-01-11T14:33:20.332709Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Hochsichere Daten zu gelabelten Daten hinzufügen\n# X_labeled = pd.concat([X_labeled, X_high_conf])\n# y_labeled = pd.concat([y_labeled, pd.Series(y_high_conf, index=X_high_conf.index)])\n\n# # Entfernen der hochsicheren Daten aus den ungelabelten Daten\n# X_unlabeled = X_unlabeled.drop(index=X_high_conf.index)\n\n# # Zielvariable für verbleibende ungelabelte Daten aktualisieren (-1 bleibt für diese erhalten)\n# y_unlabeled = pd.Series([-1] * len(X_unlabeled), index=X_unlabeled.index)\n\n# # Kombinieren der gelabelten und verbleibenden ungelabelten Daten\n# X_combined = pd.concat([X_labeled, X_unlabeled])\n# y_combined = pd.concat([y_labeled, y_unlabeled])\n\n# # Modell erneut trainieren\n# self_training_model.fit(X_combined, y_combined)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:33:20.334944Z","iopub.execute_input":"2025-01-11T14:33:20.335364Z","iopub.status.idle":"2025-01-11T14:33:20.351262Z","shell.execute_reply.started":"2025-01-11T14:33:20.335324Z","shell.execute_reply":"2025-01-11T14:33:20.350054Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# IDs der Testdaten extrahieren\nids_test = test_data['id']\n\n# Testdaten vorbereiten (entfernen der 'id'-Spalte)\nX_test = test_data.drop(['id'], axis=1)\n\n# do necessary transformations, e.g. one-hot encoding\n\nX_test['PAQ_Total'] = X_test.apply(calculate_paq_total, axis=1)\n# drop the old two columns\nX_test = X_test.drop(columns=['PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total'])\n\nX_test['Fitness_Endurance-Time'] = X_test['Fitness_Endurance-Time_Mins'] + (X_test['Fitness_Endurance-Time_Sec'] / 60)\n\n# drop old columns\n\nX_test = X_test.drop(columns=['Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec', 'BIA-Season', 'Fitness_Endurance-Season', 'PAQ_A-Season'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:33:20.352397Z","iopub.execute_input":"2025-01-11T14:33:20.352734Z","iopub.status.idle":"2025-01-11T14:33:20.378843Z","shell.execute_reply.started":"2025-01-11T14:33:20.352695Z","shell.execute_reply":"2025-01-11T14:33:20.377700Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Missing Value Treatment Attempt 1: Impute with median and mode\nX_test = X_test.copy()\n# Für numerische Spalten\nfor i in all_numerical_columns:\n    X_test[i] = X_test[i].fillna(X_test[i].median()) \n\n# Für kategorische Spalten, Werte mit dem Modus auffüllen\nfor i in all_categorical_columns:\n    X_test[i] = X_test[i].fillna(X_test[i].mode()[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:33:20.381449Z","iopub.execute_input":"2025-01-11T14:33:20.381750Z","iopub.status.idle":"2025-01-11T14:33:20.421799Z","shell.execute_reply.started":"2025-01-11T14:33:20.381724Z","shell.execute_reply":"2025-01-11T14:33:20.420656Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test = pd.get_dummies(X_test, columns=['Basic_Demos-Sex', 'Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'FGC-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season'], drop_first=True)\nX_test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:33:20.423100Z","iopub.execute_input":"2025-01-11T14:33:20.423447Z","iopub.status.idle":"2025-01-11T14:33:20.488564Z","shell.execute_reply.started":"2025-01-11T14:33:20.423421Z","shell.execute_reply":"2025-01-11T14:33:20.487747Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Vorhersagen für die Testdaten\nfinal_predictions = self_training_model.predict(X_test)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:33:20.489693Z","iopub.execute_input":"2025-01-11T14:33:20.490038Z","iopub.status.idle":"2025-01-11T14:33:20.498460Z","shell.execute_reply.started":"2025-01-11T14:33:20.490005Z","shell.execute_reply":"2025-01-11T14:33:20.497693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ergebnisse in einem DataFrame speichern\nsubmission = pd.DataFrame({\n    'id': ids_test,  # IDs der Testdaten\n    'sii': final_predictions  # Vorhergesagte sii-Werte\n})\n\n# Konvertieren der sii-Werte in ganze Zahlen\nsubmission['sii'] = submission['sii'].astype(int)\n\n# Ergebnisse als CSV speichern\nsubmission.to_csv('submission.csv', index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:33:20.499070Z","iopub.execute_input":"2025-01-11T14:33:20.499320Z","iopub.status.idle":"2025-01-11T14:33:20.528859Z","shell.execute_reply.started":"2025-01-11T14:33:20.499295Z","shell.execute_reply":"2025-01-11T14:33:20.527519Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\n# Absoluten Pfad der Datei anzeigen\nprint(os.path.abspath('submission.csv'))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-11T14:33:20.530074Z","iopub.execute_input":"2025-01-11T14:33:20.530636Z","iopub.status.idle":"2025-01-11T14:33:20.539045Z","shell.execute_reply.started":"2025-01-11T14:33:20.530589Z","shell.execute_reply":"2025-01-11T14:33:20.538098Z"}},"outputs":[],"execution_count":null}]}