{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.stats import normaltest\nfrom scipy import stats\nfrom sklearn.impute import KNNImputer\nfrom sklearn.model_selection import KFold\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import classification_report, accuracy_score, confusion_matrix\nfrom sklearn.preprocessing import StandardScaler\nfrom imblearn.over_sampling import SMOTE\n\npd.set_option('display.max_columns', None)","metadata":{"_uuid":"ec897340-d6e7-4efc-be32-ee8e26a4b66b","_cell_guid":"90de7b24-8e81-4c50-b87b-1a033177d50e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:27.708548Z","iopub.execute_input":"2024-10-24T16:24:27.708971Z","iopub.status.idle":"2024-10-24T16:24:27.715538Z","shell.execute_reply.started":"2024-10-24T16:24:27.708930Z","shell.execute_reply":"2024-10-24T16:24:27.714396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the training data\ntrain_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n\n# Display the first few rows\ntrain_df.head(250)","metadata":{"_uuid":"e5053942-3751-4d36-bb8a-aa7939f9dde1","_cell_guid":"fe2d82a5-f6b6-45bb-8469-ed234a0ef740","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:27.717806Z","iopub.execute_input":"2024-10-24T16:24:27.718518Z","iopub.status.idle":"2024-10-24T16:24:27.874657Z","shell.execute_reply.started":"2024-10-24T16:24:27.718466Z","shell.execute_reply":"2024-10-24T16:24:27.873587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.dropna(subset=['sii'])","metadata":{"execution":{"iopub.status.busy":"2024-10-24T16:24:27.876135Z","iopub.execute_input":"2024-10-24T16:24:27.876606Z","iopub.status.idle":"2024-10-24T16:24:27.885369Z","shell.execute_reply.started":"2024-10-24T16:24:27.876558Z","shell.execute_reply":"2024-10-24T16:24:27.884301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get basic information about the DataFrame\ntrain_df.info()","metadata":{"_uuid":"9a0c1ae7-c410-4f1e-a64f-8690f4f95606","_cell_guid":"3d300641-e867-42a9-ba5c-a284adcac792","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:27.888479Z","iopub.execute_input":"2024-10-24T16:24:27.889088Z","iopub.status.idle":"2024-10-24T16:24:27.907466Z","shell.execute_reply.started":"2024-10-24T16:24:27.889050Z","shell.execute_reply":"2024-10-24T16:24:27.906422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set pandas to display all rows\npd.set_option('display.max_rows', None)\n\n# Calculate the percentage of missing values for each column\nmissing_values = train_df.isnull().mean().sort_values(ascending=False) * 100\n\n# Display columns with missing values\nmissing_columns = missing_values[missing_values > 0]\nprint(\"Columns with missing values:\\n\")\nprint(missing_columns)\n\n# Reset pandas display options back to default\npd.reset_option('display.max_rows')","metadata":{"_uuid":"bc684cd7-d422-43f4-899d-f84f6b9a2115","_cell_guid":"49555c46-b48a-4a4c-b1c5-4c728a037058","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:27.908856Z","iopub.execute_input":"2024-10-24T16:24:27.909219Z","iopub.status.idle":"2024-10-24T16:24:27.922361Z","shell.execute_reply.started":"2024-10-24T16:24:27.909175Z","shell.execute_reply":"2024-10-24T16:24:27.921284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get statistical summary of numerical columns\ntrain_df.describe()","metadata":{"_uuid":"10f09507-8800-48d9-921b-332b2cd28bc7","_cell_guid":"2c1d7a57-271a-4b59-b83e-f925d314fef4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:27.923819Z","iopub.execute_input":"2024-10-24T16:24:27.924150Z","iopub.status.idle":"2024-10-24T16:24:28.106926Z","shell.execute_reply.started":"2024-10-24T16:24:27.924116Z","shell.execute_reply":"2024-10-24T16:24:28.105939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop all PCIAT columns and unnecessary columns\npciat_columns = [col for col in train_df.columns if col.startswith('PCIAT-PCIAT_')]\ncolumns_to_drop = pciat_columns + ['PCIAT-Season', 'PCIAT-PCIAT_Total']\ntrain_df.drop(columns=columns_to_drop, inplace=True)","metadata":{"_uuid":"36a25d35-bc04-414c-96a6-3545e3c921a4","_cell_guid":"ed95d87e-1328-415f-8409-ab2dcd7e027f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:28.108143Z","iopub.execute_input":"2024-10-24T16:24:28.108452Z","iopub.status.idle":"2024-10-24T16:24:28.115467Z","shell.execute_reply.started":"2024-10-24T16:24:28.108420Z","shell.execute_reply":"2024-10-24T16:24:28.114332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set all weights of 0 to the median weight\nmedian_weight = train_df['Physical-Weight'].replace(0, np.nan).median()\ntrain_df['Physical-Weight'] = train_df['Physical-Weight'].replace(0, median_weight)","metadata":{"_uuid":"cc1d832e-3685-4b09-9fe3-d089687502c0","_cell_guid":"7fee3a38-510f-44ad-b6d9-0420533b6c66","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:28.116875Z","iopub.execute_input":"2024-10-24T16:24:28.117767Z","iopub.status.idle":"2024-10-24T16:24:28.128884Z","shell.execute_reply.started":"2024-10-24T16:24:28.117720Z","shell.execute_reply":"2024-10-24T16:24:28.127791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Handle PAQ_A-Season column\n# One-hot encode 'PAQ_A-Season' column to retain valuable information\nif 'PAQ_A-Season' in train_df.columns:\n    season_dummies = pd.get_dummies(train_df['PAQ_A-Season'], prefix='PAQ_A_Season', dummy_na=True).rename(columns=lambda x: x.replace('Season_nan', 'Season_na'))\n    train_df = train_df.join(season_dummies)\n    train_df.drop('PAQ_A-Season', axis=1, inplace=True)\n\n# Handle PAQ_C-Season column\n# One-hot encode 'PAQ_C-Season' column to retain valuable information\nif 'PAQ_C-Season' in train_df.columns:\n    season_dummies = pd.get_dummies(train_df['PAQ_C-Season'], prefix='PAQ_C_Season', dummy_na=True).rename(columns=lambda x: x.replace('Season_nan', 'Season_na'))\n    train_df = train_df.join(season_dummies)\n    train_df.drop('PAQ_C-Season', axis=1, inplace=True)","metadata":{"_uuid":"5176011e-c6f0-43da-a733-4d0e293e2e30","_cell_guid":"9cfb4353-cc40-4038-ba30-71933d24276f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:28.133456Z","iopub.execute_input":"2024-10-24T16:24:28.133949Z","iopub.status.idle":"2024-10-24T16:24:28.152907Z","shell.execute_reply.started":"2024-10-24T16:24:28.133898Z","shell.execute_reply":"2024-10-24T16:24:28.151744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Analyze PAQ_A-PAQ_A_Total and PAQ_C-PAQ_C_Total columns\n\n# Display descriptive statistics\nfor column in ['PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total']:\n    if column in train_df.columns:\n        print(f\"{column} description:\")\n        print(train_df[column].describe())\n\n        # Plot the distribution\n        plt.figure(figsize=(10, 5))\n        sns.histplot(train_df[column].dropna(), kde=True)\n        plt.title(f'Distribution of {column}')\n        plt.xlabel(column)\n        plt.ylabel('Frequency')\n        plt.show()\n\n        # Test for normality\n        stat, p_value = normaltest(train_df[column].dropna())\n        print(f\"Normality test for {column}: Statistic={stat}, p-value={p_value}\")\n        if p_value > 0.05:\n            print(f\"{column} appears to be normally distributed.\")\n        else:\n            print(f\"{column} does not appear to be normally distributed.\")","metadata":{"_uuid":"58e7f441-0a4a-4741-9e49-0b22dbf37dd1","_cell_guid":"af6c574e-8bdb-40f1-82b3-cc2a6cfa68ee","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:28.154616Z","iopub.execute_input":"2024-10-24T16:24:28.155047Z","iopub.status.idle":"2024-10-24T16:24:28.844303Z","shell.execute_reply.started":"2024-10-24T16:24:28.154989Z","shell.execute_reply":"2024-10-24T16:24:28.843305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check correlations with 'sii' column\ncolumns_to_check = ['PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total', 'sii']\ncorrelation_matrix = train_df[columns_to_check].corr()\n\n# Plot heatmap of correlations\nplt.figure(figsize=(8, 6))\nsns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', fmt='.2f')\nplt.title('Correlation Heatmap')\nplt.show()","metadata":{"_uuid":"cb5db3f0-57cd-471c-a12c-928511cb8f4a","_cell_guid":"316beffb-4760-4778-a623-0b370d67d366","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:28.845632Z","iopub.execute_input":"2024-10-24T16:24:28.845968Z","iopub.status.idle":"2024-10-24T16:24:29.134724Z","shell.execute_reply.started":"2024-10-24T16:24:28.845933Z","shell.execute_reply":"2024-10-24T16:24:29.133766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_drop = ['PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total']\ntrain_df.drop(columns=columns_to_drop, inplace=True)","metadata":{"_uuid":"db1ba4da-be2b-4a92-bb81-aadd9723f530","_cell_guid":"731cc7d2-8796-42a1-bf62-d5e1e20986ee","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:29.135874Z","iopub.execute_input":"2024-10-24T16:24:29.136188Z","iopub.status.idle":"2024-10-24T16:24:29.143004Z","shell.execute_reply.started":"2024-10-24T16:24:29.136156Z","shell.execute_reply":"2024-10-24T16:24:29.142005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Handle categorical columns with one-hot encoding and missing value handling\nobject_columns = train_df.select_dtypes(include='object').columns.difference(['id'])\n\nfor column in object_columns:\n    # Manually handle missing values by adding '_missing' columns for original object columns with NaN\n    if column in train_df.columns and train_df[column].isna().any():\n        na_column = f\"{column}_na\"\n    if na_column not in train_df.columns:\n        train_df[na_column] = train_df[column].isna().astype(int)\n    \n    # One-hot encode the column\n    dummies = pd.get_dummies(train_df[column], prefix=column, dummy_na=False)\n    \n    train_df = train_df.join(dummies)\n    train_df.drop(column, axis=1, inplace=True)","metadata":{"_uuid":"f781a077-436c-40a7-af26-b2da0c452bad","_cell_guid":"49f6483c-4600-4da1-a345-1d72f053dcdb","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:29.144759Z","iopub.execute_input":"2024-10-24T16:24:29.145173Z","iopub.status.idle":"2024-10-24T16:24:29.191845Z","shell.execute_reply.started":"2024-10-24T16:24:29.145137Z","shell.execute_reply":"2024-10-24T16:24:29.190746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate Z-scores to identify outliers\nnumeric_columns = [col for col in train_df.select_dtypes(include=[np.number]).columns.difference(['id', 'Basic_Demos-Age']) if not train_df[col].dropna().isin([0, 1, 2, 3, 4, 5]).all()]\nz_scores = np.abs(stats.zscore(train_df[numeric_columns], nan_policy='omit'))\noutliers = (z_scores >= 3)\n\n# Drop 'id' column from correlations calculation\ncorrelation_matrix = train_df.drop(columns=['id']).corr()\nsii_correlations = correlation_matrix[['sii']].sort_values(by='sii', ascending=False)\n\n# Get percentage of missing values for each column\nmissing_values = train_df.isnull().mean() * 100\nmissing_columns = missing_values[missing_values > 0]\n\n# Display data for each numeric column\nfor column in numeric_columns:\n   \n    # Print number of zero values for the column\n    num_zeros = (train_df[column] == 0).sum()\n    print(f\"Number of zero values in {column}: {num_zeros}\")\n    \n    # Print number of outliers for the column\n    num_outliers = outliers[column].sum()\n    print(f\"Number of outliers in {column}: {num_outliers}\")\n    \n    # Print the values of the outliers that would be removed\n    outlier_values = train_df[column][outliers[column]].values\n    print(f\"Outlier values in {column}: {outlier_values}\")\n    \n    # Print correlation with 'sii'\n    correlation = sii_correlations.loc[column, 'sii']\n    print(f\"Correlation with 'sii' for {column}: {correlation:.5f}\")\n    \n    # Print percentage of missing values\n    missing_percentage = missing_columns.get(column, 0)\n    print(f\"Percentage of missing values in {column}: {missing_percentage:.2f}%\")\n    print(\"-\" * 50)","metadata":{"_uuid":"ee5c44f5-3c0a-4fff-a46e-76615ce59a00","_cell_guid":"7d409fd0-f62a-40ab-9749-b6ed75dec515","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:29.193218Z","iopub.execute_input":"2024-10-24T16:24:29.193668Z","iopub.status.idle":"2024-10-24T16:24:29.340279Z","shell.execute_reply.started":"2024-10-24T16:24:29.193621Z","shell.execute_reply":"2024-10-24T16:24:29.339266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate the average weight of the entries which have 0 in FGC-FGC_CU\navg_weight_zero_curl_up = train_df.loc[train_df['FGC-FGC_CU'] == 0, 'Physical-Weight'].mean()\nprint(f\"Average weight of entries with 0 curl-ups: {avg_weight_zero_curl_up:.2f}\")","metadata":{"_uuid":"46ff6bc2-294f-4f93-94b1-bff4b69c9c24","_cell_guid":"3741b6f4-ac0e-430e-96b1-35b8a877d811","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:29.341587Z","iopub.execute_input":"2024-10-24T16:24:29.341926Z","iopub.status.idle":"2024-10-24T16:24:29.348567Z","shell.execute_reply.started":"2024-10-24T16:24:29.341891Z","shell.execute_reply":"2024-10-24T16:24:29.347320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate the average weight of the entries which have 0 in FGC-FGC_CU\navg_weight_zero_curl_up = train_df.loc[train_df['FGC-FGC_PU'] == 0, 'Physical-Weight'].mean()\nprint(f\"Average weight of entries with 0 curl-ups: {avg_weight_zero_curl_up:.2f}\")","metadata":{"_uuid":"b31df95d-5760-428f-af40-5fcd4d1117b9","_cell_guid":"50ba083a-aaca-4e01-bd84-369e5427f084","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:29.349901Z","iopub.execute_input":"2024-10-24T16:24:29.350317Z","iopub.status.idle":"2024-10-24T16:24:29.361445Z","shell.execute_reply.started":"2024-10-24T16:24:29.350270Z","shell.execute_reply":"2024-10-24T16:24:29.360101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate the average weight of all participants (excluding zeros)\navg_weight_all = train_df['Physical-Weight'].replace(0, np.nan).mean()\nprint(f\"Average weight of all participants (excluding zeros): {avg_weight_all:.2f}\")","metadata":{"_uuid":"23b37560-4d2d-417a-b093-6158d2baaf4f","_cell_guid":"3f33603e-b6e9-45df-a7d7-0d01ed2246a7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:29.363030Z","iopub.execute_input":"2024-10-24T16:24:29.363995Z","iopub.status.idle":"2024-10-24T16:24:29.374946Z","shell.execute_reply.started":"2024-10-24T16:24:29.363946Z","shell.execute_reply":"2024-10-24T16:24:29.373615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set all weights of 0 to the median weight\nmedian_weight = train_df['Physical-Weight'].replace(0, np.nan).median()\ntrain_df['Physical-Weight'] = train_df['Physical-Weight'].replace(0, median_weight)\n\n# Set all curl-up counts of 0 to the median count\nmedian_curl_up = train_df['FGC-FGC_CU'].replace(0, np.nan).median()\ntrain_df['FGC-FGC_CU'] = train_df['FGC-FGC_CU'].replace(0, median_curl_up)\n\n# Set all grip strength dominant values of 0 to the median count\nmedian_grip_strength_dominant = train_df['FGC-FGC_GSD'].replace(0, np.nan).median()\ntrain_df['FGC-FGC_GSD'] = train_df['FGC-FGC_GSD'].replace(0, median_grip_strength_dominant)\n\n# Set all grip strength non-dominant values of 0 to the median count\nmedian_grip_strength_nondominant = train_df['FGC-FGC_GSND'].replace(0, np.nan).median()\ntrain_df['FGC-FGC_GSND'] = train_df['FGC-FGC_GSND'].replace(0, median_grip_strength_nondominant)\n\n# Set all push-up counts of 0 to the median count\nmedian_push_up = train_df['FGC-FGC_PU'].replace(0, np.nan).median()\ntrain_df['FGC-FGC_PU'] = train_df['FGC-FGC_PU'].replace(0, median_push_up)\n\n# Set all FGC-FGC_SRL counts of 0 to the median count\nmedian_push_up = train_df['FGC-FGC_SRL'].replace(0, np.nan).median()\ntrain_df['FGC-FGC_SRL'] = train_df['FGC-FGC_SRL'].replace(0, median_push_up)\n\n# Set all FGC-FGC_SRR counts of 0 to the median count\nmedian_push_up = train_df['FGC-FGC_SRR'].replace(0, np.nan).median()\ntrain_df['FGC-FGC_SRR'] = train_df['FGC-FGC_SRR'].replace(0, median_push_up)\n\n# Set all FGC-FGC_TL counts of 0 to the median count\nmedian_push_up = train_df['FGC-FGC_TL'].replace(0, np.nan).median()\ntrain_df['FGC-FGC_TL'] = train_df['FGC-FGC_TL'].replace(0, median_push_up)\n\n# Set all Fitness_Endurance-Max_Stage counts of 0 to the median count\nmedian_push_up = train_df['Fitness_Endurance-Max_Stage'].replace(0, np.nan).median()\ntrain_df['Fitness_Endurance-Max_Stage'] = train_df['Fitness_Endurance-Max_Stage'].replace(0, median_push_up)\n\n# Set all   Physical-BMI counts of 0 to the median count\nmedian_push_up = train_df['Physical-BMI'].replace(0, np.nan).median()\ntrain_df['Physical-BMI'] = train_df['Physical-BMI'].replace(0, median_push_up)\n\n# Set all Fitness_Endurance-Time_Sec counts of 0 to the median count\nmedian_push_up = train_df['Fitness_Endurance-Time_Sec'].replace(0, np.nan).median()\ntrain_df['Fitness_Endurance-Time_Sec'] = train_df['Fitness_Endurance-Time_Sec'].replace(0, median_push_up)","metadata":{"_uuid":"87280109-5df5-4a78-b547-343d3044aff7","_cell_guid":"c10e3add-e4cb-4a8e-a700-0eb3857eb95e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:29.376598Z","iopub.execute_input":"2024-10-24T16:24:29.377440Z","iopub.status.idle":"2024-10-24T16:24:29.400803Z","shell.execute_reply.started":"2024-10-24T16:24:29.377391Z","shell.execute_reply":"2024-10-24T16:24:29.399727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate Z-scores to identify outliers\nnumeric_columns = [col for col in train_df.select_dtypes(include=[np.number]).columns.difference(['id', 'Basic_Demos-Age']) if not train_df[col].dropna().isin([0, 1, 2, 3, 4, 5]).all()]\nz_scores = np.abs(stats.zscore(train_df[numeric_columns], nan_policy='omit'))\noutliers = (z_scores >= 3)\n\n# Remove outliers from the dataframe\noutliers_indices = outliers.any(axis=1)\ntrain_df = train_df.loc[~outliers_indices]\n\n# Drop 'id' column from correlations calculation\ncorrelation_matrix = train_df.drop(columns=['id']).corr()\nsii_correlations = correlation_matrix[['sii']].sort_values(by='sii', ascending=False)\n\n# Get percentage of missing values for each column\nmissing_values = train_df.isnull().mean() * 100\nmissing_columns = missing_values[missing_values > 0]\n\n# Display data for each numeric column\nfor column in numeric_columns:\n     \n    # Print correlation with 'sii'\n    correlation = sii_correlations.loc[column, 'sii']\n    print(f\"Correlation with 'sii' for {column}: {correlation:.5f}\")\n    \n    # Print percentage of missing values\n    missing_percentage = missing_columns.get(column, 0)\n    print(f\"Percentage of missing values in {column}: {missing_percentage:.2f}%\")\n    print(\"-\" * 50)","metadata":{"_uuid":"7f51293a-78a8-40d1-a2cf-40f03a5dbce0","_cell_guid":"dd4898ac-2646-4474-8fd2-e7dda13a4662","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:29.401954Z","iopub.execute_input":"2024-10-24T16:24:29.402269Z","iopub.status.idle":"2024-10-24T16:24:29.530281Z","shell.execute_reply.started":"2024-10-24T16:24:29.402236Z","shell.execute_reply":"2024-10-24T16:24:29.529216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop columns with a high percentage of missing values\ndrop_columns = [\n    'Physical-Waist_Circumference',\n    'Fitness_Endurance-Time_Mins',\n    'Fitness_Endurance-Time_Sec',\n    'Fitness_Endurance-Max_Stage',\n    'FGC-FGC_GSD_Zone',\n    'FGC-FGC_GSND_Zone',\n    'FGC-FGC_GSD',\n    'FGC-FGC_GSND',\n    'FGC-FGC_SRR',\n    'FGC-FGC_SRL'\n]\ntrain_df = train_df.drop(columns=drop_columns)","metadata":{"_uuid":"49bc8606-4810-49b6-90b4-c966aeab0071","_cell_guid":"99746ec5-50b1-46ed-ad4a-d12c189fa1a4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:29.531544Z","iopub.execute_input":"2024-10-24T16:24:29.531881Z","iopub.status.idle":"2024-10-24T16:24:29.539132Z","shell.execute_reply.started":"2024-10-24T16:24:29.531847Z","shell.execute_reply":"2024-10-24T16:24:29.538095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate Z-scores to identify outliers, excluding columns with only 0, 1 or values from 0 to 5\nnumeric_columns = [\n    col for col in train_df.select_dtypes(include=[np.float64]).columns\n    if not train_df[col].dropna().isin([0, 1, 2, 3, 4, 5]).all()\n]\nz_scores = np.abs(stats.zscore(train_df[numeric_columns], nan_policy='omit'))\noutliers = (z_scores >= 3)\n\n# Drop 'id' column from correlations calculation\ncorrelation_matrix = train_df.drop(columns=['id']).corr()\nsii_correlations = correlation_matrix[['sii']].sort_values(by='sii', ascending=False)\n\n# Get percentage of missing values for each column\nmissing_values = train_df.isnull().mean() * 100\nmissing_columns = missing_values[missing_values > 0]\n\n# Display data for each numeric column\nfor column in numeric_columns:\n    \n    # Print number of zero values for the column\n    num_zeros = (train_df[column] == 0).sum()\n    print(f\"Number of zero values in {column}: {num_zeros}\")\n    \n    # Print number of outliers for the column\n    num_outliers = outliers[column].sum()\n    print(f\"Number of outliers in {column}: {num_outliers}\")\n    \n    # Print the values of the outliers that would be removed\n    outlier_values = train_df[column][outliers[column]].values\n    print(f\"Outlier values in {column}: {outlier_values}\")\n    \n    # Print correlation with 'sii'\n    correlation = sii_correlations.loc[column, 'sii']\n    print(f\"Correlation with 'sii' for {column}: {correlation:.5f}\")\n    \n    # Print percentage of missing values\n    missing_percentage = missing_columns.get(column, 0)\n    print(f\"Percentage of missing values in {column}: {missing_percentage:.2f}%\")\n    print(\"-\" * 50)","metadata":{"_uuid":"1af4841b-4c42-4fea-8c8d-06b83026fb4e","_cell_guid":"f5077c9f-1a21-4dd9-a59c-86bffa1620c5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:29.540602Z","iopub.execute_input":"2024-10-24T16:24:29.541012Z","iopub.status.idle":"2024-10-24T16:24:29.662057Z","shell.execute_reply.started":"2024-10-24T16:24:29.540967Z","shell.execute_reply":"2024-10-24T16:24:29.661004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate Z-scores to identify outliers\nnumeric_columns = [col for col in train_df.select_dtypes(include=[np.float64]).columns.difference(['id', 'Basic_Demos-Age']) if not train_df[col].dropna().isin([0, 1, 2, 3, 4, 5]).all()]\nz_scores = np.abs(stats.zscore(train_df[numeric_columns], nan_policy='omit'))\noutliers = (z_scores >= 3)\n\n# Remove outliers from the dataframe\noutliers_indices = outliers.any(axis=1)\ntrain_df = train_df.loc[~outliers_indices]\n\n# Recalculate numeric columns after removing outliers\nnumeric_columns = [col for col in train_df.select_dtypes(include=[np.float64]).columns.difference(['id', 'Basic_Demos-Age']) if not train_df[col].dropna().isin([0, 1, 2, 3, 4, 5]).all()]\n\n# Drop 'id' column from correlations calculation\ncorrelation_matrix = train_df.drop(columns=['id']).corr()\nsii_correlations = correlation_matrix[['sii']].sort_values(by='sii', ascending=False)\n\n# Get percentage of missing values for each column\nmissing_values = train_df.isnull().mean() * 100\nmissing_columns = missing_values[missing_values > 0]\n\n# Display data for each numeric column\nfor column in numeric_columns:\n   \n    # Print correlation with 'sii'\n    correlation = sii_correlations.loc[column, 'sii']\n    print(f\"Correlation with 'sii' for {column}: {correlation:.5f}\")\n    \n    # Print percentage of missing values\n    missing_percentage = missing_columns.get(column, 0)\n    print(f\"Percentage of missing values in {column}: {missing_percentage:.2f}%\")\n    print(\"-\" * 50)","metadata":{"_uuid":"de87d77c-a748-4c98-80a0-54df138f9819","_cell_guid":"489aa56a-ca6f-4d88-b320-d093909dd68a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:29.663528Z","iopub.execute_input":"2024-10-24T16:24:29.663974Z","iopub.status.idle":"2024-10-24T16:24:29.786927Z","shell.execute_reply.started":"2024-10-24T16:24:29.663915Z","shell.execute_reply":"2024-10-24T16:24:29.785888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"_uuid":"269732ce-74e6-4b33-9657-cd9fe78880d8","_cell_guid":"1a0635c1-55b2-487b-8691-796c901ddf32","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:29.788177Z","iopub.execute_input":"2024-10-24T16:24:29.788527Z","iopub.status.idle":"2024-10-24T16:24:29.806591Z","shell.execute_reply.started":"2024-10-24T16:24:29.788474Z","shell.execute_reply":"2024-10-24T16:24:29.805560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize KNN imputer with default settings (n_neighbors=5)\nimputer = KNNImputer(n_neighbors=5)\n\n# Identify the numeric columns to impute (excluding boolean columns and 'id')\nnumeric_columns = [col for col in train_df.select_dtypes(include=[np.float64]).columns.difference(['id', 'Basic_Demos-Age']) if not train_df[col].dropna().isin([0, 1, 2, 3, 4, 5]).all()]\n\n# Apply KNN imputer to the numeric columns\ntrain_df[numeric_columns] = imputer.fit_transform(train_df[numeric_columns])","metadata":{"_uuid":"822e0087-910a-4628-9b1d-09ff3eb6105c","_cell_guid":"a56bafd2-dabd-46e6-ae9e-57ff76ea7513","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:29.810816Z","iopub.execute_input":"2024-10-24T16:24:29.811166Z","iopub.status.idle":"2024-10-24T16:24:30.701536Z","shell.execute_reply.started":"2024-10-24T16:24:29.811131Z","shell.execute_reply":"2024-10-24T16:24:30.700523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set pandas to display all rows\npd.set_option('display.max_rows', None)\n\n# Calculate the percentage of missing values for each column\nmissing_values = train_df.isnull().mean().sort_values(ascending=False) * 100\n\n# Display columns with missing values\nmissing_columns = missing_values[missing_values > 0]\nprint(\"Columns with missing values:\\n\")\nprint(missing_columns)\n\n# Reset pandas display options back to default\npd.reset_option('display.max_rows')","metadata":{"_uuid":"c702acb1-5d6b-4573-91c6-7accccdefb83","_cell_guid":"cb0af325-3ce6-4e5d-9cc0-75845348bc1b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:30.702829Z","iopub.execute_input":"2024-10-24T16:24:30.703186Z","iopub.status.idle":"2024-10-24T16:24:30.714913Z","shell.execute_reply.started":"2024-10-24T16:24:30.703150Z","shell.execute_reply":"2024-10-24T16:24:30.714058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_encode = [\n    'BIA-BIA_Activity_Level_num',\n    'BIA-BIA_Frame_num',\n    'FGC-FGC_PU_Zone',\n    'FGC-FGC_SRL_Zone',\n    'FGC-FGC_SRR_Zone',\n    'FGC-FGC_CU_Zone',\n    'FGC-FGC_TL_Zone',\n    'PreInt_EduHx-computerinternet_hoursday'\n]\nfor column in columns_to_encode:\n    dummies = pd.get_dummies(train_df[column], prefix=column, dummy_na=True).rename(columns=lambda x: x.replace('nan', 'na'))\n    train_df = train_df.join(dummies)\n    train_df.drop(column, axis=1, inplace=True)","metadata":{"_uuid":"5de93340-2b69-48f0-b3dd-321e7cf0b056","_cell_guid":"f1dec4b1-729a-4123-b93f-b038dbb9efca","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:30.717334Z","iopub.execute_input":"2024-10-24T16:24:30.718333Z","iopub.status.idle":"2024-10-24T16:24:30.751353Z","shell.execute_reply.started":"2024-10-24T16:24:30.718293Z","shell.execute_reply":"2024-10-24T16:24:30.750286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for duplicate IDs in train_df and test_df\ntrain_duplicates = train_df['id'].duplicated().sum()\ntest_duplicates = test_df['id'].duplicated().sum()\nif train_duplicates == 0:\n    print(\"No duplicate IDs found in train_df.\")\nelse:\n    print(f\"Duplicate IDs found in train_df: {train_duplicates}\")\n\nif test_duplicates == 0:\n    print(\"No duplicate IDs found in test_df.\")\nelse:\n    print(f\"Duplicate IDs found in test_df: {test_duplicates}\")","metadata":{"_uuid":"0caa1d35-fba1-40a1-975e-939f18077aa8","_cell_guid":"3c563b88-eb86-43f1-99e9-94872be8bd0e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:30.753193Z","iopub.execute_input":"2024-10-24T16:24:30.753637Z","iopub.status.idle":"2024-10-24T16:24:30.761262Z","shell.execute_reply.started":"2024-10-24T16:24:30.753589Z","shell.execute_reply":"2024-10-24T16:24:30.760255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Remove duplicate ID '105258' from both train_df and test_df\nduplicate_id = '00105258'\ntrain_df = train_df[train_df['id'] != duplicate_id]\ntest_df = test_df[test_df['id'] != duplicate_id]\n\nprint(f\"Removed duplicate ID '{duplicate_id}' from both train_df and test_df.\")","metadata":{"_uuid":"b8b721bc-75ff-4894-9b6e-a5b602dcde13","_cell_guid":"64431da5-c02a-4c59-92b8-5b2dc3acb321","execution":{"iopub.status.busy":"2024-10-23T02:03:41.661968Z","iopub.execute_input":"2024-10-23T02:03:41.662338Z","iopub.status.idle":"2024-10-23T02:03:41.671889Z","shell.execute_reply.started":"2024-10-23T02:03:41.662297Z","shell.execute_reply":"2024-10-23T02:03:41.670641Z"},"trusted":true}},{"cell_type":"markdown","source":"# Check for duplicate IDs between train_df and test_df\nduplicate_ids_between = set(train_df['id']).intersection(set(test_df['id']))\nif len(duplicate_ids_between) == 0:\n    print(\"No duplicate IDs found between train_df and test_df.\")\nelse:\n    print(f\"Duplicate IDs found between train_df and test_df: {len(duplicate_ids_between)}\")","metadata":{"_uuid":"235a1f88-56aa-4ab0-89b8-01b9c20d038c","_cell_guid":"956f88da-6d3f-49f1-b6fb-cb777639b08f","execution":{"iopub.status.busy":"2024-10-23T01:50:00.382985Z","iopub.execute_input":"2024-10-23T01:50:00.383387Z","iopub.status.idle":"2024-10-23T01:50:00.391214Z","shell.execute_reply.started":"2024-10-23T01:50:00.383348Z","shell.execute_reply":"2024-10-23T01:50:00.389959Z"},"trusted":true}},{"cell_type":"markdown","source":"# Remove duplicate IDs from test_df that are present in train_df\nduplicate_ids_between = set(train_df['id']).intersection(set(test_df['id']))\nif len(duplicate_ids_between) > 0:\n    test_df = test_df[~test_df['id'].isin(duplicate_ids_between)]\n    print(f\"Removed {len(duplicate_ids_between)} duplicate IDs from test_df that were present in train_df.\")","metadata":{"_uuid":"ec55a6e7-1c81-4d9d-a116-7e6cc82c9883","_cell_guid":"d327d15e-5639-415c-bfe6-936301fcf248","execution":{"iopub.status.busy":"2024-10-23T01:50:00.392746Z","iopub.execute_input":"2024-10-23T01:50:00.393335Z","iopub.status.idle":"2024-10-23T01:50:00.406901Z","shell.execute_reply.started":"2024-10-23T01:50:00.393272Z","shell.execute_reply":"2024-10-23T01:50:00.405710Z"},"trusted":true}},{"cell_type":"markdown","source":"# Remove duplicate IDs from train_df that are present in test_df\nduplicate_ids_between = set(train_df['id']).intersection(set(test_df['id']))\nif len(duplicate_ids_between) > 0:\n    train_df = train_df[~train_df['id'].isin(duplicate_ids_between)]\n    print(f\"Removed {len(duplicate_ids_between)} duplicate IDs from train_df that were present in test_df.\")","metadata":{"_uuid":"fb808095-cca7-4976-8d5f-08098513cd13","_cell_guid":"753096cb-fa48-457d-ac38-27553f883c2d","trusted":true}},{"cell_type":"code","source":"# Prepare data for modeling\nX = train_df.drop(columns=['sii', 'id'])  # Features\ny = train_df['sii']  # Target\n\n# Perform k-fold cross-validation\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\ninitial_accuracies = []\nall_classification_reports = []\nall_confusion_matrices = []\n\nfor train_index, test_index in kf.split(X):\n    X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n    y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n\n    # Train and evaluate a Random Forest model\n    rf_model = RandomForestClassifier(random_state=42)\n    rf_model.fit(X_train, y_train)\n    y_pred = rf_model.predict(X_test)\n\n    # Evaluate the model\n    accuracy = accuracy_score(y_test, y_pred)\n    initial_accuracies.append(accuracy)\n    classification_rep = classification_report(y_test, y_pred, output_dict=True)\n    all_classification_reports.append(classification_rep)\n    confusion_mat = confusion_matrix(y_test, y_pred)\n    all_confusion_matrices.append(confusion_mat)\n\n# Print average accuracies\nprint(\"Initial Model Evaluation:\")\nprint(f\"Average Accuracy: {np.mean(initial_accuracies):.4f}\")\n\n# Print classification reports and confusion matrices\nfor i, (report, matrix) in enumerate(zip(all_classification_reports, all_confusion_matrices)):\n    print(f\"Fold {i // 2 + 1} {'Initial' if i % 2 == 0 else 'Normalized'} Classification Report:\")\n    print(pd.DataFrame(report).transpose())\n    print(f\"Fold {i // 2 + 1} {'Initial' if i % 2 == 0 else 'Normalized'} Confusion Matrix:\")\n    print(matrix)\n    print(\"-\" * 50)\n\n# Feature importance\nrf_model.fit(X, y)\nfeature_importances = pd.Series(rf_model.feature_importances_, index=X.columns).sort_values(ascending=False)\nprint(\"Feature Importances:\")\nprint(feature_importances)\n\n# Plot feature importances\nplt.figure(figsize=(10, 6))\nfeature_importances.head(10).plot(kind='barh')\nplt.title('Top 10 Feature Importances')\nplt.xlabel('Importance')\nplt.ylabel('Feature')\nplt.show()","metadata":{"_uuid":"52cfd213-7ecb-4c61-8b2a-9f140ed84037","_cell_guid":"81402d74-8148-46e2-ac60-8873e30b066e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:30.762381Z","iopub.execute_input":"2024-10-24T16:24:30.763111Z","iopub.status.idle":"2024-10-24T16:24:35.395175Z","shell.execute_reply.started":"2024-10-24T16:24:30.763063Z","shell.execute_reply":"2024-10-24T16:24:35.394070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to align test_df with train_df to ensure same columns after preprocessing\ndef align_test_with_train(train_df, test_df):\n    missing_cols = set(train_df.columns) - set(test_df.columns)\n    for col in missing_cols:\n        test_df[col] = 0\n    test_df = test_df[train_df.columns]\n    return test_df\n\n# Apply the same manipulations to test_df as train_df\nif 'test_df' in locals():\n    # Remove outliers from test data (using z-scores calculated on train data)\n    numeric_columns = [col for col in test_df.select_dtypes(include=[np.number]).columns.difference(['id', 'Basic_Demos-Age']) if not test_df[col].dropna().isin([0, 1, 2, 3, 4, 5]).all()]\n    z_scores = np.abs(stats.zscore(test_df[numeric_columns], nan_policy='omit'))\n    outliers = (z_scores >= 3)\n    outliers_indices = outliers.any(axis=1)\n    test_df = test_df.loc[~outliers_indices]\n\n    # Set median value for columns with zeros in specific features\n    columns_with_zeros = ['Physical-Weight', 'FGC-FGC_CU', 'FGC-FGC_PU', 'FGC-FGC_GSD', 'FGC-FGC_GSND']\n    for col in columns_with_zeros:\n        if col in test_df.columns:\n            median_value = test_df[col].replace(0, np.nan).median()\n            test_df[col] = test_df[col].replace(0, median_value)\n    \n    # One-hot encode categorical columns and handle missing values for test data\n    object_columns = test_df.select_dtypes(include='object').columns.difference(['id', 'PCIAT-Season'])\n    for column in object_columns:\n        if column in test_df.columns:\n            # One-hot encode the column\n            if test_df[column].isna().any():\n                dummies = pd.get_dummies(test_df[column], prefix=column, dummy_na=True).rename(columns=lambda x: x.replace('nan', 'na'))\n            else:\n                dummies = pd.get_dummies(test_df[column], prefix=column, dummy_na=False)\n            test_df = test_df.join(dummies)\n            test_df.drop(column, axis=1, inplace=True)\n\n    # Align columns of test_df with train_df to ensure they match\n    test_df = align_test_with_train(train_df, test_df)\n\n    # Ensure data types match between train_df and test_df for boolean columns\n    bool_columns = train_df.select_dtypes(include='bool').columns\n    for col in bool_columns:\n        if col in test_df.columns:\n            test_df[col] = test_df[col].astype(bool)\n\n# Apply the same imputation to test_df as to train_df\ntest_numeric_columns = [col for col in test_df.select_dtypes(include=[np.float64]).columns.difference(['id', 'Basic_Demos-Age']) if not test_df[col].dropna().isin([0, 1, 2, 3, 4, 5]).all()]\ntest_df[test_numeric_columns] = imputer.transform(test_df[test_numeric_columns])\n\n# Align data types between train_df and test_df\ntrain_df['sii'] = train_df['sii'].astype('int64')\n\nfor column in ['BIA-Season_na', 'CGAS-Season_na', 'FGC-Season_na', 'Fitness_Endurance-Season_na', 'Physical-Season_na', 'PreInt_EduHx-Season_na', 'SDS-Season_na']:\n    if column in train_df.columns and column in test_df.columns:\n        train_df[column] = train_df[column].astype('int64')\n        test_df[column] = test_df[column].astype('int64')\n\n# Check compatibility between train_df and test_df\nif 'train_df' in locals() and 'test_df' in locals():\n    # Check if columns match\n    if list(train_df.columns) == list(test_df.columns):\n        print(\"train_df and test_df have matching columns.\")\n    else:\n        print(\"train_df and test_df do NOT have matching columns.\")\n        print(\"Columns in train_df but not in test_df:\", set(train_df.columns) - set(test_df.columns))\n        print(\"Columns in test_df but not in train_df:\", set(test_df.columns) - set(train_df.columns))\n\n    # Check data types\n    type_mismatch = False\n    for col in train_df.columns:\n        if col in test_df.columns and train_df[col].dtype != test_df[col].dtype:\n            print(f\"Data type mismatch in column '{col}': train_df type = {train_df[col].dtype}, test_df type = {test_df[col].dtype}\")\n            type_mismatch = True\n    if not type_mismatch:\n        print(\"All columns have matching data types between train_df and test_df.\")","metadata":{"_uuid":"4abcf04e-096d-495c-a16d-52e25d3489ad","_cell_guid":"c80b8cef-2b42-4767-b8e4-7185b5a05511","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:35.396367Z","iopub.execute_input":"2024-10-24T16:24:35.396712Z","iopub.status.idle":"2024-10-24T16:24:35.564169Z","shell.execute_reply.started":"2024-10-24T16:24:35.396678Z","shell.execute_reply":"2024-10-24T16:24:35.562957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ensure test_df has the same columns as train_df\nexpected_features = train_df.drop(columns=['sii', 'id']).columns\nmissing_in_test = set(expected_features) - set(X_test.columns)\nextra_in_test = set(X_test.columns) - set(expected_features)\n\nif missing_in_test:\n    print(f\"Columns missing in test_df that are present in train_df: {missing_in_test}\")\nif extra_in_test:\n    print(f\"Extra columns in test_df that are not in train_df: {extra_in_test}\")\n\n# Ensure there are no NaN values in test features\nif X_test.isnull().sum().sum() > 0:\n    print(\"Warning: X_test contains NaN values after preprocessing.\")\n\n# Ensure all columns are numeric\nnon_numeric_cols = X_test.select_dtypes(exclude=[np.number]).columns\nif len(non_numeric_cols) > 0:\n    print(f\"Warning: X_test contains non-numeric columns: {non_numeric_cols}\")\n\n# Remove any erroneous columns that shouldn't be in X_test\nX_test = X_test[expected_features]","metadata":{"_uuid":"1af351d2-c1c9-4a1b-8825-6def2a55a91d","_cell_guid":"812843f5-89b0-4e32-a992-9fbbb1a43521","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:35.570838Z","iopub.execute_input":"2024-10-24T16:24:35.571338Z","iopub.status.idle":"2024-10-24T16:24:35.587283Z","shell.execute_reply.started":"2024-10-24T16:24:35.571287Z","shell.execute_reply":"2024-10-24T16:24:35.586175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Explicitly convert all one-hot encoded columns to int64 to ensure consistency\none_hot_columns = [\n    'PAQ_A_Season_Fall', 'PAQ_A_Season_Spring', 'PAQ_A_Season_Summer',\n    'PAQ_A_Season_Winter', 'PAQ_A_Season_na', 'PAQ_C_Season_Fall',\n    'PAQ_C_Season_Spring', 'PAQ_C_Season_Summer', 'PAQ_C_Season_Winter',\n    'PAQ_C_Season_na', 'BIA-Season_Fall', 'BIA-Season_Spring',\n    'BIA-Season_Summer', 'BIA-Season_Winter',\n    'Basic_Demos-Enroll_Season_Fall', 'Basic_Demos-Enroll_Season_Spring',\n    'Basic_Demos-Enroll_Season_Summer', 'Basic_Demos-Enroll_Season_Winter',\n    'CGAS-Season_Fall', 'CGAS-Season_Spring', 'CGAS-Season_Summer',\n    'CGAS-Season_Winter', 'FGC-Season_Fall', 'FGC-Season_Spring',\n    'FGC-Season_Summer', 'FGC-Season_Winter',\n    'Fitness_Endurance-Season_Fall', 'Fitness_Endurance-Season_Spring',\n    'Fitness_Endurance-Season_Summer', 'Fitness_Endurance-Season_Winter',\n    'Physical-Season_Fall', 'Physical-Season_Spring',\n    'Physical-Season_Summer', 'Physical-Season_Winter',\n    'PreInt_EduHx-Season_Fall', 'PreInt_EduHx-Season_Spring',\n    'PreInt_EduHx-Season_Summer', 'PreInt_EduHx-Season_Winter',\n    'SDS-Season_Fall', 'SDS-Season_Spring', 'SDS-Season_Summer',\n    'SDS-Season_Winter', 'BIA-BIA_Activity_Level_num_1.0',\n    'BIA-BIA_Activity_Level_num_2.0', 'BIA-BIA_Activity_Level_num_3.0',\n    'BIA-BIA_Activity_Level_num_4.0', 'BIA-BIA_Activity_Level_num_5.0',\n    'BIA-BIA_Activity_Level_num_na', 'BIA-BIA_Frame_num_1.0',\n    'BIA-BIA_Frame_num_2.0', 'BIA-BIA_Frame_num_3.0',\n    'BIA-BIA_Frame_num_na', 'FGC-FGC_PU_Zone_0.0', 'FGC-FGC_PU_Zone_1.0',\n    'FGC-FGC_PU_Zone_na', 'FGC-FGC_SRL_Zone_0.0', 'FGC-FGC_SRL_Zone_1.0',\n    'FGC-FGC_SRL_Zone_na', 'FGC-FGC_SRR_Zone_0.0', 'FGC-FGC_SRR_Zone_1.0',\n    'FGC-FGC_SRR_Zone_na', 'FGC-FGC_CU_Zone_0.0', 'FGC-FGC_CU_Zone_1.0',\n    'FGC-FGC_CU_Zone_na', 'FGC-FGC_TL_Zone_0.0', 'FGC-FGC_TL_Zone_1.0',\n    'FGC-FGC_TL_Zone_na', 'PreInt_EduHx-computerinternet_hoursday_0.0',\n    'PreInt_EduHx-computerinternet_hoursday_1.0',\n    'PreInt_EduHx-computerinternet_hoursday_2.0',\n    'PreInt_EduHx-computerinternet_hoursday_3.0',\n    'PreInt_EduHx-computerinternet_hoursday_na'\n]\n\n# Convert all one-hot encoded columns in both train_df and test_df to int64\nfor column in one_hot_columns:\n    if column in train_df.columns and column in test_df.columns:\n        train_df[column] = train_df[column].astype('int64')\n        test_df[column] = test_df[column].astype('int64')\n\n# Confirm all features in X_test are numeric\nX_test = test_df.drop(columns=['id', 'sii'], errors='ignore')\nnon_numeric_cols = X_test.select_dtypes(exclude=[np.number]).columns\n\nif len(non_numeric_cols) > 0:\n    print(f\"Warning: X_test still contains non-numeric columns: {non_numeric_cols}\")\nelse:\n    print(\"All columns in X_test are numeric.\")","metadata":{"_uuid":"9d06bf50-4aa9-417b-96dd-4932d3390774","_cell_guid":"42c030e3-bb95-4aab-a8e7-bb3984685c14","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:35.589157Z","iopub.execute_input":"2024-10-24T16:24:35.589951Z","iopub.status.idle":"2024-10-24T16:24:35.654778Z","shell.execute_reply.started":"2024-10-24T16:24:35.589902Z","shell.execute_reply":"2024-10-24T16:24:35.653790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make predictions on test_df\nX_test = test_df.drop(columns=['id', 'sii'], errors='ignore')\ntest_df.loc[:, 'sii'] = rf_model.predict(X_test).astype(int)\n\n# Prepare submission file\nsubmission = test_df[['id', 'sii']].copy()\nsubmission['id'] = submission['id'].astype(str)\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Submission file 'submission.csv' created successfully.\")\n\n# Validation checks\nprint(\"Validation Checks:\")\n\n# Check row count\nexpected_rows = test_df.shape[0]\nsubmission_rows = submission.shape[0]\nif expected_rows == submission_rows:\n    print(f\"Row count check passed: {submission_rows} rows.\")\nelse:\n    print(f\"Row count mismatch: expected {expected_rows}, but got {submission_rows}.\")\n\n# Check for missing values in 'sii'\nmissing_values = submission['sii'].isnull().sum()\nif missing_values == 0:\n    print(\"No missing values in 'sii' column.\")\nelse:\n    print(f\"Missing values found in 'sii' column: {missing_values}\")\n\n# Check ID consistency\nmissing_ids = set(test_df['id'].astype(str)) - set(submission['id'])\nif len(missing_ids) == 0:\n    print(\"All IDs from test set are present in the submission.\")\nelse:\n    print(f\"Missing IDs in submission: {missing_ids}\")","metadata":{"_uuid":"b2590151-3353-4dae-9110-877014e7aab7","_cell_guid":"31b1e7b9-0340-4c0b-947f-5e2c2b64250d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-24T16:24:35.656046Z","iopub.execute_input":"2024-10-24T16:24:35.656383Z","iopub.status.idle":"2024-10-24T16:24:35.686593Z","shell.execute_reply.started":"2024-10-24T16:24:35.656344Z","shell.execute_reply":"2024-10-24T16:24:35.685625Z"},"trusted":true},"execution_count":null,"outputs":[]}]}