{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.307326Z","iopub.execute_input":"2024-12-10T07:38:57.307743Z","iopub.status.idle":"2024-12-10T07:38:57.312635Z","shell.execute_reply.started":"2024-12-10T07:38:57.307706Z","shell.execute_reply":"2024-12-10T07:38:57.311431Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Know Your Data","metadata":{}},{"cell_type":"code","source":"dftrain1=pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ndftest1=pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.314178Z","iopub.execute_input":"2024-12-10T07:38:57.314544Z","iopub.status.idle":"2024-12-10T07:38:57.364515Z","shell.execute_reply.started":"2024-12-10T07:38:57.314512Z","shell.execute_reply":"2024-12-10T07:38:57.363488Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dftrain1.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.366730Z","iopub.execute_input":"2024-12-10T07:38:57.367225Z","iopub.status.idle":"2024-12-10T07:38:57.395182Z","shell.execute_reply.started":"2024-12-10T07:38:57.367174Z","shell.execute_reply":"2024-12-10T07:38:57.393851Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dftest1.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.396499Z","iopub.execute_input":"2024-12-10T07:38:57.396844Z","iopub.status.idle":"2024-12-10T07:38:57.422907Z","shell.execute_reply.started":"2024-12-10T07:38:57.396812Z","shell.execute_reply":"2024-12-10T07:38:57.421776Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dftrain1.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.425848Z","iopub.execute_input":"2024-12-10T07:38:57.426318Z","iopub.status.idle":"2024-12-10T07:38:57.445636Z","shell.execute_reply.started":"2024-12-10T07:38:57.426270Z","shell.execute_reply":"2024-12-10T07:38:57.444442Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dftrain1.id.nunique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.447134Z","iopub.execute_input":"2024-12-10T07:38:57.447592Z","iopub.status.idle":"2024-12-10T07:38:57.455800Z","shell.execute_reply.started":"2024-12-10T07:38:57.447543Z","shell.execute_reply":"2024-12-10T07:38:57.454764Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dftest1.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.457059Z","iopub.execute_input":"2024-12-10T07:38:57.457488Z","iopub.status.idle":"2024-12-10T07:38:57.476680Z","shell.execute_reply.started":"2024-12-10T07:38:57.457454Z","shell.execute_reply":"2024-12-10T07:38:57.475568Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dict = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.477954Z","iopub.execute_input":"2024-12-10T07:38:57.478318Z","iopub.status.idle":"2024-12-10T07:38:57.494865Z","shell.execute_reply.started":"2024-12-10T07:38:57.478277Z","shell.execute_reply":"2024-12-10T07:38:57.493832Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.496104Z","iopub.execute_input":"2024-12-10T07:38:57.496479Z","iopub.status.idle":"2024-12-10T07:38:57.522021Z","shell.execute_reply.started":"2024-12-10T07:38:57.496445Z","shell.execute_reply":"2024-12-10T07:38:57.520820Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{}},{"cell_type":"code","source":"train1=dftrain1.copy()\ntest1=dftest1.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.523637Z","iopub.execute_input":"2024-12-10T07:38:57.523951Z","iopub.status.idle":"2024-12-10T07:38:57.533345Z","shell.execute_reply.started":"2024-12-10T07:38:57.523920Z","shell.execute_reply":"2024-12-10T07:38:57.532576Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train1.duplicated().any()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.536650Z","iopub.execute_input":"2024-12-10T07:38:57.536961Z","iopub.status.idle":"2024-12-10T07:38:57.566799Z","shell.execute_reply.started":"2024-12-10T07:38:57.536930Z","shell.execute_reply":"2024-12-10T07:38:57.565779Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display column summary for dftrain\nsummary = train1.isnull().sum().reset_index()\nsummary.columns = ['Column Name', 'Null Values']\nsummary['Data Type'] = train1.dtypes.values\nsummary['Unique Values'] = train1.nunique().values\nsummary['Percentage of Null Values'] = (summary['Null Values'] / len(train1)) * 100\n\n# Rearrange columns for better readability\nsummary = summary[['Column Name', 'Data Type', 'Unique Values', 'Null Values', 'Percentage of Null Values']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.568179Z","iopub.execute_input":"2024-12-10T07:38:57.568908Z","iopub.status.idle":"2024-12-10T07:38:57.597092Z","shell.execute_reply.started":"2024-12-10T07:38:57.568862Z","shell.execute_reply":"2024-12-10T07:38:57.596185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.set_option('display.max_rows', None)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.598357Z","iopub.execute_input":"2024-12-10T07:38:57.598701Z","iopub.status.idle":"2024-12-10T07:38:57.603856Z","shell.execute_reply.started":"2024-12-10T07:38:57.598668Z","shell.execute_reply":"2024-12-10T07:38:57.602700Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.605252Z","iopub.execute_input":"2024-12-10T07:38:57.605678Z","iopub.status.idle":"2024-12-10T07:38:57.630457Z","shell.execute_reply.started":"2024-12-10T07:38:57.605630Z","shell.execute_reply":"2024-12-10T07:38:57.629333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define threshold for missing values\nthreshold = 0.5\n\n# Calculate percentage of missing values in train1 dataset\nmissing_percentage_train = train1.isnull().mean()\n\n# Identify columns to be removed (more than 50% missing values)\ncolumns_to_remove = missing_percentage_train[missing_percentage_train > threshold].index.tolist()\n\n# Remove these columns from train1\ntrain1_filtered = train1.drop(columns=columns_to_remove)\n\n# Remove the same columns from test1 if they exist\ncommon_columns_to_remove = [col for col in columns_to_remove if col in test1.columns]\ntest1_filtered = test1.drop(columns=common_columns_to_remove)\n\n# Output the list of columns to be removed and shapes of the datasets\nprint(f\"Columns removed from train1: {columns_to_remove}\")\nprint(f\"Columns removed from test1: {common_columns_to_remove}\")\nprint(f\"Train1 shape after filtering: {train1_filtered.shape}\")\nprint(f\"Test1 shape after filtering: {test1_filtered.shape}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.631868Z","iopub.execute_input":"2024-12-10T07:38:57.632243Z","iopub.status.idle":"2024-12-10T07:38:57.648677Z","shell.execute_reply.started":"2024-12-10T07:38:57.632205Z","shell.execute_reply":"2024-12-10T07:38:57.647577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Separate the target variable\ntarget_column = 'sii'\ntrain_features = train1_filtered.drop(columns=[target_column])\n\n# Get column sets\ntrain_columns = set(train_features.columns)\ntest_columns = set(test1_filtered.columns)\n\n# Identify column groups (exclude 'sii')\ncommon_columns = train_columns.intersection(test_columns)\ntrain_only_columns = train_columns - test_columns\ntest_only_columns = test_columns - train_columns\n\n# Print results\nprint(\"Common Columns:\", common_columns)\nprint(\"Train-Only Columns:\", train_only_columns)\nprint(\"Test-Only Columns:\", test_only_columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.650113Z","iopub.execute_input":"2024-12-10T07:38:57.650613Z","iopub.status.idle":"2024-12-10T07:38:57.659869Z","shell.execute_reply.started":"2024-12-10T07:38:57.650562Z","shell.execute_reply":"2024-12-10T07:38:57.658663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train1_filtered.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.661321Z","iopub.execute_input":"2024-12-10T07:38:57.661820Z","iopub.status.idle":"2024-12-10T07:38:57.686051Z","shell.execute_reply.started":"2024-12-10T07:38:57.661718Z","shell.execute_reply":"2024-12-10T07:38:57.685085Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test1_filtered.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.687422Z","iopub.execute_input":"2024-12-10T07:38:57.688117Z","iopub.status.idle":"2024-12-10T07:38:57.700438Z","shell.execute_reply.started":"2024-12-10T07:38:57.688067Z","shell.execute_reply":"2024-12-10T07:38:57.699334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Find common columns\ncommon_columns = train1_filtered.columns.intersection(test1_filtered.columns)\n\n# Find columns only in imputed_train\ntrain_only_columns = train1_filtered.columns.difference(test1_filtered.columns)\n\n# Find columns only in imputed_test\ntest_only_columns = test1_filtered.columns.difference(train1_filtered.columns)\n\n# Display the results\nprint(\"Common Columns:\", common_columns.tolist())\nprint(\"Train Only Columns:\", train_only_columns.tolist())\nprint(\"Test Only Columns:\", test_only_columns.tolist())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.701806Z","iopub.execute_input":"2024-12-10T07:38:57.702202Z","iopub.status.idle":"2024-12-10T07:38:57.715448Z","shell.execute_reply.started":"2024-12-10T07:38:57.702156Z","shell.execute_reply":"2024-12-10T07:38:57.714281Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Columns present in both train and test datasets (common columns)\ncommon_columns = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex', \n                  'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI', \n                  'Physical-Height', 'Physical-Weight', 'Physical-Diastolic_BP', \n                  'Physical-HeartRate', 'Physical-Systolic_BP', 'FGC-Season', 'FGC-FGC_CU', \n                  'FGC-FGC_CU_Zone', 'FGC-FGC_PU', 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', \n                  'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR', 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', \n                  'FGC-FGC_TL_Zone', 'BIA-Season', 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', \n                  'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM', \n                  'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num', 'BIA-BIA_ICW', \n                  'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW', 'SDS-Season', \n                  'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T', 'PreInt_EduHx-Season', \n                  'PreInt_EduHx-computerinternet_hoursday']\n\n# Ensure the target column 'sii' remains in the train dataset\ntrain_columns_to_keep = common_columns + ['sii']\n\n# Drop columns not present in the test dataset from the train dataset (except 'sii')\ntrain1_filtered = train1_filtered[train_columns_to_keep]\n\n# Ensure test dataset has only the common columns\ntest1_filtered = test1_filtered[common_columns]\n\n# Output the shape to confirm alignment\nprint(f\"Train shape: {train1_filtered.shape}\")\nprint(f\"Test shape: {test1_filtered.shape}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.716813Z","iopub.execute_input":"2024-12-10T07:38:57.717244Z","iopub.status.idle":"2024-12-10T07:38:57.731020Z","shell.execute_reply.started":"2024-12-10T07:38:57.717194Z","shell.execute_reply":"2024-12-10T07:38:57.729975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from scipy.stats import normaltest\nfrom sklearn.impute import KNNImputer","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.732402Z","iopub.execute_input":"2024-12-10T07:38:57.732724Z","iopub.status.idle":"2024-12-10T07:38:57.741672Z","shell.execute_reply.started":"2024-12-10T07:38:57.732693Z","shell.execute_reply":"2024-12-10T07:38:57.740731Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import warnings\n\n# Suppress warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.743008Z","iopub.execute_input":"2024-12-10T07:38:57.743326Z","iopub.status.idle":"2024-12-10T07:38:57.753595Z","shell.execute_reply.started":"2024-12-10T07:38:57.743295Z","shell.execute_reply":"2024-12-10T07:38:57.752645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to check if a column follows a normal distribution\ndef is_normal_distribution(column):\n    _, p_value = normaltest(column.dropna())  # Perform D'Agostino and Pearson's test\n    return p_value > 0.05  # If p-value > 0.05, assume normal distribution\n\n# Separate target variable\ntarget = 'sii'\n\n# Identify numerical and categorical columns\nnumerical_cols = train1_filtered.select_dtypes(include=['float64', 'int64']).columns.difference([target])\ncategorical_cols = train1_filtered.select_dtypes(include=['object']).columns\n\n# Imputation for numerical columns\nimputed_train = train1_filtered.copy()\nimputed_test = test1_filtered.copy()\n\nfor col in numerical_cols:\n    if is_normal_distribution(train1_filtered[col]):\n        # Use median imputation for normal distribution\n        median_value = train1_filtered[col].median()\n        imputed_train[col].fillna(median_value, inplace=True)\n        if col in test1_filtered.columns:\n            imputed_test[col].fillna(median_value, inplace=True)\n    else:\n        # Use KNN imputation for non-normal distribution\n        knn_imputer = KNNImputer(n_neighbors=3)\n        # Reshape for KNN and handle only the column\n        knn_train_values = train1_filtered[[col]].values\n        imputed_train[col] = knn_imputer.fit_transform(knn_train_values)\n        if col in test1_filtered.columns:\n            knn_test_values = test1_filtered[[col]].values\n            imputed_test[col] = knn_imputer.transform(knn_test_values)\n\n# Imputation for categorical columns\nfor col in categorical_cols:\n    mode_value = train1_filtered[col].mode()[0]  # Get the most frequent value\n    imputed_train[col].fillna(mode_value, inplace=True)\n    if col in test1_filtered.columns:\n        imputed_test[col].fillna(mode_value, inplace=True)\n\n# Resulting datasets\nprint(\"Imputation completed.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:38:57.754697Z","iopub.execute_input":"2024-12-10T07:38:57.755013Z","iopub.status.idle":"2024-12-10T07:39:08.885563Z","shell.execute_reply.started":"2024-12-10T07:38:57.754982Z","shell.execute_reply":"2024-12-10T07:39:08.884307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imputed_train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:39:08.886842Z","iopub.execute_input":"2024-12-10T07:39:08.887251Z","iopub.status.idle":"2024-12-10T07:39:08.902840Z","shell.execute_reply.started":"2024-12-10T07:39:08.887216Z","shell.execute_reply":"2024-12-10T07:39:08.901888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imputed_test.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:39:08.904171Z","iopub.execute_input":"2024-12-10T07:39:08.904507Z","iopub.status.idle":"2024-12-10T07:39:08.928067Z","shell.execute_reply.started":"2024-12-10T07:39:08.904476Z","shell.execute_reply":"2024-12-10T07:39:08.926970Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Drop 'id' column from both train and test datasets\ntrain = imputed_train.drop(columns=['id'], errors='ignore')\ntest = imputed_test.drop(columns=['id'], errors='ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:39:08.929495Z","iopub.execute_input":"2024-12-10T07:39:08.929961Z","iopub.status.idle":"2024-12-10T07:39:08.945826Z","shell.execute_reply.started":"2024-12-10T07:39:08.929911Z","shell.execute_reply":"2024-12-10T07:39:08.944795Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train.shape)\nprint(test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:39:08.946944Z","iopub.execute_input":"2024-12-10T07:39:08.947243Z","iopub.status.idle":"2024-12-10T07:39:08.954974Z","shell.execute_reply.started":"2024-12-10T07:39:08.947210Z","shell.execute_reply":"2024-12-10T07:39:08.954058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.cluster import KMeans\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom scipy.stats import mode","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:39:08.956045Z","iopub.execute_input":"2024-12-10T07:39:08.956392Z","iopub.status.idle":"2024-12-10T07:39:08.968295Z","shell.execute_reply.started":"2024-12-10T07:39:08.956339Z","shell.execute_reply":"2024-12-10T07:39:08.967284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 1: Check which categorical columns exist in the train data\ncategorical_columns = train.select_dtypes(include='object').columns\n\n# Ensure that the columns exist in train and test data\ntrain_categorical_columns = [col for col in categorical_columns if col in train.columns]\ntest_categorical_columns = [col for col in categorical_columns if col in test.columns]\n\n# Step 2: Apply One-Hot Encoding to train and test data\nohe = OneHotEncoder(sparse=False, handle_unknown='ignore')\n\n# Fit on train categorical columns\ntrain_encoded_categorical = ohe.fit_transform(train[train_categorical_columns])\ntest_encoded_categorical = ohe.transform(test[test_categorical_columns])\n\n# Convert to DataFrames\ntrain_encoded_df = pd.DataFrame(train_encoded_categorical, columns=ohe.get_feature_names_out(train_categorical_columns))\ntest_encoded_df = pd.DataFrame(test_encoded_categorical, columns=ohe.get_feature_names_out(test_categorical_columns))\n\n# Step 3: Merge encoded features back with numerical features (excluding categorical columns and 'sii' in train data)\ntrain_encoded = pd.concat([train.drop(columns=train_categorical_columns + ['sii'], errors='ignore').reset_index(drop=True),\n                           train_encoded_df,\n                           train['sii'].reset_index(drop=True)], axis=1)\n\ntest_encoded = pd.concat([test.drop(columns=test_categorical_columns, errors='ignore').reset_index(drop=True),\n                          test_encoded_df], axis=1)\n\n# Step 4: Scale train and test data\nscaler = StandardScaler()\n\n# Scale all features excluding 'sii' in train data\ntrain_features = [col for col in train_encoded.columns if col != 'sii']\ntrain_scaled = scaler.fit_transform(train_encoded[train_features])\ntrain_scaled_df = pd.DataFrame(train_scaled, columns=train_features)\n\n# Scale all features in test data\ntest_scaled = scaler.transform(test_encoded)\ntest_scaled_df = pd.DataFrame(test_scaled, columns=test_encoded.columns)\n\n# --- Outputs ---\nprint(\"Final train data shape:\", train_scaled_df.shape)\nprint(\"Scaled test data shape:\", test_scaled_df.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:39:08.973559Z","iopub.execute_input":"2024-12-10T07:39:08.974326Z","iopub.status.idle":"2024-12-10T07:39:09.017070Z","shell.execute_reply.started":"2024-12-10T07:39:08.974286Z","shell.execute_reply":"2024-12-10T07:39:09.016057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_scaled_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:40:55.977678Z","iopub.execute_input":"2024-12-10T07:40:55.978061Z","iopub.status.idle":"2024-12-10T07:40:56.000437Z","shell.execute_reply.started":"2024-12-10T07:40:55.978026Z","shell.execute_reply":"2024-12-10T07:40:55.999391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add 'sii' column back to the scaled train data\ntrain_scaled_df['sii'] = train['sii'].reset_index(drop=True)\n\n# --- Outputs ---\nprint(\"Final train data shape with 'sii' added:\", train_scaled_df.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:43:32.776257Z","iopub.execute_input":"2024-12-10T07:43:32.776684Z","iopub.status.idle":"2024-12-10T07:43:32.783463Z","shell.execute_reply.started":"2024-12-10T07:43:32.776650Z","shell.execute_reply":"2024-12-10T07:43:32.782362Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Checking Class Distribution**","metadata":{}},{"cell_type":"code","source":"from imblearn.over_sampling import SMOTE\nfrom collections import Counter","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T08:26:20.020192Z","iopub.execute_input":"2024-12-10T08:26:20.020577Z","iopub.status.idle":"2024-12-10T08:26:20.180230Z","shell.execute_reply.started":"2024-12-10T08:26:20.020544Z","shell.execute_reply":"2024-12-10T08:26:20.179386Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 1: Remove rows with missing 'sii'\ntrain_scaled_df = train_scaled_df.dropna(subset=['sii'])\nprint(f\"Dataset size after removing missing 'sii': {train_scaled_df.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T08:26:56.700445Z","iopub.execute_input":"2024-12-10T08:26:56.701520Z","iopub.status.idle":"2024-12-10T08:26:56.710110Z","shell.execute_reply.started":"2024-12-10T08:26:56.701475Z","shell.execute_reply":"2024-12-10T08:26:56.708971Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 2: Check class distribution\nclass_distribution = train_scaled_df['sii'].value_counts()\nprint(\"Class distribution before SMOTE:\")\nprint(class_distribution)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T08:27:13.158892Z","iopub.execute_input":"2024-12-10T08:27:13.159241Z","iopub.status.idle":"2024-12-10T08:27:13.166667Z","shell.execute_reply.started":"2024-12-10T08:27:13.159209Z","shell.execute_reply":"2024-12-10T08:27:13.165664Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Applying SMOTE**","metadata":{}},{"cell_type":"code","source":"# Step 3: Separate features and target\nX = train_scaled_df.drop(columns=['sii'])  # Features\ny = train_scaled_df['sii'].astype(int)     # Target (as integers)\n\n# Step 4: Apply SMOTE to balance the classes\nsmote = SMOTE(random_state=42)\nX_resampled, y_resampled = smote.fit_resample(X, y)\n\n# Step 5: Check class distribution after SMOTE\nresampled_distribution = Counter(y_resampled)\nprint(\"Class distribution after SMOTE:\")\nprint(resampled_distribution)\n\n# Data is now ready for modeling\nprint(f\"Resampled dataset shape: {X_resampled.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T08:34:10.203098Z","iopub.execute_input":"2024-12-10T08:34:10.203512Z","iopub.status.idle":"2024-12-10T08:34:10.250267Z","shell.execute_reply.started":"2024-12-10T08:34:10.203479Z","shell.execute_reply":"2024-12-10T08:34:10.249161Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Classification Models & Quadratic Weighted Kappa ","metadata":{}},{"cell_type":"code","source":"# Import necessary libraries\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom xgboost import XGBClassifier","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T08:36:58.796513Z","iopub.execute_input":"2024-12-10T08:36:58.797557Z","iopub.status.idle":"2024-12-10T08:36:58.996839Z","shell.execute_reply.started":"2024-12-10T08:36:58.797512Z","shell.execute_reply":"2024-12-10T08:36:58.995814Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 1: Define the models\nmodels = {\n    \"Random Forest\": RandomForestClassifier(random_state=42),\n    \"Logistic Regression\": LogisticRegression(max_iter=1000, random_state=42, multi_class='multinomial'),\n    \"XGBoost\": XGBClassifier(use_label_encoder=False, eval_metric='mlogloss', random_state=42)\n}\n\n# Step 2: Initialize Stratified K-Fold Cross-Validation\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\nresults = {}\n\n# Step 3: Evaluate each model\nfor name, model in models.items():\n    print(f\"Evaluating {name}...\")\n    kappas = []\n    \n    for train_index, test_index in skf.split(X_resampled, y_resampled):\n        X_train, X_test = X_resampled.iloc[train_index], X_resampled.iloc[test_index]\n        y_train, y_test = y_resampled.iloc[train_index], y_resampled.iloc[test_index]\n        \n        # Train the model\n        model.fit(X_train, y_train)\n        \n        # Make predictions\n        y_pred = model.predict(X_test)\n        \n        # Compute Quadratic Weighted Kappa\n        kappa = cohen_kappa_score(y_test, y_pred, weights='quadratic')\n        kappas.append(kappa)\n    \n    # Store average kappa score\n    results[name] = np.mean(kappas)\n\n# Step 4: Display results\nprint(\"\\nModel Evaluation Results:\")\nfor name, kappa in results.items():\n    print(f\"{name}: Quadratic Weighted Kappa = {kappa:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T08:37:30.141491Z","iopub.execute_input":"2024-12-10T08:37:30.141842Z","iopub.status.idle":"2024-12-10T08:37:58.587545Z","shell.execute_reply.started":"2024-12-10T08:37:30.141810Z","shell.execute_reply":"2024-12-10T08:37:58.586582Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Final Submission","metadata":{}},{"cell_type":"code","source":"test_scaled_df['id'] = test1['id']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T08:50:53.889539Z","iopub.execute_input":"2024-12-10T08:50:53.889930Z","iopub.status.idle":"2024-12-10T08:50:53.896500Z","shell.execute_reply.started":"2024-12-10T08:50:53.889896Z","shell.execute_reply":"2024-12-10T08:50:53.895287Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Make predictions on the test set\nbest_model = XGBClassifier(use_label_encoder=False, eval_metric='mlogloss', random_state=42) \nbest_model.fit(X_resampled, y_resampled)\n\ntest_scaled_df['sii'] = best_model.predict(test_scaled_df.drop(columns=['id']))\n\n# Save submission file\nsubmission = test_scaled_df[['id', 'sii']]\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Submission file created!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T08:53:31.155480Z","iopub.execute_input":"2024-12-10T08:53:31.155865Z","iopub.status.idle":"2024-12-10T08:53:34.229506Z","shell.execute_reply.started":"2024-12-10T08:53:31.155830Z","shell.execute_reply":"2024-12-10T08:53:34.228298Z"}},"outputs":[],"execution_count":null}]}