{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Import required libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import make_scorer, cohen_kappa_score\nfrom sklearn.model_selection import GridSearchCV, cross_val_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:35.494109Z","iopub.execute_input":"2024-12-21T14:12:35.494337Z","iopub.status.idle":"2024-12-21T14:12:36.737679Z","shell.execute_reply.started":"2024-12-21T14:12:35.494312Z","shell.execute_reply":"2024-12-21T14:12:36.736554Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Data Visualization","metadata":{}},{"cell_type":"markdown","source":"## Read the csv file","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\", index_col=\"id\")\ntest_data = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\", index_col=\"id\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:36.738709Z","iopub.execute_input":"2024-12-21T14:12:36.739456Z","iopub.status.idle":"2024-12-21T14:12:36.820973Z","shell.execute_reply.started":"2024-12-21T14:12:36.739394Z","shell.execute_reply":"2024-12-21T14:12:36.819754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = train_data.copy()\ntest_df = test_data.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:36.823049Z","iopub.execute_input":"2024-12-21T14:12:36.823501Z","iopub.status.idle":"2024-12-21T14:12:36.830484Z","shell.execute_reply.started":"2024-12-21T14:12:36.823462Z","shell.execute_reply":"2024-12-21T14:12:36.829197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:36.832504Z","iopub.execute_input":"2024-12-21T14:12:36.833023Z","iopub.status.idle":"2024-12-21T14:12:36.884479Z","shell.execute_reply.started":"2024-12-21T14:12:36.832989Z","shell.execute_reply":"2024-12-21T14:12:36.88353Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:36.885203Z","iopub.execute_input":"2024-12-21T14:12:36.888549Z","iopub.status.idle":"2024-12-21T14:12:36.951749Z","shell.execute_reply.started":"2024-12-21T14:12:36.888498Z","shell.execute_reply":"2024-12-21T14:12:36.950812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_df.shape)\nprint(test_df.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:36.952683Z","iopub.execute_input":"2024-12-21T14:12:36.955669Z","iopub.status.idle":"2024-12-21T14:12:36.964521Z","shell.execute_reply.started":"2024-12-21T14:12:36.955632Z","shell.execute_reply":"2024-12-21T14:12:36.963583Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### --> The test df has less features than train df","metadata":{}},{"cell_type":"markdown","source":"# 2. Data Cleaning","metadata":{}},{"cell_type":"markdown","source":"### Visualize how many NaN values are there in the training dataset","metadata":{}},{"cell_type":"code","source":"# Count the number of missing values in each feature\nmissing_values = train_df.isnull().sum()\n\n# Plotting the chart\nplt.figure(figsize=(10, 30))\nax = missing_values.plot(kind='barh', color='skyblue')\n\nfor index, value in enumerate(missing_values):\n    ax.text(value, index, str(value), va='center', ha='left', color='black', fontweight='bold')\n\nplt.title('Number of Missing Values in Each Feature')\nplt.ylabel('Features')\nplt.xlabel('Number of Missing Values')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:36.965271Z","iopub.execute_input":"2024-12-21T14:12:36.965617Z","iopub.status.idle":"2024-12-21T14:12:38.163402Z","shell.execute_reply.started":"2024-12-21T14:12:36.965586Z","shell.execute_reply":"2024-12-21T14:12:38.162264Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Handle missing output samples","metadata":{}},{"cell_type":"markdown","source":"#### The presence of missing output samples is not suitable for training the model, so we will remove them.","metadata":{}},{"cell_type":"code","source":"#Remove samples without output label\ntrain_df = train_df.dropna(subset=['sii'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:38.164628Z","iopub.execute_input":"2024-12-21T14:12:38.165103Z","iopub.status.idle":"2024-12-21T14:12:38.174937Z","shell.execute_reply.started":"2024-12-21T14:12:38.165061Z","shell.execute_reply":"2024-12-21T14:12:38.173814Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:38.176037Z","iopub.execute_input":"2024-12-21T14:12:38.176336Z","iopub.status.idle":"2024-12-21T14:12:38.201471Z","shell.execute_reply.started":"2024-12-21T14:12:38.176308Z","shell.execute_reply":"2024-12-21T14:12:38.200244Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"##### First take a look how train and test dataset look like after data cleaning","metadata":{}},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:38.202754Z","iopub.execute_input":"2024-12-21T14:12:38.203039Z","iopub.status.idle":"2024-12-21T14:12:38.229707Z","shell.execute_reply.started":"2024-12-21T14:12:38.20301Z","shell.execute_reply":"2024-12-21T14:12:38.228265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:38.230827Z","iopub.execute_input":"2024-12-21T14:12:38.231255Z","iopub.status.idle":"2024-12-21T14:12:38.268595Z","shell.execute_reply.started":"2024-12-21T14:12:38.231211Z","shell.execute_reply":"2024-12-21T14:12:38.267223Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.1. Feature Extraction","metadata":{}},{"cell_type":"markdown","source":"### Drop features that do not appear in the test set","metadata":{}},{"cell_type":"code","source":"train_df.drop(columns=[col for col in train_df if 'PCIAT' in col], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:38.273298Z","iopub.execute_input":"2024-12-21T14:12:38.273781Z","iopub.status.idle":"2024-12-21T14:12:38.290231Z","shell.execute_reply.started":"2024-12-21T14:12:38.273733Z","shell.execute_reply":"2024-12-21T14:12:38.288961Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Check missing values per feature","metadata":{}},{"cell_type":"code","source":"missing_ratio = train_df.isna().mean()\n\nmissing_ratio = missing_ratio[missing_ratio > 0]\n\nmissing_df = missing_ratio.reset_index()\nmissing_df.columns = ['Feature', 'Missing Ratio']\n\nplt.figure(figsize=(15, 6))\nsns.barplot(data=missing_df, x='Feature', y='Missing Ratio', palette='coolwarm')\nplt.title('Missing Ratio per Feature', fontsize=12)\nplt.ylabel('Missing Ratio (%)', fontsize=12)\nplt.xlabel('Features', fontsize=4)\nplt.xticks(rotation=45, ha='right')\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:38.292746Z","iopub.execute_input":"2024-12-21T14:12:38.293076Z","iopub.status.idle":"2024-12-21T14:12:39.034729Z","shell.execute_reply.started":"2024-12-21T14:12:38.293045Z","shell.execute_reply":"2024-12-21T14:12:39.033338Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Remove features which miss more than a half","metadata":{}},{"cell_type":"code","source":"threshold = 0.5\nmissing_ratio = train_df.isnull().mean()\ndropped_columns = missing_ratio[missing_ratio > threshold].index.tolist()\n\ntrain_df.drop(columns=dropped_columns, inplace=True)\ntest_df.drop(columns=dropped_columns, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:39.035784Z","iopub.execute_input":"2024-12-21T14:12:39.036075Z","iopub.status.idle":"2024-12-21T14:12:39.047946Z","shell.execute_reply.started":"2024-12-21T14:12:39.036051Z","shell.execute_reply":"2024-12-21T14:12:39.046811Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.2. Feature Categorization","metadata":{}},{"cell_type":"markdown","source":"#### We split the features into 3 types: categorical, binary and numerical","metadata":{}},{"cell_type":"markdown","source":"After reviewing the data dictionary, we found that:\n- Features have dtype=number but has meaning as categorical: `BIA-BIA_Activity_Level_num`,  `FGC-FGC_GSD_Zone`,`FGC-FGC_GSND_Zone`,`BIA-BIA_Frame_num`, `PreInt_EduHx-computerinternet_hoursday`\n- The feature has dtype=object has 'Season' in its name","metadata":{}},{"cell_type":"code","source":"int_cols = ['BIA-BIA_Activity_Level_num', 'FGC-FGC_GSD_Zone','FGC-FGC_GSND_Zone','BIA-BIA_Frame_num', 'PreInt_EduHx-computerinternet_hoursday']\ncategorical_int_cols = [col for col in int_cols if col not in dropped_columns]\n\ncategorical_str_cols = [col for col in train_df.columns if 'Season' in col and col not in dropped_columns]\n\n#Combine 2 types of categorical features\ncategorical_cols = categorical_str_cols + categorical_int_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:39.049569Z","iopub.execute_input":"2024-12-21T14:12:39.049991Z","iopub.status.idle":"2024-12-21T14:12:39.069354Z","shell.execute_reply.started":"2024-12-21T14:12:39.04995Z","shell.execute_reply":"2024-12-21T14:12:39.068018Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"binary_cols = [col for col in train_df.columns if train_df[col].nunique() == 2]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:39.070555Z","iopub.execute_input":"2024-12-21T14:12:39.070889Z","iopub.status.idle":"2024-12-21T14:12:39.105714Z","shell.execute_reply.started":"2024-12-21T14:12:39.07086Z","shell.execute_reply":"2024-12-21T14:12:39.104257Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numerical_cols = [col for col in train_df.columns if col != 'sii' and col not in categorical_cols and col not in binary_cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:39.106762Z","iopub.execute_input":"2024-12-21T14:12:39.107158Z","iopub.status.idle":"2024-12-21T14:12:39.113412Z","shell.execute_reply.started":"2024-12-21T14:12:39.107124Z","shell.execute_reply":"2024-12-21T14:12:39.11202Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.3. Imputation","metadata":{}},{"cell_type":"markdown","source":"The strategy we will use is as follows:\n- Fill missing values in categorical and binary features with the most frequently occurring value.\n- Fill missing values in numerical features with the mean value.","metadata":{}},{"cell_type":"code","source":"for col in categorical_cols:\n    mode_value = train_df[col].mode()[0]\n    train_df[col] = train_df[col].fillna(mode_value)\n    train_df[col] = train_df[col].astype(object)\n\nfor col in binary_cols:\n    mode_value = train_df[col].mode()[0]\n    train_df[col] = train_df[col].fillna(mode_value)\n    train_df[col] = train_df[col].astype(int)\n\nfor col in numerical_cols:\n    mean_value = train_df[col].mean()\n    train_df[col] = train_df[col].fillna(mean_value)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:39.114927Z","iopub.execute_input":"2024-12-21T14:12:39.115252Z","iopub.status.idle":"2024-12-21T14:12:39.168633Z","shell.execute_reply.started":"2024-12-21T14:12:39.115226Z","shell.execute_reply":"2024-12-21T14:12:39.167587Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in categorical_cols:\n    mode_value = test_df[col].mode()[0]\n    test_df[col] = test_df[col].fillna(mode_value)\n    test_df[col] = test_df[col].astype(object)\n\nfor col in binary_cols:\n    mode_value = test_df[col].mode()[0]\n    test_df[col] = test_df[col].fillna(mode_value)\n    test_df[col] = test_df[col].astype(int)\n\nfor col in numerical_cols:\n    mean_value = test_df[col].mean()\n    test_df[col] = test_df[col].fillna(mean_value)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:39.169844Z","iopub.execute_input":"2024-12-21T14:12:39.170227Z","iopub.status.idle":"2024-12-21T14:12:39.208037Z","shell.execute_reply.started":"2024-12-21T14:12:39.170181Z","shell.execute_reply":"2024-12-21T14:12:39.206586Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.4. Feature Scaling","metadata":{}},{"cell_type":"markdown","source":"#### We use standard scaler to scale the numerical features","metadata":{}},{"cell_type":"code","source":"scaler = StandardScaler()\n\n# Helper function\ndef standardize(df):\n    columns_to_standardize = [col for col in numerical_cols]\n    df[columns_to_standardize] = scaler.fit_transform(df[columns_to_standardize])\n    return df\n\n#Proceed with standardizing\ntrain_df = standardize(train_df)\ntest_df = standardize(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:39.209334Z","iopub.execute_input":"2024-12-21T14:12:39.209699Z","iopub.status.idle":"2024-12-21T14:12:39.245443Z","shell.execute_reply.started":"2024-12-21T14:12:39.209667Z","shell.execute_reply":"2024-12-21T14:12:39.244503Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.5. Feature Encoding","metadata":{}},{"cell_type":"markdown","source":"#### We use one-hot encoding to encode the numerical features","metadata":{}},{"cell_type":"code","source":"# Helper function\ndef OneHot_Encoding(original_dataframe, feature_to_encode):\n    dummies = pd.get_dummies(original_dataframe[[feature_to_encode]], dtype=int)\n    original_dataframe = pd.concat([original_dataframe, dummies], axis=1)\n    original_dataframe = original_dataframe.drop([feature_to_encode], axis=1)\n    return original_dataframe\n\nfor col in categorical_cols:\n    train_df = OneHot_Encoding(train_df, col)\n    test_df = OneHot_Encoding(test_df, col)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:39.246609Z","iopub.execute_input":"2024-12-21T14:12:39.246945Z","iopub.status.idle":"2024-12-21T14:12:39.339442Z","shell.execute_reply.started":"2024-12-21T14:12:39.246916Z","shell.execute_reply":"2024-12-21T14:12:39.338339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Remove feature which does not appear in test data after encoding, excluding 'sii'\ntrain_miss = (set(train_df.columns) - set(test_df.columns)) - {'sii'}\n\ntrain_df = train_df.drop(columns=train_miss)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:39.340339Z","iopub.execute_input":"2024-12-21T14:12:39.340687Z","iopub.status.idle":"2024-12-21T14:12:39.347558Z","shell.execute_reply.started":"2024-12-21T14:12:39.34066Z","shell.execute_reply":"2024-12-21T14:12:39.345915Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Now, let’s review the data after applying these feature processing steps.","metadata":{}},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:39.348915Z","iopub.execute_input":"2024-12-21T14:12:39.349321Z","iopub.status.idle":"2024-12-21T14:12:39.382792Z","shell.execute_reply.started":"2024-12-21T14:12:39.349274Z","shell.execute_reply":"2024-12-21T14:12:39.381549Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:39.38416Z","iopub.execute_input":"2024-12-21T14:12:39.384525Z","iopub.status.idle":"2024-12-21T14:12:39.417074Z","shell.execute_reply.started":"2024-12-21T14:12:39.384492Z","shell.execute_reply":"2024-12-21T14:12:39.41583Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. Define and train model","metadata":{}},{"cell_type":"markdown","source":"## 4.1. Extract data frame","metadata":{}},{"cell_type":"code","source":"features = [col for col in train_df.columns if col != 'sii']\nX = train_df[features]\ny = train_df.sii","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:39.41793Z","iopub.execute_input":"2024-12-21T14:12:39.418228Z","iopub.status.idle":"2024-12-21T14:12:39.438134Z","shell.execute_reply.started":"2024-12-21T14:12:39.418198Z","shell.execute_reply":"2024-12-21T14:12:39.437079Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4.2. Split training data","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:39.4395Z","iopub.execute_input":"2024-12-21T14:12:39.439862Z","iopub.status.idle":"2024-12-21T14:12:39.461691Z","shell.execute_reply.started":"2024-12-21T14:12:39.439834Z","shell.execute_reply":"2024-12-21T14:12:39.460497Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4.3. Model selection","metadata":{}},{"cell_type":"code","source":"value_counts = y.value_counts()\nvalue_counts.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title('Frequency of Categorical Values in y')\nplt.xlabel('Values')\nplt.ylabel('Frequency')\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:39.462671Z","iopub.execute_input":"2024-12-21T14:12:39.463009Z","iopub.status.idle":"2024-12-21T14:12:39.729843Z","shell.execute_reply.started":"2024-12-21T14:12:39.462974Z","shell.execute_reply":"2024-12-21T14:12:39.728634Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**The output distribution is imbalanced, so we have chosen the `Random Forest Classifier` to train the model. This classifier is highly effective for handling imbalanced data by setting the `class_weight` parameter to `balanced`.**","metadata":{}},{"cell_type":"markdown","source":"## 4.4. Train model","metadata":{}},{"cell_type":"code","source":"# Validation model,train on X_val test, to evaluate the model\nval_model = RandomForestClassifier(\n    class_weight='balanced',\n    random_state=42\n)\n\n# Train the validation model\nval_model.fit(X_train, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:39.731083Z","iopub.execute_input":"2024-12-21T14:12:39.731402Z","iopub.status.idle":"2024-12-21T14:12:40.476072Z","shell.execute_reply.started":"2024-12-21T14:12:39.731364Z","shell.execute_reply":"2024-12-21T14:12:40.475058Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4.5. Evaluate on validation df","metadata":{}},{"cell_type":"code","source":"def quadratic_weight_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:40.477344Z","iopub.execute_input":"2024-12-21T14:12:40.477788Z","iopub.status.idle":"2024-12-21T14:12:40.483003Z","shell.execute_reply.started":"2024-12-21T14:12:40.477745Z","shell.execute_reply":"2024-12-21T14:12:40.481725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_preds = val_model.predict(X_val)\nval_preds = np.array(val_preds).astype(int)\n\ny_val = np.array(y_val).astype(int)\n\nprint(quadratic_weight_kappa(y_val, val_preds))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:40.484303Z","iopub.execute_input":"2024-12-21T14:12:40.48475Z","iopub.status.idle":"2024-12-21T14:12:40.528847Z","shell.execute_reply.started":"2024-12-21T14:12:40.484707Z","shell.execute_reply":"2024-12-21T14:12:40.527531Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**---> Not a good score, so we decide to optimize the model by using GridSearch CV to find out the best hyperparameters**","metadata":{}},{"cell_type":"markdown","source":"# 5. Model Optimization","metadata":{}},{"cell_type":"markdown","source":"## Run GridSearch CV","metadata":{}},{"cell_type":"code","source":"%%time\n\nqwk_scorer = make_scorer(quadratic_weight_kappa)\n\nparam_grid = {\n    'n_estimators': [150, 200, 250, 300],\n    'max_depth': [5, 8, 10],\n    'min_samples_split': [2, 3, 4, 5],\n    'min_samples_leaf': [1, 2, 3, 4],\n    'criterion':['entropy', 'gini'],\n}\n\n# Model initialization\nrf_model = RandomForestClassifier(\n    random_state=42,\n    class_weight='balanced',\n    max_features='sqrt',\n)\n\n# GridSearchCV with 3-fold cv\ngrid_search = GridSearchCV(\n    estimator=rf_model,\n    param_grid=param_grid,\n    scoring=qwk_scorer,\n    cv=3,\n    verbose=1,\n    n_jobs=-1\n)\n\n# Searching\ngrid_search.fit(X, y)\n\nprint(\"Best parameters found:\", grid_search.best_params_)\nprint(\"Best cross-validation accuracy:\", grid_search.best_score_)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:12:40.530046Z","iopub.execute_input":"2024-12-21T14:12:40.530506Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_model = RandomForestClassifier(\n    n_estimators=300,\n    max_depth=8,\n    max_features='sqrt',\n    min_samples_split=5,\n    min_samples_leaf=1,\n    class_weight='balanced',\n    random_state=42,\n)\n\nfinal_model.fit(X, y)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 6. Model Evaluation","metadata":{}},{"cell_type":"code","source":"scores = cross_val_score(final_model, X, y, cv=3, scoring=qwk_scorer)\nprint(\"QWK Scores:\", scores)\nprint(\"----> Mean QWK Score:\", np.mean(scores))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 7. Submit to competition","metadata":{}},{"cell_type":"markdown","source":"## Predict the test df","metadata":{}},{"cell_type":"code","source":"preds = final_model.predict(test_df)\npreds = np.array(preds).astype(int)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Save output to submission file","metadata":{}},{"cell_type":"code","source":"output = pd.DataFrame({'id': test_df.index,\n                       'sii': preds})\noutput.to_csv('submission.csv',index=False)\noutput","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}