{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":31012,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split, KFold, GridSearchCV\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.metrics import cohen_kappa_score, mean_squared_error, r2_score\nimport matplotlib.pyplot as plt\nfrom scipy.optimize import minimize\nimport joblib\nimport optuna\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:13:58.878977Z","iopub.execute_input":"2025-04-24T09:13:58.879256Z","iopub.status.idle":"2025-04-24T09:14:03.048789Z","shell.execute_reply.started":"2025-04-24T09:13:58.879229Z","shell.execute_reply":"2025-04-24T09:14:03.048069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TRAIN_PATH = '/kaggle/input/child-mind-institute-problematic-internet-use/train.csv'\nTEST_PATH = '/kaggle/input/child-mind-institute-problematic-internet-use/test.csv'\nTRAIN_TS_PATH = '/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet'\nTEST_TS_PATH = '/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet'\nSUBMISSION_PATH = '/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv'\nOUTPUT_PATH = '/kaggle/working/'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:14:03.049558Z","iopub.execute_input":"2025-04-24T09:14:03.049872Z","iopub.status.idle":"2025-04-24T09:14:03.054681Z","shell.execute_reply.started":"2025-04-24T09:14:03.049853Z","shell.execute_reply":"2025-04-24T09:14:03.053532Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define QWK function\ndef quadratic_weighted_kappa(y_true, y_pred):\n    \"\"\"Calculate quadratic weighted kappa.\"\"\"\n    # Ensure inputs are in the right format\n    y_true = np.array(y_true, dtype=int)\n    y_pred = np.round(np.array(y_pred)).astype(int)\n    \n    # Clip predictions to be within the range of observed values\n    min_val, max_val = min(y_true), max(y_true)\n    y_pred = np.clip(y_pred, min_val, max_val)\n    \n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:14:03.057117Z","iopub.execute_input":"2025-04-24T09:14:03.057356Z","iopub.status.idle":"2025-04-24T09:14:03.075229Z","shell.execute_reply.started":"2025-04-24T09:14:03.057319Z","shell.execute_reply":"2025-04-24T09:14:03.074322Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Custom calibration function to optimize QWK\ndef optimize_predictions_for_qwk(y_true, y_pred, return_params=False):\n    \"\"\"Apply a linear transformation to predictions to maximize QWK.\"\"\"\n    def objective(params):\n        a, b = params\n        calibrated = a * y_pred + b\n        return -quadratic_weighted_kappa(y_true, calibrated)\n    \n    # Starting with identity transformation\n    initial_params = [1.0, 0.0]\n    \n    # Optimize the parameters\n    result = minimize(objective, initial_params, method='Nelder-Mead')\n    a, b = result.x\n    \n    if return_params:\n        return a, b\n    \n    # Apply the transformation\n    calibrated_preds = a * y_pred + b\n    \n    # Clip to observed range\n    min_val, max_val = min(y_true), max(y_true)\n    calibrated_preds = np.clip(calibrated_preds, min_val, max_val)\n    \n    return calibrated_preds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:14:03.076183Z","iopub.execute_input":"2025-04-24T09:14:03.076552Z","iopub.status.idle":"2025-04-24T09:14:03.091939Z","shell.execute_reply.started":"2025-04-24T09:14:03.076523Z","shell.execute_reply":"2025-04-24T09:14:03.090886Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_sample_weights(X):\n    weights = np.ones(len(X))\n\n    # Feature importances (from your graph), normalized so they sum to 1\n    feature_weights = {\n        'Basic_Demos-Age': 0.1731,\n        'PreInt_EduHx-computerinternet_hoursday': 0.13,\n        'SDS-SDS_Total_Raw' : 0.0742,\n        'SDS-SDS_Total_T': 0.0607,\n        'Physical-Height': 0.0531,\n        'Physical-Weight': 0.0461,\n        'FGC-FGC_CU': 0.0342,\n        'Basic_Demos-Sex': 0.0243,\n    }\n\n    # Normalize weights to sum to 1 or scale as needed\n    total_importance = sum(feature_weights.values())\n    for feature, importance in feature_weights.items():\n        if feature in X.columns:\n            scaled_importance = importance / total_importance\n            feature_series = X[feature]\n            # Mean/std deviation safely with skipna=True\n            mean = feature_series.mean(skipna=True)\n            std = feature_series.std(skipna=True)\n            deviation = np.abs(feature_series - mean) / (std + 1e-8)\n            weights += scaled_importance * deviation.fillna(0)\n    \n    return weights","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:14:03.092996Z","iopub.execute_input":"2025-04-24T09:14:03.093316Z","iopub.status.idle":"2025-04-24T09:14:03.112016Z","shell.execute_reply.started":"2025-04-24T09:14:03.093295Z","shell.execute_reply":"2025-04-24T09:14:03.110992Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load and prepare data\nprint(\"Loading data...\")\ndata = pd.read_csv(TRAIN_PATH)\nprint(f\"Original data shape: {data.shape}\")\nprint(f\"Number of NaN values in 'sii': {data['sii'].isna().sum()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:14:03.112909Z","iopub.execute_input":"2025-04-24T09:14:03.113207Z","iopub.status.idle":"2025-04-24T09:14:03.209144Z","shell.execute_reply.started":"2025-04-24T09:14:03.113177Z","shell.execute_reply":"2025-04-24T09:14:03.208117Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Remove rows where 'sii' is NaN\ndata_clean = data.dropna(subset=['sii'])\nprint(f\"Data shape after removing NaN targets: {data_clean.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:14:03.210151Z","iopub.execute_input":"2025-04-24T09:14:03.210472Z","iopub.status.idle":"2025-04-24T09:14:03.223109Z","shell.execute_reply.started":"2025-04-24T09:14:03.210449Z","shell.execute_reply":"2025-04-24T09:14:03.222445Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Separate target variable\ny = data_clean['sii'].astype(int)  # Ensure target is integer for QWK\nprint(f\"Target values range: {y.min()} to {y.max()}\")\nprint(f\"Unique target values: {sorted(y.unique())}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:14:03.224039Z","iopub.execute_input":"2025-04-24T09:14:03.224636Z","iopub.status.idle":"2025-04-24T09:14:03.239208Z","shell.execute_reply.started":"2025-04-24T09:14:03.224606Z","shell.execute_reply":"2025-04-24T09:14:03.238562Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select features, excluding PCIAT columns\nX_columns = [col for col in data_clean.columns if not col.startswith('PCIAT') and col != 'sii' and col != 'id']\nX = data_clean[X_columns]\nprint(f\"Number of features: {len(X_columns)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:14:03.242608Z","iopub.execute_input":"2025-04-24T09:14:03.242818Z","iopub.status.idle":"2025-04-24T09:14:03.256052Z","shell.execute_reply.started":"2025-04-24T09:14:03.242802Z","shell.execute_reply":"2025-04-24T09:14:03.255205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Split data - use stratified split if possible to maintain distribution\n# For regression, we can bin the target for stratification\ntry:\n    from sklearn.model_selection import StratifiedShuffleSplit\n    # Create bins for stratification\n    bins = pd.qcut(y, q=min(10, len(y.unique())), labels=False, duplicates='drop')\n    stratified_split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n    for train_idx, test_idx in stratified_split.split(X, bins):\n        X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]\n    print(\"Used stratified split to maintain target distribution\")\nexcept:\n    # Fallback to regular split if stratification fails\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n    print(\"Used regular train-test split\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:14:03.257033Z","iopub.execute_input":"2025-04-24T09:14:03.257320Z","iopub.status.idle":"2025-04-24T09:14:03.282736Z","shell.execute_reply.started":"2025-04-24T09:14:03.257291Z","shell.execute_reply":"2025-04-24T09:14:03.281835Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Identify numeric and categorical columns\nnumeric_features = X.select_dtypes(include=['int64', 'float64']).columns.tolist()\ncategorical_features = X.select_dtypes(include=['object', 'category']).columns.tolist()\nprint(f\"Numeric features: {len(numeric_features)}\")\nprint(f\"Categorical features: {len(categorical_features)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:14:03.283741Z","iopub.execute_input":"2025-04-24T09:14:03.284122Z","iopub.status.idle":"2025-04-24T09:14:03.290933Z","shell.execute_reply.started":"2025-04-24T09:14:03.284095Z","shell.execute_reply":"2025-04-24T09:14:03.290310Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transformer_list = [\n    ('num', Pipeline(steps=[\n        ('imputer', SimpleImputer(strategy='median')),\n        ('scaler', StandardScaler())\n    ]), numeric_features)\n]\n\nif categorical_features:\n    transformer_list.append(\n        ('cat', Pipeline(steps=[\n            ('imputer', SimpleImputer(strategy='most_frequent')),\n            ('onehot', OneHotEncoder(handle_unknown='ignore'))\n        ]), categorical_features)\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:14:03.291837Z","iopub.execute_input":"2025-04-24T09:14:03.292094Z","iopub.status.idle":"2025-04-24T09:14:03.306041Z","shell.execute_reply.started":"2025-04-24T09:14:03.292064Z","shell.execute_reply":"2025-04-24T09:14:03.305153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocessor = ColumnTransformer(transformers=transformer_list)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:14:03.306922Z","iopub.execute_input":"2025-04-24T09:14:03.307159Z","iopub.status.idle":"2025-04-24T09:14:03.326265Z","shell.execute_reply.started":"2025-04-24T09:14:03.307141Z","shell.execute_reply":"2025-04-24T09:14:03.325175Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the objective function for Optuna optimization\ndef objective(trial):\n    # Hyperparameters for GradientBoostingRegressor specifically tuned for QWK\n    params = {\n    'n_estimators': trial.suggest_int('n_estimators', 400, 700),\n    'learning_rate': trial.suggest_float('learning_rate', 0.005, 0.03, log=True),\n    'max_depth': trial.suggest_int('max_depth', 3, 6),\n    'min_samples_split': trial.suggest_int('min_samples_split', 5, 15),\n    'min_samples_leaf': trial.suggest_int('min_samples_leaf', 5, 15),\n    'subsample': trial.suggest_float('subsample', 0.5, 0.9),\n    'max_features': trial.suggest_float('max_features', 0.5, 0.9),\n    'loss': trial.suggest_categorical('loss', ['squared_error', 'huber', 'quantile']),\n    'alpha': trial.suggest_float('alpha', 0.5, 0.9),\n    'random_state': 42\n}\n    \n    # Create model\n    model = GradientBoostingRegressor(**params)\n    \n    # Create pipeline\n    pipeline = Pipeline(steps=[\n        ('preprocessor', preprocessor),\n        ('model', model)\n    ])\n    \n    # Use cross-validation with QWK optimization\n    k_fold = KFold(n_splits=5, shuffle=True, random_state=42)\n    qwk_scores = []\n    \n    for fold_idx, (train_idx, val_idx) in enumerate(k_fold.split(X_train)):\n        # Split data\n        X_fold_train, X_fold_val = X_train.iloc[train_idx], X_train.iloc[val_idx]\n        y_fold_train, y_fold_val = y_train.iloc[train_idx], y_train.iloc[val_idx]\n        \n        sample_weights = create_sample_weights(X_fold_train)\n        \n        # Fit model\n        pipeline.fit(X_fold_train, y_fold_train, model__sample_weight=sample_weights)\n        \n        # Get predictions\n        y_pred = pipeline.predict(X_fold_val)\n        \n        # Optimize predictions for QWK and evaluate\n        y_pred_calibrated = optimize_predictions_for_qwk(y_fold_val, y_pred)\n        qwk = quadratic_weighted_kappa(y_fold_val, y_pred_calibrated)\n        qwk_scores.append(qwk)\n        \n        # Report intermediate value for pruning\n        trial.report(-qwk, fold_idx)\n        \n        # Handle pruning based on the intermediate value\n        if trial.should_prune():\n            raise optuna.TrialPruned()\n        \n    mean_qwk = np.mean(qwk_scores)\n    return -mean_qwk  # Negative because Optuna minimizes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:14:03.327434Z","iopub.execute_input":"2025-04-24T09:14:03.327646Z","iopub.status.idle":"2025-04-24T09:14:03.342928Z","shell.execute_reply.started":"2025-04-24T09:14:03.327630Z","shell.execute_reply":"2025-04-24T09:14:03.341922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Run Optuna study\nprint(\"\\nStarting Optuna optimization to maximize QWK...\")\nstudy = optuna.create_study(direction='minimize')  # Using minimize since we return -QWK\nn_trials = 50  # Increase for better results if you have the compute resources\nstudy.optimize(objective, n_trials=n_trials)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:14:03.343834Z","iopub.execute_input":"2025-04-24T09:14:03.344079Z","iopub.status.idle":"2025-04-24T09:36:17.576837Z","shell.execute_reply.started":"2025-04-24T09:14:03.344062Z","shell.execute_reply":"2025-04-24T09:36:17.576131Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Print optimization results\nprint(\"\\nOptimization completed!\")\nprint(f\"Best trial: {study.best_trial.number}\")\nprint(f\"Best QWK: {-study.best_value:.4f}\")\nprint(\"Best hyperparameters:\")\nfor key, value in study.best_params.items():\n    print(f\"    {key}: {value}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:36:17.577603Z","iopub.execute_input":"2025-04-24T09:36:17.577885Z","iopub.status.idle":"2025-04-24T09:36:17.583513Z","shell.execute_reply.started":"2025-04-24T09:36:17.577858Z","shell.execute_reply":"2025-04-24T09:36:17.582591Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create and train the final model with best parameters\nprint(\"\\nTraining final model with best parameters...\")\nbest_params = study.best_params.copy()\nbest_model = GradientBoostingRegressor(**best_params)\n\nfinal_pipeline = Pipeline(steps=[\n    ('preprocessor', preprocessor),\n    ('model', best_model)\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:36:17.584397Z","iopub.execute_input":"2025-04-24T09:36:17.584692Z","iopub.status.idle":"2025-04-24T09:36:17.602850Z","shell.execute_reply.started":"2025-04-24T09:36:17.584666Z","shell.execute_reply":"2025-04-24T09:36:17.602128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train on full training data\nfinal_pipeline.fit(X_train, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:36:17.603928Z","iopub.execute_input":"2025-04-24T09:36:17.604220Z","iopub.status.idle":"2025-04-24T09:36:30.713662Z","shell.execute_reply.started":"2025-04-24T09:36:17.604195Z","shell.execute_reply":"2025-04-24T09:36:30.712690Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data = pd.read_csv(TEST_PATH)\n# need to feature engineer this as well and then do prediction","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:36:30.714594Z","iopub.execute_input":"2025-04-24T09:36:30.714886Z","iopub.status.idle":"2025-04-24T09:36:30.739644Z","shell.execute_reply.started":"2025-04-24T09:36:30.714867Z","shell.execute_reply":"2025-04-24T09:36:30.738682Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get predictions on test data\ny_pred = final_pipeline.predict(X_test)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:36:30.740592Z","iopub.execute_input":"2025-04-24T09:36:30.740855Z","iopub.status.idle":"2025-04-24T09:36:30.763734Z","shell.execute_reply.started":"2025-04-24T09:36:30.740837Z","shell.execute_reply":"2025-04-24T09:36:30.762893Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calibrate predictions to maximize QWK\nprint(\"\\nCalibrating predictions to maximize QWK...\")\ny_pred_calibrated = optimize_predictions_for_qwk(y_test, y_pred)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:36:30.764604Z","iopub.execute_input":"2025-04-24T09:36:30.764899Z","iopub.status.idle":"2025-04-24T09:36:30.808595Z","shell.execute_reply.started":"2025-04-24T09:36:30.764872Z","shell.execute_reply":"2025-04-24T09:36:30.807870Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Evaluate both raw and calibrated predictions\nqwk_raw = quadratic_weighted_kappa(y_test, y_pred)\nqwk_calibrated = quadratic_weighted_kappa(y_test, y_pred_calibrated)\nrmse_raw = np.sqrt(mean_squared_error(y_test, y_pred))\nrmse_calibrated = np.sqrt(mean_squared_error(y_test, y_pred_calibrated))\nr2_raw = r2_score(y_test, y_pred)\nr2_calibrated = r2_score(y_test, y_pred_calibrated)\n\nprint(\"\\nTest set evaluation:\")\nprint(f\"Raw predictions - QWK: {qwk_raw:.4f}, RMSE: {rmse_raw:.4f}, R²: {r2_raw:.4f}\")\nprint(f\"Calibrated predictions - QWK: {qwk_calibrated:.4f}, RMSE: {rmse_calibrated:.4f}, R²: {r2_calibrated:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:36:30.809412Z","iopub.execute_input":"2025-04-24T09:36:30.809664Z","iopub.status.idle":"2025-04-24T09:36:30.820856Z","shell.execute_reply.started":"2025-04-24T09:36:30.809646Z","shell.execute_reply":"2025-04-24T09:36:30.819834Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load test data\ntest_data = pd.read_csv(TEST_PATH)\ntest_ids = test_data['id']  # Save IDs for submission\n\n# Select the same features as used in training\nX_test_submission = test_data[X_columns]\n\n# Get raw predictions using the trained pipeline\ntest_preds_raw = final_pipeline.predict(X_test_submission)\n\n# We need to calibrate these predictions for optimal QWK\n# For this, we'll use the training data that the model has already seen\ncalibration_preds = final_pipeline.predict(X_train)\ncalibration_params = optimize_predictions_for_qwk(y_train, calibration_preds, return_params=True)\n\n# Apply calibration to test predictions\na, b = calibration_params\ntest_preds_calibrated = a * test_preds_raw + b\n\n# Clip to the valid range (0-3 based on your training data)\ntest_preds_calibrated = np.clip(test_preds_calibrated, 0, 3)\n\n# Round to integers since SII appears to be an integer score\ntest_preds_calibrated = np.round(test_preds_calibrated).astype(int)\n\n# Create submission dataframe\nsubmission = pd.DataFrame({\n    'id': test_ids,\n    'sii': test_preds_calibrated\n})\nimport os\n# Save submission file\nsubmission_path = os.path.join(OUTPUT_PATH, \"submission.csv\")\nsubmission.to_csv(submission_path, index=False)\nprint(f\"Submission file saved to {submission_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:36:30.821655Z","iopub.execute_input":"2025-04-24T09:36:30.821948Z","iopub.status.idle":"2025-04-24T09:36:30.954196Z","shell.execute_reply.started":"2025-04-24T09:36:30.821924Z","shell.execute_reply":"2025-04-24T09:36:30.953369Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Feature importance analysis\n# print(\"\\nAnalyzing feature importance...\")\n# try:\n#     # Get feature names after preprocessing\n#     feature_names = []\n#     for name, transformer, features in preprocessor.transformers_:\n#         if name == 'cat':\n#             # Get one-hot encoded feature names\n#             encoder = transformer.named_steps['onehot']\n#             cats = encoder.categories_\n#             for i, feature in enumerate(features):\n#                 feature_names.extend([f\"{feature}_{cat}\" for cat in cats[i]])\n#         else:\n#             feature_names.extend(features)\n    \n#     # Get feature importances\n#     importances = final_pipeline.named_steps['model'].feature_importances_\n#     indices = np.argsort(importances)[::-1]\n    \n#     # Print top features\n#     print(\"\\nTop 20 features by importance:\")\n#     top_n = min(20, len(feature_names))\n#     for i in range(top_n):\n#         try:\n#             print(f\"{i+1}. {feature_names[indices[i]]}: {importances[indices[i]]:.4f}\")\n#         except:\n#             print(f\"{i+1}. Feature #{indices[i]}: {importances[indices[i]]:.4f}\")\n    \n#     # Plot feature importances\n#     plt.figure(figsize=(12, 10))\n#     top_indices = indices[:top_n]\n#     plt.barh(range(top_n), importances[top_indices])\n#     plt.yticks(range(top_n), [feature_names[i] if i < len(feature_names) else f\"Feature #{i}\" for i in top_indices])\n#     plt.xlabel('Importance')\n#     plt.title('Top 20 Feature Importances')\n#     plt.tight_layout()\n#     # plt.savefig('qwk_feature_importance.png')\n#     # print(\"Feature importance plot saved as 'qwk_feature_importance.png'\")\n# except Exception as e:\n#     print(f\"Error analyzing feature importance: {str(e)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:36:30.954984Z","iopub.execute_input":"2025-04-24T09:36:30.955186Z","iopub.status.idle":"2025-04-24T09:36:30.959856Z","shell.execute_reply.started":"2025-04-24T09:36:30.955170Z","shell.execute_reply":"2025-04-24T09:36:30.958963Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Visualize predictions\n# plt.figure(figsize=(10, 6))\n# plt.scatter(y_test, y_pred_calibrated, alpha=0.5)\n# plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--')\n# plt.xlabel('Actual SII')\n# plt.ylabel('Predicted SII (Calibrated)')\n# plt.title(f'Actual vs Predicted SII (QWK: {qwk_calibrated:.4f})')\n# plt.tight_layout()\n# # plt.savefig('qwk_predictions.png')\n# print(\"Prediction plot saved as 'qwk_predictions.png'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:36:30.960797Z","iopub.execute_input":"2025-04-24T09:36:30.961072Z","iopub.status.idle":"2025-04-24T09:36:30.979964Z","shell.execute_reply.started":"2025-04-24T09:36:30.961054Z","shell.execute_reply":"2025-04-24T09:36:30.979116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Error analysis - examine instances with largest errors\n# errors = abs(y_test - y_pred_calibrated)\n# error_df = pd.DataFrame({\n#     'Actual': y_test,\n#     'Predicted': y_pred_calibrated,\n#     'Error': errors\n# })\n# error_df = error_df.sort_values('Error', ascending=False)\n\n# print(\"\\nLargest prediction errors:\")\n# print(error_df.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:36:30.980914Z","iopub.execute_input":"2025-04-24T09:36:30.981157Z","iopub.status.idle":"2025-04-24T09:36:30.999610Z","shell.execute_reply.started":"2025-04-24T09:36:30.981139Z","shell.execute_reply":"2025-04-24T09:36:30.998683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Distribution of errors\n# plt.figure(figsize=(10, 6))\n# plt.hist(errors, bins=20)\n# plt.xlabel('Absolute Error')\n# plt.ylabel('Frequency')\n# plt.title('Distribution of Prediction Errors')\n# plt.tight_layout()\n# # plt.savefig('qwk_error_distribution.png')\n# print(\"Error distribution plot saved as 'qwk_error_distribution.png'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:36:31.003457Z","iopub.execute_input":"2025-04-24T09:36:31.003728Z","iopub.status.idle":"2025-04-24T09:36:31.017100Z","shell.execute_reply.started":"2025-04-24T09:36:31.003706Z","shell.execute_reply":"2025-04-24T09:36:31.016258Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Save the model\n# joblib.dump({\n#     'pipeline': final_pipeline,\n#     'calibration_function': optimize_predictions_for_qwk\n# }, 'qwk_optimized_model.pkl')\n# print(\"\\nQWK-optimized model saved as 'qwk_optimized_model.pkl'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:36:31.018083Z","iopub.execute_input":"2025-04-24T09:36:31.018937Z","iopub.status.idle":"2025-04-24T09:36:31.033085Z","shell.execute_reply.started":"2025-04-24T09:36:31.018903Z","shell.execute_reply":"2025-04-24T09:36:31.032331Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to make predictions with the saved model\n# print(\"\\nExample code to use the saved model:\")\n# print(\"\"\"\n# # Load model\n# import joblib\n# model_data = joblib.load('qwk_optimized_model.pkl')\n# pipeline = model_data['pipeline']\n# calibration_func = model_data['calibration_function']\n\n# # Make predictions (with new data)\n# raw_predictions = pipeline.predict(new_data)\n\n# # For optimal QWK, use the saved model on some validation data first\n# # and then apply the calibration\n# validation_actual = ... # Some known labels\n# validation_pred = pipeline.predict(validation_data)\n# calibrated_predictions = calibration_func(validation_actual, validation_pred)\n# \"\"\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T09:36:31.034406Z","iopub.execute_input":"2025-04-24T09:36:31.034705Z","iopub.status.idle":"2025-04-24T09:36:31.050277Z","shell.execute_reply.started":"2025-04-24T09:36:31.034675Z","shell.execute_reply":"2025-04-24T09:36:31.049620Z"}},"outputs":[],"execution_count":null}]}