{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10205783,"sourceType":"datasetVersion","datasetId":6307015}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install optuna-integration","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-17T19:06:09.850344Z","iopub.execute_input":"2024-12-17T19:06:09.851228Z","iopub.status.idle":"2024-12-17T19:06:22.908690Z","shell.execute_reply.started":"2024-12-17T19:06:09.851171Z","shell.execute_reply":"2024-12-17T19:06:22.907246Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport optuna\nimport pandas as pd\nimport optuna.integration.lightgbm as lgbo\nimport sklearn.datasets\nimport sklearn.metrics\nfrom sklearn.model_selection import train_test_split, GridSearchCV\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn.impute import SimpleImputer, KNNImputer","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T19:06:29.738041Z","iopub.execute_input":"2024-12-17T19:06:29.738477Z","iopub.status.idle":"2024-12-17T19:06:32.618595Z","shell.execute_reply.started":"2024-12-17T19:06:29.738436Z","shell.execute_reply":"2024-12-17T19:06:32.617190Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/child-mind/train.csv\")\ntarget_col='sii'\nX = train.drop([target_col], axis=1)\ny = train[target_col]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T19:06:35.736740Z","iopub.execute_input":"2024-12-17T19:06:35.737253Z","iopub.status.idle":"2024-12-17T19:06:35.816476Z","shell.execute_reply.started":"2024-12-17T19:06:35.737200Z","shell.execute_reply":"2024-12-17T19:06:35.815156Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imputer = KNNImputer(n_neighbors=5)\nX_imputer = imputer.fit_transform(X)\nX = pd.DataFrame(X_imputer, columns=X.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T19:07:07.240803Z","iopub.execute_input":"2024-12-17T19:07:07.241174Z","iopub.status.idle":"2024-12-17T19:07:13.222517Z","shell.execute_reply.started":"2024-12-17T19:07:07.241142Z","shell.execute_reply":"2024-12-17T19:07:13.221466Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T19:07:16.222871Z","iopub.execute_input":"2024-12-17T19:07:16.223724Z","iopub.status.idle":"2024-12-17T19:07:16.238164Z","shell.execute_reply.started":"2024-12-17T19:07:16.223664Z","shell.execute_reply":"2024-12-17T19:07:16.237079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T15:51:22.146363Z","iopub.execute_input":"2024-12-15T15:51:22.146850Z","iopub.status.idle":"2024-12-15T15:51:22.158921Z","shell.execute_reply.started":"2024-12-15T15:51:22.146811Z","shell.execute_reply":"2024-12-15T15:51:22.157614Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"params2 = {'learning_rate': 0.057885574535498495,\n 'lambda_l1': 2.08185822086368e-05,\n 'lambda_l2': 4.538468316534467,\n 'num_leaves': 209,\n 'feature_fraction': 0.930147922801105,\n 'bagging_fraction': 0.6831325500550711,\n 'bagging_freq': 1,\n 'min_child_samples': 37}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T15:56:40.495238Z","iopub.execute_input":"2024-12-15T15:56:40.495730Z","iopub.status.idle":"2024-12-15T15:56:40.504389Z","shell.execute_reply.started":"2024-12-15T15:56:40.495690Z","shell.execute_reply":"2024-12-15T15:56:40.503357Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optuna\nimport lightgbm as lgb\nfrom sklearn.metrics import cohen_kappa_score, make_scorer\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\nqwk_scorer = make_scorer(quadratic_weighted_kappa, greater_is_better=True)\n\ndef objective(trial):\n    # Suggest hyperparameters\n    param = {\n        \"objective\": \"multiclass\",\n        \"num_class\": 4,  # since sii has 4 classes\n        \"metric\": \"multi_logloss\",  # can optimize different metrics\n        \"verbosity\": -1,\n        \"boosting_type\": \"gbdt\",\n        \"lambda_l1\": trial.suggest_float(\"lambda_l1\", 1e-8, 10.0, log=True),\n        \"lambda_l2\": trial.suggest_float(\"lambda_l2\", 1e-8, 10.0, log=True),\n        \"num_leaves\": trial.suggest_int(\"num_leaves\", 2, 256),\n        \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.4, 1.0),\n        \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.4, 1.0),\n        \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 1, 7),\n        \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 5, 100),\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.1, log=True)\n    }\n\n    # Create datasets for LightGBM\n    dtrain = lgb.Dataset(X_train, label=y_train)\n    dval = lgb.Dataset(X_val, label=y_val, reference=dtrain)\n\n    # Train model with early stopping\n    gbm = lgb.train(\n        param,\n        dtrain,\n        num_boost_round=1000,\n        valid_sets=[dval],\n    )\n\n    # Predict on validation set\n    y_pred = gbm.predict(X_val)\n    # Convert probabilities to predicted classes\n    y_pred_classes = y_pred.argmax(axis=1)\n\n    # Calculate QWK\n    score = quadratic_weighted_kappa(y_val, y_pred_classes)\n    return score\n\n# Optimize with Optuna, aiming to maximize QWK\nstudy = optuna.create_study(direction=\"maximize\")\nstudy.optimize(objective, n_trials=50)\n\nprint(\"Best Value:\", study.best_value)\nprint(\"Best Params:\", study.best_params)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T19:08:29.126673Z","iopub.execute_input":"2024-12-17T19:08:29.127090Z","iopub.status.idle":"2024-12-17T19:10:38.038237Z","shell.execute_reply.started":"2024-12-17T19:08:29.127051Z","shell.execute_reply":"2024-12-17T19:10:38.036535Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"params3 = {'lambda_l1': 2.9209409268664883,\n 'lambda_l2': 0.00012976118112242174,\n 'num_leaves': 202,\n 'feature_fraction': 0.8038056112369449,\n 'bagging_fraction': 0.6991108077097957,\n 'bagging_freq': 3,\n \"learning_rate\": 0.05,\n 'min_child_samples': 85}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T16:52:22.225529Z","iopub.execute_input":"2024-12-15T16:52:22.226039Z","iopub.status.idle":"2024-12-15T16:52:22.235335Z","shell.execute_reply.started":"2024-12-15T16:52:22.225999Z","shell.execute_reply":"2024-12-15T16:52:22.234217Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import catboost as cb\nfrom sklearn.metrics import mean_squared_error\n\ndef objective_cat(trial):\n    params = {\n        \"iterations\": 1000,\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 1e-3, 0.1, log=True),\n        \"depth\": trial.suggest_int(\"depth\", 1, 10),\n        \"subsample\": trial.suggest_float(\"subsample\", 0.05, 1.0),\n        \"colsample_bylevel\": trial.suggest_float(\"colsample_bylevel\", 0.05, 1.0),\n        \"min_data_in_leaf\": trial.suggest_int(\"min_data_in_leaf\", 1, 100),\n    }\n\n    model = cb.CatBoostRegressor(**params, silent=True)\n    model.fit(X_train, y_train)\n    predictions = model.predict(X_val)\n    rmse = mean_squared_error(y_val, predictions, squared=False)\n    return rmse","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T17:06:07.406023Z","iopub.execute_input":"2024-12-15T17:06:07.406464Z","iopub.status.idle":"2024-12-15T17:06:07.834420Z","shell.execute_reply.started":"2024-12-15T17:06:07.406430Z","shell.execute_reply":"2024-12-15T17:06:07.833436Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study = optuna.create_study(direction='minimize')\nstudy.optimize(objective_cat, n_trials=50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T17:27:01.800060Z","iopub.execute_input":"2024-12-15T17:27:01.800522Z","iopub.status.idle":"2024-12-15T17:52:23.040450Z","shell.execute_reply.started":"2024-12-15T17:27:01.800475Z","shell.execute_reply":"2024-12-15T17:52:23.039488Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study.best_params","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T17:53:14.690038Z","iopub.execute_input":"2024-12-15T17:53:14.690631Z","iopub.status.idle":"2024-12-15T17:53:14.699973Z","shell.execute_reply.started":"2024-12-15T17:53:14.690575Z","shell.execute_reply":"2024-12-15T17:53:14.698740Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective_xgb(trial):\n    dtrain = xgb.DMatrix(X_train, label=y_train)\n    dvalid = xgb.DMatrix(X_val, label=y_val)\n\n    param = {\n        \"verbosity\": 0,\n        \"objective\": \"binary:logistic\",\n        # use exact for small dataset.\n        \"tree_method\": \"approx\",\n        # defines booster, gblinear for linear functions.\n        \"booster\": trial.suggest_categorical(\"booster\", [\"gbtree\", \"gblinear\", \"dart\"]),\n        # L2 regularization weight.\n        \"lambda\": trial.suggest_float(\"lambda\", 1e-8, 1.0, log=True),\n        # L1 regularization weight.\n        \"alpha\": trial.suggest_float(\"alpha\", 1e-8, 1.0, log=True),\n        # sampling ratio for training data.\n        \"subsample\": trial.suggest_float(\"subsample\", 0.2, 1.0),\n        # sampling according to each tree.\n        \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.2, 1.0),\n        \"learning_rate\": 0.05\n    }\n\n    if param[\"booster\"] in [\"gbtree\", \"dart\"]:\n        # maximum depth of the tree, signifies complexity of the tree.\n        param[\"max_depth\"] = trial.suggest_int(\"max_depth\", 3, 9, step=2)\n        # minimum child weight, larger the term more conservative the tree.\n        param[\"min_child_weight\"] = trial.suggest_int(\"min_child_weight\", 2, 10)\n        param[\"eta\"] = trial.suggest_float(\"eta\", 1e-8, 1.0, log=True)\n        # defines how selective algorithm is.\n        param[\"gamma\"] = trial.suggest_float(\"gamma\", 1e-8, 1.0, log=True)\n        param[\"grow_policy\"] = trial.suggest_categorical(\"grow_policy\", [\"depthwise\", \"lossguide\"])\n\n    if param[\"booster\"] == \"dart\":\n        param[\"sample_type\"] = trial.suggest_categorical(\"sample_type\", [\"uniform\", \"weighted\"])\n        param[\"normalize_type\"] = trial.suggest_categorical(\"normalize_type\", [\"tree\", \"forest\"])\n        param[\"rate_drop\"] = trial.suggest_float(\"rate_drop\", 1e-8, 1.0, log=True)\n        param[\"skip_drop\"] = trial.suggest_float(\"skip_drop\", 1e-8, 1.0, log=True)\n\n    bst = xgb.train(param, dtrain)\n    preds = bst.predict(dvalid)\n    pred_labels = np.rint(preds)\n    accuracy = sklearn.metrics.accuracy_score(y_val, pred_labels)\n    return accuracy\n\n# Create a study that tries to maximize accuracy\nstudy = optuna.create_study(direction=\"maximize\")\nstudy.optimize(objective, n_trials=100)\n\nprint(\"Number of finished trials:\", len(study.trials))\nprint(\"Best trial:\")\nbest_trial = study.best_trial\nprint(\"  Value:\", best_trial.value)\nprint(\"  Params:\")\nfor key, value in best_trial.params.items():\n    print(f\"    {key}: {value}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T19:18:14.137114Z","iopub.execute_input":"2024-12-15T19:18:14.137562Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study.best_params","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T19:16:01.946790Z","iopub.execute_input":"2024-12-15T19:16:01.947220Z","iopub.status.idle":"2024-12-15T19:16:01.955055Z","shell.execute_reply.started":"2024-12-15T19:16:01.947188Z","shell.execute_reply":"2024-12-15T19:16:01.953900Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study.best_params","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T05:18:18.518795Z","iopub.execute_input":"2024-12-17T05:18:18.519861Z","iopub.status.idle":"2024-12-17T05:18:18.526650Z","shell.execute_reply.started":"2024-12-17T05:18:18.519813Z","shell.execute_reply":"2024-12-17T05:18:18.525610Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective_ridge(trial):\n    # Load dataset\n    # Suggest hyperparameters\n    alpha = trial.suggest_float (\"alpha\", 0.0, 1.0)\n    # Train and evaluate model\n    model = Ridge(alpha=alpha)\n    model.fit(X_train, y_train)\n    y_pred = model.predict(X_val)\n    score = mean_squared_error(y_val, y_val)\n    return score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T19:11:28.469445Z","iopub.execute_input":"2024-12-17T19:11:28.469843Z","iopub.status.idle":"2024-12-17T19:11:28.476970Z","shell.execute_reply.started":"2024-12-17T19:11:28.469808Z","shell.execute_reply":"2024-12-17T19:11:28.475402Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create a study object\nstudy = optuna.create_study(direction=\"minimize\")\n\n# Optimize the objective function\nstudy.optimize(objective, n_trials=50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T19:11:30.368710Z","iopub.execute_input":"2024-12-17T19:11:30.369234Z","iopub.status.idle":"2024-12-17T19:35:30.313499Z","shell.execute_reply.started":"2024-12-17T19:11:30.369186Z","shell.execute_reply":"2024-12-17T19:35:30.311882Z"}},"outputs":[],"execution_count":null}]}