{"cells":[{"metadata":{},"cell_type":"markdown","source":"Hello everyone, today I would like to introduce you to a great open source hyperparameter optimization framework called [OPTUNA](https://github.com/optuna/optuna)\n
"},{"metadata":{},"cell_type":"markdown","source":"### 1. Load libs"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport optuna\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_absolute_error","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = \"/kaggle/input/predict-volcanic-eruptions-ingv-oe\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = pd.read_csv(os.path.join(PATH, 'train.csv'))\ny.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2. Load train and simple aggregation"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.DataFrame()\n\nfor s_id in tqdm(os.listdir(os.path.join(PATH, 'train'))):\n temp = pd.read_csv(os.path.join(PATH, 'train', s_id))\n temp = temp.groupby(lambda x: True).aggregate(['min','max','sum','mean','std','median'])\n temp.columns = [f'{col1}_{col2}' for col1, col2 in temp.columns]\n temp[\"segment_id\"] = int(s_id.split('.')[0])\n train = train.append(temp.reset_index(drop=True), ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(y, how='left', on='segment_id')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3. Let Optuna do its job)"},{"metadata":{},"cell_type":"markdown","source":"#### Thanks for the idea of implementing the class [Toshihiko Yanase](https://stackoverflow.com/a/62164601)"},{"metadata":{},"cell_type":"markdown","source":"#### 3.1 Init class for callback best models LightGBM + CV(5)"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Objective(object):\n\n best_models = None\n def __init__(self, kf, X, y):\n self.kf = kf\n self.X = X\n self.y = y\n\n def __call__(self, trial):\n\n mae_list = []\n param = {\n \"objective\": \"l1\",\n \"metric\": \"mae\",\n \"verbosity\": -1,\n \"boosting_type\": \"gbdt\",\n \"lambda_l1\": trial.suggest_float(\"lambda_l1\", 1e-8, 10.0, log=True),\n \"lambda_l2\": trial.suggest_float(\"lambda_l2\", 1e-8, 10.0, log=True),\n \"num_leaves\": trial.suggest_int(\"num_leaves\", 2, 256),\n \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.4, 1.0),\n \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.4, 1.0),\n \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 1, 7),\n \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 5, 100),\n }\n \n self.models = []\n for i, (train_idx, val_idx) in enumerate(self.kf.split(self.X, self.y)):\n \n train_x, valid_x = self.X.loc[train_idx], self.X.loc[val_idx]\n train_y, valid_y = self.y[train_idx], self.y[val_idx]\n \n dtrain = lgb.Dataset(train_x, label=train_y)\n dvalid = lgb.Dataset(valid_x, label=valid_y)\n \n pruning_callback = optuna.integration.LightGBMPruningCallback(trial, \"l1\")\n gbm = lgb.train(\n param, dtrain, valid_sets=[dvalid], verbose_eval=False, callbacks=[pruning_callback]\n )\n\n preds = gbm.predict(valid_x)\n mae_list.append(mean_absolute_error(valid_y, preds))\n self.models.append(gbm)\n \n return np.mean(mae_list)\n\n def callback(self, study, trial):\n if study.best_trial == trial:\n self.best_models = self.models\n self.best_params = trial.params","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 3.2 Main func for \"study\""},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_best_models(df, n_trials):\n kf = KFold(n_splits=5, shuffle=True, random_state=17)\n X = df.drop(['segment_id', 'time_to_eruption'], axis=1)\n y = df.time_to_eruption.values\n \n objective = Objective(kf, X, y)\n\n study = optuna.create_study(\n pruner=optuna.pruners.MedianPruner(n_warmup_steps=10), direction=\"minimize\"\n )\n study.optimize(objective, n_trials=n_trials, callbacks=[objective.callback])\n\n print(\"Number of finished trials: {}\".format(len(study.trials)))\n\n print(\"Best trial:\")\n trial = study.best_trial\n\n print(\" Value: {}\".format(trial.value))\n\n print(\" Params: \")\n for key, value in trial.params.items():\n print(\" {}: {}\".format(key, value))\n\n best_models = objective.best_models\n return best_models","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 3.3 Run study with 10 trials"},{"metadata":{"trusted":true},"cell_type":"code","source":"models = get_best_models(train, n_trials=10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 4. Make a submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame()\n\nfor s_id in tqdm(os.listdir(os.path.join(PATH, 'test'))):\n temp = pd.read_csv(os.path.join(PATH, 'test', s_id))\n temp = temp.groupby(lambda x: True).aggregate(['min','max','sum','mean','std','median'])\n temp.columns = [f'{col1}_{col2}' for col1, col2 in temp.columns]\n temp = temp.reset_index(drop=True)\n preds = 0\n for model in models:\n preds += model.predict(temp) / len(models)\n submission = submission.append({\"segment_id\": s_id.split('.')[0], \"time_to_eruption\": preds[0]}, ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}