{"cells":[{"metadata":{},"cell_type":"markdown","source":"Idea of the notebook is how to use hyperopt and flaml library to tune parameters for lightgbm.\n\nIf you're interested, @mlconsult also published a great notebook on [Tuning lightgbm with optuna](https://www.kaggle.com/mlconsult/how-to-tune-lgbm-with-optuna)\n\n### Reference\n\nMost of this notebook is inspired from the wonderful gitrepos\n\n1. [ml course ai hyperopt](https://github.com/Yorko/mlcourse.ai/blob/master/jupyter_english/tutorials/hyperparameters_tunning_ilya_larchenko.ipynb)\n\n2. [Flaml github](https://github.com/microsoft/FLAML)\n\n\nThanks @devinanzelmo for the [wifi features](https://www.kaggle.com/devinanzelmo/wifi-features) on how to use wifi features. Here I'm using 1000 as the min count for wifi bssid"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install flaml[notebook]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Libraries import"},{"metadata":{"trusted":true},"cell_type":"code","source":"# common imports\nimport os\nimport glob\nimport random\nimport numpy as np\nimport pandas as pd\nfrom scipy.stats import randint\n\n# models libraries\n#from lightgbm.sklearn import LGBMRegressor\nfrom lightgbm import LGBMRegressor,LGBMClassifier\nfrom sklearn.svm import SVR\n\n# sklearn imports \nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import GroupKFold, cross_val_score\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.pipeline import make_pipeline\n\n# hyperopt imports to perform bayesian optimisation \nfrom hyperopt import Trials, anneal, fmin, hp, tpe\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"''' import AutoML class from flaml package '''\nfrom flaml import AutoML\nautoml = AutoML()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# the metric used in this competition\ndef comp_metric(xhat, yhat, fhat, x, y, f):\n    intermediate = np.sqrt(np.power(xhat - x,2) + np.power(yhat-y,2)) + 15 * np.abs(fhat-f)\n    return intermediate.sum()/xhat.shape[0]\n\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    #torch.manual_seed(seed)\n    #torch.cuda.manual_seed(seed)\n    #torch.backends.cudnn.deterministic = True\n\nSEED = 42\nseed_everything(SEED)\n\n\n# cv strategy \nN_FOLDS = 5\nfolds = GroupKFold(n_splits=N_FOLDS)\n\n# which optimisation to perform\nperform_RandomCVSearch = False\nperform_hyperoptParsenEstimator = False\nperform_hyperoptSimpleAnnealing = False\nperfom_flaml = True\n\n\n# number of experiments to perform for hyperopt\nn_iter = 100\n\n# target time for flaml, in seconds\ntimeLimit = 30 ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read sample data"},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_dir = \"../input/indoor-navigation-and-location-wifi-features/wifi_features\"\n\n# get our train and test files\ntrain_files = sorted(glob.glob(os.path.join(feature_dir, 'train/*_train.csv')))\ntest_files = sorted(glob.glob(os.path.join(feature_dir, 'test/*_test.csv')))\nssubm = pd.read_csv('../input/indoor-location-navigation/sample_submission.csv', index_col=0)\nprint(len(train_files),len(test_files))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# selecting a particular site and choosing y coorindate\ne = 0\ndata = pd.read_csv(train_files[e], index_col=0)\nprint(data.shape)\ndata.head(3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare model inputs and outputs"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = data.iloc[:,:-4].values.astype(int)\ny_trainy = data.iloc[:,-3].values.astype(float)\ny_trainx = data.iloc[:,-4].values.astype(float)\ny_trainf = data.iloc[:,-2].values.astype(float)\ngroups = data[\"path\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# normlise inputs\nstdScaler = StandardScaler()\nx_train = stdScaler.fit_transform(x_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Baseline Lightgbm and SVR model "},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# baseline lightgbm model\nmodel = LGBMRegressor(n_estimators=125, num_leaves=90, random_state=SEED)\nresults = -cross_val_score(model, X=x_train, y=y_trainy, groups=groups, \n                              scoring=\"neg_mean_squared_error\", cv=folds, n_jobs=-1)\nprint(f\"Cross val score for y coordinate is {results.mean()}\")\nprint(results)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# baseline svm model\n# svrModel = SVR(C=100.0, epsilon=0.01)\n# results = -cross_val_score(svrModel, X=x_train, y=y_trainy, groups=groups, \n#                              scoring=\"neg_mean_squared_error\", cv=folds, n_jobs=-1)\n# print(f\"Cross val score for y coordinate is {results.mean()}\")\n# print(results)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Randomized grid search"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nif perform_RandomCVSearch == True:\n\n    param_grid_rand = {\n    \"learning_rate\": np.logspace(-5, 0, 100),\n    \"max_depth\": randint(2, 20),\n    \"n_estimators\": randint(100, 2000),\n    \"random_state\": [SEED],\n    }\n    \n    rs = RandomizedSearchCV(model,\n        param_grid_rand,\n        n_iter=n_iter,\n        scoring=\"neg_mean_squared_error\",\n        #fit_params=None,\n        n_jobs=-1,\n        cv=folds,\n        verbose=True,\n        random_state=SEED,\n    )\n\n    rs.fit(x_train, y_trainy, groups=groups)\n    print(\"Best MSE {:.3f} params {}\".format(-rs.best_score_, rs.best_params_))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if perform_RandomCVSearch == True:\n    rs_results_df = pd.DataFrame(\n        np.transpose(\n            [\n                -rs.cv_results_[\"mean_test_score\"],\n                rs.cv_results_[\"param_learning_rate\"].data,\n                rs.cv_results_[\"param_max_depth\"].data,\n                rs.cv_results_[\"param_n_estimators\"].data,\n            ]\n        ),\n        columns=[\"score\", \"learning_rate\", \"max_depth\", \"n_estimators\"],\n    )\n    rs_results_df.plot(subplots=True, figsize=(10, 10))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Hyperopt tuning methods\n### Tree-structured Parzen Estimator and Simple Annealing"},{"metadata":{"trusted":true},"cell_type":"code","source":"def gb_mse_cv(params, X=x_train, y=y_trainy, cv=folds,random_state=SEED):\n    # the function gest a set of variable parameters in \"param\"\n    lgb_params = {\n        \"objective\": \"regression\",\n        \"metric\": \"l2\",\n        \"verbosity\": -1,\n        \n        # fixed params\n        \"boosting_type\": \"gbdt\", \n        \"subsample_freq\":20,\n        \"max_depth\":6,\n\n        # variable parameters\n        \"num_leaves\": int(params[\"num_leaves\"]),\n        \"feature_fraction\": float(params[\"feature_fraction\"]),\n        \"bagging_fraction\": float(params[\"bagging_fraction\"]),        \n        \"learning_rate\": float(params[\"learning_rate\"]),\n        \"n_estimators\": int(params[\"n_estimators\"]),\n        \"lambda_l1\": float(params[\"lambda_l1\"]),\n        \"lambda_l2\": float(params[\"lambda_l2\"]),\n        \"min_child_samples\": int(params[\"min_child_samples\"]),\n    }\n    \n    # we use this params to create a new LGBM Regressor\n    model = LGBMRegressor(random_state=SEED, **lgb_params)\n\n    # and then conduct the cross validation with the same folds as before\n    score = -cross_val_score(model, X=X, y=y, groups=groups, scoring=\"neg_mean_squared_error\",\n                             cv=folds, n_jobs=-1).mean()\n    return score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# possible values of parameters\nspace = {\n        # variable parameters\n        \"num_leaves\": hp.quniform(\"num_leaves\", 10, 100, 1),\n        \"feature_fraction\": hp.choice('feature_fraction', np.linspace(0.4, 0.7, 3,dtype=float)),\n        \"bagging_fraction\": hp.choice('bagging_fraction', np.linspace(0.4, 0.7, 3,dtype=float)),        \n        \"learning_rate\": hp.loguniform(\"learning_rate\", -2, -1), \n        \"n_estimators\": hp.quniform(\"n_estimators\", 500, 10000, 1),\n        \"lambda_l1\": hp.loguniform(\"lambda_l1\", -6, 1.0), \n        \"lambda_l2\": hp.loguniform(\"lambda_l2\", -6, 1.0), \n        \"min_child_samples\": hp.quniform(\"min_child_samples\", 5, 100, 1)\n        }\n\n# trials will contain logging information\ntrials = Trials()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tuningAlgorithm = None\n\n# choice of tuning algorithm\nif perform_hyperoptParsenEstimator == True:\n    tuningAlgorithm = tpe.suggest\nif perform_hyperoptSimpleAnnealing == True:\n    tuningAlgorithm = anneal.suggest\nif perfom_flaml == True:\n    tuningAlgorithm = 'flaml'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tuningAlgorithm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nif((perform_hyperoptParsenEstimator == True) or (perform_hyperoptSimpleAnnealing == True)):\n    best = fmin(\n        fn=gb_mse_cv,                       # function to optimize\n        space=space,                        # search space\n        algo=tuningAlgorithm,               # optimization algorithm, hyperotp will select its parameters automatically\n        max_evals=n_iter,                   # maximum number of iterations\n        trials=trials,                      # logging\n        show_progressbar=True,\n        rstate=np.random.RandomState(SEED), # fixing random state for the reproducibility\n    )\n    print(\"Best MSE {:.3f} params {}\".format(gb_mse_cv(best), best))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot optimizer results"},{"metadata":{"trusted":true},"cell_type":"code","source":"if((perform_hyperoptParsenEstimator == True) or (perform_hyperoptSimpleAnnealing == True)):\n    optimizer_results = np.array([[\n                x[\"result\"][\"loss\"],  \n                x[\"misc\"][\"vals\"][\"n_estimators\"][0],    \n                x[\"misc\"][\"vals\"][\"learning_rate\"][0],\n                x[\"misc\"][\"vals\"][\"num_leaves\"][0],\n                x[\"misc\"][\"vals\"][\"feature_fraction\"][0],\n                x[\"misc\"][\"vals\"][\"bagging_fraction\"][0],\n                x[\"misc\"][\"vals\"][\"lambda_l1\"][0],\n                x[\"misc\"][\"vals\"][\"lambda_l2\"][0],\n                x[\"misc\"][\"vals\"][\"min_child_samples\"][0],        \n            ] for x in trials.trials ])\n\n    # create a df to plot\n    results_columns = [\"score\", \"n_estimators\", \"learning_rate\", \"num_leaves\", \"feature_fraction\",\n                       \"bagging_fraction\", \"lambda_l1\", \"lambda_l2\", \"min_child_samples\"]\n    optimizer_results_df = pd.DataFrame(optimizer_results, columns=results_columns)\n    optimizer_results_df.plot(subplots=True, figsize=(10, 10));","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if perfom_flaml == True:\n    settings = {\n        \"metric\": 'mse', # primary metrics for regression can be chosen from: ['mae','mse','r2']\n        \"task\": 'regression', # task type        \n        \"log_file_name\": 'lightgbm_ycoorindate.log', # flaml log file    \n        \"estimator_list\": ['lgbm', 'xgboost'], # list of ML learners; we tune lightgbm in this example\n        \"time_budget\": timeLimit, # total running time in seconds\n        \"eval_method\": 'cv',\n        \"n_splits\" : N_FOLDS, \n    }\n\n    # fit algorithms\n    automl.fit(X_train = x_train, y_train = y_trainy, **settings)\n\n    print('Best hyperparmeter config:', automl.best_config)\n    print('Best mse on validation data: {0:.4g}'.format(automl.best_loss))\n    print('Training duration of best run: {0:.4g} s'.format(automl.best_config_train_time))\n    \n    print(automl.model)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"kaggle","language":"python","name":"kaggle"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.8"}},"nbformat":4,"nbformat_minor":4}