{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport warnings\nwarnings.filterwarnings('ignore')\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2022-10-18T19:39:23.286797Z","iopub.status.busy":"2022-10-18T19:39:23.286265Z","iopub.status.idle":"2022-10-18T19:39:23.297306Z","shell.execute_reply":"2022-10-18T19:39:23.295924Z"},"papermill":{"duration":0.026809,"end_time":"2022-10-18T19:39:23.300462","exception":false,"start_time":"2022-10-18T19:39:23.273653","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get data","metadata":{"papermill":{"duration":0.009043,"end_time":"2022-10-18T19:39:23.319607","exception":false,"start_time":"2022-10-18T19:39:23.310564","status":"completed"},"tags":[]}},{"cell_type":"code","source":"dtypes_df = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_dtypes.csv')\ndtypes = {k: v for (k, v) in zip(dtypes_df.column, dtypes_df.dtype)}\ntrain0 = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_0.csv', dtype=dtypes)","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:39:23.340930Z","iopub.status.busy":"2022-10-18T19:39:23.340506Z","iopub.status.idle":"2022-10-18T19:40:01.528591Z","shell.execute_reply":"2022-10-18T19:40:01.527298Z"},"papermill":{"duration":38.202392,"end_time":"2022-10-18T19:40:01.531782","exception":false,"start_time":"2022-10-18T19:39:23.329390","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train0 = train0.sample(frac=0.1) # With NaN (214938, 61)\nprint(\"With NaN: {}\".format(train0.shape)) ","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:01.554195Z","iopub.status.busy":"2022-10-18T19:40:01.553740Z","iopub.status.idle":"2022-10-18T19:40:01.559357Z","shell.execute_reply":"2022-10-18T19:40:01.558469Z"},"papermill":{"duration":0.019714,"end_time":"2022-10-18T19:40:01.562678","exception":false,"start_time":"2022-10-18T19:40:01.542964","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test0 = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/test.csv')","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:01.614682Z","iopub.status.busy":"2022-10-18T19:40:01.614275Z","iopub.status.idle":"2022-10-18T19:40:13.299218Z","shell.execute_reply":"2022-10-18T19:40:13.298123Z"},"papermill":{"duration":11.698555,"end_time":"2022-10-18T19:40:13.301969","exception":false,"start_time":"2022-10-18T19:40:01.603414","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Memory reduction\ntrain0 7.2GB","metadata":{"papermill":{"duration":0.009793,"end_time":"2022-10-18T19:40:13.321657","exception":false,"start_time":"2022-10-18T19:40:13.311864","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:13.343647Z","iopub.status.busy":"2022-10-18T19:40:13.342921Z","iopub.status.idle":"2022-10-18T19:40:13.454573Z","shell.execute_reply":"2022-10-18T19:40:13.453473Z"},"papermill":{"duration":0.125734,"end_time":"2022-10-18T19:40:13.457248","exception":false,"start_time":"2022-10-18T19:40:13.331514","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### [Dtypes transformation (Reduce Memory Usage 75%)](https://www.kaggle.com/competitions/tabular-playground-series-oct-2022/discussion/356540) thanks by [@sergiosaharovskiy](https://www.kaggle.com/sergiosaharovskiy)","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int8', 'int16', 'int32', 'int64',\n            'float16', 'float32', 'float64']\n    \n    for col in df.columns:\n        if col == 'team_scoring_next':\n            continue\n        col_type = df[col].dtypes\n        limit = abs(df[col]).max()\n\n        for tp in numerics:\n            cond1 = str(col_type)[0] == tp[0]\n            if tp[0] == 'i': cond2 = limit <= np.iinfo(tp).max\n            else: cond2 = limit <= np.finfo(tp).max\n\n            if cond1 and cond2:\n                df[col] = df[col].astype(tp)\n                break\n    return df","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:13.478579Z","iopub.status.busy":"2022-10-18T19:40:13.478186Z","iopub.status.idle":"2022-10-18T19:40:13.485844Z","shell.execute_reply":"2022-10-18T19:40:13.484640Z"},"papermill":{"duration":0.021385,"end_time":"2022-10-18T19:40:13.488354","exception":false,"start_time":"2022-10-18T19:40:13.466969","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train1 = reduce_mem_usage(train0)\ntest1 = reduce_mem_usage(test0)","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:13.509910Z","iopub.status.busy":"2022-10-18T19:40:13.509493Z","iopub.status.idle":"2022-10-18T19:40:20.425537Z","shell.execute_reply":"2022-10-18T19:40:20.424364Z"},"papermill":{"duration":6.930227,"end_time":"2022-10-18T19:40:20.428326","exception":false,"start_time":"2022-10-18T19:40:13.498099","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\n","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:20.450587Z","iopub.status.busy":"2022-10-18T19:40:20.449671Z","iopub.status.idle":"2022-10-18T19:40:20.562580Z","shell.execute_reply":"2022-10-18T19:40:20.561562Z"},"papermill":{"duration":0.12656,"end_time":"2022-10-18T19:40:20.565025","exception":false,"start_time":"2022-10-18T19:40:20.438465","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare the data to better expose\n## Feature selection\n### Drop the attributes that provide no useful information","metadata":{"papermill":{"duration":0.009419,"end_time":"2022-10-18T19:40:20.584575","exception":false,"start_time":"2022-10-18T19:40:20.575156","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train1 = train1.drop(columns=[\"game_num\",\"event_id\",\"event_time\",\"player_scoring_next\",\"team_scoring_next\"])","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:20.607506Z","iopub.status.busy":"2022-10-18T19:40:20.606790Z","iopub.status.idle":"2022-10-18T19:40:21.202849Z","shell.execute_reply":"2022-10-18T19:40:21.201610Z"},"papermill":{"duration":0.610972,"end_time":"2022-10-18T19:40:21.205681","exception":false,"start_time":"2022-10-18T19:40:20.594709","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature engineering\n### Aggregate features into promising new features\n#### Euclidean distance\nCalculate Euclidean distance position-based player data, position-based ball data\n\n##### [Data Representation and Feature Engineering](https://www.kaggle.com/competitions/tabular-playground-series-oct-2022/discussion/356718) thanks by [Ravi Shah](https://www.kaggle.com/ravishah1)","metadata":{"papermill":{"duration":0.009391,"end_time":"2022-10-18T19:40:21.225371","exception":false,"start_time":"2022-10-18T19:40:21.215980","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_p0 = train1[['p0_pos_x', 'p0_pos_y', 'p0_pos_z']].values\ntrain_p1 = train1[['p1_pos_x', 'p1_pos_y', 'p1_pos_z']].values\ntrain_p2 = train1[['p2_pos_x', 'p2_pos_y', 'p2_pos_z']].values\none_soccer_goal_to_p0_2 = np.array([-10.0, 104.3125, 0.0], dtype=np.float32)\ntwo_soccer_goal_to_p0_2 = np.array([10.0, 104.3125, 0.0], dtype=np.float32)\n\ntrain_p3 = train1[['p3_pos_x', 'p3_pos_y', 'p3_pos_z']].values\ntrain_p4 = train1[['p4_pos_x', 'p4_pos_y', 'p4_pos_z']].values\ntrain_p5 = train1[['p5_pos_x', 'p5_pos_y', 'p5_pos_z']].values\ntrain_ball = train1[['ball_pos_x', 'ball_pos_y', 'ball_pos_z']].values\none_soccer_goal_to_p3_5 = np.array([-10.0, -104.3125, 0.0], dtype=np.float32)\ntwo_soccer_goal_to_p3_5 = np.array([10.0, -104.3125, 0.0], dtype=np.float32)\n\ntrain1['dist_p0_ball'] = np.linalg.norm(train_p0 - train_ball, axis=1)\ntrain1['dist_p1_ball'] = np.linalg.norm(train_p1 - train_ball, axis=1)\ntrain1['dist_p2_ball'] = np.linalg.norm(train_p2 - train_ball, axis=1)\ntrain1['dist_p3_ball'] = np.linalg.norm(train_p3 - train_ball, axis=1)\ntrain1['dist_p4_ball'] = np.linalg.norm(train_p4 - train_ball, axis=1)\ntrain1['dist_p5_ball'] = np.linalg.norm(train_p5 - train_ball, axis=1)\ntrain1['dist_p0_goal'] = np.linalg.norm(train_p0 - one_soccer_goal_to_p0_2, axis=1)\ntrain1['dist_p1_goal'] = np.linalg.norm(train_p1 - one_soccer_goal_to_p0_2, axis=1)\ntrain1['dist_p2_goal'] = np.linalg.norm(train_p2 - one_soccer_goal_to_p0_2, axis=1)\ntrain1['dist_p3_goal'] = np.linalg.norm(train_p3 - one_soccer_goal_to_p3_5, axis=1)\ntrain1['dist_p4_goal'] = np.linalg.norm(train_p4 - one_soccer_goal_to_p3_5, axis=1)\ntrain1['dist_p5_goal'] = np.linalg.norm(train_p5 - one_soccer_goal_to_p3_5, axis=1)","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:21.247280Z","iopub.status.busy":"2022-10-18T19:40:21.246870Z","iopub.status.idle":"2022-10-18T19:40:23.446498Z","shell.execute_reply":"2022-10-18T19:40:23.445448Z"},"papermill":{"duration":2.214037,"end_time":"2022-10-18T19:40:23.449439","exception":false,"start_time":"2022-10-18T19:40:21.235402","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_p0 = test1[['p0_pos_x', 'p0_pos_y', 'p0_pos_z']].values\ntest_p1 = test1[['p1_pos_x', 'p1_pos_y', 'p1_pos_z']].values\ntest_p2 = test1[['p2_pos_x', 'p2_pos_y', 'p2_pos_z']].values\ntest_p3 = test1[['p3_pos_x', 'p3_pos_y', 'p3_pos_z']].values\ntest_p4 = test1[['p4_pos_x', 'p4_pos_y', 'p4_pos_z']].values\ntest_p5 = test1[['p5_pos_x', 'p5_pos_y', 'p5_pos_z']].values\ntest_ball = test1[['ball_pos_x', 'ball_pos_y', 'ball_pos_z']].values \ntest1['dist_p0_ball'] = np.linalg.norm(test_p0 - test_ball, axis=1)\ntest1['dist_p1_ball'] = np.linalg.norm(test_p1 - test_ball, axis=1)\ntest1['dist_p2_ball'] = np.linalg.norm(test_p2 - test_ball, axis=1)\ntest1['dist_p3_ball'] = np.linalg.norm(test_p3 - test_ball, axis=1)\ntest1['dist_p4_ball'] = np.linalg.norm(test_p4 - test_ball, axis=1)\ntest1['dist_p5_ball'] = np.linalg.norm(test_p5 - test_ball, axis=1)\ntest1['dist_p0_goal'] = np.linalg.norm(test_p0 - one_soccer_goal_to_p0_2, axis=1)\ntest1['dist_p1_goal'] = np.linalg.norm(test_p1 - one_soccer_goal_to_p0_2, axis=1)\ntest1['dist_p2_goal'] = np.linalg.norm(test_p2 - one_soccer_goal_to_p0_2, axis=1)\ntest1['dist_p3_goal'] = np.linalg.norm(test_p3 - one_soccer_goal_to_p3_5, axis=1)\ntest1['dist_p4_goal'] = np.linalg.norm(test_p4 - one_soccer_goal_to_p3_5, axis=1)\ntest1['dist_p5_goal'] = np.linalg.norm(test_p5 - one_soccer_goal_to_p3_5, axis=1)","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:23.471902Z","iopub.status.busy":"2022-10-18T19:40:23.470857Z","iopub.status.idle":"2022-10-18T19:40:24.271126Z","shell.execute_reply":"2022-10-18T19:40:24.269965Z"},"papermill":{"duration":0.814583,"end_time":"2022-10-18T19:40:24.274183","exception":false,"start_time":"2022-10-18T19:40:23.459600","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Drop the attributes that provide no useful information","metadata":{"papermill":{"duration":0.009447,"end_time":"2022-10-18T19:40:24.293850","exception":false,"start_time":"2022-10-18T19:40:24.284403","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#Remove p{i}_pos_[xyz] cols\ncols = [col for col in train1.columns if \"pos\" in col]\ncols = [col for col in cols if not \"ball\" in col]\ntrain1 = train1.drop(columns=cols)\ntest1 = test1.drop(columns=cols)\n#train1.columns","metadata":{"_kg_hide-output":true,"execution":{"iopub.execute_input":"2022-10-18T19:40:24.315569Z","iopub.status.busy":"2022-10-18T19:40:24.315152Z","iopub.status.idle":"2022-10-18T19:40:25.003585Z","shell.execute_reply":"2022-10-18T19:40:25.002319Z"},"papermill":{"duration":0.70279,"end_time":"2022-10-18T19:40:25.006457","exception":false,"start_time":"2022-10-18T19:40:24.303667","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train2 = train1.copy()\ndel train1","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:25.027683Z","iopub.status.busy":"2022-10-18T19:40:25.027272Z","iopub.status.idle":"2022-10-18T19:40:25.138646Z","shell.execute_reply":"2022-10-18T19:40:25.136124Z"},"papermill":{"duration":0.126881,"end_time":"2022-10-18T19:40:25.143125","exception":false,"start_time":"2022-10-18T19:40:25.016244","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prepare train data: X, y","metadata":{"papermill":{"duration":0.009834,"end_time":"2022-10-18T19:40:25.165107","exception":false,"start_time":"2022-10-18T19:40:25.155273","status":"completed"},"tags":[]}},{"cell_type":"code","source":"yA = train2[\"team_A_scoring_within_10sec\"].astype('int8')\nyB = train2[\"team_B_scoring_within_10sec\"].astype('int8')","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:25.187004Z","iopub.status.busy":"2022-10-18T19:40:25.186172Z","iopub.status.idle":"2022-10-18T19:40:25.193512Z","shell.execute_reply":"2022-10-18T19:40:25.192703Z"},"papermill":{"duration":0.020929,"end_time":"2022-10-18T19:40:25.195869","exception":false,"start_time":"2022-10-18T19:40:25.174940","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train3 = train2.drop(columns=['team_A_scoring_within_10sec', 'team_B_scoring_within_10sec'])\nX = train3.copy()","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:25.217596Z","iopub.status.busy":"2022-10-18T19:40:25.216806Z","iopub.status.idle":"2022-10-18T19:40:25.645548Z","shell.execute_reply":"2022-10-18T19:40:25.644133Z"},"papermill":{"duration":0.442927,"end_time":"2022-10-18T19:40:25.648683","exception":false,"start_time":"2022-10-18T19:40:25.205756","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:25.670606Z","iopub.status.busy":"2022-10-18T19:40:25.670208Z","iopub.status.idle":"2022-10-18T19:40:25.784174Z","shell.execute_reply":"2022-10-18T19:40:25.783022Z"},"papermill":{"duration":0.127797,"end_time":"2022-10-18T19:40:25.786588","exception":false,"start_time":"2022-10-18T19:40:25.658791","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine-tune your model","metadata":{"papermill":{"duration":0.009395,"end_time":"2022-10-18T19:40:25.806086","exception":false,"start_time":"2022-10-18T19:40:25.796691","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from sklearn.ensemble import GradientBoostingRegressor, HistGradientBoostingRegressor\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom xgboost import XGBRegressor\nfrom sklearn.model_selection import cross_validate\n\nimport optuna","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:25.827878Z","iopub.status.busy":"2022-10-18T19:40:25.826701Z","iopub.status.idle":"2022-10-18T19:40:28.969173Z","shell.execute_reply":"2022-10-18T19:40:28.967930Z"},"papermill":{"duration":3.156051,"end_time":"2022-10-18T19:40:28.971932","exception":false,"start_time":"2022-10-18T19:40:25.815881","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fine-tune the hyperparameters\n#### Team A","metadata":{"papermill":{"duration":0.009518,"end_time":"2022-10-18T19:40:28.991498","exception":false,"start_time":"2022-10-18T19:40:28.981980","status":"completed"},"tags":[]}},{"cell_type":"code","source":"KFOLDS = 5","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:29.012949Z","iopub.status.busy":"2022-10-18T19:40:29.012505Z","iopub.status.idle":"2022-10-18T19:40:29.017951Z","shell.execute_reply":"2022-10-18T19:40:29.016803Z"},"papermill":{"duration":0.019064,"end_time":"2022-10-18T19:40:29.020362","exception":false,"start_time":"2022-10-18T19:40:29.001298","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def HGBRegressor_objective_yA(trial):\n    max_leaf_nodes = trial.suggest_int('max_leaf_nodes', 10, 40)\n    learning_rate = trial.suggest_float('learning_rate', 0, 0.5)\n    max_depth = trial.suggest_int('max_depth', 3, 20)\n    min_samples_leaf = trial.suggest_int('min_samples_leaf', 10, 40)\n    max_bins = trial.suggest_int('max_bins', 50, 600)\n    \n    model = HistGradientBoostingRegressor(\n        #task_type='GPU',\n        learning_rate= learning_rate,\n        max_depth= max_depth,\n        min_samples_leaf = min_samples_leaf,\n        max_bins= max_bins,\n        max_leaf_nodes = max_leaf_nodes,\n        random_state=973\n    )\n\n    cv_a = cross_validate(model, X, yA, cv=KFOLDS, scoring='neg_mean_squared_error') \n    test_score = cv_a['test_score'].sum()/KFOLDS\n    return test_score","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:40:29.042002Z","iopub.status.busy":"2022-10-18T19:40:29.041617Z","iopub.status.idle":"2022-10-18T19:40:29.050263Z","shell.execute_reply":"2022-10-18T19:40:29.049091Z"},"papermill":{"duration":0.022228,"end_time":"2022-10-18T19:40:29.052581","exception":false,"start_time":"2022-10-18T19:40:29.030353","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_1 = optuna.create_study(direction= 'maximize')\nstudy_1.optimize(HGBRegressor_objective_yA, n_trials=15)","metadata":{"_kg_hide-output":true,"papermill":{"duration":878.937647,"end_time":"2022-10-18T19:55:08.000111","exception":false,"start_time":"2022-10-18T19:40:29.062464","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optuna.visualization.plot_contour(study_1) ","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:55:08.023551Z","iopub.status.busy":"2022-10-18T19:55:08.023136Z","iopub.status.idle":"2022-10-18T19:55:09.089212Z","shell.execute_reply":"2022-10-18T19:55:09.088071Z"},"papermill":{"duration":1.080694,"end_time":"2022-10-18T19:55:09.091695","exception":false,"start_time":"2022-10-18T19:55:08.011001","status":"completed"},"tags":[],"collapsed":true,"jupyter":{"outputs_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optuna.visualization.plot_optimization_history(study_1)","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:55:09.117029Z","iopub.status.busy":"2022-10-18T19:55:09.116642Z","iopub.status.idle":"2022-10-18T19:55:09.131859Z","shell.execute_reply":"2022-10-18T19:55:09.130547Z"},"papermill":{"duration":0.030702,"end_time":"2022-10-18T19:55:09.134420","exception":false,"start_time":"2022-10-18T19:55:09.103718","status":"completed"},"tags":[],"collapsed":true,"jupyter":{"outputs_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Team B","metadata":{"papermill":{"duration":0.01146,"end_time":"2022-10-18T19:55:09.159744","exception":false,"start_time":"2022-10-18T19:55:09.148284","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def HGBRegressor_objective_yB(trial):\n    max_leaf_nodes = trial.suggest_int('max_leaf_nodes', 10, 40)\n    learning_rate = trial.suggest_float('learning_rate', 0, 0.5)\n    max_depth = trial.suggest_int('max_depth', 3, 20)\n    min_samples_leaf = trial.suggest_int('min_samples_leaf', 10, 40)\n    max_bins = trial.suggest_int('max_bins', 50, 600)\n    \n    model = HistGradientBoostingRegressor(\n        #task_type='GPU',\n        learning_rate= learning_rate,\n        max_depth= max_depth,\n        min_samples_leaf = min_samples_leaf,\n        max_bins= max_bins,\n        max_leaf_nodes = max_leaf_nodes,\n        random_state=973\n    )\n\n    cv_b = cross_validate(model, X, yB, cv=KFOLDS, scoring='neg_mean_squared_error') \n    test_score = cv_b['test_score'].sum()/KFOLDS\n    return test_score","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:55:09.185867Z","iopub.status.busy":"2022-10-18T19:55:09.184955Z","iopub.status.idle":"2022-10-18T19:55:09.194350Z","shell.execute_reply":"2022-10-18T19:55:09.192871Z"},"papermill":{"duration":0.025356,"end_time":"2022-10-18T19:55:09.196905","exception":false,"start_time":"2022-10-18T19:55:09.171549","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_2 = optuna.create_study(direction= 'maximize')\nstudy_2.optimize(HGBRegressor_objective_yB, n_trials=15)","metadata":{"execution":{"iopub.execute_input":"2022-10-18T19:55:09.222541Z","iopub.status.busy":"2022-10-18T19:55:09.222144Z","iopub.status.idle":"2022-10-18T20:09:33.897576Z","shell.execute_reply":"2022-10-18T20:09:33.896310Z"},"papermill":{"duration":864.703827,"end_time":"2022-10-18T20:09:33.912732","exception":false,"start_time":"2022-10-18T19:55:09.208905","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optuna.visualization.plot_contour(study_2) ","metadata":{"execution":{"iopub.execute_input":"2022-10-18T20:09:33.941074Z","iopub.status.busy":"2022-10-18T20:09:33.940696Z","iopub.status.idle":"2022-10-18T20:09:34.517110Z","shell.execute_reply":"2022-10-18T20:09:34.515861Z"},"papermill":{"duration":0.593318,"end_time":"2022-10-18T20:09:34.519401","exception":false,"start_time":"2022-10-18T20:09:33.926083","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optuna.visualization.plot_optimization_history(study_2)","metadata":{"execution":{"iopub.execute_input":"2022-10-18T20:09:34.548874Z","iopub.status.busy":"2022-10-18T20:09:34.548437Z","iopub.status.idle":"2022-10-18T20:09:34.563087Z","shell.execute_reply":"2022-10-18T20:09:34.562068Z"},"papermill":{"duration":0.032659,"end_time":"2022-10-18T20:09:34.565881","exception":false,"start_time":"2022-10-18T20:09:34.533222","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Measure its performance","metadata":{"papermill":{"duration":0.013362,"end_time":"2022-10-18T20:09:34.593392","exception":false,"start_time":"2022-10-18T20:09:34.580030","status":"completed"},"tags":[]}},{"cell_type":"code","source":"est_A = HistGradientBoostingRegressor(random_state=973, **study_1.best_params)\nest_A.fit(X, yA)\nest_B = HistGradientBoostingRegressor(random_state=973, **study_2.best_params)\nest_B.fit(X, yB)","metadata":{"execution":{"iopub.execute_input":"2022-10-18T20:09:34.622609Z","iopub.status.busy":"2022-10-18T20:09:34.622235Z","iopub.status.idle":"2022-10-18T20:10:47.779931Z","shell.execute_reply":"2022-10-18T20:10:47.778782Z"},"papermill":{"duration":73.189982,"end_time":"2022-10-18T20:10:47.797003","exception":false,"start_time":"2022-10-18T20:09:34.607021","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv_a = cross_validate(est_A, X, yA, cv=KFOLDS, return_estimator=True)\ncv_b = cross_validate(est_B, X, yB, cv=KFOLDS, return_estimator=True)","metadata":{"execution":{"iopub.execute_input":"2022-10-18T20:10:47.827175Z","iopub.status.busy":"2022-10-18T20:10:47.826512Z","iopub.status.idle":"2022-10-18T20:15:54.355950Z","shell.execute_reply":"2022-10-18T20:15:54.354645Z"},"papermill":{"duration":306.547789,"end_time":"2022-10-18T20:15:54.358840","exception":false,"start_time":"2022-10-18T20:10:47.811051","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_a = np.zeros(test1.shape[0])\nfor estimator in cv_a['estimator']:\n    pred_a += estimator.predict(test1.drop(columns=['id']).values)\n    cv_a['test_score']\npred_a /= KFOLDS","metadata":{"execution":{"iopub.execute_input":"2022-10-18T20:15:54.389550Z","iopub.status.busy":"2022-10-18T20:15:54.389110Z","iopub.status.idle":"2022-10-18T20:16:11.472311Z","shell.execute_reply":"2022-10-18T20:16:11.471307Z"},"papermill":{"duration":17.101284,"end_time":"2022-10-18T20:16:11.474849","exception":false,"start_time":"2022-10-18T20:15:54.373565","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_b = np.zeros(test1.shape[0])\nfor estimator in cv_b['estimator']:\n    pred_b += estimator.predict(test1.drop(columns=['id']).values)\n    cv_b['test_score']\npred_b /= KFOLDS\n","metadata":{"execution":{"iopub.execute_input":"2022-10-18T20:16:11.505559Z","iopub.status.busy":"2022-10-18T20:16:11.505104Z","iopub.status.idle":"2022-10-18T20:16:23.590315Z","shell.execute_reply":"2022-10-18T20:16:23.589256Z"},"papermill":{"duration":12.103269,"end_time":"2022-10-18T20:16:23.593287","exception":false,"start_time":"2022-10-18T20:16:11.490018","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.013971,"end_time":"2022-10-18T20:16:23.622618","exception":false,"start_time":"2022-10-18T20:16:23.608647","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_submission = pd.DataFrame(\n    {\n        \"id\": test1['id'],\n        \"team_A_scoring_within_10sec\": pred_a,\n        \"team_B_scoring_within_10sec\": pred_b\n    }\n) \n\ndf_submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.execute_input":"2022-10-18T20:16:23.652613Z","iopub.status.busy":"2022-10-18T20:16:23.652129Z","iopub.status.idle":"2022-10-18T20:16:26.329667Z","shell.execute_reply":"2022-10-18T20:16:26.328525Z"},"papermill":{"duration":2.696002,"end_time":"2022-10-18T20:16:26.332603","exception":false,"start_time":"2022-10-18T20:16:23.636601","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}