{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Tabular Playground Oct 2022 - Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"In this notebook, we experiment with new features. More specifically, we compute the distance between the ball and the players.  The intuition is that the team that controls the ball tends to have better chance of scoring a goal. Empirical results confirms improvement in the model performance.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport gc\nfrom itertools import chain\nimport lightgbm\nfrom sklearn.metrics import log_loss\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:18:44.319900Z","iopub.execute_input":"2022-10-02T19:18:44.320415Z","iopub.status.idle":"2022-10-02T19:18:45.836230Z","shell.execute_reply.started":"2022-10-02T19:18:44.320310Z","shell.execute_reply":"2022-10-02T19:18:45.835211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option(\"display.max_columns\", 999)\nplt.rcParams['figure.figsize'] = (10,10)","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:18:45.838048Z","iopub.execute_input":"2022-10-02T19:18:45.838588Z","iopub.status.idle":"2022-10-02T19:18:45.843902Z","shell.execute_reply.started":"2022-10-02T19:18:45.838551Z","shell.execute_reply":"2022-10-02T19:18:45.842279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DAT_DIR = '../input/tabular-playground-series-oct-2022'","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:19:59.227135Z","iopub.execute_input":"2022-10-02T19:19:59.227583Z","iopub.status.idle":"2022-10-02T19:19:59.233267Z","shell.execute_reply.started":"2022-10-02T19:19:59.227546Z","shell.execute_reply":"2022-10-02T19:19:59.232068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = list(range(7))\nvalid_files = list(range(7,8))\ntest_files = list(range(8, 10))","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:20:10.845730Z","iopub.execute_input":"2022-10-02T19:20:10.846132Z","iopub.status.idle":"2022-10-02T19:20:10.851813Z","shell.execute_reply.started":"2022-10-02T19:20:10.846099Z","shell.execute_reply":"2022-10-02T19:20:10.850495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'train_files = {train_files}')\nprint(f'valid_files = {valid_files}')\nprint(f'test_files = {test_files}')","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:20:10.857545Z","iopub.execute_input":"2022-10-02T19:20:10.858814Z","iopub.status.idle":"2022-10-02T19:20:10.865053Z","shell.execute_reply.started":"2022-10-02T19:20:10.858772Z","shell.execute_reply":"2022-10-02T19:20:10.863879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"We want to add the distance between the ball and players as features. We add 3-d Euclidean distances.","metadata":{}},{"cell_type":"markdown","source":"### Distances between the Ball and the Players","metadata":{}},{"cell_type":"code","source":"dist_p_ball_cols = [f'dist_p_ball_{i}' for i in range(6)]","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:20:10.870290Z","iopub.execute_input":"2022-10-02T19:20:10.870706Z","iopub.status.idle":"2022-10-02T19:20:10.876322Z","shell.execute_reply.started":"2022-10-02T19:20:10.870645Z","shell.execute_reply":"2022-10-02T19:20:10.875158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'dist_p_ball_cols = {dist_p_ball_cols}')","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:20:10.881622Z","iopub.execute_input":"2022-10-02T19:20:10.882001Z","iopub.status.idle":"2022-10-02T19:20:10.888301Z","shell.execute_reply.started":"2022-10-02T19:20:10.881969Z","shell.execute_reply":"2022-10-02T19:20:10.887015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(os.path.join(DAT_DIR, f\"train_{train_files[0]}.csv\"))","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:20:10.890102Z","iopub.execute_input":"2022-10-02T19:20:10.892208Z","iopub.status.idle":"2022-10-02T19:20:51.653260Z","shell.execute_reply.started":"2022-10-02T19:20:10.892152Z","shell.execute_reply":"2022-10-02T19:20:51.652144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:20:51.655400Z","iopub.execute_input":"2022-10-02T19:20:51.656147Z","iopub.status.idle":"2022-10-02T19:20:51.738719Z","shell.execute_reply.started":"2022-10-02T19:20:51.656098Z","shell.execute_reply":"2022-10-02T19:20:51.737695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us first calculate the distance of each player to the ball.","metadata":{}},{"cell_type":"code","source":"def add_dist_cols(df):\n    for i in range(6):\n        df[f'dist_p_ball_{i}'] = np.sqrt((df.ball_pos_x - df[f'p{i}_pos_x'])**2 + (df.ball_pos_y - df[f'p{i}_pos_y'])**2 + (df.ball_pos_z - df[f'p{i}_pos_z'])**2)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:22:21.413975Z","iopub.execute_input":"2022-10-02T19:22:21.415589Z","iopub.status.idle":"2022-10-02T19:22:21.425435Z","shell.execute_reply.started":"2022-10-02T19:22:21.415530Z","shell.execute_reply":"2022-10-02T19:22:21.424216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = add_dist_cols(train)","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:22:21.427170Z","iopub.execute_input":"2022-10-02T19:22:21.427932Z","iopub.status.idle":"2022-10-02T19:22:21.718626Z","shell.execute_reply.started":"2022-10-02T19:22:21.427884Z","shell.execute_reply":"2022-10-02T19:22:21.717415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:22:21.720404Z","iopub.execute_input":"2022-10-02T19:22:21.720802Z","iopub.status.idle":"2022-10-02T19:22:21.810728Z","shell.execute_reply.started":"2022-10-02T19:22:21.720766Z","shell.execute_reply":"2022-10-02T19:22:21.809544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid = pd.read_csv(os.path.join(DAT_DIR, f\"train_{valid_files[0]}.csv\"))","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:22:33.813744Z","iopub.execute_input":"2022-10-02T19:22:33.814619Z","iopub.status.idle":"2022-10-02T19:23:10.664315Z","shell.execute_reply.started":"2022-10-02T19:22:33.814573Z","shell.execute_reply":"2022-10-02T19:23:10.663121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid = add_dist_cols(valid)","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:23:10.666223Z","iopub.execute_input":"2022-10-02T19:23:10.666841Z","iopub.status.idle":"2022-10-02T19:23:10.915859Z","shell.execute_reply.started":"2022-10-02T19:23:10.666803Z","shell.execute_reply":"2022-10-02T19:23:10.914718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:23:10.917324Z","iopub.execute_input":"2022-10-02T19:23:10.918220Z","iopub.status.idle":"2022-10-02T19:23:11.000485Z","shell.execute_reply.started":"2022-10-02T19:23:10.918180Z","shell.execute_reply":"2022-10-02T19:23:10.999278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ball_pos_cols = [f'ball_pos_{i}' for i in ['x', 'y', 'z']]\nball_vel_cols = [f'ball_vel_{i}' for i in ['x', 'y', 'z']]\nplayer_pos_cols = list(chain.from_iterable([[f'p{i}_pos_{j}' for j in ['x', 'y', 'z']] for i in range(6)]))\nplayer_vel_cols = list(chain.from_iterable([[f'p{i}_vel_{j}' for j in ['x', 'y', 'z']] for i in range(6)]))\nboost_cols = [f'p{i}_boost' for i in range(6)]\nboost_timer_cols = [f'boost{i}_timer' for i in range(6)]","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:23:11.003032Z","iopub.execute_input":"2022-10-02T19:23:11.003901Z","iopub.status.idle":"2022-10-02T19:23:11.012035Z","shell.execute_reply.started":"2022-10-02T19:23:11.003858Z","shell.execute_reply":"2022-10-02T19:23:11.010782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'ball_pos_cols = {ball_pos_cols}')\nprint(f'ball_vel_cols = {ball_vel_cols}')\nprint(f'player_pos_cols = {player_pos_cols}')\nprint(f'boost_cols = {boost_cols}')\nprint(f'boost_timer_cols = {boost_timer_cols}')","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:23:11.013757Z","iopub.execute_input":"2022-10-02T19:23:11.014160Z","iopub.status.idle":"2022-10-02T19:23:11.025570Z","shell.execute_reply.started":"2022-10-02T19:23:11.014117Z","shell.execute_reply":"2022-10-02T19:23:11.024695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_cols = ball_pos_cols + ball_vel_cols + player_pos_cols + player_vel_cols + boost_cols + boost_timer_cols + dist_p_ball_cols\nprint(f'x_cols = {x_cols}')","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:23:11.026755Z","iopub.execute_input":"2022-10-02T19:23:11.028112Z","iopub.status.idle":"2022-10-02T19:23:11.043435Z","shell.execute_reply.started":"2022-10-02T19:23:11.028063Z","shell.execute_reply":"2022-10-02T19:23:11.041969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_cols = ['team_A_scoring_within_10sec', 'team_B_scoring_within_10sec']\nprint(f'y_cols = {y_cols}')","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:23:19.438295Z","iopub.execute_input":"2022-10-02T19:23:19.438731Z","iopub.status.idle":"2022-10-02T19:23:19.445108Z","shell.execute_reply.started":"2022-10-02T19:23:19.438693Z","shell.execute_reply":"2022-10-02T19:23:19.443483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fitting and Predicting Team A","metadata":{}},{"cell_type":"code","source":"MAX_ITER = 1000\nPATIENCE = 100\nDISPLAY_FREQ = 100","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:23:23.193322Z","iopub.execute_input":"2022-10-02T19:23:23.194502Z","iopub.status.idle":"2022-10-02T19:23:23.199255Z","shell.execute_reply.started":"2022-10-02T19:23:23.194461Z","shell.execute_reply":"2022-10-02T19:23:23.198140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_PARAMS = {    \n    'objective': 'binary',\n    'seed': 42,\n    'num_leaves': 120,\n    'n_estimators': MAX_ITER,\n    'max_depth': 8,\n    'learning_rate': 0.01,\n    'feature_fraction': 0.75,\n    'subsample': 0.7,\n    'subsample_freq': 8,\n    'n_jobs': -1,\n    'reg_alpha': 1,\n    'reg_lambda': 2,\n    'min_child_samples': 80}\n\nmdl_A = lightgbm.LGBMClassifier(**MODEL_PARAMS)\n\nmdl_A.fit(X=train[x_cols], y=train[y_cols[0]],\n          eval_set=[(valid[x_cols], valid[y_cols[0]])],\n          eval_names=['valid'],\n          callbacks=[lightgbm.log_evaluation(period=PATIENCE, show_stdv=True),\n                     lightgbm.early_stopping(stopping_rounds=DISPLAY_FREQ)])","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:23:26.198742Z","iopub.execute_input":"2022-10-02T19:23:26.199204Z","iopub.status.idle":"2022-10-02T19:32:45.753974Z","shell.execute_reply.started":"2022-10-02T19:23:26.199163Z","shell.execute_reply":"2022-10-02T19:32:45.752510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in train_files[1:]:\n    print(f'Iteration {i}, Reading train_{i}.csv')\n    train = pd.read_csv(os.path.join(DAT_DIR, f'train_{i}.csv'))\n    train = add_dist_cols(train)\n    print('Fitting model')\n    mdl_A.fit(X=train[x_cols], \n              y=train[y_cols[0]],\n              eval_set=[(valid[x_cols], valid[y_cols[0]])],\n              eval_names=['valid'],\n              callbacks=[lightgbm.log_evaluation(period=PATIENCE, show_stdv=True),\n                         lightgbm.early_stopping(stopping_rounds=DISPLAY_FREQ)],\n             init_model=mdl_A)","metadata":{"execution":{"iopub.status.busy":"2022-10-02T19:32:55.671959Z","iopub.execute_input":"2022-10-02T19:32:55.672499Z","iopub.status.idle":"2022-10-02T20:22:12.750368Z","shell.execute_reply.started":"2022-10-02T19:32:55.672448Z","shell.execute_reply":"2022-10-02T20:22:12.749291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in test_files:\n    test = pd.read_csv(os.path.join(DAT_DIR, f\"train_{i}.csv\"))\n    test = add_dist_cols(test)\n    pred_probs = mdl_A.predict_proba(test[x_cols])[:, 1]\n    test['pred_probs'] = pred_probs\n    print(f'LogLoss of test on train_{i}.csv: {round(log_loss(test[\"team_A_scoring_within_10sec\"], test[\"pred_probs\"]), 3)}')","metadata":{"execution":{"iopub.status.busy":"2022-10-02T20:23:10.452949Z","iopub.execute_input":"2022-10-02T20:23:10.453388Z","iopub.status.idle":"2022-10-02T20:31:21.387164Z","shell.execute_reply.started":"2022-10-02T20:23:10.453352Z","shell.execute_reply":"2022-10-02T20:31:21.385763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us check the importance of variables.","metadata":{}},{"cell_type":"code","source":"lightgbm.plot_importance(mdl_A, max_num_features=30)","metadata":{"execution":{"iopub.status.busy":"2022-10-02T20:31:28.665676Z","iopub.execute_input":"2022-10-02T20:31:28.666125Z","iopub.status.idle":"2022-10-02T20:31:29.423016Z","shell.execute_reply.started":"2022-10-02T20:31:28.666089Z","shell.execute_reply":"2022-10-02T20:31:29.421742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It is quite interesting to note that the feature importances are not what I have assumed in my [EDA notebook](https://www.kaggle.com/code/stautxie/tabular-challenge-oct-2022-eda). As I have previously assumed $y$-coordinates are most important. Here, it turns out the $x$-coordinates of ball as well as boost items are much more important.","metadata":{}},{"cell_type":"markdown","source":"### Fitting and Predicting Team B","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(os.path.join(DAT_DIR, f\"train_{train_files[0]}.csv\"))\ntrain = add_dist_cols(train)","metadata":{"execution":{"iopub.status.busy":"2022-10-02T20:31:37.060689Z","iopub.execute_input":"2022-10-02T20:31:37.061136Z","iopub.status.idle":"2022-10-02T20:32:12.721309Z","shell.execute_reply.started":"2022-10-02T20:31:37.061099Z","shell.execute_reply":"2022-10-02T20:32:12.719783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_PARAMS = {    \n    'objective': 'binary',\n    'seed': 42,\n    'num_leaves': 120,\n    'n_estimators': MAX_ITER,\n    'max_depth': 8,\n    'learning_rate': 0.01,\n    'feature_fraction': 0.75,\n    'subsample': 0.7,\n    'subsample_freq': 8,\n    'n_jobs': -1,\n    'reg_alpha': 1,\n    'reg_lambda': 2,\n    'min_child_samples': 80}\n\nmdl_B = lightgbm.LGBMClassifier(**MODEL_PARAMS)\n\nmdl_B.fit(X=train[x_cols], y=train[y_cols[1]],\n          eval_set=[(valid[x_cols], valid[y_cols[1]])],\n          eval_names=['valid'],\n          callbacks=[lightgbm.log_evaluation(period=PATIENCE, show_stdv=True),\n                     lightgbm.early_stopping(stopping_rounds=DISPLAY_FREQ)])","metadata":{"execution":{"iopub.status.busy":"2022-10-02T20:37:57.799826Z","iopub.execute_input":"2022-10-02T20:37:57.800319Z","iopub.status.idle":"2022-10-02T20:46:46.691395Z","shell.execute_reply.started":"2022-10-02T20:37:57.800284Z","shell.execute_reply":"2022-10-02T20:46:46.690151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in train_files[1:]:\n    print(f'Iteration {i}, Reading train_{i}.csv')\n    train = pd.read_csv(os.path.join(DAT_DIR, f'train_{i}.csv'))\n    train = add_dist_cols(train)\n    print('Fitting model')\n    mdl_B.fit(X=train[x_cols], \n              y=train[y_cols[1]],\n              eval_set=[(valid[x_cols], valid[y_cols[1]])],\n              eval_names=['valid'],\n              callbacks=[lightgbm.log_evaluation(period=PATIENCE, show_stdv=True),\n                         lightgbm.early_stopping(stopping_rounds=DISPLAY_FREQ)],\n             init_model=mdl_B)","metadata":{"execution":{"iopub.status.busy":"2022-10-02T21:24:35.417937Z","iopub.execute_input":"2022-10-02T21:24:35.418515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in test_files:\n    test = pd.read_csv(os.path.join(DAT_DIR, f\"train_{i}.csv\"))\n    test = add_dist_cols(test)\n    pred_probs = mdl_B.predict_proba(test[x_cols])[:, 1]\n    test['pred_probs'] = pred_probs\n    print(f'LogLoss of test on train_{i}.csv: {round(log_loss(test[\"team_B_scoring_within_10sec\"], test[\"pred_probs\"]), 3)}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lightgbm.plot_importance(mdl_A, max_num_features=30)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predicting on Test","metadata":{}},{"cell_type":"code","source":"final_test = pd.read_csv(os.path.join(DAT_DIR, \"test.csv\"))\nsample_submit = pd.read_csv(os.path.join(DAT_DIR, \"sample_submission.csv\"))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test = add_dist_cols(final_test)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submit[\"team_A_scoring_within_10sec\"] = mdl_A.predict_proba(final_test[x_cols])[:,1]\nsample_submit[\"team_B_scoring_within_10sec\"] = mdl_B.predict_proba(final_test[x_cols])[:,1]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submit","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submit.to_csv(\"submission.csv\", index=None)","metadata":{},"execution_count":null,"outputs":[]}]}