{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#####   ---XGBOOST ON GPU ----   #####\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-19T19:42:31.407625Z","iopub.execute_input":"2022-10-19T19:42:31.408404Z","iopub.status.idle":"2022-10-19T19:42:31.437531Z","shell.execute_reply.started":"2022-10-19T19:42:31.408301Z","shell.execute_reply":"2022-10-19T19:42:31.436583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm  import LGBMClassifier, log_evaluation, early_stopping\nfrom sklearn.metrics import log_loss\nfrom sklearn.model_selection import train_test_split, GridSearchCV\nfrom xgboost import XGBClassifier\nfrom tensorflow.config import list_physical_devices\nfrom sklearn.model_selection import cross_validate\nimport gc\n","metadata":{"execution":{"iopub.status.busy":"2022-10-19T19:42:31.439209Z","iopub.execute_input":"2022-10-19T19:42:31.440181Z","iopub.status.idle":"2022-10-19T19:42:39.610605Z","shell.execute_reply.started":"2022-10-19T19:42:31.440143Z","shell.execute_reply":"2022-10-19T19:42:39.609627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"column_names = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_3.csv', nrows=1).columns\ncolumn_names = list(column_names)\nCOLUMNS_TO_READ = column_names[3:-4] + column_names[-2:]\nCOLUMNS_TO_READ","metadata":{"execution":{"iopub.status.busy":"2022-10-19T19:42:43.094559Z","iopub.execute_input":"2022-10-19T19:42:43.096042Z","iopub.status.idle":"2022-10-19T19:42:43.131486Z","shell.execute_reply.started":"2022-10-19T19:42:43.095994Z","shell.execute_reply":"2022-10-19T19:42:43.130519Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_dtypes.csv')\ndtypes = {col:dtype for col,dtype in dtypes.values}\nCOLUMNS_TO_READ = list(dtypes.keys())[3:-4] + list(dtypes.keys())[-2:]\nCOLUMNS_TO_READ","metadata":{"execution":{"iopub.status.busy":"2022-10-19T19:48:23.166080Z","iopub.execute_input":"2022-10-19T19:48:23.166497Z","iopub.status.idle":"2022-10-19T19:48:23.183773Z","shell.execute_reply.started":"2022-10-19T19:48:23.166466Z","shell.execute_reply":"2022-10-19T19:48:23.182628Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"READING IN THE TRAINING SET\")\ntrain = pd.DataFrame()\nfor i in range(0,10):\n    df_tmp = pd.read_csv(f'/kaggle/input/tabular-playground-series-oct-2022/train_{i}.csv', usecols = COLUMNS_TO_READ, dtype=dtypes)\n    sample = df_tmp.sample(frac=0.18)\n    train = pd.concat([train, sample])\n    del df_tmp,sample\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T19:48:44.972744Z","iopub.execute_input":"2022-10-19T19:48:44.973137Z","iopub.status.idle":"2022-10-19T19:48:59.561462Z","shell.execute_reply.started":"2022-10-19T19:48:44.973102Z","shell.execute_reply":"2022-10-19T19:48:59.560448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-10-19T19:44:41.301965Z","iopub.execute_input":"2022-10-19T19:44:41.302368Z","iopub.status.idle":"2022-10-19T19:44:41.423059Z","shell.execute_reply.started":"2022-10-19T19:44:41.302317Z","shell.execute_reply":"2022-10-19T19:44:41.422020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_features(df, return_combined_target=False):\n    \n    if return_combined_target:\n        ## Create combined_target feature\n        df.loc[:,\"combined_target\"] = 3 - (2 * df.loc[:,\"team_A_scoring_within_10sec\"] + df.loc[:,\"team_B_scoring_within_10sec\"]).astype(np.int8)\n            \n      \n    ## Add distance of the ball to goals A and B\n    goal_a = [0,100,5]     ## z here is quite arbitrary, just took the mid point as goal post extends to 20 units in z^\n    goal_b = [0,-100,5]    ## experiment with other values of z perhaps , or perhaps get rid of the z direction completely\n    df['dist_ball_goal_a'] = (((df[\"ball_pos_x\"]-goal_a[0])**2 + (df[\"ball_pos_y\"]-goal_a[1])**2 + (df[\"ball_pos_z\"]-goal_a[2])**2)**0.5).astype(np.float32)\n    df['dist_ball_goal_b'] = (((df[\"ball_pos_x\"]-goal_b[0])**2 + (df[\"ball_pos_y\"]-goal_b[1])**2 + (df[\"ball_pos_z\"]-goal_b[2])**2)**0.5).astype(np.float32)\n    \n    \n    ## Add the distance between the ball and all 6 players\n    for i in range(6):\n       df[f'dist_ball_p{i}'] = (((df[f\"p{i}_pos_x\"]-df[\"ball_pos_x\"])**2 + (df[\"ball_pos_y\"]-df[f\"p{i}_pos_y\"])**2 + (df[\"ball_pos_z\"]-df[f\"p{i}_pos_z\"])**2)**0.5).astype(np.float32)\n        \n        \n    ## Add the speeds of the ball and all the players\n    df['speed_ball'] = ((df['ball_vel_x']**2 + df['ball_vel_y']**2 + df['ball_vel_z']**2) ** 0.5).astype(np.float32)\n    for i in range(6):\n        df[f'speed_p{i}'] = ((df[f'p{i}_vel_x']**2 + df[f'p{i}_vel_y']**2 + df[f'p{i}_vel_z']**2) ** 0.5).astype(np.float32)\n        \n        \n    ## Add number of players in play (not demolished at the moment) for both teams\n    df[\"team_A_num_players\"]  = (3 - df['p0_pos_x'].isna()  - df['p1_pos_x'].isna()  - df['p2_pos_x'].isna()).astype(np.int8)\n    df[\"team_B_num_players\"]  = (3 - df['p3_pos_x'].isna()  - df['p4_pos_x'].isna()  - df['p5_pos_x'].isna()).astype(np.int8)\n    \n    ## Add the next location of the ball , next_ball_pos_[xyz]\n    df['next_ball_pos_x'] = (df['ball_pos_x'] + 0.2 * df['ball_vel_x']).astype(np.float32)\n    df['next_ball_pos_y'] = (df['ball_pos_y'] + 0.2 * df['ball_vel_y']).astype(np.float32)\n    df['next_ball_pos_z'] = (df['ball_pos_z'] + 0.2 * df['ball_vel_z']).astype(np.float32)\n    \n    ## Add the next distance of the ball from the two goals\n    df['next_dist_ball_goal_a'] = (((df[\"next_ball_pos_x\"]-goal_a[0])**2 + (df[\"next_ball_pos_y\"]-goal_a[1])**2 + (df[\"next_ball_pos_z\"]-goal_a[2])**2)**0.5).astype(np.float32)\n    df['next_dist_ball_goal_b'] = (((df[\"next_ball_pos_x\"]-goal_b[0])**2 + (df[\"next_ball_pos_y\"]-goal_b[1])**2 + (df[\"next_ball_pos_z\"]-goal_b[2])**2)**0.5).astype(np.float32)\n    \n    ## Add the next position of the players , next_p{i}_pos[xyz] \n    for i in range(6):\n        df[f'next_p{i}_pos_x'] = (df[f'p{i}_pos_x'] + 0.2 * df[f'p{i}_vel_x']).astype(np.float32)\n        df[f'next_p{i}_pos_y'] = (df[f'p{i}_pos_y'] + 0.2 * df[f'p{i}_vel_y']).astype(np.float32)\n        df[f'next_p{i}_pos_z'] = (df[f'p{i}_pos_z'] + 0.2 * df[f'p{i}_vel_z']).astype(np.float32)   \n        df[f'next_dist_ball_p{i}'] = (((df[f\"next_p{i}_pos_x\"]-df[\"next_ball_pos_x\"])**2 + (df[\"next_ball_pos_y\"]-df[f\"next_p{i}_pos_y\"])**2 + (df[\"next_ball_pos_z\"]-df[f\"next_p{i}_pos_z\"])**2)**0.5).astype(np.float32)\n        \n    ## Add the cosine similarity between the player->goal and \n    ball_pos = df[['ball_pos_x', 'ball_pos_y', 'ball_pos_z']].values\n    for i in range(6):\n        player_ball_vec = df[[f'p{i}_pos_x', f'p{i}_pos_y', f'p{i}_pos_z']].values - ball_pos\n        ball_goalA_vec = ball_pos - goal_a\n        ball_goalB_vec = ball_pos - goal_b\n        df[f'p{i}_goalA_ball_similarity'] = np.sum(player_ball_vec*ball_goalA_vec,axis=1)/(np.linalg.norm(player_ball_vec, axis=1)*np.linalg.norm(ball_goalA_vec, axis=1)).astype(np.float16)\n        df[f'p{i}_goalB_ball_similarity'] = np.sum(player_ball_vec*ball_goalB_vec,axis=1)/(np.linalg.norm(player_ball_vec, axis=1)*np.linalg.norm(ball_goalB_vec, axis=1)).astype(np.float16)\n   \n\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-10-19T19:57:23.072931Z","iopub.execute_input":"2022-10-19T19:57:23.073313Z","iopub.status.idle":"2022-10-19T19:57:23.093773Z","shell.execute_reply.started":"2022-10-19T19:57:23.073281Z","shell.execute_reply":"2022-10-19T19:57:23.092541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint(\"ADDING FEATURES TO THE TRAIN SET\")\ntrain = add_features(train)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T19:55:36.167206Z","iopub.execute_input":"2022-10-19T19:55:36.168023Z","iopub.status.idle":"2022-10-19T19:55:36.348355Z","shell.execute_reply.started":"2022-10-19T19:55:36.167984Z","shell.execute_reply":"2022-10-19T19:55:36.347133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-10-19T19:55:39.099515Z","iopub.execute_input":"2022-10-19T19:55:39.100225Z","iopub.status.idle":"2022-10-19T19:55:39.210223Z","shell.execute_reply.started":"2022-10-19T19:55:39.100185Z","shell.execute_reply":"2022-10-19T19:55:39.209063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GPU = list_physical_devices('GPU') != []\nGPU ","metadata":{"execution":{"iopub.status.busy":"2022-10-18T11:28:42.114348Z","iopub.execute_input":"2022-10-18T11:28:42.115234Z","iopub.status.idle":"2022-10-18T11:28:42.192521Z","shell.execute_reply.started":"2022-10-18T11:28:42.115197Z","shell.execute_reply":"2022-10-18T11:28:42.191514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GPU = list_physical_devices('GPU') != []\nxgb_a = XGBClassifier(\n    n_estimators=2000,\n    max_depth=8,\n    learning_rate=0.01,\n    objective='binary:logistic',\n    tree_method='gpu_hist' if GPU else 'hist'\n)","metadata":{"execution":{"iopub.status.busy":"2022-10-18T11:28:42.194275Z","iopub.execute_input":"2022-10-18T11:28:42.195006Z","iopub.status.idle":"2022-10-18T11:28:42.201057Z","shell.execute_reply.started":"2022-10-18T11:28:42.194966Z","shell.execute_reply":"2022-10-18T11:28:42.199973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('   --- FITTING CROSS VALIDATION FOR team_A_scoring_within_10sec ---')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv_a = cross_validate(\n    xgb_a, \n    X=train.drop(columns=['team_A_scoring_within_10sec', 'team_B_scoring_within_10sec']).values,\n    y=train['team_A_scoring_within_10sec'].values,\n    scoring=\"neg_log_loss\",\n    cv=5,\n    verbose=2,\n    return_estimator=True\n)","metadata":{"execution":{"iopub.status.busy":"2022-10-18T11:28:42.204848Z","iopub.execute_input":"2022-10-18T11:28:42.205445Z","iopub.status.idle":"2022-10-18T11:53:27.081568Z","shell.execute_reply.started":"2022-10-18T11:28:42.205414Z","shell.execute_reply":"2022-10-18T11:53:27.080415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GPU = list_physical_devices('GPU') != []\nxgb_b = XGBClassifier(\n    n_estimators=2000,\n    max_depth=8,\n    learning_rate=0.01,\n    objective='binary:logistic',\n    tree_method='gpu_hist' if GPU else 'hist'\n)","metadata":{"execution":{"iopub.status.busy":"2022-10-18T11:53:27.083556Z","iopub.execute_input":"2022-10-18T11:53:27.084297Z","iopub.status.idle":"2022-10-18T11:53:27.090857Z","shell.execute_reply.started":"2022-10-18T11:53:27.084251Z","shell.execute_reply":"2022-10-18T11:53:27.089482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('   --- FITTING CROSS VALIDATION XGBOOST FOR team_B_scoring_within_10sec ---')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv_b = cross_validate(\n    xgb_b, \n    X=train.drop(columns=['team_A_scoring_within_10sec', 'team_B_scoring_within_10sec']).values,\n    y=train['team_B_scoring_within_10sec'].values,\n    scoring=\"neg_log_loss\",\n    cv=5,\n    verbose=2,\n    return_estimator=True\n)","metadata":{"execution":{"iopub.status.busy":"2022-10-18T11:53:27.092660Z","iopub.execute_input":"2022-10-18T11:53:27.093115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dtypes = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/test_dtypes.csv')\ntest_dtypes = {col:dtype for col,dtype in test_dtypes.values}\nTEST_COLUMNS_TO_READ = list(test_dtypes.keys())[1:]\nTEST_COLUMNS_TO_READ","metadata":{"execution":{"iopub.status.busy":"2022-10-19T19:53:25.211361Z","iopub.execute_input":"2022-10-19T19:53:25.211744Z","iopub.status.idle":"2022-10-19T19:53:25.229505Z","shell.execute_reply.started":"2022-10-19T19:53:25.211711Z","shell.execute_reply":"2022-10-19T19:53:25.228383Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test = pd.read_feather('/kaggle/input/tpsoct22-feather-files/test.feather')\ntest = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/test.csv', dtype = test_dtypes, usecols=TEST_COLUMNS_TO_READ)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T19:53:45.419741Z","iopub.execute_input":"2022-10-19T19:53:45.421120Z","iopub.status.idle":"2022-10-19T19:53:54.165864Z","shell.execute_reply.started":"2022-10-19T19:53:45.421078Z","shell.execute_reply":"2022-10-19T19:53:54.164357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/sample_submission.csv')\nsample","metadata":{"execution":{"iopub.status.busy":"2022-10-18T12:36:31.779249Z","iopub.execute_input":"2022-10-18T12:36:31.779709Z","iopub.status.idle":"2022-10-18T12:36:31.929572Z","shell.execute_reply.started":"2022-10-18T12:36:31.779667Z","shell.execute_reply":"2022-10-18T12:36:31.928533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('    ---  ADDING FEATURES TO THE TEST SET  ---')\ntest = add_features(test)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T19:54:08.386071Z","iopub.execute_input":"2022-10-19T19:54:08.386427Z","iopub.status.idle":"2022-10-19T19:54:08.598221Z","shell.execute_reply.started":"2022-10-19T19:54:08.386395Z","shell.execute_reply":"2022-10-19T19:54:08.596917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_a = sample.team_A_scoring_within_10sec","metadata":{"execution":{"iopub.status.busy":"2022-10-18T12:38:34.548380Z","iopub.execute_input":"2022-10-18T12:38:34.548806Z","iopub.status.idle":"2022-10-18T12:38:34.553938Z","shell.execute_reply.started":"2022-10-18T12:38:34.548768Z","shell.execute_reply":"2022-10-18T12:38:34.552779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('   --- PREDICTING team_B_scoring_within_10sec ---')\nfor estimator in cv_a['estimator']:\n    pred_a += estimator.predict_proba(test.values)[:, 1]\n    ","metadata":{"execution":{"iopub.status.busy":"2022-10-18T12:42:02.283588Z","iopub.execute_input":"2022-10-18T12:42:02.283981Z","iopub.status.idle":"2022-10-18T12:45:49.219216Z","shell.execute_reply.started":"2022-10-18T12:42:02.283945Z","shell.execute_reply":"2022-10-18T12:45:49.218323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_b = sample.team_B_scoring_within_10sec","metadata":{"execution":{"iopub.status.busy":"2022-10-18T12:45:49.220710Z","iopub.execute_input":"2022-10-18T12:45:49.221443Z","iopub.status.idle":"2022-10-18T12:45:49.228254Z","shell.execute_reply.started":"2022-10-18T12:45:49.221406Z","shell.execute_reply":"2022-10-18T12:45:49.227508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('   --- PREDICTING team_B_scoring_within_10sec ---')\nfor estimator in cv_b['estimator']:\n    pred_b += estimator.predict_proba(test.values)[:, 1]","metadata":{"execution":{"iopub.status.busy":"2022-10-18T12:45:49.231154Z","iopub.execute_input":"2022-10-18T12:45:49.232177Z","iopub.status.idle":"2022-10-18T12:49:34.204034Z","shell.execute_reply.started":"2022-10-18T12:45:49.232120Z","shell.execute_reply":"2022-10-18T12:49:34.203143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_a /= 5\npred_b /= 5","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample.team_A_scoring_within_10sec = pred_a\nsample.team_B_scoring_within_10sec = pred_b","metadata":{"execution":{"iopub.status.busy":"2022-10-18T13:29:29.360648Z","iopub.execute_input":"2022-10-18T13:29:29.361014Z","iopub.status.idle":"2022-10-18T13:29:29.370035Z","shell.execute_reply.started":"2022-10-18T13:29:29.360983Z","shell.execute_reply":"2022-10-18T13:29:29.368917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('   --- CREATING SUBMISSION FILE ---')\nsample.to_csv('submission___.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-10-18T13:29:33.600494Z","iopub.execute_input":"2022-10-18T13:29:33.600889Z","iopub.status.idle":"2022-10-18T13:29:35.832373Z","shell.execute_reply.started":"2022-10-18T13:29:33.600855Z","shell.execute_reply":"2022-10-18T13:29:35.830927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sample","metadata":{"execution":{"iopub.status.busy":"2022-10-18T13:36:35.491263Z","iopub.execute_input":"2022-10-18T13:36:35.491614Z","iopub.status.idle":"2022-10-18T13:36:35.495580Z","shell.execute_reply.started":"2022-10-18T13:36:35.491582Z","shell.execute_reply":"2022-10-18T13:36:35.494623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from IPython.display import FileLink\n#FileLink('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-10-18T13:36:17.860439Z","iopub.execute_input":"2022-10-18T13:36:17.860845Z","iopub.status.idle":"2022-10-18T13:36:17.865577Z","shell.execute_reply.started":"2022-10-18T13:36:17.860810Z","shell.execute_reply":"2022-10-18T13:36:17.864199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}