{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🏁 Summary 🏁\n\nIn this notebook I will try 2 approaches to training this data\n1. Merging the Target columns to form 1 encoded Target column:\n* 00 = draw \n* 10 = A team scores in 10 secs \n* 01 = B team scores in 10 secs \nThe assumption for the above is that if Team A is expected to score this will impact team B's probability of scoring. This interaction is not present if we have 2 seperate models to predict each column \n\n2. Continuous learning of model \n* For each file I will pass the previously training model and continue its training \n\n**Note**: Ive taken samples from mutliple notebooks, however I wasnt able to note each persons code as this was a testing notebook. Please message me if your code is present in the notebook so I can reference you.","metadata":{}},{"cell_type":"markdown","source":"# 📑 Import Libraries 📑","metadata":{"papermill":{"duration":0.007803,"end_time":"2022-10-02T10:29:28.946528","exception":false,"start_time":"2022-10-02T10:29:28.938725","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import random\nimport gc\n!pip install datatable\nimport datatable as dt\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, RobustScaler,QuantileTransformer, MinMaxScaler\nimport lightgbm as lgb\nfrom sklearn.calibration import CalibratedClassifierCV","metadata":{"papermill":{"duration":15.167978,"end_time":"2022-10-02T10:29:44.131987","exception":false,"start_time":"2022-10-02T10:29:28.964009","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-08T06:41:46.645771Z","iopub.execute_input":"2022-10-08T06:41:46.646575Z","iopub.status.idle":"2022-10-08T06:41:59.928274Z","shell.execute_reply.started":"2022-10-08T06:41:46.646428Z","shell.execute_reply":"2022-10-08T06:41:59.927247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Scaling = False # removed as we cant scale each file seperately as well as the test data \nscaler =   StandardScaler() \n\nDEBUG = False\nEPOCHS = 3000\nEARLY_STOPPING = 30\nCALIBRATION = True # still to implement","metadata":{"execution":{"iopub.status.busy":"2022-10-08T06:41:59.930358Z","iopub.execute_input":"2022-10-08T06:41:59.930696Z","iopub.status.idle":"2022-10-08T06:41:59.937576Z","shell.execute_reply.started":"2022-10-08T06:41:59.930662Z","shell.execute_reply":"2022-10-08T06:41:59.935317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🖋️ Import Data 🖋️","metadata":{}},{"cell_type":"code","source":"dtypes_dict = {\n    'game_num': 'int8', 'event_id': 'int8', 'event_time': 'float16',\n    'ball_pos_x': 'float16', 'ball_pos_y': 'float16', 'ball_pos_z': 'float16',\n    'ball_vel_x': 'float16', 'ball_vel_y': 'float16', 'ball_vel_z': 'float16',\n    'p0_pos_x': 'float16', 'p0_pos_y': 'float16', 'p0_pos_z': 'float16',\n    'p0_vel_x': 'float16', 'p0_vel_y': 'float16', 'p0_vel_z': 'float16',\n    'p0_boost': 'float16', 'p1_pos_x': 'float16', 'p1_pos_y': 'float16',\n    'p1_pos_z': 'float16', 'p1_vel_x': 'float16', 'p1_vel_y': 'float16',\n    'p1_vel_z': 'float16', 'p1_boost': 'float16', 'p2_pos_x': 'float16',\n    'p2_pos_y': 'float16', 'p2_pos_z': 'float16', 'p2_vel_x': 'float16',\n    'p2_vel_y': 'float16', 'p2_vel_z': 'float16', 'p2_boost': 'float16',\n    'p3_pos_x': 'float16', 'p3_pos_y': 'float16', 'p3_pos_z': 'float16',\n    'p3_vel_x': 'float16', 'p3_vel_y': 'float16', 'p3_vel_z': 'float16',\n    'p3_boost': 'float16', 'p4_pos_x': 'float16', 'p4_pos_y': 'float16',\n    'p4_pos_z': 'float16', 'p4_vel_x': 'float16', 'p4_vel_y': 'float16',\n    'p4_vel_z': 'float16', 'p4_boost': 'float16', 'p5_pos_x': 'float16',\n    'p5_pos_y': 'float16', 'p5_pos_z': 'float16', 'p5_vel_x': 'float16',\n    'p5_vel_y': 'float16', 'p5_vel_z': 'float16', 'p5_boost': 'float16',\n    'boost0_timer': 'float16', 'boost1_timer': 'float16', 'boost2_timer': 'float16',\n    'boost3_timer': 'float16', 'boost4_timer': 'float16', 'boost5_timer': 'float16',\n    'player_scoring_next': 'O', 'team_scoring_next': 'O', 'team_A_scoring_within_10sec': 'int8',\n    'team_B_scoring_within_10sec': 'int8'\n}","metadata":{"execution":{"iopub.status.busy":"2022-10-08T06:41:59.939714Z","iopub.execute_input":"2022-10-08T06:41:59.940133Z","iopub.status.idle":"2022-10-08T06:41:59.949812Z","shell.execute_reply.started":"2022-10-08T06:41:59.940039Z","shell.execute_reply":"2022-10-08T06:41:59.948854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_to_data = '../input/fast-loading-high-compression-with-feather/feather_data/'\ndf_test = pd.read_csv(\"../input/tabular-playground-series-oct-2022/test.csv\",index_col =0)\n\ntrn_drop_cols = [\"game_num\",\"event_id\",\"event_time\",\"player_scoring_next\",\n                 \"team_scoring_next\", \"team_A_scoring_within_10sec\", \n                 \"team_B_scoring_within_10sec\"] ","metadata":{"execution":{"iopub.status.busy":"2022-10-08T06:41:59.952324Z","iopub.execute_input":"2022-10-08T06:41:59.954010Z","iopub.status.idle":"2022-10-08T06:42:08.297816Z","shell.execute_reply.started":"2022-10-08T06:41:59.953973Z","shell.execute_reply":"2022-10-08T06:42:08.296839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🧹 Data Engineering🧹","metadata":{"papermill":{"duration":0.008018,"end_time":"2022-10-02T10:29:44.148736","exception":false,"start_time":"2022-10-02T10:29:44.140718","status":"completed"},"tags":[]}},{"cell_type":"code","source":"goal1_location = (0,100,0) #assumption that goal centre is at z= 0  (unknown how high the goal is)\ngoal2_location = (0,-100,0) ","metadata":{"execution":{"iopub.status.busy":"2022-10-08T06:42:08.299467Z","iopub.execute_input":"2022-10-08T06:42:08.299850Z","iopub.status.idle":"2022-10-08T06:42:08.305868Z","shell.execute_reply.started":"2022-10-08T06:42:08.299786Z","shell.execute_reply":"2022-10-08T06:42:08.304000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = [\n    \"p0_pos_x\",\"p1_pos_x\",\"p2_pos_x\",\"p3_pos_x\",\"p4_pos_x\",\"p5_pos_x\",\n    \"p0_pos_y\",\"p1_pos_y\",\"p2_pos_y\",\"p3_pos_y\",\"p4_pos_y\",\"p5_pos_y\",\n    \"p0_pos_z\",\"p1_pos_z\",\"p2_pos_z\",\"p3_pos_z\",\"p4_pos_z\",\"p5_pos_z\",\n    \n    \"p0_vel_x\",\"p1_vel_x\",\"p2_vel_x\",\"p3_vel_x\",\"p4_vel_x\",\"p5_vel_x\",\n    \"p0_vel_y\",\"p1_vel_y\",\"p2_vel_y\",\"p3_vel_y\",\"p4_vel_y\",\"p5_vel_y\",\n    \"p0_vel_z\",\"p1_vel_z\",\"p2_vel_z\",\"p3_vel_z\",\"p4_vel_z\",\"p5_vel_z\",\n    \n    'p0_boost','p1_boost','p2_boost','p3_boost','p4_boost','p5_boost'\n       ]\n\n\ndef fillNan(cols, data):\n    random.seed()\n    num = random.random()\n    for i in cols:\n        data[i] = data[i].fillna(0)\n        data[i] = data[i].replace(0, (sum(data[i])/10676072)-num)\n\ndef FeatureEngineering(data):\n    # distance of ball to goal1 (0,100,0) and goal2 (0,-100,0) \n    data[f\"goal1_distance\"] = ((data[\"ball_pos_x\"]-goal1_location[0])**2 + (data[\"ball_pos_y\"]-goal1_location[1])**2 + (data[\"ball_pos_z\"]-goal1_location[2])**2)**0.5\n    data[f\"goal2_distance\"] = ((data[\"ball_pos_x\"]-goal2_location[0])**2 + (data[\"ball_pos_y\"]-goal2_location[1])**2 + (data[\"ball_pos_z\"]-goal2_location[2])**2)**0.5\n    \n    #Ball speed \n    #data[\"ball_direction\"] = ((data[\"ball_vel_x\"])**2 + (data[\"ball_vel_y\"])**2 + (data[\"ball_vel_z\"])**2)**0.5\n    \n    for i in range(6):\n        # ball distance to player\n        data[f\"p{i}_ball_distance\"] = ((data[\"ball_pos_x\"]-data[f\"p{i}_pos_x\"])**2 + (data[\"ball_pos_y\"]-data[f\"p{i}_pos_y\"])**2 + (data[\"ball_pos_z\"]-data[f\"p{i}_pos_z\"])**2)**0.5\n        # boost timer\n        #data[f\"p{i}_boost_timer\"] = data[f\"p{i}_boost\"]*data[f\"boost{i}_timer\"]\n        \n        #Player direction  \n        data[f\"p{i}_direction\"] = ((data[f\"p{i}_vel_x\"])**2 + (data[f\"p{i}_vel_y\"])**2 + (data[f\"p{i}_vel_z\"])**2)**0.5\n        \n        #if the player hit the ball what would its direction/velocity be\n        #multiplied by the inverse of the ball distance (assumption is that the further the player is from the ball the less likely they would collide with ball)\n        #data[f\"p{i}_ball_hit_direction\"] = (data[f\"p{i}_direction\"]**2 + data[\"ball_direction\"]**2)**0.5 * (1/data[f\"p{i}_ball_distance\"]) \n        ","metadata":{"papermill":{"duration":0.024085,"end_time":"2022-10-02T10:30:47.697068","exception":false,"start_time":"2022-10-02T10:30:47.672983","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-08T06:42:08.307625Z","iopub.execute_input":"2022-10-08T06:42:08.308337Z","iopub.status.idle":"2022-10-08T06:42:08.322166Z","shell.execute_reply.started":"2022-10-08T06:42:08.308301Z","shell.execute_reply":"2022-10-08T06:42:08.321322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📄 Training Requirements 📄","metadata":{}},{"cell_type":"code","source":"params = {'num_boost_round' : 3000,\n    'objective': 'multiclass', #'multiclassova' \n         'num_class':3,\n          'force_col_wise': True,\n          \"metric\":  'multi_logloss',\n          'boosting' : 'gbdt',\n          #testing below\n          'feature_fraction': 0.75,\n          'reg_lambda': 2,\n          'verbosity' :-1,\n          'device' : 'cpu',\n          \n          'num_threads': -1\n\n        }","metadata":{"execution":{"iopub.status.busy":"2022-10-08T06:42:08.323327Z","iopub.execute_input":"2022-10-08T06:42:08.323918Z","iopub.status.idle":"2022-10-08T06:42:08.334310Z","shell.execute_reply.started":"2022-10-08T06:42:08.323883Z","shell.execute_reply":"2022-10-08T06:42:08.333383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Model_Training(df, model= None):\n    score = []\n    \n    #split \n    y = df[\"Target\"].astype('int32')\n    X = df.drop(\"Target\",axis =1)\n    \n    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.33, random_state=42)\n\n    trn_data = lgb.Dataset(X_train,label = y_train)\n    val_data = lgb.Dataset(X_val,label = y_val)\n\n    # Continuous training \n    if model is None:  #cold start\n        model = lgb.train(params, trn_data, valid_sets=[trn_data, val_data],\n                          callbacks= [lgb.early_stopping(EARLY_STOPPING)])\n    else:         #warm start\n        model = lgb.train(params, trn_data, valid_sets=[trn_data, val_data],\n                          callbacks= [lgb.early_stopping(EARLY_STOPPING)],\n                         init_model = model)\n\n    logloss = model.best_score[\"valid_1\"][\"multi_logloss\"]\n\n    print(f\"\\n Logloss: {logloss}\")\n    \n    return model, logloss","metadata":{"execution":{"iopub.status.busy":"2022-10-08T06:42:08.337927Z","iopub.execute_input":"2022-10-08T06:42:08.338178Z","iopub.status.idle":"2022-10-08T06:42:08.346930Z","shell.execute_reply.started":"2022-10-08T06:42:08.338155Z","shell.execute_reply":"2022-10-08T06:42:08.345857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🤖 Run All 🤖\nDue to the size of the datasets we will load them one at a time and train our model.\\\nWe will pass the previous trained model to the next iteration for continuous learning.","metadata":{}},{"cell_type":"code","source":"%%time \n\nscore = []\n\nfor i in range(10):\n    print(f\"\\n####### Loading file {i} #########\")\n    df = pd.read_feather(f'{path_to_data}/train_{i}_compressed.ftr')\n    df = df.astype(dtypes_dict)\n    \n    if DEBUG:\n        df = df.iloc[:20000,:]\n        \n    # feat engineering\n    fillNan(cols, df)\n    fillNan(cols, df_test)\n    FeatureEngineering(df)\n    FeatureEngineering(df_test)\n    \n    # Merge and encode target\n    df[\"Target\"] = df[\"team_A_scoring_within_10sec\"].astype(str)+df[\"team_B_scoring_within_10sec\"].astype(str)\n    encoder = LabelEncoder()\n    df[\"Target\"] = encoder.fit_transform(df[\"Target\"])\n    \n    #drop cols \n    df.drop(columns=trn_drop_cols,inplace = True)\n    \n    #training\n    print(f\"Running model training\")\n    if i ==0:  #cold start\n        model_in, logloss= Model_Training(df, None)\n    else:         #warm start\n        model,logloss = Model_Training(df,model= model_in)\n        \n    score.append(logloss)\n    \n    \nprint(\"\\n Mean Logloss for each file: \",np.mean(score))","metadata":{"execution":{"iopub.status.busy":"2022-10-08T06:42:08.348574Z","iopub.execute_input":"2022-10-08T06:42:08.348925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\n#predict A and B (note: team A is class 2)\nprint(\"predicting A\")\nA_preds= model.predict(df_test)[:,2] \nprint(\"predicting B\")\nB_preds= model.predict(df_test)[:,1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_importance = pd.DataFrame(model.feature_importance(),index = df.drop(\"Target\",axis =1).columns, columns = [\"Importance\"])\n\nplt.figure(figsize=(20, 15))\nsns.barplot(x=\"Importance\", y=feature_importance.index, data=feature_importance.sort_values(\"Importance\",ascending = False))\nplt.title('LightGBM Features')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del feature_importance, df_test\ngc.collect()  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🏆 Submit🏆","metadata":{"papermill":{"duration":0.023294,"end_time":"2022-10-02T10:44:11.484529","exception":false,"start_time":"2022-10-02T10:44:11.461235","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sub = pd.read_csv(\"../input/tabular-playground-series-oct-2022/sample_submission.csv\")\n\n#Cross validation\nsub[\"team_A_scoring_within_10sec\"] = A_preds \nsub[\"team_B_scoring_within_10sec\"] = B_preds\n\nsub.to_csv(\"sub_cv.csv\",index=False)\nsub","metadata":{"papermill":{"duration":3.017746,"end_time":"2022-10-02T10:44:14.529695","exception":false,"start_time":"2022-10-02T10:44:11.511949","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub[[\"team_A_scoring_within_10sec\",\"team_B_scoring_within_10sec\"]].hist(figsize = (20,5))\nplt.show()","metadata":{"papermill":{"duration":0.025027,"end_time":"2022-10-02T10:44:14.732378","exception":false,"start_time":"2022-10-02T10:44:14.707351","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# To do \n* multiclassova\n* Mirror data (players","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}