{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import precision_recall_fscore_support","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-06T14:14:04.729100Z","iopub.execute_input":"2022-10-06T14:14:04.730163Z","iopub.status.idle":"2022-10-06T14:14:05.439662Z","shell.execute_reply.started":"2022-10-06T14:14:04.730037Z","shell.execute_reply":"2022-10-06T14:14:05.438635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('max_columns', None)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:14:05.690405Z","iopub.execute_input":"2022-10-06T14:14:05.691164Z","iopub.status.idle":"2022-10-06T14:14:05.696663Z","shell.execute_reply.started":"2022-10-06T14:14:05.691105Z","shell.execute_reply":"2022-10-06T14:14:05.695577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read in Train Data","metadata":{}},{"cell_type":"code","source":"dtypes_dict = {\n    'game_num': 'int8', 'event_id': 'int8', 'event_time': 'float16',\n    'ball_pos_x': 'float16', 'ball_pos_y': 'float16', 'ball_pos_z': 'float16',\n    'ball_vel_x': 'float16', 'ball_vel_y': 'float16', 'ball_vel_z': 'float16',\n    'p0_pos_x': 'float16', 'p0_pos_y': 'float16', 'p0_pos_z': 'float16',\n    'p0_vel_x': 'float16', 'p0_vel_y': 'float16', 'p0_vel_z': 'float16',\n    'p0_boost': 'float16', 'p1_pos_x': 'float16', 'p1_pos_y': 'float16',\n    'p1_pos_z': 'float16', 'p1_vel_x': 'float16', 'p1_vel_y': 'float16',\n    'p1_vel_z': 'float16', 'p1_boost': 'float16', 'p2_pos_x': 'float16',\n    'p2_pos_y': 'float16', 'p2_pos_z': 'float16', 'p2_vel_x': 'float16',\n    'p2_vel_y': 'float16', 'p2_vel_z': 'float16', 'p2_boost': 'float16',\n    'p3_pos_x': 'float16', 'p3_pos_y': 'float16', 'p3_pos_z': 'float16',\n    'p3_vel_x': 'float16', 'p3_vel_y': 'float16', 'p3_vel_z': 'float16',\n    'p3_boost': 'float16', 'p4_pos_x': 'float16', 'p4_pos_y': 'float16',\n    'p4_pos_z': 'float16', 'p4_vel_x': 'float16', 'p4_vel_y': 'float16',\n    'p4_vel_z': 'float16', 'p4_boost': 'float16', 'p5_pos_x': 'float16',\n    'p5_pos_y': 'float16', 'p5_pos_z': 'float16', 'p5_vel_x': 'float16',\n    'p5_vel_y': 'float16', 'p5_vel_z': 'float16', 'p5_boost': 'float16',\n    'boost0_timer': 'float16', 'boost1_timer': 'float16', 'boost2_timer': 'float16',\n    'boost3_timer': 'float16', 'boost4_timer': 'float16', 'boost5_timer': 'float16',\n    'team_A_scoring_within_10sec': 'int8', 'team_B_scoring_within_10sec': 'int8'}","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:14:06.539262Z","iopub.execute_input":"2022-10-06T14:14:06.539677Z","iopub.status.idle":"2022-10-06T14:14:06.549346Z","shell.execute_reply.started":"2022-10-06T14:14:06.539642Z","shell.execute_reply":"2022-10-06T14:14:06.547961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Combine All 10 Parquet Files","metadata":{}},{"cell_type":"code","source":"%%time\nfor i in range(10):\n    if i == 0:\n        train = pd.read_parquet('../input/tps-rocket-league-data-float16-parquet-format/train_0.parquet.gzip')\n    else:\n        temp = pd.read_parquet('../input/tps-rocket-league-data-float16-parquet-format/train_{}.parquet.gzip'.format(i))\n        train = pd.concat([train, temp], axis=0)\n    \n        del temp\n        gc.collect()\n    \ntrain = train.astype(dtypes_dict)\ntrain = train.fillna(0.0) # strong assumption to place missing values at center of field (0,0,0)\ntrain = train.sort_values(['game_num','event_id'], ascending=[True,True])\\\n             .reset_index(drop=True)\ntrain['game_event_index'] = train.groupby(['game_num','event_id']).cumcount() # get index of each game/event\nprint(train.shape, train[['game_num','event_id']].drop_duplicates().shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:14:07.867400Z","iopub.execute_input":"2022-10-06T14:14:07.867776Z","iopub.status.idle":"2022-10-06T14:15:47.797537Z","shell.execute_reply.started":"2022-10-06T14:14:07.867745Z","shell.execute_reply":"2022-10-06T14:15:47.796388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sample 100k records to test out embeddings on","metadata":{}},{"cell_type":"code","source":"train_subset = train.sample(100000)\ntrain_subset.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:16:48.798385Z","iopub.execute_input":"2022-10-06T14:16:48.798888Z","iopub.status.idle":"2022-10-06T14:16:50.269264Z","shell.execute_reply.started":"2022-10-06T14:16:48.798849Z","shell.execute_reply":"2022-10-06T14:16:50.268105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_rows = train.groupby(['game_num','event_id'])\\\n              .agg({'game_event_index':'max'})\\\n              .reset_index()\nn_rows.columns = ['game_num','event_id','n_rows']\nn_rows.sort_values('n_rows', ascending=False).head(5)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:16:53.740128Z","iopub.execute_input":"2022-10-06T14:16:53.740875Z","iopub.status.idle":"2022-10-06T14:16:54.960007Z","shell.execute_reply.started":"2022-10-06T14:16:53.740837Z","shell.execute_reply":"2022-10-06T14:16:54.958833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_game = train[(train['game_num'] == -27) & (train['event_id'] == -119)]\nexample_game.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:16:56.025360Z","iopub.execute_input":"2022-10-06T14:16:56.026533Z","iopub.status.idle":"2022-10-06T14:16:56.087204Z","shell.execute_reply.started":"2022-10-06T14:16:56.026484Z","shell.execute_reply":"2022-10-06T14:16:56.086051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We want to take in a frame and be able to predict/reconstruct the next frame. That includes the ball and players X,Y,Z coordinates, velocities and boosting. I'm going to structure them as 7x7xp (ball+players)x(ball+players)x(features). I'll have the game/event split into even frames and odd frames. The beginning frame will be an even index, and the end frame will be the subsequent odd index. For example, the first frame in a game/event (index 0) will be my beginning frame, which will then need to predict/reconstruct the next frame (odd index of 1). The next input will be for frames indexed at 2 (input) and 3 (output). Basically I don't want the beginning frames to include any of the output frames with overlap. For example, index 0 predicts index 1 and then index 1 predicts index 2.\n\nThis format runs out of memory if using all of the data, so I'm going to just use the game/event with most records to start with. If this ends up looking promising, then it will be worth figuring out a way to handle more of the training data.","metadata":{}},{"cell_type":"markdown","source":"### Build Matrix Relating Pos/Vel/Boost Differences","metadata":{}},{"cell_type":"code","source":"def create_matrix(df):\n    ball = df[['game_num','event_id','game_event_index','ball_pos_x','ball_pos_y','ball_pos_z','ball_vel_x','ball_vel_y','ball_vel_z']]\n    ball['boost'] = 0.0\n    ball.columns = ['game_num','event_id','indexes','x','y','z','vx','vy','vz','boost']\n    p0 = df[['game_num','event_id','game_event_index','p0_pos_x','p0_pos_y','p0_pos_z','p0_vel_x','p0_vel_y','p0_vel_z','p0_boost']]\n    p0.columns = ['game_num','event_id','indexes','x','y','z','vx','vy','vz','boost']\n    p1 = df[['game_num','event_id','game_event_index','p1_pos_x','p1_pos_y','p1_pos_z','p1_vel_x','p1_vel_y','p1_vel_z','p1_boost']]\n    p1.columns = ['game_num','event_id','indexes','x','y','z','vx','vy','vz','boost']\n    p2 = df[['game_num','event_id','game_event_index','p2_pos_x','p2_pos_y','p2_pos_z','p2_vel_x','p2_vel_y','p2_vel_z','p2_boost']]\n    p2.columns = ['game_num','event_id','indexes','x','y','z','vx','vy','vz','boost']\n    p3 = df[['game_num','event_id','game_event_index','p3_pos_x','p3_pos_y','p3_pos_z','p3_vel_x','p3_vel_y','p3_vel_z','p3_boost']]\n    p3.columns = ['game_num','event_id','indexes','x','y','z','vx','vy','vz','boost']\n    p4 = df[['game_num','event_id','game_event_index','p4_pos_x','p4_pos_y','p4_pos_z','p4_vel_x','p4_vel_y','p4_vel_z','p4_boost']]\n    p4.columns = ['game_num','event_id','indexes','x','y','z','vx','vy','vz','boost']\n    p5 = df[['game_num','event_id','game_event_index','p5_pos_x','p5_pos_y','p5_pos_z','p5_vel_x','p5_vel_y','p5_vel_z','p5_boost']]\n    p5.columns = ['game_num','event_id','indexes','x','y','z','vx','vy','vz','boost']\n\n    Xs = pd.concat([ball,p0,p1,p2,p3,p4,p5], axis=0)\n    XxX = pd.merge(Xs, Xs, on=['game_num','event_id','indexes'], how='inner').drop('indexes', axis=1)\n    \n    XxX['X_diff'] = XxX['x_x'] - XxX['x_y']\n    XxX['Y_diff'] = XxX['y_x'] - XxX['y_y']\n    XxX['Z_diff'] = XxX['z_x'] - XxX['z_y']\n    XxX['Xv_diff'] = XxX['vx_x'] - XxX['vx_y']\n    XxX['Yv_diff'] = XxX['vy_x'] - XxX['vy_y']\n    XxX['Zv_diff'] = XxX['vz_x'] - XxX['vz_y']\n    XxX['boost_diff'] = XxX['boost_x'] - XxX['boost_y']\n    XxX['distance'] = np.sqrt((XxX['x_x'] - XxX['x_y'])**2 + (XxX['y_x'] - XxX['y_y'])**2 + (XxX['z_x'] - XxX['z_y'])**2)\n\n    XxX = XxX[['X_diff','Y_diff','Z_diff','Xv_diff','Yv_diff','Zv_diff','boost_diff','distance']]\n\n    scaler = MinMaxScaler()\n    XxX_scaled = scaler.fit_transform(XxX).reshape(df.shape[0], 7, 7, XxX.shape[1])\n    \n    return XxX_scaled","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:16:59.637820Z","iopub.execute_input":"2022-10-06T14:16:59.638252Z","iopub.status.idle":"2022-10-06T14:16:59.658205Z","shell.execute_reply.started":"2022-10-06T14:16:59.638216Z","shell.execute_reply":"2022-10-06T14:16:59.657199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frames = create_matrix(example_game)\nprint(frames.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:17:01.610007Z","iopub.execute_input":"2022-10-06T14:17:01.610463Z","iopub.status.idle":"2022-10-06T14:17:02.685027Z","shell.execute_reply.started":"2022-10-06T14:17:01.610425Z","shell.execute_reply":"2022-10-06T14:17:02.683947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_shape = frames.shape[3]\noutput_shape","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:17:05.707737Z","iopub.execute_input":"2022-10-06T14:17:05.708161Z","iopub.status.idle":"2022-10-06T14:17:05.715706Z","shell.execute_reply.started":"2022-10-06T14:17:05.708102Z","shell.execute_reply":"2022-10-06T14:17:05.714483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This would need to be better written to handle the variable number of records per game/event if we wanted to include more than just 1 example to our set.","metadata":{}},{"cell_type":"code","source":"pos_indexes = [x for x in range(frames.shape[0]//2) if x%2 == 0]\nneg_indexes = [x+1 for x in pos_indexes]\nfirst_frame = frames[pos_indexes]\nlast_frame = frames[neg_indexes]\n\nprint(first_frame.shape, last_frame.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:17:07.081884Z","iopub.execute_input":"2022-10-06T14:17:07.082903Z","iopub.status.idle":"2022-10-06T14:17:07.109647Z","shell.execute_reply.started":"2022-10-06T14:17:07.082859Z","shell.execute_reply":"2022-10-06T14:17:07.108667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split Data into Train/Val Sets","metadata":{}},{"cell_type":"code","source":"X_train, X_val, Y_train, Y_val = train_test_split(first_frame, last_frame, test_size=0.10)\nprint((X_train.shape, Y_train.shape), (X_val.shape, Y_val.shape))","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:17:10.122821Z","iopub.execute_input":"2022-10-06T14:17:10.123229Z","iopub.status.idle":"2022-10-06T14:17:10.150722Z","shell.execute_reply.started":"2022-10-06T14:17:10.123192Z","shell.execute_reply":"2022-10-06T14:17:10.149362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train, frames, first_frame, last_frame\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:17:10.809187Z","iopub.execute_input":"2022-10-06T14:17:10.809596Z","iopub.status.idle":"2022-10-06T14:17:11.006644Z","shell.execute_reply.started":"2022-10-06T14:17:10.809560Z","shell.execute_reply":"2022-10-06T14:17:11.005521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build CNN","metadata":{}},{"cell_type":"markdown","source":"Autoencoder architecture inspired by: https://keras.io/examples/vision/oxford_pets_image_segmentation/\n\nNot sure if this makes any sense, but figured it would be interesting to use CNN to extract features and then CNN Transpose to get back to original size. The output is the prediction/reconstruction of the next frame. I figure this could be an interesting way to extract dense embeddings from the middle of the architecture before having to reconstruct the next frame. If so, the embeddings could be used as features for the actual prediction of no score, Team A score or Team B score in the next 10 seconds. Basically treat this model as a pre-training task to learn game state representations on ball+player positioning and velocity.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.models import Model\nfrom tensorflow.keras.backend import clear_session\nfrom tensorflow.keras.layers import (Add, AvgPool2D, AvgPool1D, BatchNormalization, Concatenate, Conv1D, Conv1DTranspose, Conv2D, Conv2DTranspose, Dense,\n                                     Dropout, Flatten, Input, Lambda, LayerNormalization, MaxPool1D, MaxPool2D, Reshape)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:17:21.121737Z","iopub.execute_input":"2022-10-06T14:17:21.122197Z","iopub.status.idle":"2022-10-06T14:17:24.888342Z","shell.execute_reply.started":"2022-10-06T14:17:21.122149Z","shell.execute_reply":"2022-10-06T14:17:24.887312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clear_session()\n\n# Input\ninputs = Input(shape=(7, 7, output_shape))\n\n# Encoder\nx = Conv2D(128, (1, 1), activation=\"relu\", padding=\"same\")(inputs)\nx = Conv2D(256, (1, 1), activation=\"relu\", padding=\"same\")(x)\nx = Conv2D(512, (1, 1), activation=\"relu\", padding=\"same\")(x)\nx = MaxPool2D((1, 7), padding=\"same\")(x)\nx = BatchNormalization()(x)\n\nx = Reshape((7, 512))(x)\n\nx = Conv1D(512, 1, 1, activation=\"relu\", padding=\"same\")(x)\nx = Conv1D(256, 1, 1, activation=\"relu\", padding=\"same\")(x)\n\nx = MaxPool1D(7)(x)\nx = BatchNormalization()(x)\n\n# embedding layer\nembed = Flatten()(x)\nembed = Dense(256, activation=\"relu\")(embed)\nembed = Dense(128, activation=\"sigmoid\")(embed) # output representation with dim 128\n\n# Decoder\nx = Reshape((1, 128))(embed)\nx = Conv1DTranspose(128, 7, activation=\"relu\", padding=\"valid\")(x)\n# x = Conv1DTranspose(128, 3, activation=\"relu\", padding=\"valid\")(x)\n# x = Conv1DTranspose(128, 3, activation=\"relu\", padding=\"valid\")(x)\nx = Reshape((7, 1, 128))(x)\nx = Conv2DTranspose(128, (1, 7), activation=\"relu\", padding=\"valid\")(x)\nx = Conv2DTranspose(64, (1, 1), activation=\"relu\", padding=\"valid\")(x)\nx = Conv2DTranspose(32, (1, 1), activation=\"relu\", padding=\"valid\")(x)\nx = BatchNormalization()(x)\nx = Dropout(0.2)(x)\nnext_play_prediction = Conv2DTranspose(output_shape, (1, 1), activation=\"sigmoid\", padding=\"same\")(x)\n\n# Autoencoder\nembeds = Model(inputs, embed)\nautoencoder = Model(inputs, next_play_prediction)\nautoencoder.compile(optimizer=\"adam\", loss=\"binary_crossentropy\")\nautoencoder.summary()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:17:28.444794Z","iopub.execute_input":"2022-10-06T14:17:28.445599Z","iopub.status.idle":"2022-10-06T14:17:31.141572Z","shell.execute_reply.started":"2022-10-06T14:17:28.445556Z","shell.execute_reply":"2022-10-06T14:17:31.140456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nhistory = autoencoder.fit(X_train, Y_train, \n                          validation_data=(X_val, Y_val),\n                          batch_size=256, \n                          epochs=100,\n                          shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:17:35.735167Z","iopub.execute_input":"2022-10-06T14:17:35.736172Z","iopub.status.idle":"2022-10-06T14:18:59.015842Z","shell.execute_reply.started":"2022-10-06T14:17:35.736092Z","shell.execute_reply":"2022-10-06T14:18:59.014371Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.legend(['train','val'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:18:59.017767Z","iopub.execute_input":"2022-10-06T14:18:59.018085Z","iopub.status.idle":"2022-10-06T14:18:59.251664Z","shell.execute_reply.started":"2022-10-06T14:18:59.018058Z","shell.execute_reply":"2022-10-06T14:18:59.250584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_embeds = embeds.predict(X_train)\nval_embeds = embeds.predict(X_val)\nprint(train_embeds.shape, val_embeds.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:18:59.253249Z","iopub.execute_input":"2022-10-06T14:18:59.253932Z","iopub.status.idle":"2022-10-06T14:19:00.261341Z","shell.execute_reply.started":"2022-10-06T14:18:59.253887Z","shell.execute_reply":"2022-10-06T14:19:00.259836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save Model","metadata":{}},{"cell_type":"code","source":"embeds.save('./CNN_Model_Embedder')\nautoencoder.save('./CNN_Autoencoder')","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:19:21.310783Z","iopub.execute_input":"2022-10-06T14:19:21.311191Z","iopub.status.idle":"2022-10-06T14:19:27.662299Z","shell.execute_reply.started":"2022-10-06T14:19:21.311149Z","shell.execute_reply":"2022-10-06T14:19:27.661199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Embeddings for our sample","metadata":{}},{"cell_type":"code","source":"def scoring_team(teamA,teamB):\n    if teamA+teamB == 0:\n        return 0\n    elif teamA == 1:\n        return 1\n    else:\n        return 2","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:19:48.463300Z","iopub.execute_input":"2022-10-06T14:19:48.464176Z","iopub.status.idle":"2022-10-06T14:19:48.471792Z","shell.execute_reply.started":"2022-10-06T14:19:48.464093Z","shell.execute_reply":"2022-10-06T14:19:48.468946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import to_categorical\n\nY_labels = train_subset[['team_A_scoring_within_10sec','team_B_scoring_within_10sec']].apply(lambda x: scoring_team(x[0],x[1]), axis=1)\nY = to_categorical(Y_labels)\nY.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:36:19.876745Z","iopub.execute_input":"2022-10-06T14:36:19.877203Z","iopub.status.idle":"2022-10-06T14:36:21.070317Z","shell.execute_reply.started":"2022-10-06T14:36:19.877163Z","shell.execute_reply":"2022-10-06T14:36:21.069082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embed_subset = create_matrix(train_subset)\nembeddings = embeds.predict(embed_subset)\nprint(embed_subset.shape, embeddings.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:20:01.800706Z","iopub.execute_input":"2022-10-06T14:20:01.801090Z","iopub.status.idle":"2022-10-06T14:20:15.844560Z","shell.execute_reply.started":"2022-10-06T14:20:01.801059Z","shell.execute_reply":"2022-10-06T14:20:15.843372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split into train/val","metadata":{}},{"cell_type":"code","source":"X_train, X_val, y_train, y_val, y_train_labels, y_val_labels = train_test_split(embeddings, Y, Y_labels, test_size=0.1, stratify=Y)\nprint(X_train.shape, X_val.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:37:05.729970Z","iopub.execute_input":"2022-10-06T14:37:05.730383Z","iopub.status.idle":"2022-10-06T14:37:06.239956Z","shell.execute_reply.started":"2022-10-06T14:37:05.730347Z","shell.execute_reply":"2022-10-06T14:37:06.238853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dense NN for scoring team output","metadata":{}},{"cell_type":"code","source":"clear_session()\n\ninput_shape = embeddings.shape[1]\ninputs = Input((input_shape,))\n\nx = Dense(512, activation=\"relu\")(inputs)\nfor i in [256,128,64,32,16,8]:\n    x = Dense(i, activation=\"relu\")(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.3)(x)\n    \noutputs = Dense(3, activation=\"softmax\")(x)\n\ndense_model = Model(inputs=inputs, outputs=outputs)\ndense_model.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\")\ndense_model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:37:09.866853Z","iopub.execute_input":"2022-10-06T14:37:09.867306Z","iopub.status.idle":"2022-10-06T14:37:10.046801Z","shell.execute_reply.started":"2022-10-06T14:37:09.867263Z","shell.execute_reply":"2022-10-06T14:37:10.042152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndense_history = dense_model.fit(X_train, y_train, \n                                validation_data=(X_val, y_val),\n                                batch_size=128, \n                                epochs=100,\n                                shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:49:32.152983Z","iopub.execute_input":"2022-10-06T14:49:32.154086Z","iopub.status.idle":"2022-10-06T14:57:54.204527Z","shell.execute_reply.started":"2022-10-06T14:49:32.154047Z","shell.execute_reply":"2022-10-06T14:57:54.203222Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(dense_history.history['loss'])\nplt.plot(dense_history.history['val_loss'])\nplt.legend(['train','val'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:57:54.206968Z","iopub.execute_input":"2022-10-06T14:57:54.207460Z","iopub.status.idle":"2022-10-06T14:57:54.443825Z","shell.execute_reply.started":"2022-10-06T14:57:54.207414Z","shell.execute_reply":"2022-10-06T14:57:54.442056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_preds = dense_model.predict(X_train)\nval_preds = dense_model.predict(X_val)\n\ntrain_labels = np.argmax(train_preds, axis=1)\nval_labels = np.argmax(val_preds, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:57:54.445570Z","iopub.execute_input":"2022-10-06T14:57:54.446790Z","iopub.status.idle":"2022-10-06T14:57:58.605353Z","shell.execute_reply.started":"2022-10-06T14:57:54.446744Z","shell.execute_reply":"2022-10-06T14:57:58.604272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print((np.mean(y_train, axis=0), np.mean(train_preds, axis=0)), (np.mean(y_val, axis=0), np.mean(val_preds, axis=0)))","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:57:58.608536Z","iopub.execute_input":"2022-10-06T14:57:58.608958Z","iopub.status.idle":"2022-10-06T14:57:58.622150Z","shell.execute_reply.started":"2022-10-06T14:57:58.608920Z","shell.execute_reply":"2022-10-06T14:57:58.620850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pr = precision_recall_fscore_support(y_true=y_train_labels, y_pred=train_labels, labels=[0,1,2], average=\"weighted\")\nval_pr = precision_recall_fscore_support(y_true=y_val_labels, y_pred=val_labels, labels=[0,1,2], average=\"weighted\")\nprint(train_pr)\nprint(val_pr)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:57:58.631538Z","iopub.execute_input":"2022-10-06T14:57:58.632502Z","iopub.status.idle":"2022-10-06T14:57:58.664533Z","shell.execute_reply.started":"2022-10-06T14:57:58.632454Z","shell.execute_reply":"2022-10-06T14:57:58.663517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train[:10]","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:59:06.045191Z","iopub.execute_input":"2022-10-06T14:59:06.045856Z","iopub.status.idle":"2022-10-06T14:59:06.064586Z","shell.execute_reply.started":"2022-10-06T14:59:06.045806Z","shell.execute_reply":"2022-10-06T14:59:06.061623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_preds[:10]","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:59:18.377066Z","iopub.execute_input":"2022-10-06T14:59:18.377593Z","iopub.status.idle":"2022-10-06T14:59:18.386260Z","shell.execute_reply.started":"2022-10-06T14:59:18.377551Z","shell.execute_reply":"2022-10-06T14:59:18.385208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_val[:10]","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:57:58.666173Z","iopub.execute_input":"2022-10-06T14:57:58.667067Z","iopub.status.idle":"2022-10-06T14:57:58.675505Z","shell.execute_reply.started":"2022-10-06T14:57:58.667017Z","shell.execute_reply":"2022-10-06T14:57:58.674272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_preds[:10]","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:57:58.677094Z","iopub.execute_input":"2022-10-06T14:57:58.678332Z","iopub.status.idle":"2022-10-06T14:57:58.687652Z","shell.execute_reply.started":"2022-10-06T14:57:58.678285Z","shell.execute_reply":"2022-10-06T14:57:58.686315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Can clearly see it is overfitting the training data.","metadata":{}},{"cell_type":"markdown","source":"# Save Model","metadata":{}},{"cell_type":"code","source":"dense_model.save('./Dense_Model')","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:59:45.791994Z","iopub.execute_input":"2022-10-06T14:59:45.792743Z","iopub.status.idle":"2022-10-06T14:59:49.420159Z","shell.execute_reply.started":"2022-10-06T14:59:45.792701Z","shell.execute_reply":"2022-10-06T14:59:49.419164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extract Embeddings for Test Records","metadata":{}},{"cell_type":"code","source":"# test = pd.read_parquet('../input/tps-rocket-league-data-float16-parquet-format/test.parquet.gzip')\\\n#          .rename(columns={'id':'game_num'})\n# test['event_id'] = 0\n# test['game_event_index'] = 0\n# N = test.shape[0]\n# N","metadata":{"execution":{"iopub.status.busy":"2022-10-06T15:08:24.722465Z","iopub.execute_input":"2022-10-06T15:08:24.723025Z","iopub.status.idle":"2022-10-06T15:08:25.818961Z","shell.execute_reply.started":"2022-10-06T15:08:24.722985Z","shell.execute_reply":"2022-10-06T15:08:25.817845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_test = create_matrix(test)\n# X_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-06T15:08:34.296069Z","iopub.execute_input":"2022-10-06T15:08:34.296738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_embeds = embeds.predict(X_test)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate Predictions for Test Records","metadata":{}},{"cell_type":"code","source":"# test_preds = dense_model.predict(test_embeds)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save Output File","metadata":{}},{"cell_type":"code","source":"# sub = pd.read_csv('../input/tabular-playground-series-oct-2022/sample_submission.csv')\n# sub['team_A_scoring_within_10sec'] = test_preds[:, 1]\n# sub['team_B_scoring_within_10sec'] = test_preds[:, 2]\n# sub.to_csv('./sample_submission.csv', header=True, index=False)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T15:02:06.668818Z","iopub.execute_input":"2022-10-06T15:02:06.669677Z","iopub.status.idle":"2022-10-06T15:02:06.950974Z","shell.execute_reply.started":"2022-10-06T15:02:06.669629Z","shell.execute_reply":"2022-10-06T15:02:06.949871Z"},"trusted":true},"execution_count":null,"outputs":[]}]}