{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ❗❗❗ DISCALIMER ❗❗❗\nThis code is based on [this notebook](https://www.kaggle.com/code/ryanholbrook/tfrecords-basics) and [this other notebook](https://www.kaggle.com/code/paddykb/tps-2022-10-fastai). \n\nThe first one is the reference of converting the dataset to a format suitable for convert to TFrecords, the second one is the notebook from which the base logic to read the dataset is taken.\n\nI already finished my GPU quota for this week. So for me and anyone else struggling with GPU I wanted to try and move everything to TPU and see if TF + TPU is the correct recipe for success.\n\nThis notebook generates the TFrecords in 10 folders to allow the use of this dataset also with 10-fold CV with no risk of leakage.\n\nAt the end of the notebook I provide an example code for an easy NN training (Here we are on CPU so the example is minimal).\n\nWhen I'll have more time I'll polish this notebook and publish a TPU starter notebook with this dataset.\n\nI'm new to TPU so I don't know if I did everything correctly.\n","metadata":{}},{"cell_type":"code","source":"import random\nimport numpy as np\nimport pandas as pd\nimport gc\nfrom pathlib import Path\nimport tensorflow as tf\nfrom tensorflow.data import Dataset, TFRecordDataset\n#from tensorflow.io import TFRecordWriter\nimport os\n#from tensorflow.train import BytesList, FloatList, Int64List\n#from tensorflow.train import Example, Features, Feature","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-24T11:35:42.108876Z","iopub.execute_input":"2022-10-24T11:35:42.109345Z","iopub.status.idle":"2022-10-24T11:35:47.860419Z","shell.execute_reply.started":"2022-10-24T11:35:42.109251Z","shell.execute_reply":"2022-10-24T11:35:47.859210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option(\"display.max_columns\", None)\npd.set_option(\"display.max_rows\", None)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:35:47.862276Z","iopub.execute_input":"2022-10-24T11:35:47.862885Z","iopub.status.idle":"2022-10-24T11:35:47.868374Z","shell.execute_reply.started":"2022-10-24T11:35:47.862851Z","shell.execute_reply":"2022-10-24T11:35:47.866831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [\n    'ball_pos_x', 'ball_pos_y','ball_pos_z', 'ball_vel_x', 'ball_vel_y', 'ball_vel_z', \n    'p0_pos_x', 'p0_pos_y', 'p0_pos_z', 'p0_vel_x', 'p0_vel_y', 'p0_vel_z', 'p0_boost', 'p0_na',\n    'p1_pos_x', 'p1_pos_y', 'p1_pos_z', 'p1_vel_x', 'p1_vel_y', 'p1_vel_z', 'p1_boost', 'p1_na',\n    'p2_pos_x', 'p2_pos_y', 'p2_pos_z', 'p2_vel_x', 'p2_vel_y', 'p2_vel_z', 'p2_boost', 'p2_na',\n    'p3_pos_x', 'p3_pos_y', 'p3_pos_z', 'p3_vel_x', 'p3_vel_y', 'p3_vel_z', 'p3_boost', 'p3_na',\n    'p4_pos_x', 'p4_pos_y', 'p4_pos_z', 'p4_vel_x', 'p4_vel_y', 'p4_vel_z', 'p4_boost', 'p4_na',\n    'p5_pos_x', 'p5_pos_y', 'p5_pos_z', 'p5_vel_x', 'p5_vel_y', 'p5_vel_z', 'p5_boost', 'p5_na',\n    'boost0_timer', 'boost1_timer', \n    'boost2_timer', 'boost3_timer',\n    'boost4_timer', 'boost5_timer']\n\nfloat_features = [\n    'ball_pos_x', 'ball_pos_y','ball_pos_z', 'ball_vel_x', 'ball_vel_y', 'ball_vel_z', \n    'p0_pos_x', 'p0_pos_y', 'p0_pos_z', 'p0_vel_x', 'p0_vel_y', 'p0_vel_z',\n    'p1_pos_x', 'p1_pos_y', 'p1_pos_z', 'p1_vel_x', 'p1_vel_y', 'p1_vel_z', \n    'p2_pos_x', 'p2_pos_y', 'p2_pos_z', 'p2_vel_x', 'p2_vel_y', 'p2_vel_z', \n    'p3_pos_x', 'p3_pos_y', 'p3_pos_z', 'p3_vel_x', 'p3_vel_y', 'p3_vel_z', \n    'p4_pos_x', 'p4_pos_y', 'p4_pos_z', 'p4_vel_x', 'p4_vel_y', 'p4_vel_z',\n    'p5_pos_x', 'p5_pos_y', 'p5_pos_z', 'p5_vel_x', 'p5_vel_y', 'p5_vel_z',\n    ]\n\nfeatures_x_pos = [pos for pos, feature in enumerate(features) if feature.endswith('_x')]\nfeatures_y_pos = [pos for pos, feature in enumerate(features) if feature.endswith('_y')]\n\ntargets = [\n    'team_A_scoring_within_10sec',\n    'team_B_scoring_within_10sec']","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:35:47.869566Z","iopub.execute_input":"2022-10-24T11:35:47.870192Z","iopub.status.idle":"2022-10-24T11:35:47.886712Z","shell.execute_reply.started":"2022-10-24T11:35:47.870159Z","shell.execute_reply":"2022-10-24T11:35:47.885794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create folders to hold tfrecords\nfor i in range(10):\n    os.system(f\"mkdir train_{i}\")\n    \nos.system(f\"mkdir test\")  ","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:35:47.888924Z","iopub.execute_input":"2022-10-24T11:35:47.889284Z","iopub.status.idle":"2022-10-24T11:35:47.936611Z","shell.execute_reply.started":"2022-10-24T11:35:47.889254Z","shell.execute_reply":"2022-10-24T11:35:47.935451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#list of all features in the dataset\nall_features=[\n    'ball_pos_x', 'ball_pos_y', 'ball_pos_z', 'ball_vel_x', 'ball_vel_y', 'ball_vel_z','distance_ball_goal_1',\n    'distance_ball_goal_2','cos_goal_1_ball_speed_angle','cos_goal_2_ball_speed_angle','sin_goal_1_ball_speed_angle','sin_goal_2_ball_speed_angle',\n    \"distance_boundary_upper_b\",\"distance_boundary_lower_b\",\"distance_boundary_right_b\",\"distance_boundary_left_b\",\"speed_modulus_b\",\n    #17 features of the ball\n    \n    'p0_pos_x', 'p0_pos_y', 'p0_pos_z', 'p0_vel_x', 'p0_vel_y', 'p0_vel_z', 'p0_boost','boost0_timer','p0_na',\n    'distance_player_0_goal_1', 'distance_player_0_goal_2', 'distance_ball_0', 'relative_speed_0',\n    'cos_angle_0', 'cos_goal_1_angle_0', 'cos_goal_2_angle_0', 'cos_goal_1_p_speed_angle_0','cos_goal_2_p_speed_angle_0',\n    'sin_angle_0', 'sin_goal_1_angle_0', 'sin_goal_2_angle_0', 'sin_goal_1_p_speed_angle_0','sin_goal_2_p_speed_angle_0',\n    \"distance_boundary_upper_0\",\"distance_boundary_lower_0\",\"distance_boundary_right_0\",\"distance_boundary_left_0\",\"speed_modulus_0\",\n    #28 features of player\n    \n    'p1_pos_x', 'p1_pos_y', 'p1_pos_z', 'p1_vel_x', 'p1_vel_y', 'p1_vel_z', 'p1_boost','boost1_timer','p1_na',\n     'distance_player_1_goal_1', 'distance_player_1_goal_2', 'distance_ball_1', 'relative_speed_1', \n    'cos_angle_1', 'cos_goal_1_angle_1', 'cos_goal_2_angle_1', 'cos_goal_1_p_speed_angle_1','cos_goal_2_p_speed_angle_1',\n    'sin_angle_1', 'sin_goal_1_angle_1', 'sin_goal_2_angle_1', 'sin_goal_1_p_speed_angle_1','sin_goal_2_p_speed_angle_1',\n    \"distance_boundary_upper_1\",\"distance_boundary_lower_1\",\"distance_boundary_right_1\",\"distance_boundary_left_1\",\"speed_modulus_1\",\n    \n    'p2_pos_x', 'p2_pos_y', 'p2_pos_z', 'p2_vel_x', 'p2_vel_y', 'p2_vel_z', 'p2_boost','boost2_timer','p2_na',\n    'distance_player_2_goal_1', 'distance_player_2_goal_2', 'distance_ball_2', 'relative_speed_2', \n    'cos_angle_2', 'cos_goal_1_angle_2', 'cos_goal_2_angle_2', 'cos_goal_1_p_speed_angle_2', 'cos_goal_2_p_speed_angle_2',\n    'sin_angle_2', 'sin_goal_1_angle_2', 'sin_goal_2_angle_2', 'sin_goal_1_p_speed_angle_2','sin_goal_2_p_speed_angle_2',\n    \"distance_boundary_upper_2\",\"distance_boundary_lower_2\",\"distance_boundary_right_2\",\"distance_boundary_left_2\",\"speed_modulus_2\",\n    \n    'p3_pos_x', 'p3_pos_y', 'p3_pos_z', 'p3_vel_x', 'p3_vel_y', 'p3_vel_z', 'p3_boost','boost3_timer','p3_na',\n    'distance_player_3_goal_1', 'distance_player_3_goal_2', 'distance_ball_3', 'relative_speed_3', \n    'cos_angle_3', 'cos_goal_1_angle_3', 'cos_goal_2_angle_3', 'cos_goal_1_p_speed_angle_3', 'cos_goal_2_p_speed_angle_3',\n    'sin_angle_3', 'sin_goal_1_angle_3', 'sin_goal_2_angle_3', 'sin_goal_1_p_speed_angle_3','sin_goal_2_p_speed_angle_3',\n    \"distance_boundary_upper_3\",\"distance_boundary_lower_3\",\"distance_boundary_right_3\",\"distance_boundary_left_3\",\"speed_modulus_3\",\n    \n    'p4_pos_x', 'p4_pos_y', 'p4_pos_z', 'p4_vel_x', 'p4_vel_y', 'p4_vel_z', 'p4_boost','boost4_timer','p4_na',\n    'distance_player_4_goal_1', 'distance_player_4_goal_2', 'distance_ball_4', 'relative_speed_4', \n    'cos_angle_4', 'cos_goal_1_angle_4', 'cos_goal_2_angle_4', 'cos_goal_1_p_speed_angle_4', 'cos_goal_2_p_speed_angle_4',\n    'sin_angle_4', 'sin_goal_1_angle_4', 'sin_goal_2_angle_4', 'sin_goal_1_p_speed_angle_4','sin_goal_2_p_speed_angle_4',\n    \"distance_boundary_upper_4\",\"distance_boundary_lower_4\",\"distance_boundary_right_4\",\"distance_boundary_left_4\",\"speed_modulus_4\",\n    \n    'p5_pos_x', 'p5_pos_y', 'p5_pos_z', 'p5_vel_x', 'p5_vel_y', 'p5_vel_z', 'p5_boost','boost5_timer','p5_na',\n    'distance_player_5_goal_1', 'distance_player_5_goal_2', 'distance_ball_5', 'relative_speed_5', \n    'cos_angle_5', 'cos_goal_1_angle_5', 'cos_goal_2_angle_5', 'cos_goal_1_p_speed_angle_5', 'cos_goal_2_p_speed_angle_5',\n    'sin_angle_5', 'sin_goal_1_angle_5', 'sin_goal_2_angle_5', 'sin_goal_1_p_speed_angle_5','sin_goal_2_p_speed_angle_5',\n    \"distance_boundary_upper_5\",\"distance_boundary_lower_5\",\"distance_boundary_right_5\",\"distance_boundary_left_5\",\"speed_modulus_5\",\n    \n    'team_A_scoring_within_10sec',\n    'team_B_scoring_within_10sec'\n ]","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:35:47.937991Z","iopub.execute_input":"2022-10-24T11:35:47.938554Z","iopub.status.idle":"2022-10-24T11:35:47.951569Z","shell.execute_reply.started":"2022-10-24T11:35:47.938523Z","shell.execute_reply":"2022-10-24T11:35:47.950509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False\nNUM_SHARDS=2\ninput_path = Path('../input/fast-loading-high-compression-with-feather/feather_data')\ncolumns={}\n\n\nrandom.seed(10)#always run the below with same seed -> find issues with generated files is easier\n\ndef fe(x,is_train=True,index=0):\n    if is_train:\n        #we are not going to use this now\n        x=x.drop(\"team_scoring_next\",axis=1)\n    \n    # indicators for respawns...\n    x['p0_na'] = x['p0_pos_x'].isna().astype('int8')\n    x['p1_na'] = x['p1_pos_x'].isna().astype('int8')\n    x['p2_na'] = x['p2_pos_x'].isna().astype('int8')\n    x['p3_na'] = x['p3_pos_x'].isna().astype('int8')\n    x['p4_na'] = x['p4_pos_x'].isna().astype('int8')\n    x['p5_na'] = x['p5_pos_x'].isna().astype('int8')\n    x['p0_na'] = x['p0_pos_x'].isna().astype('int8')\n    x['p1_na'] = x['p1_pos_x'].isna().astype('int8')\n    x['p2_na'] = x['p2_pos_x'].isna().astype('int8')\n    x['p3_na'] = x['p3_pos_x'].isna().astype('int8')\n    x['p4_na'] = x['p4_pos_x'].isna().astype('int8')\n    x['p5_na'] = x['p5_pos_x'].isna().astype('int8')\n    \n    #convert to float16 -> better use only one dtype for our NN -> don't convert game_num that is not an input\n    x[float_features]=x[float_features].astype(\"float32\")\n    x=x.fillna(0.0)\n    \n        \n    x[\"distance_ball_goal_1\"]=((x['ball_pos_x']**2+(x['ball_pos_y']-100)**2)**0.5 /120)#.astype('float16')\n    \n    x[\"distance_ball_goal_2\"]=((x['ball_pos_x']**2+(x['ball_pos_y']+100)**2)**0.5 /120)#.astype('float16')\n    \n    #cos -> dot(a,b)/[mod(a)*mod(b)] -> dot(a,b)=ax*bx + ay*by ; mod(a)=(ax**2+ay**2)**0.5\n    x[f\"cos_goal_1_ball_speed_angle\"]=(((x[f\"ball_vel_x\"])*(x[f\"ball_pos_x\"])+(x[f\"ball_vel_y\"])*(x[f\"ball_pos_y\"]-100))\\\n                                /(np.sqrt(((x[f\"ball_vel_x\"])**2+(x[f\"ball_vel_y\"])**2)*((x[f\"ball_pos_x\"]-0)**2+(x[f\"ball_pos_y\"]-100)**2))+1e-15))#.astype('float16')\n    #sin -> cross(a,b)/[mod(a)*mod(b)]\n    x[f\"sin_goal_1_ball_speed_angle\"]=(((x[f\"ball_vel_x\"])*(x[f\"ball_pos_y\"]-100)-(x[f\"ball_vel_y\"])*(x[f\"ball_pos_x\"]))\\\n                                /(np.sqrt(((x[f\"ball_vel_x\"])**2+(x[f\"ball_vel_y\"])**2)*((x[f\"ball_pos_x\"]-0)**2+(x[f\"ball_pos_y\"]-100)**2))+1e-15))#.astype('float16')\n        \n    x[f\"cos_goal_2_ball_speed_angle\"]=(((x[f\"ball_vel_x\"])*(x[f\"ball_pos_x\"])+(x[f\"ball_vel_y\"])*(x[f\"ball_pos_y\"]+100))\\\n                                /(np.sqrt(((x[f\"ball_vel_x\"])**2+(x[f\"ball_vel_y\"])**2)*((x[f\"ball_pos_x\"]-0)**2+(x[f\"ball_pos_y\"]+100)**2))+1e-15))#.astype('float16')\n    x[f\"sin_goal_2_ball_speed_angle\"]=(((x[f\"ball_vel_x\"])*(x[f\"ball_pos_y\"]+100)-(x[f\"ball_vel_y\"])*(x[f\"ball_pos_x\"]))\\\n                                /(np.sqrt(((x[f\"ball_vel_x\"])**2+(x[f\"ball_vel_y\"])**2)*((x[f\"ball_pos_x\"]-0)**2+(x[f\"ball_pos_y\"]+100)**2))+1e-15))#.astype('float16')\n    \n    \n    x[f\"distance_boundary_upper_b\"]=(x[f'ball_pos_y']-100)/120\n    x[f\"distance_boundary_lower_b\"]=(x[f'ball_pos_y']+100)/120\n    x[f\"distance_boundary_right_b\"]=(x[f'ball_pos_x']-80)/120\n    x[f\"distance_boundary_left_b\"]=(x[f'ball_pos_x']+80)/120\n    x[f\"speed_modulus_b\"]=np.sqrt(x[\"ball_vel_x\"]**2+x[\"ball_vel_y\"]**2+x[\"ball_vel_z\"]**2)/100\n    \n    \n    for i in range(6):\n        print(f\"generating player {i} features\")\n        temp=pd.DataFrame()\n        temp[f\"distance_player_{i}_goal_1\"]=((x[f'p{i}_pos_x']**2+(x[f'p{i}_pos_y']-100)**2)**0.5 /120)#.astype('float16')\n        temp[f\"distance_player_{i}_goal_2\"]=((x[f'p{i}_pos_x']**2+(x[f'p{i}_pos_y']+100)**2)**0.5 /120)#.astype('float16')\n        \n        temp[f\"distance_boundary_upper_{i}\"]=(x[f'p{i}_pos_y']-100)/120\n        temp[f\"distance_boundary_lower_{i}\"]=(x[f'p{i}_pos_y']+100)/120\n        temp[f\"distance_boundary_right_{i}\"]=(x[f'p{i}_pos_x']-80)/120\n        temp[f\"distance_boundary_left_{i}\"]=(x[f'p{i}_pos_x']+80)/120\n        \n        temp[f\"speed_modulus_{i}\"]=np.sqrt(x[f\"p{i}_vel_x\"]**2+x[f\"p{i}_vel_y\"]**2+x[f\"p{i}_vel_z\"]**2)/100\n        \n        temp[f\"distance_ball_{i}\"] = ((x[f\"p{i}_pos_x\"]-x[f\"ball_pos_x\"])**2+(x[f\"p{i}_pos_y\"]-x[f\"ball_pos_y\"])**2+(x[f\"p{i}_pos_z\"]-x[f\"ball_pos_y\"])**2)**0.5 /120\n        temp[f\"distance_ball_{i}\"]#.astype('float16')\n        \n        temp[f\"relative_speed_{i}\"] = ((x[f\"p{i}_vel_x\"]-x[f\"ball_vel_x\"])**2+(x[f\"p{i}_vel_y\"]-x[f\"ball_vel_y\"])**2+(x[f\"p{i}_vel_z\"]-x[f\"ball_vel_y\"])**2)**0.5 /120\n        temp[f\"relative_speed_{i}\"]#.astype('float16')\n        \n        # 3D angle -> let's try only 2D for now\n        #x[f\"cos_angle_{i}\"]=(x[f\"p{i}_vel_x\"]*x[f\"ball_vel_x\"]+x[f\"p{i}_vel_y\"]*x[f\"ball_vel_y\"]+x[f\"p{i}_vel_z\"]*x[f\"ball_vel_z\"])\\\n        #                        /(np.sqrt((x[f\"p{i}_vel_x\"]**2+x[f\"p{i}_vel_y\"]**2+x[f\"p{i}_vel_z\"]**2)*(x[f\"ball_vel_x\"]**2+x[f\"ball_vel_y\"]**2+x[f\"ball_vel_z\"]**2)+1e-15))#.astype('float16')\n        \n        temp[f\"cos_angle_{i}\"]=(x[f\"p{i}_vel_x\"]*x[f\"ball_vel_x\"]+x[f\"p{i}_vel_y\"]*x[f\"ball_vel_y\"])\\\n                                /(np.sqrt((x[f\"p{i}_vel_x\"]**2+x[f\"p{i}_vel_y\"]**2)*(x[f\"ball_vel_x\"]**2+x[f\"ball_vel_y\"]**2))+1e-15)#.astype('float16')\n        \n        temp[f\"sin_angle_{i}\"]=(x[f\"p{i}_vel_x\"]*x[f\"ball_vel_y\"]-x[f\"p{i}_vel_y\"]*x[f\"ball_vel_x\"])\\\n                                /(np.sqrt((x[f\"p{i}_vel_x\"]**2+x[f\"p{i}_vel_y\"]**2)*(x[f\"ball_vel_x\"]**2+x[f\"ball_vel_y\"]**2))+1e-15)#.astype('float16')\n        \n        \n        temp[f\"cos_goal_1_angle_{i}\"]=((x[f\"ball_pos_x\"]-x[f\"p{i}_pos_x\"])*(x[f\"ball_pos_x\"])+(x[f\"ball_pos_y\"]-x[f\"p{i}_pos_y\"])*(x[f\"ball_pos_y\"]-100))\\\n                                /(np.sqrt(((x[f\"p{i}_pos_x\"]-x[f\"ball_pos_x\"])**2+(x[f\"p{i}_pos_y\"]-x[f\"ball_pos_y\"])**2)*((x[f\"ball_pos_x\"]-0)**2+(x[f\"ball_pos_y\"]-100)**2))+1e-15)#.astype('float16')\n        \n        temp[f\"sin_goal_1_angle_{i}\"]=((x[f\"ball_pos_x\"]-x[f\"p{i}_pos_x\"])*(x[f\"ball_pos_y\"]-100)-(x[f\"ball_pos_y\"]-x[f\"p{i}_pos_y\"])*(x[f\"ball_pos_x\"]))\\\n                                /(np.sqrt(((x[f\"p{i}_pos_x\"]-x[f\"ball_pos_x\"])**2+(x[f\"p{i}_pos_y\"]-x[f\"ball_pos_y\"])**2)*((x[f\"ball_pos_x\"]-0)**2+(x[f\"ball_pos_y\"]-100)**2))+1e-15)#.astype('float16')\n        \n        \n        temp[f\"cos_goal_2_angle_{i}\"]=((x[f\"ball_pos_x\"]-x[f\"p{i}_pos_x\"])*(x[f\"ball_pos_x\"])+(x[f\"ball_pos_y\"]-x[f\"p{i}_pos_y\"])*(x[f\"ball_pos_y\"]+100))\\\n                                /(np.sqrt(((x[f\"p{i}_pos_x\"]-x[f\"ball_pos_x\"])**2+(x[f\"p{i}_pos_y\"]-x[f\"ball_pos_y\"])**2)*((x[f\"ball_pos_x\"]-0)**2+(x[f\"ball_pos_y\"]+100)**2))+1e-15)#.astype('float16')\n        temp[f\"sin_goal_2_angle_{i}\"]=((x[f\"ball_pos_x\"]-x[f\"p{i}_pos_x\"])*(x[f\"ball_pos_y\"]+100)-(x[f\"ball_pos_y\"]-x[f\"p{i}_pos_y\"])*(x[f\"ball_pos_x\"]))\\\n                                /(np.sqrt(((x[f\"p{i}_pos_x\"]-x[f\"ball_pos_x\"])**2+(x[f\"p{i}_pos_y\"]-x[f\"ball_pos_y\"])**2)*((x[f\"ball_pos_x\"]-0)**2+(x[f\"ball_pos_y\"]+100)**2))+1e-15)#.astype('float16')\n        \n        temp[f\"cos_goal_1_p_speed_angle_{i}\"]=(((x[f\"p{i}_vel_x\"])*(x[f\"p{i}_pos_x\"])+(x[f\"p{i}_vel_y\"])*(x[f\"p{i}_pos_y\"]-100))\\\n                                /(np.sqrt(((x[f\"p{i}_vel_x\"])**2+(x[f\"p{i}_vel_y\"])**2)*((x[f\"p{i}_pos_x\"]-0)**2+(x[f\"p{i}_pos_y\"]-100)**2)+1e-15)))#.astype('float16')\n        temp[f\"sin_goal_1_p_speed_angle_{i}\"]=(((x[f\"p{i}_vel_x\"])*(x[f\"p{i}_pos_y\"]-100)-(x[f\"p{i}_vel_y\"])*(x[f\"p{i}_pos_x\"]))\\\n                                /(np.sqrt(((x[f\"p{i}_vel_x\"])**2+(x[f\"p{i}_vel_y\"])**2)*((x[f\"p{i}_pos_x\"]-0)**2+(x[f\"p{i}_pos_y\"]-100)**2)+1e-15)))#.astype('float16')\n        \n        temp[f\"cos_goal_2_p_speed_angle_{i}\"]=(((x[f\"p{i}_vel_x\"])*(x[f\"p{i}_pos_x\"])+(x[f\"p{i}_vel_y\"])*(x[f\"p{i}_pos_y\"]+100))\\\n                                /(np.sqrt(((x[f\"p{i}_vel_x\"])**2+(x[f\"p{i}_vel_y\"])**2)*((x[f\"p{i}_pos_x\"]-0)**2+(x[f\"p{i}_pos_y\"]+100)**2))+1e-15))#.astype('float16')\n        temp[f\"sin_goal_2_p_speed_angle_{i}\"]=(((x[f\"p{i}_vel_x\"])*(x[f\"p{i}_pos_y\"]+100)+(x[f\"p{i}_vel_y\"])*(x[f\"p{i}_pos_x\"]))\\\n                                /(np.sqrt(((x[f\"p{i}_vel_x\"])**2+(x[f\"p{i}_vel_y\"])**2)*((x[f\"p{i}_pos_x\"]-0)**2+(x[f\"p{i}_pos_y\"]+100)**2))+1e-15))#.astype('float16')\n        x=pd.concat([x, temp], axis = 1)\n        \n        del temp\n        gc.collect()\n        \n        \n   \n        \n    for feature in features:\n        if feature.endswith('_na'):\n            continue\n        # this is just scaling the features to something reasonable\n        # it might make sense to apply a transformation to the z-dimension.\n        if feature.endswith('_x'):\n            x[feature] = (x[feature] / 120).fillna(0)#.astype('float16')\n        if feature.endswith('_y'):\n            x[feature] = (x[feature] / 120).fillna(0)#.astype('float16')\n        if feature.endswith('_z'):\n            x[feature] = (x[feature] / 120).fillna(0)#.astype('float16')\n        if feature.endswith('_boost'):\n            x[feature] = (x[feature] / 100).fillna(0)#.astype('float16')\n        if feature.endswith('_timer'):\n            x[feature] = (-x[feature] / 10).fillna(0)#.astype('float16')\n        elif feature[-1] in [i for i in range(6)]:\n            x[feature] = (x[feature] / 10)#.astype('float16')\n\n    \n    gc.collect()\n    if is_train:\n        x=x[[\"game_num\"]+all_features]\n        nans=np.sum(x.isna().sum())\n        print(f\"number of nulls before conversion {nans}\")\n    else:\n        x=x[[\"id\"]+all_features[:-2]]\n        nans=np.sum(x.isna().sum())\n        print(f\"number of nulls before conversion {nans}\")\n    #description=x.describe()\n    #description=description.iloc[[3,7]]#min and max\n    #mask=(description.iloc[1]>10)|(description.iloc[0]<-10) # boolean mask for out of range variables -> we consider here -10,10 as acceptable range\n    #mask[0]=False #game num will always have out of range values -> ignore\n    #if np.sum(mask)>0:\n    #    print(description.transpose()[mask])#see where there are features with modulus >10 \n    #else:\n    #    print(\"no out of bounds\")\n    \n    \n    if is_train:\n        #shuffle games \n        game_nums = x['game_num'].unique()\n        random.shuffle(game_nums)\n\n        #split games into 10 bins\n        game_numbers = np.array_split(game_nums,10)\n        #save games of each bin into tfrecords -> bins are created to allow k fold.\n        for i in range(10):\n            print(f\"saving tfrecords {i}\")\n            current_games=game_numbers[i]\n            \n            \n            df=x.query(\"game_num in @current_games\").drop([\"game_num\"],axis=1)[all_features]\n           \n        \n            PATH_PREFIX = f'/kaggle/working/train_{i}/feats.tfrecord'\n            \n            \n            ds = Dataset.from_tensor_slices((df.to_numpy().astype(\"float32\")))\n            \n            \n            ds_feats= ds.map(tf.io.serialize_tensor)\n            \n            #from tensorflow documentation https://www.tensorflow.org/api_docs/python/tf/data/experimental/TFRecordWriter\n            #code for sharding\n            def reduce_func(key, dataset):\n                filename = tf.strings.join([PATH_PREFIX, tf.strings.as_string(key+index*NUM_SHARDS)])#place into different shards different parts of dataset\n                writer=tf.data.experimental.TFRecordWriter(filename)\n                writer.write(dataset.map(lambda _, x: x))\n                del writer\n                return tf.data.Dataset.from_tensors(np.array([0]))#something ro\n            \n            \n            ds_feats = ds_feats.enumerate()\n            dataset = ds_feats.apply(tf.data.experimental.group_by_window(\n              lambda i, _: i % NUM_SHARDS, reduce_func, tf.int64.max\n            ))\n            \n            # Iterate through the dataset to trigger data writing.\n            for _ in dataset:\n                pass\n            del dataset\n            del ds_feats\n            del df\n            del ds\n            gc.collect()\n            \n            \n    else:\n        # clean writing without sharding\n        print(f\"saving tfrecords TEST\")\n        writer = tf.data.experimental.TFRecordWriter(f'/kaggle/working/test/feats.tfrecord')\n        df=x.drop(\"id\",axis=1)[all_features[:-2]]#get all columns except the last 2 (the targets. They are not available in the test set)\n        ds=Dataset.from_tensor_slices((df.to_numpy().astype(\"float32\")))\n        writer.write(ds.map(tf.io.serialize_tensor))\n        columns[\"cols\"]=df.columns\n    del x \n        \n\ndef read_train():\n    for i in range(10):\n        print(f\"-------------------- started dataset {i} --------------------\")\n        fe(pd.read_feather(input_path / f'train_{i}_compressed.ftr'),is_train=True,index=i)\n        \n        gc.collect()\n    \ndef read_test():\n    print(f\"-------------------- started TEST dataset --------------------\")\n    return fe(pd.read_feather(input_path / 'test_compressed.ftr'),is_train=False)\n\n\nx=read_train()\nread_test()\n\n_=gc.collect()\n\n","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-10-24T11:46:28.704384Z","iopub.execute_input":"2022-10-24T11:46:28.706697Z","iopub.status.idle":"2022-10-24T11:47:58.721516Z","shell.execute_reply.started":"2022-10-24T11:46:28.706631Z","shell.execute_reply":"2022-10-24T11:47:58.720113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check dataset","metadata":{}},{"cell_type":"code","source":"#utility\ndef get_next(ds):\n    return next(iter(ds))","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:47:58.726327Z","iopub.execute_input":"2022-10-24T11:47:58.727030Z","iopub.status.idle":"2022-10-24T11:47:58.732406Z","shell.execute_reply.started":"2022-10-24T11:47:58.726980Z","shell.execute_reply":"2022-10-24T11:47:58.731598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get dataset from a folder of tfrecords\nPATH=tf.io.gfile.glob(f'/kaggle/working/train_0/feats.tfrecord*')\nds = TFRecordDataset(PATH)\nds_feats = ds.map(lambda x: tf.ensure_shape(tf.io.parse_tensor(x, out_type=tf.float32),(len(all_features)))) #we statically give the number of features to expect \nget_next(ds_feats).shape,len(all_features) #check shape","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.161285Z","iopub.status.idle":"2022-10-24T11:38:48.161722Z","shell.execute_reply.started":"2022-10-24T11:38:48.161511Z","shell.execute_reply":"2022-10-24T11:38:48.161531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#convert dataset to x,(y0,y1) \nds1=ds_feats.map(lambda x: (x[:-2],([x[-2]],[x[-1]])))\nexample=get_next(ds1)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.164546Z","iopub.status.idle":"2022-10-24T11:38:48.165598Z","shell.execute_reply.started":"2022-10-24T11:38:48.165380Z","shell.execute_reply":"2022-10-24T11:38:48.165401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#extract features and targets\nfeats=example[0]\ntarget0,target1=example[1]","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.166816Z","iopub.status.idle":"2022-10-24T11:38:48.167770Z","shell.execute_reply.started":"2022-10-24T11:38:48.167570Z","shell.execute_reply":"2022-10-24T11:38:48.167591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check shapes\nfeats.shape,target0.shape,target1.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.169230Z","iopub.status.idle":"2022-10-24T11:38:48.170054Z","shell.execute_reply.started":"2022-10-24T11:38:48.169848Z","shell.execute_reply":"2022-10-24T11:38:48.169869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Combine 2 datasets","metadata":{}},{"cell_type":"code","source":"#get dataset from another folder of tfrecords\nPATH=tf.io.gfile.glob(f'/kaggle/working/train_1/feats.tfrecord*')\nds = TFRecordDataset(PATH)\nds_feats = ds.map(lambda x: tf.ensure_shape(tf.io.parse_tensor(x, out_type=tf.float32),(len(all_features))))","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.171310Z","iopub.status.idle":"2022-10-24T11:38:48.171791Z","shell.execute_reply.started":"2022-10-24T11:38:48.171554Z","shell.execute_reply":"2022-10-24T11:38:48.171573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds2=ds_feats.map(lambda x: (x[:-2],([x[-2]],[x[-1]])))","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.173921Z","iopub.status.idle":"2022-10-24T11:38:48.174329Z","shell.execute_reply.started":"2022-10-24T11:38:48.174138Z","shell.execute_reply":"2022-10-24T11:38:48.174157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds1_2=ds1.concatenate(ds2)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.175550Z","iopub.status.idle":"2022-10-24T11:38:48.175957Z","shell.execute_reply.started":"2022-10-24T11:38:48.175745Z","shell.execute_reply":"2022-10-24T11:38:48.175763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"ds1_2 now contains both ds1 and ds2","metadata":{}},{"cell_type":"markdown","source":"# Easy example usage","metadata":{}},{"cell_type":"code","source":"PATH=tf.io.gfile.glob(f'/kaggle/working/train_2/feats.tfrecord*')\nds = TFRecordDataset(PATH)\nds_feats = ds.map(lambda x: tf.ensure_shape(tf.io.parse_tensor(x, out_type=tf.float32),(len(all_features))))","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.177364Z","iopub.status.idle":"2022-10-24T11:38:48.177833Z","shell.execute_reply.started":"2022-10-24T11:38:48.177606Z","shell.execute_reply":"2022-10-24T11:38:48.177626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds3=ds_feats.map(lambda x: (x[:-2],([x[-2]],[x[-1]])))","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.179302Z","iopub.status.idle":"2022-10-24T11:38:48.180406Z","shell.execute_reply.started":"2022-10-24T11:38:48.180198Z","shell.execute_reply":"2022-10-24T11:38:48.180219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds=ds1_2.shuffle(100000).batch(128)\nvalid_ds=ds3.batch(128)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.181888Z","iopub.status.idle":"2022-10-24T11:38:48.182359Z","shell.execute_reply.started":"2022-10-24T11:38:48.182108Z","shell.execute_reply":"2022-10-24T11:38:48.182147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds,valid_ds","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.183550Z","iopub.status.idle":"2022-10-24T11:38:48.183965Z","shell.execute_reply.started":"2022-10-24T11:38:48.183748Z","shell.execute_reply":"2022-10-24T11:38:48.183768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.layers import Dense,Dropout,Input,GaussianNoise,Lambda\nfrom tensorflow.keras.losses import  BinaryCrossentropy","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.185381Z","iopub.status.idle":"2022-10-24T11:38:48.185826Z","shell.execute_reply.started":"2022-10-24T11:38:48.185601Z","shell.execute_reply":"2022-10-24T11:38:48.185621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"displacement=tf.constant([*([i for i in range(28)]*3)],dtype=tf.int64)\nindices=tf.range(0,3,dtype=tf.int64)\nindices_ball=tf.range(0,17,dtype=tf.int64)\ndef augment(x):\n    #shuffle team A\n    shuffled_indices = tf.random.shuffle(indices)\n    indices_teamA=tf.repeat(shuffled_indices,28,axis=-1)*28+displacement+17\n    original_indicesA=tf.repeat(indices,28,axis=-1)*28+displacement+17\n    a=tf.gather(x,indices_teamA,axis=-1)\n    \n    shuffled_indices = tf.random.shuffle(indices)\n    indices_teamB=tf.repeat(shuffled_indices,28,axis=-1)*28+displacement+101\n    original_indicesB=tf.repeat(indices,28,axis=-1)*28+displacement+17\n    b=tf.gather(x,indices_teamB,axis=-1)\n    \n    ball=tf.gather(x,indices_ball,axis=-1)\n    return tf.concat([ball,a,b],axis=-1)\n    \n    \ninputs=Input(shape=[len(all_features)-2])\nx = Lambda(augment)(inputs)\nx=GaussianNoise(0.01)(x)\nx=Dense(64,activation=\"gelu\")(x)\nx=Dropout(0.1)(x)\n\nout1=Dense(1,activation=\"sigmoid\",name=\"teamA\")(x)\nout2=Dense(1,activation=\"sigmoid\",name=\"teamB\")(x)\n\nmodel=Model(inputs=inputs,outputs=[out1,out2])\n\nmodel.compile(\"adam\",[BinaryCrossentropy(from_logits=False),BinaryCrossentropy(from_logits=False)],[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.187738Z","iopub.status.idle":"2022-10-24T11:38:48.188171Z","shell.execute_reply.started":"2022-10-24T11:38:48.187970Z","shell.execute_reply":"2022-10-24T11:38:48.187990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train_ds,validation_data=valid_ds,epochs=3,verbose=2) # just 3 epochs to try out the approach","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.189812Z","iopub.status.idle":"2022-10-24T11:38:48.190233Z","shell.execute_reply.started":"2022-10-24T11:38:48.190037Z","shell.execute_reply":"2022-10-24T11:38:48.190057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"PATH=tf.io.gfile.glob(f'/kaggle/working/test/feats.tfrecord')\nds = TFRecordDataset(PATH)\nds_feats = ds.map(lambda x: tf.ensure_shape(tf.io.parse_tensor(x, out_type=tf.float32),(len(all_features)-2)))","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.191435Z","iopub.status.idle":"2022-10-24T11:38:48.191833Z","shell.execute_reply.started":"2022-10-24T11:38:48.191621Z","shell.execute_reply":"2022-10-24T11:38:48.191639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds=model.predict(ds_feats.batch(512),verbose=2)\npreds2=model.predict(ds_feats.batch(512),verbose=2)\npreds3=model.predict(ds_feats.batch(512),verbose=2)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.193732Z","iopub.status.idle":"2022-10-24T11:38:48.194146Z","shell.execute_reply.started":"2022-10-24T11:38:48.193954Z","shell.execute_reply":"2022-10-24T11:38:48.193973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds[0].shape,preds[1].shape","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.196109Z","iopub.status.idle":"2022-10-24T11:38:48.196524Z","shell.execute_reply.started":"2022-10-24T11:38:48.196287Z","shell.execute_reply":"2022-10-24T11:38:48.196304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.read_csv(\"../input/tabular-playground-series-oct-2022/sample_submission.csv\")\nlen(df)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.198063Z","iopub.status.idle":"2022-10-24T11:38:48.198469Z","shell.execute_reply.started":"2022-10-24T11:38:48.198264Z","shell.execute_reply":"2022-10-24T11:38:48.198283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"team_A_scoring_within_10sec\"]=(preds[0]+preds2[0]+preds3[0])/3\ndf[\"team_B_scoring_within_10sec\"]=(preds[1]+preds2[1]+preds3[1])/3","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.200027Z","iopub.status.idle":"2022-10-24T11:38:48.200368Z","shell.execute_reply.started":"2022-10-24T11:38:48.200196Z","shell.execute_reply":"2022-10-24T11:38:48.200212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-24T11:38:48.202362Z","iopub.status.idle":"2022-10-24T11:38:48.202744Z","shell.execute_reply.started":"2022-10-24T11:38:48.202555Z","shell.execute_reply":"2022-10-24T11:38:48.202574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(\"submission.csv\",index=False)","metadata":{},"execution_count":null,"outputs":[]}]}