{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# GENERATE PERSONAL DATASETS\nThis is a notebook where explain how to create my personal dataset, that it's used in other complementary notebook. https://www.kaggle.com/code/josmejagamarra/tps-oct-22-personal-subm","metadata":{}},{"cell_type":"code","source":"#IMPORT THE LIBRARIES\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_columns', None) \nimport gc\nimport tensorflow as tf\nimport time","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. CONFIGURE THE TPU","metadata":{}},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    BATCH_SIZE = 4096 * strategy.num_replicas_in_sync\n    print(\"TPU\")\nexcept:\n    tpu = None\n    strategy = tf.distribute.get_strategy()\n    BATCH_SIZE=512\n    print(\"CPU\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. LOAD THE TRAINING DATA","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\n\n#Load the dtypes\ndtypes_df = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_dtypes.csv')\ndtypes_df['dtype']=dtypes_df['dtype'].replace(['float64','float32'],'float16') #Use float16\ndtypes = {k: v for (k, v) in zip(dtypes_df.column, dtypes_df.dtype)}\n\n#Filter the useless columns\nall_columns =list(pd.read_csv(\"/kaggle/input/tabular-playground-series-oct-2022/train_0.csv\",nrows=1))\nuseless_columns = ['event_id','event_time','player_scoring_next','team_scoring_next']\nusecols = [i for i in all_columns if i not in useless_columns]\n\n#Read the train data\ntrain0 = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_0.csv',usecols = usecols ,dtype=dtypes)\ntrain1 = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_1.csv', usecols = usecols ,dtype=dtypes)\ntrain2 = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_2.csv', usecols = usecols ,dtype=dtypes)\ntrain3 = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_3.csv',usecols = usecols , dtype=dtypes)\ntrain4 = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_4.csv',usecols = usecols , dtype=dtypes)\ntrain5 = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_5.csv',usecols = usecols , dtype=dtypes)\ntrain6 = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_6.csv',usecols = usecols , dtype=dtypes)\ntrain7 = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_7.csv', usecols = usecols ,dtype=dtypes)\ntrain8 = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_8.csv',usecols = usecols , dtype=dtypes)\ntrain9 = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_9.csv', usecols = usecols ,dtype=dtypes)\n\n#Concatenate the data\ndata = pd.concat([train0,train1,train2,train3,train4,train5,train6,train7,train8,train9],axis=0).fillna(0)\ndel(dtypes_df,dtypes,all_columns,useless_columns,usecols,train0,train1,train2,train3,train4,train5,train6,train7,train8,train9)\n_=gc.collect()\n\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. FEATURE ENGINEERING","metadata":{}},{"cell_type":"code","source":"#Distance between player and the ball\ndef dist_ball_player (data):\n    for i in np.arange(0,6,1):\n        data[f'p{i}_ball'] = np.sqrt(\n                               (data[f'p{i}_pos_x']-data['ball_pos_x'])**2+\n                               (data[f'p{i}_pos_y']-data['ball_pos_y'])**2+\n                               (data[f'p{i}_pos_z']-data['ball_pos_z'])**2\n                                )\n    return data\n\n#VECTOR BETWEEN BALL AND GATE\ndef vect_ball_gate (data):\n    for i,n in [(\"A\",-100),(\"B\",100)]:\n        data[f'ball_gate{i}_x'] = data['ball_pos_x']-(0)\n        data[f'ball_gate{i}_y'] = data['ball_pos_y']-(n)\n        data[f'ball_gate{i}_z'] = data['ball_pos_z']-(0)\n    return data\n\n#DISTANCE BEETWEEN BOOST POSITION AND PLAYER\nboost_pos_dic={'boost0_pos':[-61.4, -81.9, 0],\n              'boost1_pos':[61.4, -81.9, 0],\n              'boost2_pos':[-71.7, 0, 0],\n              'boost3_pos':[71.7, 0, 0],\n              'boost4_pos':[-61.4, 81.9, 0],\n              'boost5_pos':[61.4, 81.9, 0]}\n\ndef dist_boost_player (data):\n    for i in np.arange(0,6,1):\n        for j in np.arange(0,6,1):\n            data[f'p{i}_boost{j}'] = np.sqrt(\n                (data[f'p{i}_pos_x']-boost_pos_dic[f'boost{i}_pos'][0])**2+\n                (data[f'p{i}_pos_y']-boost_pos_dic[f'boost{i}_pos'][1])**2+\n                (data[f'p{i}_pos_z']-boost_pos_dic[f'boost{i}_pos'][2])**2\n            )\n    return data\n\n#BALL VELOCITY VALUE\ndef ball_vel_val (data):\n    data['ball_vel']=np.sqrt(\n        data['ball_vel_x']**2+\n        data['ball_vel_y']**2+\n        data['ball_vel_z']**2\n    )\n    return data\n\n#PLAYER VELOCITY VALUE\ndef player_vel_val (data):\n    for i in np.arange(0,6,1):\n        data[f'p{i}_vel']=np.sqrt(\n            data[f'p{i}_vel_x']**2+\n            data[f'p{i}_vel_y']**2+\n            data[f'p{i}_vel_z']**2\n        )\n    return data\n\n#APLY THE FEATURE ENGINEERING\nstart_time = time.time()\n##\ndata = dist_ball_player (data)\ndata = vect_ball_gate (data)\ndata = dist_boost_player (data)\ndata = ball_vel_val (data)\ndata = player_vel_val (data)\n##\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# MOVE THE TARGET COLUMNS TO THE END\ndf_temp = data.pop('team_A_scoring_within_10sec')\ndata['team_A_scoring_within_10sec'] =df_temp\n\ndf_temp = data.pop('team_B_scoring_within_10sec')\ndata['team_B_scoring_within_10sec'] =df_temp\n\ndel(df_temp)\n_=gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. GENERATE USEFULL COLUMNS LIST","metadata":{}},{"cell_type":"code","source":"P0=[k for k in data.columns.to_list() if k.startswith(\"p0\")]\nP1=[k for k in data.columns.to_list() if k.startswith(\"p1\")]\nP2=[k for k in data.columns.to_list() if k.startswith(\"p2\")]\nP3=[k for k in data.columns.to_list() if k.startswith(\"p3\")]\nP4=[k for k in data.columns.to_list() if k.startswith(\"p4\")]\nP5=[k for k in data.columns.to_list() if k.startswith(\"p5\")]\n\nBasic=[k for k in data.columns.to_list() if not k.startswith((\"p0\",\n                                                               \"p1\",\n                                                               \"p2\",\n                                                               \"p3\",\n                                                               \"p4\",\n                                                               \"p5\",\n                                                               \"team_A_scoring_within_10sec\",\n                                                              \"team_B_scoring_within_10sec\"))]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#DEFINE UTILS\ngame_nums = data['game_num'].unique()\ngame_numbers = np.array_split(game_nums,10) #split games into 10 bins\n\ncolumns_t = data.drop(columns=['game_num']).columns\nds_size = data.shape[0]\nNUM_SHARDS = 1\n\nfrom tensorflow.data import Dataset, TFRecordDataset\nimport os","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. SAVE THE INPUT & TARGET DATA","metadata":{}},{"cell_type":"code","source":"# SAVE THE DATA FROM TEAM A - INPUT 1\nwith strategy.scope():\n    start_time = time.time()\n    ##\n    for i in range(10):\n        print(f\"saving tfrecords {i}\")\n        current_games=game_numbers[i]\n        df=data.query(\"game_num in @current_games\")\n        df=df[Basic+P0+P1+P2]\n        df=df.drop(columns=['game_num'])\n        PATH_PREFIX = f'/kaggle/working/Team_A_input/train_{i}/feats.tfrecord'\n\n        ds_feats = Dataset.from_tensor_slices(df.astype(\"float16\"))\n        ds_feats= ds_feats.map(tf.io.serialize_tensor)\n\n        def reduce_func(key, dataset):\n            filename = tf.strings.join([PATH_PREFIX, tf.strings.as_string(key)])#place into different shards different parts of dataset\n            writer = tf.data.experimental.TFRecordWriter(filename) \n            writer.write(dataset.map(lambda _, x: x))\n            return tf.data.Dataset.from_tensors(filename)\n\n        ds_feats = ds_feats.enumerate()\n        dataset = ds_feats.apply(tf.data.experimental.group_by_window( \n            lambda i, _: i % NUM_SHARDS, reduce_func, tf.int64.max\n        ))\n\n        # Iterate through the dataset to trigger data writing.\n        for _ in dataset:\n            pass\n    ##\n    print(\"--- %s seconds ---\" % (time.time() - start_time))\n\ndel(df, ds_feats,dataset)\n_=gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SAVE DATA FROM TEAM B - INPUT 2\nwith strategy.scope():\n    import time\n    start_time = time.time()\n    ##\n    for i in range(10):\n        print(f\"saving tfrecords {i}\")\n        current_games=game_numbers[i]\n        df=data.query(\"game_num in @current_games\")\n        df=df[Basic+P3+P4+P5]\n        df=df.drop(columns=['game_num'])\n        PATH_PREFIX = f'/kaggle/working/Team_B_input/train_{i}/feats.tfrecord'\n\n        ds_feats = Dataset.from_tensor_slices(df.astype(\"float16\"))\n        ds_feats= ds_feats.map(tf.io.serialize_tensor)\n\n        def reduce_func(key, dataset):\n            filename = tf.strings.join([PATH_PREFIX, tf.strings.as_string(key)])#place into different shards different parts of dataset\n            writer = tf.data.experimental.TFRecordWriter(filename)\n            writer.write(dataset.map(lambda _, x: x))\n            return tf.data.Dataset.from_tensors(filename)\n\n        ds_feats = ds_feats.enumerate()\n        dataset = ds_feats.apply(tf.data.experimental.group_by_window( \n            lambda i, _: i % NUM_SHARDS, reduce_func, tf.int64.max\n        ))\n\n        # Iterate through the dataset to trigger data writing.\n        for _ in dataset:\n            pass\n    ##\n    print(\"--- %s seconds ---\" % (time.time() - start_time))\n    \ndel(df, ds_feats,dataset)\n_=gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#SAVE DATA FROM TARGET\nPATH_PREFIX = '/kaggle/working/Target/target.tfrecord'\ndf = data[[\"team_A_scoring_within_10sec\",\"team_B_scoring_within_10sec\"]]\ndf = Dataset.from_tensor_slices(df.astype(\"float16\"))\ndf = df.map(tf.io.serialize_tensor)\nwriter = tf.data.experimental.TFRecordWriter(PATH_PREFIX)\nwriter.write(df)\n\ndel(df, data)\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. SAVE THE TEST DATA","metadata":{}},{"cell_type":"code","source":"# LOAD THE DATASET FROM TEST\nstart_time = time.time()\n##\ndtypes_df = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/test_dtypes.csv')\ndtypes_df['dtype']=dtypes_df['dtype'].replace(['float64','float32'],'float16') #Use float16\ndtypes = {k: v for (k, v) in zip(dtypes_df.column, dtypes_df.dtype)}\nall_columns =list(pd.read_csv(\"/kaggle/input/tabular-playground-series-oct-2022/test.csv\",nrows=1))\ntest = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/test.csv',dtype=dtypes)\ndata_t = test.drop(columns=['id']).fillna(0)\n##\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# APPLYING ALL FEATURE FUNCTIONS TO THE TEST DATA\ndata_t = dist_ball_player (data_t)\ndata_t = vect_ball_gate (data_t)\ndata_t = dist_boost_player (data_t)\ndata_t = ball_vel_val (data_t)\ndata_t = player_vel_val (data_t)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SAVE DATA FROM TEST- INPUT 1\nPATH_PREFIX = '/kaggle/working/Test/test_in_1.tfrecord'\ndf = data_t[Basic[1:]+P0+P1+P2]\ndf = tf.data.Dataset.from_tensor_slices(df.astype(\"float16\"))\ndf = df.map(tf.io.serialize_tensor)\nwriter = tf.data.experimental.TFRecordWriter(PATH_PREFIX)\nwriter.write(df)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SAVE DATA FROM TEST- INPUT 2\nPATH_PREFIX = '/kaggle/working/Test/test_in_2.tfrecord'\ndf = data_t[Basic[1:]+P3+P4+P5]\ndf = tf.data.Dataset.from_tensor_slices(df.astype(\"float16\"))\ndf = df.map(tf.io.serialize_tensor)\nwriter = tf.data.experimental.TFRecordWriter(PATH_PREFIX)\nwriter.write(df)","metadata":{},"execution_count":null,"outputs":[]}]}