{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#!pip install tensorflow-addons","metadata":{"papermill":{"duration":0.019159,"end_time":"2022-10-19T09:07:05.574131","exception":false,"start_time":"2022-10-19T09:07:05.554972","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:21.029339Z","iopub.execute_input":"2022-10-20T09:01:21.029801Z","iopub.status.idle":"2022-10-20T09:01:21.048451Z","shell.execute_reply.started":"2022-10-20T09:01:21.029721Z","shell.execute_reply":"2022-10-20T09:01:21.047541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd \nimport numpy as np \nimport matplotlib.pyplot as plt\nimport gc\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nimport random \nfrom sklearn.model_selection import KFold\n\nimport tensorflow as tf \nfrom tensorflow import keras\nfrom keras import Model, layers\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom keras.utils.vis_utils import plot_model\n\nimport tensorflow_addons as tfa\nimport random ","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":5.965,"end_time":"2022-10-19T09:07:11.545668","exception":false,"start_time":"2022-10-19T09:07:05.580668","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:21.051907Z","iopub.execute_input":"2022-10-20T09:01:21.052847Z","iopub.status.idle":"2022-10-20T09:01:28.151537Z","shell.execute_reply.started":"2022-10-20T09:01:21.052799Z","shell.execute_reply":"2022-10-20T09:01:28.150570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))\ntf.debugging.set_log_device_placement(False)\ngpus = tf.config.list_logical_devices('GPU')\nstrategy = tf.distribute.MirroredStrategy(gpus)","metadata":{"papermill":{"duration":2.721724,"end_time":"2022-10-19T09:07:14.273685","exception":false,"start_time":"2022-10-19T09:07:11.551961","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:28.152919Z","iopub.execute_input":"2022-10-20T09:01:28.154518Z","iopub.status.idle":"2022-10-20T09:01:31.023749Z","shell.execute_reply.started":"2022-10-20T09:01:28.154478Z","shell.execute_reply":"2022-10-20T09:01:31.022808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FILES = random.sample(range(10), 7)\nprint(\"Files used\",FILES)\nEARLY_STOPPING = 10\nEPOCHS = 50\nBATCH_SIZE = 4096 \n\nSCALING = True\nACTIVATION = 'relu'  # tfa.activations.mish 'swish' 'selu'\n\nCROSS_VAL = True\nSPLITS = 5","metadata":{"papermill":{"duration":0.014934,"end_time":"2022-10-19T09:07:14.294914","exception":false,"start_time":"2022-10-19T09:07:14.279980","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:03:39.771697Z","iopub.execute_input":"2022-10-20T09:03:39.772609Z","iopub.status.idle":"2022-10-20T09:03:39.779455Z","shell.execute_reply.started":"2022-10-20T09:03:39.772572Z","shell.execute_reply":"2022-10-20T09:03:39.778353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes_dict = {\n    'game_num': 'int8', 'event_id': 'int8', 'event_time': 'float16',\n    'ball_pos_x': 'float16', 'ball_pos_y': 'float16', 'ball_pos_z': 'float16',\n    'ball_vel_x': 'float16', 'ball_vel_y': 'float16', 'ball_vel_z': 'float16',\n    'p0_pos_x': 'float16', 'p0_pos_y': 'float16', 'p0_pos_z': 'float16',\n    'p0_vel_x': 'float16', 'p0_vel_y': 'float16', 'p0_vel_z': 'float16',\n    'p0_boost': 'float16', 'p1_pos_x': 'float16', 'p1_pos_y': 'float16',\n    'p1_pos_z': 'float16', 'p1_vel_x': 'float16', 'p1_vel_y': 'float16',\n    'p1_vel_z': 'float16', 'p1_boost': 'float16', 'p2_pos_x': 'float16',\n    'p2_pos_y': 'float16', 'p2_pos_z': 'float16', 'p2_vel_x': 'float16',\n    'p2_vel_y': 'float16', 'p2_vel_z': 'float16', 'p2_boost': 'float16',\n    'p3_pos_x': 'float16', 'p3_pos_y': 'float16', 'p3_pos_z': 'float16',\n    'p3_vel_x': 'float16', 'p3_vel_y': 'float16', 'p3_vel_z': 'float16',\n    'p3_boost': 'float16', 'p4_pos_x': 'float16', 'p4_pos_y': 'float16',\n    'p4_pos_z': 'float16', 'p4_vel_x': 'float16', 'p4_vel_y': 'float16',\n    'p4_vel_z': 'float16', 'p4_boost': 'float16', 'p5_pos_x': 'float16',\n    'p5_pos_y': 'float16', 'p5_pos_z': 'float16', 'p5_vel_x': 'float16',\n    'p5_vel_y': 'float16', 'p5_vel_z': 'float16', 'p5_boost': 'float16',\n    'boost0_timer': 'float16', 'boost1_timer': 'float16', 'boost2_timer': 'float16',\n    'boost3_timer': 'float16', 'boost4_timer': 'float16', 'boost5_timer': 'float16',\n    'player_scoring_next': 'O', 'team_scoring_next': 'O', 'team_A_scoring_within_10sec': 'O',\n    'team_B_scoring_within_10sec': 'O'\n}\n\npath_to_data = '../input/fast-loading-high-compression-with-feather/feather_data'\ndf = pd.DataFrame({}, columns=dtypes_dict.keys())\nfor i in FILES:\n    dt_read = pd.read_feather(f'{path_to_data}/train_{i}_compressed.ftr').astype(dtypes_dict)\n    dt_read = dt_read.astype(dtypes_dict)\n    df = pd.concat([df, dt_read])\n    del dt_read\n    gc.collect()\n\ndf_test = pd.read_feather(\"../input/fast-loading-high-compression-with-feather/feather_data/test_compressed.ftr\")\ndf_test.drop(\"id\",axis =1, inplace = True)","metadata":{"papermill":{"duration":31.783448,"end_time":"2022-10-19T09:07:46.084332","exception":false,"start_time":"2022-10-19T09:07:14.300884","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:02:35.563551Z","iopub.execute_input":"2022-10-20T09:02:35.563930Z","iopub.status.idle":"2022-10-20T09:03:02.632049Z","shell.execute_reply.started":"2022-10-20T09:02:35.563898Z","shell.execute_reply":"2022-10-20T09:03:02.631036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trn_drop_cols = [\"game_num\",\"event_id\",\"event_time\",\"player_scoring_next\",\"team_scoring_next\"]\ndf.drop(columns=trn_drop_cols,inplace = True)","metadata":{"papermill":{"duration":8.837393,"end_time":"2022-10-19T09:07:54.928572","exception":false,"start_time":"2022-10-19T09:07:46.091179","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:03:02.633836Z","iopub.execute_input":"2022-10-20T09:03:02.634235Z","iopub.status.idle":"2022-10-20T09:03:10.088456Z","shell.execute_reply.started":"2022-10-20T09:03:02.634199Z","shell.execute_reply":"2022-10-20T09:03:10.087499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{"papermill":{"duration":0.005909,"end_time":"2022-10-19T09:07:54.940893","exception":false,"start_time":"2022-10-19T09:07:54.934984","status":"completed"},"tags":[]}},{"cell_type":"code","source":"goal1_location = (0,100,0) #assumption that goal centre is at z= 0  (unknown how high the goal is)\ngoal2_location = (0,-100,0) ","metadata":{"papermill":{"duration":0.015323,"end_time":"2022-10-19T09:07:54.962216","exception":false,"start_time":"2022-10-19T09:07:54.946893","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.389536Z","iopub.status.idle":"2022-10-20T09:01:31.390038Z","shell.execute_reply.started":"2022-10-20T09:01:31.389775Z","shell.execute_reply":"2022-10-20T09:01:31.389798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = [\n    \"p0_pos_x\",\"p1_pos_x\",\"p2_pos_x\",\"p3_pos_x\",\"p4_pos_x\",\"p5_pos_x\",\n    \"p0_pos_y\",\"p1_pos_y\",\"p2_pos_y\",\"p3_pos_y\",\"p4_pos_y\",\"p5_pos_y\",\n    \"p0_pos_z\",\"p1_pos_z\",\"p2_pos_z\",\"p3_pos_z\",\"p4_pos_z\",\"p5_pos_z\",\n    \n    \"p0_vel_x\",\"p1_vel_x\",\"p2_vel_x\",\"p3_vel_x\",\"p4_vel_x\",\"p5_vel_x\",\n    \"p0_vel_y\",\"p1_vel_y\",\"p2_vel_y\",\"p3_vel_y\",\"p4_vel_y\",\"p5_vel_y\",\n    \"p0_vel_z\",\"p1_vel_z\",\"p2_vel_z\",\"p3_vel_z\",\"p4_vel_z\",\"p5_vel_z\",\n    \n    'p0_boost','p1_boost','p2_boost','p3_boost','p4_boost','p5_boost'\n       ]\n\ndef fillNan(cols, data):\n    random.seed()\n    num = random.random()\n    for i in cols:\n        data[i] = data[i].fillna(0)\n        data[i] = data[i].replace(0, (sum(data[i])/10676072)-num)\n        \ndef FeatureEngineering(data):\n    # distance of ball to goal1 (0,100,0) and goal2 (0,-100,0) \n    data[f\"goal1_distance\"] = ((data[\"ball_pos_x\"]-goal1_location[0])**2 + (data[\"ball_pos_y\"]-goal1_location[1])**2 + (data[\"ball_pos_z\"]-goal1_location[2])**2)**0.5\n    data[f\"goal2_distance\"] = ((data[\"ball_pos_x\"]-goal2_location[0])**2 + (data[\"ball_pos_y\"]-goal2_location[1])**2 + (data[\"ball_pos_z\"]-goal2_location[2])**2)**0.5\n    \n    #Ball direction \n    #data[\"ball_direction\"] = ((data[\"ball_vel_x\"])**2 + (data[\"ball_vel_y\"])**2 + (data[\"ball_vel_z\"])**2)**0.5\n    \n    #general direction of play of all players\n#     data[\"direction_of_play\"] = ((data[\"p0_vel_x\"]+data[\"p1_vel_x\"]+ data[\"p2_vel_x\"]+ data[\"p3_vel_x\"]+ data[\"p4_vel_x\"]+ data[\"p5_vel_x\"])**2 +\n#                                 (data[\"p0_vel_y\"]+data[\"p1_vel_y\"]+ data[\"p2_vel_y\"]+ data[\"p3_vel_y\"]+ data[\"p4_vel_y\"]+ data[\"p5_vel_y\"])**2 +\n#                                 (data[\"p0_vel_z\"]+data[\"p1_vel_z\"]+ data[\"p2_vel_z\"]+ data[\"p3_vel_z\"]+ data[\"p4_vel_z\"]+ data[\"p5_vel_z\"])**2)**0.5\n    \n    for i in range(6):\n        ## ball distance to player\n        data[f\"p{i}_ball_distance\"] = ((data[\"ball_pos_x\"]-data[f\"p{i}_pos_x\"])**2 + (data[\"ball_pos_y\"]-data[f\"p{i}_pos_y\"])**2 + (data[\"ball_pos_z\"]-data[f\"p{i}_pos_z\"])**2)**0.5\n        ## boost timer\n        #data[f\"p{i}_boost_timer\"] = data[f\"p{i}_boost\"]*data[f\"boost{i}_timer\"]\n        ##Player speed  \n        #data[f\"p{i}_speed\"] = ((data[f\"p{i}_vel_x\"])**2 + (data[f\"p{i}_vel_y\"])**2 + (data[f\"p{i}_vel_z\"])**2)**0.5\n        \n        #Player direction  \n        #data[f\"p{i}_direction\"] = ((data[f\"p{i}_vel_x\"])**2 + (data[f\"p{i}_vel_y\"])**2 + (data[f\"p{i}_vel_z\"])**2)**0.5\n        \n        #if the player hit the ball what would its direction be\n        #multiplied by the inverse of the ball distance (assumption is that the further the player is from the ball the less likely they would collide with ball)\n        #data[f\"p{i}_ball_hit_direction\"] = (1/data[f\"p{i}_ball_distance\"]) * ((data[\"ball_vel_x\"] + data[f\"p{i}_vel_x\"])**2 + (data[\"ball_vel_y\"] + data[f\"p{i}_vel_y\"])**2 + (data[\"ball_vel_z\"] + data[f\"p{i}_vel_z\"])**2)**0.5\n        ","metadata":{"papermill":{"duration":0.019683,"end_time":"2022-10-19T09:07:54.987859","exception":false,"start_time":"2022-10-19T09:07:54.968176","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.391897Z","iopub.status.idle":"2022-10-20T09:01:31.392378Z","shell.execute_reply.started":"2022-10-20T09:01:31.392131Z","shell.execute_reply":"2022-10-20T09:01:31.392154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fillNan(cols, df)\nfillNan(cols, df_test)\n\n# FeatureEngineering(df)\n# FeatureEngineering(df_test)\n\ndel goal1_location,goal2_location, FeatureEngineering, fillNan","metadata":{"papermill":{"duration":66.949854,"end_time":"2022-10-19T09:09:01.943717","exception":false,"start_time":"2022-10-19T09:07:54.993863","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.394317Z","iopub.status.idle":"2022-10-20T09:01:31.394791Z","shell.execute_reply.started":"2022-10-20T09:01:31.394542Z","shell.execute_reply":"2022-10-20T09:01:31.394564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split ","metadata":{"papermill":{"duration":0.005946,"end_time":"2022-10-19T09:09:01.956246","exception":false,"start_time":"2022-10-19T09:09:01.950300","status":"completed"},"tags":[]}},{"cell_type":"code","source":"yA = df[\"team_A_scoring_within_10sec\"].astype('int8')\nyB = df[\"team_B_scoring_within_10sec\"].astype('int8')\nX = df.copy()\n\ndel X[\"team_A_scoring_within_10sec\"], X[\"team_B_scoring_within_10sec\"], df","metadata":{"papermill":{"duration":2.42695,"end_time":"2022-10-19T09:09:04.389407","exception":false,"start_time":"2022-10-19T09:09:01.962457","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.397439Z","iopub.status.idle":"2022-10-20T09:01:31.398460Z","shell.execute_reply.started":"2022-10-20T09:01:31.398212Z","shell.execute_reply":"2022-10-20T09:01:31.398236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Scaling ","metadata":{"papermill":{"duration":0.006113,"end_time":"2022-10-19T09:09:04.401876","exception":false,"start_time":"2022-10-19T09:09:04.395763","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def scaling_half(X_train, X_test,df_test):\n    \n    scaler1 = StandardScaler()\n    scaler2 = StandardScaler()\n    \n    X_train1 = scaler1.fit_transform(X_train.iloc[:,:int(X_train.shape[1]/2)])\n    X_train2 = scaler2.fit_transform(X_train.iloc[:,int(X_train.shape[1]/2):])\n    \n    X_train = pd.concat([pd.DataFrame(X_train1,columns=cols[:int(X_train.shape[1]/2)]),pd.DataFrame(X_train2,columns=cols[int(X_train.shape[1]/2):])],axis =1)\n    \n    del X_train1,X_train2\n    gc.collect()\n    \n    X_test1 = scaler1.transform(X_test.iloc[:,:int(X_test.shape[1]/2)])\n    X_test2 = scaler2.transform(X_test.iloc[:,int(X_test.shape[1]/2):])\n    \n    X_test = pd.concat([pd.DataFrame(X_test1,columns=cols[:int(X_test.shape[1]/2)]),pd.DataFrame(X_test2,columns=cols[int(X_test.shape[1]/2):])],axis =1)\n    del X_test1,X_test2\n    gc.collect()  \n    \n    df_test1 = scaler1.transform(df_test.iloc[:,:int(df_test.shape[1]/2)])\n    df_test2 = scaler2.transform(df_test.iloc[:,int(df_test.shape[1]/2):])\n    \n    df_test = pd.concat([pd.DataFrame(df_test1,columns=cols[:int(df_test.shape[1]/2)]),pd.DataFrame(df_test2,columns=cols[int(df_test.shape[1]/2):])],axis =1)\n    del df_test1,df_test2\n    gc.collect()\n    \n    return X_train,X_test,df_test","metadata":{"papermill":{"duration":0.020193,"end_time":"2022-10-19T09:09:04.428061","exception":false,"start_time":"2022-10-19T09:09:04.407868","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.399630Z","iopub.status.idle":"2022-10-20T09:01:31.400471Z","shell.execute_reply.started":"2022-10-20T09:01:31.400219Z","shell.execute_reply":"2022-10-20T09:01:31.400243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model ","metadata":{"papermill":{"duration":0.006122,"end_time":"2022-10-19T09:09:04.440264","exception":false,"start_time":"2022-10-19T09:09:04.434142","status":"completed"},"tags":[]}},{"cell_type":"code","source":"es = EarlyStopping(\n    monitor=\"val_loss\",\n    patience=EARLY_STOPPING,\n    verbose=0,\n    mode=\"auto\",\n    baseline=None,\n    restore_best_weights=True)\n\nlr = ReduceLROnPlateau(\n    monitor=\"val_loss\",\n    factor=0.1,\n    patience=5,\n    verbose=0,\n    mode=\"min\",\n    min_delta=0.0001,\n    cooldown=0,\n    min_lr=0,\n)","metadata":{"papermill":{"duration":0.014897,"end_time":"2022-10-19T09:09:04.461088","exception":false,"start_time":"2022-10-19T09:09:04.446191","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.401829Z","iopub.status.idle":"2022-10-20T09:01:31.402680Z","shell.execute_reply.started":"2022-10-20T09:01:31.402422Z","shell.execute_reply":"2022-10-20T09:01:31.402446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build model ","metadata":{"papermill":{"duration":0.005795,"end_time":"2022-10-19T09:09:04.472874","exception":false,"start_time":"2022-10-19T09:09:04.467079","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def build_model(X_in):\n    input_layer = keras.Input(shape = X_in.shape[1])\n    x1 = layers.Dense(256,activation = ACTIVATION)(input_layer)\n    x1 = layers.BatchNormalization()(x1)\n    x1 = layers.Dropout(0.1)(x1)\n\n    x1 = layers.Dense(256,activation = ACTIVATION)(x1)\n    x1 = layers.BatchNormalization()(x1)\n    x1 = layers.Dropout(0.1)(x1)\n\n    x1 = layers.Dense(128,activation = ACTIVATION)(x1)\n    x1 = layers.BatchNormalization()(x1)\n    x1 = layers.Dropout(0.1)(x1)\n\n    x1 = layers.Dense(64,activation = ACTIVATION)(x1)\n    x1 = layers.BatchNormalization()(x1)\n    x1 = layers.Dropout(0.1)(x1)\n\n    output_A = layers.Dense(1,activation = \"sigmoid\", name = \"outputA\")(x1)\n    output_B = layers.Dense(1,activation = \"sigmoid\", name = \"outputB\")(x1)\n\n    model = Model(inputs = input_layer, outputs =  [output_A,output_B])\n    return model \n\nmodel = build_model(X)\nplot_model(model)","metadata":{"papermill":{"duration":1.112597,"end_time":"2022-10-19T09:09:05.591529","exception":false,"start_time":"2022-10-19T09:09:04.478932","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.404016Z","iopub.status.idle":"2022-10-20T09:01:31.405017Z","shell.execute_reply.started":"2022-10-20T09:01:31.404702Z","shell.execute_reply":"2022-10-20T09:01:31.404734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model","metadata":{"papermill":{"duration":0.016884,"end_time":"2022-10-19T09:09:05.615790","exception":false,"start_time":"2022-10-19T09:09:05.598906","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.406438Z","iopub.status.idle":"2022-10-20T09:01:31.407413Z","shell.execute_reply.started":"2022-10-20T09:01:31.407163Z","shell.execute_reply":"2022-10-20T09:01:31.407186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = X.columns","metadata":{"papermill":{"duration":0.014687,"end_time":"2022-10-19T09:09:05.637547","exception":false,"start_time":"2022-10-19T09:09:05.622860","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.409208Z","iopub.status.idle":"2022-10-20T09:01:31.409674Z","shell.execute_reply.started":"2022-10-20T09:01:31.409433Z","shell.execute_reply":"2022-10-20T09:01:31.409455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# One Model ","metadata":{"papermill":{"duration":0.006521,"end_time":"2022-10-19T09:09:05.651025","exception":false,"start_time":"2022-10-19T09:09:05.644504","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if not CROSS_VAL:\n    X_train, X_test, yA_train, yA_test = train_test_split(X, yA, test_size=0.33, shuffle = True, random_state=42)\n    yB_train = yB.iloc[yA_train.index]\n    yB_test = yB.iloc[yA_test.index]\n   \n    print(\"Scaling\")\n    X_train, X_test,test_s = scaling_half(X_train, X_test,df_test)\n    \n    with strategy.scope():   \n        model = build_model(X)\n        model.compile(optimizer= \"adam\", loss = \"binary_crossentropy\" )\n\n        print(\"Predicting\")\n        model.fit(x = X_train,\n                  y = {\"outputA\": yA_train, \"outputB\": yB_train},\n                  epochs =EPOCHS, \n                  batch_size = BATCH_SIZE, \n                  validation_data = (X_test,yA_test), \n                  callbacks = [es,lr] )\n        history = model.history.history\n\n        A_preds, B_preds = model.predict(test_s)","metadata":{"papermill":{"duration":0.018542,"end_time":"2022-10-19T09:09:05.676433","exception":false,"start_time":"2022-10-19T09:09:05.657891","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.411642Z","iopub.status.idle":"2022-10-20T09:01:31.412209Z","shell.execute_reply.started":"2022-10-20T09:01:31.411916Z","shell.execute_reply":"2022-10-20T09:01:31.411942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* 256-256-128-64 + scaling + relu + feats   + drop :::loss = 0.3810 ,val_loss: 0.3969\n* **256-256-128-64 + scaling + relu -NO feats + drop:::: loss= 0.3826, val_loss= 0.3978**\n* 256-256-128-64 + scaling + MISH+ feats ::: **loss: 0.3785**, **val_loss: 0.3968**\n* 256-256-128-64 + scaling + relu + feats - no drop::: **loss: 0.3459** ,val_loss: 0.4114\n* 256-256-128-64 + scaling + SELU +feats + drop :::loss: 0.3832, val_loss: 0.3975\n* 256-128-64 + scaling + MISH+feats :::loss: 0.3914, val_loss: 0.4039\n* 64-32-16+relu -NO feats+drop::: loss: 0.4073, val_loss: 0.4126\n* 64-64-32-16+relu -NO feats+drop::  loss: 0.4066, val_loss: 0.4116\n* 128-64-32+relu -NO feats+drop :::: loss: 0.4034, val_loss: 0.4098 ","metadata":{"papermill":{"duration":0.006802,"end_time":"2022-10-19T09:09:05.690619","exception":false,"start_time":"2022-10-19T09:09:05.683817","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# CV","metadata":{"papermill":{"duration":0.006623,"end_time":"2022-10-19T09:09:05.704179","exception":false,"start_time":"2022-10-19T09:09:05.697556","status":"completed"},"tags":[]}},{"cell_type":"code","source":"cv = KFold(n_splits=SPLITS,  shuffle=True , random_state=42\n          )","metadata":{"papermill":{"duration":0.014603,"end_time":"2022-10-19T09:09:05.725482","exception":false,"start_time":"2022-10-19T09:09:05.710879","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.414417Z","iopub.status.idle":"2022-10-20T09:01:31.414910Z","shell.execute_reply.started":"2022-10-20T09:01:31.414647Z","shell.execute_reply":"2022-10-20T09:01:31.414669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"options = tf.data.Options()\noptions.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.OFF","metadata":{"papermill":{"duration":0.014661,"end_time":"2022-10-19T09:09:05.746693","exception":false,"start_time":"2022-10-19T09:09:05.732032","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.416950Z","iopub.status.idle":"2022-10-20T09:01:31.417426Z","shell.execute_reply.started":"2022-10-20T09:01:31.417182Z","shell.execute_reply":"2022-10-20T09:01:31.417204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model (X, yA,yB, df_test):\n    oof_A = []\n    oof_B = []\n    score = []\n\n    for fold, (train_idx, val_idx) in enumerate(cv.split(X,yA)):\n\n        print(f\"\\n###### Fold {fold} ######\")\n\n        X_train, X_test = X.iloc[train_idx], X.iloc[val_idx]\n        yA_train, yA_test = yA.iloc[train_idx], yA.iloc[val_idx]\n        yB_train, yB_test = yB.iloc[train_idx], yB.iloc[val_idx]  \n\n        #scaling\n        if SCALING:\n            print(\"Scaling\")\n            X_train, X_test,test_s = scaling_half(X_train, X_test,df_test)\n        else:\n            test_s = df_test.copy()\n        \n        #build , compile & fit\n        print(\"predicting\")\n        with strategy.scope():   \n            model = build_model(X)\n            \n        # Wrap data in Dataset objects.\n        train_data = tf.data.Dataset.from_tensor_slices((X_train,{\"outputA\": yA_train, \"outputB\": yB_train}))\n        val_data = tf.data.Dataset.from_tensor_slices((X_test, yA_test))\n        test_data = tf.data.Dataset.from_tensor_slices((test_s))\n        # The batch size must now be set on the Dataset objects.\n        train_data = train_data.batch(BATCH_SIZE)\n        val_data = val_data.batch(BATCH_SIZE)\n        test_data = test_data.batch(BATCH_SIZE)\n        # Disable AutoShard.\n        options = tf.data.Options()\n        options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.OFF\n        train_data = train_data.with_options(options)\n        val_data = val_data.with_options(options)\n        test_data = test_data.with_options(options)\n        \n        model.compile(optimizer= \"adam\", loss = \"binary_crossentropy\" )\n        model.fit(train_data,\n                  validation_data=val_data ,\n                  epochs =EPOCHS, \n                  #batch_size = BATCH_SIZE, \n                  #validation_data = (X_test,yA_test), \n                  callbacks = [es,lr] )\n\n            \n#         model.compile(optimizer= \"adam\", loss = \"binary_crossentropy\" )\n#         model.fit(x = X_train,\n#                   y = {\"outputA\": yA_train, \"outputB\": yB_train},\n#                   epochs =EPOCHS, \n#                   batch_size = BATCH_SIZE, \n#                   #validation_data = (X_test,yA_test), \n#                   callbacks = [es,lr] )\n        history = model.history.history \n\n        del X_train,X_test,yA_train, yA_test, yB_train, yB_test\n        gc.collect()\n\n        #predict\n        print(\"Predicting\")\n        Apreds, Bpreds = model.predict(test_data)\n        #Apreds, Bpreds = model.predict(test_s)\n\n        oof_A.append(Apreds)\n        oof_B.append(Bpreds)\n        score.append(min(history[\"loss\"]))\n\n        del test_s , model\n        gc.collect()\n\n    print(f\"\\n Mean logloss: {np.mean(score)}\")\n    \n    return score, oof_A,oof_B, history\n\n# Run \nif CROSS_VAL:\n    scoreA, A_preds,B_preds,history = train_model(X, yA,yB, df_test)","metadata":{"papermill":{"duration":5080.400971,"end_time":"2022-10-19T10:33:46.154333","exception":false,"start_time":"2022-10-19T09:09:05.753362","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.419415Z","iopub.status.idle":"2022-10-20T09:01:31.419918Z","shell.execute_reply.started":"2022-10-20T09:01:31.419646Z","shell.execute_reply":"2022-10-20T09:01:31.419671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize and submit","metadata":{"papermill":{"duration":3.417633,"end_time":"2022-10-19T10:33:52.684350","exception":false,"start_time":"2022-10-19T10:33:49.266717","status":"completed"},"tags":[]}},{"cell_type":"code","source":"fig, ax = plt.subplots(1,2, figsize = (20,7))\nax[0].plot(history[\"loss\"], marker = 'o', label = \"Total\", c= \"g\")\nax[1].plot(history[\"outputA_loss\"], marker = 'o',label=\"Team A\")\nax[1].plot(history[\"outputB_loss\"], marker = 'o',label=\"Team B\")\nfig.legend()\nfig.suptitle(\"Loss of each Target and Total loss (last model interation)\")\nplt.show()","metadata":{"papermill":{"duration":3.481393,"end_time":"2022-10-19T10:33:59.339326","exception":false,"start_time":"2022-10-19T10:33:55.857933","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.421509Z","iopub.status.idle":"2022-10-20T09:01:31.422327Z","shell.execute_reply.started":"2022-10-20T09:01:31.422049Z","shell.execute_reply":"2022-10-20T09:01:31.422073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(\"../input/tabular-playground-series-oct-2022/sample_submission.csv\")\n\nif CROSS_VAL:\n    sub[\"team_A_scoring_within_10sec\"] = np.mean(A_preds,axis=0)\n    sub[\"team_B_scoring_within_10sec\"] = np.mean(B_preds,axis=0)\nelse: \n    sub[\"team_A_scoring_within_10sec\"] = A_preds\n    sub[\"team_B_scoring_within_10sec\"] = B_preds\n\n\nsub.to_csv(\"submission.csv\",index=False)\nsub","metadata":{"papermill":{"duration":5.28965,"end_time":"2022-10-19T10:34:07.739788","exception":false,"start_time":"2022-10-19T10:34:02.450138","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.423749Z","iopub.status.idle":"2022-10-20T09:01:31.424541Z","shell.execute_reply.started":"2022-10-20T09:01:31.424286Z","shell.execute_reply":"2022-10-20T09:01:31.424310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub[[\"team_A_scoring_within_10sec\",\"team_B_scoring_within_10sec\"]].hist(figsize = (20,5))\nplt.show()","metadata":{"papermill":{"duration":4.213274,"end_time":"2022-10-19T10:34:15.069259","exception":false,"start_time":"2022-10-19T10:34:10.855985","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-10-20T09:01:31.425938Z","iopub.status.idle":"2022-10-20T09:01:31.426719Z","shell.execute_reply.started":"2022-10-20T09:01:31.426462Z","shell.execute_reply":"2022-10-20T09:01:31.426487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":3.065013,"end_time":"2022-10-19T10:34:21.313387","exception":false,"start_time":"2022-10-19T10:34:18.248374","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}