{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# PERSONAL SUBMISSION\nThe data was create in a previoous notebook: https://www.kaggle.com/code/josmejagamarra/tps-oct-22-personal-datasets\n\nLoad the tfrecords from: https://www.kaggle.com/datasets/josmejagamarra/tps-oct-2022-personal-ds-2","metadata":{}},{"cell_type":"code","source":"#Import libraries\nimport pandas as pd\nimport numpy as np\nimport gc\nimport tensorflow as tf\nfrom tensorflow.data import TFRecordDataset\nimport os\nfrom tensorflow.keras import Model","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-31T12:46:01.344895Z","iopub.execute_input":"2022-10-31T12:46:01.345639Z","iopub.status.idle":"2022-10-31T12:46:07.117954Z","shell.execute_reply.started":"2022-10-31T12:46:01.345515Z","shell.execute_reply":"2022-10-31T12:46:07.117320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. CONFIGURE THE TPU","metadata":{}},{"cell_type":"code","source":"#Get google cloud path of the dataset while on CPU \nfrom kaggle_datasets import KaggleDatasets\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(\"tps-oct-2022-personal-ds-2\")\nGCS_DS_PATH","metadata":{"execution":{"iopub.status.busy":"2022-10-31T12:46:07.119294Z","iopub.execute_input":"2022-10-31T12:46:07.119646Z","iopub.status.idle":"2022-10-31T12:46:07.408377Z","shell.execute_reply.started":"2022-10-31T12:46:07.119610Z","shell.execute_reply":"2022-10-31T12:46:07.407464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Configure the TPU\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    BATCH_SIZE = 4096 * strategy.num_replicas_in_sync\n    print(\"TPU\")\nexcept:\n    tpu = None\n    strategy = tf.distribute.get_strategy()\n    BATCH_SIZE=512\n    print(\"CPU\")","metadata":{"execution":{"iopub.status.busy":"2022-10-31T12:46:07.409501Z","iopub.execute_input":"2022-10-31T12:46:07.409732Z","iopub.status.idle":"2022-10-31T12:46:13.544343Z","shell.execute_reply.started":"2022-10-31T12:46:07.409701Z","shell.execute_reply":"2022-10-31T12:46:13.543476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. LOAD THE INPUT 1 / INPUT 2 / TARGET DATA","metadata":{}},{"cell_type":"code","source":"# Used to autotune tensorflow dataset transformations\nAUTO = tf.data.experimental.AUTOTUNE\n\n# Get 10 datasets independently and store all in a list for later use\nds_input_1=[]  #For the input 1\nds_input_2=[]  #For the input 2\n\n# Option for faster TPU data read\nignore_order = tf.data.Options()\nignore_order.experimental_deterministic = False\n\nwith strategy.scope():\n    # LOAD INPUT 1\n    for i in range(10):\n        PATH=tf.io.gfile.glob(os.path.join(GCS_DS_PATH,f'Combining/Team_A_input/train_{i}/feats.tfrecord*'))\n        ds = TFRecordDataset(PATH, num_parallel_reads=AUTO)\n        ds = ds.with_options(ignore_order)\n        dataset = ds.map(lambda x: tf.ensure_shape(tf.io.parse_tensor(x, out_type=tf.float16),(64)), num_parallel_calls=AUTO)\n        ds_input_1.append(dataset)\n\n    Input_1 = ds_input_1[0]\n    for i in np.arange(1,10,1):\n        Input_1 = Input_1.concatenate(ds_input_1[i])\n    \n    # LOAD INPUT 2\n    for i in range(10):\n        PATH=tf.io.gfile.glob(os.path.join(GCS_DS_PATH,f'Combining/Team_B_input/train_{i}/feats.tfrecord*'))\n        ds = TFRecordDataset(PATH, num_parallel_reads=AUTO)\n        ds = ds.with_options(ignore_order)\n        dataset = ds.map(lambda x: tf.ensure_shape(tf.io.parse_tensor(x, out_type=tf.float16),(64)), num_parallel_calls=AUTO)\n        ds_input_2.append(dataset)\n\n    Input_2 = ds_input_2[0]\n    for i in np.arange(1,10,1):\n        Input_2 = Input_2.concatenate(ds_input_2[i])\n        \n    #LOAD TARGET\n    PATH=tf.io.gfile.glob(os.path.join(GCS_DS_PATH,f'Combining/Target/target.tfrecord'))\n    ds = TFRecordDataset(PATH, num_parallel_reads=AUTO)\n    ds = ds.map(lambda x: tf.ensure_shape(tf.io.parse_tensor(x, tf.float16), (2)), num_parallel_calls=AUTO)\n    target = ds.map(lambda x: ([x[-2]],[x[-1]]) )","metadata":{"execution":{"iopub.status.busy":"2022-10-31T12:46:15.568689Z","iopub.execute_input":"2022-10-31T12:46:15.568975Z","iopub.status.idle":"2022-10-31T12:46:16.848336Z","shell.execute_reply.started":"2022-10-31T12:46:15.568947Z","shell.execute_reply":"2022-10-31T12:46:16.846574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Zip the data from Input_1 / Input_2 / target\ninputs = tf.data.Dataset.zip((Input_1, Input_2))\nds_all = tf.data.Dataset.zip((inputs, target))","metadata":{"execution":{"iopub.status.busy":"2022-10-31T12:46:20.575614Z","iopub.execute_input":"2022-10-31T12:46:20.575881Z","iopub.status.idle":"2022-10-31T12:46:20.583259Z","shell.execute_reply.started":"2022-10-31T12:46:20.575849Z","shell.execute_reply":"2022-10-31T12:46:20.581879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. DEFINE THE TRAINING / VALIDATION SET","metadata":{}},{"cell_type":"code","source":"def get_dataset_partitions_tf(ds, ds_size, train_split=0.9, shuffle_size=1000, batch_s=BATCH_SIZE):\n    \n    # Shuffle the dataset\n    ds=ds.shuffle(shuffle_size, seed=2022)\n    # Split the data (train / valid)\n    train_size = int(train_split * ds_size)\n    train_ds = ds.take(train_size)    \n    val_ds = ds.skip(train_size)\n    # Generate the batches\n    train_ds = train_ds.batch(batch_s).cache().prefetch(tf.data.AUTOTUNE)\n    val_ds = val_ds.batch(batch_s).cache().prefetch(tf.data.AUTOTUNE)\n    \n    return train_ds, val_ds","metadata":{"execution":{"iopub.status.busy":"2022-10-31T12:46:22.573313Z","iopub.execute_input":"2022-10-31T12:46:22.574244Z","iopub.status.idle":"2022-10-31T12:46:22.580854Z","shell.execute_reply.started":"2022-10-31T12:46:22.574195Z","shell.execute_reply":"2022-10-31T12:46:22.579786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Define the number of rowns\nds_size=21198036\n#Generate the train and valid dataset\ntrain_dataset, valid_dataset = get_dataset_partitions_tf(ds_all, ds_size=ds_size, batch_s=BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2022-10-31T12:46:24.602892Z","iopub.execute_input":"2022-10-31T12:46:24.603781Z","iopub.status.idle":"2022-10-31T12:46:24.619074Z","shell.execute_reply.started":"2022-10-31T12:46:24.603733Z","shell.execute_reply":"2022-10-31T12:46:24.618119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. DEFINE PERSONAL CALLBACK","metadata":{}},{"cell_type":"code","source":"def Personal_callback(metrics, threshold_metric, ep, lr_i=0.001):\n    \"\"\"Generate a list of personal Callbacks to use in the training process\n    Args:\n        model_name (string)        - Contain the name of the model\n        metrics (string)           - Contains the metric to be evaluated\n        threshold_metric (string)  - Specify the threshold for the metric\n        ep (int)                   - The number of epoch in the training process\n        lr_i                       - Specify the initial learning rate\n        \n    Returns:\n        A list of callback functions to use in the training process\n    \"\"\"\n    \n    \"\"\"\n    Stop training\n    \"\"\"\n    class stop_training(tf.keras.callbacks.Callback):                                               #Define the class\n        def on_epoch_end(self, epoch, logs = {}):                                                   #Use in the end of the epoch\n            if(logs.get(metrics)<threshold_metric and logs.get('val_'+metrics) <threshold_metric):  #Define threshold for metrics\n                print(\"\\Cancelling training!\")\n                self.model.stop_training = True                                                     #Stop the training process\n    stop_train = stop_training()\n    \n    \"\"\"\n    Learning Rate Decay\n    \"\"\"\n    global LR_init        #Define global variable\n    LR_init=lr_i          #Specify the initial learning rate\n    \n    class learning_decay(tf.keras.callbacks.Callback):                        #Define the class\n        def on_epoch_end(self, batch, logs={}):                               #Use in the end of the epoch\n            lr = self.model.optimizer.lr                                      #Call the leraning rate from the model\n            global LR_init                                                    #Define global variable\n            new_lr = (LR_init) * 10.**(-(batch+1.)/(ep*10))                   #Define the learning rate decay function\n            if lr > new_lr:                                                   #If the previous lr is greater than actual lr \n                tf.keras.backend.set_value(self.model.optimizer.lr, new_lr)   #Update the value of the learning rate\n            else: \n                LR_init=lr                                                    #This used because we use the callback 'ReduceLROnPlateau'\n\n    lr_decay = learning_decay()\n    \n    \"\"\"\n    Reduce Learning Rate\n    \"\"\"\n    # Creating learning rate reduction callback\n    reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor= 'val_'+metrics,   #Specify the metrics\n                                                     factor=0.2,         # new_lr = lr * factor\n                                                     patience=2,         # number of epochs with no improvement after which learning rate will be reduced\n                                                     verbose=1,          # print out when learning rate goes down \n                                                     min_lr=1e-15)       # lower bound on the learning rate\n    \"\"\"\n    Early Stopping\n    \"\"\"\n    # Setup EarlyStopping callback to stop training if model's val_loss doesn't improve for 10 epochs\n    early_stopping = tf.keras.callbacks.EarlyStopping(monitor = metrics,         # watch the val loss metric\n                                                  patience = 5,                 # if val loss decreases for 10 epochs in a row, stop training\n                                                  min_delta = 0.00001,           # Minimum change in the monitored quantity to qualify as an improvement\n                                                  restore_best_weights = False,  #Don't sabe the best weights, because we use the callback \"ModelCheckpoint\"\n                                                     verbose=1)                  #Display a message\n    \n    return [stop_train, lr_decay, reduce_lr, early_stopping]   #Return a list of Callbacks","metadata":{"execution":{"iopub.status.busy":"2022-10-31T14:35:31.361454Z","iopub.execute_input":"2022-10-31T14:35:31.361975Z","iopub.status.idle":"2022-10-31T14:35:31.373320Z","shell.execute_reply.started":"2022-10-31T14:35:31.361927Z","shell.execute_reply":"2022-10-31T14:35:31.372557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. BUILD THE MODEL","metadata":{}},{"cell_type":"code","source":"#BUILD THE MODEL\ndef create_model():\n\n    ############ 1. Setup input_1 ############\n    input_1=tf.keras.layers.Input(shape=[64])\n    in_1=tf.keras.layers.BatchNormalization()(input_1)\n    in_1=tf.keras.layers.GaussianNoise(0.001)(in_1)\n    in_1=tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=1))(in_1)\n    \n    in_1=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(512, return_sequences=True))(in_1)\n    in_1=tf.keras.layers.BatchNormalization()(in_1)\n    in_1=tf.keras.layers.Dropout(0.5)(in_1)\n    in_1=tf.keras.layers.Dense(units=1024, activation=\"tanh\")(in_1)\n    \n    in_1=tf.keras.layers.BatchNormalization()(in_1)\n    in_1=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(256, return_sequences=True))(in_1)\n    in_1=tf.keras.layers.BatchNormalization()(in_1)\n    in_1=tf.keras.layers.Dropout(0.5)(in_1)\n    in_1=tf.keras.layers.Dense(units=512, activation=\"tanh\")(in_1)\n    \n    in_1=tf.keras.layers.BatchNormalization()(in_1)\n    in_1=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(128, return_sequences=True))(in_1)\n    in_1=tf.keras.layers.BatchNormalization()(in_1)\n    in_1=tf.keras.layers.Dropout(0.5)(in_1)\n    in_1=tf.keras.layers.Dense(units=256, activation=\"tanh\")(in_1)\n    \n    output_1=tf.keras.layers.BatchNormalization()(in_1)\n    model_1=tf.keras.Model(inputs=input_1,outputs=output_1)\n    \n    ############ 2. Setup input_2 ############\n    input_2=tf.keras.layers.Input(shape=[64])\n    in_2=tf.keras.layers.BatchNormalization()(input_2)\n    in_2=tf.keras.layers.GaussianNoise(0.001)(in_2)\n    in_2=tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=1))(in_2)\n    \n    in_2=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(512, return_sequences=True))(in_2)\n    in_2=tf.keras.layers.BatchNormalization()(in_2)\n    in_2=tf.keras.layers.Dropout(0.5)(in_2)\n    in_2=tf.keras.layers.Dense(units=1024, activation=\"tanh\")(in_2)\n    \n    in_2=tf.keras.layers.BatchNormalization()(in_2)\n    in_2=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(256, return_sequences=True))(in_2)\n    in_2=tf.keras.layers.BatchNormalization()(in_2)\n    in_2=tf.keras.layers.Dropout(0.5)(in_2)\n    in_2=tf.keras.layers.Dense(units=512, activation=\"tanh\")(in_2)\n    \n    in_2=tf.keras.layers.BatchNormalization()(in_2)\n    in_2=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(128, return_sequences=True))(in_2)\n    in_2=tf.keras.layers.BatchNormalization()(in_2)\n    in_2=tf.keras.layers.Dropout(0.5)(in_2)\n    in_2=tf.keras.layers.Dense(units=256, activation=\"tanh\")(in_2)\n    \n    output_2=tf.keras.layers.BatchNormalization()(in_2)\n    model_2=tf.keras.Model(inputs=input_2,outputs=output_2)\n\n    ############ 3. Concatenate ############\n    concat = tf.keras.layers.Concatenate()([model_1.output, \n                                            model_2.output])\n    \n    ############ 4. Setup the rest ############\n    x=tf.keras.layers.BatchNormalization()(concat)\n    x=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(256, return_sequences=True))(x)\n    x=tf.keras.layers.BatchNormalization()(x)\n    x=tf.keras.layers.Dropout(0.5)(x)\n    x=tf.keras.layers.Dense(units=512, activation=\"tanh\")(x)\n    \n    x=tf.keras.layers.BatchNormalization()(x)\n    x=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(128, return_sequences=True))(x)\n    x=tf.keras.layers.BatchNormalization()(x)\n    x=tf.keras.layers.Dropout(0.5)(x)\n    x=tf.keras.layers.Dense(units=256, activation=\"tanh\")(x)\n    \n    x=tf.keras.layers.BatchNormalization()(x)\n    x=tf.keras.layers.Dropout(0.5)(x)\n    x=tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True))(x)\n    x=tf.keras.layers.BatchNormalization()(x)\n    x=tf.keras.layers.Dense(units=128, activation=\"tanh\")(x)\n\n    x=tf.keras.layers.BatchNormalization()(x)\n    x=tf.keras.layers.Dropout(0.5)(x)\n\n    out1=tf.keras.layers.Dense(1,activation=\"sigmoid\",name=\"teamA\")(x)\n    out2=tf.keras.layers.Dense(1,activation=\"sigmoid\",name=\"teamB\")(x)\n\n    model=tf.keras.Model(inputs=[model_1.input,model_2.input],\n                         outputs=[out1,out2])\n\n    ############ COMPILE THE MODEL ############\n    model.compile(loss=[tf.keras.losses.BinaryCrossentropy(from_logits=False),\n                        tf.keras.losses.BinaryCrossentropy(from_logits=False)],                      \n                  optimizer='adam',  \n                  metrics=['accuracy'])                 \n\n    #SAVE THE INITIAL WEIGHTS (Use when try with different models)\n    InitialW = model.get_weights()\n    \n    return model, InitialW","metadata":{"execution":{"iopub.status.busy":"2022-10-31T15:34:58.543953Z","iopub.execute_input":"2022-10-31T15:34:58.544706Z","iopub.status.idle":"2022-10-31T15:34:58.583568Z","shell.execute_reply.started":"2022-10-31T15:34:58.544671Z","shell.execute_reply":"2022-10-31T15:34:58.582929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"############ MODEL 1 ############\nwith strategy.scope():\n    gc.collect()\n    \n    #Create the model\n    model_1,InitialW_1 = create_model()\n    #Load the initial weights\n    model_1.set_weights(InitialW_1)\n    #Reset the value of learning rate in the model\n    tf.keras.backend.set_value(model_1.optimizer.lr, 0.001)\n\n    #Fit the model\n    history = model_1.fit(train_dataset,\n                        epochs=15,\n                        validation_data=valid_dataset,\n                        callbacks=Personal_callback(metrics='loss', \n                                                    threshold_metric=0.01,\n                                                    ep=100))\n\n    #Save the model weights (Use when you train again)\n    FIT_1_1=model_1.get_weights()","metadata":{"execution":{"iopub.status.busy":"2022-10-31T12:47:03.339626Z","iopub.execute_input":"2022-10-31T12:47:03.339906Z","iopub.status.idle":"2022-10-31T13:34:02.072771Z","shell.execute_reply.started":"2022-10-31T12:47:03.339878Z","shell.execute_reply":"2022-10-31T13:34:02.071684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. FIT THE MODELS\nIn this case we used 5 models, after the training, predict the data with the mean of all models prediction.","metadata":{}},{"cell_type":"code","source":"############ MODEL 1 ############\nwith strategy.scope():\n    gc.collect()\n    \n    #Create the model\n    model_1,InitialW_1 = create_model()\n    #Load the initial weights\n    model_1.set_weights(FIT_1_2)\n    #Reset the value of learning rate in the model\n    tf.keras.backend.set_value(model_1.optimizer.lr, 0.001)\n\n    #Fit the model\n    history = model_1.fit(train_dataset,\n                        epochs=25,\n                        validation_data=valid_dataset,\n                        callbacks=Personal_callback(metrics='loss', \n                                                    threshold_metric=0.01,\n                                                    ep=100))\n\n    #Save the model weights (Use when you train again)\n    FIT_2_1=model_1.get_weights()","metadata":{"execution":{"iopub.status.busy":"2022-10-31T15:35:47.250666Z","iopub.execute_input":"2022-10-31T15:35:47.250941Z","iopub.status.idle":"2022-10-31T15:50:35.411212Z","shell.execute_reply.started":"2022-10-31T15:35:47.250912Z","shell.execute_reply":"2022-10-31T15:50:35.410352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"############ MODEL 2 ############\nwith strategy.scope():\n    gc.collect()\n\n    #Create the model\n    model_2,InitialW_2 = create_model()\n    #Load the initial weights\n    model_2.set_weights(InitialW_2)\n    #Reset the value of learning rate in the model\n    tf.keras.backend.set_value(model_2.optimizer.lr, 0.001)\n\n    #Fit the model\n    history = model_2.fit(train_dataset,\n                        epochs=15,\n                        validation_data=valid_dataset,\n                        callbacks=Personal_callback(metrics='loss', \n                                                    threshold_metric=0.01,\n                                                    ep=100))\n\n    #Save the model weights (Use when you train again)\n    FIT_1_2=model_2.get_weights()","metadata":{"execution":{"iopub.status.busy":"2022-10-31T13:35:41.710073Z","iopub.execute_input":"2022-10-31T13:35:41.710352Z","iopub.status.idle":"2022-10-31T13:45:13.982820Z","shell.execute_reply.started":"2022-10-31T13:35:41.710321Z","shell.execute_reply":"2022-10-31T13:45:13.980841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"############ MODEL 2 ############\nwith strategy.scope():\n    gc.collect()\n\n    #Create the model\n    model_2,InitialW_2 = create_model()\n    #Load the initial weights\n    model_2.set_weights(FIT_1_2)\n    #Reset the value of learning rate in the model\n    tf.keras.backend.set_value(model_2.optimizer.lr, 0.001)\n\n    #Fit the model\n    history = model_2.fit(train_dataset,\n                        epochs=25,\n                        validation_data=valid_dataset,\n                        callbacks=Personal_callback(metrics='loss', \n                                                    threshold_metric=0.01,\n                                                    ep=100))\n\n    #Save the model weights (Use when you train again)\n    FIT_2_2=model_2.get_weights()","metadata":{"execution":{"iopub.status.busy":"2022-10-31T15:50:35.413336Z","iopub.execute_input":"2022-10-31T15:50:35.413831Z","iopub.status.idle":"2022-10-31T16:05:59.407125Z","shell.execute_reply.started":"2022-10-31T15:50:35.413797Z","shell.execute_reply":"2022-10-31T16:05:59.405831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"############ MODEL 3 ############\nwith strategy.scope():\n    gc.collect()\n\n    #Create the model\n    model_3,InitialW_3 = create_model()\n    #Load the initial weights\n    model_3.set_weights(InitialW_3)\n    #Reset the value of learning rate in the model\n    tf.keras.backend.set_value(model_3.optimizer.lr, 0.001)\n\n    #Fit the model\n    history = model_3.fit(train_dataset,\n                        epochs=15,\n                        validation_data=valid_dataset,\n                        callbacks=Personal_callback(metrics='loss', \n                                                    threshold_metric=0.01,\n                                                    ep=100))\n\n    #Save the model weights (Use when you train again)\n    FIT_1_3=model_3.get_weights()","metadata":{"execution":{"iopub.status.busy":"2022-10-31T13:45:13.984877Z","iopub.execute_input":"2022-10-31T13:45:13.985170Z","iopub.status.idle":"2022-10-31T13:54:50.474794Z","shell.execute_reply.started":"2022-10-31T13:45:13.985138Z","shell.execute_reply":"2022-10-31T13:54:50.473446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"############ MODEL 3 ############\nwith strategy.scope():\n    gc.collect()\n\n    #Create the model\n    model_3,InitialW_3 = create_model()\n    #Load the initial weights\n    model_3.set_weights(FIT_1_3)\n    #Reset the value of learning rate in the model\n    tf.keras.backend.set_value(model_3.optimizer.lr, 0.001)\n\n    #Fit the model\n    history = model_3.fit(train_dataset,\n                        epochs=25,\n                        validation_data=valid_dataset,\n                        callbacks=Personal_callback(metrics='loss', \n                                                    threshold_metric=0.01,\n                                                    ep=100))\n\n    #Save the model weights (Use when you train again)\n    FIT_2_3=model_3.get_weights()","metadata":{"execution":{"iopub.status.busy":"2022-10-31T16:05:59.409008Z","iopub.execute_input":"2022-10-31T16:05:59.409298Z","iopub.status.idle":"2022-10-31T16:19:40.584199Z","shell.execute_reply.started":"2022-10-31T16:05:59.409262Z","shell.execute_reply":"2022-10-31T16:19:40.583317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"############ MODEL 4 ############\nwith strategy.scope():\n    gc.collect()\n\n    #Create the model\n    model_4,InitialW_4 = create_model()\n    #Load the initial weights\n    model_4.set_weights(InitialW_4)\n    #Reset the value of learning rate in the model\n    tf.keras.backend.set_value(model_4.optimizer.lr, 0.001)\n\n    #Fit the model\n    history = model_4.fit(train_dataset,\n                        epochs=15,\n                        validation_data=valid_dataset,\n                        callbacks=Personal_callback(metrics='loss', \n                                                    threshold_metric=0.01,\n                                                    ep=100))\n\n    #Save the model weights (Use when you train again)\n    FIT_1_4=model_4.get_weights()","metadata":{"execution":{"iopub.status.busy":"2022-10-31T13:54:50.476353Z","iopub.execute_input":"2022-10-31T13:54:50.476679Z","iopub.status.idle":"2022-10-31T14:04:19.895600Z","shell.execute_reply.started":"2022-10-31T13:54:50.476648Z","shell.execute_reply":"2022-10-31T14:04:19.894490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"############ MODEL 4 ############\nwith strategy.scope():\n    gc.collect()\n\n    #Create the model\n    model_4,InitialW_4 = create_model()\n    #Load the initial weights\n    model_4.set_weights(FIT_1_4)\n    #Reset the value of learning rate in the model\n    tf.keras.backend.set_value(model_4.optimizer.lr, 0.001)\n\n    #Fit the model\n    history = model_4.fit(train_dataset,\n                        epochs=25,\n                        validation_data=valid_dataset,\n                        callbacks=Personal_callback(metrics='loss', \n                                                    threshold_metric=0.01,\n                                                    ep=100))\n\n    #Save the model weights (Use when you train again)\n    FIT_2_4=model_4.get_weights()","metadata":{"execution":{"iopub.status.busy":"2022-10-31T16:19:40.585771Z","iopub.execute_input":"2022-10-31T16:19:40.586068Z","iopub.status.idle":"2022-10-31T16:35:24.459627Z","shell.execute_reply.started":"2022-10-31T16:19:40.586035Z","shell.execute_reply":"2022-10-31T16:35:24.458651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"############ MODEL 5 ############\nwith strategy.scope():\n    gc.collect()\n\n    #Create the model\n    model_5,InitialW_5 = create_model()\n    #Load the initial weights\n    model_5.set_weights(InitialW_5)\n    #Reset the value of learning rate in the model\n    tf.keras.backend.set_value(model_5.optimizer.lr, 0.001)\n\n    #Fit the model\n    history = model_5.fit(train_dataset,\n                        epochs=15,\n                        validation_data=valid_dataset,\n                        callbacks=Personal_callback(metrics='loss', \n                                                    threshold_metric=0.01,\n                                                    ep=100))\n\n    #Save the model weights (Use when you train again)\n    FIT_1_5=model_5.get_weights()","metadata":{"execution":{"iopub.status.busy":"2022-10-31T14:04:19.897509Z","iopub.execute_input":"2022-10-31T14:04:19.897759Z","iopub.status.idle":"2022-10-31T14:13:55.806821Z","shell.execute_reply.started":"2022-10-31T14:04:19.897734Z","shell.execute_reply":"2022-10-31T14:13:55.805652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"############ MODEL 5 ############\nwith strategy.scope():\n    gc.collect()\n\n    #Create the model\n    model_5,InitialW_5 = create_model()\n    #Load the initial weights\n    model_5.set_weights(FIT_1_5)\n    #Reset the value of learning rate in the model\n    tf.keras.backend.set_value(model_5.optimizer.lr, 0.001)\n\n    #Fit the model\n    history = model_5.fit(train_dataset,\n                        epochs=25,\n                        validation_data=valid_dataset,\n                        callbacks=Personal_callback(metrics='loss', \n                                                    threshold_metric=0.01,\n                                                    ep=100))\n\n    #Save the model weights (Use when you train again)\n    FIT_2_5=model_5.get_weights()","metadata":{"execution":{"iopub.status.busy":"2022-10-31T16:35:24.460994Z","iopub.execute_input":"2022-10-31T16:35:24.461221Z","iopub.status.idle":"2022-10-31T16:50:55.170451Z","shell.execute_reply.started":"2022-10-31T16:35:24.461195Z","shell.execute_reply":"2022-10-31T16:50:55.169642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. LOAD THE TEST DATA","metadata":{}},{"cell_type":"code","source":"PATH=tf.io.gfile.glob(os.path.join(GCS_DS_PATH,f'Combining/Test/test_in_1.tfrecord'))\ntest_in_1 = TFRecordDataset(PATH, num_parallel_reads=AUTO)\ntest_in_1 = test_in_1.map(lambda x: tf.ensure_shape(tf.io.parse_tensor(x, tf.float16), (64)))\n\nPATH=tf.io.gfile.glob(os.path.join(GCS_DS_PATH,f'Combining/Test/test_in_2.tfrecord'))\ntest_in_2 = TFRecordDataset(PATH, num_parallel_reads=AUTO)\ntest_in_2 = test_in_2.map(lambda x: tf.ensure_shape(tf.io.parse_tensor(x, tf.float16), (64)))","metadata":{"execution":{"iopub.status.busy":"2022-10-31T16:50:55.171633Z","iopub.execute_input":"2022-10-31T16:50:55.171934Z","iopub.status.idle":"2022-10-31T16:50:55.320702Z","shell.execute_reply.started":"2022-10-31T16:50:55.171907Z","shell.execute_reply":"2022-10-31T16:50:55.319449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_inputs = tf.data.Dataset.zip(((test_in_1, test_in_2), ))\ntest_inputs = test_inputs.batch(512*8).cache().prefetch(tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2022-10-31T16:50:55.322257Z","iopub.execute_input":"2022-10-31T16:50:55.322500Z","iopub.status.idle":"2022-10-31T16:50:55.332402Z","shell.execute_reply.started":"2022-10-31T16:50:55.322471Z","shell.execute_reply":"2022-10-31T16:50:55.331593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 8. PREDICT AND SAVE THE SUBMISSION","metadata":{}},{"cell_type":"code","source":"with strategy.scope():\n    import time\n    start_time = time.time()\n    ##\n    #Predict the target values using the previous test dataset and the load model\n    x_1=model_1.predict(test_inputs)\n    predict_1 = np.squeeze(x_1)\n    \n    x_2=model_2.predict(test_inputs)\n    predict_2 = np.squeeze(x_2)\n    \n    x_3=model_3.predict(test_inputs)\n    predict_3 = np.squeeze(x_3)\n    \n    x_4=model_4.predict(test_inputs)\n    predict_4 = np.squeeze(x_4)\n    \n    x_5=model_5.predict(test_inputs)\n    predict_5 = np.squeeze(x_5)\n    ##\n    print(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2022-10-31T16:50:55.333643Z","iopub.execute_input":"2022-10-31T16:50:55.333846Z","iopub.status.idle":"2022-10-31T16:52:44.163818Z","shell.execute_reply.started":"2022-10-31T16:50:55.333822Z","shell.execute_reply":"2022-10-31T16:52:44.162485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.read_csv(\"../input/tabular-playground-series-oct-2022/sample_submission.csv\")\ndf[\"team_A_scoring_within_10sec\"]=0\ndf[\"team_B_scoring_within_10sec\"]=0\n\ndf[\"team_A_scoring_within_10sec\"]=(predict_1[0]+predict_2[0]+predict_3[0]+predict_4[0]+predict_5[0])/5\ndf[\"team_B_scoring_within_10sec\"]=(predict_1[1]+predict_2[1]+predict_3[1]+predict_4[1]+predict_5[1])/5\n\ndf.to_csv(\"submission_final.csv\",index=False)\ndf","metadata":{"execution":{"iopub.status.busy":"2022-10-31T16:52:44.165496Z","iopub.execute_input":"2022-10-31T16:52:44.166408Z","iopub.status.idle":"2022-10-31T16:52:45.796481Z","shell.execute_reply.started":"2022-10-31T16:52:44.166359Z","shell.execute_reply":"2022-10-31T16:52:45.794938Z"},"trusted":true},"execution_count":null,"outputs":[]}]}