{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ❗️❗️❗️DISCLAIMER: My notebook uses the model architecture of [this notebook](https://www.kaggle.com/code/alexryzhkov/tps-2022-10-fastai-with-multistart-and-tta) by [@alexryzhkov](https://www.kaggle.com/alexryzhkov) as the main branch. My notebook wants to showcase the *SPEED* of training using TPU and how it is possible to train a reconstruction head together with a standard logloss head without particular changes in speed. If you enjoyed first like [@alexryzhkov](https://www.kaggle.com/alexryzhkov)'s notebook and leave a like also to this one if you want.❗️❗️❗️\n\n\n\n\n","metadata":{}},{"cell_type":"markdown","source":"The notebook results slower than the previous one since the number of epochs and the early stopping patience has been increased.\nThe model could improve by tuning learning rate, and other NN hyperparameters. A possibility could also be increase the number of inferences on the test set by enabling dropout also in inference. \n\nAn interesting aspect I found is that even though I shuffle players inside the network the reconstruction loss keeps decreasing.\n\nThis month's TPS allowed me to understand how TPU is incredibly fast and how complex is setting up a tabular pipeline for working on TPU, bugs are often difficult to spot and hard to fix.\n\nHope this notebook helps someone learn something new. \n\nI hope to learn from other kagglers at the end of this TPS seeing top solutions.","metadata":{}},{"cell_type":"markdown","source":"# Required imports","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport gc\nfrom pathlib import Path\nimport tensorflow as tf\nfrom tensorflow.data import Dataset, TFRecordDataset\nimport os\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.layers import Dense,Dropout,Input,GaussianNoise,BatchNormalization,Lambda\nfrom tensorflow.keras.losses import  BinaryCrossentropy,MeanSquaredError\nfrom tensorflow.keras.optimizers import Adam\nimport tensorflow_addons as tfa","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-29T18:35:19.957214Z","iopub.execute_input":"2022-10-29T18:35:19.957572Z","iopub.status.idle":"2022-10-29T18:35:19.963615Z","shell.execute_reply.started":"2022-10-29T18:35:19.957533Z","shell.execute_reply":"2022-10-29T18:35:19.962775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TPU preparation","metadata":{"execution":{"iopub.status.busy":"2022-10-19T11:34:01.477237Z","iopub.execute_input":"2022-10-19T11:34:01.477526Z","iopub.status.idle":"2022-10-19T11:34:01.61326Z","shell.execute_reply.started":"2022-10-19T11:34:01.477496Z","shell.execute_reply":"2022-10-19T11:34:01.612315Z"}}},{"cell_type":"code","source":"#get google cloud path of the dataset while on CPU \n#Uncomment this cell, execute it on cpu, write down GCS_DS_PATH below in the variable\nfrom kaggle_datasets import KaggleDatasets\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(\"tps-oct-2022-tfrecords\")\nGCS_DS_PATH","metadata":{"execution":{"iopub.status.busy":"2022-10-29T18:35:19.965039Z","iopub.execute_input":"2022-10-29T18:35:19.965280Z","iopub.status.idle":"2022-10-29T18:35:20.293314Z","shell.execute_reply.started":"2022-10-29T18:35:19.965252Z","shell.execute_reply":"2022-10-29T18:35:20.291880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    BATCH_SIZE = 4096 * strategy.num_replicas_in_sync\n    print(\"TPU\")\nexcept:\n    tpu = None\n    strategy = tf.distribute.get_strategy()\n    BATCH_SIZE=512\n    print(\"CPU\")","metadata":{"execution":{"iopub.status.busy":"2022-10-29T18:35:20.295585Z","iopub.execute_input":"2022-10-29T18:35:20.296018Z","iopub.status.idle":"2022-10-29T18:35:25.835966Z","shell.execute_reply.started":"2022-10-29T18:35:20.295971Z","shell.execute_reply":"2022-10-29T18:35:25.835032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset setup","metadata":{}},{"cell_type":"code","source":"# used to autotune tensorflow dataset transformations\nAUTO = tf.data.experimental.AUTOTUNE\n\n#get 10 datasets independently and store all in a list for later use\ndatasets=[]\n\n#option for faster TPU data read\nignore_order = tf.data.Options()\nignore_order.experimental_deterministic = False\n\nwith strategy.scope():\n    for i in range(10):\n        PATH=tf.io.gfile.glob(os.path.join(GCS_DS_PATH,f'train_{i}/feats.tfrecord*'))\n        ds = TFRecordDataset(PATH, num_parallel_reads=AUTO)\n        ds = ds.with_options(ignore_order)\n        ds_all_feats = ds.map(lambda x: tf.ensure_shape(tf.io.parse_tensor(x, out_type=tf.float32),(187)), num_parallel_calls=AUTO)\n        dataset=ds_all_feats.map(lambda x: (x[:-2],([x[-2]],[x[-1]],x[:-2])), num_parallel_calls=AUTO)\n\n        datasets.append(dataset)","metadata":{"execution":{"iopub.status.busy":"2022-10-29T18:35:25.841517Z","iopub.execute_input":"2022-10-29T18:35:25.841826Z","iopub.status.idle":"2022-10-29T18:35:26.606854Z","shell.execute_reply.started":"2022-10-29T18:35:25.841793Z","shell.execute_reply":"2022-10-29T18:35:26.605087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train  validation split\ndef get_train_valid(datasets,valid_idx=0):\n    #return datasets[(valid_idx+1)%10],datasets[valid_idx]\n    train_ds=None\n    for i,dataset in enumerate(datasets):\n        if i==valid_idx:\n            valid_ds=dataset\n        elif train_ds is not None:\n            train_ds=train_ds.concatenate(dataset)\n        else:\n            train_ds=dataset\n            \n            \n    return train_ds,valid_ds","metadata":{"execution":{"iopub.status.busy":"2022-10-29T18:35:26.609042Z","iopub.execute_input":"2022-10-29T18:35:26.609389Z","iopub.status.idle":"2022-10-29T18:35:26.615804Z","shell.execute_reply.started":"2022-10-29T18:35:26.609355Z","shell.execute_reply":"2022-10-29T18:35:26.614803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model definition","metadata":{}},{"cell_type":"markdown","source":"## scheduler from [this notebook](https://www.kaggle.com/code/tolgadincer/tf-keras-learning-rate-schedulers/notebook)","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\ndef plot_scheduler2(step, schedulers):\n    if not isinstance(schedulers, list):\n        schedulers = [schedulers]\n\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=[10, 3])\n    for scheduler in schedulers:\n        x = range(step)\n        y = [scheduler(i).numpy() for i in x]\n        ax1.plot(x, y, label=scheduler.name)\n        ax1.set_xlabel('Epoch')\n        ax1.set_ylabel('Learning Rate')\n        ax1.legend()\n\n        ax2.plot(x, y, label=scheduler.name)\n        ax2.set_xlabel('Epoch')\n        ax2.set_ylabel('Learning Rate')\n        ax2.set_yscale('log')\n        ax2.legend()\n    plt.show()\n    \nexp_clr = tfa.optimizers.ExponentialCyclicalLearningRate(\n    initial_learning_rate=7e-5,\n    maximal_learning_rate=8e-3,\n    step_size=150,\n    scale_mode='iterations',\n    gamma=0.9985,\n    name='ExponentialCyclicalLearningRate'\n)\nplot_scheduler2(15000, exp_clr)","metadata":{"execution":{"iopub.status.busy":"2022-10-29T18:38:03.779276Z","iopub.execute_input":"2022-10-29T18:38:03.780085Z","iopub.status.idle":"2022-10-29T18:39:09.492097Z","shell.execute_reply.started":"2022-10-29T18:38:03.780014Z","shell.execute_reply":"2022-10-29T18:39:09.491331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Support constants that will be used in the augment function\ndisplacement=tf.constant([*([i for i in range(28)]*3)],dtype=tf.int64)\n\nindices=tf.range(0,3,dtype=tf.int64)\n\nindices_ball=tf.range(0,17,dtype=tf.int64)\n\n#Augmentation function -> maybe we can even shuffle teams, the model seems to be resistent to player reordering ???!!!\ndef augment(x):\n    #shuffle team A\n    shuffled_indices = tf.random.shuffle(indices)\n    indices_teamA=tf.repeat(shuffled_indices,28,axis=-1)*28+displacement+17\n    original_indicesA=tf.repeat(indices,28,axis=-1)*28+displacement+17\n    a=tf.gather(x,indices_teamA,axis=-1)\n    \n    shuffled_indices = tf.random.shuffle(indices)\n    indices_teamB=tf.repeat(shuffled_indices,28,axis=-1)*28+displacement+101\n    original_indicesB=tf.repeat(indices,28,axis=-1)*28+displacement+101\n    b=tf.gather(x,indices_teamB,axis=-1)\n    \n    ball=tf.gather(x,indices_ball,axis=-1)\n    \n    return tf.concat([ball,a,b],axis=-1)","metadata":{"execution":{"iopub.status.busy":"2022-10-29T18:35:26.617285Z","iopub.execute_input":"2022-10-29T18:35:26.617520Z","iopub.status.idle":"2022-10-29T18:35:26.635021Z","shell.execute_reply.started":"2022-10-29T18:35:26.617491Z","shell.execute_reply":"2022-10-29T18:35:26.633028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model():\n    inputs=Input(shape=[185])\n    \n    x=inputs\n    x2=inputs\n    \n    \n    \n    #need to check better this logic\n    #Shuffle players inside their own team\n    x=Lambda(augment)(x)#this should make reconstruction very hard -> has to reorder players , well actually it seems that having this or not is not so important for reconstruction loss\n    \n    sequential=tf.keras.Sequential([\n        BatchNormalization(),\n        GaussianNoise(0.005),\n        Dense(4096,activation=tfa.activations.mish),\n        BatchNormalization(),\n        Dropout(0.75),\n        Dense(2048,activation=tfa.activations.mish),\n        BatchNormalization(),\n        Dropout(0.75),\n        Dense(2048,activation=tfa.activations.mish),\n        BatchNormalization(),\n        Dropout(0.75),\n        Dense(1024,activation=tfa.activations.mish),\n        BatchNormalization(),\n        Dropout(0.75),\n        Dense(512,activation=tfa.activations.mish),\n        Dropout(0.7)\n    ])\n    \n    x=sequential(x)\n    x2=sequential(x2)#skip augmentation, this helps if we want to add some kind of preprocessing\n    \n    denseA=Dense(1,activation=\"sigmoid\",name=\"teamA\")\n    denseB=Dense(1,activation=\"sigmoid\",name=\"teamB\")\n    \n    out1=denseA(x)\n    out2=denseB(x)\n    \n    \n    x=Dense(1024,activation=tfa.activations.mish)(x)\n    x=BatchNormalization()(x)    \n    x=Dropout(0.8)(x)\n    \n    x=Dense(1024,activation=tfa.activations.mish)(x)\n    x=BatchNormalization()(x)    \n    x=Dropout(0.8)(x)\n    \n    out3=Dense(185,name=\"reconstruction\")(x)\n    \n    \n    model=Model(inputs=inputs,outputs=[out1,out2,out3])\n    \n    model_inference=Model(inputs=inputs,outputs=[out1,out2])\n    \n    #higher BS -> better increase the Learning rate\n    model.compile(Adam(learning_rate=exp_clr),[BinaryCrossentropy(from_logits=False),BinaryCrossentropy(from_logits=False),MeanSquaredError()],[\"accuracy\"],loss_weights=[1,1,0.7])\n    return model,model_inference","metadata":{"execution":{"iopub.status.busy":"2022-10-29T18:36:38.775695Z","iopub.status.idle":"2022-10-29T18:36:38.776063Z","shell.execute_reply.started":"2022-10-29T18:36:38.775873Z","shell.execute_reply":"2022-10-29T18:36:38.775890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.read_csv(\"../input/tabular-playground-series-oct-2022/sample_submission.csv\")\ndf[\"team_A_scoring_within_10sec\"]=0\ndf[\"team_B_scoring_within_10sec\"]=0\nwith strategy.scope():\n    PATH=tf.io.gfile.glob(os.path.join(GCS_DS_PATH,f'test/feats.tfrecord'))\n    ds = TFRecordDataset(PATH, num_parallel_reads=AUTO)\n\n    ds_test = ds.map(lambda x: tf.ensure_shape(tf.io.parse_tensor(x, out_type=tf.float32),(185)), num_parallel_calls=AUTO)\n\n    batched_dataset=ds_test.batch(512*8).cache()","metadata":{"execution":{"iopub.status.busy":"2022-10-29T18:36:38.778194Z","iopub.status.idle":"2022-10-29T18:36:38.778457Z","shell.execute_reply.started":"2022-10-29T18:36:38.778319Z","shell.execute_reply":"2022-10-29T18:36:38.778332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training ","metadata":{}},{"cell_type":"code","source":"model,_=create_model()\nNumber_of_parameters=model.count_params()\nprint(f\"The model has {Number_of_parameters:,} parameters\")","metadata":{"execution":{"iopub.status.busy":"2022-10-29T18:36:38.780526Z","iopub.status.idle":"2022-10-29T18:36:38.781137Z","shell.execute_reply.started":"2022-10-29T18:36:38.780831Z","shell.execute_reply":"2022-10-29T18:36:38.780860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n#code adapted from https://stackoverflow.com/questions/64556120/early-stopping-with-multiple-conditions\n#NEXT step -> save best for A, best for B -> stop when both stop improving (so if only A improve keep going and update only best for A)\n#             can do inference using the 2 models separately         \nclass CustomEarlyStopping(tf.keras.callbacks.Callback):\n    def __init__(self, patience=0):\n        super(CustomEarlyStopping, self).__init__()\n        self.patience = patience\n        self.best_weights = None\n        \n    def on_train_begin(self, logs=None):\n        # The number of epoch it has waited when loss is no longer minimum.\n        self.wait = 0\n        # The epoch the training stops at.\n        self.stopped_epoch = 0\n        # Initialize the best as infinity.\n        self.loss = np.Inf\n\n    def on_epoch_end(self, epoch, logs=None): \n        loss1=logs.get('val_teamA_loss')\n        loss2=logs.get('val_teamB_loss')\n        loss=loss1+loss2\n        # If BOTH the validation loss AND map10 does not improve for 'patience' epochs, stop training early.\n        if np.less(loss, self.loss):\n            self.loss = loss\n            self.wait = 0\n            # Record the best weights if current results is better (less).\n            self.best_weights = self.model.get_weights()\n        else:\n            self.wait += 1\n            if self.wait >= self.patience:\n                self.stopped_epoch = epoch\n                self.model.stop_training = True\n                print(\"Restoring model weights from the end of the best epoch.\")\n                self.model.set_weights(self.best_weights)\n                \n    def on_train_end(self, logs=None):\n        pass","metadata":{"execution":{"iopub.status.busy":"2022-10-29T18:36:38.783451Z","iopub.status.idle":"2022-10-29T18:36:38.784546Z","shell.execute_reply.started":"2022-10-29T18:36:38.784325Z","shell.execute_reply":"2022-10-29T18:36:38.784350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predsA=[]\npredsB=[]\nwith strategy.scope():\n    gc.collect()\n    for i in range(10):\n        #callbacks=[tf.keras.callbacks.EarlyStopping(monitor=\"val_teamA_loss\",mode=\"min\",restore_best_weights=True,patience=5),\n        #           tf.keras.callbacks.EarlyStopping(monitor=\"val_teamB_loss\",mode=\"min\",restore_best_weights=True,patience=5)]\n        callbacks=[CustomEarlyStopping(patience=10)]\n        print(f\"---------------------------  start fold {i}  ---------------------------\")\n        train_ds,valid_ds=get_train_valid(datasets,i)\n        #Very important caching reduces training time to 10% after first epoch!!!!\n        train=train_ds.shuffle(500000).batch(BATCH_SIZE).cache().prefetch(AUTO).shuffle(150)\n        valid=valid_ds.shuffle(500000).batch(BATCH_SIZE).cache().prefetch(AUTO).shuffle(150)\n        model,model_inference=create_model()\n        model.fit(train,validation_data=valid,epochs=60,callbacks=callbacks,verbose=2)\n        preds=model_inference.predict(batched_dataset,verbose=2)\n        \n        preds=model_inference.predict(batched_dataset,verbose=2)\n        preds2=model_inference.predict(batched_dataset,verbose=2)\n        preds3=model_inference.predict(batched_dataset,verbose=2)\n        preds4=model_inference.predict(batched_dataset,verbose=2)\n        preds5=model_inference.predict(batched_dataset,verbose=2)\n        \n        predsA.append(preds[0][:,0])\n        predsB.append(preds[1][:,0])\n        predsA.append(preds2[0][:,0])\n        predsB.append(preds2[1][:,0])\n        predsA.append(preds3[0][:,0])\n        predsB.append(preds3[1][:,0])\n        predsA.append(preds4[0][:,0])\n        predsB.append(preds4[1][:,0])\n        predsA.append(preds5[0][:,0])\n        predsB.append(preds5[1][:,0])\n        \n        df[\"team_A_scoring_within_10sec\"]+=(preds[0][:,0]+preds2[0][:,0]+preds3[0][:,0]+preds4[0][:,0]+preds5[0][:,0])/5\n        df[\"team_B_scoring_within_10sec\"]+=(preds[1][:,0]+preds2[1][:,0]+preds3[1][:,0]+preds4[1][:,0]+preds5[1][:,0])/5\n        \n        #save models to disk \n        save_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\n        model.save(f'./model_{i}', options=save_locally) # saving in Tensorflow's \"SavedModel\" format\n        del model,preds,preds2,preds3,preds4,preds5,train,valid\n        gc.collect()\n\n    df[\"team_A_scoring_within_10sec\"]/=10\n    df[\"team_B_scoring_within_10sec\"]/=10","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-10-29T18:36:38.785457Z","iopub.status.idle":"2022-10-29T18:36:38.786164Z","shell.execute_reply.started":"2022-10-29T18:36:38.785971Z","shell.execute_reply":"2022-10-29T18:36:38.785990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"df.to_csv(\"submission.csv\",index=False)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-29T18:36:38.787229Z","iopub.status.idle":"2022-10-29T18:36:38.788242Z","shell.execute_reply.started":"2022-10-29T18:36:38.788060Z","shell.execute_reply":"2022-10-29T18:36:38.788080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"team_A_scoring_within_10sec\"]=np.median(np.array(predsA),axis=0)\ndf[\"team_B_scoring_within_10sec\"]=np.median(np.array(predsB),axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-10-29T18:36:38.789381Z","iopub.status.idle":"2022-10-29T18:36:38.789732Z","shell.execute_reply.started":"2022-10-29T18:36:38.789554Z","shell.execute_reply":"2022-10-29T18:36:38.789572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-29T18:36:38.791119Z","iopub.status.idle":"2022-10-29T18:36:38.791439Z","shell.execute_reply.started":"2022-10-29T18:36:38.791277Z","shell.execute_reply":"2022-10-29T18:36:38.791294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(\"submission_median.csv\",index=False)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-29T18:36:38.792465Z","iopub.status.idle":"2022-10-29T18:36:38.792767Z","shell.execute_reply.started":"2022-10-29T18:36:38.792611Z","shell.execute_reply":"2022-10-29T18:36:38.792627Z"},"trusted":true},"execution_count":null,"outputs":[]}]}