{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ❗️❗️❗️DISCLAIMER: My notebook uses the model architecture of [this notebook](https://www.kaggle.com/code/alexryzhkov/tps-2022-10-fastai-with-multistart-and-tta) by [@alexryzhkov](https://www.kaggle.com/alexryzhkov) . My notebook wants to showcase the *SPEED* of training using TPU. If you enjoyed first like [@alexryzhkov](https://www.kaggle.com/alexryzhkov)'s notebook and leave a like also to this one if you want.❗️❗️❗️\n\n\n\n\n","metadata":{}},{"cell_type":"markdown","source":"In this version I'll try invert the usual folding , I use 10% for training and 90% for validation. I train 10 models so that overall every part of the dataset is used for training, but this time every part is used only by one model in training and by 9 models for validation.","metadata":{}},{"cell_type":"markdown","source":"# Required imports","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport gc\nfrom pathlib import Path\nimport tensorflow as tf\nfrom tensorflow.data import Dataset, TFRecordDataset\nimport os\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.layers import Dense,Dropout,Input,GaussianNoise,BatchNormalization,Lambda\nfrom tensorflow.keras.losses import  BinaryCrossentropy\nfrom tensorflow.keras.optimizers import Adam\nimport tensorflow_addons as tfa","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-24T17:55:04.702361Z","iopub.execute_input":"2022-10-24T17:55:04.702713Z","iopub.status.idle":"2022-10-24T17:55:04.708820Z","shell.execute_reply.started":"2022-10-24T17:55:04.702680Z","shell.execute_reply":"2022-10-24T17:55:04.708068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TPU preparation","metadata":{"execution":{"iopub.status.busy":"2022-10-19T11:34:01.477237Z","iopub.execute_input":"2022-10-19T11:34:01.477526Z","iopub.status.idle":"2022-10-19T11:34:01.613260Z","shell.execute_reply.started":"2022-10-19T11:34:01.477496Z","shell.execute_reply":"2022-10-19T11:34:01.612315Z"}}},{"cell_type":"code","source":"#get google cloud path of the dataset while on CPU \n#Uncomment this cell, execute it on cpu, write down GCS_DS_PATH below in the variable\nfrom kaggle_datasets import KaggleDatasets\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(\"tps-oct-2022-tfrecords\")\nGCS_DS_PATH","metadata":{"execution":{"iopub.status.busy":"2022-10-24T17:55:04.710323Z","iopub.execute_input":"2022-10-24T17:55:04.710807Z","iopub.status.idle":"2022-10-24T17:55:05.152865Z","shell.execute_reply.started":"2022-10-24T17:55:04.710773Z","shell.execute_reply":"2022-10-24T17:55:05.151932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    BATCH_SIZE = 4096 * strategy.num_replicas_in_sync\n    print(\"TPU\")\nexcept:\n    tpu = None\n    strategy = tf.distribute.get_strategy()\n    BATCH_SIZE=512\n    print(\"CPU\")","metadata":{"execution":{"iopub.status.busy":"2022-10-24T17:55:05.154186Z","iopub.execute_input":"2022-10-24T17:55:05.154451Z","iopub.status.idle":"2022-10-24T17:55:11.776864Z","shell.execute_reply.started":"2022-10-24T17:55:05.154421Z","shell.execute_reply":"2022-10-24T17:55:11.775919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset setup","metadata":{}},{"cell_type":"code","source":"# used to autotune tensorflow dataset transformations\nAUTO = tf.data.experimental.AUTOTUNE\n#get 10 datasets independently and store all in a list for later use\ndatasets=[]\n\n#option for faster TPU data read\nignore_order = tf.data.Options()\nignore_order.experimental_deterministic = False\n\nwith strategy.scope():\n    for i in range(10):\n        PATH=tf.io.gfile.glob(os.path.join(GCS_DS_PATH,f'train_{i}/feats.tfrecord*'))\n        ds = TFRecordDataset(PATH, num_parallel_reads=AUTO)\n        ds = ds.with_options(ignore_order)\n        ds_all_feats = ds.map(lambda x: tf.ensure_shape(tf.io.parse_tensor(x, out_type=tf.float32),(187)), num_parallel_calls=AUTO)\n        dataset=ds_all_feats.map(lambda x: (x[:-2],([x[-2]],[x[-1]])), num_parallel_calls=AUTO)\n\n        datasets.append(dataset)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T17:55:11.779106Z","iopub.execute_input":"2022-10-24T17:55:11.779521Z","iopub.status.idle":"2022-10-24T17:55:12.768202Z","shell.execute_reply.started":"2022-10-24T17:55:11.779477Z","shell.execute_reply":"2022-10-24T17:55:12.767250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train  validation split\ndef get_train_valid(datasets,valid_idx=0):\n    #return datasets[(valid_idx+1)%10],datasets[valid_idx]\n    train_ds=None\n    for i,dataset in enumerate(datasets):\n        if i==valid_idx:\n            valid_ds=dataset\n        elif train_ds is not None:\n            train_ds=train_ds.concatenate(dataset)\n        else:\n            train_ds=dataset\n            \n    #Here I swap train and valid\n    return valid_ds,train_ds","metadata":{"execution":{"iopub.status.busy":"2022-10-24T17:55:12.769388Z","iopub.execute_input":"2022-10-24T17:55:12.769726Z","iopub.status.idle":"2022-10-24T17:55:12.775425Z","shell.execute_reply.started":"2022-10-24T17:55:12.769693Z","shell.execute_reply":"2022-10-24T17:55:12.774501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model definition","metadata":{}},{"cell_type":"code","source":"#Support constants that will be used in the augment function\ndisplacement=tf.constant([*([i for i in range(28)]*3)],dtype=tf.int64)\n\nindices=tf.range(0,3,dtype=tf.int64)\n\nindices_ball=tf.range(0,17,dtype=tf.int64)\n\n#Augmentation function\ndef augment(x):\n    #shuffle team A\n    shuffled_indices = tf.random.shuffle(indices)\n    indices_teamA=tf.repeat(shuffled_indices,28,axis=-1)*28+displacement+17\n    original_indicesA=tf.repeat(indices,28,axis=-1)*28+displacement+17\n    a=tf.gather(x,indices_teamA,axis=-1)\n    \n    shuffled_indices = tf.random.shuffle(indices)\n    indices_teamB=tf.repeat(shuffled_indices,28,axis=-1)*28+displacement+101\n    original_indicesB=tf.repeat(indices,28,axis=-1)*28+displacement+101\n    b=tf.gather(x,indices_teamB,axis=-1)\n    \n    ball=tf.gather(x,indices_ball,axis=-1)\n    \n    return tf.concat([ball,a,b],axis=-1)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T17:55:12.776485Z","iopub.execute_input":"2022-10-24T17:55:12.776722Z","iopub.status.idle":"2022-10-24T17:55:12.790329Z","shell.execute_reply.started":"2022-10-24T17:55:12.776696Z","shell.execute_reply":"2022-10-24T17:55:12.789234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model():\n    inputs=Input(shape=[185])\n    \n    x=inputs\n    \n    \n    \n    #need to check better this logic\n    #Shuffle players inside their own team\n    x=Lambda(augment)(x)\n    \n    \n    #architecture from @alexryzhkov notebook\n    x=BatchNormalization()(x)\n    \n    x=GaussianNoise(0.001)(x)\n    x=Dense(4096,activation=tfa.activations.mish)(x)\n    x=BatchNormalization()(x)    \n    x=Dropout(0.3)(x)\n    \n    x=Dense(2048,activation=tfa.activations.mish)(x)\n    x=BatchNormalization()(x)    \n    x=Dropout(0.3)(x)\n    \n    x=Dense(2048,activation=tfa.activations.mish)(x)\n    x=BatchNormalization()(x)    \n    x=Dropout(0.3)(x)\n    \n    x=Dense(1024,activation=tfa.activations.mish)(x)\n    x=BatchNormalization()(x)    \n    x=Dropout(0.3)(x)\n    \n    x=Dense(512,activation=tfa.activations.mish)(x)\n    x=BatchNormalization()(x)    \n    x=Dropout(0.3)(x)\n    \n    out1=Dense(1,activation=\"sigmoid\",name=\"teamA\")(x)\n    out2=Dense(1,activation=\"sigmoid\",name=\"teamB\")(x)\n    #NEXT STEP try setting initial bias for inbalanced classification, should speed up learning: https://www.tensorflow.org/tutorials/structured_data/imbalanced_data#optional_set_the_correct_initial_bias\n    model=Model(inputs=inputs,outputs=[out1,out2])\n    \n    #higher BS -> better increase the Learning rate\n    model.compile(Adam(learning_rate=0.004),[BinaryCrossentropy(from_logits=False),BinaryCrossentropy(from_logits=False)],[\"accuracy\"])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-10-24T17:55:12.791583Z","iopub.execute_input":"2022-10-24T17:55:12.791820Z","iopub.status.idle":"2022-10-24T17:55:12.808044Z","shell.execute_reply.started":"2022-10-24T17:55:12.791793Z","shell.execute_reply":"2022-10-24T17:55:12.807207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.read_csv(\"../input/tabular-playground-series-oct-2022/sample_submission.csv\")\ndf[\"team_A_scoring_within_10sec\"]=0\ndf[\"team_B_scoring_within_10sec\"]=0\nwith strategy.scope():\n    PATH=tf.io.gfile.glob(os.path.join(GCS_DS_PATH,f'test/feats.tfrecord'))\n    ds = TFRecordDataset(PATH, num_parallel_reads=AUTO)\n\n    ds_test = ds.map(lambda x: tf.ensure_shape(tf.io.parse_tensor(x, out_type=tf.float32),(185)), num_parallel_calls=AUTO)\n\n    batched_dataset=ds_test.batch(512*8).cache()","metadata":{"execution":{"iopub.status.busy":"2022-10-24T17:55:12.809357Z","iopub.execute_input":"2022-10-24T17:55:12.809581Z","iopub.status.idle":"2022-10-24T17:55:13.099015Z","shell.execute_reply.started":"2022-10-24T17:55:12.809555Z","shell.execute_reply":"2022-10-24T17:55:13.098170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training ","metadata":{}},{"cell_type":"code","source":"with strategy.scope():\n    gc.collect()\n    for i in range(10):\n        callbacks=[tf.keras.callbacks.EarlyStopping(restore_best_weights=True,patience=3)]\n        print(f\"---------------------------  start fold {i}  ---------------------------\")\n        train_ds,valid_ds=get_train_valid(datasets,i)\n        #Very important caching reduces training time to 10% after first epoch!!!!\n        train=train_ds.batch(BATCH_SIZE).cache().prefetch(AUTO)\n        valid=valid_ds.batch(BATCH_SIZE).cache().prefetch(AUTO)\n        model=create_model()\n        model.fit(train,validation_data=valid,epochs=25,callbacks=callbacks,verbose=2)\n        preds=model.predict(batched_dataset,verbose=2)\n        preds2=model.predict(batched_dataset,verbose=2)\n        preds3=model.predict(batched_dataset,verbose=2)\n        preds4=model.predict(batched_dataset,verbose=2)\n        preds5=model.predict(batched_dataset,verbose=2)\n\n        df[\"team_A_scoring_within_10sec\"]+=(preds[0][:,0]+preds2[0][:,0]+preds3[0][:,0]+preds4[0][:,0]+preds5[0][:,0])/5\n        df[\"team_B_scoring_within_10sec\"]+=(preds[1][:,0]+preds2[1][:,0]+preds3[1][:,0]+preds4[1][:,0]+preds5[1][:,0])/5\n        \n        #save models to disk \n        save_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\n        model.save(f'./model_{i}', options=save_locally) # saving in Tensorflow's \"SavedModel\" format\n        del model,preds,preds2,preds3,preds4,preds5,train,valid\n        gc.collect()\n    df[\"team_A_scoring_within_10sec\"]/=10\n    df[\"team_B_scoring_within_10sec\"]/=10","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-10-24T17:55:13.100160Z","iopub.execute_input":"2022-10-24T17:55:13.100976Z","iopub.status.idle":"2022-10-24T18:03:58.700202Z","shell.execute_reply.started":"2022-10-24T17:55:13.100939Z","shell.execute_reply":"2022-10-24T18:03:58.698763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"df.to_csv(\"submission.csv\",index=False)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-24T18:03:58.701504Z","iopub.status.idle":"2022-10-24T18:03:58.702352Z","shell.execute_reply.started":"2022-10-24T18:03:58.702106Z","shell.execute_reply":"2022-10-24T18:03:58.702127Z"},"trusted":true},"execution_count":null,"outputs":[]}]}