{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Team members are junseonglee11 (@junseonglee11), Ayaan Jang(@ayaanjang).\nIn this notebook, We just combined many great public notebooks and fine-tuned the model.  \nAnd We usually executed this notebook on the Google Colab TPU with Google Drive (for fast saving). ","metadata":{}},{"cell_type":"markdown","source":"# References","metadata":{}},{"cell_type":"markdown","source":"1. Robin smith's: notebooks:   \nhttps://www.kaggle.com/code/rsmits/tensorflow-lstm-model-inference  \nhttps://www.kaggle.com/code/rsmits/tensorflow-lstm-model-training-tpu  \nhttps://www.kaggle.com/code/rsmits/tensorflow-lstm-model-data-preprocessor/notebook  \nRobin Smith's reference  \nhttps://www.kaggle.com/code/seungmoklee/lstm-preprocessing-point-picker\n   I modified his notebook   \n   a. Converted the dataset to TFRecords  \n   b. Additional inputs and some preprocessing  \n   c. Changed the model (more RNN layers, GRU --> LSTM, GELU activation, rectified adam optimizer)\n\n2. Robert Hatch's notebook  \nhttps://www.kaggle.com/code/roberthatch/lb-1-183-lightning-fast-baseline-with-polars  \nIt was crucial to improve our score.\nUsed the results of this notebook as additional inputs in our model.","metadata":{}},{"cell_type":"markdown","source":"TFRecord dataset generation: https://www.kaggle.com/code/junseonglee11/icecube-data-to-tfrecord-v2-1","metadata":{}},{"cell_type":"markdown","source":"# Google Colab Part","metadata":{}},{"cell_type":"markdown","source":"1. Connect to Google drive and install require libraries  \n2. Save weights to Google drive with filename = save_name","metadata":{}},{"cell_type":"code","source":"#from google.colab import drive\n#drive.mount('/content/drive')\n\n#from google.colab import files\n#files.upload()","metadata":{"id":"LtSV5FPog6Mv","outputId":"c12d54c9-53f3-430a-852d-ba791c0b3ef7","execution":{"iopub.status.busy":"2023-04-20T00:45:24.268582Z","iopub.execute_input":"2023-04-20T00:45:24.269265Z","iopub.status.idle":"2023-04-20T00:45:24.277048Z","shell.execute_reply.started":"2023-04-20T00:45:24.269230Z","shell.execute_reply":"2023-04-20T00:45:24.276198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip -qq install kaggle\n#!pip -qq install datasets\n!pip -qq install tensorflow-addons\n!pip -qq install wandb","metadata":{"id":"h-8ZfrO_t8n7","outputId":"64246780-dab1-4c98-c350-ef52b353231b","execution":{"iopub.status.busy":"2023-04-20T00:46:21.726078Z","iopub.execute_input":"2023-04-20T00:46:21.726820Z","iopub.status.idle":"2023-04-20T00:46:30.074729Z","shell.execute_reply.started":"2023-04-20T00:46:21.726782Z","shell.execute_reply":"2023-04-20T00:46:30.073419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_name = '230420_LSTM_90file_line-aux_2layer_radam_gelu_drop_f0_5th'\nIS_WANDB = False\nWANDB_KEY = None","metadata":{"id":"CHzZWXz0U-s3","execution":{"iopub.status.busy":"2023-04-20T00:46:33.427083Z","iopub.execute_input":"2023-04-20T00:46:33.427993Z","iopub.status.idle":"2023-04-20T00:46:33.432021Z","shell.execute_reply.started":"2023-04-20T00:46:33.427956Z","shell.execute_reply":"2023-04-20T00:46:33.431270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import libraries","metadata":{}},{"cell_type":"code","source":"# Import\nimport glob\nimport numpy as np\nimport os\nimport gc\nimport tensorflow as tf\nimport random\nfrom tqdm.notebook import tqdm\nfrom kaggle_datasets import KaggleDatasets\nfrom keras import backend as K\nimport itertools\nimport math\nfrom matplotlib import pyplot as plt\nimport tensorflow_addons as tfa\nimport wandb\nfrom wandb.keras import WandbCallback\n\n#for wandb logging\nif (WANDB_KEY is not None):\n    wandb.login(key = WANDB_KEY)\n    os.environ[\"WANDB_SILENT\"] = \"true\"\n    IS_WANDB = True\n    print('Use WandB for logging!')\n\ntf.config.optimizer.set_jit(True)\n#tf.compat.v1.disable_eager_execution()\n#tf.compat.v1.experimental.output_all_intermediates(True)\n#tf.compat.v1.enable_eager_execution()\n#tf.config.run_functions_eagerly(True)\n#tf.compat.v1.enable_eager_execution()\n#tf.data.experimental.enable_debug_mode()","metadata":{"id":"s7s5npp9ULuR","outputId":"60549e4e-764b-4a15-b69d-dee985301679","execution":{"iopub.status.busy":"2023-04-20T00:48:28.446797Z","iopub.execute_input":"2023-04-20T00:48:28.447797Z","iopub.status.idle":"2023-04-20T00:48:28.455724Z","shell.execute_reply.started":"2023-04-20T00:48:28.447757Z","shell.execute_reply":"2023-04-20T00:48:28.454676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TPU setting","metadata":{}},{"cell_type":"code","source":"# Configure Strategy. Assume TPU...if not set default for GPU\ntpu = None\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.MirroredStrategy()\n    \n\nN_REPLICAS = strategy.num_replicas_in_sync\nprint(f'N_REPLICAS: {N_REPLICAS}, IS_TPU: {tpu is not None}')","metadata":{"id":"XSlDvRx3ULuS","outputId":"30df107d-69ea-41fa-a128-0df9969486c0","execution":{"iopub.status.busy":"2023-04-20T00:46:40.418915Z","iopub.execute_input":"2023-04-20T00:46:40.419292Z","iopub.status.idle":"2023-04-20T00:46:50.209444Z","shell.execute_reply.started":"2023-04-20T00:46:40.419261Z","shell.execute_reply":"2023-04-20T00:46:50.208534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Constants","metadata":{"id":"uobPPXECULuS"}},{"cell_type":"code","source":"VERBOSE = 1\n\n# Training\nN_FEATS = 9\nN_WARMUP_EPOCHS = 10\nLR_MAX = 0.0005# *N_REPLICAS\nLR_MIN = LR_MAX/4\nSTEP_SIZE = 4\n\nWD_RATIO = 0.01\nN_EPOCHS = 5\nn_folds = 10\nselected_fold = 0\nbatch_size = 896*N_REPLICAS    # Local Training: 2048\n\n# Model Parameters\npulse_count = 96\nfeature_count = N_FEATS\nlstm_units = 192\nbin_num = 32\n\nconfig = {\n    'project': 'IceCube',    \n    'name'  : 'cnn0_LSTM_90file_line-2layer_radam_gelu_drop_f0_5th',\n    'group'  : 'line_fit',\n    'N_FEATS': N_FEATS,\n    'N_WARMUP_EPOCHS': N_WARMUP_EPOCHS,\n    'LR_MAX' : LR_MAX,\n    'LR_MIN' : LR_MIN,\n    'STEP_SIZE': STEP_SIZE,\n    'WD_RATIO': WD_RATIO,\n    'N_EPOCHS': N_EPOCHS,\n    'N_FOLDS' : n_folds,\n    'SELECTED_FOLD': selected_fold,\n    'BATCH_SIZE': batch_size,\n    'pulse_count': pulse_count,\n    'feature_count':N_FEATS,\n    'lstm_units': lstm_units,\n    'bin_num' : bin_num,\n    'save_name': save_name,\n    'N_FILES'  : 660,\n    'IS_TPU': tpu is not None\n    }\n\n","metadata":{"id":"BreAWNbUULuS","execution":{"iopub.status.busy":"2023-04-20T00:46:53.921359Z","iopub.execute_input":"2023-04-20T00:46:53.921734Z","iopub.status.idle":"2023-04-20T00:46:53.930851Z","shell.execute_reply.started":"2023-04-20T00:46:53.921704Z","shell.execute_reply":"2023-04-20T00:46:53.929918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set Seed\nseed = 4242\ntf.random.set_seed(seed)\nrandom.seed(seed)\nnp.random.seed(seed)","metadata":{"id":"w6fUBHpMULuT","execution":{"iopub.status.busy":"2023-04-20T00:46:54.784646Z","iopub.execute_input":"2023-04-20T00:46:54.785390Z","iopub.status.idle":"2023-04-20T00:46:54.790324Z","shell.execute_reply.started":"2023-04-20T00:46:54.785346Z","shell.execute_reply":"2023-04-20T00:46:54.789487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define Azimuth and Zenith Bins","metadata":{"id":"4Os1M4GJULuT"}},{"cell_type":"markdown","source":"# TFRecord dataset\n230317 junseonglee11 for full data training  \nConverted azimuth and zenith values (float) to category using keras backend operations","metadata":{"id":"gDR3tIMLULuU"}},{"cell_type":"code","source":"tensor_azimuth_edges = tf.constant(azimuth_edges, dtype = tf.float32)\ntensor_zenith_edges = tf.constant(zenith_edges,dtype = tf.float32)\n\ndef decode_tfrecord(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'event_pulses': tf.io.FixedLenFeature([], tf.string),\n        'fitted_azimuth': tf.io.FixedLenFeature([], tf.float32),               \n        'fitted_zenith' : tf.io.FixedLenFeature([], tf.float32),\n        'origin_azimuth': tf.io.FixedLenFeature([], tf.float32),               \n        'origin_zenith' : tf.io.FixedLenFeature([], tf.float32),\n    })        \n\n    event_pulses = tf.io.parse_tensor(features['event_pulses'], out_type=tf.float16)\n    event_pulses = tf.cast(event_pulses, tf.float16)\n    event_pulses = tf.reshape(event_pulses, [96, 9])\n    event_pulses = tf.slice(event_pulses, [0, 0], [96, feature_count])\n    \n    fitted_azimuth = features['fitted_azimuth']\n    fitted_azimuth = tf.cast(fitted_azimuth, tf.float16)\n    fitted_zenith  = features['fitted_zenith']\n    fitted_zenith  = tf.cast(fitted_zenith, tf.float16)\n\n    fitted_targets = tf.stack([fitted_azimuth, fitted_zenith])\n    \n    origin_azimuth = features['origin_azimuth']    \n    origin_azimuth = tf.cast(origin_azimuth, tf.float32)\n    \n    origin_zenith =  features['origin_zenith']\n    origin_zenith = tf.cast(origin_zenith, tf.float32)\n    \n    \n    azimuth_array = tf.repeat(origin_azimuth, bin_num+1)\n    azimuth_array = K.greater(origin_azimuth, tensor_azimuth_edges)\n    azimuth_array = tf.cast(azimuth_array, tf.int32)\n    azimuth = K.sum(azimuth_array, axis = 0)-1\n    \n    #for encodeing reproduce            \n    zenith_array = tf.repeat(origin_zenith, (bin_num+1))\n    zenith_array = K.greater(origin_zenith, tensor_zenith_edges)\n    zenith_array = tf.cast(zenith_array, tf.int32)\n    zenith = K.sum(zenith_array, axis = 0)-1\n    \n\n    \n    encoded_angle = azimuth*(bin_num)+zenith\n    origin_targets = tf.stack([origin_azimuth, origin_zenith])\n    \n    \n    return {'event_pulses': event_pulses, 'fitted_targets': fitted_targets }, {\n            'encoded_angle': encoded_angle}","metadata":{"id":"eSBagVZiULuU","execution":{"iopub.status.busy":"2023-04-20T00:47:03.096786Z","iopub.execute_input":"2023-04-20T00:47:03.097750Z","iopub.status.idle":"2023-04-20T00:47:03.112792Z","shell.execute_reply.started":"2023-04-20T00:47:03.097713Z","shell.execute_reply":"2023-04-20T00:47:03.111937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tensorflow AUTO flag\nAUTO = tf.data.experimental.AUTOTUNE\n\ndef get_dataset(tfrecords, bs=batch_size, val=False, debug=False):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False\n    \n    # Initialize dataset with TFRecords\n    dataset = tf.data.TFRecordDataset(tfrecords, num_parallel_reads=AUTO, compression_type='GZIP')\n    \n    # Decode mapping\n    dataset = dataset.map(decode_tfrecord, num_parallel_calls=AUTO)\n\n    if not val:                \n        dataset = dataset.with_options(ignore_order)\n        if not debug:\n            dataset = dataset.shuffle(2000000)\n        dataset = dataset.repeat()        \n\n    dataset = dataset.batch(bs, drop_remainder= not val)\n    dataset = dataset.prefetch(AUTO)    \n    return dataset","metadata":{"id":"ilUvcONQULuV","execution":{"iopub.status.busy":"2023-04-20T00:47:04.486794Z","iopub.execute_input":"2023-04-20T00:47:04.487604Z","iopub.status.idle":"2023-04-20T00:47:04.496332Z","shell.execute_reply.started":"2023-04-20T00:47:04.487567Z","shell.execute_reply":"2023-04-20T00:47:04.495007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TFRECORDS_FILE_PATHS = sorted(glob.glob('/kaggle/input/icecube-tfrecords-dataset-v3-*/*'))\n\nfile_inds = np.zeros(len(TFRECORDS_FILE_PATHS), int)\ncounter = 0\nfor file_path in TFRECORDS_FILE_PATHS:\n    tmp = file_path.split('/')[-1][:-9]\n    file_inds[counter] = int(tmp)    \n    counter+=1\nfile_ind_order = np.argsort(file_inds)\n\nprint('# of TFRecords: %d'%len(TFRECORDS_FILE_PATHS))\nTFRECORDS_TRAIN =[]\nTFRECORDS_VALID =[]\nfor i in range(0, config['N_FILES']):#len(TFRECORDS_FILE_PATHS)):\n    if(i%n_folds == selected_fold):\n        TFRECORDS_VALID.append(TFRECORDS_FILE_PATHS[file_ind_order[i]])\n        print(TFRECORDS_FILE_PATHS[file_ind_order[i]])\n    else:\n        TFRECORDS_TRAIN.append(TFRECORDS_FILE_PATHS[file_ind_order[i]])\n                              \nprint('# of train TFRECORDS: %d'%len(TFRECORDS_TRAIN))\nprint('# of valid TFRECORDS: %d'%len(TFRECORDS_VALID))\nTRAIN_STEPS_PER_EPOCH = len(TFRECORDS_TRAIN)*200000//batch_size\nprint('Train steps per epoch: %d'%TRAIN_STEPS_PER_EPOCH)","metadata":{"id":"6UfNyi8sULuV","outputId":"68191a01-b3d6-45b4-fbe7-893fa0183630","execution":{"iopub.status.busy":"2023-04-20T00:54:31.416514Z","iopub.execute_input":"2023-04-20T00:54:31.417268Z","iopub.status.idle":"2023-04-20T00:54:31.488296Z","shell.execute_reply.started":"2023-04-20T00:54:31.417232Z","shell.execute_reply":"2023-04-20T00:54:31.487118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model and callbacks definition","metadata":{"id":"VhHjwXkHULuW"}},{"cell_type":"code","source":"def create_model():\n    with strategy.scope(): \n        inputs_1 = tf.keras.layers.Input((pulse_count, feature_count))\n        inputs_2 = tf.keras.layers.Input((2,))\n\n        x = tf.keras.layers.Masking(mask_value = 0., input_shape = (pulse_count, feature_count))(inputs_1)\n\n        x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(lstm_units, return_sequences = True))(x)\n        x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(lstm_units, return_sequences = True))(x)\n        x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(lstm_units, return_sequences = True))(x)\n        x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(lstm_units, return_sequences = True))(x)\n        x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(lstm_units, return_sequences = True))(x)\n        x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(lstm_units, return_sequences = True))(x)\n        x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(lstm_units))(x)\n        x = tf.keras.layers.BatchNormalization()(x)\n        x = tf.keras.layers.Dense(256, activation = 'gelu')(x)\n        x = tf.keras.layers.BatchNormalization()(x)\n\n        x = tf.concat([x, inputs_2], axis = 1)\n        x = tf.keras.layers.Dense(258, activation = 'gelu')(x)\n        x = tf.keras.layers.Dense(258, activation = 'gelu')(x)\n        x = tf.keras.layers.Dropout(0.1)(x)\n        x = tf.keras.layers.Dense(258, activation = 'gelu')(x)\n\n        outputs = tf.keras.layers.Dense(bin_num**2, activation = 'softmax')(x)\n\n        # Finalize Model\n        model = tf.keras.models.Model(inputs = {'event_pulses': inputs_1, 'fitted_targets': inputs_2}, outputs = {'encoded_angle': outputs})\n\n        # Compile model\n        model.compile(loss = 'sparse_categorical_crossentropy',\n                      #optimizer= tf.keras.optimizers.Adam(learning_rate = LR_MAX),\n                      optimizer = tfa.optimizers.RectifiedAdam(learning_rate = LR_MAX),\n                      metrics = ['accuracy'])\n\n        # Show Model Summary\n        model.summary()\n\n        return model","metadata":{"id":"TpVO-DtJULuX","execution":{"iopub.status.busy":"2023-04-20T00:54:36.923263Z","iopub.execute_input":"2023-04-20T00:54:36.923642Z","iopub.status.idle":"2023-04-20T00:54:36.940186Z","shell.execute_reply.started":"2023-04-20T00:54:36.923613Z","shell.execute_reply":"2023-04-20T00:54:36.939148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filepath = \"epoch{epoch:02d}-val_acc{val_accuracy:.5f}.h5\"\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(filepath, save_best_only=False)\ncallbacks = [checkpoint]","metadata":{"id":"T-Zy9csxULuW","execution":{"iopub.status.busy":"2023-04-20T00:54:37.993389Z","iopub.execute_input":"2023-04-20T00:54:37.994159Z","iopub.status.idle":"2023-04-20T00:54:37.999031Z","shell.execute_reply.started":"2023-04-20T00:54:37.994109Z","shell.execute_reply":"2023-04-20T00:54:37.998068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if(IS_WANDB):\n    wandb.init(project = config['project'], name = f\"{config['name']}\",\n                   group = config['group'], config = config)\n    wandb_callback = WandbCallback(\n        monitor=\"val_accuracy\", save_model=(False), mode = 'max'\n    )\n    callbacks.append(wandb_callback)","metadata":{"id":"CoX5cCv64Z26","outputId":"189e09a2-5a86-4e3f-a301-00f5e23d819d","execution":{"iopub.status.busy":"2023-04-20T00:54:38.492866Z","iopub.execute_input":"2023-04-20T00:54:38.493749Z","iopub.status.idle":"2023-04-20T00:54:38.499447Z","shell.execute_reply.started":"2023-04-20T00:54:38.493717Z","shell.execute_reply":"2023-04-20T00:54:38.498257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = get_dataset(TFRECORDS_TRAIN, val=False, debug=False)\nvalid_dataset = get_dataset(TFRECORDS_VALID, val=True, debug=False)\n\nK.clear_session()\n\nmodel = create_model()\n                    \nhistory = model.fit(\n        train_dataset,\n        steps_per_epoch = TRAIN_STEPS_PER_EPOCH ,        \n        validation_data = valid_dataset,\n        epochs = N_EPOCHS,\n        verbose = VERBOSE,\n        callbacks = callbacks\n    )\nmodel.save('last_model.h5')\n\n#del model    \n#K.clear_session()\n#gc.collect()\n    \nwandb.finish()","metadata":{"id":"CnlBW0-wULuX","outputId":"fb1c45a8-828a-433f-95e4-b180d0057380","execution":{"iopub.status.busy":"2023-04-20T00:55:06.316392Z","iopub.execute_input":"2023-04-20T00:55:06.317447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model\ngc.collect()\nK.clear_session()","metadata":{"id":"XmKSu5vkULuX","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save to Google Drive (for Colab)","metadata":{}},{"cell_type":"code","source":"import glob\nimport os\nimport shutil \n\ntry:    \n    file_list = glob.glob('*h5')\n    print(file_list)\n\n\n    os.mkdir(save_name)\n    for i in range(0, len(file_list)):\n        shutil.copyfile(file_list[i], '%s/%s'%(save_name, file_list[i]))\n    shutil.make_archive('/content/drive/MyDrive/%s'%save_name, 'gztar', save_name)\n\nexcept: \n    print('It is not Colab notebook')","metadata":{"id":"UIDNASh6ULuY","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}