{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.10"},"papermill":{"default_parameters":{},"duration":283.724921,"end_time":"2023-06-13T03:07:09.700085","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-06-13T03:02:25.975164","version":"2.4.0"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":41880,"databundleVersionId":5677426,"sourceType":"competition"},{"sourceId":7619962,"sourceType":"datasetVersion","datasetId":4438398}],"dockerImageVersionId":30498,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Configuration**","metadata":{"papermill":{"duration":0.00482,"end_time":"2023-06-13T03:02:36.847375","exception":false,"start_time":"2023-06-13T03:02:36.842555","status":"completed"},"tags":[]}},{"cell_type":"code","source":"CFG = {'TPU': 0, \n       'block_size': 15552, \n       'block_stride': 15552//16,\n       'patch_size': 18, \n       \n       'fog_model_dim': 320,\n       'fog_model_num_heads': 6,\n       'fog_model_num_encoder_layers': 5,\n       'fog_model_num_lstm_layers': 2,\n       'fog_model_first_dropout': 0.1,\n       'fog_model_encoder_dropout': 0.1,\n       'fog_model_mha_dropout': 0.0,\n      }\n\nassert CFG['block_size'] % CFG['patch_size'] == 0\nassert CFG['block_size'] % CFG['block_stride'] == 0\n\n'''\nMean-std normalization function. \nExample input: shape (5000), dtype np.float32\nExample output: shape (5000), dtype np.float32\n\nUsed to normalize AccV, AccML, AccAP values.\n\n'''\n\ndef sample_normalize(sample):\n    mean = tf.math.reduce_mean(sample)\n    std = tf.math.reduce_std(sample)\n    sample = tf.math.divide_no_nan(sample-mean, std)\n    \n    return sample.numpy()\n\n'''\nFunction for splitting a series into blocks. Blocks can overlap. \nHow the function works:\nSuppose we have a series with AccV, AccML, AccAP columns and len of 50000, that is (50000, 3). \nFirst, the series is padded so that the final length is divisible by CFG['block_size'] = 15552. Now the series shape is (62208, 3).\nThen we get blocks: first block is series[0:15552, :], second block is series[972:16524, :], ... , last block is series[46656:62208, :].\n\n'''\n\ndef get_blocks(series, columns):\n    series = series.copy()\n    series = series[columns]\n    series = series.values\n    series = series.astype(np.float32)\n    \n    block_count = math.ceil(len(series) / CFG['block_size'])\n    \n    series = np.pad(series, pad_width=[[0, block_count*CFG['block_size']-len(series)], [0, 0]])\n    \n    block_begins = list(range(0, len(series), CFG['block_stride']))\n    block_begins = [x for x in block_begins if x+CFG['block_size'] <= len(series)]\n    \n    blocks = []\n    for begin in block_begins:\n        values = series[begin:begin+CFG['block_size']]\n        blocks.append({'begin': begin,\n                       'end': begin+CFG['block_size'],\n                       'values': values})\n    \n    return blocks\n\n'''\nTrain and inference batch size\n\n'''\n\nGPU_BATCH_SIZE = 32\nTPU_BATCH_SIZE = GPU_BATCH_SIZE*8","metadata":{"papermill":{"duration":0.029145,"end_time":"2023-06-13T03:02:36.881046","exception":false,"start_time":"2023-06-13T03:02:36.851901","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-01T09:23:32.892563Z","iopub.execute_input":"2024-04-01T09:23:32.892817Z","iopub.status.idle":"2024-04-01T09:23:32.909551Z","shell.execute_reply.started":"2024-04-01T09:23:32.892793Z","shell.execute_reply":"2024-04-01T09:23:32.908729Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Imports and Utils**","metadata":{"papermill":{"duration":0.004152,"end_time":"2023-06-13T03:02:36.889496","exception":false,"start_time":"2023-06-13T03:02:36.885344","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport math\nimport random\nimport warnings\n\nif CFG['TPU']:\n    !pip install -q /lib/wheels/tensorflow-2.9.1-cp38-cp38-linux_x86_64.whl\n    !pip install -qU scikit-learn\n    \nimport numpy as np \nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport scipy\n\nfrom tqdm import tqdm\nfrom itertools import cycle\nfrom joblib import Parallel, delayed\nfrom sklearn.metrics import average_precision_score\n\nif CFG['TPU']:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu='local') \n    tpu_strategy = tf.distribute.TPUStrategy(tpu)\n\n\nwarnings.filterwarnings(\"ignore\")\npd.set_option('display.max_colwidth', None)\n\ntf.random.set_seed(3)\n\ndef folder(path): \n    if not os.path.exists(path): os.makedirs(path)\n        \ndef plot(e, size=(20, 4)):\n    plt.figure(figsize=size)\n    plt.plot(e)\n    plt.show()","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":9.402423,"end_time":"2023-06-13T03:02:46.296225","exception":false,"start_time":"2023-06-13T03:02:36.893802","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2024-04-01T09:23:32.911260Z","iopub.execute_input":"2024-04-01T09:23:32.911540Z","iopub.status.idle":"2024-04-01T09:23:41.405406Z","shell.execute_reply.started":"2024-04-01T09:23:32.911516Z","shell.execute_reply":"2024-04-01T09:23:41.404618Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model**","metadata":{"papermill":{"duration":0.003974,"end_time":"2023-06-13T03:02:46.304584","exception":false,"start_time":"2023-06-13T03:02:46.300610","status":"completed"},"tags":[]}},{"cell_type":"code","source":"'''\nThe transformer encoder layer\nFor more details, see https://arxiv.org/pdf/1706.03762.pdf [Attention Is All You Need]\n'''\n\nclass EncoderLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super().__init__()\n        \n        self.mha = tf.keras.layers.MultiHeadAttention(num_heads=CFG['fog_model_num_heads'], key_dim=CFG['fog_model_dim'], dropout=CFG['fog_model_mha_dropout'])\n        \n        self.add = tf.keras.layers.Add()\n        \n        self.layernorm = tf.keras.layers.LayerNormalization()\n        \n        self.seq = tf.keras.Sequential([tf.keras.layers.Dense(CFG['fog_model_dim'], activation='relu'), \n                                        tf.keras.layers.Dropout(CFG['fog_model_encoder_dropout']), \n                                        tf.keras.layers.Dense(CFG['fog_model_dim']), \n                                        tf.keras.layers.Dropout(CFG['fog_model_encoder_dropout']),\n                                       ])\n        \n    def call(self, x):\n        attn_output = self.mha(query=x, key=x, value=x)\n        x = self.add([x, attn_output])\n        x = self.layernorm(x)\n        x = self.add([x, self.seq(x)])\n        x = self.layernorm(x)\n        \n        return x\n    \n'''\nFOGEncoder is a combination of transformer encoder (D=320, H=6, L=5) and two BidirectionalLSTM layers\n'''\n\nclass FOGEncoder(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.first_linear = tf.keras.layers.Dense(CFG['fog_model_dim'])\n        \n        self.add = tf.keras.layers.Add()\n        \n        self.first_dropout = tf.keras.layers.Dropout(CFG['fog_model_first_dropout'])\n        \n        self.enc_layers = [EncoderLayer() for _ in range(CFG['fog_model_num_encoder_layers'])]\n        \n        self.lstm_layers = [tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(CFG['fog_model_dim'], return_sequences=True)) for _ in range(CFG['fog_model_num_lstm_layers'])]\n        \n        self.sequence_len = CFG['block_size'] // CFG['patch_size']\n        self.pos_encoding = tf.Variable(initial_value=tf.random.normal(shape=(1, self.sequence_len, CFG['fog_model_dim']), stddev=0.02), trainable=True)\n        \n    def call(self, x, training=None): # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (4, 864, 54)\n        x = x / 25.0 # Normalization attempt in the segment [-1, 1]\n        x = self.first_linear(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']), Example shape (4, 864, 320)\n          \n        if training: # augmentation by randomly roll of the position encoding tensor\n            random_pos_encoding = tf.roll(tf.tile(self.pos_encoding, multiples=[GPU_BATCH_SIZE, 1, 1]), \n                                          shift=tf.random.uniform(shape=(GPU_BATCH_SIZE,), minval=-self.sequence_len, maxval=0, dtype=tf.int32),\n                                          axis=GPU_BATCH_SIZE * [1],\n                                          )\n            x = self.add([x, random_pos_encoding])\n        \n        else: # without augmentation \n            x = self.add([x, tf.tile(self.pos_encoding, multiples=[GPU_BATCH_SIZE, 1, 1])])\n            \n        x = self.first_dropout(x)\n        \n        for i in range(CFG['fog_model_num_encoder_layers']): x = self.enc_layers[i](x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']), Example shape (4, 864, 320)\n        for i in range(CFG['fog_model_num_lstm_layers']): x = self.lstm_layers[i](x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']*2), Example shape (4, 864, 640)\n            \n        return x\n    \nclass FOGModel(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.encoder = FOGEncoder()\n        self.last_linear = tf.keras.layers.Dense(3) \n        \n    def call(self, x): # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (4, 864, 54)\n        x = self.encoder(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']*2), Example shape (4, 864, 640)\n        x = self.last_linear(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], 3), Example shape (4, 864, 3)\n        x = tf.nn.sigmoid(x) # Sigmoid activation\n        \n        return x","metadata":{"papermill":{"duration":0.028218,"end_time":"2023-06-13T03:02:46.336980","exception":false,"start_time":"2023-06-13T03:02:46.308762","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-01T09:23:41.406666Z","iopub.execute_input":"2024-04-01T09:23:41.407253Z","iopub.status.idle":"2024-04-01T09:23:41.427359Z","shell.execute_reply.started":"2024-04-01T09:23:41.407223Z","shell.execute_reply":"2024-04-01T09:23:41.426463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Tdcsfog preparing**","metadata":{"papermill":{"duration":0.003967,"end_time":"2023-06-13T03:02:46.345108","exception":false,"start_time":"2023-06-13T03:02:46.341141","status":"completed"},"tags":[]}},{"cell_type":"code","source":"'''\nCreate train blocks with AccV, AccML, AccAP, StartHesitation, Turn, Walking, Valid, Mask columns and save in the directory\n'''\n\n\nsave_path = '/kaggle/working/train/tdcsfog'; folder(save_path); \ntdcsfog_metadata = pd.read_csv('/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/tdcsfog_metadata.csv').set_index('Id')\n\nblocks_descriptions = []\nfor Id in tqdm(tdcsfog_metadata.index, total=len(tdcsfog_metadata.index), desc='Preparing'):\n    series = pd.read_csv(f'/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/tdcsfog/{Id}.csv')\n    \n    series['AccV'] = sample_normalize(series['AccV'].values)\n    series['AccML'] = sample_normalize(series['AccML'].values)\n    series['AccAP'] = sample_normalize(series['AccAP'].values)\n    series['Valid'] = 1\n    series['Mask'] = 1\n\n    blocks = get_blocks(series, ['AccV', 'AccML', 'AccAP', 'StartHesitation', 'Turn', 'Walking', 'Valid', 'Mask'])\n\n    for block_count, block in enumerate(blocks):\n        fname, values = f'{Id}_{block_count}.npy', block['values']\n        block_description = {}\n        block_description['Id'] = Id\n        block_description['Count'] = block_count\n        block_description['File'] = fname\n        block_description['Path'] = f'{save_path}/{fname}'\n        block_description['Source'] = 'tsfog'\n        block_description['StartHesitation_size'] = np.sum(values[:, 3])\n        block_description['Turn_size'] = np.sum(values[:, 4])\n        block_description['Walking_size'] = np.sum(values[:, 5])\n        block_description['Valid_size'] = np.sum(values[:, 6])\n        block_description['Mask_size'] = np.sum(values[:, 7])\n\n        blocks_descriptions.append(block_description)\n        np.save(f'{save_path}/{fname}', values)\n\nblocks_descriptions = pd.DataFrame(blocks_descriptions)","metadata":{"papermill":{"duration":34.250404,"end_time":"2023-06-13T03:03:20.599674","exception":false,"start_time":"2023-06-13T03:02:46.349270","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-01T09:23:41.428647Z","iopub.execute_input":"2024-04-01T09:23:41.428978Z","iopub.status.idle":"2024-04-01T09:24:15.330254Z","shell.execute_reply.started":"2024-04-01T09:23:41.428946Z","shell.execute_reply":"2024-04-01T09:24:15.329292Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Train Dataset**","metadata":{"papermill":{"duration":0.018247,"end_time":"2023-06-13T03:03:20.636481","exception":false,"start_time":"2023-06-13T03:03:20.618234","status":"completed"},"tags":[]}},{"cell_type":"code","source":"'''\nSelecting validation subjects\nFOGModel train data preparing\n'''\n\ndef write_to_ram(fog):\n    fog = fog[['Id', 'Count', 'Path']]\n    for _, row in tqdm(fog.iterrows(), total=len(fog), desc='Write'):\n        Id, Count, path = row['Id'], row['Count'], row['Path']\n        \n        # Read data\n        series = np.load(path) # ['AccV', 'AccML', 'AccAP', 'StartHesitation', 'Turn', 'Walking', 'Valid', 'Mask']\n\n        # Create patches\n        series = tf.reshape(series, shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size'], series.shape[1]))\n\n        # Create input\n        series_input = series[:, :, 0:3]\n        series_input = tf.reshape(series_input, shape=(CFG['block_size'] // CFG['patch_size'], -1))\n\n        # Create target\n        series_target = series[:, :, 3:]\n        series_target = tf.transpose(series_target, perm=[0, 2, 1])\n        series_target = tf.reduce_max(series_target, axis=-1)\n        series_target = tf.cast(series_target, tf.int64)\n\n        RAM[(Id, Count)] = (series_input, series_target)\n        \nval_subjects = ['07285e', '220a17', '54ee6e', '312788', '24a59d', '4bb5d0', '48fd62', '79011a', '7688c1']\n\ntrain_ids = tdcsfog_metadata[tdcsfog_metadata['Subject'].apply(lambda x: x not in val_subjects)].index.tolist()\nval_ids = tdcsfog_metadata[tdcsfog_metadata['Subject'].apply(lambda x: x in val_subjects)].index.tolist()\n\ntrain_blocks_descriptions = blocks_descriptions[blocks_descriptions['Id'].apply(lambda x: x in train_ids)]\n\nRAM = {} \n\nwrite_to_ram(train_blocks_descriptions)\n\nprint(f'\\n[Train ids] {len(train_ids)} [Val ids] {len(val_ids)} ({100*len(val_ids)/(len(train_ids)+len(val_ids)):.1f})')\nprint(f'[Train blocks] {len(train_blocks_descriptions )}\\n')","metadata":{"papermill":{"duration":5.544187,"end_time":"2023-06-13T03:03:26.198708","exception":false,"start_time":"2023-06-13T03:03:20.654521","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-01T09:24:15.333688Z","iopub.execute_input":"2024-04-01T09:24:15.334047Z","iopub.status.idle":"2024-04-01T09:24:21.433823Z","shell.execute_reply.started":"2024-04-01T09:24:15.334019Z","shell.execute_reply":"2024-04-01T09:24:21.432908Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nCreate a random train dataset from train_blocks_descriptions DataFrame\n'''\n\ndef read(row):\n    \n    def read_from_ram(Id, Count):  \n        series_inputs, series_targets = RAM[(Id.numpy().decode('utf-8'), Count.numpy())]\n        series_targets = series_targets.numpy().astype(np.float32)\n        \n        return series_inputs, series_targets\n\n    [series_input, series_target] = tf.py_function(read_from_ram, [row['Id'], row['Count']], [tf.float32, tf.float32])\n    series_input.set_shape(shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3))\n    series_target.set_shape(shape=(CFG['block_size'] // CFG['patch_size'], 5))\n    \n    return series_input, series_target\n\ngroups = [group.aggregate(dict, axis=1).tolist() for Id, group in train_blocks_descriptions.groupby('Id')]\nrandom.shuffle(groups)\ngroups = cycle(groups)\n\ndataset, iterator = [], 0\nwhile len(dataset) <= 500000:\n    group = next(groups)\n    sample = random.choice(group)\n    dataset.append(sample)\n    iterator += 1\n    \ndataset = tf.data.Dataset.from_tensor_slices(dict(pd.DataFrame(dataset)))\ndataset = dataset.map(read).batch(TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE, drop_remainder=True)","metadata":{"papermill":{"duration":3.303548,"end_time":"2023-06-13T03:03:29.524798","exception":false,"start_time":"2023-06-13T03:03:26.221250","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-01T09:24:21.489766Z","iopub.execute_input":"2024-04-01T09:24:21.490061Z","iopub.status.idle":"2024-04-01T09:24:25.384107Z","shell.execute_reply.started":"2024-04-01T09:24:21.490035Z","shell.execute_reply":"2024-04-01T09:24:25.383307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Train**","metadata":{"papermill":{"duration":0.021969,"end_time":"2023-06-13T03:03:29.569296","exception":false,"start_time":"2023-06-13T03:03:29.547327","status":"completed"},"tags":[]}},{"cell_type":"code","source":"'''\nloss_function args exp\n\nreal is a tensor with the shape (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], 5) where the last axis means:\n0 - StartHesitation \n1 - Turn\n2 - Walking\n3 - Valid\n4 - Mask\n\noutput is a tensor with the shape (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], 3) where the last axis means:\n0 - StartHesitation predicted\n1 - Turn predicted\n2 - Walking predicted\n\n'''\n\nce = tf.keras.losses.BinaryCrossentropy(reduction='none')\n\ndef loss_function(real, output, name='loss_function'):\n    loss = ce(tf.expand_dims(real[:, :, 0:3], axis=-1), tf.expand_dims(output, axis=-1)) # Example shape (32, 864, 3)\n    \n    mask = tf.math.multiply(real[:, :, 3], real[:, :, 4]) # Example shape (32, 864)\n    mask = tf.cast(mask, dtype=loss.dtype)\n    mask = tf.expand_dims(mask, axis=-1) # Example shape (32, 864, 1)\n    mask = tf.tile(mask, multiples=[1, 1, 3]) # Example shape (32, 864, 3)\n    loss *= mask # Example shape (32, 864, 3)\n    \n    return tf.reduce_sum(loss) / tf.reduce_sum(mask)\n\n'''\nSimple learning rate schedule with warm up steps\n\n'''\n        \nclass CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n    def __init__(self, initial_lr, warmup_steps=1):\n        super(CustomSchedule, self).__init__()\n\n        self.initial_lr = tf.cast(initial_lr, tf.float32)\n        self.warmup_steps = tf.cast(warmup_steps, tf.float32)\n\n    def __call__(self, step):\n        step = tf.cast(step, tf.float32)\n        return tf.math.minimum(self.initial_lr, self.initial_lr * (step/self.warmup_steps))  \n\n\n    \n\n'''\nPredictionFnCallback is used for:\n1. Loading validation data\n2. FOGModel data preparation\n3. Prediction\n4. Scoring and save\n\n'''\n\nclass PredictionFnCallback(tf.keras.callbacks.Callback):\n    \n    def __init__(self, model=None, verbose=0):\n        \n        if not model is None: self.model = model\n        self.verbose = verbose\n        \n        def init(Id, path):\n            series = pd.read_csv(path).reset_index(drop=True)\n            series['Id'] = Id\n            series['AccV'] = sample_normalize(series['AccV'].values)\n            series['AccML'] = sample_normalize(series['AccML'].values)\n            series['AccAP'] = sample_normalize(series['AccAP'].values)\n            series['Event'] = series[['StartHesitation', 'Turn', 'Walking']].aggregate('max', axis=1)\n            \n            series_blocks=[]\n            for block in get_blocks(series, ['AccV', 'AccML', 'AccAP']): # Example shape (15552, 3)\n                values = tf.reshape(block['values'], shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size'], 3)) # Example shape (864, 18, 3)\n                values = tf.reshape(values, shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3)) # Example shape (864, 54)\n                values = tf.expand_dims(values, axis=0) # Example shape (1, 864, 54)\n                \n                \"\"\"\n                Creating the val DATA\n                \"\"\"\n                \n                self.blocks.append(values)\n                series_blocks.append((self.blocks_counter, block['begin'], block['end']))\n                self.blocks_counter += 1\n            \n            description = {}\n            description['series'] = series\n            description['series_blocks'] = series_blocks\n            self.descriptions.append(description)\n            \n        self.descriptions = [] # Blocks metadata\n        self.blocks = [] # Validation data blocks\n        self.blocks_counter=0 # Blocks counter\n        \n        tsfog_ids = val_ids\n        tsfog_paths = [f'/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/tdcsfog/{tsfog_id}.csv' for tsfog_id in tsfog_ids]\n        for tsfog_id, tsfog_path in tqdm(zip(tsfog_ids, tsfog_paths), total=len(tsfog_ids), desc='PredictionFnCallback Initialization', disable=1-verbose): \n            init(tsfog_id, tsfog_path)\n        \n        self.blocks = tf.concat(self.blocks, axis=0) # Example shape (self.blocks_counter, 864, 54) # still no labels\n        \n        '''\n        self.blocks is padded so that the final length is divisible by inference batch size for error-free operation of model.predict function\n        Padded values have no effect on the predictions\n        '''\n        \n        self.blocks = tf.pad(self.blocks, \n                             paddings=[[0, math.ceil(self.blocks_counter / GPU_BATCH_SIZE)*(GPU_BATCH_SIZE)-self.blocks_counter], \n                                                    [0, 0], \n                                                    [0, 0],\n                                      ]) # Example shape (self.blocks_counter+pad_value, 864, 54)\n        \n        print(f'\\n[EventPredictionFnCallback Initialization] [Series] {len(self.descriptions)} [Blocks] {self.blocks_counter}\\n')\n    \n    def prediction(self):\n        predictions = model.predict(self.blocks, batch_size=TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE, verbose=self.verbose) # Example shape (self.blocks_counter+pad_value, 864, 3)\n        predictions = tf.expand_dims(predictions, axis=-1) # Example shape (self.blocks_counter+pad_value, 864, 3, 1)\n        predictions = tf.transpose(predictions, perm=[0, 1, 3, 2]) # Example shape (self.blocks_counter+pad_value, 864, 1, 3)\n        predictions = tf.tile(predictions, multiples=[1, 1, CFG['patch_size'], 1]) # Example shape (self.blocks_counter+pad_value, 864, 18, 3)\n        predictions = tf.reshape(predictions, shape=(predictions.shape[0], predictions.shape[1]*predictions.shape[2], 3)) # Example shape (self.blocks_counter+pad_value, 15552, 3)\n        predictions = predictions.numpy()\n        \n        \n        '''\n        The following function aggregates predictions blocks and creates dataframes with StartHesitation_prediction, Turn_prediction, Walking_prediction columns.\n        '''\n        \n        def create_target(description):\n            series, series_blocks = description['series'].copy(), description['series_blocks']\n            \n            values = np.zeros((series_blocks[-1][2], 4))\n            for series_block in series_blocks:\n                i, begin, end = series_block\n                values[begin:end, 0:3] += predictions[i]\n                values[begin:end, 3] += 1\n            \n            values = values[:len(series)]\n            \n            \n            series['StartHesitation_prediction'] = values[:, 0] / values[:, 3]\n            series['Turn_prediction'] = values[:, 1] / values[:, 3]\n            series['Walking_prediction'] = values[:, 2] / values[:, 3]\n            series['Prediction_count'] = values[:, 3]\n            series['Event_prediction'] = series[['StartHesitation_prediction', 'Turn_prediction', 'Walking_prediction']].aggregate('max', axis=1)\n            \n            return series\n            \n        targets = Parallel(n_jobs=-1)(delayed(create_target)(self.descriptions[i]) for i in tqdm(range(len(self.descriptions)), disable=1-self.verbose))\n        targets = pd.concat(targets)\n        \n        return targets\n    \n    def on_epoch_end(self, epoch, logs=None):\n        scores=[]\n        scores.append(f'{(epoch+1):03d}')\n        \n        loss = logs['loss'] if epoch >= 0 else 1.0\n        \n        targets = self.prediction()\n        \n        # Score\n        \n        StartHesitation_mAP = average_precision_score(targets['StartHesitation'], targets['StartHesitation_prediction'])\n        Turn_mAP = average_precision_score(targets['Turn'], targets['Turn_prediction'])\n        Walking_mAP = average_precision_score(targets['Walking'], targets['Walking_prediction'])\n        mAP = (Walking_mAP+Turn_mAP+StartHesitation_mAP)/3\n\n        # print(f'\\n\\n[0] StartHesitation mAP - {StartHesitation_mAP:.3f} Turn mAP - {Turn_mAP:.3f} Walking mAP - {Walking_mAP:.3f} mAP - {mAP:.3f}')\n        \n        scores.append(f'{mAP:.3f}')\n        \n        # Score\n        \n        Event_mAP = average_precision_score(targets['Event'], targets['Event_prediction'])\n        \n        # print(f'[1] Event mAP - {Event_mAP:.3f}\\n')\n        \n        scores.append(f'{Event_mAP:.3f}')\n        \n        # Save\n        \n        scores.append(f'{loss:.4f}')\n        \n        save_name = '_'.join(scores)\n        save_path = f'/kaggle/working/{save_name}_model.h5'\n        self.model.save_weights(save_path)\n        \n","metadata":{"papermill":{"duration":216.819576,"end_time":"2023-06-13T03:07:06.410804","exception":false,"start_time":"2023-06-13T03:03:29.591228","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-01T09:24:25.385430Z","iopub.execute_input":"2024-04-01T09:24:25.385727Z","iopub.status.idle":"2024-04-01T09:24:25.420518Z","shell.execute_reply.started":"2024-04-01T09:24:25.385700Z","shell.execute_reply":"2024-04-01T09:24:25.419429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Search models**","metadata":{"papermill":{"duration":0.031954,"end_time":"2023-06-13T03:07:06.479630","exception":false,"start_time":"2023-06-13T03:07:06.447676","status":"completed"},"tags":[]}},{"cell_type":"code","source":"for batch in dataset.take(1):\n    first_batch = batch\n    break  # Exit after the first batch\n    \nfirst_batch[1][0]","metadata":{"execution":{"iopub.status.busy":"2024-04-01T09:57:58.296372Z","iopub.execute_input":"2024-04-01T09:57:58.297115Z","iopub.status.idle":"2024-04-01T09:57:58.361805Z","shell.execute_reply.started":"2024-04-01T09:57:58.297078Z","shell.execute_reply":"2024-04-01T09:57:58.360987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nTraining\n'''\n\nLEARNING_RATE = 0.01/38\nSTEPS_PER_EPOCH = 64\nWARMUP_STEPS = 64\nEPOCHS = 19\nWEIGHTS = ''\n\nif CFG['TPU']:\n    with tpu_strategy.scope():\n        model = FOGModel()\n        model.build(input_shape=(GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3))\n        if len(WEIGHTS): model.load_weights(WEIGHTS)\n        model.compile(loss=loss_function, optimizer=tf.keras.optimizers.Adam(learning_rate=CustomSchedule(LEARNING_RATE, WARMUP_STEPS), beta_1=0.9, beta_2=0.98, epsilon=1e-9))\n        !rm -r /kaggle/working/*\n        model.fit(dataset, epochs=EPOCHS, steps_per_epoch=STEPS_PER_EPOCH, callbacks=[PredictionFnCallback()])\n        \nelse:\n    \n    model = FOGModel()\n    model.build(input_shape=(GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3))\n    if len(WEIGHTS): model.load_weights(WEIGHTS)\n    model.compile(loss=loss_function, optimizer=tf.keras.optimizers.Adam(learning_rate=CustomSchedule(LEARNING_RATE, WARMUP_STEPS), beta_1=0.9, beta_2=0.98, epsilon=1e-9))\n    !rm -r /kaggle/working/*\n    model.fit(dataset, epochs=EPOCHS, steps_per_epoch=STEPS_PER_EPOCH, callbacks=[PredictionFnCallback()])","metadata":{"execution":{"iopub.status.busy":"2024-04-01T09:24:25.421949Z","iopub.execute_input":"2024-04-01T09:24:25.422237Z","iopub.status.idle":"2024-04-01T09:45:26.323422Z","shell.execute_reply.started":"2024-04-01T09:24:25.422213Z","shell.execute_reply":"2024-04-01T09:45:26.321898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nSearch for saved models in the working directory and sort them\n'''\n\nmodels = []\nfor fname in os.listdir('/kaggle/working/'):\n    if 'model.h5' in fname:\n        m = {}\n        m['Path'] = '/kaggle/working/' + fname\n        for i, elem in enumerate(fname.split('_')): \n            try:\n                m[i+1] = float(elem)\n            except:\n                m[i+1] = elem\n        models.append(m)\n\nif len(models): \n    models = pd.DataFrame(models)\n    plot(models.sort_values(1)[2].values)\n    models = models.sort_values(2, ascending=False)\n    display(models.head(15))","metadata":{"papermill":{"duration":0.33069,"end_time":"2023-06-13T03:07:06.843066","exception":false,"start_time":"2023-06-13T03:07:06.512376","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-01T09:45:26.324521Z","iopub.status.idle":"2024-04-01T09:45:26.324881Z","shell.execute_reply.started":"2024-04-01T09:45:26.324711Z","shell.execute_reply":"2024-04-01T09:45:26.324727Z"},"trusted":true},"execution_count":null,"outputs":[]}]}