{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Imports**","metadata":{"papermill":{"duration":0.012455,"end_time":"2023-06-08T03:06:00.752051","exception":false,"start_time":"2023-06-08T03:06:00.739596","status":"completed"},"tags":[]}},{"cell_type":"code","source":"'''\nThe submission code consists of 3 parts: \n1) Tdcsfog. Here we define the tdcsfog model, then load tdcsfog test data and predict targets.\n2) Defog. Here we define the defog model, then load defog test data and predict targets.\n3) Submission. Here the predicted values are collected, uniformly averaged (if several models were used), then the submission.csv is created.\n\nTdcsfog model and defog model differ in parameters and pretrained weights.\n\n'''\n\nimport os \nimport math\n\nimport numpy as np \nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed","metadata":{"papermill":{"duration":7.424407,"end_time":"2023-06-08T03:06:08.187487","exception":false,"start_time":"2023-06-08T03:06:00.763080","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Load**","metadata":{"papermill":{"duration":0.010723,"end_time":"2023-06-08T03:06:08.209881","exception":false,"start_time":"2023-06-08T03:06:08.199158","status":"completed"},"tags":[]}},{"cell_type":"code","source":"all_submissions = [] # All predicted values are put here\n\ntsfog_ids = [fname.split('.')[0] for fname in os.listdir('/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/tdcsfog')] # Get tdcsfog ids from test directory\ndefog_ids = [fname.split('.')[0] for fname in os.listdir('/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/defog')] # Get defog ids from test directory","metadata":{"papermill":{"duration":0.030364,"end_time":"2023-06-08T03:06:08.251059","exception":false,"start_time":"2023-06-08T03:06:08.220695","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Part 1. TDCSFOG**","metadata":{"papermill":{"duration":0.010563,"end_time":"2023-06-08T03:06:08.272556","exception":false,"start_time":"2023-06-08T03:06:08.261993","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Configuration**","metadata":{"papermill":{"duration":0.010664,"end_time":"2023-06-08T03:06:08.294076","exception":false,"start_time":"2023-06-08T03:06:08.283412","status":"completed"},"tags":[]}},{"cell_type":"code","source":"CFG = {'TPU': 0,\n       'block_size': 15552, \n       'block_stride': 15552//16,\n       'patch_size': 18, \n       \n       'fog_model_dim': 320,\n       'fog_model_num_heads': 6,\n       'fog_model_num_encoder_layers': 5,\n       'fog_model_num_lstm_layers': 2,\n       'fog_model_first_dropout': 0.1,\n       'fog_model_encoder_dropout': 0.1,\n       'fog_model_mha_dropout': 0.0,\n      }\n\nassert CFG['block_size'] % CFG['patch_size'] == 0\nassert CFG['block_size'] % CFG['block_stride'] == 0\n\n'''\nMean-std normalization function. \nExample input: shape (5000), dtype np.float32\nExample output: shape (5000), dtype np.float32\n\nUsed to normalize AccV, AccML, AccAP values.\n\n'''\n\ndef sample_normalize(sample):\n    mean = tf.math.reduce_mean(sample)\n    std = tf.math.reduce_std(sample)\n    sample = tf.math.divide_no_nan(sample-mean, std)\n    \n    return sample.numpy()\n\n'''\nFunction for splitting a series into blocks. Blocks can overlap. \nHow the function works:\nSuppose we have a series with AccV, AccML, AccAP columns and len of 50000, that is (50000, 3). \nFirst, the series is padded so that the final length is divisible by CFG['block_size'] = 15552. Now the series shape is (62208, 3).\nThen we get blocks: first block is series[0:15552, :], second block is series[972:16524, :], ... , last block is series[46656:62208, :].\n\n'''\n\ndef get_blocks(series, columns):\n    series = series.copy()\n    series = series[columns]\n    series = series.values\n    series = series.astype(np.float32)\n    \n    block_count = math.ceil(len(series) / CFG['block_size'])\n    \n    series = np.pad(series, pad_width=[[0, block_count*CFG['block_size']-len(series)], [0, 0]])\n    \n    block_begins = list(range(0, len(series), CFG['block_stride']))\n    block_begins = [x for x in block_begins if x+CFG['block_size'] <= len(series)]\n    \n    blocks = []\n    for begin in block_begins:\n        values = series[begin:begin+CFG['block_size']]\n        blocks.append({'begin': begin,\n                       'end': begin+CFG['block_size'],\n                       'values': values})\n    \n    return blocks\n\n'''\nTrain and inference batch size\n\n'''\n\nGPU_BATCH_SIZE = 4\nTPU_BATCH_SIZE = GPU_BATCH_SIZE*8\n\n'''\nThe transformer encoder layer\nFor more details, see https://arxiv.org/pdf/1706.03762.pdf [Attention Is All You Need]\n\n'''\n\nclass EncoderLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super().__init__()\n        \n        self.mha = tf.keras.layers.MultiHeadAttention(num_heads=CFG['fog_model_num_heads'], key_dim=CFG['fog_model_dim'], dropout=CFG['fog_model_mha_dropout'])\n        \n        self.add = tf.keras.layers.Add()\n        \n        self.layernorm = tf.keras.layers.LayerNormalization()\n        \n        self.seq = tf.keras.Sequential([tf.keras.layers.Dense(CFG['fog_model_dim'], activation='relu'), \n                                        tf.keras.layers.Dropout(CFG['fog_model_encoder_dropout']), \n                                        tf.keras.layers.Dense(CFG['fog_model_dim']), \n                                        tf.keras.layers.Dropout(CFG['fog_model_encoder_dropout']),\n                                       ])\n        \n    def call(self, x):\n        attn_output = self.mha(query=x, key=x, value=x)\n        x = self.add([x, attn_output])\n        x = self.layernorm(x)\n        x = self.add([x, self.seq(x)])\n        x = self.layernorm(x)\n        \n        return x\n    \n'''\nFOGEncoder is a combination of transformer encoder (D=320, H=6, L=5) and two BidirectionalLSTM layers\n\n'''\n\nclass FOGEncoder(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.first_linear = tf.keras.layers.Dense(CFG['fog_model_dim'])\n        \n        self.add = tf.keras.layers.Add()\n        \n        self.first_dropout = tf.keras.layers.Dropout(CFG['fog_model_first_dropout'])\n        \n        self.enc_layers = [EncoderLayer() for _ in range(CFG['fog_model_num_encoder_layers'])]\n        \n        self.lstm_layers = [tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(CFG['fog_model_dim'], return_sequences=True)) for _ in range(CFG['fog_model_num_lstm_layers'])]\n        \n        self.sequence_len = CFG['block_size'] // CFG['patch_size']\n        self.pos_encoding = tf.Variable(initial_value=tf.random.normal(shape=(1, self.sequence_len, CFG['fog_model_dim']), stddev=0.02), trainable=True)\n        \n    def call(self, x, training=None): # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (4, 864, 54)\n        x = x / 25.0 # Normalization attempt in the segment [-1, 1]\n        x = self.first_linear(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']), Example shape (4, 864, 320)\n          \n        if training: # augmentation by randomly roll of the position encoding tensor\n            random_pos_encoding = tf.roll(tf.tile(self.pos_encoding, multiples=[GPU_BATCH_SIZE, 1, 1]), \n                                          shift=tf.random.uniform(shape=(GPU_BATCH_SIZE,), minval=-self.sequence_len, maxval=0, dtype=tf.int32),\n                                          axis=GPU_BATCH_SIZE * [1],\n                                          )\n            x = self.add([x, random_pos_encoding])\n        \n        else: # without augmentation \n            x = self.add([x, tf.tile(self.pos_encoding, multiples=[GPU_BATCH_SIZE, 1, 1])])\n            \n        x = self.first_dropout(x)\n        \n        for i in range(CFG['fog_model_num_encoder_layers']): x = self.enc_layers[i](x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']), Example shape (4, 864, 320)\n        for i in range(CFG['fog_model_num_lstm_layers']): x = self.lstm_layers[i](x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']*2), Example shape (4, 864, 640)\n            \n        return x\n    \nclass FOGModel(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.encoder = FOGEncoder()\n        self.last_linear = tf.keras.layers.Dense(3) \n        \n    def call(self, x): # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (4, 864, 54)\n        x = self.encoder(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']*2), Example shape (4, 864, 640)\n        x = self.last_linear(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], 3), Example shape (4, 864, 3)\n        x = tf.nn.sigmoid(x) # Sigmoid activation\n        \n        return x\n    \nWEIGHTS = '/kaggle/input/parkinsons-freezing-submission-models/044_0.609_0.911_0.0481_model.h5' # TDCSFOG weights\n    \nmodel = FOGModel()\nmodel.build(input_shape=(GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3))\nif len(WEIGHTS): model.load_weights(WEIGHTS)","metadata":{"papermill":{"duration":6.40324,"end_time":"2023-06-08T03:06:14.708156","exception":false,"start_time":"2023-06-08T03:06:08.304916","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Prediction**","metadata":{"papermill":{"duration":0.011323,"end_time":"2023-06-08T03:06:14.731077","exception":false,"start_time":"2023-06-08T03:06:14.719754","status":"completed"},"tags":[]}},{"cell_type":"code","source":"'''\nPredictionFnCallback is used for:\n1. Loading test data\n2. FOGModel data preparation\n3. Prediction\n\n'''\n\nclass PredictionFnCallback(tf.keras.callbacks.Callback):\n    \n    def __init__(self, prediction_ids, model=None, verbose=0):\n        \n        if not model is None: self.model = model\n        self.verbose = verbose\n         \n        def init(Id, path):\n            series = pd.read_csv(path).reset_index(drop=True)\n            series['Id'] = Id\n            series['AccV'] = sample_normalize(series['AccV'].values)\n            series['AccML'] = sample_normalize(series['AccML'].values)\n            series['AccAP'] = sample_normalize(series['AccAP'].values)\n            \n            series_blocks=[]\n            for block in get_blocks(series, ['AccV', 'AccML', 'AccAP']): # Example shape (15552, 3)\n                values = tf.reshape(block['values'], shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size'], 3)) # Example shape (864, 18, 3)\n                values = tf.reshape(values, shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3)) # Example shape (864, 54)\n                values = tf.expand_dims(values, axis=0) # Example shape (1, 864, 54)\n                \n                self.blocks.append(values)\n                series_blocks.append((self.blocks_counter, block['begin'], block['end']))\n                self.blocks_counter += 1\n            \n            description = {}\n            description['series'] = series\n            description['series_blocks'] = series_blocks\n            self.descriptions.append(description)\n            \n        self.descriptions = [] # Blocks metadata\n        self.blocks = [] # Test data blocks\n        self.blocks_counter=0 # Blocks counter\n        \n        tsfog_ids = prediction_ids\n        tsfog_paths = [f'/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/tdcsfog/{tsfog_id}.csv' for tsfog_id in tsfog_ids]\n        for tsfog_id, tsfog_path in tqdm(zip(tsfog_ids, tsfog_paths), total=len(tsfog_ids), desc='PredictionFnCallback Initialization', disable=1-verbose): \n            init(tsfog_id, tsfog_path)\n            \n        self.blocks = tf.concat(self.blocks, axis=0) # Example shape (self.blocks_counter, 864, 54)\n        \n        '''\n        self.blocks is padded so that the final length is divisible by inference batch size for error-free operation of model.predict function\n        Padded values have no effect on the predictions\n        \n        '''\n        \n        self.blocks = tf.pad(self.blocks, \n                             paddings=[[0, math.ceil(self.blocks_counter / (TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE))*(TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE)-self.blocks_counter], \n                                                    [0, 0], \n                                                    [0, 0],\n                                      ]) # Example shape (self.blocks_counter+pad_value, 864, 54)\n        \n        print(f'\\n[EventPredictionFnCallback Initialization] [Series] {len(self.descriptions)} [Blocks] {self.blocks_counter}\\n')\n    \n    def prediction(self):\n        predictions = model.predict(self.blocks, batch_size=TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE, verbose=self.verbose) # Example shape (self.blocks_counter+pad_value, 864, 3)\n        predictions = tf.expand_dims(predictions, axis=-1) # Example shape (self.blocks_counter+pad_value, 864, 3, 1)\n        predictions = tf.transpose(predictions, perm=[0, 1, 3, 2]) # Example shape (self.blocks_counter+pad_value, 864, 1, 3)\n        predictions = tf.tile(predictions, multiples=[1, 1, CFG['patch_size'], 1]) # Example shape (self.blocks_counter+pad_value, 864, 18, 3)\n        predictions = tf.reshape(predictions, shape=(predictions.shape[0], predictions.shape[1]*predictions.shape[2], 3)) # Example shape (self.blocks_counter+pad_value, 15552, 3)\n        predictions = predictions.numpy()\n        \n        '''\n        The following function aggregates predictions blocks and creates dataframes with StartHesitation_prediction, Turn_prediction, Walking_prediction columns.\n        \n        '''\n        \n        def create_target(description):\n            series, series_blocks = description['series'].copy(), description['series_blocks']\n            \n            values = np.zeros((series_blocks[-1][2], 4))\n            for series_block in series_blocks:\n                i, begin, end = series_block\n                values[begin:end, 0:3] += predictions[i]\n                values[begin:end, 3] += 1\n\n            values = values[:len(series)]\n            \n            series['StartHesitation_prediction'] = values[:, 0] / values[:, 3]\n            series['Turn_prediction'] = values[:, 1] / values[:, 3]\n            series['Walking_prediction'] = values[:, 2] / values[:, 3]\n            series['Prediction_count'] = values[:, 3]\n            series['Event_prediction'] = series[['StartHesitation_prediction', 'Turn_prediction', 'Walking_prediction']].aggregate('max', axis=1)\n            \n            return series\n            \n        targets = Parallel(n_jobs=-1)(delayed(create_target)(self.descriptions[i]) for i in tqdm(range(len(self.descriptions)), disable=1-self.verbose))\n        targets = pd.concat(targets)\n        \n        return targets\n    \n\n'''\nPrediction\n        \n'''\n\nfor Id in tsfog_ids:\n    targets = PredictionFnCallback(prediction_ids=[Id], model=model).prediction()\n    submission = pd.DataFrame({'Id': (targets['Id'].values + '_' + targets['Time'].astype('str')).values,\n                               'StartHesitation': targets['StartHesitation_prediction'].values,\n                               'Turn': targets['Turn_prediction'].values,\n                               'Walking': targets['Walking_prediction'].values,\n                              })\n    \n    all_submissions.append(submission)","metadata":{"papermill":{"duration":6.74492,"end_time":"2023-06-08T03:06:21.487080","exception":false,"start_time":"2023-06-08T03:06:14.742160","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Configuration**","metadata":{"papermill":{"duration":0.011392,"end_time":"2023-06-08T03:06:21.511055","exception":false,"start_time":"2023-06-08T03:06:21.499663","status":"completed"},"tags":[]}},{"cell_type":"code","source":"CFG = {'TPU': 0,\n       'block_size': 15552, \n       'block_stride': 15552//16,\n       'patch_size': 18, \n       'batch_size': 16,\n      }\n\nassert CFG['block_size'] % CFG['patch_size'] == 0\nassert CFG['block_size'] % CFG['block_stride'] == 0\n\n'''\nMean-std normalization function. \nExample input: shape (5000), dtype np.float32\nExample output: shape (5000), dtype np.float32\n\nUsed to normalize AccV, AccML, AccAP values.\n\n'''\n\ndef sample_normalize(sample):\n    mean = tf.math.reduce_mean(sample)\n    std = tf.math.reduce_std(sample)\n    sample = tf.math.divide_no_nan(sample-mean, std)\n    \n    return sample.numpy()\n\n'''\nFunction for splitting a series into blocks. Blocks can overlap. \nHow the function works:\nSuppose we have a series with AccV, AccML, AccAP columns and len of 50000, that is (50000, 3). \nFirst, the series is padded so that the final length is divisible by CFG['block_size'] = 15552. Now the series shape is (62208, 3).\nThen we get blocks: first block is series[0:15552, :], second block is series[972:16524, :], ... , last block is series[46656:62208, :].\n\n'''\n\ndef get_blocks(series, columns):\n    series = series.copy()\n    series = series[columns]\n    series = series.values\n    series = series.astype(np.float32)\n    \n    block_count = math.ceil(len(series) / CFG['block_size'])\n    \n    series = np.pad(series, pad_width=[[0, block_count*CFG['block_size']-len(series)], [0, 0]])\n    \n    block_begins = list(range(0, len(series), CFG['block_stride']))\n    block_begins = [x for x in block_begins if x+CFG['block_size'] <= len(series)]\n    \n    blocks = []\n    for begin in block_begins:\n        values = series[begin:begin+CFG['block_size']]\n        blocks.append({'begin': begin,\n                       'end': begin+CFG['block_size'],\n                       'values': values})\n    \n    return blocks\n\n'''\nThe transformer encoder layer\nFor more details, see https://arxiv.org/pdf/1706.03762.pdf [Attention Is All You Need]\n\n'''\n\nclass EncoderLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super().__init__()\n        \n        self.mha = tf.keras.layers.MultiHeadAttention(num_heads=6, key_dim=256)\n        self.add = tf.keras.layers.Add()\n        self.layernorm = tf.keras.layers.LayerNormalization()\n        self.seq = tf.keras.Sequential([tf.keras.layers.Dense(256, activation='relu'), tf.keras.layers.Dropout(0.1), tf.keras.layers.Dense(256), tf.keras.layers.Dropout(0.1)])\n        \n    def call(self, x):\n        attn_output, attn_scores = self.mha(query=x, key=x, value=x, \n                                            return_attention_scores=True)\n        \n        self.attn_scores = attn_scores\n        \n        x = self.add([x, attn_output])\n        x = self.layernorm(x)\n        x = self.add([x, self.seq(x)])\n        x = self.layernorm(x)\n        \n        return x\n    \n'''\nFOGEncoder is a combination of transformer encoder (D=256, H=6, L=3) and two BidirectionalLSTM layers\n\n'''\n\nclass FOGEncoder(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.num_layers=3\n        self.num_lstm_layers=2\n        self.key_dim=256\n        self.dropout_rate = 0.1\n    \n        self.masking = tf.keras.layers.Masking()\n        self.first_linear = tf.keras.layers.Dense(self.key_dim)\n        self.add = tf.keras.layers.Add()\n        self.first_dropout = tf.keras.layers.Dropout(self.dropout_rate)\n        self.enc_layers = [EncoderLayer() for _ in range(self.num_layers)]\n        self.lstm_layers = [tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(self.key_dim, return_sequences=True)) for _ in range(self.num_lstm_layers)]\n        \n        self.sequence_len = CFG['block_size'] // CFG['patch_size']\n        self.pos_encoding = tf.Variable(initial_value=tf.random.normal(shape=(1, self.sequence_len, self.key_dim), stddev=0.02), trainable=True)\n        \n        \n    \n    def call(self, x, training=None): # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3+1), Example shape (16, 864, 55)\n        x = x / 25.0 # Normalization attempt in the segment [-1, 1]\n        x = self.masking(x) # Masks a padded timesteps from multi head attention and lstm layers\n        x = self.first_linear(x) # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], self.key_dim), Example shape (16, 864, 256)\n        \n        if training: # augmentation by randomly roll of the position encoding tensor\n            random_pos_encoding = tf.roll(tf.tile(self.pos_encoding, multiples=[CFG['batch_size'], 1, 1]), \n                                          shift=tf.random.uniform(shape=(CFG['batch_size'],), minval=-self.sequence_len, maxval=0, dtype=tf.int32),\n                                          axis=CFG['batch_size'] * [1],\n                                          )\n            x = self.add([x, random_pos_encoding])\n        else: # without augmentation\n            x = self.add([x, tf.tile(self.pos_encoding, multiples=[CFG['batch_size'], 1, 1])])\n            \n        x = self.first_dropout(x)\n        \n        for i in range(self.num_layers): x = self.enc_layers[i](x) # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], self.key_dim), Example shape (16, 864, 256)\n        for i in range(self.num_lstm_layers): x = self.lstm_layers[i](x) # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], self.key_dim*2), Example shape (16, 864, 512)\n            \n        return x\n    \nclass FOGModel(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.encoder = FOGEncoder()\n        self.last_linear = tf.keras.layers.Dense(3) \n        \n    def call(self, x): # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3+1), Example shape (16, 864, 55)\n        x = self.encoder(x) # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], self.key_dim*2), Example shape (16, 864, 512)\n        x = self.last_linear(x) # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], 3), Example shape (16, 864, 3)\n        x = tf.nn.sigmoid(x) # Sigmoid activation\n        \n        return x\n    \nmodel = FOGModel()\nmodel.build(input_shape=(CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3+1))\nmodel.load_weights('/kaggle/input/parkinsons-freezing-submission-models/062_0.601_0.913_0.0490_model.h5') # TDCSFOG weights","metadata":{"papermill":{"duration":5.379867,"end_time":"2023-06-08T03:06:26.902216","exception":false,"start_time":"2023-06-08T03:06:21.522349","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Prediction**","metadata":{"papermill":{"duration":0.012024,"end_time":"2023-06-08T03:06:26.926032","exception":false,"start_time":"2023-06-08T03:06:26.914008","status":"completed"},"tags":[]}},{"cell_type":"code","source":"'''\nPredictionFnCallback is used for:\n1. Loading test data\n2. FOGModel data preparation\n3. Prediction\n\n'''\n\nclass PredictionFnCallback(tf.keras.callbacks.Callback):\n    \n    def __init__(self, prediction_ids, model=None, verbose=0):\n        \n        if not model is None: self.model = model\n        self.verbose = verbose\n         \n        def init(Id, path):\n            series = pd.read_csv(path).reset_index(drop=True)\n            series['Id'] = Id\n            series['AccV'] = sample_normalize(series['AccV'].values)\n            series['AccML'] = sample_normalize(series['AccML'].values)\n            series['AccAP'] = sample_normalize(series['AccAP'].values)\n            \n            series_blocks=[]\n            for block in get_blocks(series, ['AccV', 'AccML', 'AccAP']): # Example shape (15552, 3)\n                values = tf.reshape(block['values'], shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size'], 3)) # Example shape (864, 18, 3)\n                values = tf.reshape(values, shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3)) # Example shape (864, 54)\n                values = tf.concat([values, \n                                    tf.constant(0.0, dtype=tf.float32, shape=(CFG['block_size'] // CFG['patch_size'], 1)),\n                                   ], axis=-1) # Example shape (864, 55) (Add a constant to the right. Not used. Has no effect on the model)\n                values = tf.expand_dims(values, axis=0) # Example shape (1, 864, 55)\n                \n                self.blocks.append(values)\n                series_blocks.append((self.blocks_counter, block['begin'], block['end']))\n                self.blocks_counter += 1\n            \n            description = {}\n            description['series'] = series\n            description['series_blocks'] = series_blocks\n            self.descriptions.append(description)\n            \n        self.descriptions = [] # Blocks metadata\n        self.blocks = [] # Test data blocks\n        self.blocks_counter=0 # Blocks counter\n        \n        notype_ids = prediction_ids\n        notype_paths = [f'/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/tdcsfog/{notype_id}.csv' for notype_id in notype_ids]\n        for notype_id, notype_path in tqdm(zip(notype_ids, notype_paths), total=len(notype_ids), desc='PredictionFnCallback Initialization', disable=1-verbose): \n            init(notype_id, notype_path)\n            \n        self.blocks = tf.concat(self.blocks, axis=0) # Example shape (self.blocks_counter, 864, 55)\n        \n        '''\n        self.blocks is padded so that the final length is divisible by inference batch size for error-free operation of model.predict function\n        Padded values have no effect on the predictions\n        \n        '''\n        \n        self.blocks = tf.pad(self.blocks, \n                             paddings=[[0, math.ceil(self.blocks_counter / CFG['batch_size'])*CFG['batch_size']-self.blocks_counter], \n                                                    [0, 0], \n                                                    [0, 0],\n                                      ]) # Example shape (self.blocks_counter+pad_value, 864, 55)\n        \n        print(f'[PredictionFnCallback Initialization] [Series] {len(self.descriptions)} [Blocks] {self.blocks_counter}')\n    \n    def prediction(self):\n        predictions = model.predict(self.blocks, batch_size=CFG['batch_size'], verbose=self.verbose) # Example shape (self.blocks_counter+pad_value, 864, 3)\n        predictions = tf.expand_dims(predictions, axis=-1) # Example shape (self.blocks_counter+pad_value, 864, 3, 1)\n        predictions = tf.transpose(predictions, perm=[0, 1, 3, 2])  # Example shape (self.blocks_counter+pad_value, 864, 1, 3)\n        predictions = tf.tile(predictions, multiples=[1, 1, CFG['patch_size'], 1]) # Example shape (self.blocks_counter+pad_value, 864, 18, 3)\n        predictions = tf.reshape(predictions, shape=(predictions.shape[0], predictions.shape[1]*predictions.shape[2], 3)) # Example shape (self.blocks_counter+pad_value, 15552, 3)\n        predictions = predictions.numpy() \n        \n        '''\n        The following function aggregates predictions blocks and creates dataframes with StartHesitation_prediction, Turn_prediction, Walking_prediction columns.\n        \n        '''\n        \n        def create_target(description):\n            series, series_blocks = description['series'].copy(), description['series_blocks']\n            \n            values = np.zeros((series_blocks[-1][2], 4))\n            for series_block in series_blocks:\n                i, begin, end = series_block\n                values[begin:end, 0:3] += predictions[i]\n                values[begin:end, 3] += 1\n\n            values = values[:len(series)]\n            \n            series['StartHesitation_prediction'] = values[:, 0] / values[:, 3]\n            series['Turn_prediction'] = values[:, 1] / values[:, 3]\n            series['Walking_prediction'] = values[:, 2] / values[:, 3]\n            series['Prediction_count'] = values[:, 3]\n            \n            return series\n            \n        targets = Parallel(n_jobs=-1)(delayed(create_target)(self.descriptions[i]) for i in tqdm(range(len(self.descriptions)), disable=1-self.verbose))\n        targets = pd.concat(targets)\n        \n        return targets\n    \n'''\nPrediction\n        \n'''\n\nfor Id in tsfog_ids:\n    targets = PredictionFnCallback(prediction_ids=[Id], model=model).prediction()\n    submission = pd.DataFrame({'Id': (targets['Id'].values + '_' + targets['Time'].astype('str')).values,\n                               'StartHesitation': targets['StartHesitation_prediction'].values,\n                               'Turn': targets['Turn_prediction'].values,\n                               'Walking': targets['Walking_prediction'].values,\n                              })\n    \n    all_submissions.append(submission)","metadata":{"papermill":{"duration":5.628638,"end_time":"2023-06-08T03:06:32.566200","exception":false,"start_time":"2023-06-08T03:06:26.937562","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Configuration**","metadata":{"papermill":{"duration":0.01127,"end_time":"2023-06-08T03:06:32.589528","exception":false,"start_time":"2023-06-08T03:06:32.578258","status":"completed"},"tags":[]}},{"cell_type":"code","source":"CFG = {'TPU': 0,\n       'block_size': 15552, \n       'block_stride': 15552//16,\n       'patch_size': 18, \n       \n       'fog_model_dim': 320,\n       'fog_model_num_heads': 6,\n       'fog_model_num_encoder_layers': 5,\n       'fog_model_num_lstm_layers': 2,\n       'fog_model_first_dropout': 0.1,\n       'fog_model_encoder_dropout': 0.1,\n       'fog_model_mha_dropout': 0.0,\n      }\n\nassert CFG['block_size'] % CFG['patch_size'] == 0\nassert CFG['block_size'] % CFG['block_stride'] == 0\n\n'''\nMean-std normalization function. \nExample input: shape (5000), dtype np.float32\nExample output: shape (5000), dtype np.float32\n\nUsed to normalize AccV, AccML, AccAP values.\n\n'''\n\ndef sample_normalize(sample):\n    mean = tf.math.reduce_mean(sample)\n    std = tf.math.reduce_std(sample)\n    sample = tf.math.divide_no_nan(sample-mean, std)\n    \n    return sample.numpy()\n\n'''\nFunction for splitting a series into blocks. Blocks can overlap. \nHow the function works:\nSuppose we have a series with AccV, AccML, AccAP columns and len of 50000, that is (50000, 3). \nFirst, the series is padded so that the final length is divisible by CFG['block_size'] = 15552. Now the series shape is (62208, 3).\nThen we get blocks: first block is series[0:15552, :], second block is series[972:16524, :], ... , last block is series[46656:62208, :].\n\n'''\n\ndef get_blocks(series, columns):\n    series = series.copy()\n    series = series[columns]\n    series = series.values\n    series = series.astype(np.float32)\n    \n    block_count = math.ceil(len(series) / CFG['block_size'])\n    \n    series = np.pad(series, pad_width=[[0, block_count*CFG['block_size']-len(series)], [0, 0]])\n    \n    block_begins = list(range(0, len(series), CFG['block_stride']))\n    block_begins = [x for x in block_begins if x+CFG['block_size'] <= len(series)]\n    \n    blocks = []\n    for begin in block_begins:\n        values = series[begin:begin+CFG['block_size']]\n        blocks.append({'begin': begin,\n                       'end': begin+CFG['block_size'],\n                       'values': values})\n    \n    return blocks\n\n'''\nTrain and inference batch size\n\n'''\n\nGPU_BATCH_SIZE = 4\nTPU_BATCH_SIZE = GPU_BATCH_SIZE*8\n\n'''\nThe transformer encoder layer\nFor more details, see https://arxiv.org/pdf/1706.03762.pdf [Attention Is All You Need]\n\n'''\n\nclass EncoderLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super().__init__()\n        \n        self.mha = tf.keras.layers.MultiHeadAttention(num_heads=CFG['fog_model_num_heads'], key_dim=CFG['fog_model_dim'], dropout=CFG['fog_model_mha_dropout'])\n        \n        self.add = tf.keras.layers.Add()\n        \n        self.layernorm = tf.keras.layers.LayerNormalization()\n        \n        self.seq = tf.keras.Sequential([tf.keras.layers.Dense(CFG['fog_model_dim'], activation='relu'), \n                                        tf.keras.layers.Dropout(CFG['fog_model_encoder_dropout']), \n                                        tf.keras.layers.Dense(CFG['fog_model_dim']), \n                                        tf.keras.layers.Dropout(CFG['fog_model_encoder_dropout']),\n                                       ])\n        \n    def call(self, x):\n        attn_output = self.mha(query=x, key=x, value=x)\n        x = self.add([x, attn_output])\n        x = self.layernorm(x)\n        x = self.add([x, self.seq(x)])\n        x = self.layernorm(x)\n        \n        return x\n    \n'''\nFOGEncoder is a combination of transformer encoder (D=320, H=6, L=5) and two BidirectionalLSTM layers\n\n'''\n\nclass FOGEncoder(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.masking = tf.keras.layers.Masking()\n        \n        self.first_linear = tf.keras.layers.Dense(CFG['fog_model_dim'])\n        \n        self.add = tf.keras.layers.Add()\n        \n        self.first_dropout = tf.keras.layers.Dropout(CFG['fog_model_first_dropout'])\n        \n        self.enc_layers = [EncoderLayer() for _ in range(CFG['fog_model_num_encoder_layers'])]\n        \n        self.lstm_layers = [tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(CFG['fog_model_dim'], return_sequences=True)) for _ in range(CFG['fog_model_num_lstm_layers'])]\n        \n        self.sequence_len = CFG['block_size'] // CFG['patch_size']\n        self.pos_encoding = tf.Variable(initial_value=tf.random.normal(shape=(1, self.sequence_len, CFG['fog_model_dim']), stddev=0.02), trainable=True)\n        \n    def call(self, x, training=None): # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (4, 864, 54)\n        x = x / 25.0 # Normalization attempt in the segment [-1, 1]\n        x = self.masking(x) # Masks a padded timesteps from multi head attention and lstm layers\n        x = self.first_linear(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']), Example shape (4, 864, 320)\n          \n        if training: # augmentation by randomly roll of the position encoding tensor\n            random_pos_encoding = tf.roll(tf.tile(self.pos_encoding, multiples=[GPU_BATCH_SIZE, 1, 1]), \n                                          shift=tf.random.uniform(shape=(GPU_BATCH_SIZE,), minval=-self.sequence_len, maxval=0, dtype=tf.int32),\n                                          axis=GPU_BATCH_SIZE * [1],\n                                          )\n            x = self.add([x, random_pos_encoding])\n        \n        else: # without augmentation \n            x = self.add([x, tf.tile(self.pos_encoding, multiples=[GPU_BATCH_SIZE, 1, 1])])\n            \n        x = self.first_dropout(x)\n        \n        km = x._keras_mask # Bug fix (Multi head attention masking does not work on TPU training)\n        del x._keras_mask # Bug fix\n        for i in range(CFG['fog_model_num_encoder_layers']): x = self.enc_layers[i](x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']), Example shape (4, 864, 320)\n        x._keras_mask = km # Bug fix\n        for i in range(CFG['fog_model_num_lstm_layers']): x = self.lstm_layers[i](x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']*2), Example shape (4, 864, 640)\n            \n        return x\n    \nclass FOGModel(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.encoder = FOGEncoder()\n        self.last_linear = tf.keras.layers.Dense(3) \n        \n    def call(self, x): # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (4, 864, 54)\n        x = self.encoder(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']*2), Example shape (4, 864, 640)\n        x = self.last_linear(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], 3), Example shape (4, 864, 3)\n        x = tf.nn.sigmoid(x) # Sigmoid activation\n        \n        return x\n    \nmodel = FOGModel()\nmodel.build(input_shape=(GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3))\nmodel.load_weights('/kaggle/input/parkinsons-freezing-submission-models/025_0.582_0.982_0.0463_model.h5') # TDCSFOG weights","metadata":{"papermill":{"duration":5.462751,"end_time":"2023-06-08T03:06:38.063948","exception":false,"start_time":"2023-06-08T03:06:32.601197","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Prediction**","metadata":{"papermill":{"duration":0.011976,"end_time":"2023-06-08T03:06:38.088102","exception":false,"start_time":"2023-06-08T03:06:38.076126","status":"completed"},"tags":[]}},{"cell_type":"code","source":"'''\nPredictionFnCallback is used for:\n1. Loading test data\n2. FOGModel data preparation\n3. Prediction\n\n'''\n\nclass PredictionFnCallback(tf.keras.callbacks.Callback):\n    \n    def __init__(self, prediction_ids, model=None, verbose=0):\n        \n        if not model is None: self.model = model\n        self.verbose = verbose\n         \n        def init(Id, path):\n            series = pd.read_csv(path).reset_index(drop=True)\n            series['Id'] = Id\n            series['AccV'] = sample_normalize(series['AccV'].values)\n            series['AccML'] = sample_normalize(series['AccML'].values)\n            series['AccAP'] = sample_normalize(series['AccAP'].values)\n            \n            series_blocks=[]\n            for block in get_blocks(series, ['AccV', 'AccML', 'AccAP']): # Example shape (15552, 3)\n                values = tf.reshape(block['values'], shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size'], 3)) # Example shape (864, 18, 3)\n                values = tf.reshape(values, shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3)) # Example shape (864, 54)\n                values = tf.expand_dims(values, axis=0) # Example shape (1, 864, 54)\n                \n                self.blocks.append(values)\n                series_blocks.append((self.blocks_counter, block['begin'], block['end']))\n                self.blocks_counter += 1\n            \n            description = {}\n            description['series'] = series\n            description['series_blocks'] = series_blocks\n            self.descriptions.append(description)\n            \n        self.descriptions = [] # Blocks metadata\n        self.blocks = [] # Test data blocks\n        self.blocks_counter=0 # Blocks counter\n        \n        notype_ids = prediction_ids\n        notype_paths = [f'/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/tdcsfog/{notype_id}.csv' for notype_id in notype_ids]\n        for notype_id, notype_path in tqdm(zip(notype_ids, notype_paths), total=len(notype_ids), desc='PredictionFnCallback Initialization', disable=1-verbose): \n            init(notype_id, notype_path)\n            \n        self.blocks = tf.concat(self.blocks, axis=0) # Example shape (self.blocks_counter, 864, 54)\n        \n        '''\n        self.blocks is padded so that the final length is divisible by inference batch size for error-free operation of model.predict function\n        Padded values have no effect on the predictions\n        \n        '''\n        \n        self.blocks = tf.pad(self.blocks, \n                             paddings=[[0, math.ceil(self.blocks_counter / (TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE))*(TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE)-self.blocks_counter], \n                                                    [0, 0], \n                                                    [0, 0],\n                                      ]) # Example shape (self.blocks_counter+pad_value, 864, 54)\n        \n        print(f'\\n[EventPredictionFnCallback Initialization] [Series] {len(self.descriptions)} [Blocks] {self.blocks_counter}\\n')\n    \n    def prediction(self):\n        predictions = model.predict(self.blocks, batch_size=TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE, verbose=self.verbose) # Example shape (self.blocks_counter+pad_value, 864, 3)\n        predictions = tf.expand_dims(predictions, axis=-1) # Example shape (self.blocks_counter+pad_value, 864, 3, 1)\n        predictions = tf.transpose(predictions, perm=[0, 1, 3, 2]) # Example shape (self.blocks_counter+pad_value, 864, 1, 3)\n        predictions = tf.tile(predictions, multiples=[1, 1, CFG['patch_size'], 1]) # Example shape (self.blocks_counter+pad_value, 864, 18, 3)\n        predictions = tf.reshape(predictions, shape=(predictions.shape[0], predictions.shape[1]*predictions.shape[2], 3)) # Example shape (self.blocks_counter+pad_value, 15552, 3)\n        predictions = predictions.numpy()\n        \n        '''\n        The following function aggregates predictions blocks and creates dataframes with StartHesitation_prediction, Turn_prediction, Walking_prediction columns.\n        \n        '''\n        \n        def create_target(description):\n            series, series_blocks = description['series'].copy(), description['series_blocks']\n            \n            values = np.zeros((series_blocks[-1][2], 4))\n            for series_block in series_blocks:\n                i, begin, end = series_block\n                values[begin:end, 0:3] += predictions[i]\n                values[begin:end, 3] += 1\n\n            values = values[:len(series)]\n            \n            series['StartHesitation_prediction'] = values[:, 0] / values[:, 3]\n            series['Turn_prediction'] = values[:, 1] / values[:, 3]\n            series['Walking_prediction'] = values[:, 2] / values[:, 3]\n            series['Prediction_count'] = values[:, 3]\n            \n            return series\n            \n        targets = Parallel(n_jobs=-1)(delayed(create_target)(self.descriptions[i]) for i in tqdm(range(len(self.descriptions)), disable=1-self.verbose))\n        targets = pd.concat(targets)\n        \n        return targets\n    \n'''\nPrediction\n        \n'''\n\nfor Id in tsfog_ids:\n    targets = PredictionFnCallback(prediction_ids=[Id], model=model).prediction()\n    submission = pd.DataFrame({'Id': (targets['Id'].values + '_' + targets['Time'].astype('str')).values,\n                               'StartHesitation': targets['StartHesitation_prediction'].values,\n                               'Turn': targets['Turn_prediction'].values,\n                               'Walking': targets['Walking_prediction'].values,\n                              })\n    \n    all_submissions.append(submission)","metadata":{"papermill":{"duration":5.741103,"end_time":"2023-06-08T03:06:43.840484","exception":false,"start_time":"2023-06-08T03:06:38.099381","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Configuration**","metadata":{"papermill":{"duration":0.012088,"end_time":"2023-06-08T03:06:43.865376","exception":false,"start_time":"2023-06-08T03:06:43.853288","status":"completed"},"tags":[]}},{"cell_type":"code","source":"CFG = {'TPU': 0,\n       'block_size': 15552, \n       'block_stride': 15552//16,\n       'patch_size': 18, \n       \n       'fog_model_dim': 320,\n       'fog_model_num_heads': 6,\n       'fog_model_num_encoder_layers': 5,\n       'fog_model_num_lstm_layers': 2,\n       'fog_model_first_dropout': 0.1,\n       'fog_model_encoder_dropout': 0.1,\n       'fog_model_mha_dropout': 0.0,\n      }\n\nassert CFG['block_size'] % CFG['patch_size'] == 0\nassert CFG['block_size'] % CFG['block_stride'] == 0\n\n'''\nMean-std normalization function. \nExample input: shape (5000), dtype np.float32\nExample output: shape (5000), dtype np.float32\n\nUsed to normalize AccV, AccML, AccAP values.\n\n'''\n\ndef sample_normalize(sample):\n    mean = tf.math.reduce_mean(sample)\n    std = tf.math.reduce_std(sample)\n    sample = tf.math.divide_no_nan(sample-mean, std)\n    \n    return sample.numpy()\n\n'''\nFunction for splitting a series into blocks. Blocks can overlap. \nHow the function works:\nSuppose we have a series with AccV, AccML, AccAP columns and len of 50000, that is (50000, 3). \nFirst, the series is padded so that the final length is divisible by CFG['block_size'] = 15552. Now the series shape is (62208, 3).\nThen we get blocks: first block is series[0:15552, :], second block is series[972:16524, :], ... , last block is series[46656:62208, :].\n\n'''\n\ndef get_blocks(series, columns):\n    series = series.copy()\n    series = series[columns]\n    series = series.values\n    series = series.astype(np.float32)\n    \n    block_count = math.ceil(len(series) / CFG['block_size'])\n    \n    series = np.pad(series, pad_width=[[0, block_count*CFG['block_size']-len(series)], [0, 0]])\n    \n    block_begins = list(range(0, len(series), CFG['block_stride']))\n    block_begins = [x for x in block_begins if x+CFG['block_size'] <= len(series)]\n    \n    blocks = []\n    for begin in block_begins:\n        values = series[begin:begin+CFG['block_size']]\n        blocks.append({'begin': begin,\n                       'end': begin+CFG['block_size'],\n                       'values': values})\n    \n    return blocks\n\n'''\nTrain and inference batch size\n\n'''\n\nGPU_BATCH_SIZE = 4\nTPU_BATCH_SIZE = GPU_BATCH_SIZE*8\n\n'''\nThe transformer encoder layer\nFor more details, see https://arxiv.org/pdf/1706.03762.pdf [Attention Is All You Need]\n\n'''\n\nclass EncoderLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super().__init__()\n        \n        self.mha = tf.keras.layers.MultiHeadAttention(num_heads=CFG['fog_model_num_heads'], key_dim=CFG['fog_model_dim'], dropout=CFG['fog_model_mha_dropout'])\n        \n        self.add = tf.keras.layers.Add()\n        \n        self.layernorm = tf.keras.layers.LayerNormalization()\n        \n        self.seq = tf.keras.Sequential([tf.keras.layers.Dense(CFG['fog_model_dim'], activation='relu'), \n                                        tf.keras.layers.Dropout(CFG['fog_model_encoder_dropout']), \n                                        tf.keras.layers.Dense(CFG['fog_model_dim']), \n                                        tf.keras.layers.Dropout(CFG['fog_model_encoder_dropout']),\n                                       ])\n        \n    def call(self, x):\n        attn_output = self.mha(query=x, key=x, value=x)\n        x = self.add([x, attn_output])\n        x = self.layernorm(x)\n        x = self.add([x, self.seq(x)])\n        x = self.layernorm(x)\n        \n        return x\n    \n'''\nFOGEncoder is a combination of transformer encoder (D=320, H=6, L=5) and two BidirectionalLSTM layers\n\n'''\n\nclass FOGEncoder(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.first_linear = tf.keras.layers.Dense(CFG['fog_model_dim'])\n        \n        self.add = tf.keras.layers.Add()\n        \n        self.first_dropout = tf.keras.layers.Dropout(CFG['fog_model_first_dropout'])\n        \n        self.enc_layers = [EncoderLayer() for _ in range(CFG['fog_model_num_encoder_layers'])]\n        \n        self.lstm_layers = [tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(CFG['fog_model_dim'], return_sequences=True)) for _ in range(CFG['fog_model_num_lstm_layers'])]\n        \n        self.sequence_len = CFG['block_size'] // CFG['patch_size']\n        self.pos_encoding = tf.Variable(initial_value=tf.random.normal(shape=(1, self.sequence_len, CFG['fog_model_dim']), stddev=0.02), trainable=True)\n        \n    def call(self, x, training=None): # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (4, 864, 54)\n        x = x / 25.0 # Normalization attempt in the segment [-1, 1]\n        x = self.first_linear(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']), Example shape (4, 864, 320)\n          \n        if training: # augmentation by randomly roll of the position encoding tensor\n            random_pos_encoding = tf.roll(tf.tile(self.pos_encoding, multiples=[GPU_BATCH_SIZE, 1, 1]), \n                                          shift=tf.random.uniform(shape=(GPU_BATCH_SIZE,), minval=-self.sequence_len, maxval=0, dtype=tf.int32),\n                                          axis=GPU_BATCH_SIZE * [1],\n                                          )\n            x = self.add([x, random_pos_encoding])\n        \n        else: # without augmentation \n            x = self.add([x, tf.tile(self.pos_encoding, multiples=[GPU_BATCH_SIZE, 1, 1])])\n            \n        x = self.first_dropout(x)\n        \n        for i in range(CFG['fog_model_num_encoder_layers']): x = self.enc_layers[i](x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']), Example shape (4, 864, 320)\n        for i in range(CFG['fog_model_num_lstm_layers']): x = self.lstm_layers[i](x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']*2), Example shape (4, 864, 640)\n            \n        return x\n    \nclass FOGModel(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.encoder = FOGEncoder()\n        self.last_linear = tf.keras.layers.Dense(3) \n        \n    def call(self, x): # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (4, 864, 54)\n        x = self.encoder(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']*2), Example shape (4, 864, 640)\n        x = self.last_linear(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], 3), Example shape (4, 864, 3)\n        x = tf.nn.sigmoid(x) # Sigmoid activation\n        \n        return x\n    \nWEIGHTS = '/kaggle/input/parkinsons-freezing-submission-models/035_0.480_0.899_0.0480_model.h5' # TDCSFOG weights\n    \nmodel = FOGModel()\nmodel.build(input_shape=(GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3))\nif len(WEIGHTS): model.load_weights(WEIGHTS)","metadata":{"papermill":{"duration":3.66713,"end_time":"2023-06-08T03:06:47.544390","exception":false,"start_time":"2023-06-08T03:06:43.877260","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Prediction**","metadata":{"papermill":{"duration":0.012106,"end_time":"2023-06-08T03:06:47.568474","exception":false,"start_time":"2023-06-08T03:06:47.556368","status":"completed"},"tags":[]}},{"cell_type":"code","source":"'''\nPredictionFnCallback is used for:\n1. Loading test data\n2. FOGModel data preparation\n3. Prediction\n\n'''\n\nclass PredictionFnCallback(tf.keras.callbacks.Callback):\n    \n    def __init__(self, prediction_ids, model=None, verbose=0):\n        \n        if not model is None: self.model = model\n        self.verbose = verbose\n         \n        def init(Id, path):\n            series = pd.read_csv(path).reset_index(drop=True)\n            series['Id'] = Id\n            series['AccV'] = sample_normalize(series['AccV'].values)\n            series['AccML'] = sample_normalize(series['AccML'].values)\n            series['AccAP'] = sample_normalize(series['AccAP'].values)\n            \n            series_blocks=[]\n            for block in get_blocks(series, ['AccV', 'AccML', 'AccAP']): # Example shape (15552, 3)\n                values = tf.reshape(block['values'], shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size'], 3)) # Example shape (864, 18, 3)\n                values = tf.reshape(values, shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3)) # Example shape (864, 54)\n                values = tf.expand_dims(values, axis=0) # Example shape (1, 864, 54)\n                \n                self.blocks.append(values)\n                series_blocks.append((self.blocks_counter, block['begin'], block['end']))\n                self.blocks_counter += 1\n            \n            description = {}\n            description['series'] = series\n            description['series_blocks'] = series_blocks\n            self.descriptions.append(description)\n            \n        self.descriptions = [] # Blocks metadata\n        self.blocks = [] # Test data blocks\n        self.blocks_counter=0 # Blocks counter\n        \n        tsfog_ids = prediction_ids\n        tsfog_paths = [f'/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/tdcsfog/{tsfog_id}.csv' for tsfog_id in tsfog_ids]\n        for tsfog_id, tsfog_path in tqdm(zip(tsfog_ids, tsfog_paths), total=len(tsfog_ids), desc='PredictionFnCallback Initialization', disable=1-verbose): \n            init(tsfog_id, tsfog_path)\n            \n        self.blocks = tf.concat(self.blocks, axis=0) # Example shape (self.blocks_counter, 864, 54)\n        \n        '''\n        self.blocks is padded so that the final length is divisible by inference batch size for error-free operation of model.predict function\n        Padded values have no effect on the predictions\n        \n        '''\n        \n        self.blocks = tf.pad(self.blocks, \n                             paddings=[[0, math.ceil(self.blocks_counter / (TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE))*(TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE)-self.blocks_counter], \n                                                    [0, 0], \n                                                    [0, 0],\n                                      ]) # Example shape (self.blocks_counter+pad_value, 864, 54)\n        \n        print(f'\\n[EventPredictionFnCallback Initialization] [Series] {len(self.descriptions)} [Blocks] {self.blocks_counter}\\n')\n    \n    def prediction(self):\n        predictions = model.predict(self.blocks, batch_size=TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE, verbose=self.verbose) # Example shape (self.blocks_counter+pad_value, 864, 3)\n        predictions = tf.expand_dims(predictions, axis=-1) # Example shape (self.blocks_counter+pad_value, 864, 3, 1)\n        predictions = tf.transpose(predictions, perm=[0, 1, 3, 2]) # Example shape (self.blocks_counter+pad_value, 864, 1, 3)\n        predictions = tf.tile(predictions, multiples=[1, 1, CFG['patch_size'], 1]) # Example shape (self.blocks_counter+pad_value, 864, 18, 3)\n        predictions = tf.reshape(predictions, shape=(predictions.shape[0], predictions.shape[1]*predictions.shape[2], 3)) # Example shape (self.blocks_counter+pad_value, 15552, 3)\n        predictions = predictions.numpy()\n        \n        '''\n        The following function aggregates predictions blocks and creates dataframes with StartHesitation_prediction, Turn_prediction, Walking_prediction columns.\n        \n        '''\n        \n        def create_target(description):\n            series, series_blocks = description['series'].copy(), description['series_blocks']\n            \n            values = np.zeros((series_blocks[-1][2], 4))\n            for series_block in series_blocks:\n                i, begin, end = series_block\n                values[begin:end, 0:3] += predictions[i]\n                values[begin:end, 3] += 1\n\n            values = values[:len(series)]\n            \n            series['StartHesitation_prediction'] = values[:, 0] / values[:, 3]\n            series['Turn_prediction'] = values[:, 1] / values[:, 3]\n            series['Walking_prediction'] = values[:, 2] / values[:, 3]\n            series['Prediction_count'] = values[:, 3]\n            series['Event_prediction'] = series[['StartHesitation_prediction', 'Turn_prediction', 'Walking_prediction']].aggregate('max', axis=1)\n            \n            return series\n            \n        targets = Parallel(n_jobs=-1)(delayed(create_target)(self.descriptions[i]) for i in tqdm(range(len(self.descriptions)), disable=1-self.verbose))\n        targets = pd.concat(targets)\n        \n        return targets\n    \n'''\nPrediction\n        \n'''\n\nfor Id in tsfog_ids:\n    targets = PredictionFnCallback(prediction_ids=[Id], model=model).prediction()\n    submission = pd.DataFrame({'Id': (targets['Id'].values + '_' + targets['Time'].astype('str')).values,\n                               'StartHesitation': targets['StartHesitation_prediction'].values,\n                               'Turn': targets['Turn_prediction'].values,\n                               'Walking': targets['Walking_prediction'].values,\n                              })\n    \n    all_submissions.append(submission)","metadata":{"papermill":{"duration":2.18559,"end_time":"2023-06-08T03:06:49.765996","exception":false,"start_time":"2023-06-08T03:06:47.580406","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Part 2. DEFOG**","metadata":{"papermill":{"duration":0.011521,"end_time":"2023-06-08T03:06:49.789916","exception":false,"start_time":"2023-06-08T03:06:49.778395","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Configuration**","metadata":{"papermill":{"duration":0.011445,"end_time":"2023-06-08T03:06:49.813005","exception":false,"start_time":"2023-06-08T03:06:49.801560","status":"completed"},"tags":[]}},{"cell_type":"code","source":"CFG = {'TPU': 0,\n       'block_size': 12096, \n       'block_stride': 12096//16,\n       'patch_size': 14, \n       \n       'fog_model_dim': 320,\n       'fog_model_num_heads': 6,\n       'fog_model_num_encoder_layers': 5,\n       'fog_model_num_lstm_layers': 2,\n       'fog_model_first_dropout': 0.1,\n       'fog_model_encoder_dropout': 0.1,\n       'fog_model_mha_dropout': 0.0,\n      }\n\nassert CFG['block_size'] % CFG['patch_size'] == 0\nassert CFG['block_size'] % CFG['block_stride'] == 0\n\n'''\nMean-std normalization function. \nExample input: shape (5000), dtype np.float32\nExample output: shape (5000), dtype np.float32\n\nUsed to normalize AccV, AccML, AccAP values.\n\n'''\n\ndef sample_normalize(sample):\n    mean = tf.math.reduce_mean(sample)\n    std = tf.math.reduce_std(sample)\n    sample = tf.math.divide_no_nan(sample-mean, std)\n    \n    return sample.numpy()\n\n'''\nFunction for splitting a series into blocks. Blocks can overlap. \nHow the function works:\nSuppose we have a series with AccV, AccML, AccAP columns and len of 50000, that is (50000, 3). \nFirst, the series is padded so that the final length is divisible by CFG['block_size'] = 15552. Now the series shape is (62208, 3).\nThen we get blocks: first block is series[0:15552, :], second block is series[972:16524, :], ... , last block is series[46656:62208, :].\n\n'''\n\ndef get_blocks(series, columns):\n    series = series.copy()\n    series = series[columns]\n    series = series.values\n    series = series.astype(np.float32)\n    \n    block_count = math.ceil(len(series) / CFG['block_size'])\n    \n    series = np.pad(series, pad_width=[[0, block_count*CFG['block_size']-len(series)], [0, 0]])\n    \n    block_begins = list(range(0, len(series), CFG['block_stride']))\n    block_begins = [x for x in block_begins if x+CFG['block_size'] <= len(series)]\n    \n    blocks = []\n    for begin in block_begins:\n        values = series[begin:begin+CFG['block_size']]\n        blocks.append({'begin': begin,\n                       'end': begin+CFG['block_size'],\n                       'values': values})\n    \n    return blocks\n\n'''\nTrain and inference batch size\n\n'''\n\nGPU_BATCH_SIZE = 4\nTPU_BATCH_SIZE = GPU_BATCH_SIZE*8\n\n'''\nThe transformer encoder layer\nFor more details, see https://arxiv.org/pdf/1706.03762.pdf [Attention Is All You Need]\n\n'''\n\nclass EncoderLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super().__init__()\n        \n        self.mha = tf.keras.layers.MultiHeadAttention(num_heads=CFG['fog_model_num_heads'], key_dim=CFG['fog_model_dim'], dropout=CFG['fog_model_mha_dropout'])\n        \n        self.add = tf.keras.layers.Add()\n        \n        self.layernorm = tf.keras.layers.LayerNormalization()\n        \n        self.seq = tf.keras.Sequential([tf.keras.layers.Dense(CFG['fog_model_dim'], activation='relu'), \n                                        tf.keras.layers.Dropout(CFG['fog_model_encoder_dropout']), \n                                        tf.keras.layers.Dense(CFG['fog_model_dim']), \n                                        tf.keras.layers.Dropout(CFG['fog_model_encoder_dropout']),\n                                       ])\n        \n    def call(self, x):\n        attn_output = self.mha(query=x, key=x, value=x)\n        x = self.add([x, attn_output])\n        x = self.layernorm(x)\n        x = self.add([x, self.seq(x)])\n        x = self.layernorm(x)\n        \n        return x\n    \n'''\nFOGEncoder is a combination of transformer encoder (D=320, H=6, L=5) and two BidirectionalLSTM layers\n\n'''\n\nclass FOGEncoder(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.first_linear = tf.keras.layers.Dense(CFG['fog_model_dim'])\n        \n        self.add = tf.keras.layers.Add()\n        \n        self.first_dropout = tf.keras.layers.Dropout(CFG['fog_model_first_dropout'])\n        \n        self.enc_layers = [EncoderLayer() for _ in range(CFG['fog_model_num_encoder_layers'])]\n        \n        self.lstm_layers = [tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(CFG['fog_model_dim'], return_sequences=True)) for _ in range(CFG['fog_model_num_lstm_layers'])]\n        \n        self.sequence_len = CFG['block_size'] // CFG['patch_size']\n        self.pos_encoding = tf.Variable(initial_value=tf.random.normal(shape=(1, self.sequence_len, CFG['fog_model_dim']), stddev=0.02), trainable=True)\n        \n    def call(self, x, training=None): # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (4, 864, 42)\n        x = x / 50.0 # Normalization attempt in the segment [-1, 1]\n        x = self.first_linear(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']), Example shape (4, 864, 320)\n          \n        if training: # augmentation by randomly roll of the position encoding tensor\n            random_pos_encoding = tf.roll(tf.tile(self.pos_encoding, multiples=[GPU_BATCH_SIZE, 1, 1]), \n                                          shift=tf.random.uniform(shape=(GPU_BATCH_SIZE,), minval=-self.sequence_len, maxval=0, dtype=tf.int32),\n                                          axis=GPU_BATCH_SIZE * [1],\n                                          )\n            x = self.add([x, random_pos_encoding])\n        \n        else: # without augmentation \n            x = self.add([x, tf.tile(self.pos_encoding, multiples=[GPU_BATCH_SIZE, 1, 1])])\n            \n        x = self.first_dropout(x)\n        \n        for i in range(CFG['fog_model_num_encoder_layers']): x = self.enc_layers[i](x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']), Example shape (4, 864, 320)\n        for i in range(CFG['fog_model_num_lstm_layers']): x = self.lstm_layers[i](x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']*2), Example shape (4, 864, 640)\n            \n        return x\n    \nclass FOGModel(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.encoder = FOGEncoder()\n        self.last_linear = tf.keras.layers.Dense(4) \n        \n    def call(self, x): # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (4, 864, 42)\n        x = self.encoder(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']*2), Example shape (4, 864, 640)\n        x = self.last_linear(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], 3), Example shape (4, 864, 4)\n        x = tf.nn.sigmoid(x) # Sigmoid activation\n        \n        return x\n\nWEIGHTS = '/kaggle/input/parkinsons-freezing-submission-models/034_0.432_0.800_0.0351_model.h5' # DEFOG weights\n\nmodel = FOGModel()\nmodel.build(input_shape=(GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3))\nif len(WEIGHTS): model.load_weights(WEIGHTS)","metadata":{"papermill":{"duration":3.872949,"end_time":"2023-06-08T03:06:53.697626","exception":false,"start_time":"2023-06-08T03:06:49.824677","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Prediction**","metadata":{"papermill":{"duration":0.011738,"end_time":"2023-06-08T03:06:53.721575","exception":false,"start_time":"2023-06-08T03:06:53.709837","status":"completed"},"tags":[]}},{"cell_type":"code","source":"'''\nPredictionFnCallback is used for:\n1. Loading test data\n2. FOGModel data preparation\n3. Prediction\n\n'''\n\nclass PredictionFnCallback(tf.keras.callbacks.Callback):\n    \n    def __init__(self, prediction_ids, model=None, verbose=0):\n        \n        if not model is None: self.model = model\n        self.verbose = verbose\n         \n        def init(Id, path):\n            series = pd.read_csv(path).reset_index(drop=True)\n            series['Id'] = Id\n            series['AccV'] = sample_normalize(series['AccV'].values)\n            series['AccML'] = sample_normalize(series['AccML'].values)\n            series['AccAP'] = sample_normalize(series['AccAP'].values)\n            \n            series_blocks=[]\n            for block in get_blocks(series, ['AccV', 'AccML', 'AccAP']): # Example shape (12096, 3)\n                values = tf.reshape(block['values'], shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size'], 3)) # Example shape (864, 14, 3)\n                values = tf.reshape(values, shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3)) # Example shape (864, 42)\n                values = tf.expand_dims(values, axis=0) # Example shape (1, 864, 42)\n                \n                self.blocks.append(values)\n                series_blocks.append((self.blocks_counter, block['begin'], block['end']))\n                self.blocks_counter += 1\n            \n            description = {}\n            description['series'] = series\n            description['series_blocks'] = series_blocks\n            self.descriptions.append(description)\n            \n        self.descriptions = [] # Blocks metadata\n        self.blocks = [] # Test data blocks\n        self.blocks_counter=0 # Blocks counter\n                \n        defog_ids = prediction_ids\n        defog_paths = [f'/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/defog/{defog_id}.csv' for defog_id in defog_ids]\n        for defog_id, defog_path in tqdm(zip(defog_ids, defog_paths), total=len(defog_ids), desc='PredictionFnCallback Initialization', disable=1-verbose): \n            init(defog_id, defog_path)\n                \n        self.blocks = tf.concat(self.blocks, axis=0)  # Example shape (self.blocks_counter, 864, 42)\n        \n        '''\n        self.blocks is padded so that the final length is divisible by inference batch size for error-free operation of model.predict function\n        Padded values have no effect on the predictions\n        \n        '''\n        \n        self.blocks = tf.pad(self.blocks, \n                             paddings=[[0, math.ceil(self.blocks_counter / (TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE))*(TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE)-self.blocks_counter], \n                                                    [0, 0], \n                                                    [0, 0],\n                                      ]) # Example shape (self.blocks_counter+pad_value, 864, 42)\n        \n        print(f'\\n[PredictionFnCallback Initialization] [Series] {len(self.descriptions)} [Blocks] {self.blocks_counter}\\n')\n    \n    def prediction(self):\n        predictions = model.predict(self.blocks, batch_size=TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE, verbose=self.verbose) # Example shape (self.blocks_counter+pad_value, 864, 4)\n        predictions = predictions[:, :, :3] # Example shape (self.blocks_counter+pad_value, 864, 3)\n        predictions = tf.expand_dims(predictions, axis=-1) # Example shape (self.blocks_counter+pad_value, 864, 3, 1)\n        predictions = tf.transpose(predictions, perm=[0, 1, 3, 2]) # Example shape (self.blocks_counter+pad_value, 864, 1, 3)\n        predictions = tf.tile(predictions, multiples=[1, 1, CFG['patch_size'], 1]) # Example shape (self.blocks_counter+pad_value, 864, 14, 3)\n        predictions = tf.reshape(predictions, shape=(predictions.shape[0], predictions.shape[1]*predictions.shape[2], 3)) # Example shape (self.blocks_counter+pad_value, 12096, 3)\n        predictions = predictions.numpy()\n        \n        '''\n        The following function aggregates predictions blocks and creates dataframes with StartHesitation_prediction, Turn_prediction, Walking_prediction columns.\n        \n        '''\n        \n        def create_target(description):\n            series, series_blocks = description['series'].copy(), description['series_blocks']\n            \n            values = np.zeros((series_blocks[-1][2], 4))\n            for series_block in series_blocks:\n                i, begin, end = series_block\n                values[begin:end, 0:3] += predictions[i]\n                values[begin:end, 3] += 1\n\n            values = values[:len(series)]\n            \n            series['StartHesitation_prediction'] = values[:, 0] / values[:, 3]\n            series['Turn_prediction'] = values[:, 1] / values[:, 3]\n            series['Walking_prediction'] = values[:, 2] / values[:, 3]\n            series['Prediction_count'] = values[:, 3]\n            \n            return series\n            \n        targets = Parallel(n_jobs=-1)(delayed(create_target)(self.descriptions[i]) for i in tqdm(range(len(self.descriptions)), disable=1-self.verbose))\n        targets = pd.concat(targets).reset_index(drop=True)\n        \n        return targets\n    \n'''\nPrediction\n        \n'''\n\nfor Id in defog_ids:\n    targets = PredictionFnCallback(prediction_ids=[Id], model=model).prediction()\n    submission = pd.DataFrame({'Id': (targets['Id'].values + '_' + targets['Time'].astype('str')).values,\n                               'StartHesitation': targets['StartHesitation_prediction'].values,\n                               'Turn': targets['Turn_prediction'].values,\n                               'Walking': targets['Walking_prediction'].values,\n                              })\n    \n    all_submissions.append(submission)","metadata":{"papermill":{"duration":24.14632,"end_time":"2023-06-08T03:07:17.879767","exception":false,"start_time":"2023-06-08T03:06:53.733447","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Configuration**","metadata":{"papermill":{"duration":0.011684,"end_time":"2023-06-08T03:07:17.904023","exception":false,"start_time":"2023-06-08T03:07:17.892339","status":"completed"},"tags":[]}},{"cell_type":"code","source":"CFG = {'TPU': 0,\n       'block_size': 12096, \n       'block_stride': 12096//16,\n       'patch_size': 14, \n       'batch_size': 128,\n      }\n\nassert CFG['block_size'] % CFG['patch_size'] == 0\nassert CFG['block_size'] % CFG['block_stride'] == 0\n\n'''\nMean-std normalization function. \nExample input: shape (5000), dtype np.float32\nExample output: shape (5000), dtype np.float32\n\nUsed to normalize AccV, AccML, AccAP values.\n\n'''\n\ndef sample_normalize(sample):\n    mean = tf.math.reduce_mean(sample)\n    std = tf.math.reduce_std(sample)\n    sample = tf.math.divide_no_nan(sample-mean, std)\n    \n    return sample.numpy()\n\n'''\nFunction for splitting a series into blocks. Blocks can overlap. \nHow the function works:\nSuppose we have a series with AccV, AccML, AccAP columns and len of 50000, that is (50000, 3). \nFirst, the series is padded so that the final length is divisible by CFG['block_size'] = 15552. Now the series shape is (62208, 3).\nThen we get blocks: first block is series[0:15552, :], second block is series[972:16524, :], ... , last block is series[46656:62208, :].\n\n'''\n\ndef get_blocks(series, columns):\n    series = series.copy()\n    series = series[columns]\n    series = series.values\n    series = series.astype(np.float32)\n    \n    block_count = math.ceil(len(series) / CFG['block_size'])\n    \n    series = np.pad(series, pad_width=[[0, block_count*CFG['block_size']-len(series)], [0, 0]])\n    \n    block_begins = list(range(0, len(series), CFG['block_stride']))\n    block_begins = [x for x in block_begins if x+CFG['block_size'] <= len(series)]\n    \n    blocks = []\n    for begin in block_begins:\n        values = series[begin:begin+CFG['block_size']]\n        blocks.append({'begin': begin,\n                       'end': begin+CFG['block_size'],\n                       'values': values})\n    \n    return blocks\n\n'''\nThe transformer encoder layer\nFor more details, see https://arxiv.org/pdf/1706.03762.pdf [Attention Is All You Need]\n\n'''\n \nclass EncoderLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super().__init__()\n        \n        self.mha = tf.keras.layers.MultiHeadAttention(num_heads=5, key_dim=320)\n        self.add = tf.keras.layers.Add()\n        self.layernorm = tf.keras.layers.LayerNormalization()\n        self.seq = tf.keras.Sequential([tf.keras.layers.Dense(320, activation='relu'), tf.keras.layers.Dropout(0.1), tf.keras.layers.Dense(320), tf.keras.layers.Dropout(0.1)])\n        \n    def call(self, x):\n        attn_output, attn_scores = self.mha(query=x, key=x, value=x, \n                                            return_attention_scores=True)\n        \n        self.attn_scores = attn_scores\n        \n        x = self.add([x, attn_output])\n        x = self.layernorm(x)\n        x = self.add([x, self.seq(x)])\n        x = self.layernorm(x)\n        \n        return x\n    \n'''\nFOGEncoder is a combination of transformer encoder (D=320, H=5, L=5) and two BidirectionalLSTM layers\n\n'''\n\nclass FOGEncoder(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.num_layers=5\n        self.num_lstm_layers=2\n        self.key_dim=320\n        self.dropout_rate = 0.1\n    \n        self.masking = tf.keras.layers.Masking()\n        self.first_linear = tf.keras.layers.Dense(self.key_dim)\n        self.first_dropout = tf.keras.layers.Dropout(self.dropout_rate)\n        self.enc_layers = [EncoderLayer() for _ in range(self.num_layers)]\n        self.lstm_layers = [tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(self.key_dim, return_sequences=True)) for _ in range(self.num_lstm_layers)]\n        \n        self.sequence_len = CFG['block_size'] // CFG['patch_size']\n        self.pos_encoding = tf.Variable(initial_value=tf.random.normal(shape=(1, self.sequence_len, self.key_dim), stddev=0.02), trainable=True)\n    \n    def call(self, x, training=None): # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (128, 864, 42)\n        x = self.masking(x) # Masks a padded timesteps from multi head attention and lstm layers\n        x = x / 50.0 # Normalization attempt in the segment [-1, 1]\n        x = self.first_linear(x) # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], self.key_dim), Example shape (128, 864, 320)\n        \n        x = x + self.pos_encoding \n            \n        x = self.first_dropout(x)\n        \n        for i in range(self.num_layers): x = self.enc_layers[i](x) # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], self.key_dim), Example shape (128, 864, 320)\n        for i in range(self.num_lstm_layers): x = self.lstm_layers[i](x) # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], self.key_dim*2), Example shape (128, 864, 640)\n        \n        return x\n    \nclass FOGModel(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.encoder = FOGEncoder()\n        self.last_linear = tf.keras.layers.Dense(3) \n        \n    def call(self, x): # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (128, 864, 42)\n        x = self.encoder(x) # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], self.key_dim*2), Example shape (128, 864, 640)\n        x = self.last_linear(x) # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], 3), Example shape (128, 864, 3)\n        x = tf.nn.sigmoid(x) # Sigmoid activation\n        \n        return x","metadata":{"papermill":{"duration":0.048608,"end_time":"2023-06-08T03:07:17.964499","exception":false,"start_time":"2023-06-08T03:07:17.915891","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Prediction**","metadata":{"papermill":{"duration":0.011674,"end_time":"2023-06-08T03:07:17.988146","exception":false,"start_time":"2023-06-08T03:07:17.976472","status":"completed"},"tags":[]}},{"cell_type":"code","source":"'''\nPredictionFnCallback is used for:\n1. Loading test data\n2. FOGModel data preparation\n3. Prediction\n\nDeprecated. See previous implementations of the PredictionFnCallback.\n\n'''\n\nclass PredictionFnCallback(tf.keras.callbacks.Callback):\n    \n    def prediction_fn(self, all_targets, all_blocks, model, verbose=0):\n        if len(all_blocks) == 0: return pd.DataFrame()\n        \n        all_values = tf.concat([block['values'] for block in all_blocks], axis=0)\n        batch_count = math.ceil(len(all_values) / CFG['batch_size'])\n        pad_count = batch_count*CFG['batch_size']-len(all_values)\n        all_values = tf.pad(all_values, paddings=[[0, pad_count], [0, 0], [0, 0]])\n\n        all_predictions = model.predict(all_values, batch_size=CFG['batch_size'], verbose=verbose)\n        all_predictions = tf.expand_dims(all_predictions, axis=-1)\n        all_predictions = tf.transpose(all_predictions, perm=[0, 1, 3, 2])\n        all_predictions = tf.tile(all_predictions, multiples=[1, 1, CFG['patch_size'], 1])\n        all_predictions = tf.reshape(all_predictions, shape=(all_predictions.shape[0], all_predictions.shape[1]*all_predictions.shape[2], 3))\n\n        for Id in all_targets:\n            target = all_targets[Id]\n            target[[0, 1, 2, 'output_count']] = 0.0\n\n        for block, prediction in tqdm(zip(all_blocks, all_predictions), total=len(all_blocks), desc='Set prediction', disable=1-verbose):\n            target = all_targets[block['Id']]\n            prediction = pd.DataFrame(prediction.numpy(), index=range(block['begin'], block['end']))\n\n            indexes = prediction.index.intersection(target.index)\n\n            target.loc[indexes, [0, 1, 2]] += prediction.loc[indexes, [0, 1, 2]]\n            target.loc[indexes, 'output_count'] += 1\n\n        all_targets = pd.concat([target for target in all_targets.values()]).copy()\n\n        assert not (all_targets['output_count'] == 0.0).any()\n\n        for i in [0, 1, 2]: all_targets[i] /= all_targets['output_count']\n\n        all_targets['StartHesitation_prediction'] = all_targets[0]\n        all_targets['Turn_prediction'] = all_targets[1]\n        all_targets['Walking_prediction'] = all_targets[2]\n\n        return all_targets\n    \n    def __init__(self, Ids, prediction_folder, model=None, verbose=0):\n        \n        if not model is None: self.model = model\n        self.verbose = verbose\n        \n        def add_blocks_and_target(all_targets, all_blocks, Id, path, source):\n            \n            series = pd.read_csv(path).reset_index(drop=True)\n            series['AccV'] = sample_normalize(series['AccV'].values)\n            series['AccML'] = sample_normalize(series['AccML'].values)\n            series['AccAP'] = sample_normalize(series['AccAP'].values)\n            \n            all_targets[Id] = series.copy()\n            \n            blocks = get_blocks(series, ['AccV', 'AccML', 'AccAP'])\n            for block in blocks: \n                begin, end, values = block['begin'], block['end'], block['values'] # Example shape (15552, 3)\n                values = tf.reshape(values, shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size'], 3)) # Example shape (864, 14, 3)\n                values = tf.reshape(values, shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3)) # Example shape (864, 42)\n                series_info = tf.constant(0.0, dtype=tf.float32, shape=(CFG['block_size'] // CFG['patch_size'], 1)) if source == 'tsfog' else tf.constant(1.0, dtype=tf.float32, shape=(CFG['block_size'] // CFG['patch_size'], 1))\n                values = tf.concat([values, series_info], axis=-1) # Example shape (864, 43) (Add a constant to the right. Not used. Has no effect on the model)\n                values = tf.expand_dims(values, axis=0) # Example shape (1, 864, 43)\n                all_blocks.append({'Id': Id, 'begin': begin, 'end': end, 'values': values})\n                \n        self.defog_all_targets = {}\n        self.defog_all_blocks = []\n        \n        for Id in tqdm(Ids, total=len(Ids), desc='defog initialization', disable=1-verbose): add_blocks_and_target(all_targets=self.defog_all_targets,\n                                                                                                                   all_blocks=self.defog_all_blocks,\n                                                                                                                   Id=Id, \n                                                                                                                   path=f'{prediction_folder}/defog/{Id}.csv',\n                                                                                                                   source='defog',\n                                                                                                             )\n\n            \n\n'''\nPrediction\n        \n'''\n\nmodel_paths = ['/kaggle/input/parkinsons-freezing-submission-models/023_0.485_0.118_0.047_mAP_mAP_loss_model.h5'] # DEFOG weights\n\nfor model_path in model_paths:\n    model = FOGModel()\n    model.build(input_shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3+1))\n    model.load_weights(model_path)\n\n    for Id in tqdm(defog_ids, total=len(defog_ids)):\n        prediction_fn_callback = PredictionFnCallback(Ids=[Id],\n                                                      prediction_folder='/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test',\n                                                      model=model,\n                                                     )\n        all_targets = prediction_fn_callback.prediction_fn(prediction_fn_callback.defog_all_targets, \n                                                           prediction_fn_callback.defog_all_blocks, \n                                                           prediction_fn_callback.model, \n                                                           verbose=prediction_fn_callback.verbose)\n\n        submission = pd.DataFrame({'Id': (Id + '_' + all_targets['Time'].astype('str')).values,\n                                   'StartHesitation': all_targets['StartHesitation_prediction'].values,\n                                   'Turn': all_targets['Turn_prediction'].values,\n                                   'Walking': all_targets['Walking_prediction'].values,\n                                  })\n\n        all_submissions.append(submission)","metadata":{"papermill":{"duration":16.510774,"end_time":"2023-06-08T03:07:34.511034","exception":false,"start_time":"2023-06-08T03:07:18.000260","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Configuration**","metadata":{"papermill":{"duration":0.012254,"end_time":"2023-06-08T03:07:34.536316","exception":false,"start_time":"2023-06-08T03:07:34.524062","status":"completed"},"tags":[]}},{"cell_type":"code","source":"CFG = {'TPU': 0,\n       'block_size': 12096, \n       'block_stride': 12096//16,\n       'patch_size': 14, \n       'batch_size': 16,\n       \n       'fog_model_dim': 320,\n       'fog_model_num_heads': 6,\n       'fog_model_num_encoder_layers': 4,\n       'fog_model_num_lstm_layers': 2,\n       'fog_model_first_dropout': 0.1,\n       'fog_model_encoder_dropout': 0.1,\n       'fog_model_mha_dropout': 0.0,\n      }\n\nassert CFG['block_size'] % CFG['patch_size'] == 0\nassert CFG['block_size'] % CFG['block_stride'] == 0\n\n'''\nMean-std normalization function. \nExample input: shape (5000), dtype np.float32\nExample output: shape (5000), dtype np.float32\n\nUsed to normalize AccV, AccML, AccAP values.\n\n'''\n\ndef sample_normalize(sample):\n    mean = tf.math.reduce_mean(sample)\n    std = tf.math.reduce_std(sample)\n    sample = tf.math.divide_no_nan(sample-mean, std)\n    \n    return sample.numpy()\n\n'''\nFunction for splitting a series into blocks. Blocks can overlap. \nHow the function works:\nSuppose we have a series with AccV, AccML, AccAP columns and len of 50000, that is (50000, 3). \nFirst, the series is padded so that the final length is divisible by CFG['block_size'] = 15552. Now the series shape is (62208, 3).\nThen we get blocks: first block is series[0:15552, :], second block is series[972:16524, :], ... , last block is series[46656:62208, :].\n\n'''\n\ndef get_blocks(series, columns):\n    series = series.copy()\n    series = series[columns]\n    series = series.values\n    series = series.astype(np.float32)\n    \n    block_count = math.ceil(len(series) / CFG['block_size'])\n    \n    series = np.pad(series, pad_width=[[0, block_count*CFG['block_size']-len(series)], [0, 0]])\n    \n    block_begins = list(range(0, len(series), CFG['block_stride']))\n    block_begins = [x for x in block_begins if x+CFG['block_size'] <= len(series)]\n    \n    blocks = []\n    for begin in block_begins:\n        values = series[begin:begin+CFG['block_size']]\n        blocks.append({'begin': begin,\n                       'end': begin+CFG['block_size'],\n                       'values': values})\n    \n    return blocks\n\n'''\nThe transformer encoder layer\nFor more details, see https://arxiv.org/pdf/1706.03762.pdf [Attention Is All You Need]\n\n'''\n \nclass EncoderLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super().__init__()\n        \n        self.mha = tf.keras.layers.MultiHeadAttention(num_heads=CFG['fog_model_num_heads'], key_dim=CFG['fog_model_dim'])\n        \n        self.add = tf.keras.layers.Add()\n        \n        self.layernorm = tf.keras.layers.LayerNormalization()\n        \n        self.seq = tf.keras.Sequential([tf.keras.layers.Dense(CFG['fog_model_dim'], activation='relu'), \n                                        tf.keras.layers.Dropout(CFG['fog_model_encoder_dropout']), \n                                        tf.keras.layers.Dense(CFG['fog_model_dim']), \n                                        tf.keras.layers.Dropout(CFG['fog_model_encoder_dropout']),\n                                       ])\n        \n    def call(self, x):\n        attn_output, attn_scores = self.mha(query=x, key=x, value=x, \n                                            return_attention_scores=True)\n        \n        self.attn_scores = attn_scores\n        \n        x = self.add([x, attn_output])\n        x = self.layernorm(x)\n        x = self.add([x, self.seq(x)])\n        x = self.layernorm(x)\n        \n        return x\n    \n'''\nFOGEncoder is a combination of transformer encoder (D=320, H=6, L=4) and two BidirectionalLSTM layers\n\n'''\n\nclass FOGEncoder(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.masking = tf.keras.layers.Masking()\n        \n        self.first_linear = tf.keras.layers.Dense(CFG['fog_model_dim'])\n        \n        self.add = tf.keras.layers.Add()\n        \n        self.first_dropout = tf.keras.layers.Dropout(CFG['fog_model_first_dropout'])\n        \n        self.enc_layers = [EncoderLayer() for _ in range(CFG['fog_model_num_encoder_layers'])]\n        \n        self.lstm_layers = [tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(CFG['fog_model_dim'], return_sequences=True)) for _ in range(CFG['fog_model_num_lstm_layers'])]\n        \n        self.sequence_len = CFG['block_size'] // CFG['patch_size']\n        self.pos_encoding = tf.Variable(initial_value=tf.random.normal(shape=(1, self.sequence_len, CFG['fog_model_dim']), stddev=0.02), trainable=True)\n        \n    def call(self, x, training=None): # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (16, 864, 42)\n        x = x / 50.0 # Normalization attempt in the segment [0, 1]\n        x = self.masking(x) # Masks a padded timesteps from multi head attention and lstm layers\n        x = self.first_linear(x) # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']), Example shape (16, 864, 320)\n          \n        x = self.add([x, tf.tile(self.pos_encoding, multiples=[CFG['batch_size'], 1, 1])])\n            \n        x = self.first_dropout(x)\n        \n        km = x._keras_mask # Bug fix (Multi head attention masking does not work on TPU training)\n        del x._keras_mask # Bug fix \n        for i in range(CFG['fog_model_num_encoder_layers']): x = self.enc_layers[i](x) # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']), Example shape (16, 864, 320)\n        x._keras_mask = km # Bug fix\n        for i in range(CFG['fog_model_num_lstm_layers']): x = self.lstm_layers[i](x) # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']*2), Example shape (16, 864, 640)\n            \n        return x\n    \nclass FOGModel(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.encoder = FOGEncoder()\n        self.last_linear = tf.keras.layers.Dense(3) \n        \n    def call(self, x): # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (16, 864, 42)\n        x = self.encoder(x) # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']*2), Example shape (16, 864, 640)\n        x = self.last_linear(x) # (CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], 3), Example shape (16, 864, 3)\n        x = tf.nn.sigmoid(x) # Sigmoid activation\n        \n        return x\n\nmodel = FOGModel()\nmodel.build(input_shape=(CFG['batch_size'], CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3+1))\nmodel.load_weights('/kaggle/input/parkinsons-freezing-submission-models/033_0.764_0.0659_mAP_loss_model.h5') # DEFOG weights","metadata":{"papermill":{"duration":5.37982,"end_time":"2023-06-08T03:07:39.928635","exception":false,"start_time":"2023-06-08T03:07:34.548815","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Prediction**","metadata":{"papermill":{"duration":0.012971,"end_time":"2023-06-08T03:07:39.955847","exception":false,"start_time":"2023-06-08T03:07:39.942876","status":"completed"},"tags":[]}},{"cell_type":"code","source":"'''\nPredictionFnCallback is used for:\n1. Loading test data\n2. FOGModel data preparation\n3. Prediction\n\n'''\n\nclass PredictionFnCallback(tf.keras.callbacks.Callback):\n    \n    def __init__(self, prediction_ids, model=None, verbose=0):\n        \n        if not model is None: self.model = model\n        self.verbose = verbose\n         \n        def init(Id, path):\n            series = pd.read_csv(path).reset_index(drop=True)\n            series['Id'] = Id\n            series['AccV'] = sample_normalize(series['AccV'].values)\n            series['AccML'] = sample_normalize(series['AccML'].values)\n            series['AccAP'] = sample_normalize(series['AccAP'].values)\n            \n            series_blocks=[]\n            for block in get_blocks(series, ['AccV', 'AccML', 'AccAP']): # Example shape (12096, 3)\n                values = tf.reshape(block['values'], shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size'], 3)) # Example shape (864, 14, 3)\n                values = tf.reshape(values, shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3)) # Example shape (864, 42)\n                values = tf.concat([values, \n                                    tf.constant(1.0, dtype=tf.float32, shape=(CFG['block_size'] // CFG['patch_size'], 1)),\n                                   ], axis=-1) # Example shape (864, 43) (Add a constant to the right. Not used. Has no effect on the model)\n                values = tf.expand_dims(values, axis=0) # Example shape (1, 864, 43)\n                \n                self.blocks.append(values)\n                series_blocks.append((self.blocks_counter, block['begin'], block['end']))\n                self.blocks_counter += 1\n            \n            description = {}\n            description['series'] = series\n            description['series_blocks'] = series_blocks\n            self.descriptions.append(description)\n            \n        self.descriptions = [] # Blocks metadata\n        self.blocks = [] # Test data blocks\n        self.blocks_counter=0 # Blocks counter\n        \n        notype_ids = prediction_ids\n        notype_paths = [f'/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/defog/{notype_id}.csv' for notype_id in notype_ids]\n        for notype_id, notype_path in tqdm(zip(notype_ids, notype_paths), total=len(notype_ids), desc='PredictionFnCallback Initialization', disable=1-verbose): \n            init(notype_id, notype_path)\n            \n        self.blocks = tf.concat(self.blocks, axis=0) # Example shape (self.blocks_counter, 864, 42)\n        \n        '''\n        self.blocks is padded so that the final length is divisible by inference batch size for error-free operation of model.predict function\n        Padded values have no effect on the predictions\n        \n        '''\n        \n        self.blocks = tf.pad(self.blocks, \n                             paddings=[[0, math.ceil(self.blocks_counter / CFG['batch_size'])*CFG['batch_size']-self.blocks_counter], \n                                                    [0, 0], \n                                                    [0, 0],\n                                      ]) # Example shape (self.blocks_counter+pad_value, 864, 42)\n        \n        print(f'[PredictionFnCallback Initialization] [Series] {len(self.descriptions)} [Blocks] {self.blocks_counter}')\n    \n    def prediction(self):\n        predictions = model.predict(self.blocks, batch_size=CFG['batch_size'], verbose=self.verbose) # Example shape (self.blocks_counter+pad_value, 864, 3)\n        predictions = tf.expand_dims(predictions, axis=-1) # Example shape (self.blocks_counter+pad_value, 864, 3, 1)\n        predictions = tf.transpose(predictions, perm=[0, 1, 3, 2]) # Example shape (self.blocks_counter+pad_value, 864, 1, 3)\n        predictions = tf.tile(predictions, multiples=[1, 1, CFG['patch_size'], 1]) # Example shape (self.blocks_counter+pad_value, 864, 14, 3)\n        predictions = tf.reshape(predictions, shape=(predictions.shape[0], predictions.shape[1]*predictions.shape[2], 3)) # Example shape (self.blocks_counter+pad_value, 12096, 3)\n        predictions = predictions.numpy() \n        \n        '''\n        The following function aggregates predictions blocks and creates dataframes with StartHesitation_prediction, Turn_prediction, Walking_prediction columns.\n        \n        '''\n        \n        def create_target(description):\n            series, series_blocks = description['series'].copy(), description['series_blocks']\n            \n            values = np.zeros((series_blocks[-1][2], 4))\n            for series_block in series_blocks:\n                i, begin, end = series_block\n                values[begin:end, 0:3] += predictions[i]\n                values[begin:end, 3] += 1\n\n            values = values[:len(series)]\n            \n            series['StartHesitation_prediction'] = values[:, 0] / values[:, 3]\n            series['Turn_prediction'] = values[:, 1] / values[:, 3]\n            series['Walking_prediction'] = values[:, 2] / values[:, 3]\n            series['Prediction_count'] = values[:, 3]\n            \n            return series\n            \n        targets = Parallel(n_jobs=-1)(delayed(create_target)(self.descriptions[i]) for i in tqdm(range(len(self.descriptions)), disable=1-self.verbose))\n        targets = pd.concat(targets)\n        \n        return targets\n    \n'''\nPrediction\n        \n'''\n\nfor Id in defog_ids:\n    targets = PredictionFnCallback(prediction_ids=[Id], model=model).prediction()\n    submission = pd.DataFrame({'Id': (targets['Id'].values + '_' + targets['Time'].astype('str')).values,\n                               'StartHesitation': targets['StartHesitation_prediction'].values,\n                               'Turn': targets['Turn_prediction'].values,\n                               'Walking': targets['Walking_prediction'].values,\n                              })\n    \n    all_submissions.append(submission)","metadata":{"papermill":{"duration":11.944945,"end_time":"2023-06-08T03:07:51.913562","exception":false,"start_time":"2023-06-08T03:07:39.968617","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Configuration**","metadata":{"papermill":{"duration":0.0126,"end_time":"2023-06-08T03:07:51.939574","exception":false,"start_time":"2023-06-08T03:07:51.926974","status":"completed"},"tags":[]}},{"cell_type":"code","source":"CFG = {'TPU': 0,\n       'block_size': 12096, \n       'block_stride': 12096//16,\n       'patch_size': 14, \n       \n       'fog_model_dim': 320,\n       'fog_model_num_heads': 6,\n       'fog_model_num_encoder_layers': 5,\n       'fog_model_num_lstm_layers': 2,\n       'fog_model_first_dropout': 0.1,\n       'fog_model_encoder_dropout': 0.1,\n       'fog_model_mha_dropout': 0.0,\n      }\n\nassert CFG['block_size'] % CFG['patch_size'] == 0\nassert CFG['block_size'] % CFG['block_stride'] == 0\n\n'''\nMean-std normalization function. \nExample input: shape (5000), dtype np.float32\nExample output: shape (5000), dtype np.float32\n\nUsed to normalize AccV, AccML, AccAP values.\n\n'''\n\ndef sample_normalize(sample):\n    mean = tf.math.reduce_mean(sample)\n    std = tf.math.reduce_std(sample)\n    sample = tf.math.divide_no_nan(sample-mean, std)\n    \n    return sample.numpy()\n\n'''\nFunction for splitting a series into blocks. Blocks can overlap. \nHow the function works:\nSuppose we have a series with AccV, AccML, AccAP columns and len of 50000, that is (50000, 3). \nFirst, the series is padded so that the final length is divisible by CFG['block_size'] = 15552. Now the series shape is (62208, 3).\nThen we get blocks: first block is series[0:15552, :], second block is series[972:16524, :], ... , last block is series[46656:62208, :].\n\n'''\n\ndef get_blocks(series, columns):\n    series = series.copy()\n    series = series[columns]\n    series = series.values\n    series = series.astype(np.float32)\n    \n    block_count = math.ceil(len(series) / CFG['block_size'])\n    \n    series = np.pad(series, pad_width=[[0, block_count*CFG['block_size']-len(series)], [0, 0]])\n    \n    block_begins = list(range(0, len(series), CFG['block_stride']))\n    block_begins = [x for x in block_begins if x+CFG['block_size'] <= len(series)]\n    \n    blocks = []\n    for begin in block_begins:\n        values = series[begin:begin+CFG['block_size']]\n        blocks.append({'begin': begin,\n                       'end': begin+CFG['block_size'],\n                       'values': values})\n    \n    return blocks\n\n'''\nTrain and inference batch size\n\n'''\n\nGPU_BATCH_SIZE = 4\nTPU_BATCH_SIZE = GPU_BATCH_SIZE*8\n\n'''\nThe transformer encoder layer\nFor more details, see https://arxiv.org/pdf/1706.03762.pdf [Attention Is All You Need]\n\n'''\n\nclass EncoderLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super().__init__()\n        \n        self.mha = tf.keras.layers.MultiHeadAttention(num_heads=CFG['fog_model_num_heads'], key_dim=CFG['fog_model_dim'], dropout=CFG['fog_model_mha_dropout'])\n        \n        self.add = tf.keras.layers.Add()\n        \n        self.layernorm = tf.keras.layers.LayerNormalization()\n        \n        self.seq = tf.keras.Sequential([tf.keras.layers.Dense(CFG['fog_model_dim'], activation='relu'), \n                                        tf.keras.layers.Dropout(CFG['fog_model_encoder_dropout']), \n                                        tf.keras.layers.Dense(CFG['fog_model_dim']), \n                                        tf.keras.layers.Dropout(CFG['fog_model_encoder_dropout']),\n                                       ])\n        \n    def call(self, x):\n        attn_output = self.mha(query=x, key=x, value=x)\n        x = self.add([x, attn_output])\n        x = self.layernorm(x)\n        x = self.add([x, self.seq(x)])\n        x = self.layernorm(x)\n        \n        return x\n    \n'''\nFOGEncoder is a combination of transformer encoder (D=320, H=6, L=5) and two BidirectionalLSTM layers\n\n'''\n\nclass FOGEncoder(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.first_linear = tf.keras.layers.Dense(CFG['fog_model_dim'])\n        \n        self.add = tf.keras.layers.Add()\n        \n        self.first_dropout = tf.keras.layers.Dropout(CFG['fog_model_first_dropout'])\n        \n        self.enc_layers = [EncoderLayer() for _ in range(CFG['fog_model_num_encoder_layers'])]\n        \n        self.lstm_layers = [tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(CFG['fog_model_dim'], return_sequences=True)) for _ in range(CFG['fog_model_num_lstm_layers'])]\n        \n        self.sequence_len = CFG['block_size'] // CFG['patch_size']\n        self.pos_encoding = tf.Variable(initial_value=tf.random.normal(shape=(1, self.sequence_len, CFG['fog_model_dim']), stddev=0.02), trainable=True)\n        \n    def call(self, x, training=None): # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (4, 864, 42)\n        x = x / 50.0 # Normalization attempt in the segment [-1, 1]\n        x = self.first_linear(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']), Example shape (4, 864, 320)\n          \n        if training: # augmentation by randomly roll of the position encoding tensor\n            random_pos_encoding = tf.roll(tf.tile(self.pos_encoding, multiples=[GPU_BATCH_SIZE, 1, 1]), \n                                          shift=tf.random.uniform(shape=(GPU_BATCH_SIZE,), minval=-self.sequence_len, maxval=0, dtype=tf.int32),\n                                          axis=GPU_BATCH_SIZE * [1],\n                                          )\n            x = self.add([x, random_pos_encoding])\n        \n        else: # without augmentation \n            x = self.add([x, tf.tile(self.pos_encoding, multiples=[GPU_BATCH_SIZE, 1, 1])])\n            \n        x = self.first_dropout(x)\n        \n        for i in range(CFG['fog_model_num_encoder_layers']): x = self.enc_layers[i](x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']), Example shape (4, 864, 320)\n        for i in range(CFG['fog_model_num_lstm_layers']): x = self.lstm_layers[i](x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']*2), Example shape (4, 864, 640)\n            \n        return x\n    \nclass FOGModel(tf.keras.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.encoder = FOGEncoder()\n        self.last_linear = tf.keras.layers.Dense(4) \n        \n    def call(self, x): # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3), Example shape (4, 864, 42)\n        x = self.encoder(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['fog_model_dim']*2), Example shape (4, 864, 640)\n        x = self.last_linear(x) # (GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], 4), Example shape (4, 864, 4)\n        x = tf.nn.sigmoid(x) # Sigmoid activation\n        \n        return x\n\nWEIGHTS = '/kaggle/input/parkinsons-freezing-submission-models/019_0.489_0.744_0.0817_model.h5' # DEFOG weights\n\nmodel = FOGModel()\nmodel.build(input_shape=(GPU_BATCH_SIZE, CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3))\nif len(WEIGHTS): model.load_weights(WEIGHTS)","metadata":{"papermill":{"duration":3.428986,"end_time":"2023-06-08T03:07:55.381569","exception":false,"start_time":"2023-06-08T03:07:51.952583","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Prediction**","metadata":{"papermill":{"duration":0.012606,"end_time":"2023-06-08T03:07:55.407616","exception":false,"start_time":"2023-06-08T03:07:55.395010","status":"completed"},"tags":[]}},{"cell_type":"code","source":"'''\nPredictionFnCallback is used for:\n1. Loading test data\n2. FOGModel data preparation\n3. Prediction\n\n'''\n\nclass PredictionFnCallback(tf.keras.callbacks.Callback):\n    \n    def __init__(self, prediction_ids, model=None, verbose=0):\n        \n        if not model is None: self.model = model\n        self.verbose = verbose\n         \n        def init(Id, path):\n            series = pd.read_csv(path).reset_index(drop=True)\n            series['Id'] = Id\n            series['AccV'] = sample_normalize(series['AccV'].values)\n            series['AccML'] = sample_normalize(series['AccML'].values)\n            series['AccAP'] = sample_normalize(series['AccAP'].values)\n            \n            series_blocks=[]\n            for block in get_blocks(series, ['AccV', 'AccML', 'AccAP']): # Example shape (12096, 3)\n                values = tf.reshape(block['values'], shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size'], 3)) # Example shape (864, 14, 3)\n                values = tf.reshape(values, shape=(CFG['block_size'] // CFG['patch_size'], CFG['patch_size']*3)) # Example shape (864, 42)\n                values = tf.expand_dims(values, axis=0) # Example shape (1, 864, 42)\n                \n                self.blocks.append(values)\n                series_blocks.append((self.blocks_counter, block['begin'], block['end']))\n                self.blocks_counter += 1\n            \n            description = {}\n            description['series'] = series\n            description['series_blocks'] = series_blocks\n            self.descriptions.append(description)\n            \n        self.descriptions = [] # Blocks metadata\n        self.blocks = [] # Test data blocks\n        self.blocks_counter=0 # Blocks counter\n                \n        defog_ids = prediction_ids\n        defog_paths = [f'/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/test/defog/{defog_id}.csv' for defog_id in defog_ids]\n        for defog_id, defog_path in tqdm(zip(defog_ids, defog_paths), total=len(defog_ids), desc='PredictionFnCallback Initialization', disable=1-verbose): \n            init(defog_id, defog_path)\n                \n        self.blocks = tf.concat(self.blocks, axis=0) # Example shape (self.blocks_counter, 864, 42)\n        \n        '''\n        self.blocks is padded so that the final length is divisible by inference batch size for error-free operation of model.predict function\n        Padded values have no effect on the predictions\n        \n        '''\n        \n        self.blocks = tf.pad(self.blocks, \n                             paddings=[[0, math.ceil(self.blocks_counter / (TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE))*(TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE)-self.blocks_counter], \n                                                    [0, 0], \n                                                    [0, 0],\n                                      ]) # Example shape (self.blocks_counter+pad_value, 864, 42)\n        \n        print(f'\\n[PredictionFnCallback Initialization] [Series] {len(self.descriptions)} [Blocks] {self.blocks_counter}\\n')\n    \n    def prediction(self):\n        predictions = model.predict(self.blocks, batch_size=TPU_BATCH_SIZE if CFG['TPU'] else GPU_BATCH_SIZE, verbose=self.verbose) # Example shape (self.blocks_counter+pad_value, 864, 4)\n        predictions = predictions[:, :, :3] # Example shape (self.blocks_counter+pad_value, 864, 3)\n        predictions = tf.expand_dims(predictions, axis=-1) # Example shape (self.blocks_counter+pad_value, 864, 3, 1)\n        predictions = tf.transpose(predictions, perm=[0, 1, 3, 2]) # Example shape (self.blocks_counter+pad_value, 864, 1, 3)\n        predictions = tf.tile(predictions, multiples=[1, 1, CFG['patch_size'], 1]) # Example shape (self.blocks_counter+pad_value, 864, 14, 3)\n        predictions = tf.reshape(predictions, shape=(predictions.shape[0], predictions.shape[1]*predictions.shape[2], 3)) # Example shape (self.blocks_counter+pad_value, 12096, 3)\n        predictions = predictions.numpy()\n        \n        '''\n        The following function aggregates predictions blocks and creates dataframes with StartHesitation_prediction, Turn_prediction, Walking_prediction columns.\n        \n        '''\n        \n        def create_target(description):\n            series, series_blocks = description['series'].copy(), description['series_blocks']\n            \n            values = np.zeros((series_blocks[-1][2], 4))\n            for series_block in series_blocks:\n                i, begin, end = series_block\n                values[begin:end, 0:3] += predictions[i]\n                values[begin:end, 3] += 1\n\n            values = values[:len(series)]\n            \n            series['StartHesitation_prediction'] = values[:, 0] / values[:, 3]\n            series['Turn_prediction'] = values[:, 1] / values[:, 3]\n            series['Walking_prediction'] = values[:, 2] / values[:, 3]\n            series['Prediction_count'] = values[:, 3]\n            \n            return series\n            \n        targets = Parallel(n_jobs=-1)(delayed(create_target)(self.descriptions[i]) for i in tqdm(range(len(self.descriptions)), disable=1-self.verbose))\n        targets = pd.concat(targets).reset_index(drop=True)\n        \n        return targets\n    \n'''\nPrediction\n        \n'''\n\nfor Id in defog_ids:\n    targets = PredictionFnCallback(prediction_ids=[Id], model=model).prediction()\n    submission = pd.DataFrame({'Id': (targets['Id'].values + '_' + targets['Time'].astype('str')).values,\n                               'StartHesitation': targets['StartHesitation_prediction'].values,\n                               'Turn': targets['Turn_prediction'].values,\n                               'Walking': targets['Walking_prediction'].values,\n                              })\n    \n    all_submissions.append(submission)","metadata":{"papermill":{"duration":14.14782,"end_time":"2023-06-08T03:08:09.568162","exception":false,"start_time":"2023-06-08T03:07:55.420342","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Part 3. SUBMISSION**","metadata":{"papermill":{"duration":0.012874,"end_time":"2023-06-08T03:08:09.594581","exception":false,"start_time":"2023-06-08T03:08:09.581707","status":"completed"},"tags":[]}},{"cell_type":"code","source":"submission = pd.concat(all_submissions).reset_index(drop=True)\nsubmission = submission.groupby('Id').agg('mean').reset_index()\nsubmission.to_csv('submission.csv', index=False)","metadata":{"papermill":{"duration":2.739339,"end_time":"2023-06-08T03:08:12.346845","exception":false,"start_time":"2023-06-08T03:08:09.607506","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**PUBLIC IMAGE TEST**","metadata":{"papermill":{"duration":0.01369,"end_time":"2023-06-08T03:08:12.374106","exception":false,"start_time":"2023-06-08T03:08:12.360416","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if len(all_submissions) <= 10:\n    for series in all_submissions:\n        plt.figure(figsize=(20, 6))\n        plt.plot(series['StartHesitation'], label='StartHesitation')\n        plt.plot(series['Turn'], label='Turn')\n        plt.plot(series['Walking'], label='Walking')\n        plt.ylim([-0.1, 1.1])\n        plt.legend()\n        plt.show()","metadata":{"papermill":{"duration":4.304523,"end_time":"2023-06-08T03:08:16.692207","exception":false,"start_time":"2023-06-08T03:08:12.387684","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}