{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# - 2023/6/10 added -\n\n### This submission ensembles two models.\n### One is the [Public kernel](https://www.kaggle.com/code/coderrkj/parkinson-fog-pred-conv1d-separate-tf-model) and the other is [the original model with the hyperparameters changed (Version2 of this Notebook)](https://www.kaggle.com/code/itsuki9180/34th-place-solution?scriptVersionId=124699166).\n\n### Final submission is [Version10 of this Notebook](https://www.kaggle.com/code/itsuki9180/34th-place-solution?scriptVersionId=124996308).\n\n# - Postscript up to here -","metadata":{}},{"cell_type":"markdown","source":"## Conv1D TF Model with Separate Pipelines for defog and tdcsfog data\nFeature Column TimeSeries grouping adapted from https://www.kaggle.com/code/mayukh18/pytorch-fog-end-to-end-baseline-lb-0-254\n\nSubject-wise GroupKFold splitting adapted from https://www.kaggle.com/code/xzj19013742/groupkfold-cross-validation-tsflex\n### In this Notebook\n- Tensorflow Model with Conv1D blocks \n- Models trained separately for defog and tdcsfog data\n- Event Stratified Subject Grouped KFold splitting","metadata":{}},{"cell_type":"markdown","source":"## Imports and Config","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport numpy as np\nfrom numpy.random import default_rng\nimport pandas as pd\nfrom tqdm.auto import tqdm\nfrom glob import glob\nfrom os.path import basename, dirname, join, exists\nfrom time import perf_counter\nfrom collections import defaultdict as dd\nfrom functools import partial\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, StratifiedGroupKFold\nfrom sklearn.metrics import average_precision_score\nfrom sklearn.preprocessing import StandardScaler as Scaler\nfrom scipy.special import expit\n\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nprint(f\"TF version: {tf.__version__}\")\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-14T10:01:43.702349Z","iopub.execute_input":"2023-04-14T10:01:43.703629Z","iopub.status.idle":"2023-04-14T10:01:52.865442Z","shell.execute_reply.started":"2023-04-14T10:01:43.703578Z","shell.execute_reply":"2023-04-14T10:01:52.864204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Constants\n\nBASE_DIR = \"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction\"\nTRAIN_DIR = join(BASE_DIR, \"train\")\nTEST_DIR = join(BASE_DIR, \"test\")\n\nIS_PUBLIC = len(glob(join(TEST_DIR, \"*/*.csv\")))==2","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:01:52.868020Z","iopub.execute_input":"2023-04-14T10:01:52.868879Z","iopub.status.idle":"2023-04-14T10:01:52.880431Z","shell.execute_reply.started":"2023-04-14T10:01:52.868834Z","shell.execute_reply":"2023-04-14T10:01:52.879447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    train_sub_dirs = [\n        join(TRAIN_DIR, \"defog\"),\n        join(TRAIN_DIR, \"tdcsfog\")\n    ]\n    \n    metadata_paths = [\n        join(BASE_DIR, \"defog_metadata.csv\"),\n        join(BASE_DIR, \"tdcsfog_metadata.csv\")\n    ]\n    \n    splits = 5\n\n    batch_size = 1024\n    window_size = 64\n    window_future = 16\n    window_past = window_size - window_future # Includes current value\n    \n    wx = 8\n    \n    model_dropout = 0.2\n    model_hidden = 192\n    model_nblocks = 3\n    \n    lr = 0.00015\n    num_epochs = 6\n    \n    feature_list = ['AccV', 'AccML', 'AccAP']\n    label_list = ['StartHesitation', 'Turn', 'Walking']\n    \n    n_features = len(feature_list)\n    n_labels = len(label_list)    \n    \ncfg = Config()","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:01:52.882023Z","iopub.execute_input":"2023-04-14T10:01:52.882844Z","iopub.status.idle":"2023-04-14T10:01:52.891260Z","shell.execute_reply.started":"2023-04-14T10:01:52.882805Z","shell.execute_reply":"2023-04-14T10:01:52.890293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Stratified Group K Fold","metadata":{}},{"cell_type":"code","source":"# Create Mapping between Id and Subject\nid2sub_df = pd.concat([\n    pd.read_csv(f, usecols=['Id', 'Subject']).assign(Module=basename(f).split('_')[0]) for f in cfg.metadata_paths\n]).astype(\"category\").set_index(\"Id\")\nprint(f\"id2sub_df length: {len(id2sub_df)}, unique Ids: {id2sub_df.index.nunique()}, unique Subjects: {id2sub_df.Subject.nunique()}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:01:52.893913Z","iopub.execute_input":"2023-04-14T10:01:52.894982Z","iopub.status.idle":"2023-04-14T10:01:52.937262Z","shell.execute_reply.started":"2023-04-14T10:01:52.894945Z","shell.execute_reply":"2023-04-14T10:01:52.936225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read csv files and add metadata (Id, Subject, Event)\ndef reader(filepath, usecols, getid=False, getsub=False, getevent=False, dtype=None, exclude=['notype']):\n    fog_type = basename(dirname(filepath))\n    if fog_type in exclude:\n        return None\n    df = pd.read_csv(filepath, index_col=\"Time\", usecols=usecols, dtype=dtype)\n    if getid:\n        df['Id'] = basename(filepath).split('.')[0] + '_' + df.index.astype(str)\n    if getsub:\n        df['Subject'] = id2sub_df.loc[basename(filepath).split('.')[0], 'Subject']\n    if getevent:\n        df['Event'] = np.select(\n            [df[col].astype(bool) for col in cfg.label_list], \n            np.arange(1,cfg.n_labels+1), default=0\n        ).astype('int8')\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:01:52.938670Z","iopub.execute_input":"2023-04-14T10:01:52.939025Z","iopub.status.idle":"2023-04-14T10:01:52.947171Z","shell.execute_reply.started":"2023-04-14T10:01:52.938989Z","shell.execute_reply":"2023-04-14T10:01:52.945925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create common train Dataframe\ntrain_paths = glob(join(TRAIN_DIR, '*/*.csv'))\ndtype = {col:'int8' for col in cfg.label_list}\ndtype['Time'] = 'int32'\nusecols = ['Time', *cfg.label_list]\n\ntrain_reader = partial(reader, usecols=usecols, dtype=dtype, getsub=True, getevent=True)\ntrain_df = pd.concat([train_reader(f) for f in tqdm(train_paths)]).reset_index(drop=True)\ntrain_df.Subject = train_df.Subject.astype('category')\ndisplay(train_df.Event.value_counts().to_frame().style.background_gradient())","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:01:52.948720Z","iopub.execute_input":"2023-04-14T10:01:52.949459Z","iopub.status.idle":"2023-04-14T10:02:31.005541Z","shell.execute_reply.started":"2023-04-14T10:01:52.949422Z","shell.execute_reply":"2023-04-14T10:02:31.004121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save paths for each Stratified Group Fold for defog and tdcsfog separately\nsgkf = StratifiedGroupKFold(n_splits=cfg.splits, random_state=42, shuffle=True)\nfold_train_fpaths, fold_valid_fpaths = {'defog': [], 'tdcsfog':[]}, {'defog': [], 'tdcsfog':[]}\ndf_paths = {'defog':glob(join(cfg.train_sub_dirs[0],'*.csv')), 'tdcsfog': glob(join(cfg.train_sub_dirs[1],'*.csv'))}\nfor module, paths in df_paths.items():\n    print(f\"{module}:\")\n    sub_train_df = train_df[train_df.Subject.isin(id2sub_df.loc[id2sub_df.Module==module, 'Subject'])].reset_index(drop=True)\n    for i, (train_index, test_index) in enumerate(sgkf.split(sub_train_df.index, sub_train_df.Event, groups=sub_train_df.Subject)):\n        print(f\"\\tFold {i}:\", end=\" \")\n        train_subs = sub_train_df.loc[train_index, 'Subject'].unique()\n        test_subs = sub_train_df.loc[test_index, 'Subject'].unique()\n        print(f\"Subjects->train:{len(train_subs)}|test:{len(test_subs)}\")\n        train_ids = set(id2sub_df[id2sub_df.Subject.isin(train_subs)].index)\n        test_ids = set(id2sub_df[id2sub_df.Subject.isin(test_subs)].index)\n        fold_train_fpaths[module].append([f for f in paths if basename(f).split('.')[0] in train_ids])\n        fold_valid_fpaths[module].append([f for f in paths if basename(f).split('.')[0] in test_ids])\n        del train_subs, test_subs, train_ids, test_ids\n        gc.collect()\n    del sub_train_df\ndel train_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:02:31.007426Z","iopub.execute_input":"2023-04-14T10:02:31.007817Z","iopub.status.idle":"2023-04-14T10:03:22.101520Z","shell.execute_reply.started":"2023-04-14T10:02:31.007777Z","shell.execute_reply":"2023-04-14T10:03:22.100363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"# Adapted from FOGDataset of https://www.kaggle.com/code/mayukh18/pytorch-fog-end-to-end-baseline-lb-0-254\nclass FOGSequence(tf.keras.utils.Sequence):\n\n    def __init__(self, df_paths, cfg=cfg, split=\"train\"):\n        _time = perf_counter()\n        \n        self.rng = default_rng(42)\n        self.cfg = cfg\n        self.split = split\n        \n        self.past_pad = self.cfg.wx*(self.cfg.window_past-1)\n        self.future_pad = self.cfg.wx*self.cfg.window_future\n        \n        if self.split == \"test\":\n            self.Ids = []\n        _values = [self._read(f) for f in df_paths]\n        \n        self.mapping = []\n        _length = 0\n        for _value in _values:\n            _shape = _value.shape[0]\n            self.mapping.extend(range(_length+self.past_pad, _length+_shape-self.future_pad))\n            _length += _shape\n            \n        self.values = np.concatenate(_values, axis=0)\n        self.mapping = np.array(self.mapping)\n        if self.split != \"test\":\n            # Keep only vaild and task rows\n            _valid_pos = self.values[self.mapping,self.valid_position] > 0\n            _task_pos = self.values[self.mapping,self.task_position] > 0\n            self.mapping = self.mapping[_valid_pos&_task_pos]\n        self.length = self.mapping.shape[0]\n        \n        print(f\"Valid Dataset of size {self.length:,} initialized in {perf_counter() - _time:.3f} secs!\")\n        gc.collect()\n    \n    def _read(self, path):\n        _is_tdcs = basename(dirname(path)).startswith('tdcs')\n        df = pd.read_csv(path)\n        \n        if self.split == \"test\":\n            _ids = basename(path).split('.')[0] + '_' + df.Time.astype(str)\n            self.Ids.extend(_ids.tolist())\n            return self._df_to_array(df, self.cfg.feature_list)\n        \n        _cols = [*self.cfg.feature_list, *self.cfg.label_list, 'Valid', 'Task']\n        self.valid_position = self.cfg.n_features + self.cfg.n_labels\n        self.task_position = self.valid_position + 1\n        \n        if _is_tdcs:\n            # Fill Valid and Task columns for tdcsfog\n            df['Valid'] = 1\n            df['Task'] = 1\n            \n        return self._df_to_array(df, _cols)\n    \n    def _df_to_array(self, df, cols):\n        # Pads past and future rows to dataframe values for indexing \n        _values = df[cols].values.astype(np.float16)\n        return np.pad(_values, ((self.past_pad, self.future_pad),(0,0)), 'edge')\n    \n    def __len__(self):\n        return int(np.ceil(self.length / self.cfg.batch_size))\n    \n    def __getitem__(self, idx):\n        \n        if self.split == \"train\":\n            # Onlt train set has randomly selected batches\n            _idxs = self.rng.choice(self.mapping, size=self.cfg.batch_size, replace=False)\n        else:\n            _idxs = self._get_indices(idx)\n            \n        # For test return only features\n        if self.split == \"test\":\n            return self._get_X(_idxs)\n        # For train and val splits return y also\n        return self._get_X_y(_idxs)\n    \n    def _get_indices(self, idx):\n        _low = idx * self.cfg.batch_size\n        # Cap high at self.length so overflow does not occur\n        _high = min(_low + self.cfg.batch_size, self.length)\n        return self.mapping[_low:_high]\n    \n    def _get_X(self, indices):\n        _X = np.empty((len(indices), self.cfg.window_size, self.cfg.n_features), dtype=np.float16)\n        for i, idx in enumerate(indices):\n            _X[i] = self.values[idx-self.past_pad:idx+self.future_pad+1:self.cfg.wx, :self.cfg.n_features]\n        return _X\n    \n    def _get_X_y(self, indices):\n        _X = np.empty((len(indices), self.cfg.window_size, self.cfg.n_features), dtype=np.float16)\n        for i, idx in enumerate(indices):\n            _X[i] = self.values[idx-self.past_pad: idx+self.future_pad+1:self.cfg.wx, :self.cfg.n_features]\n        return _X, self.values[indices, self.cfg.n_features:self.cfg.n_features+self.cfg.n_labels]","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:03:22.103342Z","iopub.execute_input":"2023-04-14T10:03:22.103987Z","iopub.status.idle":"2023-04-14T10:03:22.124824Z","shell.execute_reply.started":"2023-04-14T10:03:22.103946Z","shell.execute_reply":"2023-04-14T10:03:22.123514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"# average_precision_score with positive sample added if no true positive cases are present\ndef calculate_precision(y_true, y_pred):\n    pad_width = ((0,0),(0,0)) if y_true.any(axis=0).all() else ((1,0),(0,0))\n    y_true, y_pred = np.pad(y_true, pad_width, constant_values=1), np.pad(y_pred, pad_width, constant_values=1)\n    return average_precision_score(y_true, y_pred)","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:03:22.126391Z","iopub.execute_input":"2023-04-14T10:03:22.126836Z","iopub.status.idle":"2023-04-14T10:03:22.141838Z","shell.execute_reply.started":"2023-04-14T10:03:22.126798Z","shell.execute_reply":"2023-04-14T10:03:22.140866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Note: Not same result as average_precision_score\nclass AveragePrecision(tf.keras.metrics.Metric):\n\n    def __init__(self, num_classes, thresholds=None, name='avg_precision', **kwargs):\n        super(AveragePrecision, self).__init__(name=name, **kwargs)\n        self.class_precision = [tf.keras.metrics.Precision(thresholds) for _ in range(num_classes)]\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        for i, precision in enumerate(self.class_precision):\n            precision.update_state(y_true[...,i], y_pred[..., i])\n\n    def result(self):\n        return tf.math.reduce_mean([precision.result() for precision in self.class_precision])\n    \n    def reset_state(self):\n        for precision in self.class_precision:\n            precision.reset_state()","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:03:22.146743Z","iopub.execute_input":"2023-04-14T10:03:22.147068Z","iopub.status.idle":"2023-04-14T10:03:22.157266Z","shell.execute_reply.started":"2023-04-14T10:03:22.147041Z","shell.execute_reply":"2023-04-14T10:03:22.156232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model adapted from https://keras.io/examples/timeseries/timeseries_classification_from_scratch/\ndef get_model(checkpoint_path = None):\n    model = tf.keras.models.Sequential()\n    model.add(tf.keras.Input(shape=(cfg.window_size, cfg.n_features), dtype='float16'))\n    for _ in range(cfg.model_nblocks):\n        model.add(tf.keras.layers.Conv1D(filters=cfg.model_hidden, kernel_size=31, padding=\"same\"))\n        model.add(tf.keras.layers.BatchNormalization())\n        model.add(tf.keras.layers.PReLU())\n        model.add(tf.keras.layers.Dropout(cfg.model_dropout))\n    model.add(tf.keras.layers.GlobalAveragePooling1D())\n    model.add(tf.keras.layers.Dense(cfg.n_labels, activation=None))\n\n    if checkpoint_path is not None:\n        model.load_weights(checkpoint_path)\n    model.compile(\n        tf.keras.optimizers.Adam(learning_rate=cfg.lr), \n        loss = tf.keras.losses.BinaryCrossentropy(from_logits=True),\n        metrics=[AveragePrecision(cfg.n_labels, thresholds=0.0)]\n    )\n    return model\n\nget_model().summary()","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:03:22.159143Z","iopub.execute_input":"2023-04-14T10:03:22.159523Z","iopub.status.idle":"2023-04-14T10:03:24.767922Z","shell.execute_reply.started":"2023-04-14T10:03:22.159487Z","shell.execute_reply":"2023-04-14T10:03:24.767067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model_pub(checkpoint_path = None):\n    model = tf.keras.models.Sequential()\n    model.add(tf.keras.Input(shape=(cfg.window_size, cfg.n_features), dtype='float16'))\n    for _ in range(cfg.model_nblocks):\n        model.add(tf.keras.layers.Conv1D(filters=128, kernel_size=15, padding=\"same\"))\n        model.add(tf.keras.layers.BatchNormalization())\n        model.add(tf.keras.layers.ReLU())\n        model.add(tf.keras.layers.Dropout(cfg.model_dropout))\n    model.add(tf.keras.layers.GlobalAveragePooling1D())\n    model.add(tf.keras.layers.Dense(cfg.n_labels, activation=None))\n\n    if checkpoint_path is not None:\n        model.load_weights(checkpoint_path)\n    model.compile(\n        tf.keras.optimizers.Adam(learning_rate=cfg.lr), \n        loss = tf.keras.losses.BinaryCrossentropy(from_logits=True),\n        metrics=[AveragePrecision(cfg.n_labels, thresholds=0.0)]\n    )\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:03:24.769032Z","iopub.execute_input":"2023-04-14T10:03:24.769419Z","iopub.status.idle":"2023-04-14T10:03:24.793128Z","shell.execute_reply.started":"2023-04-14T10:03:24.769361Z","shell.execute_reply":"2023-04-14T10:03:24.792220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import *\n\ndef get_model_wn(checkpoint_path=None):\n    \n    def cbr(x, out_layer, kernel, stride, dilation):\n        x = Conv1D(out_layer, kernel_size=kernel, dilation_rate=dilation, strides=stride, padding=\"same\")(x)\n        x = BatchNormalization()(x)\n        x = Activation(\"relu\")(x)\n        return x\n    \n    def wave_block(x, filters, kernel_size, n):\n        dilation_rates = [2**i for i in range(n)]\n        x = Conv1D(filters = filters,\n                   kernel_size = 1,\n                   padding = 'same')(x)\n        res_x = x\n        for dilation_rate in dilation_rates:\n            tanh_out = Conv1D(filters = filters,\n                              kernel_size = kernel_size,\n                              padding = 'same', \n                              activation = 'tanh', \n                              dilation_rate = dilation_rate)(x)\n            sigm_out = Conv1D(filters = filters,\n                              kernel_size = kernel_size,\n                              padding = 'same',\n                              activation = 'sigmoid', \n                              dilation_rate = dilation_rate)(x)\n            x = Multiply()([tanh_out, sigm_out])\n            x = Conv1D(filters = filters,\n                       kernel_size = 1,\n                       padding = 'same')(x)\n            res_x = Add()([res_x, x])\n        return res_x\n    \n    inp = Input(shape = (cfg.window_size, cfg.n_features))\n    x = cbr(inp, 64, 7, 1, 1)\n    x = BatchNormalization()(x)\n    x = wave_block(x, 16, 3, 12)\n    x = BatchNormalization()(x)\n    x = wave_block(x, 32, 3, 8)\n    x = BatchNormalization()(x)\n    x = wave_block(x, 64, 3, 4)\n    x = BatchNormalization()(x)\n    x = wave_block(x, 128, 3, 1)\n    x = cbr(x, 32, 7, 1, 1)\n    #x = BatchNormalization()(x)\n    x = tf.keras.layers.GlobalAveragePooling1D()(x)\n    #x = Dropout(0.2)(x)\n    out = Dense(3, activation = None, name = 'out')(x)\n    \n    model = tf.keras.models.Model(inputs = inp, outputs = out)\n    \n    opt = tf.keras.optimizers.Adam(lr = cfg.lr)\n    #opt = tfa.optimizers.SWA(opt)\n    #model.compile(loss = losses.CategoricalCrossentropy(), optimizer = opt, metrics = ['accuracy'])\n    if checkpoint_path is not None:\n        model.load_weights(checkpoint_path)\n    model.compile(\n        opt, \n        loss = tf.keras.losses.BinaryCrossentropy(from_logits=True),\n        metrics=[AveragePrecision(cfg.n_labels, thresholds=0.0)]\n    )\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:03:24.794297Z","iopub.execute_input":"2023-04-14T10:03:24.794669Z","iopub.status.idle":"2023-04-14T10:03:24.813727Z","shell.execute_reply.started":"2023-04-14T10:03:24.794628Z","shell.execute_reply":"2023-04-14T10:03:24.812354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"def predict_select_model(fold, ds, model_save_dir=''):\n    best_path, best_score = None, -1\n    print(f\"Validation for fold{fold}:\")\n    for model_path in sorted(glob(join(model_save_dir, f\"fold{fold}_*.h5\"))):\n        pred_time = perf_counter()\n        gc.collect()\n        score = calculate_precision(\n            ds.values[ds.mapping, cfg.n_features:cfg.n_features + cfg.n_labels], \n            expit(get_model(model_path).predict(ds, verbose=0)) # expit converts to sigmoid output\n        )\n        if best_score < score:\n            best_score = score\n            best_path = model_path\n        gc.collect()\n        print(\"\\t\", basename(model_path), f\": score-{score:.4f} in {(perf_counter()-pred_time)/60:.2f} mins\")\n    print(basename(best_path), \"selected with score\", best_score)\n    return best_path","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:03:24.815463Z","iopub.execute_input":"2023-04-14T10:03:24.816150Z","iopub.status.idle":"2023-04-14T10:03:24.827194Z","shell.execute_reply.started":"2023-04-14T10:03:24.816114Z","shell.execute_reply":"2023-04-14T10:03:24.826230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import LearningRateScheduler\ndef get_lr_callback(FOG_TYPE):\n    lr_start   = 0.00015 if FOG_TYPE=='tdcsfog' else 1e-4\n    lr_max     = 0.00015 if FOG_TYPE=='tdcsfog' else 1e-4\n    lr_min     = 1e-6\n    lr_ramp_ep = 0\n    lr_sus_ep  = 0\n    lr_decay   = 0.713 if FOG_TYPE=='tdcsfog' else 0.69\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start   \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max    \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min    \n        return lr\n\n    lr_callback = LearningRateScheduler(lrfn, verbose = 1)\n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:03:24.828320Z","iopub.execute_input":"2023-04-14T10:03:24.830400Z","iopub.status.idle":"2023-04-14T10:03:24.838098Z","shell.execute_reply.started":"2023-04-14T10:03:24.830363Z","shell.execute_reply":"2023-04-14T10:03:24.837069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_loop(train_paths, valid_paths, fold, model_save_dir=''):\n    gc.collect()\n    \n    # train_paths, test_paths = train_test_split(train_paths, test_size=0.4)\n    train_ds = FOGSequence(train_paths)\n    val_ds = FOGSequence(valid_paths, split=\"val\")\n    # test_ds = FOGSequence(test_paths, split=\"val\")\n    \n    model = get_model()\n    ckpt = tf.keras.callbacks.ModelCheckpoint(join(model_save_dir, f\"fold{fold}_model_\"+\"{epoch:02d}.h5\"), save_weights_only=True)\n    history = model.fit(train_ds,\n                        epochs=cfg.num_epochs,\n                        verbose=2, \n                        workers=5, \n                        validation_data=val_ds, \n                        use_multiprocessing=True, \n                        callbacks=[ckpt, get_lr_callback(model_save_dir)])\n    \n    best_model_path = predict_select_model(fold, val_ds, model_save_dir)\n    # score = calculate_precision(test_ds.values[test_ds.mapping, cfg.n_features:cfg.n_features + cfg.n_labels], expit(get_model(best_model_path).predict(test_ds, verbose=0)))\n    \n    del train_ds, val_ds, model, ckpt, history\n    gc.collect()\n    return best_model_path\n","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:03:24.841205Z","iopub.execute_input":"2023-04-14T10:03:24.841496Z","iopub.status.idle":"2023-04-14T10:03:24.849926Z","shell.execute_reply.started":"2023-04-14T10:03:24.841462Z","shell.execute_reply":"2023-04-14T10:03:24.848870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Main training loop\nmodel_paths = {'defog': [], 'tdcsfog':[]}\nfor module in model_paths:\n    module_start = perf_counter()\n    print(f\"***Training {module}{'*'*75}\")\n    if not exists(module): \n        os.mkdir(module)\n    for fold, (train_fpaths, valid_fpaths) in enumerate(zip(fold_train_fpaths[module], fold_valid_fpaths[module])):\n        #if fold!=0: continue\n        fold_start = perf_counter()\n        print(f\"Fold {fold}{'-'*25}\")\n        model_paths[module].append(train_loop(train_fpaths, valid_fpaths, fold, model_save_dir=module))\n        print(f\"Fold {fold} done in {(perf_counter()-fold_start)/60:.2f} min\")\n    print(f\"***{module} done in {(perf_counter()-module_start)/3600:.2f} hrs{'*'*50}\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-04-14T10:03:24.853113Z","iopub.execute_input":"2023-04-14T10:03:24.853526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_paths = {\n#     'defog': [\n#         '/kaggle/input/fog-conv1d-models/defog/fold0_model_02.h5',\n#         '/kaggle/input/fog-conv1d-models/defog/fold1_model_01.h5',\n#         '/kaggle/input/fog-conv1d-models/defog/fold2_model_01.h5',\n#         '/kaggle/input/fog-conv1d-models/defog/fold3_model_05.h5',\n#         '/kaggle/input/fog-conv1d-models/defog/fold4_model_02.h5',\n        \n#         '/kaggle/input/fog-conv1d-models-pub/defog/fold0_model_02.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/defog/fold1_model_02.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/defog/fold3_model_05.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/defog/fold4_model_02.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/defog/fold5_model_05.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/defog/fold6_model_05.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/defog/fold7_model_04.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/defog/fold8_model_01.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/defog/fold9_model_05.h5',\n#     ], \n#     'tdcsfog':[\n#         '/kaggle/input/fog-conv1d-models/tdcsfog/fold0_model_01.h5',\n#         '/kaggle/input/fog-conv1d-models/tdcsfog/fold1_model_01.h5',\n#         '/kaggle/input/fog-conv1d-models/tdcsfog/fold2_model_05.h5',\n#         '/kaggle/input/fog-conv1d-models/tdcsfog/fold3_model_05.h5',\n#         '/kaggle/input/fog-conv1d-models/tdcsfog/fold4_model_04.h5',\n        \n#         '/kaggle/input/fog-conv1d-models-pub/tdcsfog/fold0_model_05.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/tdcsfog/fold1_model_03.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/tdcsfog/fold2_model_04.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/tdcsfog/fold3_model_04.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/tdcsfog/fold4_model_02.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/tdcsfog/fold5_model_04.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/tdcsfog/fold6_model_02.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/tdcsfog/fold7_model_02.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/tdcsfog/fold8_model_05.h5',\n#         '/kaggle/input/fog-conv1d-models-pub/tdcsfog/fold9_model_05.h5',\n#     ]}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"test_defog_paths = glob(join(TEST_DIR, \"defog/*.csv\"))\ntest_tdcsfog_paths = glob(join(TEST_DIR, \"tdcsfog/*.csv\"))\n\ntest_ds_dict = {\n    'defog':FOGSequence(test_defog_paths, split=\"test\"), \n    'tdcsfog':FOGSequence(test_tdcsfog_paths, split=\"test\")\n}\n\n# Get test predictions\ndf_list = []\nfor module, test_ds in test_ds_dict.items():\n    y_pred_list = []\n    for model_path in model_paths[module]:\n        if 'models-pub' in model_path:\n            model = get_model_pub(model_path)\n        else:\n            model = get_model(model_path)\n        y_pred_list.append(expit(model.predict(test_ds, verbose=0)))  # expit converts to sigmoid output\n    y_pred = np.mean(y_pred_list, axis=0)\n    df_list.append(pd.DataFrame(\n        {'Id': test_ds.Ids, 'StartHesitation': y_pred[:,0], 'Turn': y_pred[:,1], 'Walking': y_pred[:,2]}))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Concatenate Prediction to DataFrames\nsubmission = pd.concat(df_list)\n\n# Only keep Ids in sample_submission\nsample_submission = pd.read_csv(join(BASE_DIR, \"sample_submission.csv\"))\nsubmission = pd.merge(sample_submission[['Id']], submission, how='left', on='Id').fillna(0.0)\nsubmission.to_csv(\"submission.csv\", index=False, float_format='%.6f') # round to 5 decimal places while keeping point notation","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head -5 submission.csv","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}