{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport warnings\nimport random\nimport tensorflow\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow import keras\nfrom tensorflow.keras.optimizers import Adam\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom scipy.interpolate import splev, splrep\nfrom scipy import interpolate","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:36:54.075047Z","iopub.execute_input":"2023-05-30T19:36:54.075816Z","iopub.status.idle":"2023-05-30T19:36:54.082935Z","shell.execute_reply.started":"2023-05-30T19:36:54.075775Z","shell.execute_reply":"2023-05-30T19:36:54.081736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random.seed(12345)\nrandomState = np.random.RandomState(123456)","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:36:54.085361Z","iopub.execute_input":"2023-05-30T19:36:54.086398Z","iopub.status.idle":"2023-05-30T19:36:54.099044Z","shell.execute_reply.started":"2023-05-30T19:36:54.086359Z","shell.execute_reply":"2023-05-30T19:36:54.097657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = '/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/'","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:36:54.101263Z","iopub.execute_input":"2023-05-30T19:36:54.102001Z","iopub.status.idle":"2023-05-30T19:36:54.113626Z","shell.execute_reply.started":"2023-05-30T19:36:54.101894Z","shell.execute_reply":"2023-05-30T19:36:54.112516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = True\n#DEBUG = False\nREDUCED_DATA = True\n#REDUCED_DATA = False\nSUBMIT = False\n#SUBMIT = True\n\nTRAIN_BATCH_SIZE = 128\nPREDICT_BATCH_SIZE = 128\n\nNUM_THRESHOLDS = 20","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:36:54.116086Z","iopub.execute_input":"2023-05-30T19:36:54.117475Z","iopub.status.idle":"2023-05-30T19:36:54.130258Z","shell.execute_reply.started":"2023-05-30T19:36:54.117396Z","shell.execute_reply":"2023-05-30T19:36:54.129127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"G = 9.80665\nBUFFER_LENGTH = 6 # (in sec)\nTDCSFOG_SPEED = 128 # frame per second\nDEFOG_SPEED = 100 # frame per second","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:36:54.132498Z","iopub.execute_input":"2023-05-30T19:36:54.132987Z","iopub.status.idle":"2023-05-30T19:36:54.145417Z","shell.execute_reply.started":"2023-05-30T19:36:54.132950Z","shell.execute_reply":"2023-05-30T19:36:54.144167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EVENTS = ['StartHesitation', 'Turn', 'Walking', 'NoEvent']","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:36:54.151881Z","iopub.execute_input":"2023-05-30T19:36:54.152891Z","iopub.status.idle":"2023-05-30T19:36:54.163869Z","shell.execute_reply.started":"2023-05-30T19:36:54.152843Z","shell.execute_reply":"2023-05-30T19:36:54.162383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resample_spline(data, new_len):\n    x = [i for i in range(0, len(data))]\n    y = data\n    x2 = np.linspace(0, len(data) - 1, new_len)\n    y2 = splev(x2, splrep(x, y))\n    return y2    ","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:36:54.180190Z","iopub.execute_input":"2023-05-30T19:36:54.180820Z","iopub.status.idle":"2023-05-30T19:36:54.185884Z","shell.execute_reply.started":"2023-05-30T19:36:54.180785Z","shell.execute_reply":"2023-05-30T19:36:54.185005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resample_X(data, label):\n    if label == 'tdcsfog':\n        return data[['AccV', 'AccML', 'AccAP']]\n    else:\n        new_len = BUFFER_LENGTH * TDCSFOG_SPEED\n        new_data = {}\n        for x in ['AccV', 'AccML', 'AccAP']:\n            new_data[x] = resample_spline(data[x].values, new_len)\n        return pd.DataFrame(new_data, columns = ['AccV', 'AccML', 'AccAP'])","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:36:54.216213Z","iopub.execute_input":"2023-05-30T19:36:54.216683Z","iopub.status.idle":"2023-05-30T19:36:54.223392Z","shell.execute_reply.started":"2023-05-30T19:36:54.216640Z","shell.execute_reply":"2023-05-30T19:36:54.222481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Episod:\n    def __init__(self, id, type, label, file_id, dataset): # not call directly\n        assert Episod.__id == id, 'not create Episod directly'\n        self.id = id\n        self.type = type\n        self.label = label\n        self.file_id = file_id\n        self.num_ticks = dataset.shape[0]\n        for acc in ['AccV', 'AccML', 'AccAP']:\n            setattr(self, 'mean'+acc, dataset[acc].mean())\n            setattr(self, 'min'+acc, dataset[acc].min())\n            setattr(self, 'max'+acc, dataset[acc].max())\n            setattr(self, 'std'+acc, dataset[acc].std())\n        assert self.num_ticks > 0, 'empty dataset for episod!'\n        return\n\n    def keys(self):\n        return Episod.__item_keys\n    \n    def __getitem__(self, key):\n        try:\n            return getattr(self, key)\n        except Exception as e:\n            print('error on key', key)\n            raise e\n    \n    __item_keys = ['id',\n                'type', 'label', 'file_id', 'num_ticks',\n                'meanAccV', 'meanAccML', 'meanAccAP',\n                'minAccV', 'minAccML', 'minAccAP',\n                'maxAccV', 'maxAccML', 'maxAccAP',\n                'stdAccV', 'stdAccML', 'stdAccAP'\n            ]\n    __id = 1 \n    __info = {} \n    # id -> Episod\n    \n    __static_cache = {} # id -> [start, end, columns]\n    __static_cache_data = None # dataframe\n    \n    __cache = {} # id -> [cache_tick, dataframe]\n    __cache_tick = 0\n\n    __max_cache_size = 10 # const\n    __cache_free_step = 5 # const\n\n    __frame_info = pd.DataFrame(columns = [x for x in __item_keys if not x == 'id']) # placeholder\n\n    @staticmethod\n    def load_to_static_cache(ids):\n        global __static_cache_data\n        Episod.__static_cache.clear()\n        Episod.__static_cache_data = None\n        \n        ids = set(ids)\n        \n        if len(ids) > 0:\n            Episod.__static_cache_data = pd.concat([\n                Episod.get(id).load() for id in tqdm(ids, desc = 'fill static cache')\n            ], axis = 0)\n            prev_end = 0\n            for id in ids:\n                e = Episod.get(id)\n                data = e.load()\n                Episod.__static_cache[id] = (\n                    prev_end, \n                    prev_end + data.shape[0],\n                    data.columns\n                )\n                prev_end = prev_end + data.shape[0]\n\n            \n    @staticmethod\n    def __load_dataframe(type, label, file_id) -> pd.DataFrame:\n        dataset = pd.read_csv(\n            ROOT_DIR + type + '/' + label + '/' + file_id + '.csv',\n            index_col = 'Time',\n            dtype = { \n                'Time' : 'int32', \n                'StartHesitation' : 'int8', \n                'Walking' : 'int8', \n                'Turn' : 'int8' }\n        )\n        if (type == 'train') and (label == 'defog'):\n            dataset['InvalidCounter'] = (~(dataset['Valid'] & dataset['Task'])).cumsum()\n            dataset = dataset.drop(columns = 'Task')\n            dataset = dataset.drop(columns = 'Valid')\n        if label == 'tdcsfog':\n            for x in ['AccV', 'AccML', 'AccAP']:\n                dataset[x] = dataset[x] / G\n        if type == 'train':\n            dataset['NoEvent'] = 1 - \\\n                (dataset['StartHesitation'] + \\\n                dataset['Walking'] + \\\n                dataset['Turn'])\n        return dataset\n\n##### class Episod initalizer\n        \n    @staticmethod\n    def init() -> pd.DataFrame:\n        global __id\n        global __info\n        global __frame_info\n        assert Episod.__id == 1, 'Episod already inialized'\n        for type in ['train', 'test']:\n            for label in ['tdcsfog', 'defog']:\n                for *_, filenames in os.walk(ROOT_DIR + type + '/' + label):\n                    for filename in tqdm(\n                        filenames, \n                        desc = 'load ' + type + ', ' + label\n                    ):\n                        file_id = filename.removesuffix('.csv')\n                        dataset = Episod.__load_dataframe(type, label, file_id)\n                        Episod.__info[Episod.__id] = Episod(Episod.__id, type, label, file_id, dataset)\n                        Episod.__id += 1\n        Episod.__frame_info = pd.DataFrame.from_records(\n            data = [dict(values) for _, values in Episod.__info.items()],\n            index = 'id',\n            columns = Episod.__item_keys,\n            exclude = ['id']\n        )\n\n        return Episod.__frame_info\n\n##### end class Episod initalizer\n    @staticmethod\n    def get(id: int):\n        return Episod.__info[id];\n    \n    @staticmethod\n    def get_info() -> pd.DataFrame:\n        return Episod.__frame_info;\n\n    @staticmethod\n    def __ensure_can_add_to_cache():  \n        if len(Episod.__cache) >= Episod.__max_cache_size:\n            id_2_tick = sorted(\n                [(id, v[0]) for id,v in Episod.__cache.items()],\n                key=lambda x: x[1]\n            )\n            for id, tick in id_2_tick[:Episod.__cache_free_step]:\n                del Episod.__cache[id]\n        return\n         \n    def load(self) -> pd.DataFrame:\n        try:\n            start, end, columns = Episod.__static_cache[self.id]\n            return Episod.__static_cache_data.iloc[start:end][columns]\n        except KeyError:\n            try:\n                tick, data = Episod.__cache[self.id]\n            except KeyError:\n                Episod.__ensure_can_add_to_cache()\n                data = Episod.__load_dataframe(self.type, self.label, self.file_id)\n                data['AccV'] = data['AccV'] - self.meanAccV\n                data['AccML'] = data['AccML'] - self.meanAccML\n                data['AccAP'] = data['AccAP'] - self.meanAccAP\n \n            tick = Episod.__cache_tick\n            Episod.__cache_tick += 1\n            Episod.__cache[self.id] = (tick, data)\n        return data        ","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:36:54.225170Z","iopub.execute_input":"2023-05-30T19:36:54.225782Z","iopub.status.idle":"2023-05-30T19:36:54.254012Z","shell.execute_reply.started":"2023-05-30T19:36:54.225741Z","shell.execute_reply":"2023-05-30T19:36:54.252996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Episod.init()","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:36:54.256132Z","iopub.execute_input":"2023-05-30T19:36:54.256609Z","iopub.status.idle":"2023-05-30T19:37:18.661704Z","shell.execute_reply.started":"2023-05-30T19:36:54.256489Z","shell.execute_reply":"2023-05-30T19:37:18.660797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"info = Episod.get_info()\nif info[(info['type'] == 'test')].shape[0] > 2: # run on real data\n    DEBUG = False \n    SUBMIT = True","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:18.662871Z","iopub.execute_input":"2023-05-30T19:37:18.663460Z","iopub.status.idle":"2023-05-30T19:37:18.670964Z","shell.execute_reply.started":"2023-05-30T19:37:18.663413Z","shell.execute_reply":"2023-05-30T19:37:18.669575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SUBMIT or not REDUCED_DATA:\n    NUM_TRAIN_FRAMES = TRAIN_BATCH_SIZE * 1000\n    NUM_TEST_FRAMES = TRAIN_BATCH_SIZE * 10\n    NUM_EPOCHS = 6 \nelse:\n    NUM_TRAIN_FRAMES = TRAIN_BATCH_SIZE * 10\n    NUM_TEST_FRAMES = TRAIN_BATCH_SIZE * 1\n    NUM_EPOCHS = 1","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:18.673627Z","iopub.execute_input":"2023-05-30T19:37:18.673975Z","iopub.status.idle":"2023-05-30T19:37:18.686967Z","shell.execute_reply.started":"2023-05-30T19:37:18.673946Z","shell.execute_reply":"2023-05-30T19:37:18.685761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Frame:\n    @staticmethod\n    def get_length_for(episod):\n        if episod.label == 'tdcsfog':\n            return TDCSFOG_SPEED * BUFFER_LENGTH\n        else:\n            return DEFOG_SPEED * BUFFER_LENGTH\n\n    def __init__(self, episod: Episod, target_index: int):\n        self.episod = episod\n        length = Frame.get_length_for(episod)\n        self.offset = target_index - (length // 2)\n        assert self.offset >= 0, 'No frame (offset too low)'\n        assert self.offset + length <= episod.num_ticks, 'No frame (no data)'\n        \n    def load(self):\n        dataset = self.episod.load()\n        length = Frame.get_length_for(self.episod)\n        framedataset = dataset.iloc[self.offset : self.offset + length]\n        return framedataset\n    \n    def isValidTarget(self):\n        if (self.episod.type == 'train') and (self.episod.label == 'defog'):\n            dataset = Episod.get(self.episod_id).load()\n            return dataset.iloc[offset]['InvalidCounter'] == \\\n                    dataset.iloc[offset + length - 1]['InvalidCounter']\n        else:\n            return True\n    \n    def get_y(self):\n        data = self.load()[['StartHesitation', 'Turn', 'Walking','NoEvent']]\n        return data.iloc[data.shape[0]//2].values\n    \n    def get_X(self):\n        return resample_X(self.load(), self.episod.label)","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:18.688479Z","iopub.execute_input":"2023-05-30T19:37:18.688927Z","iopub.status.idle":"2023-05-30T19:37:18.704198Z","shell.execute_reply.started":"2023-05-30T19:37:18.688881Z","shell.execute_reply":"2023-05-30T19:37:18.702858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"info = Episod.get_info()\ntrain_episod_ids = info[(info['type'] == 'train')]\n\nif SUBMIT:\n    _, test_episod_ids = train_test_split(\n            train_episod_ids,\n            test_size = 0.01,\n            random_state = randomState\n    )\nelse:\n    train_episod_ids, test_episod_ids = train_test_split(\n        train_episod_ids, \n        test_size=0.05, \n        random_state= randomState\n    )\n    if REDUCED_DATA:\n        train_episod_ids, _ = train_test_split(\n            train_episod_ids,\n            train_size = 0.1,\n            random_state = randomState\n        )\n        _, test_episod_ids = train_test_split(\n            test_episod_ids,\n            test_size = 0.05,\n            random_state = randomState\n        )\n        \ntrain_episod_ids = train_episod_ids[train_episod_ids['label'] == 'tdcsfog']\n\ntrain_episod_ids = train_episod_ids.index.values\ntest_episod_ids = test_episod_ids.index.values\n\nif DEBUG:\n    print('length train ids =', len(train_episod_ids))\n    print('length test ids =', len(test_episod_ids))","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:18.705592Z","iopub.execute_input":"2023-05-30T19:37:18.706255Z","iopub.status.idle":"2023-05-30T19:37:18.725980Z","shell.execute_reply.started":"2023-05-30T19:37:18.706211Z","shell.execute_reply":"2023-05-30T19:37:18.724726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model():\n    input_shape = (int(BUFFER_LENGTH * TDCSFOG_SPEED), 3, 1)\n    model = keras.Sequential([\n        keras.layers.Conv2D(5, (3,3),  input_shape = input_shape, \n                padding='valid', activation='relu'\n            ),\n        keras.layers.Conv2D(11, (7,1),  input_shape = input_shape, \n                padding='valid', activation='relu', strides = 3\n            ),\n        keras.layers.AveragePooling2D(pool_size = (5,1), strides = (2,1)),\n        keras.layers.Conv2D(21, (7,1),  input_shape = input_shape, strides = 2,\n                padding='valid', activation='relu'\n            ),\n        keras.layers.AveragePooling2D(pool_size = (3,1), strides = (3,1)),\n        keras.layers.Conv2D(31, (7,1),  input_shape = input_shape, strides = 2,\n                padding='valid', activation='relu'\n            ),\n        keras.layers.AveragePooling2D(pool_size = (3,1), strides = (3,1)),\n        keras.layers.Flatten(),\n        keras.layers.Dense(10, activation='relu'),\n        keras.layers.Dense(4, activation='softmax')\n    ])\n    lr = 0.0001\n    optimizer = Adam(lr = lr)\n    model.compile(\n        optimizer=optimizer, \n        loss='categorical_crossentropy',\n        metrics=['acc']\n    )\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:43:25.041965Z","iopub.execute_input":"2023-05-30T19:43:25.042395Z","iopub.status.idle":"2023-05-30T19:43:25.052184Z","shell.execute_reply.started":"2023-05-30T19:43:25.042363Z","shell.execute_reply":"2023-05-30T19:43:25.050898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEBUG:\n    print(create_model().summary())","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:43:30.194512Z","iopub.execute_input":"2023-05-30T19:43:30.194961Z","iopub.status.idle":"2023-05-30T19:43:30.337505Z","shell.execute_reply.started":"2023-05-30T19:43:30.194916Z","shell.execute_reply":"2023-05-30T19:43:30.336109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BatchSlicer:\n    def __init__(self, length: int, batch_size: int):\n        assert length >=0, 'invalid length, must be > 0'\n        assert batch_size > 0, 'invalid batch_size, must be > 0'\n        self._length = length\n        self._batch_size = batch_size\n        self._num_batches = (self._length + self._batch_size - 1) // self._batch_size\n        self._real_batch_size = self._length // self._num_batches\n        self._num_full_batches = self._length - self._num_batches * self._real_batch_size \n        self._num_short_batches = self._num_batches - self._num_full_batches\n    \n    def get_slices(self, index: int): # return -> (begin, end)\n        assert index>=0 and index < self._num_batches, 'invalid batch index'\n        if index < self._num_full_batches:\n            begin = index * (self._real_batch_size + 1)\n            end = begin + (self._real_batch_size + 1)\n        else:\n            begin = self._num_full_batches * ( self._real_batch_size + 1) + \\\n                (index - self._num_full_batches) * (self._real_batch_size)\n            end = begin + self._real_batch_size\n        return (begin, end)\n\n    def num_batches(self):\n        return self._num_batches","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:19.257269Z","iopub.status.idle":"2023-05-30T19:37:19.257694Z","shell.execute_reply.started":"2023-05-30T19:37:19.257493Z","shell.execute_reply":"2023-05-30T19:37:19.257512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class XyGenerator(keras.utils.Sequence):\n    def __init__(self, episod_ids, batch_size, num_frames, frame_generator):\n        self._episod_ids = episod_ids\n        self._batch_size = batch_size\n        self._num_frames = num_frames\n        self._slicer = BatchSlicer(num_frames, batch_size)\n        self._frame_generator = frame_generator\n        \n    def __len__(self):\n        return self._slicer.num_batches()\n    \n    def __get_next_frame(self):\n        while True:\n            episod_id = random.choice(self._episod_ids)\n            episod = Episod.get(episod_id)\n            target_index = random.randrange(episod.num_ticks)\n            try:\n                frame = self._frame_generator(episod, target_index)\n                if frame.isValidTarget():\n                    return frame\n            except Exception:\n                pass\n            \n    \n    def __getitem__(self, index):\n        index = index % self._slicer.num_batches()\n        begin, end = self._slicer.get_slices(index)\n        X = []\n        y = []\n        for i in range(begin, end):\n            frame = self.__get_next_frame()\n            X.append(frame.get_X())\n            y.append(frame.get_y())\n        X = np.array(X)\n        try:\n            X = X.reshape(X.shape[0], X.shape[1], X.shape[2], 1)\n        except Exception as e:\n            print(X.shape)\n            raise e\n        y = np.array(y)\n        return X, y\n            \n    def __on_epoch_end(self):\n        pass\n        ","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:19.259233Z","iopub.status.idle":"2023-05-30T19:37:19.259709Z","shell.execute_reply.started":"2023-05-30T19:37:19.259503Z","shell.execute_reply":"2023-05-30T19:37:19.259524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_predictions_for(model, episod: Episod):\n    data = episod.load()\n    frame_length = Frame.get_length_for(episod)\n    start_offset = frame_length // 2\n    end_offset = data.shape[0] - frame_length + start_offset\n    predicted = [(0,0,0,1)]*start_offset\n    \n    if end_offset > start_offset:\n        b = BatchSlicer(end_offset - start_offset, PREDICT_BATCH_SIZE)\n        for batch_index in range(b.num_batches()):\n            begin, end = b.get_slices(batch_index)\n            X = []\n            for target_index in range(start_offset + begin, start_offset + end):\n                try:\n                    f = Frame(episod, target_index)\n                except Exception as ex:\n                    print('start_offset =', start_offset)\n                    print('end_offset =', end_offset)\n                    print('batch_index =', batch_index)\n                    print('begin =', begin)\n                    print('end =', end)\n                    print('batch._num_full_batches =', b._num_full_batches)\n                    print('batch._num_short_batches =', b._num_short_batches)\n                    print('batch._num_batches =', b._num_batches)\n                    print('batch._length =', b._length)\n                    print('batch._real_batch_size =', b._real_batch_size)\n                    print('batch._batch_size =', b._batch_size)\n                    \n                    raise ex\n                X.append(f.get_X())\n            X = np.array(X)\n            X = X.reshape(-1, X.shape[1], X.shape[2], 1)\n            y_predict = model.predict(X, verbose = 0)\n            predicted.extend(y_predict)\n    else: \n        end_offset = start_offset\n    predicted.extend([(0,0,0,1)] * (data.shape[0] - end_offset))\n    assert len(predicted) == data.shape[0], 'prediction and target not same size'\n    return pd.DataFrame(predicted, columns = ['StartHesitation', 'Turn', 'Walking','NoEvent'])\n        \n        ","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:19.261276Z","iopub.status.idle":"2023-05-30T19:37:19.261816Z","shell.execute_reply.started":"2023-05-30T19:37:19.261621Z","shell.execute_reply":"2023-05-30T19:37:19.261642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_results(model, episod, num_levels = 10, scores = {}):\n    prediction = get_predictions_for(model, episod)\n    data = episod.load()\n    for event in EVENTS:\n        try:\n            event_scores = scores[event]\n        except KeyError:\n            event_scores = {}\n            event_scores[0] = { \n                'TP' : data[event].sum(), \n                'TN' : 0,\n                'FP' : data.shape[0] - data[event].sum(),\n                'FN' : 0\n            }\n            event_scores[1.0] = { \n                'TP' : 0,  \n                'TN' : data.shape[0] - data[event].sum(),\n                'FP' : 0,\n                'FN' : data[event].sum()\n            }            \n            scores[event] = event_scores\n\n        for level in range(num_levels):\n            threshold = (level + 1)/(num_levels + 1)\n            TP = ((data[event] == 1) & (prediction[event] > threshold)).sum()\n            TN = ((data[event] == 0) & (prediction[event] <= threshold)).sum()\n            FP = ((data[event] == 0) & (prediction[event] > threshold)).sum()\n            FN = ((data[event] == 1) & (prediction[event] <= threshold)).sum()\n            assert (TP + TN + FP + FN) == data.shape[0], 'sum events not mutch'\n            try:\n                threshold_scores = event_scores[threshold]\n            except KeyError:\n                threshold_scores = { 'TP': 0, 'TN': 0, 'FP' : 0, 'FN' : 0}\n                event_scores[threshold] = threshold_scores\n                \n            threshold_scores['TP'] = threshold_scores['TP'] + TP\n            threshold_scores['TN'] = threshold_scores['TN'] + TN\n            threshold_scores['FP'] = threshold_scores['FP'] + FP\n            threshold_scores['FN'] = threshold_scores['FN'] + FN\n    return scores","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:19.263111Z","iopub.status.idle":"2023-05-30T19:37:19.263489Z","shell.execute_reply.started":"2023-05-30T19:37:19.263295Z","shell.execute_reply":"2023-05-30T19:37:19.263312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ids = []\nfor x in train_episod_ids:\n    e = Episod.get(x)\n    if e.label == 'tdcsfog':\n        train_ids.append(e.id)","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:19.264730Z","iopub.status.idle":"2023-05-30T19:37:19.266820Z","shell.execute_reply.started":"2023-05-30T19:37:19.266533Z","shell.execute_reply":"2023-05-30T19:37:19.266564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nEpisod.load_to_static_cache(list(train_episod_ids) + list(test_episod_ids))\nmodel = create_model()\nmodel.fit(\n    XyGenerator(\n        train_ids, TRAIN_BATCH_SIZE, NUM_TRAIN_FRAMES, \n        lambda episod, target_index: Frame(episod, target_index)\n    ),\n    epochs = NUM_EPOCHS, \n    validation_data = XyGenerator(\n        test_episod_ids, TRAIN_BATCH_SIZE, NUM_TEST_FRAMES, \n        lambda episod, target_index: Frame(episod, target_index)\n    )\n)\nEpisod.load_to_static_cache([])","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:19.268509Z","iopub.status.idle":"2023-05-30T19:37:19.269310Z","shell.execute_reply.started":"2023-05-30T19:37:19.269012Z","shell.execute_reply":"2023-05-30T19:37:19.269040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nif DEBUG:\n    scores = {}\n    for x in tqdm([Episod.get(id) for id in test_episod_ids]):\n        scores = calc_results(model, x, NUM_THRESHOLDS, scores)","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:19.271215Z","iopub.status.idle":"2023-05-30T19:37:19.272175Z","shell.execute_reply.started":"2023-05-30T19:37:19.271891Z","shell.execute_reply":"2023-05-30T19:37:19.271920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_precision_recall_from_matrix(tp, tn, fp, fn):\n    precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n    recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n    return precision, recall        ","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:19.273700Z","iopub.status.idle":"2023-05-30T19:37:19.274602Z","shell.execute_reply.started":"2023-05-30T19:37:19.274276Z","shell.execute_reply":"2023-05-30T19:37:19.274305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_precision_recall(threshold_results):\n    for threshold in sorted([x for x in threshold_results]):\n        value = threshold_results[threshold]\n        prec, recall = calc_precision_recall_from_matrix(\n            tp = value['TP'],\n            tn = value['TN'],\n            fp = value['FP'],\n            fn = value['FN']\n        )\n        yield threshold, prec, recall       ","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:19.276372Z","iopub.status.idle":"2023-05-30T19:37:19.276956Z","shell.execute_reply.started":"2023-05-30T19:37:19.276682Z","shell.execute_reply":"2023-05-30T19:37:19.276710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_ap(threshold_results):\n    ap = 0\n    data = []\n    for threshold in sorted([x for x in threshold_results]):\n        value = threshold_results[threshold]\n        prec, recall = calc_precision_recall_from_matrix(\n            tp = value['TP'],\n            tn = value['TN'],\n            fp = value['FP'],\n            fn = value['FN']\n        )\n        data.append((recall, prec))\n    data = sorted(data, key = lambda x: x[0])\n    prev_recall = 0\n    ap = 0\n    for x in data:\n        ap = ap + x[1] * (x[0] - prev_recall)\n        prev_recall = x[0]\n    return ap","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:19.278168Z","iopub.status.idle":"2023-05-30T19:37:19.278574Z","shell.execute_reply.started":"2023-05-30T19:37:19.278368Z","shell.execute_reply":"2023-05-30T19:37:19.278386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEBUG:\n    event_ap = {}\n    for event, event_v in scores.items():\n        ap = calc_ap(event_v)\n        print('event =', event, ' ap=', ap)\n        event_ap[event] = ap\n    estimated_map = (event_ap['StartHesitation'] + event_ap['Turn'] + event_ap['Walking']) / 3\n    print('Estimated MAP =', estimated_map)","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:19.280015Z","iopub.status.idle":"2023-05-30T19:37:19.280394Z","shell.execute_reply.started":"2023-05-30T19:37:19.280214Z","shell.execute_reply":"2023-05-30T19:37:19.280232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEBUG:\n    draw_data = pd.concat([\n        pd.DataFrame(\n            calc_precision_recall(event_v),\n            columns = ['threshold', 'precision', 'recall']\n        ).assign(event = event) \n        for event, event_v in scores.items()   \n    ], axis = 0)\n    draw_data['f1'] = draw_data['precision'] + draw_data['recall']\n    draw_data['f1'] = draw_data['f1'].where(\n        draw_data['f1'] == 0, \n        2 * draw_data['precision'] * draw_data['recall'] / draw_data['f1']\n    )\n    draw_line_data = pd.melt(draw_data, id_vars = ['threshold', 'event'])\n    draw_line_data = draw_line_data.reset_index(drop = True)\n    g = sns.relplot(\n        draw_line_data,\n        x = 'threshold',\n        y = 'value',\n        col = 'variable',\n        row = 'event',\n        kind = 'line'\n    )\n    g.fig.set_size_inches(8,8)\n    g.fig.subplots_adjust(hspace=.25)\n    g.set_titles(row_template='{col_name}', col_template='{row_name}')\n    \n    plt.show()\n    sns.relplot(\n        draw_data[['event','recall','precision']],\n        x = 'recall',\n        y = 'precision',\n        col = 'event',\n        kind = 'line'\n    )\n        ","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:19.282968Z","iopub.status.idle":"2023-05-30T19:37:19.283592Z","shell.execute_reply.started":"2023-05-30T19:37:19.283273Z","shell.execute_reply":"2023-05-30T19:37:19.283301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndata_to_submit = info[info['type'] == 'test'].index.values\nif not SUBMIT:\n    data_to_submit = [data_to_submit[0]]\n\nfirst = True\ninfo = Episod.get_info()\nfor s in tqdm(data_to_submit):\n    episod = Episod.get(s)\n    if DEBUG:\n        print('load episod ' + episod.file_id + ' num ticks =', episod.num_ticks)\n    prediction = get_predictions_for(model, episod)\n    data = episod.load()\n    data = pd.concat([data, prediction], axis = 1)\n    data['Time'] = data.index.values\n    data['Id'] = episod.file_id + '_' + data['Time'].astype('str')\n    data = data[['Id', 'StartHesitation', 'Turn', 'Walking']]\n    if DEBUG:\n        print(data.info())\n    if first:\n        if DEBUG:\n            print('write to file with headers ...')\n        data.to_csv('submission.csv', index = False)\n        if DEBUG:\n            print('write done')\n        first = False\n    else:\n        if DEBUG:\n            print('append to file with headers ...')\n        data.to_csv('submission.csv', mode='a', header=False, index = False)\n        if DEBUG:\n            print('append done')\n","metadata":{"execution":{"iopub.status.busy":"2023-05-30T19:37:19.284856Z","iopub.status.idle":"2023-05-30T19:37:19.285396Z","shell.execute_reply.started":"2023-05-30T19:37:19.285129Z","shell.execute_reply":"2023-05-30T19:37:19.285154Z"},"trusted":true},"execution_count":null,"outputs":[]}]}