{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport seaborn as sns\nimport random\nfrom matplotlib import pyplot as plt\n\nfrom functools import lru_cache \nfrom collections.abc import Callable\nfrom scipy import stats\nfrom dataclasses import dataclass","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-25T18:24:08.604112Z","iopub.execute_input":"2024-02-25T18:24:08.604923Z","iopub.status.idle":"2024-02-25T18:24:09.521639Z","shell.execute_reply.started":"2024-02-25T18:24:08.604869Z","shell.execute_reply":"2024-02-25T18:24:09.520578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q bottleneck\nimport bottleneck","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:09.523816Z","iopub.execute_input":"2024-02-25T18:24:09.524432Z","iopub.status.idle":"2024-02-25T18:24:21.806109Z","shell.execute_reply.started":"2024-02-25T18:24:09.524391Z","shell.execute_reply":"2024-02-25T18:24:21.804638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = '/kaggle/input/hms-harmful-brain-activity-classification'\nTRAIN_EEGS = os.path.join(ROOT_DIR , 'train_eegs')\nTRAIN_SPECTR =  os.path.join(ROOT_DIR, 'train_spectrograms')\nTEST_EEGS = os.path.join(ROOT_DIR, 'test_eegs')\nTEST_SPECTR = os.path.join(ROOT_DIR, 'test_spectrograms')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:21.807745Z","iopub.execute_input":"2024-02-25T18:24:21.808129Z","iopub.status.idle":"2024-02-25T18:24:21.814554Z","shell.execute_reply.started":"2024-02-25T18:24:21.808092Z","shell.execute_reply":"2024-02-25T18:24:21.813315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EEG_FRAME_PER_SECOND = 200\nEEG_FRAME = 50 * EEG_FRAME_PER_SECOND\nSPECTR_FRAME = 10 * 60 // 2","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:21.817356Z","iopub.execute_input":"2024-02-25T18:24:21.817766Z","iopub.status.idle":"2024-02-25T18:24:21.826250Z","shell.execute_reply.started":"2024-02-25T18:24:21.817711Z","shell.execute_reply":"2024-02-25T18:24:21.824946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False\nSKIP_ASSERT = True","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:21.827976Z","iopub.execute_input":"2024-02-25T18:24:21.828419Z","iopub.status.idle":"2024-02-25T18:24:21.838436Z","shell.execute_reply.started":"2024-02-25T18:24:21.828389Z","shell.execute_reply":"2024-02-25T18:24:21.837605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SPECTR_COLUMNS = [ # to assert columns order is the same\n    'time',\n    'LL_0.59', 'LL_0.78', 'LL_0.98', 'LL_1.17', 'LL_1.37', \n    'LL_1.56', 'LL_1.76', 'LL_1.95', 'LL_2.15', 'LL_2.34', \n    'LL_2.54', 'LL_2.73', 'LL_2.93', 'LL_3.13', 'LL_3.32', \n    'LL_3.52', 'LL_3.71', 'LL_3.91', 'LL_4.1', 'LL_4.3', \n    'LL_4.49', 'LL_4.69', 'LL_4.88', 'LL_5.08', 'LL_5.27', \n    'LL_5.47', 'LL_5.66', 'LL_5.86', 'LL_6.05', 'LL_6.25', \n    'LL_6.45', 'LL_6.64', 'LL_6.84', 'LL_7.03', 'LL_7.23', \n    'LL_7.42', 'LL_7.62', 'LL_7.81', 'LL_8.01', 'LL_8.2', \n    'LL_8.4', 'LL_8.59', 'LL_8.79', 'LL_8.98', 'LL_9.18', \n    'LL_9.38', 'LL_9.57', 'LL_9.77', 'LL_9.96', 'LL_10.16', \n    'LL_10.35', 'LL_10.55', 'LL_10.74', 'LL_10.94', 'LL_11.13', \n    'LL_11.33', 'LL_11.52', 'LL_11.72', 'LL_11.91', 'LL_12.11', \n    'LL_12.3', 'LL_12.5', 'LL_12.7', 'LL_12.89', 'LL_13.09', \n    'LL_13.28', 'LL_13.48', 'LL_13.67', 'LL_13.87', 'LL_14.06', \n    'LL_14.26', 'LL_14.45', 'LL_14.65', 'LL_14.84', 'LL_15.04', \n    'LL_15.23', 'LL_15.43', 'LL_15.63', 'LL_15.82', 'LL_16.02', \n    'LL_16.21', 'LL_16.41', 'LL_16.6', 'LL_16.8', 'LL_16.99', \n    'LL_17.19', 'LL_17.38', 'LL_17.58', 'LL_17.77', 'LL_17.97', \n    'LL_18.16', 'LL_18.36', 'LL_18.55', 'LL_18.75', 'LL_18.95', \n    'LL_19.14', 'LL_19.34', 'LL_19.53', 'LL_19.73', 'LL_19.92', \n    'RL_0.59', 'RL_0.78', 'RL_0.98', 'RL_1.17', 'RL_1.37', \n    'RL_1.56', 'RL_1.76', 'RL_1.95', 'RL_2.15', 'RL_2.34', \n    'RL_2.54', 'RL_2.73', 'RL_2.93', 'RL_3.13', 'RL_3.32', \n    'RL_3.52', 'RL_3.71', 'RL_3.91', 'RL_4.1', 'RL_4.3', \n    'RL_4.49', 'RL_4.69', 'RL_4.88', 'RL_5.08', 'RL_5.27', \n    'RL_5.47', 'RL_5.66', 'RL_5.86', 'RL_6.05', 'RL_6.25', \n    'RL_6.45', 'RL_6.64', 'RL_6.84', 'RL_7.03', 'RL_7.23', \n    'RL_7.42', 'RL_7.62', 'RL_7.81', 'RL_8.01', 'RL_8.2', \n    'RL_8.4', 'RL_8.59', 'RL_8.79', 'RL_8.98', 'RL_9.18', \n    'RL_9.38', 'RL_9.57', 'RL_9.77', 'RL_9.96', 'RL_10.16', \n    'RL_10.35', 'RL_10.55', 'RL_10.74', 'RL_10.94', 'RL_11.13', \n    'RL_11.33', 'RL_11.52', 'RL_11.72', 'RL_11.91', 'RL_12.11', \n    'RL_12.3', 'RL_12.5', 'RL_12.7', 'RL_12.89', 'RL_13.09', \n    'RL_13.28', 'RL_13.48', 'RL_13.67', 'RL_13.87', 'RL_14.06', \n    'RL_14.26', 'RL_14.45', 'RL_14.65', 'RL_14.84', 'RL_15.04', \n    'RL_15.23', 'RL_15.43', 'RL_15.63', 'RL_15.82', 'RL_16.02', \n    'RL_16.21', 'RL_16.41', 'RL_16.6', 'RL_16.8', 'RL_16.99',\n    'RL_17.19', 'RL_17.38', 'RL_17.58', 'RL_17.77', 'RL_17.97', \n    'RL_18.16', 'RL_18.36', 'RL_18.55', 'RL_18.75', 'RL_18.95', \n    'RL_19.14', 'RL_19.34', 'RL_19.53', 'RL_19.73', 'RL_19.92', \n    'LP_0.59', 'LP_0.78', 'LP_0.98', 'LP_1.17', 'LP_1.37', \n    'LP_1.56', 'LP_1.76', 'LP_1.95', 'LP_2.15', 'LP_2.34', \n    'LP_2.54', 'LP_2.73', 'LP_2.93', 'LP_3.13', 'LP_3.32', \n    'LP_3.52', 'LP_3.71', 'LP_3.91', 'LP_4.1', 'LP_4.3', \n    'LP_4.49', 'LP_4.69', 'LP_4.88', 'LP_5.08', 'LP_5.27', \n    'LP_5.47', 'LP_5.66', 'LP_5.86', 'LP_6.05', 'LP_6.25', \n    'LP_6.45', 'LP_6.64', 'LP_6.84', 'LP_7.03', 'LP_7.23', \n    'LP_7.42', 'LP_7.62', 'LP_7.81', 'LP_8.01', 'LP_8.2', \n    'LP_8.4', 'LP_8.59', 'LP_8.79', 'LP_8.98', 'LP_9.18', \n    'LP_9.38', 'LP_9.57', 'LP_9.77', 'LP_9.96', 'LP_10.16', \n    'LP_10.35', 'LP_10.55', 'LP_10.74', 'LP_10.94', 'LP_11.13',\n    'LP_11.33', 'LP_11.52', 'LP_11.72', 'LP_11.91', 'LP_12.11', \n    'LP_12.3', 'LP_12.5', 'LP_12.7', 'LP_12.89', 'LP_13.09', \n    'LP_13.28', 'LP_13.48', 'LP_13.67', 'LP_13.87', 'LP_14.06', \n    'LP_14.26', 'LP_14.45', 'LP_14.65', 'LP_14.84', 'LP_15.04', \n    'LP_15.23', 'LP_15.43', 'LP_15.63', 'LP_15.82', 'LP_16.02', \n    'LP_16.21', 'LP_16.41', 'LP_16.6', 'LP_16.8', 'LP_16.99', \n    'LP_17.19', 'LP_17.38', 'LP_17.58', 'LP_17.77', 'LP_17.97', \n    'LP_18.16', 'LP_18.36', 'LP_18.55', 'LP_18.75', 'LP_18.95', \n    'LP_19.14', 'LP_19.34', 'LP_19.53', 'LP_19.73', 'LP_19.92', \n    'RP_0.59', 'RP_0.78', 'RP_0.98', 'RP_1.17', 'RP_1.37', \n    'RP_1.56', 'RP_1.76', 'RP_1.95', 'RP_2.15', 'RP_2.34', \n    'RP_2.54', 'RP_2.73', 'RP_2.93', 'RP_3.13', 'RP_3.32', \n    'RP_3.52', 'RP_3.71', 'RP_3.91', 'RP_4.1', 'RP_4.3', \n    'RP_4.49', 'RP_4.69', 'RP_4.88', 'RP_5.08', 'RP_5.27', \n    'RP_5.47', 'RP_5.66', 'RP_5.86', 'RP_6.05', 'RP_6.25', \n    'RP_6.45', 'RP_6.64', 'RP_6.84', 'RP_7.03', 'RP_7.23', \n    'RP_7.42', 'RP_7.62', 'RP_7.81', 'RP_8.01', 'RP_8.2', \n    'RP_8.4', 'RP_8.59', 'RP_8.79', 'RP_8.98', 'RP_9.18', \n    'RP_9.38', 'RP_9.57', 'RP_9.77', 'RP_9.96', 'RP_10.16', \n    'RP_10.35', 'RP_10.55', 'RP_10.74', 'RP_10.94', 'RP_11.13', \n    'RP_11.33', 'RP_11.52', 'RP_11.72', 'RP_11.91', 'RP_12.11', \n    'RP_12.3', 'RP_12.5', 'RP_12.7', 'RP_12.89', 'RP_13.09', \n    'RP_13.28', 'RP_13.48', 'RP_13.67', 'RP_13.87', 'RP_14.06', \n    'RP_14.26', 'RP_14.45', 'RP_14.65', 'RP_14.84', 'RP_15.04', \n    'RP_15.23', 'RP_15.43', 'RP_15.63', 'RP_15.82', 'RP_16.02', \n    'RP_16.21', 'RP_16.41', 'RP_16.6', 'RP_16.8', 'RP_16.99', \n    'RP_17.19', 'RP_17.38', 'RP_17.58', 'RP_17.77', 'RP_17.97', \n    'RP_18.16', 'RP_18.36', 'RP_18.55', 'RP_18.75', 'RP_18.95',\n    'RP_19.14', 'RP_19.34', 'RP_19.53', 'RP_19.73', 'RP_19.92'\n]\n\n\nEEG_COLUMNS =  [ # to assert columns order is the same\n    'Fp1','F3', 'C3', 'P3', 'F7', \n    'T3', 'T5', 'O1', 'Fz', 'Cz', \n    'Pz', 'Fp2', 'F4', 'C4', 'P4',\n    'F8', 'T4', 'T6', 'O2', 'EKG'\n]\n\n\nTARGET_FEATURES = [\n    'seizure_vote', \n    'lpd_vote', \n    'gpd_vote', \n    'lrda_vote', \n    'grda_vote', \n    'other_vote'\n]","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:21.840045Z","iopub.execute_input":"2024-02-25T18:24:21.840356Z","iopub.status.idle":"2024-02-25T18:24:21.862519Z","shell.execute_reply.started":"2024-02-25T18:24:21.840331Z","shell.execute_reply":"2024-02-25T18:24:21.861189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EEG_FEATURES = EEG_COLUMNS # [ 'EKG' ]","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:21.864291Z","iopub.execute_input":"2024-02-25T18:24:21.865281Z","iopub.status.idle":"2024-02-25T18:24:21.869839Z","shell.execute_reply.started":"2024-02-25T18:24:21.865246Z","shell.execute_reply":"2024-02-25T18:24:21.868802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@lru_cache(maxsize = 1024)\ndef load_train_eeg_frame(id):\n    data = pd.read_parquet(\n        os.path.join(TRAIN_EEGS, str(id) + '.parquet'), \n        engine='pyarrow'\n    )\n    if not SKIP_ASSERT:\n        assert list(data.columns) == EEG_COLUMNS, 'EEG columns order is not the same!'\n    return data\n\n@lru_cache(maxsize = 1024)\ndef load_train_spectr_frame(id):\n    data = pd.read_parquet(\n        os.path.join(TRAIN_SPECTR, str(id) + '.parquet'), \n        engine='pyarrow'\n    )\n    if not SKIP_ASSERT:\n        assert all([(data['time'].iloc[i] == (2*i + 1)) for i in range(len(data))]), 'time inconsistent'\n        assert list(data.columns) == SPECTR_COLUMNS, 'SPECTR columns order is not the same!'\n    return data.drop(columns = 'time')\n    \ndef load_test_eeg_frame(id):\n    data = pd.read_parquet(\n        os.path.join(TEST_EEGS, str(id) + '.parquet'), \n        engine='pyarrow'\n    )\n    if not SKIP_ASSERT:\n        assert list(data.columns) == EEG_COLUMNS, 'EEG columns order is not the same!'\n    return data\n\ndef load_test_spectr_frame(id):\n    data = pd.read_parquet(\n        os.path.join(TEST_SPECTR, str(id) + '.parquet'), \n        engine='pyarrow'\n    )\n    if not SKIP_ASSERT:\n        assert list(data.columns) == SPECTR_COLUMNS, 'SPECTR columns order is not the same!'\n    return data","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:21.871259Z","iopub.execute_input":"2024-02-25T18:24:21.871599Z","iopub.status.idle":"2024-02-25T18:24:21.882186Z","shell.execute_reply.started":"2024-02-25T18:24:21.871568Z","shell.execute_reply":"2024-02-25T18:24:21.881340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decrease_int_type(column):\n    try:\n        new_column = column.astype('int8')\n        if new_column.astype(column.dtype).equals(column):\n            return new_column\n    except Exception:\n        pass\n    try:\n        new_column = column.astype('int16')\n        if new_column.astype(column.dtype).equals(column):\n            return new_column\n    except Exception:\n        pass\n    try:\n        new_column = column.astype('int32')\n        if new_column.astype(column.dtype).equals(column):\n            return new_column\n    except Exception:\n        pass\n    return column ","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:21.883681Z","iopub.execute_input":"2024-02-25T18:24:21.884329Z","iopub.status.idle":"2024-02-25T18:24:21.896887Z","shell.execute_reply.started":"2024-02-25T18:24:21.884299Z","shell.execute_reply":"2024-02-25T18:24:21.895803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TrainEegLoader(Callable):\n    def __init__(self, train_info):\n        self.data = pd.DataFrame(\n            {\n                c : decrease_int_type(train_info[c]) \n                for c in ['eeg_id','eeg_label_offset_seconds']\n            }\n        )\n    \n    def __call__(self, index):\n        eeg_id, start = self.data.iloc[index] \n        start = start * EEG_FRAME_PER_SECOND\n        end = start + EEG_FRAME\n        data = load_train_eeg_frame(eeg_id)\n        if not SKIP_ASSERT:\n            assert start >=0 and start <= len(data), 'inlvalid start = {}, len = {}'.format(start, len(data))\n            assert end <= len(data) and end >=0, 'invalid end = {}, len = {}'.format(end, len(data))\n        return data[EEG_FEATURES].iloc[start:end]\n    \nclass TrainSpectrLoader(Callable):\n    def __init__(self, train_info):\n        self.data = pd.DataFrame(\n            {\n                c : decrease_int_type(train_info[c]) \n                for c in ['spectrogram_id','spectrogram_label_offset_seconds']\n            }\n        )\n    \n    def __call__(self, index):\n        spectr_id, start = self.data.iloc[index]\n        start = start // 2\n        end = start + SPECTR_FRAME\n        data = load_train_spectr_frame(spectr_id)\n        if not SKIP_ASSERT:\n            assert start >=0 and start <= len(data), 'inlvalid start = {}, len = {}'.format(start, len(data))\n            assert end <= len(data) and end >=0, 'invalid end = {}, len = {}'.format(end, len(data))\n        return data[SPECTR_FEATURES].iloc[start:end] \n    \nclass TrainTargetLoader(Callable):\n    def __init__(self, test_info):\n        self.data = test_info[TARGET_FEATURES]\n        \n    def __call__(self, index):\n        return self.data.iloc[index]","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:21.900104Z","iopub.execute_input":"2024-02-25T18:24:21.900665Z","iopub.status.idle":"2024-02-25T18:24:21.911616Z","shell.execute_reply.started":"2024-02-25T18:24:21.900631Z","shell.execute_reply":"2024-02-25T18:24:21.910801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_train():\n    train_info = (\n        pd.read_csv(os.path.join(ROOT_DIR,'train.csv'))\n        .drop(columns = [\n            'expert_consensus',\n            'eeg_sub_id',\n            'spectrogram_sub_id',\n            'patient_id',\n            'label_id'\n        ])\n    )\n    if DEBUG:\n        train_info = train_info.sample(DEBUG_TRAIN_SIZE)\n    return (\n        len(train_info),\n        TrainEegLoader(train_info), \n        TrainSpectrLoader(train_info), \n        TrainTargetLoader(train_info)\n    )","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:21.912976Z","iopub.execute_input":"2024-02-25T18:24:21.913548Z","iopub.status.idle":"2024-02-25T18:24:21.925967Z","shell.execute_reply.started":"2024-02-25T18:24:21.913513Z","shell.execute_reply":"2024-02-25T18:24:21.924609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_size, train_eeg_loader, train_spectr_loader, train_target_loader = load_train()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:21.927505Z","iopub.execute_input":"2024-02-25T18:24:21.927959Z","iopub.status.idle":"2024-02-25T18:24:22.127348Z","shell.execute_reply.started":"2024-02-25T18:24:21.927926Z","shell.execute_reply":"2024-02-25T18:24:22.126060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EEG_FRAME_INDEXES = [17, 21, 44, 124, 1025]","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:22.128724Z","iopub.execute_input":"2024-02-25T18:24:22.129197Z","iopub.status.idle":"2024-02-25T18:24:22.134448Z","shell.execute_reply.started":"2024-02-25T18:24:22.129155Z","shell.execute_reply":"2024-02-25T18:24:22.133160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eeg_data = train_eeg_loader(EEG_FRAME_INDEXES[0])","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:22.136498Z","iopub.execute_input":"2024-02-25T18:24:22.137648Z","iopub.status.idle":"2024-02-25T18:24:22.283942Z","shell.execute_reply.started":"2024-02-25T18:24:22.137584Z","shell.execute_reply":"2024-02-25T18:24:22.283080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eeg_data.describe()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:22.285119Z","iopub.execute_input":"2024-02-25T18:24:22.285921Z","iopub.status.idle":"2024-02-25T18:24:22.353099Z","shell.execute_reply.started":"2024-02-25T18:24:22.285884Z","shell.execute_reply":"2024-02-25T18:24:22.352035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_PLOTS = 3\nPLOT_FEATURES = ['EKG', 'O2']","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:22.354627Z","iopub.execute_input":"2024-02-25T18:24:22.354981Z","iopub.status.idle":"2024-02-25T18:24:22.360593Z","shell.execute_reply.started":"2024-02-25T18:24:22.354951Z","shell.execute_reply":"2024-02-25T18:24:22.359479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WINDOW_IN_SEC = 3 # in seconds\nWINDOW = WINDOW_IN_SEC * EEG_FRAME_PER_SECOND \nWINDOW_FILTERED = WINDOW  // 8","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:22.361864Z","iopub.execute_input":"2024-02-25T18:24:22.362188Z","iopub.status.idle":"2024-02-25T18:24:22.370622Z","shell.execute_reply.started":"2024-02-25T18:24:22.362160Z","shell.execute_reply":"2024-02-25T18:24:22.369463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@dataclass\nclass EEGFilter_minmax:\n    input_window: int # in source ticks\n    output_window: int # in output ticks    \n    base_window: int = 7 * EEG_FRAME_PER_SECOND # 7 seconds\n    median_window: int = 7\n    filter_window: int = 17\n\n    def __call__(self,data):\n        assert len(data) >= self.input_window, 'no data for filter window!'\n        data = data[\n            max(0,len(data)//2 - self.input_window//2 - self.base_window) : \n            min(len(data), len(data)//2 + self.input_window//2 + self.base_window)\n        ]\n        data = np.nan_to_num(data, nan = 0)\n        data = data -  bottleneck.move_mean(data, window=self.base_window, min_count=1)\n        data = bottleneck.move_median(data, window=self.median_window, min_count=1)\n        data_max = bottleneck.move_max(data, window=self.filter_window, min_count=1)\n        data_min = bottleneck.move_min(data, window=self.filter_window, min_count=1)\n\n        total_max = max(\n            np.max(np.abs(data_max)),\n            np.max(np.abs(data_min))\n        )\n\n        step = self.input_window // self.output_window\n        data_max = data_max[::step] / total_max\n        data_min = data_min[::step] / total_max\n\n        data = np.column_stack((data_max, data_min))\n        data = data[\n            len(data)//2 - self.output_window//2 : \n            len(data)//2 + self.output_window//2\n        ]\n        return data","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:24:22.373126Z","iopub.execute_input":"2024-02-25T18:24:22.373558Z","iopub.status.idle":"2024-02-25T18:24:22.386074Z","shell.execute_reply.started":"2024-02-25T18:24:22.373514Z","shell.execute_reply":"2024-02-25T18:24:22.384803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@dataclass\nclass EEGDistribution:\n    input_window: int # in source ticks\n    output_window: int # in output levels   \n    base_window: int = 5 * EEG_FRAME_PER_SECOND # 5 seconds\n\n    def __call__(self,data):\n        assert len(data) >= self.input_window, 'no data for filter window!'\n        data = data[\n            max(0,len(data)//2 - self.input_window//2 - self.base_window) : \n            min(len(data), len(data)//2 + self.input_window//2 + self.base_window)\n        ]\n        data = np.nan_to_num(data, nan = 0)\n        data = data -  bottleneck.move_mean(data, window=self.base_window, min_count=1)\n        data_max = bottleneck.move_max(data, window=self.base_window, min_count=1)\n        data_min = bottleneck.move_min(data, window=self.base_window, min_count=1)\n        data_max = bottleneck.move_mean(data_max, window=self.base_window, min_count=1)\n        data_min = bottleneck.move_mean(data_min, window=self.base_window, min_count=1)\n        \n        data_diff = data_max - data_min\n        data_diff[data_diff == 0] = 1\n        \n        data = (data - data_min)/data_diff\n        hist = np.histogram(data, bins = self.output_window, density = False)\n        return hist[0]/len(data)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:34:35.296792Z","iopub.execute_input":"2024-02-25T18:34:35.297218Z","iopub.status.idle":"2024-02-25T18:34:35.309090Z","shell.execute_reply.started":"2024-02-25T18:34:35.297186Z","shell.execute_reply":"2024-02-25T18:34:35.307792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eeg_filter = EEGFilter_minmax(\n    input_window = WINDOW,\n    output_window = WINDOW_FILTERED\n)\neeg_hist = EEGDistribution(\n    input_window = WINDOW,\n    output_window = 20    \n)\n\nfor reason_index, reason in enumerate(TARGET_FEATURES):\n    print('For ', reason, '-------------------------------')\n    for _ in range(NUM_PLOTS):\n        start = random.randint(0, train_size)\n        targets = train_target_loader(start)\n        while(targets.idxmax() != reason):\n            start = (start + 1) % train_size\n            targets = train_target_loader(start)\n        print(targets)\n        data = train_eeg_loader(start)\n        for f in PLOT_FEATURES:\n            ax, fig = plt.subplots(1, 1, figsize = (10,5))\n            ax = sns.scatterplot(data[f].iloc[len(data)//2-WINDOW//2:len(data)//2+WINDOW//2])\n            ax.set_title('Reason = ' + reason + ' feature = ' + f)\n            plt.show()\n            filtered_data = pd.DataFrame(\n                data = eeg_filter(data[f].to_numpy()), \n                columns = ['min', 'max']\n            )\n            plot_data = filtered_data\n            ax, fig = plt.subplots(1, 1, figsize = (10,5))\n            ax = sns.scatterplot(plot_data)\n            ax.set_title('Reason = ' + reason + ' feature = ' + f + ' filtered')\n            plt.show()      \n            hist = eeg_hist(data[f].to_numpy())\n            ax = sns.heatmap(hist.reshape(20,1), annot = True)\n            plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:37:23.164874Z","iopub.execute_input":"2024-02-25T18:37:23.166104Z","iopub.status.idle":"2024-02-25T18:37:56.978049Z","shell.execute_reply.started":"2024-02-25T18:37:23.166030Z","shell.execute_reply":"2024-02-25T18:37:56.976813Z"},"trusted":true},"execution_count":null,"outputs":[]}]}