{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7435969,"sourceType":"datasetVersion","datasetId":4327669},{"sourceId":7436296,"sourceType":"datasetVersion","datasetId":4327895}],"dockerImageVersionId":30636,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"papermill":{"default_parameters":{},"duration":102.485133,"end_time":"2024-01-16T12:14:21.10455","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-01-16T12:12:38.619417","version":"2.4.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd, numpy as np, os\nimport matplotlib.pyplot as plt, gc\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom scipy.signal import butter, lfilter\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom scipy.signal import freqz\n\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom tqdm import tqdm","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.693827,"end_time":"2024-01-16T12:12:42.606147","exception":false,"start_time":"2024-01-16T12:12:41.91232","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-03T09:01:14.931760Z","iopub.execute_input":"2024-02-03T09:01:14.932164Z","iopub.status.idle":"2024-02-03T09:01:18.979703Z","shell.execute_reply.started":"2024-02-03T09:01:14.932115Z","shell.execute_reply":"2024-02-03T09:01:18.978615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nTARGETS = df.columns[-6:]\nprint('Train shape:', df.shape )\nprint('Targets', list(TARGETS))\ndf.head()","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.693827,"end_time":"2024-01-16T12:12:42.606147","exception":false,"start_time":"2024-01-16T12:12:41.91232","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-03T09:01:18.981652Z","iopub.execute_input":"2024-02-03T09:01:18.982467Z","iopub.status.idle":"2024-02-03T09:01:19.269055Z","shell.execute_reply.started":"2024-02-03T09:01:18.982430Z","shell.execute_reply":"2024-02-03T09:01:19.267934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def butter_bandpass(lowcut, highcut, fs, order=5):\n    return butter(order, [lowcut, highcut], fs=fs, btype='band')\n\ndef butter_bandpass_filter(data, lowcut, highcut, fs, order=5):\n    b, a = butter_bandpass(lowcut, highcut, fs, order=order)\n    y = lfilter(b, a, data)\n    return y\n\n\ndef denoise_filter(x):\n    # Sample rate and desired cutoff frequencies (in Hz).\n    fs = 200.0\n    lowcut = 1.0\n    highcut = 25.0\n    \n    # Filter a noisy signal.\n    T = 50\n    nsamples = T * fs\n    t = np.arange(0, nsamples) / fs\n    y = butter_bandpass_filter(x, lowcut, highcut, fs, order=6)\n    y = (y + np.roll(y,-1)+ np.roll(y,-2)+ np.roll(y,-3))/4\n    y = y[0:-1:4]\n    \n    return y","metadata":{"execution":{"iopub.status.busy":"2024-02-03T09:01:19.270655Z","iopub.execute_input":"2024-02-03T09:01:19.271052Z","iopub.status.idle":"2024-02-03T09:01:19.279971Z","shell.execute_reply.started":"2024-02-03T09:01:19.271013Z","shell.execute_reply":"2024-02-03T09:01:19.278900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_id':'first','spectrogram_label_offset_seconds':'min'})\ntrain.columns = ['spec_id','min']\n\ntmp = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_label_offset_seconds':'max'})\ntrain['max'] = tmp\n\ntmp = df.groupby('eeg_id')[['patient_id']].agg('first')\ntrain['patient_id'] = tmp\n\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum')\nfor t in TARGETS:\n    train[t] = tmp[t].values\n    \ny_data = train[TARGETS].values\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[TARGETS] = y_data\n\ntmp = df.groupby('eeg_id')[['expert_consensus']].apply(lambda x: x.mode().iloc[0]).reset_index()\ntmp2 = df.groupby(['eeg_id','expert_consensus'])[['eeg_sub_id']].agg(min).reset_index()\ntmp = pd.merge(tmp,tmp2,on=['eeg_id','expert_consensus'],how='left')\ntrain['target'] = tmp['expert_consensus'].values\ntrain['eeg_sub_id'] = tmp['eeg_sub_id'].values\n\ntrain = pd.merge(train,df[['eeg_id','eeg_sub_id','eeg_label_offset_seconds']],on=['eeg_id','eeg_sub_id'],how='left')\ntrain = train.reset_index()\nprint('Train non-overlapp eeg_id shape:', train.shape )\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-03T09:01:19.282042Z","iopub.execute_input":"2024-02-03T09:01:19.282400Z","iopub.status.idle":"2024-02-03T09:01:28.900187Z","shell.execute_reply.started":"2024-02-03T09:01:19.282369Z","shell.execute_reply":"2024-02-03T09:01:28.899007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NAMES = ['LL','LP','RP','RR']\n\nFEATS = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\nPATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/'","metadata":{"execution":{"iopub.status.busy":"2024-02-03T09:01:28.901313Z","iopub.execute_input":"2024-02-03T09:01:28.901609Z","iopub.status.idle":"2024-02-03T09:01:28.910845Z","shell.execute_reply.started":"2024-02-03T09:01:28.901582Z","shell.execute_reply":"2024-02-03T09:01:28.909659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def quantize_data(data, classes):\n    data = data-data.mean()\n    data = data/(abs(data).max())\n    mu_x = mu_law_encoding(data, classes)\n    return mu_x#quantized\n\ndef mu_law_encoding(data, mu):\n    mu_x = np.sign(data) * np.log(1 + mu * np.abs(data)) / np.log(mu + 1)\n    return mu_x\n","metadata":{"execution":{"iopub.status.busy":"2024-02-03T12:10:45.320411Z","iopub.execute_input":"2024-02-03T12:10:45.321504Z","iopub.status.idle":"2024-02-03T12:10:45.358549Z","shell.execute_reply.started":"2024-02-03T12:10:45.321448Z","shell.execute_reply":"2024-02-03T12:10:45.356944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_eegs1 = {}\nall_eegs2 = {}\nfor idx in tqdm(range(len(train))):\n    row = train.iloc[idx]\n    eeg_id = row['eeg_id']\n    eeg_sub_id = row['eeg_sub_id']\n    parq_path = f'{PATH}{eeg_id}.parquet'\n    eeg = pd.read_parquet(parq_path)\n    start = int(row['eeg_label_offset_seconds']*200)\n    eeg = eeg.iloc[start:start+10000]\n    eeg = eeg.fillna(0)\n    eeg = eeg.drop(['Fz', 'Cz','Pz','EKG'], axis=1)\n#     eeg = eeg-eeg.min()\n#     eeg = eeg/(eeg.abs().max())\n#         print(idx,eeg_id,start,eeg.shape)\n    signals1 = []\n    signals2 = []\n    for k in range(4):\n            COLS = FEATS[k]\n\n            # COMPUTE PAIR DIFFERENCES AND AVERAGE\n#             x = eeg[COLS[0]].values - eeg[COLS[1]].values\n            for j in range(4):\n                x = eeg[COLS[j]].values - eeg[COLS[j+1]].values\n                x = denoise_filter(x)\n                signals1.append(x)\n                x = quantize_data(x,1)\n                signals2.append(x)\n    signals1 = np.array(signals1,dtype='float32')\n    signals2 = np.array(signals2,dtype='float32')\n    all_eegs1[f'{eeg_id}_{eeg_sub_id}'] = signals1\n    all_eegs2[f'{eeg_id}_{eeg_sub_id}'] = signals2\n    \nnp.save('16_waves_eeg_specs_partial_train',all_eegs1)\nnp.save('16_waves_eeg_specs_partial_train_quantize',all_eegs2)","metadata":{"execution":{"iopub.status.busy":"2024-01-29T16:20:06.170647Z","iopub.execute_input":"2024-01-29T16:20:06.171024Z","iopub.status.idle":"2024-01-29T16:31:12.404551Z","shell.execute_reply.started":"2024-01-29T16:20:06.170995Z","shell.execute_reply":"2024-01-29T16:31:12.403197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataloader","metadata":{"papermill":{"duration":0.028035,"end_time":"2024-01-16T12:13:50.519598","exception":false,"start_time":"2024-01-16T12:13:50.491563","status":"completed"},"tags":[]}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}