{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7483703,"sourceType":"datasetVersion","datasetId":4356560},{"sourceId":7568180,"sourceType":"datasetVersion","datasetId":4406053}],"dockerImageVersionId":30648,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\nimport os\nimport random\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-06T07:09:34.110634Z","iopub.execute_input":"2024-02-06T07:09:34.111142Z","iopub.status.idle":"2024-02-06T07:09:34.116167Z","shell.execute_reply.started":"2024-02-06T07:09:34.111107Z","shell.execute_reply":"2024-02-06T07:09:34.115109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Paths","metadata":{}},{"cell_type":"code","source":"''' CSV '''\nTRAIN_CSV = \"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\"\nTEST_CSV = \"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\"\n\n''' EEG Parquet '''\nTRAIN_EEG = \"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs\"\nTEST_EEG = \"/kaggle/input/hms-harmful-brain-activity-classification/test_eegs\"\n\n''' Spectogram Parquet'''\nTRAIN_SPECTROGRAM = \"/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms\"\nTEST_SPECTROGRAM = \"/kaggle/input/hms-harmful-brain-activity-classification/test_eegs\"","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:34.120956Z","iopub.execute_input":"2024-02-06T07:09:34.121238Z","iopub.status.idle":"2024-02-06T07:09:34.132365Z","shell.execute_reply.started":"2024-02-06T07:09:34.121207Z","shell.execute_reply":"2024-02-06T07:09:34.131383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA `train.csv`","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:34.133731Z","iopub.execute_input":"2024-02-06T07:09:34.134014Z","iopub.status.idle":"2024-02-06T07:09:34.142713Z","shell.execute_reply.started":"2024-02-06T07:09:34.133989Z","shell.execute_reply":"2024-02-06T07:09:34.141870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read train.csv\ntrain_df = pd.read_csv(TRAIN_CSV)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:34.144154Z","iopub.execute_input":"2024-02-06T07:09:34.144453Z","iopub.status.idle":"2024-02-06T07:09:34.366414Z","shell.execute_reply.started":"2024-02-06T07:09:34.144427Z","shell.execute_reply":"2024-02-06T07:09:34.365426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`target` distribution\n> Info: extensive eda can be found [here](https://www.kaggle.com/competitions/hms-harmful-brain-activity-classification/discussion/467021)","metadata":{}},{"cell_type":"code","source":"train_df.groupby('eeg_id')[['expert_consensus']].agg({'expert_consensus':'first'}).value_counts().plot(kind='bar')","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:34.368221Z","iopub.execute_input":"2024-02-06T07:09:34.368485Z","iopub.status.idle":"2024-02-06T07:09:34.603202Z","shell.execute_reply.started":"2024-02-06T07:09:34.368463Z","shell.execute_reply":"2024-02-06T07:09:34.602147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create non-overlapping `EEG` train data","metadata":{}},{"cell_type":"markdown","source":"**From the competition data description**\n- *`train.csv` Metadata for the train set. The expert annotators reviewed `50 second` long EEG samples plus matched spectrograms covering `10 a minute window centered at the same time` and `labeled the central 10 seconds`. Many of these samples overlapped and have been consolidated. train.csv provides the metadata that allows you to extract the original subsets that the raters annotated.*","metadata":{}},{"cell_type":"code","source":"targets = train_df.columns[-6:]\ntargets","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:34.604383Z","iopub.execute_input":"2024-02-06T07:09:34.604750Z","iopub.status.idle":"2024-02-06T07:09:34.610937Z","shell.execute_reply.started":"2024-02-06T07:09:34.604722Z","shell.execute_reply":"2024-02-06T07:09:34.610103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find all unique eeg_ids\nprint(\"Unique eegs = \",len(train_df.groupby(\"eeg_id\")[\"patient_id\"]))","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:34.613100Z","iopub.execute_input":"2024-02-06T07:09:34.613409Z","iopub.status.idle":"2024-02-06T07:09:34.890164Z","shell.execute_reply.started":"2024-02-06T07:09:34.613384Z","shell.execute_reply":"2024-02-06T07:09:34.889063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\" Create non-overlaping EEG train data \"\"\"\n\n# find the `min` eeg_offset for a unique `eeg_id`\ntrain = train_df.groupby(\"eeg_id\")[[\"spectrogram_id\",\"eeg_label_offset_seconds\"]].agg({\"spectrogram_id\":\"first\",'eeg_label_offset_seconds':'min'})\ntrain.columns = ['spec_id','eeg_min']\n\n# find the `max` eeg_offset for a unique `eeg_id`\ntmp = train_df.groupby(\"eeg_id\")[[\"spectrogram_id\",\"eeg_label_offset_seconds\"]].agg({'eeg_label_offset_seconds':'max'})\ntrain[\"eeg_max\"] = tmp\n\n# find the `min` spectogram_offset for a unique `eeg_id`\ntmp = train_df.groupby(\"eeg_id\")[[\"spectrogram_id\",\"spectrogram_label_offset_seconds\"]].agg({'spectrogram_label_offset_seconds':'min'})\ntrain[\"spec_min\"] = tmp\n\n# find the `max` spectogram_offset for a unique `eeg_id`\ntmp = train_df.groupby(\"eeg_id\")[[\"spectrogram_id\",\"spectrogram_label_offset_seconds\"]].agg({'spectrogram_label_offset_seconds':'max'})\ntrain[\"spec_max\"] = tmp\n\n# aggregate the expert votes for each targets\ntmp = train_df.groupby(\"eeg_id\")[targets].agg(\"sum\")\nfor t in targets:\n    train[t] = tmp[t].values\n    \n# to probablities\ny_data = train[targets].values\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[targets] = y_data\n\ntmp = train_df.groupby(\"eeg_id\")[[\"expert_consensus\"]].agg(\"first\")\ntrain[\"target\"] = tmp\n\ntmp = train_df.groupby(\"eeg_id\")[[\"patient_id\"]].agg(\"first\")\ntrain[\"patient_id\"] = tmp\n\ntrain = train.reset_index()\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:34.891644Z","iopub.execute_input":"2024-02-06T07:09:34.892026Z","iopub.status.idle":"2024-02-06T07:09:35.000813Z","shell.execute_reply.started":"2024-02-06T07:09:34.891993Z","shell.execute_reply":"2024-02-06T07:09:34.999846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- each `eeg` sample is `50` seconds long\n- take ```eeg.iloc[min:(max+50)]```\n- we predict for the central `10` seconds for eeg\n- sampling frequency is `200` samples per second\n- center time (T) for eeg = `((eeg_max + 50)*200 + eeg_min*200) // 2`\n- predict for `eeg.iloc[T-5 : T+5]`","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm\nimport multiprocessing","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:35.002162Z","iopub.execute_input":"2024-02-06T07:09:35.002481Z","iopub.status.idle":"2024-02-06T07:09:35.007081Z","shell.execute_reply.started":"2024-02-06T07:09:35.002455Z","shell.execute_reply":"2024-02-06T07:09:35.006075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = multiprocessing.Manager().list()  # Create a shared list accessible to all processes\ny_train = multiprocessing.Manager().list()","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:35.008448Z","iopub.execute_input":"2024-02-06T07:09:35.008742Z","iopub.status.idle":"2024-02-06T07:09:35.066192Z","shell.execute_reply.started":"2024-02-06T07:09:35.008716Z","shell.execute_reply":"2024-02-06T07:09:35.064568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fueture Engineering\nFeatures will be extracted from the bipolar montages\nNumber of features 8 * len(bipolar_channels)\n- Mean (foreach channel)\n- STD (foreach channel)\n- min (foreach channel)\n- max (foreach channel)\n- skewness (foreach channel)\n- kurtosis (foreach channel)\n- energy (foreach channel)\n- entropy (foreach channel)","metadata":{}},{"cell_type":"code","source":"montage_pairs = {\n        \"Fp1-F7\": (\"Fp1\", \"F7\"),\n        \"F7-T3\": (\"F7\", \"T3\"),\n        \"T3-T5\": (\"T3\", \"T5\"),\n        \"T5-O1\": (\"T5\", \"O1\"),\n        \"Fp2-F8\": (\"Fp2\", \"F8\"),\n        \"F8-T4\": (\"F8\", \"T4\"),\n        \"T4-T6\": (\"T4\", \"T6\"),\n        \"T6-O2\": (\"T6\", \"O2\"),\n        \"T3-C3\": (\"T3\", \"C3\"),\n        \"C3-Cz\": (\"C3\", \"Cz\"),\n        \"Cz-C4\": (\"Cz\", \"C4\"),\n        \"C4-T4\": (\"C4\", \"T4\"),\n        \"Fp1-F3\": (\"Fp1\", \"F3\"),\n        \"F3-C3\": (\"F3\", \"C3\"),\n        \"C3-P3\": (\"C3\", \"P3\"),\n        \"P3-O1\": (\"P3\", \"O1\"),\n        \"Fp2-F4\": (\"Fp2\", \"F4\"),\n        \"F4-C4\": (\"F4\", \"C4\"),\n        \"C4-P4\": (\"C4\", \"P4\"),\n        \"P4-O2\": (\"P4\", \"O2\"),\n    }\nTARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:35.068429Z","iopub.execute_input":"2024-02-06T07:09:35.068787Z","iopub.status.idle":"2024-02-06T07:09:35.079163Z","shell.execute_reply.started":"2024-02-06T07:09:35.068754Z","shell.execute_reply":"2024-02-06T07:09:35.078138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Helper Functions**","metadata":{}},{"cell_type":"code","source":"from scipy.stats import skew,kurtosis,entropy\n\ndef raw_to_bipolar(raw_eeg):\n    \"\"\" Convert EEG from Unipolar to Bipolar Montage\n    Params:\n    - raw_eeg (pd.DataFrame): Subsample eeg, shape(num_channels,s_freq * 10)\n    \"\"\"\n    bipolar_eeg_dict = {}\n    for key, value in montage_pairs.items():\n        c_1 = raw_eeg[value[0].strip()].values\n        c_2 = raw_eeg[value[1].strip()].values\n    \n        bipolar_eeg_dict[key] = c_1 - c_2\n    return bipolar_eeg_dict\n\ndef extract_features(bipolar_eeg):\n    \n    num_features = 8\n    num_channels = len(bipolar_eeg)\n    # feature extraction\n    features = np.zeros((num_channels,num_features)) # 8 features per channel\n    \n    for idx,electrode_data in enumerate(bipolar_eeg.values()):\n        # Mean\n        features[idx, 0] = np.mean(electrode_data)\n        # Standard deviation\n        features[idx, 1] = np.std(electrode_data)\n        # Minimum\n        features[idx, 2] = np.min(electrode_data)\n        # Maximum\n        features[idx, 3] = np.max(electrode_data)\n        # Skewness\n        features[idx, 4] = skew(electrode_data)\n        # Kurtosis\n        features[idx, 5] = kurtosis(electrode_data)\n        # Energy\n        features[idx, 6] = np.sum(electrode_data**2) / len(electrode_data)\n        # Entropy\n        val = np.nan_to_num(electrode_data,np.mean(electrode_data))\n        features[idx, 7] = entropy(np.abs(val))\n    \n    return features.reshape(-1)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:35.080618Z","iopub.execute_input":"2024-02-06T07:09:35.081144Z","iopub.status.idle":"2024-02-06T07:09:35.095699Z","shell.execute_reply.started":"2024-02-06T07:09:35.081108Z","shell.execute_reply":"2024-02-06T07:09:35.094563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_central_eeg(data):\n    i,row = data\n    min_startT = row.eeg_min\n    max_startT = row.eeg_max\n    centralT = int(min_startT + (max_startT + 50) // 2)\n    \n    # read_eeg file\n    eeg = pd.read_parquet(f\"{TRAIN_EEG}/{row.eeg_id}.parquet\")\n    # get the target for the cental eeg\n    target = row.target\n    y_train.append(TARS[target])\n    \n    # get central 10 second eeg data\n    central_eeg = eeg.iloc[(centralT-5)*200 : (centralT+5)*200]\n    # ignore the `EKG` signal\n    central_eeg = central_eeg[central_eeg.columns[:-1]]\n    # convert to bipolar\n    bipolar_eeg = raw_to_bipolar(central_eeg)\n    # extract features\n    f = extract_features(bipolar_eeg)\n    \n    x_train.append(f)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:35.101749Z","iopub.execute_input":"2024-02-06T07:09:35.102119Z","iopub.status.idle":"2024-02-06T07:09:35.110573Z","shell.execute_reply.started":"2024-02-06T07:09:35.102079Z","shell.execute_reply":"2024-02-06T07:09:35.109747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pool = multiprocessing.Pool(processes=2)\n# d = train\n# pool.map(get_central_eeg, tqdm(d.iterrows(),total=len(d)))\n\n# pool.close()\n# pool.join()","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:35.111518Z","iopub.execute_input":"2024-02-06T07:09:35.113072Z","iopub.status.idle":"2024-02-06T07:09:35.123283Z","shell.execute_reply.started":"2024-02-06T07:09:35.113047Z","shell.execute_reply":"2024-02-06T07:09:35.122239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save features to disk\n#np.savez('/kaggle/working/handcrafted_bipolar_features_train.npz', array1=x_train, array2=y_train)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:35.124688Z","iopub.execute_input":"2024-02-06T07:09:35.125135Z","iopub.status.idle":"2024-02-06T07:09:35.133184Z","shell.execute_reply.started":"2024-02-06T07:09:35.125091Z","shell.execute_reply":"2024-02-06T07:09:35.132341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\" Load saved dataset\"\"\"\n\ntraining_data = np.load(\"/kaggle/input/training-features/handcrafted_bipolar_features_train.npz\")\nx_train = training_data[\"array1\"]\ny_train = training_data[\"array2\"].astype(np.int8)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:35.134266Z","iopub.execute_input":"2024-02-06T07:09:35.134696Z","iopub.status.idle":"2024-02-06T07:09:35.217819Z","shell.execute_reply.started":"2024-02-06T07:09:35.134658Z","shell.execute_reply":"2024-02-06T07:09:35.216938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = np.array(x_train)\ny_train = np.array(y_train)\n\nx_train.shape, y_train.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:35.219220Z","iopub.execute_input":"2024-02-06T07:09:35.219975Z","iopub.status.idle":"2024-02-06T07:09:35.237137Z","shell.execute_reply.started":"2024-02-06T07:09:35.219935Z","shell.execute_reply":"2024-02-06T07:09:35.236145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train `Catboost`","metadata":{}},{"cell_type":"code","source":"import catboost as cat\nfrom catboost import CatBoostClassifier, Pool\n\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom sklearn.metrics import (\n    accuracy_score,\n    recall_score,\n    log_loss,\n    f1_score,\n    precision_score,\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:35.238421Z","iopub.execute_input":"2024-02-06T07:09:35.238769Z","iopub.status.idle":"2024-02-06T07:09:35.244655Z","shell.execute_reply.started":"2024-02-06T07:09:35.238741Z","shell.execute_reply":"2024-02-06T07:09:35.243483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clf_score(y, preds):\n    \"\"\" Calculate some classification metrics \"\"\"\n    acc = accuracy_score(y, preds)\n    recall = recall_score(y, preds, average=\"weighted\")\n    f1 = f1_score(y, preds, average=\"weighted\")\n    precision = precision_score(y, preds, average=\"weighted\")\n\n    return (acc, recall, f1, precision)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:35.245897Z","iopub.execute_input":"2024-02-06T07:09:35.246181Z","iopub.status.idle":"2024-02-06T07:09:35.253778Z","shell.execute_reply.started":"2024-02-06T07:09:35.246156Z","shell.execute_reply":"2024-02-06T07:09:35.252937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/kaggle-kl-div')\nfrom kaggle_kl_div import score","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:35.254956Z","iopub.execute_input":"2024-02-06T07:09:35.255272Z","iopub.status.idle":"2024-02-06T07:09:35.264013Z","shell.execute_reply.started":"2024-02-06T07:09:35.255247Z","shell.execute_reply":"2024-02-06T07:09:35.263132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_catboost():\n    gkf = GroupKFold(n_splits=5)\n    scores = {}\n    models = {}\n\n    for i, (train_index, valid_index) in enumerate(gkf.split(x_train, y_train, train.patient_id)):\n        print(f\"Training for fold {i + 1}\")\n\n        #model = xgb.XGBClassifier(**model_params)\n        model = CatBoostClassifier(task_type='GPU', loss_function='MultiClass',learning_rate=0.1)\n\n        train_data,train_target = x_train[train_index],y_train[train_index]\n        val_data,val_target = x_train[valid_index],y_train[valid_index]\n        \n        train_pool = Pool(\n        data = x_train[train_index],\n        label = y_train[train_index]\n        )\n        \n        valid_pool = Pool(\n        data = x_train[valid_index],\n        label = y_train[valid_index]\n        )\n\n        model.fit(train_pool,verbose=100,eval_set=valid_pool)\n        probs = model.predict_proba(valid_pool)\n        \n        preds = np.argmax(probs,axis=-1)\n        \n        acc,recall,f1,precision = clf_score(val_target,preds)\n        \n        true = train.loc[valid_index,targets]\n        pred_df = pd.DataFrame(columns=[\"id\",\"seizure_vote\",\"lpd_vote\",\"gpd_vote\",\"lrda_vote\",\"grda_vote\",\"other_vote\"])\n        pred_df[\"id\"] = range(len(preds))\n        pred_df[targets.values.tolist()] = probs\n        \n        true_df = pd.DataFrame(columns=[\"id\",\"seizure_vote\",\"lpd_vote\",\"gpd_vote\",\"lrda_vote\",\"grda_vote\",\"other_vote\"])\n        true_df[\"id\"] = range(len(true))\n        sums = np.sum(true.values,axis=1)\n        true = true.values /sums[:, None]\n        true_df[targets.values.tolist()] = true\n\n        #kl divergence\n        kl_score = score(solution=true_df, submission=pred_df, row_id_column_name='id')\n        print('#' * 20)\n        print(\"kl_score=\",kl_score)\n        print(\"acc=\",acc)\n        print(\"f1=\",f1)\n        print('#' * 20)\n\n        models[f\"fold_{i+1}\"] = model\n\n        #model.save_model(f\"/kaggle/working/xgb_model_lr_{model_params['learning_rate']}_fold_{i+1}.json\")\n    return kl_score,models\n","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:35.265348Z","iopub.execute_input":"2024-02-06T07:09:35.265617Z","iopub.status.idle":"2024-02-06T07:09:35.279916Z","shell.execute_reply.started":"2024-02-06T07:09:35.265594Z","shell.execute_reply":"2024-02-06T07:09:35.278947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kl_score, models_dict = train_catboost()","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:09:35.281111Z","iopub.execute_input":"2024-02-06T07:09:35.281390Z","iopub.status.idle":"2024-02-06T07:10:18.508999Z","shell.execute_reply.started":"2024-02-06T07:09:35.281366Z","shell.execute_reply":"2024-02-06T07:10:18.507946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluate on test and submission","metadata":{}},{"cell_type":"code","source":"x_test = multiprocessing.Manager().list()  # Create a shared list accessible to all processes\ny_test = multiprocessing.Manager().list()","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:10:18.512494Z","iopub.execute_input":"2024-02-06T07:10:18.512807Z","iopub.status.idle":"2024-02-06T07:10:18.602899Z","shell.execute_reply.started":"2024-02-06T07:10:18.512779Z","shell.execute_reply":"2024-02-06T07:10:18.600579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_central_eeg_test(data):\n    i,row = data\n    # read_eeg file\n    eeg = pd.read_parquet(f\"{TEST_EEG}/{row.eeg_id}.parquet\")\n    # get central 10 second eeg data\n    central_eeg = eeg.iloc[20*200 : 30*200]\n    # ignore the `EKG` signal\n    central_eeg = central_eeg[central_eeg.columns[:-1]]\n    # convert to bipolar\n    bipolar_eeg = raw_to_bipolar(central_eeg)\n    # extract features\n    f = extract_features(bipolar_eeg)\n    \n    x_test.append(f)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:10:18.606285Z","iopub.execute_input":"2024-02-06T07:10:18.607478Z","iopub.status.idle":"2024-02-06T07:10:18.617026Z","shell.execute_reply.started":"2024-02-06T07:10:18.607432Z","shell.execute_reply":"2024-02-06T07:10:18.615939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(TEST_CSV)\n\npool = multiprocessing.Pool(processes=2)\npool.map(get_central_eeg_test, tqdm(test_df.iterrows(),total=len(test_df)))\n\npool.close()\npool.join()\n\nx_test = np.array(x_test)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:10:18.618393Z","iopub.execute_input":"2024-02-06T07:10:18.618856Z","iopub.status.idle":"2024-02-06T07:10:18.874214Z","shell.execute_reply.started":"2024-02-06T07:10:18.618831Z","shell.execute_reply":"2024-02-06T07:10:18.873097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(columns=[\"eeg_id\",\"seizure_vote\",\"lpd_vote\",\"gpd_vote\",\"lrda_vote\",\"grda_vote\",\"other_vote\"])","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:10:18.875839Z","iopub.execute_input":"2024-02-06T07:10:18.876190Z","iopub.status.idle":"2024-02-06T07:10:18.884385Z","shell.execute_reply.started":"2024-02-06T07:10:18.876159Z","shell.execute_reply":"2024-02-06T07:10:18.883336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = models_dict[\"fold_1\"].predict_proba(x_test)\npred","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:10:18.885735Z","iopub.execute_input":"2024-02-06T07:10:18.886107Z","iopub.status.idle":"2024-02-06T07:10:18.905570Z","shell.execute_reply.started":"2024-02-06T07:10:18.886082Z","shell.execute_reply":"2024-02-06T07:10:18.904726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"eeg_id\"].values","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:10:18.906917Z","iopub.execute_input":"2024-02-06T07:10:18.907184Z","iopub.status.idle":"2024-02-06T07:10:18.914049Z","shell.execute_reply.started":"2024-02-06T07:10:18.907161Z","shell.execute_reply":"2024-02-06T07:10:18.913056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[\"eeg_id\"] = test_df[\"eeg_id\"].values\nsubmission['seizure_vote'] = pred[...,0]\nsubmission['lpd_vote'] = pred[...,1]\nsubmission['gpd_vote'] = pred[...,2]\nsubmission['lrda_vote'] = pred[...,3]\nsubmission['grda_vote'] = pred[...,4]\nsubmission['other_vote'] = pred[...,5]","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:10:18.915457Z","iopub.execute_input":"2024-02-06T07:10:18.915733Z","iopub.status.idle":"2024-02-06T07:10:18.924858Z","shell.execute_reply.started":"2024-02-06T07:10:18.915702Z","shell.execute_reply":"2024-02-06T07:10:18.924020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv',index = False)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T07:10:18.927458Z","iopub.execute_input":"2024-02-06T07:10:18.927752Z","iopub.status.idle":"2024-02-06T07:10:18.941633Z","shell.execute_reply.started":"2024-02-06T07:10:18.927726Z","shell.execute_reply":"2024-02-06T07:10:18.940692Z"},"trusted":true},"execution_count":null,"outputs":[]}]}