{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfrom sklearn.model_selection import GroupKFold\n\nRANDOM_SEED = 1086\n\nCLASSES = [\"seizure_vote\", \"lpd_vote\", \"gpd_vote\", \"lrda_vote\", \"grda_vote\", \"other_vote\"]\nN_CLASSES = len(CLASSES)\nFOLDS = [0, 1, 2, 3, 4]\nN_FOLDS = len(FOLDS)\n\ntrain = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\n\ntrain['total_evaluators'] = train[['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']].sum(axis=1)\ntrain[CLASSES] /= train[CLASSES].sum(axis=1).values[:, None]\n# train = train.groupby([\"eeg_id\", \"seizure_vote\", \"lpd_vote\", \"gpd_vote\", \"lrda_vote\", \"grda_vote\", \"other_vote\"])\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-27T20:34:13.855602Z","iopub.execute_input":"2024-02-27T20:34:13.857007Z","iopub.status.idle":"2024-02-27T20:34:14.086994Z","shell.execute_reply.started":"2024-02-27T20:34:13.856955Z","shell.execute_reply":"2024-02-27T20:34:14.085769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nan_list = set([228018333, 2609800009, 1033402741, 3360776627, 1936370864, 3447241762, 1799086501, 2294110417, 1774977261, 837428467,\n           1511903313, 3254156300, 3030710864, 2393305422, 191408832, 1944688715, 1092242633, 2565199369, 1750767188, 352100210,\n           227051891, 3980685149, 3197027554, 689412237, 84103002, 788653799, 952997900, 1410292431, 1339041688, 725185416,\n           1267818343, 1907765019, 1293001777, 523936200, 3326234321, 1309137409, 2190373347, 2538961182, 2406368236,\n           3418300291, 3780781224, 3707844385, 975631111, 1459425125, 3932380488, 420827282, 23656323, 1301620928,\n           1916955481, 3042198969, 2525259601, 2763615057, 2641635192, 2276240743, 2728561398, 4018623581, 21746311,\n           524927151, 2185189413, 116770645, 3931449367, 4045810693, 1828657109, 1889311261, 3386127802, 2800397565,\n           1379952459, 3948834462, 1747732327, 1366044762, 4115113596, 1593385762, 1119914885, 402182162, 3595195521,\n           2572900449, 2906001188, 1782988316, 2690795118, 579740230, 2272873515, 1926819138, 1534551817, 82511342, 2081405553,\n           3168916926, 2551548463, 3939775387])\ntrain = train[~train['eeg_id'].isin(nan_list)].reset_index(drop=False)\nprint(train.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sgkf = GroupKFold(n_splits=N_FOLDS)\n\ntrain[\"fold\"] = -1\n\nfor fold_id, (_, val_idx) in enumerate(\n    sgkf.split(train, y=train[\"expert_consensus\"], groups=train[\"patient_id\"])\n):\n    train.loc[val_idx, \"fold\"] = fold_id","metadata":{"execution":{"iopub.status.busy":"2024-02-27T20:34:14.089359Z","iopub.execute_input":"2024-02-27T20:34:14.090210Z","iopub.status.idle":"2024-02-27T20:34:14.121578Z","shell.execute_reply.started":"2024-02-27T20:34:14.090162Z","shell.execute_reply":"2024-02-27T20:34:14.120524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in FOLDS:\n    train_idx = train[train[\"fold\"] != fold].index.values\n    val_idx   = train[train[\"fold\"] == fold].index.values\n    eeg_ids = []\n    raw_labels = train[CLASSES].values\n    labels = []\n    spec_ids = []\n    eeg_label_offset_seconds = []\n    spectrogram_label_offset_seconds = []\n    total_votes = []\n    for raw_label in raw_labels:\n        label = ' '.join(map(str, raw_label))\n        labels.append(label)\n    \n    for eeg_id in train[\"eeg_id\"].values:\n        eeg_ids.append(eeg_id)\n        \n    for spec_id in train[\"spectrogram_id\"].values:\n        spec_ids.append(spec_id)\n        \n    for spectrogram_label_offset_second in train[\"spectrogram_label_offset_seconds\"].values:\n        spectrogram_label_offset_seconds.append(spectrogram_label_offset_second)\n        \n    for eeg_label_offset_second in train[\"eeg_label_offset_seconds\"].values:\n        eeg_label_offset_seconds.append(eeg_label_offset_second)\n        \n    for total_vote in train[\"total_evaluators\"].values:\n        total_votes.append(int(total_vote))\n\n    train_eeg_ids = [eeg_ids[idx] for idx in train_idx]\n    train_spec_ids = [spec_ids[idx] for idx in train_idx]\n    train_labels = [labels[idx] for idx in train_idx]\n    train_spectrogram_label_offset_seconds = [spectrogram_label_offset_seconds[idx] for idx in train_idx]\n    train_eeg_label_offset_seconds = [eeg_label_offset_seconds[idx] for idx in train_idx]\n    train_total_votes = [total_votes[idx] for idx in train_idx]\n\n    val_eeg_ids = [eeg_ids[idx] for idx in val_idx]\n    val_spec_ids = [spec_ids[idx] for idx in val_idx]\n    val_labels = [labels[idx] for idx in val_idx]\n    val_spectrogram_label_offset_seconds = [spectrogram_label_offset_seconds[idx] for idx in val_idx]\n    val_eeg_label_offset_seconds = [eeg_label_offset_seconds[idx] for idx in val_idx]\n    val_total_votes = [total_votes[idx] for idx in val_idx]\n    \n    train_eeg_ids = np.array(train_eeg_ids)\n    train_spec_ids = np.array(train_spec_ids)\n    train_labels = np.array(train_labels)\n    train_spectrogram_label_offset_seconds = np.array(train_spectrogram_label_offset_seconds)\n    train_eeg_label_offset_seconds = np.array(train_eeg_label_offset_seconds)\n    train_total_votes = np.array(train_total_votes)\n    val_eeg_ids = np.array(val_eeg_ids)\n    val_spec_ids = np.array(val_spec_ids)\n    val_labels = np.array(val_labels)\n    val_spectrogram_label_offset_seconds = np.array(val_spectrogram_label_offset_seconds)\n    val_eeg_label_offset_seconds = np.array(val_eeg_label_offset_seconds)\n    val_total_votes = np.array(val_total_votes)\n    train_eeg_ids = np.expand_dims(train_eeg_ids, 1)\n    train_spec_ids = np.expand_dims(train_spec_ids, 1)\n    train_labels = np.expand_dims(train_labels, 1)\n    train_spectrogram_label_offset_seconds = np.expand_dims(train_spectrogram_label_offset_seconds, 1)\n    train_eeg_label_offset_seconds = np.expand_dims(train_eeg_label_offset_seconds, 1)\n    train_total_votes = np.expand_dims(train_total_votes, 1)\n    val_eeg_ids = np.expand_dims(val_eeg_ids, 1)\n    val_spec_ids = np.expand_dims(val_spec_ids, 1)\n    val_labels = np.expand_dims(val_labels, 1)\n    val_spectrogram_label_offset_seconds = np.expand_dims(val_spectrogram_label_offset_seconds, 1)\n    val_eeg_label_offset_seconds = np.expand_dims(val_eeg_label_offset_seconds, 1)\n    val_total_votes = np.expand_dims(val_total_votes, 1)\n    \n    train_df = pd.DataFrame(np.concatenate((train_eeg_ids, train_spec_ids, train_labels, train_eeg_label_offset_seconds, train_spectrogram_label_offset_seconds, train_total_votes), axis=1), columns=[\"eeg_id\", \"spec_id\", \"label\", \"eeg_label_offset_seconds\", \"spectrogram_label_offset_seconds\", \"total_votes\"])\n    train_df.to_csv('train_fold{}.csv'.format(fold), index=False)\n    \n    val_df = pd.DataFrame(np.concatenate((val_eeg_ids, val_spec_ids, val_labels, val_eeg_label_offset_seconds, val_spectrogram_label_offset_seconds, val_total_votes), axis=1), columns=[\"eeg_id\", \"spec_id\", \"label\", \"eeg_label_offset_seconds\", \"spectrogram_label_offset_seconds\", \"total_votes\"])\n    val_df = val_df.groupby([\"eeg_id\"]).head(1)\n    val_df.to_csv('val_fold{}.csv'.format(fold), index=False)\n    train_df_stage1 = train_df.loc[train_df['total_votes'].astype(np.int64) < 10]\n    train_df_stage1.to_csv('train_fold{}_stage1.csv'.format(fold), index=False)\n    val_df_stage1 = val_df.loc[val_df['total_votes'].astype(np.int64) < 10]\n    val_df_stage1.to_csv('val_fold{}_stage1.csv'.format(fold), index=False)\n    train_df_stage2 = train_df.loc[train_df['total_votes'].astype(np.int64) >= 10]\n    train_df_stage2.to_csv('train_fold{}_stage2.csv'.format(fold), index=False)\n    val_df_stage2 = val_df.loc[val_df['total_votes'].astype(np.int64) >= 10]\n    val_df_stage2.to_csv('val_fold{}_stage2.csv'.format(fold), index=False)\n    \n    train_df_private = train_df.loc[train_df['total_votes'].astype(np.int64) >= 3]\n    train_df_private.to_csv('train_fold{}_private.csv'.format(fold), index=False)\n    val_df_private = val_df.loc[val_df['total_votes'].astype(np.int64) >= 3]\n    val_df_private.to_csv('val_fold{}_private.csv'.format(fold), index=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T20:34:14.152160Z","iopub.execute_input":"2024-02-27T20:34:14.152544Z","iopub.status.idle":"2024-02-27T20:34:16.273914Z","shell.execute_reply.started":"2024-02-27T20:34:14.152511Z","shell.execute_reply":"2024-02-27T20:34:16.272583Z"},"trusted":true},"execution_count":null,"outputs":[]}]}