{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30636,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom sklearn import *\nfrom statistics import quantiles\npd.set_option('mode.chained_assignment', None)\n\np = '../input/hms-harmful-brain-activity-classification/'\ntrain = pd.read_csv(p+'train.csv')\ncol = ['eeg_id', 'spectrogram_id', 'seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote', 'expert_consensus']\ntrain = train[col]\ntrain = train.drop_duplicates(subset=['eeg_id']).reset_index(drop=True)\ntest = pd.read_csv(p+'test.csv')\nsub = pd.read_csv(p+'sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-01-21T13:18:18.710167Z","iopub.execute_input":"2024-01-21T13:18:18.711023Z","iopub.status.idle":"2024-01-21T13:18:20.131456Z","shell.execute_reply.started":"2024-01-21T13:18:18.710990Z","shell.execute_reply":"2024-01-21T13:18:20.130423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_parquet('../input/hms-harmful-brain-activity-classification/train_eegs/'+ str(train.eeg_id[0]) +'.parquet')[:10000][:10_000][4_000:6_000] #200 * 5\n_ = df.plot(subplots=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-21T13:18:26.450871Z","iopub.execute_input":"2024-01-21T13:18:26.452113Z","iopub.status.idle":"2024-01-21T13:18:32.037501Z","shell.execute_reply.started":"2024-01-21T13:18:26.452077Z","shell.execute_reply":"2024-01-21T13:18:32.036494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getStats(path, ids, idname):\n    all_data = []\n    for id_ in tqdm(ids):\n        try:\n            df = pd.read_parquet(path + str(id_) + '.parquet')[:10_000][4_000:6_000]\n            cols = ['Fp1', 'F3', 'C3', 'P3', 'F7', 'T3', 'T5', 'O1', 'Fz', 'Cz', 'Pz', 'Fp2', 'F4', 'C4', 'P4', 'F8', 'T4', 'T6', 'O2'] #, 'EKG'\n            head = [idname] + ['mean_'+str(i) for i in range(4000,6000)]\n            df_data = [id_] + list(df[cols].sum(axis=1).values)\n        except:\n            df_data = [id_] + [np.nan for i in range(2000)]\n        all_data.append(df_data[:])\n    return pd.DataFrame(all_data, columns=head)","metadata":{"execution":{"iopub.status.busy":"2024-01-21T13:18:37.744530Z","iopub.execute_input":"2024-01-21T13:18:37.744887Z","iopub.status.idle":"2024-01-21T13:18:37.752863Z","shell.execute_reply.started":"2024-01-21T13:18:37.744861Z","shell.execute_reply":"2024-01-21T13:18:37.751871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getStats2(path, ids, idname):\n    all_data = []\n    for id_ in tqdm(ids):\n        try:\n            df = pd.read_parquet(path + str(id_) + '.parquet')\n            df = df[df['time'].isin(range(295,306))].reset_index(drop=True)\n            cols = df.columns[1:] #remove time column\n            head = [idname] + ['_'.join([c,str(i)]) for i in range(6) for c in cols]\n            df_data = [id_] + list(df[:6][cols].values.flatten())\n        except:\n            df_data = [id_] + [np.nan for i in range(400*6)]\n        all_data.append(df_data[:])\n    return pd.DataFrame(all_data, columns=head)","metadata":{"execution":{"iopub.status.busy":"2024-01-21T13:18:45.635381Z","iopub.execute_input":"2024-01-21T13:18:45.636303Z","iopub.status.idle":"2024-01-21T13:18:45.643888Z","shell.execute_reply.started":"2024-01-21T13:18:45.636267Z","shell.execute_reply":"2024-01-21T13:18:45.642891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrainx = getStats('../input/hms-harmful-brain-activity-classification/train_eegs/', train.eeg_id.values, 'eeg_id')\ntestx = getStats('../input/hms-harmful-brain-activity-classification/test_eegs/', test.eeg_id.values, 'eeg_id')\n\ntrainx2 = getStats2('../input/hms-harmful-brain-activity-classification/train_spectrograms/', train.spectrogram_id.values, 'spectrogram_id')\ntestx2 = getStats2('../input/hms-harmful-brain-activity-classification/test_spectrograms/', test.spectrogram_id.values, 'spectrogram_id')","metadata":{"execution":{"iopub.status.busy":"2024-01-21T13:18:53.569403Z","iopub.execute_input":"2024-01-21T13:18:53.570093Z","iopub.status.idle":"2024-01-21T13:39:20.280326Z","shell.execute_reply.started":"2024-01-21T13:18:53.570057Z","shell.execute_reply":"2024-01-21T13:39:20.279332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainx = pd.concat((trainx, trainx2), axis=1)\ntrainx.to_csv('trainx.csv', index=False) #Use output with different models no rerun required\nprint(trainx.shape)\n\ntestx = pd.concat((testx, testx2), axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-01-21T13:39:20.282066Z","iopub.execute_input":"2024-01-21T13:39:20.282376Z","iopub.status.idle":"2024-01-21T13:40:38.411437Z","shell.execute_reply.started":"2024-01-21T13:39:20.282351Z","shell.execute_reply":"2024-01-21T13:40:38.410427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epsilon=10e-15\ndef kl_divergence(solution, submission, micro=True):\n    for col in solution.columns:\n        submission[col] = np.clip(submission[col], epsilon, 1 - epsilon)\n        y_nonzero_indices = solution[col] != 0\n        solution[col] = solution[col].astype(float)\n        solution.loc[y_nonzero_indices, col] = solution.loc[y_nonzero_indices, col] * np.log(solution.loc[y_nonzero_indices, col] / submission.loc[y_nonzero_indices, col])\n        solution.loc[~y_nonzero_indices, col] = 0\n        if micro:\n            return np.average(solution.sum(axis=1))\n        else:\n            return np.average(solution.mean())","metadata":{"execution":{"iopub.status.busy":"2024-01-21T13:40:38.412746Z","iopub.execute_input":"2024-01-21T13:40:38.413164Z","iopub.status.idle":"2024-01-21T13:40:38.420689Z","shell.execute_reply.started":"2024-01-21T13:40:38.413129Z","shell.execute_reply":"2024-01-21T13:40:38.419762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xcol = [c for c in trainx.columns if c not in ['eeg_id','spectrogram_id', 'expert_consensus']]\nycol = [c for c in train.columns if c not in ['eeg_id','spectrogram_id', 'expert_consensus']]\n\ncd = {'Seizure':'seizure_vote', 'GPD':'gpd_vote', 'LRDA':'lrda_vote', 'Other':'other_vote', 'GRDA':'grda_vote', 'LPD':'lpd_vote'}\ntrain['expert_consensus'] = train['expert_consensus'].map(cd)\nfor i in range(len(train)):\n    c = train['expert_consensus'][i]\n    train[c][i] = train[c][i]+10 #adding weight to expert consensus\n\nysum = train[ycol].sum(axis=1) \nfor c in ycol:\n    train[c] = (train[c] / ysum).astype(np.float64)","metadata":{"execution":{"iopub.status.busy":"2024-01-21T13:40:38.423196Z","iopub.execute_input":"2024-01-21T13:40:38.423554Z","iopub.status.idle":"2024-01-21T13:40:42.867852Z","shell.execute_reply.started":"2024-01-21T13:40:38.423521Z","shell.execute_reply":"2024-01-21T13:40:42.867097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x1, x2, y1, y2 = model_selection.train_test_split(trainx[xcol].fillna(0), train[ycol], test_size=0.3, random_state=11, stratify=train.expert_consensus)\nmodel = ensemble.ExtraTreesRegressor(n_estimators=400, max_depth=None, n_jobs=-1, random_state=1, verbose=0, max_features='sqrt')\nmodel.fit(x1, y1)\npred = pd.DataFrame(model.predict(x2), columns=ycol)\nysum = pred.sum(axis=1)\nfor c in pred.columns: pred[c] = (pred[c] / ysum).astype(np.float64)\nscore=kl_divergence(y2.reset_index(drop=True), pred)\nprint(score)\nmodel.fit(trainx[xcol].fillna(0), train[ycol])\nsub = pd.DataFrame(model.predict(testx[xcol].fillna(0)), columns=ycol)\nysum = sub.sum(axis=1)\nfor c in sub.columns: sub[c] = (sub[c] / ysum).astype(np.float64)\nsub['eeg_id'] = test['eeg_id']\nsub.to_csv('submission.csv', index=False)\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-21T13:40:42.868962Z","iopub.execute_input":"2024-01-21T13:40:42.869339Z","iopub.status.idle":"2024-01-21T13:42:28.539193Z","shell.execute_reply.started":"2024-01-21T13:40:42.869304Z","shell.execute_reply":"2024-01-21T13:42:28.538162Z"},"trusted":true},"execution_count":null,"outputs":[]}]}