{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7387907,"sourceType":"datasetVersion","datasetId":4294350}],"dockerImageVersionId":30627,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport copy\nfrom sklearn.model_selection import train_test_split\nimport numpy as np\nimport keras\nimport os\nimport concurrent.futures\nimport xgboost as xgb\nfrom sklearn.metrics import accuracy_score\nimport lightgbm as lgb","metadata":{"execution":{"iopub.status.busy":"2024-01-12T09:49:41.487910Z","iopub.execute_input":"2024-01-12T09:49:41.488295Z","iopub.status.idle":"2024-01-12T09:50:00.932928Z","shell.execute_reply.started":"2024-01-12T09:49:41.488262Z","shell.execute_reply":"2024-01-12T09:50:00.931580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\ntest = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-01-12T09:50:00.935013Z","iopub.execute_input":"2024-01-12T09:50:00.935479Z","iopub.status.idle":"2024-01-12T09:50:01.279021Z","shell.execute_reply.started":"2024-01-12T09:50:00.935438Z","shell.execute_reply":"2024-01-12T09:50:01.278019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = pd.read_parquet('/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/3911565283.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-01-12T09:50:46.041001Z","iopub.execute_input":"2024-01-12T09:50:46.041596Z","iopub.status.idle":"2024-01-12T09:50:46.314772Z","shell.execute_reply.started":"2024-01-12T09:50:46.041549Z","shell.execute_reply":"2024-01-12T09:50:46.313555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Getting features with describe on EEGS","metadata":{}},{"cell_type":"markdown","source":"**describe contains [count, mean, std, min, 25%, 50%, 75%, max] which can be flattened to get a 1D feature vector for an eeg file** ","metadata":{}},{"cell_type":"code","source":"# %%time\n# import os\n# import concurrent.futures\n# import pandas as pd\n\n# folder_path = '/kaggle/input/hms-harmful-brain-activity-classification/train_eegs'\n\n# def process_file(filename):\n#     file_path = os.path.join(folder_path, filename)\n#     df = pd.read_parquet(file_path)\n#     df = df.describe().drop(df.describe().index[0])\n#     return filename.split('.')[0], df.to_numpy().flatten()\n\n# train_eegs = {}\n# file_list = os.listdir(folder_path)\n\n# with concurrent.futures.ProcessPoolExecutor() as executor:\n#     futures = {executor.submit(process_file, filename): filename for filename in file_list}\n#     for future in concurrent.futures.as_completed(futures):\n#         filename = futures[future]\n#         try:\n#             result = future.result()\n#             train_eegs[result[0]] = result[1]\n#         except Exception as e:\n#             print(f\"Error processing file {filename}: {e}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from joblib import dump, load\n# dump(train_eegs,'train_eegs.joblib')\ntrain_eegs = load('/kaggle/input/train-eegs/train_eegs.joblib')","metadata":{"execution":{"iopub.status.busy":"2024-01-12T09:50:06.476387Z","iopub.execute_input":"2024-01-12T09:50:06.477774Z","iopub.status.idle":"2024-01-12T09:50:08.891304Z","shell.execute_reply.started":"2024-01-12T09:50:06.477728Z","shell.execute_reply":"2024-01-12T09:50:08.889885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = train[['eeg_id','seizure_vote','lpd_vote','gpd_vote','lrda_vote','grda_vote','other_vote','expert_consensus']]\nx_train = x_train.drop_duplicates()","metadata":{"execution":{"iopub.status.busy":"2024-01-12T09:50:10.608004Z","iopub.execute_input":"2024-01-12T09:50:10.608441Z","iopub.status.idle":"2024-01-12T09:50:10.676797Z","shell.execute_reply.started":"2024-01-12T09:50:10.608404Z","shell.execute_reply":"2024-01-12T09:50:10.675610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val = []\nfor i in x_train.values:\n    key = str(i[0])\n    val.append(train_eegs[key])\nval = np.asarray(val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = pd.DataFrame(val, columns= range(len(val[0])) )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = x_train.reset_index()\nx_train = pd.concat([x_train, n], axis=1)\nx_train = x_train.drop(columns = ['index'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Making x_test","metadata":{}},{"cell_type":"code","source":"x_test = test[['eeg_id']]\nx_sub = sub[['eeg_id']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfolder_path = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs'\n\ndef process_file(filename):\n    file_path = os.path.join(folder_path, filename)\n    df = pd.read_parquet(file_path)\n    df = df.describe().drop(df.describe().index[0])\n    return filename.split('.')[0], df.to_numpy().flatten()\n\ntest_eegs = {}\nfile_list = os.listdir(folder_path)\n\nwith concurrent.futures.ProcessPoolExecutor() as executor:\n    futures = {executor.submit(process_file, filename): filename for filename in file_list}\n    for future in concurrent.futures.as_completed(futures):\n        filename = futures[future]\n        try:\n            result = future.result()\n            test_eegs[result[0]] = result[1]\n        except Exception as e:\n            print(f\"Error processing file {filename}: {e}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val = []\nfor i in x_sub.values:\n    key = str(i[0])\n    val.append(test_eegs[key])\nval = np.asarray(val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = pd.DataFrame(val, columns= range(len(val[0])) )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_sub = x_sub.reset_index()\nx_sub = pd.concat([x_sub, n], axis=1)\nx_sub = x_sub.drop(columns= ['index'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# data labeling and preprocessing","metadata":{}},{"cell_type":"markdown","source":"**Considering this a classification problem **","metadata":{}},{"cell_type":"code","source":"x_train['expert_consensus'] = x_train['expert_consensus'].replace({'Seizure': 0, 'LPD': 1, 'GPD': 2,'LRDA':3, 'GRDA':4, 'Other':5 })","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_train = x_train.drop(columns = ['eeg_id','seizure_vote','lpd_vote',\t'gpd_vote',\t'lrda_vote','grda_vote','other_vote','expert_consensus']).copy()\nlgb_test = x_sub.drop(columns = ['eeg_id']).copy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LGBM Modeling","metadata":{}},{"cell_type":"code","source":"params = {\n 'objective': 'multiclass',\n 'num_class': 6,\n 'boosting_type': 'gbdt',\n 'metric': 'multi_logloss',\n 'device': 'gpu',\n 'num_leaves': 121,\n 'learning_rate': 0.018623105710769177,\n 'feature_fraction': 0.5894871939636406,\n 'bagging_fraction': 0.756777580360579,\n 'max_depth': 8\n}\n\n\nlgb_model = lgb.LGBMClassifier(**params)\nlgb_model.fit(lgb_train, x_train.expert_consensus , verbose=0)\n\ny_pred = lgb_model.predict(lgb_test)\ny_pred_proba = lgb_model.predict_proba(lgb_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {'objective': 'multi:softprob',\n 'num_class': 6,\n 'booster': 'gbtree',\n 'eval_metric': 'mlogloss',\n 'max_depth': 8,\n 'learning_rate': 0.008406279027937572,\n 'subsample': 0.7273986104941954,\n 'colsample_bytree': 0.6818816981862805,\n 'min_child_weight': 6,\n 'gamma': 2.2671779654246492e-07}\n\nxgb_model = xgb.XGBClassifier(**params, )\nxgb_model.fit(lgb_train, x_train.expert_consensus )\n\nfinal_y_pred_proba = xgb_model.predict_proba(lgb_test)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_proba = y_pred_proba*0.8 + final_y_pred_proba*0.2","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['seizure_vote'] = y_pred_proba[:,0]\nsub['lpd_vote'] = y_pred_proba[:,1]\nsub['gpd_vote'] = y_pred_proba[:,2]\nsub['lrda_vote'] = y_pred_proba[:,3]\nsub['grda_vote'] = y_pred_proba[:,4]\nsub['other_vote'] = y_pred_proba[:,5]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv',index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***An upvote would be appreciated if you find this notebook helpful :')***","metadata":{}}]}