{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-18T09:50:47.048645Z","iopub.execute_input":"2024-01-18T09:50:47.049038Z","iopub.status.idle":"2024-01-18T09:50:58.241605Z","shell.execute_reply.started":"2024-01-18T09:50:47.049003Z","shell.execute_reply":"2024-01-18T09:50:58.240154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd \nimport numpy as np \nimport matplotlib.pyplot as plt ","metadata":{"execution":{"iopub.status.busy":"2024-01-18T09:50:58.243667Z","iopub.execute_input":"2024-01-18T09:50:58.244153Z","iopub.status.idle":"2024-01-18T09:50:58.248983Z","shell.execute_reply.started":"2024-01-18T09:50:58.244118Z","shell.execute_reply":"2024-01-18T09:50:58.248027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path='/kaggle/input/hms-harmful-brain-activity-classification'\n","metadata":{"execution":{"iopub.status.busy":"2024-01-18T09:50:58.250572Z","iopub.execute_input":"2024-01-18T09:50:58.250936Z","iopub.status.idle":"2024-01-18T09:50:58.262546Z","shell.execute_reply.started":"2024-01-18T09:50:58.250906Z","shell.execute_reply":"2024-01-18T09:50:58.261759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_orignal=pd.read_csv(data_path+'/train.csv')","metadata":{"execution":{"iopub.status.busy":"2024-01-18T09:50:58.264816Z","iopub.execute_input":"2024-01-18T09:50:58.265553Z","iopub.status.idle":"2024-01-18T09:50:58.557142Z","shell.execute_reply.started":"2024-01-18T09:50:58.265517Z","shell.execute_reply":"2024-01-18T09:50:58.555612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_orignal","metadata":{"execution":{"iopub.status.busy":"2024-01-18T09:50:58.558966Z","iopub.execute_input":"2024-01-18T09:50:58.559395Z","iopub.status.idle":"2024-01-18T09:50:58.605707Z","shell.execute_reply.started":"2024-01-18T09:50:58.559362Z","shell.execute_reply":"2024-01-18T09:50:58.6042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#### our goal is to create master dataframe which is 30 gb to reduce ","metadata":{"execution":{"iopub.status.busy":"2024-01-18T09:50:58.608384Z","iopub.execute_input":"2024-01-18T09:50:58.60889Z","iopub.status.idle":"2024-01-18T09:50:58.613291Z","shell.execute_reply.started":"2024-01-18T09:50:58.608847Z","shell.execute_reply":"2024-01-18T09:50:58.612396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spectogram_dummy=pd.read_parquet('/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/353733.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-01-18T09:50:58.614345Z","iopub.execute_input":"2024-01-18T09:50:58.615275Z","iopub.status.idle":"2024-01-18T09:50:58.837384Z","shell.execute_reply.started":"2024-01-18T09:50:58.615239Z","shell.execute_reply":"2024-01-18T09:50:58.835413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eeg_dummy=pd.read_parquet('/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/1000913311.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-01-18T09:50:58.83916Z","iopub.execute_input":"2024-01-18T09:50:58.839881Z","iopub.status.idle":"2024-01-18T09:50:58.907455Z","shell.execute_reply.started":"2024-01-18T09:50:58.839839Z","shell.execute_reply":"2024-01-18T09:50:58.906504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eeg_dummy","metadata":{"execution":{"iopub.status.busy":"2024-01-18T09:50:58.90911Z","iopub.execute_input":"2024-01-18T09:50:58.910073Z","iopub.status.idle":"2024-01-18T09:50:58.939693Z","shell.execute_reply.started":"2024-01-18T09:50:58.910025Z","shell.execute_reply":"2024-01-18T09:50:58.938566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nfolder_path_spec = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/'\nfolder_path_eeg='/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/'\n# List all files in the folder\nfiles_spec = os.listdir(folder_path_spec)\nfiles_eeg = os.listdir(folder_path_eeg)\n# Extract numbers from file names\nids_spectogram = [file.split('.')[0] for file in files_spec if file.endswith('.parquet')]\nids_eeg = [file.split('.')[0] for file in files_eeg if file.endswith('.parquet')]\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-01-18T10:22:22.620624Z","iopub.execute_input":"2024-01-18T10:22:22.621067Z","iopub.status.idle":"2024-01-18T10:22:22.650913Z","shell.execute_reply.started":"2024-01-18T10:22:22.621034Z","shell.execute_reply":"2024-01-18T10:22:22.650017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids_to_check=1628180742","metadata":{"execution":{"iopub.status.busy":"2024-01-18T10:25:14.176791Z","iopub.execute_input":"2024-01-18T10:25:14.177282Z","iopub.status.idle":"2024-01-18T10:25:14.183219Z","shell.execute_reply.started":"2024-01-18T10:25:14.177241Z","shell.execute_reply":"2024-01-18T10:25:14.1816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if id_to_check in ids_eeg:\n    print(f\"{id_to_check} is present in the list.\")\nelse:\n    print(f\"{id_to_check} is not present in the list.\")","metadata":{"execution":{"iopub.status.busy":"2024-01-18T10:25:14.900042Z","iopub.execute_input":"2024-01-18T10:25:14.900756Z","iopub.status.idle":"2024-01-18T10:25:14.909058Z","shell.execute_reply.started":"2024-01-18T10:25:14.900718Z","shell.execute_reply":"2024-01-18T10:25:14.907262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids_spectogram","metadata":{"execution":{"iopub.status.busy":"2024-01-18T10:19:18.294142Z","iopub.execute_input":"2024-01-18T10:19:18.295702Z","iopub.status.idle":"2024-01-18T10:19:18.320917Z","shell.execute_reply.started":"2024-01-18T10:19:18.295643Z","shell.execute_reply":"2024-01-18T10:19:18.319732Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_orignal[train_orignal['eeg_id']==1628180742]","metadata":{"execution":{"iopub.status.busy":"2024-01-18T09:50:59.038857Z","iopub.execute_input":"2024-01-18T09:50:59.039242Z","iopub.status.idle":"2024-01-18T09:50:59.058309Z","shell.execute_reply.started":"2024-01-18T09:50:59.039208Z","shell.execute_reply":"2024-01-18T09:50:59.057387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spectogram_dummy","metadata":{"execution":{"iopub.status.busy":"2024-01-18T09:50:59.059686Z","iopub.execute_input":"2024-01-18T09:50:59.060261Z","iopub.status.idle":"2024-01-18T09:50:59.094405Z","shell.execute_reply.started":"2024-01-18T09:50:59.060222Z","shell.execute_reply":"2024-01-18T09:50:59.093309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eeg_dummy.iloc[1:50]","metadata":{"execution":{"iopub.status.busy":"2024-01-18T09:50:59.095963Z","iopub.execute_input":"2024-01-18T09:50:59.096672Z","iopub.status.idle":"2024-01-18T09:50:59.145314Z","shell.execute_reply.started":"2024-01-18T09:50:59.096629Z","shell.execute_reply":"2024-01-18T09:50:59.14412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in train_orignal[train_orignal['eeg_id']==1628180742]['eeg_label_offset_seconds'].values:\n#     i=int(i)\n#     print(i)\n    ","metadata":{"execution":{"iopub.status.busy":"2024-01-18T09:51:11.243999Z","iopub.execute_input":"2024-01-18T09:51:11.244741Z","iopub.status.idle":"2024-01-18T09:51:11.249756Z","shell.execute_reply.started":"2024-01-18T09:51:11.244706Z","shell.execute_reply":"2024-01-18T09:51:11.24864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Code to automate into function at latter stages we also need to concat","metadata":{}},{"cell_type":"code","source":"final_df=pd.DataFrame()\ntemp=pd.DataFrame()\nfor j in ids_eeg\n    for i in train_orignal[train_orignal['eeg_id']==1628180742]['eeg_label_offset_seconds'].values:\n        i=int(i)\n        temp=pd.DataFrame()\n        temp=eeg_dummy.iloc[i:i+50, :]\n        empty_df=pd.DataFrame()\n        temp_concat=pd.DataFrame()\n        for j in range(i,i+50,1):\n            empty_df=pd.concat([empty_df,train_orignal[(train_orignal['eeg_label_offset_seconds']==i)&(train_orignal['eeg_id']==1628180742)]],axis=0)\n        temp_concat=pd.concat([empty_df.reset_index(),temp.reset_index()],axis=1)\n        final_df=pd.concat([temp_concat,final_df],axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T10:16:43.695625Z","iopub.execute_input":"2024-01-18T10:16:43.696029Z","iopub.status.idle":"2024-01-18T10:16:51.14123Z","shell.execute_reply.started":"2024-01-18T10:16:43.695997Z","shell.execute_reply":"2024-01-18T10:16:51.139745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_orignal[train_orignal['eeg_id']==j]","metadata":{"execution":{"iopub.status.busy":"2024-01-18T10:17:52.951163Z","iopub.execute_input":"2024-01-18T10:17:52.951566Z","iopub.status.idle":"2024-01-18T10:17:52.96406Z","shell.execute_reply.started":"2024-01-18T10:17:52.951534Z","shell.execute_reply":"2024-01-18T10:17:52.962612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids_eeg","metadata":{"execution":{"iopub.status.busy":"2024-01-18T10:18:23.229283Z","iopub.execute_input":"2024-01-18T10:18:23.22981Z","iopub.status.idle":"2024-01-18T10:18:23.250782Z","shell.execute_reply.started":"2024-01-18T10:18:23.22977Z","shell.execute_reply":"2024-01-18T10:18:23.249289Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(empty_df)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T10:04:50.268727Z","iopub.execute_input":"2024-01-18T10:04:50.269184Z","iopub.status.idle":"2024-01-18T10:04:50.277001Z","shell.execute_reply.started":"2024-01-18T10:04:50.26915Z","shell.execute_reply":"2024-01-18T10:04:50.275682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"empty_df=pd.DataFrame()","metadata":{"execution":{"iopub.status.busy":"2024-01-18T09:57:19.142604Z","iopub.execute_input":"2024-01-18T09:57:19.143124Z","iopub.status.idle":"2024-01-18T09:57:19.149026Z","shell.execute_reply.started":"2024-01-18T09:57:19.143082Z","shell.execute_reply":"2024-01-18T09:57:19.147826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"empty_df=pd.concat([empty_df,train_orignal[(train_orignal['eeg_label_offset_seconds']==i)&(train_orignal['eeg_id']==1628180742)]],axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T09:58:16.79589Z","iopub.execute_input":"2024-01-18T09:58:16.79639Z","iopub.status.idle":"2024-01-18T09:58:16.809535Z","shell.execute_reply.started":"2024-01-18T09:58:16.796351Z","shell.execute_reply":"2024-01-18T09:58:16.80799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"empty_df","metadata":{"execution":{"iopub.status.busy":"2024-01-18T09:58:20.922936Z","iopub.execute_input":"2024-01-18T09:58:20.92338Z","iopub.status.idle":"2024-01-18T09:58:20.943037Z","shell.execute_reply.started":"2024-01-18T09:58:20.923347Z","shell.execute_reply":"2024-01-18T09:58:20.942062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}