{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport seaborn as sns\nimport torch\nimport os\nfrom pandas.plotting import scatter_matrix\nimport sys\nfrom tempfile import NamedTemporaryFile\nfrom urllib.request import urlopen\nfrom urllib.parse import unquote, urlparse\nfrom urllib.error import HTTPError\nfrom zipfile import ZipFile\nimport tarfile\nimport shutil","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:14:41.723644Z","iopub.execute_input":"2024-02-13T14:14:41.724100Z","iopub.status.idle":"2024-02-13T14:15:03.694807Z","shell.execute_reply.started":"2024-02-13T14:14:41.724066Z","shell.execute_reply":"2024-02-13T14:15:03.693714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the dataset\ndf = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:15:03.697139Z","iopub.execute_input":"2024-02-13T14:15:03.698218Z","iopub.status.idle":"2024-02-13T14:15:04.069338Z","shell.execute_reply.started":"2024-02-13T14:15:03.698178Z","shell.execute_reply":"2024-02-13T14:15:04.068407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data Overview\ndf.info()","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:15:04.070560Z","iopub.execute_input":"2024-02-13T14:15:04.071156Z","iopub.status.idle":"2024-02-13T14:15:04.119261Z","shell.execute_reply.started":"2024-02-13T14:15:04.071121Z","shell.execute_reply":"2024-02-13T14:15:04.117711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count of each class\nclass_counts = df[['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']].sum()\nsns.barplot(x=class_counts.index, y=class_counts.values)\nplt.show()\ndf.describe()","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:15:04.121043Z","iopub.execute_input":"2024-02-13T14:15:04.121389Z","iopub.status.idle":"2024-02-13T14:15:04.476426Z","shell.execute_reply.started":"2024-02-13T14:15:04.121360Z","shell.execute_reply":"2024-02-13T14:15:04.475476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count of each class\nclass_counts = df[['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']].sum()\nsns.barplot(x=class_counts.index, y=class_counts.values)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:15:04.479146Z","iopub.execute_input":"2024-02-13T14:15:04.479750Z","iopub.status.idle":"2024-02-13T14:15:04.999118Z","shell.execute_reply.started":"2024-02-13T14:15:04.479717Z","shell.execute_reply":"2024-02-13T14:15:04.997438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Boxplot for anomaly detection in votes\nsns.boxplot(data=df[['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:15:05.000920Z","iopub.execute_input":"2024-02-13T14:15:05.001490Z","iopub.status.idle":"2024-02-13T14:15:05.497372Z","shell.execute_reply.started":"2024-02-13T14:15:05.001449Z","shell.execute_reply":"2024-02-13T14:15:05.495937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Histogram of votes for each class\nsns.histplot(data=df, x='seizure_vote', bins=20, kde=True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:15:05.498976Z","iopub.execute_input":"2024-02-13T14:15:05.499591Z","iopub.status.idle":"2024-02-13T14:15:06.322404Z","shell.execute_reply.started":"2024-02-13T14:15:05.499555Z","shell.execute_reply":"2024-02-13T14:15:06.321544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Handling missing values (if any)\n# For demonstration, we'll fill missing numerical values with the median and categorical with the mode\nnumerical_cols = df.select_dtypes(include=['float64', 'int64']).columns\ncategorical_cols = df.select_dtypes(include=['object']).columns\n\nfor col in numerical_cols:\n    df[col].fillna(df[col].median(), inplace=True)\n\nfor col in categorical_cols:\n    df[col].fillna(df[col].mode()[0], inplace=True)\n\n# Distribution of Numerical Features\nfig, axes = plt.subplots(len(numerical_cols), 1, figsize=(10, 2*len(numerical_cols)))\nfor i, col in enumerate(numerical_cols):\n    sns.histplot(df[col], ax=axes[i], kde=True)\n    axes[i].set_title(f'Distribution of {col}')\nplt.tight_layout()\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:15:06.324606Z","iopub.execute_input":"2024-02-13T14:15:06.325100Z","iopub.status.idle":"2024-02-13T14:15:29.302597Z","shell.execute_reply.started":"2024-02-13T14:15:06.325060Z","shell.execute_reply":"2024-02-13T14:15:29.301165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Categorical Analysis\nplt.figure(figsize=(10, 6))\nsns.countplot(y='expert_consensus', data=df, order=df['expert_consensus'].value_counts().index)\nplt.title('Distribution of Expert Consensus')","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:15:29.303731Z","iopub.execute_input":"2024-02-13T14:15:29.304091Z","iopub.status.idle":"2024-02-13T14:15:29.675834Z","shell.execute_reply.started":"2024-02-13T14:15:29.304053Z","shell.execute_reply":"2024-02-13T14:15:29.674188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Correlation Analysis\ncorrelation_matrix = df[numerical_cols].corr()\nplt.figure(figsize=(10, 8))\nsns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', fmt=\".2f\", linewidths=.05)\nplt.title('Correlation Matrix of Numerical Features')","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:15:29.677542Z","iopub.execute_input":"2024-02-13T14:15:29.677956Z","iopub.status.idle":"2024-02-13T14:15:30.907948Z","shell.execute_reply.started":"2024-02-13T14:15:29.677923Z","shell.execute_reply":"2024-02-13T14:15:30.906726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Outlier Analysis with Box Plots\nfig, axes = plt.subplots(len(numerical_cols), 1, figsize=(10, 2*len(numerical_cols)))\nfor i, col in enumerate(numerical_cols):\n    sns.boxplot(x=df[col], ax=axes[i])\n    axes[i].set_title(f'Box Plot of {col}')\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:15:30.909647Z","iopub.execute_input":"2024-02-13T14:15:30.910034Z","iopub.status.idle":"2024-02-13T14:15:34.195958Z","shell.execute_reply.started":"2024-02-13T14:15:30.910001Z","shell.execute_reply":"2024-02-13T14:15:34.194019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pyarrow","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:48:40.396195Z","iopub.execute_input":"2024-02-13T14:48:40.397765Z","iopub.status.idle":"2024-02-13T14:48:56.031360Z","shell.execute_reply.started":"2024-02-13T14:48:40.397708Z","shell.execute_reply":"2024-02-13T14:48:56.029473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pyarrow.parquet as pq","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:48:56.034528Z","iopub.execute_input":"2024-02-13T14:48:56.034951Z","iopub.status.idle":"2024-02-13T14:48:56.042439Z","shell.execute_reply.started":"2024-02-13T14:48:56.034916Z","shell.execute_reply":"2024-02-13T14:48:56.040836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = \"/kaggle/input/hms-harmful-brain-activity-classification/\"","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:48:56.044181Z","iopub.execute_input":"2024-02-13T14:48:56.044546Z","iopub.status.idle":"2024-02-13T14:48:56.054766Z","shell.execute_reply.started":"2024-02-13T14:48:56.044517Z","shell.execute_reply":"2024-02-13T14:48:56.053304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_path1 = BASE_DIR + 'test_eegs/3911565283.parquet'\n\ntable1 = pq.read_table(file_path1)\n\n# Convert the table to a pandas DataFrame\ndf = table1.to_pandas()\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:48:56.058717Z","iopub.execute_input":"2024-02-13T14:48:56.059313Z","iopub.status.idle":"2024-02-13T14:48:56.105790Z","shell.execute_reply.started":"2024-02-13T14:48:56.059271Z","shell.execute_reply":"2024-02-13T14:48:56.104167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_path2 = BASE_DIR + 'test_spectrograms/853520.parquet'\n\ntable2 = pq.read_table(file_path2)\n\n# Convert the table to a pandas DataFrame\ndf = table2.to_pandas()\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-13T14:48:56.107659Z","iopub.execute_input":"2024-02-13T14:48:56.108028Z","iopub.status.idle":"2024-02-13T14:48:56.185530Z","shell.execute_reply.started":"2024-02-13T14:48:56.107997Z","shell.execute_reply":"2024-02-13T14:48:56.184331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BD = \"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs\"\n\nparquet_files = [f for f in os.listdir(BD) if f.endswith('.parquet')]\n\ndfs = []\n\n# Loop over the files and read each one\nfor file in parquet_files:\n    file_path = os.path.join(BD, file)\n    table = pq.read_table(file_path)\n    df = table.to_pandas()\n    dfs.append(df)","metadata":{"execution":{"iopub.status.busy":"2024-02-13T15:37:25.947991Z","iopub.execute_input":"2024-02-13T15:37:25.948838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}