{"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7656513,"sourceType":"datasetVersion","datasetId":4460771}],"dockerImageVersionId":30664,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 📽️ Imports & tools","metadata":{}},{"cell_type":"code","source":"import os\nfrom pathlib import Path\nimport socket\nfrom pprint import pprint, pformat\nfrom datetime import datetime\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n# Suppress performance warnings\nfrom warnings import simplefilter\n\nsimplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)","metadata":{"id":"GfmwIBqefQqz","execution":{"iopub.status.busy":"2024-02-28T13:51:45.119116Z","iopub.execute_input":"2024-02-28T13:51:45.120158Z","iopub.status.idle":"2024-02-28T13:51:45.126854Z","shell.execute_reply.started":"2024-02-28T13:51:45.120124Z","shell.execute_reply":"2024-02-28T13:51:45.125447Z"},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🎥 Configs","metadata":{"id":"aq-Uj66G0DdY"}},{"cell_type":"code","source":"import seaborn as sns\n\n# Using Seaborn to generate a color palette\ncolor_palette = sns.color_palette(\"bright\", 401).as_hex()","metadata":{"execution":{"iopub.status.busy":"2024-02-28T13:51:45.129023Z","iopub.execute_input":"2024-02-28T13:51:45.129259Z","iopub.status.idle":"2024-02-28T13:51:45.141076Z","shell.execute_reply.started":"2024-02-28T13:51:45.129238Z","shell.execute_reply":"2024-02-28T13:51:45.140433Z"},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_SAMPLES = 100\nroot = f\"/kaggle/input/hms-harmful-brain-activity-classification\"\n# root = f\"../data/hms-harmful-brain-activity-classification\"\n\nroot_spect = os.path.join(root, \"train_spectrograms\")\nroot_eeg = os.path.join(root, \"train_eegs\")\n\neeg_cols = eval(\n    \"['Fp1', 'F3', 'C3', 'P3', 'F7', 'T3', 'T5', 'O1', 'Fz', 'Cz', 'Pz', 'Fp2', 'F4', 'C4', 'P4', 'F8', 'T4', 'T6', 'O2', 'EKG']\"\n)\n\neeg_colors = {i: j for i, j in zip(eeg_cols, color_palette)}\n\nspect_prefix = [\"RP\", \"LP\", \"LL\", \"RL\"]\nspect_times = eval(\n    \"[0.59, 0.78, 0.98, 1.17, 1.37, 1.56, 1.76, 1.95, 2.15, 2.34, 2.54, 2.73, 2.93, 3.13, 3.32, 3.52, 3.71, 3.91, 4.1, 4.3, 4.49, 4.69, 4.88, 5.08, 5.27, 5.47, 5.66, 5.86, 6.05, 6.25, 6.45, 6.64, 6.84, 7.03, 7.23, 7.42, 7.62, 7.81, 8.01, 8.2, 8.4, 8.59, 8.79, 8.98, 9.18, 9.38, 9.57, 9.77, 9.96, 10.16, 10.35, 10.55, 10.74, 10.94, 11.13, 11.33, 11.52, 11.72, 11.91, 12.11, 12.3, 12.5, 12.7, 12.89, 13.09, 13.28, 13.48, 13.67, 13.87, 14.06, 14.26, 14.45, 14.65, 14.84, 15.04, 15.23, 15.43, 15.63, 15.82, 16.02, 16.21, 16.41, 16.6, 16.8, 16.99, 17.19, 17.38, 17.58, 17.77, 17.97, 18.16, 18.36, 18.55, 18.75, 18.95, 19.14, 19.34, 19.53, 19.73, 19.92]\"\n)\n\nspect_ll_cols = [\"LL_\" + str(i) for i in spect_times]\nspect_lp_cols = [\"LP_\" + str(i) for i in spect_times]\nspect_rl_cols = [\"RL_\" + str(i) for i in spect_times]\nspect_rp_cols = [\"RP_\" + str(i) for i in spect_times]\n\nspect_ll_colors = {i: j for i, j in zip(spect_ll_cols, color_palette)}\nspect_lp_colors = {i: j for i, j in zip(spect_lp_cols, color_palette)}\nspect_rl_colors = {i: j for i, j in zip(spect_rl_cols, color_palette)}\nspect_rp_colors = {i: j for i, j in zip(spect_rp_cols, color_palette)}\n\n\nclasses = {\"Seizure\": 0, \"GPD\": 1, \"LRDA\": 2, \"Other\": 3, \"GRDA\": 4, \"LPD\": 5}\nclasses_labels = list(classes.keys())\nclasses_ids = list(classes.values())\nclasses_num = len(classes_labels)\n\nprint(\"Loading data...\")\nprint(\n    f\"classes ids: {classes_ids}  \\nnames: {classes_labels}\",\n)\n\ntrain_columns = [\n    # \"eeg_id\",\n    # \"eeg_sub_id\",\n    # \"eeg_label_offset_seconds\",\n    # \"spectrogram_id\",\n    # \"spectrogram_sub_id\",\n    # \"spectrogram_label_offset_seconds\",\n    # \"label_id\",\n    # \"patient_id\",\n    # \"expert_consensus\",\n    # \"seizure_vote\",\n    # \"lpd_vote\",\n    # \"gpd_vote\",\n    # \"lrda_vote\",\n    # \"grda_vote\",\n    # \"other_vote\",\n    # \"expert_consensus_classes\",\n]\n\neefg_columns_with_index = [f\"eeg_{x}_{i}\" for x in eeg_cols for i in range(10_000)]\n\nX_columns = eefg_columns_with_index[:100]\ny_columns = \"expert_consensus_classes\"","metadata":{"id":"yJ-M8t9m0Dde","outputId":"80273b41-7cb9-4d41-ce8f-455c05805749","execution":{"iopub.status.busy":"2024-02-28T13:51:45.142736Z","iopub.execute_input":"2024-02-28T13:51:45.142959Z","iopub.status.idle":"2024-02-28T13:51:45.200211Z","shell.execute_reply.started":"2024-02-28T13:51:45.142941Z","shell.execute_reply":"2024-02-28T13:51:45.199521Z"},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📹 Load train\nMany thanks to Chris Deotte for [Understanding Competition Data](\nhttps://www.kaggle.com/competitions/hms-harmful-brain-activity-classification/discussion/468010)","metadata":{"id":"yir2EH3t0Ddf"}},{"cell_type":"code","source":"def get_one_sample(row):\n    eeg = pd.read_parquet(f\"{root_eeg}/{row.eeg_id}.parquet\")\n    eeg_offset = int(row.eeg_label_offset_seconds)\n    eeg = eeg.iloc[eeg_offset * 200 : (eeg_offset + 50) * 200]\n\n    spectrogram = pd.read_parquet(f\"{root_spect}/{row.spectrogram_id}.parquet\")\n    spec_offset = int(row.spectrogram_label_offset_seconds)\n    spectrogram = spectrogram.loc[\n        (spectrogram.time >= spec_offset) & (spectrogram.time < spec_offset + 600)\n    ]\n\n    return eeg, spectrogram","metadata":{"id":"BIuFqQX90Ddh","execution":{"iopub.status.busy":"2024-02-28T13:51:45.202053Z","iopub.execute_input":"2024-02-28T13:51:45.202531Z","iopub.status.idle":"2024-02-28T13:51:45.207669Z","shell.execute_reply.started":"2024-02-28T13:51:45.202502Z","shell.execute_reply":"2024-02-28T13:51:45.206836Z"},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(os.path.join(root, \"train.csv\"))\ntrain[\"expert_consensus_classes\"] = train[\"expert_consensus\"].map(classes)\n\nn_samples = train.shape[0] if N_SAMPLES == -1 else N_SAMPLES\n\npprint(list(train.columns))\nprint(f\"\\nTrain dtypes:  {train.dtypes.to_dict()}\")\nprint(f\"\\nTrain shape: {train.shape}\\n\")\ntrain.describe()","metadata":{"execution":{"iopub.status.busy":"2024-02-28T13:51:45.209109Z","iopub.execute_input":"2024-02-28T13:51:45.209697Z","iopub.status.idle":"2024-02-28T13:51:45.386484Z","shell.execute_reply.started":"2024-02-28T13:51:45.209670Z","shell.execute_reply":"2024-02-28T13:51:45.385499Z"},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📷 Analysis","metadata":{"id":"5hSBd5lMUOOy"}},{"cell_type":"markdown","source":"## on Targets","metadata":{"id":"yf7QbuFuUUWh"}},{"cell_type":"code","source":"# Calculate value counts\nvalue_counts = (\n    train[\"expert_consensus\"] + \" \" + train[\"expert_consensus_classes\"].astype(str)\n).value_counts()\n\n# Plot the count histogram\nax = value_counts.plot(kind=\"bar\", color=\"skyblue\")\n\n# Annotate each bar with its count value\nfor i, count in enumerate(value_counts):\n    ax.text(i, count + 0.1, str(count), ha=\"center\", va=\"bottom\")","metadata":{"id":"yy7Xdpi40Ddg","outputId":"ca60f456-40a9-41c2-c10d-b710ddfff8b8","_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-28T13:51:45.387804Z","iopub.execute_input":"2024-02-28T13:51:45.388129Z","iopub.status.idle":"2024-02-28T13:51:45.610467Z","shell.execute_reply.started":"2024-02-28T13:51:45.388100Z","shell.execute_reply":"2024-02-28T13:51:45.609705Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## on Files","metadata":{"id":"pXiUO6uLUZH3"}},{"cell_type":"code","source":"train.head()","metadata":{"id":"itfUtvLyUfVg","outputId":"de6074d9-4afb-40e0-ed89-95eccc965eab","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-28T13:51:45.611524Z","iopub.execute_input":"2024-02-28T13:51:45.611800Z","iopub.status.idle":"2024-02-28T13:51:45.625799Z","shell.execute_reply.started":"2024-02-28T13:51:45.611781Z","shell.execute_reply":"2024-02-28T13:51:45.624784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eeg_files = (\n    train.groupby(\"eeg_id\")\n    .size()\n    .reset_index(name=\"count\")\n    .sort_values(by=\"count\", ascending=False)\n)\nprint(\n    f\"eeg files: {len(eeg_files)} for a total samples: {len(train)} About: {len(train)/len(eeg_files):.2f} samples per file.\"\n)\n\nspectr_files = (\n    train.groupby(\"spectrogram_id\")\n    .size()\n    .reset_index(name=\"count\")\n    .sort_values(by=\"count\", ascending=False)\n)\nprint(\n    f\"eeg files: {len(spectr_files)} for a total samples: {len(train)}. About: {len(train)/len(spectr_files):.2f} samples per file.\"\n)","metadata":{"id":"IYhDT13OUawe","outputId":"64a2bb26-6641-425e-eada-9cf7b487b90d","_kg_hide-input":true,"_kg_hide-output":false,"execution":{"iopub.status.busy":"2024-02-28T13:51:45.626918Z","iopub.execute_input":"2024-02-28T13:51:45.627116Z","iopub.status.idle":"2024-02-28T13:51:45.649771Z","shell.execute_reply.started":"2024-02-28T13:51:45.627098Z","shell.execute_reply":"2024-02-28T13:51:45.648899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## on EEG and EKG","metadata":{}},{"cell_type":"code","source":"def plot_eegs(x, y, title):\n    # plot EEG\n    plt.figure(figsize=(20, 12))\n    idx = 0\n    for i in y:\n        plt.plot(x[i] + idx * 50, color=eeg_colors[i], label=i)\n        idx += 3\n    # Add title, labels, legend\n    plt.xlabel(\"Time (s)\")\n    plt.ylabel(\"Frequency (Hz)\")\n    plt.yticks([])\n    plt.legend(loc=\"upper right\")\n    plt.title(f\"{title} {row.eeg_id}\")\n    plt.show()\n    return plt","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-28T13:51:45.651033Z","iopub.execute_input":"2024-02-28T13:51:45.651733Z","iopub.status.idle":"2024-02-28T13:51:45.657851Z","shell.execute_reply.started":"2024-02-28T13:51:45.651696Z","shell.execute_reply":"2024-02-28T13:51:45.656983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot spectrograms\ndef plot_spectrum(spectrogram, cols_family, colors_family):\n    plt.figure(figsize=(20, 10))\n    idx = 0\n    for i in cols_family:\n        plt.plot(\n            spectrogram[\"time\"],\n            spectrogram[i] + idx * 3,\n            label=i,\n            color=colors_family[i],\n        )\n        idx += 3\n    # Add title, labels, legend\n    plt.xlabel(\"Time (s)\")\n    plt.ylabel(\"Frequency (Hz)\")\n    plt.legend(loc=\"upper right\")\n    plt.yticks([])\n    plt.title(f\"Spectrogram {cols_family[0]} - {cols_family[-1]}\")\n    plt.show()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-28T13:51:45.660857Z","iopub.execute_input":"2024-02-28T13:51:45.661076Z","iopub.status.idle":"2024-02-28T13:51:45.667707Z","shell.execute_reply.started":"2024-02-28T13:51:45.661058Z","shell.execute_reply":"2024-02-28T13:51:45.666800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = train.iloc[0]\neeg, spectrogram = get_one_sample(row)\n\nprint(f\"Testing one sample ...\")\nprint(f\"eeg shape: {eeg.shape} spectrogram shape: {spectrogram.shape}\")\nprint(\n    f\"eeg columns ({len(eeg.columns)}): {list( eeg.columns[:3])}... \\nspectrogram ({len(spectrogram.columns[:3])}...): {list(spectrogram.columns[:3])}...\"\n)\n# print(f\"eeg head: {eeg.head()} spectrogram head: {spectrogram.head()}\")\nprint(f\"eeg time: {eeg.index.min()} - {eeg.index.max()}\")\nprint(f\"spectrogram time: {spectrogram.time.min()} - {spectrogram.time.max()}\")\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-28T13:51:45.668884Z","iopub.execute_input":"2024-02-28T13:51:45.669136Z","iopub.status.idle":"2024-02-28T13:51:45.713109Z","shell.execute_reply.started":"2024-02-28T13:51:45.669115Z","shell.execute_reply":"2024-02-28T13:51:45.712225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_eegs(eeg, set(eeg_cols) - {\"EKG\"}, \"EEG\")\nplot_eegs(eeg, {\"EKG\"}, \"EKG\")","metadata":{"execution":{"iopub.status.busy":"2024-02-28T13:51:45.714484Z","iopub.execute_input":"2024-02-28T13:51:45.714800Z","iopub.status.idle":"2024-02-28T13:51:46.538137Z","shell.execute_reply.started":"2024-02-28T13:51:45.714772Z","shell.execute_reply":"2024-02-28T13:51:46.537214Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## on Spectrogram lines","metadata":{}},{"cell_type":"code","source":"# first 10 frequencies Hz\ntop_freq = 10\nplot_spectrum(spectrogram, spect_ll_cols[:top_freq], spect_ll_colors)\nplot_spectrum(spectrogram, spect_lp_cols[:top_freq], spect_lp_colors)\nplot_spectrum(spectrogram, spect_rl_cols[:top_freq], spect_rl_colors)\nplot_spectrum(spectrogram, spect_rp_cols[:top_freq], spect_rp_colors)\n\n# last 10 frequencies Hz\nlast_freq = 20\nplot_spectrum(spectrogram, spect_ll_cols[-last_freq:], spect_ll_colors)\nplot_spectrum(spectrogram, spect_lp_cols[-last_freq:], spect_lp_colors)\nplot_spectrum(spectrogram, spect_rl_cols[-last_freq:], spect_rl_colors)\nplot_spectrum(spectrogram, spect_rp_cols[-last_freq:], spect_rp_colors)\n\npass","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-28T13:51:46.539774Z","iopub.execute_input":"2024-02-28T13:51:46.540109Z","iopub.status.idle":"2024-02-28T13:51:49.353458Z","shell.execute_reply.started":"2024-02-28T13:51:46.540079Z","shell.execute_reply":"2024-02-28T13:51:49.352652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## on Spectrograms surface ","metadata":{}},{"cell_type":"code","source":"import plotly.graph_objects as go\n\n\ndef plot_3d_sprectogram(spectrogram, cols=spect_ll_cols, title=\"Spectrogram LL\"):\n    fig = go.Figure(\n        go.Surface(\n            contours={\n                \"x\": {\n                    \"show\": True,\n                    \"start\": 1.5,\n                    \"end\": 2,\n                    \"size\": 0.04,\n                    \"color\": \"white\",\n                },\n                \"z\": {\"show\": True, \"start\": 0.5, \"end\": 0.8, \"size\": 0.05},\n            },\n            y=spectrogram.time,\n            x=cols,\n            z=spectrogram[cols].values,\n        )\n    )\n    fig.update_layout(\n        title=title,\n        width=1000,\n        height=800,\n        scene={\n            \"xaxis_title\": \"Frequency (Hz)\",\n            \"yaxis_title\": \"Time (s)\",\n            \"zaxis_title\": \"Intensity\",\n            \"xaxis\": {\"nticks\": 25},\n            \"zaxis\": {\"nticks\": 4},\n            \"camera_eye\": {\"x\": 1, \"y\": -1, \"z\": 0.5},\n            \"aspectratio\": {\"x\": .75, \"y\": 0.75, \"z\": 0.5},\n        },\n    )\n    fig.show()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-28T14:17:09.822846Z","iopub.execute_input":"2024-02-28T14:17:09.823180Z","iopub.status.idle":"2024-02-28T14:17:09.829984Z","shell.execute_reply.started":"2024-02-28T14:17:09.823156Z","shell.execute_reply":"2024-02-28T14:17:09.829228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-28T13:51:49.362769Z","iopub.execute_input":"2024-02-28T13:51:49.362957Z","iopub.status.idle":"2024-02-28T13:51:49.385648Z","shell.execute_reply.started":"2024-02-28T13:51:49.362940Z","shell.execute_reply":"2024-02-28T13:51:49.384710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_3d_sprectogram(spectrogram, spect_ll_cols, \"LL - \" + row.expert_consensus)\nplot_3d_sprectogram(spectrogram, spect_lp_cols, \"LP - \" + row.expert_consensus)\nplot_3d_sprectogram(spectrogram, spect_rl_cols, \"RL - \" + row.expert_consensus)\nplot_3d_sprectogram(spectrogram, spect_rp_cols, \"RP - \" + row.expert_consensus)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-28T14:17:15.207628Z","iopub.execute_input":"2024-02-28T14:17:15.207947Z","iopub.status.idle":"2024-02-28T14:17:15.412964Z","shell.execute_reply.started":"2024-02-28T14:17:15.207924Z","shell.execute_reply":"2024-02-28T14:17:15.412230Z"},"trusted":true},"execution_count":null,"outputs":[]}]}