{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":155730565,"sourceType":"kernelVersion"}],"dockerImageVersionId":30626,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🧠 Overview\nHere we're going to take an initial look at what data we're working with.","metadata":{}},{"cell_type":"markdown","source":"---\n# ⚙️ Setup\nLet's get our python environment ready to go, then look at what files we've been given.","metadata":{}},{"cell_type":"code","source":"# Imports\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom datetime import datetime, timedelta\nfrom tqdm import tqdm\nfrom itertools import cycle\nimport random\nfrom random import choice\nimport os\nimport sys\nfrom collections import Counter\n\nimport tensorflow as tf\nimport tensorflow_io as tfio\n\nfrom typing import Optional\n\nfrom sklearn.neighbors import NearestNeighbors\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\nfrom scipy.stats import entropy\n\n# %run kullback-leibler-divergence/metric.py","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-11T00:26:33.810019Z","iopub.execute_input":"2024-01-11T00:26:33.811279Z","iopub.status.idle":"2024-01-11T00:26:44.514585Z","shell.execute_reply.started":"2024-01-11T00:26:33.811220Z","shell.execute_reply":"2024-01-11T00:26:44.513355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Settings\nsns.set_style(\"whitegrid\")\ncolor_pal = plt.rcParams[\"axes.prop_cycle\"].by_key()[\"color\"]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-11T00:26:44.516515Z","iopub.execute_input":"2024-01-11T00:26:44.517305Z","iopub.status.idle":"2024-01-11T00:26:44.524434Z","shell.execute_reply.started":"2024-01-11T00:26:44.517267Z","shell.execute_reply":"2024-01-11T00:26:44.522973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# View files\n\ndef list_files_and_folders_with_info(input_dir):\n    try:\n        for comp_dir in os.listdir(input_dir):\n            comp_path = '/'.join([input_dir, comp_dir])\n            print(f\"Competition Directory: {comp_path}\")\n            print(\"Contains:\")\n            with os.scandir(comp_path) as entries:\n                for entry in entries:\n                    if entry.is_file():\n                        print(f\"- (File) {entry.name}, Size: {entry.stat().st_size} bytes\")\n                    elif entry.is_dir():\n                        print(f\"- (Folder) {entry.name}\")\n\n    except FileNotFoundError:\n        print(f\"The specified directory '{directory}' does not exist.\")\n    except PermissionError:\n        print(f\"Permission error accessing directory '{directory}'.\")\n\n# Replace 'path/to/your/directory' with the actual path of the directory you want to inspect\ninput_dir = '/kaggle/input'\n\nlist_files_and_folders_with_info(input_dir)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-11T00:26:44.526245Z","iopub.execute_input":"2024-01-11T00:26:44.526663Z","iopub.status.idle":"2024-01-11T00:26:44.542916Z","shell.execute_reply.started":"2024-01-11T00:26:44.526620Z","shell.execute_reply":"2024-01-11T00:26:44.541520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load Data\ncomp_path = '/kaggle/input/hms-harmful-brain-activity-classification'\ntrain_df = pd.read_csv(f'{comp_path}/train.csv')\ntest_df = pd.read_csv(f'{comp_path}/test.csv')\nss_df = pd.read_csv(f'{comp_path}/sample_submission.csv')\ndf_dict = {\n    \"Training data\": train_df,\n    \"Testing data\": test_df,\n    \"Sample submission\": ss_df,\n}\n\ntrain_spect_dir = '/'.join([comp_path, 'train_spectrograms'])\ntrain_eeg_dir = '/'.join([comp_path, 'train_eegs'])\ntest_eeg_dir = '/'.join([comp_path, 'test_eegs'])\ntest_spect_dir = '/'.join([comp_path, 'test_spectrograms'])\n\ntrain_eeg_path_list = [entry.path for entry in os.scandir(train_eeg_dir)]\ntrain_spect_path_list = [entry.path for entry in os.scandir(train_spect_dir)]\ntest_eeg_path_list = [entry.path for entry in os.scandir(test_eeg_dir)]\ntest_spect_path_list = [entry.path for entry in os.scandir(test_spect_dir)]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-11T00:26:44.545907Z","iopub.execute_input":"2024-01-11T00:26:44.546314Z","iopub.status.idle":"2024-01-11T00:26:45.392660Z","shell.execute_reply.started":"2024-01-11T00:26:44.546276Z","shell.execute_reply":"2024-01-11T00:26:45.391293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Training EEG files: {len(train_eeg_path_list)}\")\nprint(f\"Training Spectrogram files: {len(train_spect_path_list)}\")\nprint(f\"Testing EEG files: {len(test_eeg_path_list)}\")\nprint(f\"Testing Spectrogram files: {len(test_spect_path_list)}\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-11T00:26:45.394408Z","iopub.execute_input":"2024-01-11T00:26:45.394909Z","iopub.status.idle":"2024-01-11T00:26:45.401873Z","shell.execute_reply.started":"2024-01-11T00:26:45.394864Z","shell.execute_reply":"2024-01-11T00:26:45.400648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# 🛠️ Define Tools\n\nDefine some useful functions for exploring this data.","metadata":{}},{"cell_type":"code","source":"# Functions for plotting features\ndef plot_numerical_feature(data):\n    print(f\"Max: {max(data)}, Min: {min(data)}, Mean: {np.mean(data):.2f}, Median: {np.median(data)}\")\n    fig, axes = plt.subplots(1, 2, figsize=(20, 5))\n\n    sns.histplot(data, ax=axes[0], color=color_pal[0])\n\n    sns.boxplot(data, ax=axes[1], color=color_pal[0], orient='h')\n    \n    plt.tight_layout()\n    plt.show()\n    return\n\ndef plot_categorical_feature(data):\n    if len(np.unique(data)) > 50:\n        print(f\"{len(np.unique(data))} categorical features. Examples:\")\n        top_5_items = Counter(data).most_common(5)\n        for item, count in top_5_items:\n            print(f\"{item} - {count} counts.\")\n        return\n    \n    # Column Chart\n    if len(np.unique(data)) > 10:\n        # Vertical plot layout\n        fig, axes = plt.subplots(2, 1, figsize=(20, 10), height_ratios=(1/3, 2/3))\n    else:\n        # Horizontal plot layout\n        fig, axes = plt.subplots(1, 2, figsize=(20, 5), width_ratios=(1/3, 2/3))\n    s = data.value_counts().rename_axis('class').rename('count') # if x- and y-labels are important\n    sns.barplot(x=s.index, y=s.values, order=s.index, ax=axes[0])\n    axes[0].tick_params(axis='x', rotation=90)\n    \n    # Donut chart\n    threshold_percent = 5\n    threshold_count = s.sum() * threshold_percent / 100\n    small_segments = s[s < threshold_count]\n    pie_data = s.copy()\n    if len(small_segments):\n        pie_data['Other'] = small_segments.sum()\n        pie_data.loc[small_segments.index] = np.nan\n    pie_data = pie_data.dropna()\n    \n    axes[1].pie(pie_data, labels=pie_data.index, autopct='%1.1f%%', startangle=90, pctdistance=0.85, wedgeprops=dict(width=0.4))\n    centre_circle = plt.Circle((0, 0), 0.70, fc='white')\n    axes[1].add_artist(centre_circle)\n    \n    plt.tight_layout()\n    plt.show()\n    return\n\ndef plot_feature(data):\n    if str(data[0]).replace(\".\", \"\").isdecimal():\n        plot_numerical_feature(data)\n    elif isinstance(data[0], list):\n        plot_categorical_feature(data.explode().tolist())\n    else:\n        plot_categorical_feature(data)\n    return\n\ndef plot_features(df):\n    for feature in df.columns.values:\n        print('-'*20)\n        print(f\"Feautre: {feature}\")\n        plot_feature(df[feature])\n    return\n\n# Functions for plotting features as a function of a target\ndef plot_numerical_feature_v_numerical_target(data, target):\n    \n    fig, axes = plt.subplots(1, 3, figsize=(20, 5))\n\n    sns.histplot(data, ax=axes[0], color=color_pal[0])\n    axes[0].set_title('Distribution Plot')\n\n    sns.boxplot(data, ax=axes[1], color=color_pal[0], orient='h')\n    axes[1].set_title('Box Plot')\n    \n    sns.scatterplot(x=data, y=target, ax=axes[2], color=color_pal[0])\n    pfit, residuals, rank, singular_values, rcond = np.polyfit(x=data, y=target, deg=3, full=True)\n    xfitted = np.linspace(min(data), max(data), 1000)\n    fitmodel = np.poly1d(pfit)\n    yfitted = fitmodel(xfitted)\n    plt.plot(xfitted, yfitted, color=color_pal[1])\n    r2 = float(1-residuals/(sum(np.square(target))))\n    axes[2].set_title(f'Scatter Plot (R2 = {r2:.3f})')\n    \n    plt.tight_layout()\n    plt.show()\n    return\n    \ndef plot_categorical_feature_v_numerical_target(data, target):\n    if len(np.unique(data)) > 50:\n        print(f\"{len(np.unique(data))} categorical features. Examples:\")\n        top_5_items = Counter(data).most_common(5)\n        for item, count in top_5_items:\n            print(f\"{item} - {count} counts.\")\n        return\n    \n    if len(np.unique(data)) > 10:\n        fig, axes = plt.subplots(2, 1, figsize=(20, 10), height_ratios=(1/3, 2/3))\n    else:\n        fig, axes = plt.subplots(1, 2, figsize=(20, 5), width_ratios=(1/3, 2/3))\n    s = data.value_counts().rename_axis('class').rename('count') # if x- and y-labels are important\n    sns.barplot(x=s.index, y=s.values, order=s.index, ax=axes[0])\n#     sns.countplot(x=data, ax=axes[0])\n    sns.boxplot(x=target, y=data, ax=axes[1], order=s.index, orient='h')\n    plt.tight_layout()\n    plt.show()\n    return\n\ndef plot_feature_v_numerical_target(data, target):\n    if str(data[0]).replace(\".\", \"\").isdecimal():\n        plot_numerical_feature(data, target)\n    else:\n        plot_categorical_feature(data, target)\n    return\n\ndef plot_features_v_numerical_target(df, target_name):\n    for feature in df.columns.values:\n        if feature == target_name:\n            continue\n        print(f\"Feature: {feature}\")\n        plot_feature(df[feature], df[target_name])\n    return\n\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-01-11T00:26:45.403522Z","iopub.execute_input":"2024-01-11T00:26:45.403896Z","iopub.status.idle":"2024-01-11T00:26:45.436616Z","shell.execute_reply.started":"2024-01-11T00:26:45.403863Z","shell.execute_reply":"2024-01-11T00:26:45.435477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Scoring functions taken from https://www.kaggle.com/code/metric/kullback-leibler-divergence","metadata":{}},{"cell_type":"code","source":"def kl_divergence(solution: pd.DataFrame, submission: pd.DataFrame, epsilon: float, micro_average: bool, sample_weights: Optional[pd.Series]):\n    # Overwrite solution for convenience\n    for col in solution.columns:\n        # Prevent issue with populating int columns with floats\n        if not pd.api.types.is_float_dtype(solution[col]):\n            solution[col] = solution[col].astype(float)\n\n        # Clip both the min and max following Kaggle conventions for related metrics like log loss\n        # Clipping the max avoids cases where the loss would be infinite or undefined, clipping the min\n        # prevents users from playing games with the 20th decimal place of predictions.\n        submission[col] = np.clip(submission[col], epsilon, 1 - epsilon)\n\n        y_nonzero_indices = solution[col] != 0\n        solution[col] = solution[col].astype(float)\n        solution.loc[y_nonzero_indices, col] = solution.loc[y_nonzero_indices, col] * np.log(solution.loc[y_nonzero_indices, col] / submission.loc[y_nonzero_indices, col])\n        # Set the loss equal to zero where y_true equals zero following the scipy convention:\n        # https://docs.scipy.org/doc/scipy/reference/generated/scipy.special.rel_entr.html#scipy.special.rel_entr\n        solution.loc[~y_nonzero_indices, col] = 0\n\n    if micro_average:\n        return np.average(solution.sum(axis=1), weights=sample_weights)\n    else:\n        return np.average(solution.mean())\n    \ndef score(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        row_id_column_name: str,\n        epsilon: float=10**-15,\n        micro_average: bool=True,\n        sample_weights_column_name: Optional[str]=None\n    ) -> float:\n    ''' The Kullback–Leibler divergence.\n    The KL divergence is technically undefined/infinite where the target equals zero.\n\n    This implementation always assigns those cases a score of zero; effectively removing them from consideration.\n    The predictions in each row must add to one so any probability assigned to a case where y == 0 reduces\n    another prediction where y > 0, so crucially there is an important indirect effect.\n\n    https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence\n\n    solution: pd.DataFrame\n    submission: pd.DataFrame\n    epsilon: KL divergence is undefined for p=0 or p=1. If epsilon is not null, solution and submission probabilities are clipped to max(eps, min(1 - eps, p).\n    row_id_column_name: str\n    micro_average: bool. Row-wise average if True, column-wise average if False.\n\n    Examples\n    --------\n    >>> import pandas as pd\n    >>> row_id_column_name = \"id\"\n    >>> score(pd.DataFrame({'id': range(4), 'ham': [0, 1, 1, 0], 'spam': [1, 0, 0, 1]}), pd.DataFrame({'id': range(4), 'ham': [.1, .9, .8, .35], 'spam': [.9, .1, .2, .65]}), row_id_column_name=row_id_column_name)\n    0.216161...\n    >>> solution = pd.DataFrame({'id': range(3), 'ham': [0, 0.5, 0.5], 'spam': [0.1, 0.5, 0.5], 'other': [0.9, 0, 0]})\n    >>> submission = pd.DataFrame({'id': range(3), 'ham': [0, 0.5, 0.5], 'spam': [0.1, 0.5, 0.5], 'other': [0.9, 0, 0]})\n    >>> score(solution, submission, 'id')\n    0.0\n    >>> solution = pd.DataFrame({'id': range(3), 'ham': [0, 0.5, 0.5], 'spam': [0.1, 0.5, 0.5], 'other': [0.9, 0, 0]})\n    >>> submission = pd.DataFrame({'id': range(3), 'ham': [0.2, 0.3, 0.5], 'spam': [0.1, 0.5, 0.5], 'other': [0.7, 0.2, 0]})\n    >>> score(solution, submission, 'id')\n    0.160531...\n    '''\n    del solution[row_id_column_name]\n    del submission[row_id_column_name]\n\n    sample_weights = None\n    if sample_weights_column_name:\n        if sample_weights_column_name not in solution.columns:\n            raise ParticipantVisibleError(f'{sample_weights_column_name} not found in solution columns')\n        sample_weights = solution.pop(sample_weights_column_name)\n\n    if sample_weights_column_name and not micro_average:\n        raise ParticipantVisibleError('Sample weights are only valid if `micro_average` is `True`')\n\n    for col in solution.columns:\n        if col not in submission.columns:\n            raise ParticipantVisibleError(f'Missing submission column {col}')\n\n\n    return kl_divergence(solution, submission, epsilon=epsilon, micro_average=micro_average, sample_weights=sample_weights)","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:26:45.443074Z","iopub.execute_input":"2024-01-11T00:26:45.443823Z","iopub.status.idle":"2024-01-11T00:26:45.465058Z","shell.execute_reply.started":"2024-01-11T00:26:45.443770Z","shell.execute_reply":"2024-01-11T00:26:45.463621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# 👀 Quick look at the data\nNow let's take a look at the data that we're working with. What we're looking for is:\n\n* What data do I need to reformat/adapt before I even start looking at it?\n* What does the data mean?\n* Is there any data missing?","metadata":{}},{"cell_type":"code","source":"# Print out an exerpt from each of the dataframes we've been given\nfor df_key in df_dict.keys():\n    print(f\"- {df_key} -\\nShape: {df_dict[df_key].shape} - {df_dict[df_key].shape[0]} Rows x {df_dict[df_key].shape[1]} Columns\\nFeatures: {df_dict[df_key].columns.values}\\n\")\n    print(df_dict[df_key].head(10))\n    print('─'*70)","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:26:45.467070Z","iopub.execute_input":"2024-01-11T00:26:45.468067Z","iopub.status.idle":"2024-01-11T00:26:45.506599Z","shell.execute_reply.started":"2024-01-11T00:26:45.468015Z","shell.execute_reply":"2024-01-11T00:26:45.505647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Feature Engineering","metadata":{}},{"cell_type":"code","source":"\nvote_list = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\ntrain_df['total_vote'] = train_df[vote_list].sum(axis=1)\ntrain_df['patient_samples'] = train_df['patient_id'].map(train_df['patient_id'].value_counts())\n","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:26:45.507765Z","iopub.execute_input":"2024-01-11T00:26:45.508711Z","iopub.status.idle":"2024-01-11T00:26:45.548253Z","shell.execute_reply.started":"2024-01-11T00:26:45.508674Z","shell.execute_reply":"2024-01-11T00:26:45.547040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Exploratory Data Analysis (EDA)","metadata":{}},{"cell_type":"markdown","source":"## Examine Train.csv\n","metadata":{}},{"cell_type":"code","source":"plot_features(train_df)","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:26:45.552030Z","iopub.execute_input":"2024-01-11T00:26:45.552446Z","iopub.status.idle":"2024-01-11T00:27:10.598760Z","shell.execute_reply.started":"2024-01-11T00:26:45.552409Z","shell.execute_reply":"2024-01-11T00:27:10.597585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Unique Patient IDs: {len(train_df['patient_id'].unique())}\")\nprint(f\"Unique Label IDs: {len(train_df['label_id'].unique())}\")\nprint(f\"Unique EEG IDs: {len(train_df['eeg_id'].unique())}\")\nprint(f\"Unique Spectrogram IDs: {len(train_df['spectrogram_id'].unique())}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:27:10.600101Z","iopub.execute_input":"2024-01-11T00:27:10.600437Z","iopub.status.idle":"2024-01-11T00:27:10.618574Z","shell.execute_reply.started":"2024-01-11T00:27:10.600407Z","shell.execute_reply":"2024-01-11T00:27:10.617263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_items = Counter(train_df['expert_consensus']).most_common()\nfor i, (item, count) in enumerate(top_items):\n    print(f\"{i+1:{2}} {item:{10}} - {count} counts.\")","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:27:10.620109Z","iopub.execute_input":"2024-01-11T00:27:10.621157Z","iopub.status.idle":"2024-01-11T00:27:10.643193Z","shell.execute_reply.started":"2024-01-11T00:27:10.621115Z","shell.execute_reply":"2024-01-11T00:27:10.642176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_consensus_count_list = [len(train_df[train_df['patient_id'] == patient_id]['expert_consensus'].unique()) for patient_id in train_df['patient_id'].unique()]\nplt.figure()\nplt.hist(unique_consensus_count_list, bins=np.arange(0.5,7.5,1))\nplt.xlabel(\"Number of different consensus across all samples\")\nplt.ylabel(\"Number of patients\")\nplt.title(\"Diversity in conensus for each patient\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:27:10.644746Z","iopub.execute_input":"2024-01-11T00:27:10.645171Z","iopub.status.idle":"2024-01-11T00:27:12.309246Z","shell.execute_reply.started":"2024-01-11T00:27:10.645128Z","shell.execute_reply":"2024-01-11T00:27:12.308116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Examine EEG Data\nLet's look at some of the EEG Data","metadata":{}},{"cell_type":"code","source":"sample_eeg = pd.read_parquet(train_eeg_path_list[0])\nsample_eeg.info()\nsample_eeg.describe()","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:27:12.310820Z","iopub.execute_input":"2024-01-11T00:27:12.311227Z","iopub.status.idle":"2024-01-11T00:27:12.628486Z","shell.execute_reply.started":"2024-01-11T00:27:12.311192Z","shell.execute_reply":"2024-01-11T00:27:12.627594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=7, ncols=3, figsize=(20, 30))\naxes = axes.flatten()\nfor i, column in enumerate(sample_eeg.columns.tolist()):\n    ax = axes[i]\n    ax.plot(sample_eeg[column])\n    ax.set_title(column)","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:27:12.630033Z","iopub.execute_input":"2024-01-11T00:27:12.630788Z","iopub.status.idle":"2024-01-11T00:27:20.797991Z","shell.execute_reply.started":"2024-01-11T00:27:12.630744Z","shell.execute_reply":"2024-01-11T00:27:20.796261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Examine Spectrogram Data\n","metadata":{}},{"cell_type":"code","source":"sample_spect = pd.read_parquet(train_spect_path_list[0])\nsample_spect.info()\nsample_spect.describe()\nsample_spect","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:27:20.800769Z","iopub.execute_input":"2024-01-11T00:27:20.801494Z","iopub.status.idle":"2024-01-11T00:27:21.841557Z","shell.execute_reply.started":"2024-01-11T00:27:20.801451Z","shell.execute_reply":"2024-01-11T00:27:21.840211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"split_spect = {\n    \"LL\": sample_spect.filter(regex='^LL', axis=1),\n    \"RL\": sample_spect.filter(regex='^RL', axis=1),\n    \"RP\": sample_spect.filter(regex='^RP', axis=1),\n    \"LP\": sample_spect.filter(regex='^LP', axis=1),\n}","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:27:21.843355Z","iopub.execute_input":"2024-01-11T00:27:21.844405Z","iopub.status.idle":"2024-01-11T00:27:21.860599Z","shell.execute_reply.started":"2024-01-11T00:27:21.844349Z","shell.execute_reply":"2024-01-11T00:27:21.858814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_spectrograms(spectrogram_df):\n    \n    split_spect = {\n        \"LL\": spectrogram_df.filter(regex='^LL', axis=1),\n        \"RL\": spectrogram_df.filter(regex='^RL', axis=1),\n        \"RP\": spectrogram_df.filter(regex='^RP', axis=1),\n        \"LP\": spectrogram_df.filter(regex='^LP', axis=1),\n    }\n    \n    fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(15, 12))\n    axes = axes.flatten()\n    label_interval = 5\n    for i, split_name in enumerate(split_spect.keys()):\n        ax = axes[i]\n        img = ax.imshow(np.log(split_spect[split_name]).T, cmap='viridis', aspect='auto', origin='lower')  # You can choose any colormap (cmap) that suits your preferences\n        cbar = fig.colorbar(img, ax=ax)\n        cbar.set_label('Log(Value)')\n        ax.set_title(split_name)\n        ax.set_ylabel(\"Frequency (Hz)\")\n        ax.set_xlabel(\"Time\")\n\n        ax.set_yticks(np.arange(len(split_spect[split_name].columns)))\n        ax.set_yticklabels([column_name[3:] for column_name in split_spect[split_name].columns])\n        frequencies = [column_name[3:] for column_name in split_spect[split_name].columns]\n        ax.set_yticks(np.arange(0, len(split_spect[split_name].columns), label_interval))\n        ax.set_yticklabels(frequencies[::label_interval])\n    plt.tight_layout()\n    plt.show()\n    \nplot_spectrograms(sample_spect)","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:27:21.862449Z","iopub.execute_input":"2024-01-11T00:27:21.862958Z","iopub.status.idle":"2024-01-11T00:27:26.180305Z","shell.execute_reply.started":"2024-01-11T00:27:21.862914Z","shell.execute_reply":"2024-01-11T00:27:26.178881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Analysis\n* Some code borrowed from https://www.kaggle.com/code/datajl/fit-tensorflow-on-parquet-dataset-via-tfio ","metadata":{}},{"cell_type":"code","source":"# output_parquet_features_dataset = tfio.IODataset.from_parquet(train_eeg_path_list[0])\n# output_parquet_features_dataset.element_spec","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:27:26.181646Z","iopub.execute_input":"2024-01-11T00:27:26.182420Z","iopub.status.idle":"2024-01-11T00:27:27.118999Z","shell.execute_reply.started":"2024-01-11T00:27:26.182384Z","shell.execute_reply":"2024-01-11T00:27:27.117806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(type(output_parquet_features_dataset))\n# for data in output_parquet_features_dataset.take(2):\n#     print(data)","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:27:27.120206Z","iopub.execute_input":"2024-01-11T00:27:27.120552Z","iopub.status.idle":"2024-01-11T00:27:27.406095Z","shell.execute_reply.started":"2024-01-11T00:27:27.120510Z","shell.execute_reply":"2024-01-11T00:27:27.404927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eeg_samples_per_second = 200\neeg_seconds_per_subsample = 50\nX = []\ny = []\nfor sample_index in range(5000):\n    eeg_id = train_df['eeg_id'].iloc[sample_index]\n    eeg_sub_id = train_df['eeg_sub_id'].iloc[sample_index]\n    eeg_offset_seconds = train_df['eeg_label_offset_seconds'].iloc[sample_index]\n#     print(f\"EEG id: {eeg_id}, Sub id: {eeg_sub_id}, offset seconds: {eeg_offset_seconds}\")\n    eeg_data = pd.read_parquet('/'.join([train_eeg_dir, str(eeg_id)])+ '.parquet')\n    start_ind = int(eeg_samples_per_second*eeg_offset_seconds)\n    subsample_eeg_data = eeg_data[start_ind:start_ind + eeg_samples_per_second*eeg_seconds_per_subsample]\n    total_votes = train_df[vote_list].iloc[sample_index].sum()\n    y.append([int(votes)/total_votes for votes in train_df[vote_list].iloc[sample_index]])\n    X.append(np.nan_to_num(subsample_eeg_data['EKG'], nan=0))","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:30:08.122429Z","iopub.execute_input":"2024-01-11T00:30:08.123765Z","iopub.status.idle":"2024-01-11T00:31:41.857234Z","shell.execute_reply.started":"2024-01-11T00:30:08.123714Z","shell.execute_reply":"2024-01-11T00:31:41.856022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# K Nearest Neighbour attempt\n","metadata":{}},{"cell_type":"code","source":"# Assuming X contains your timeseries data (vectors) and y contains the labels (categories)\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)\n\n# Initialize NearestNeighbors\nnn_model = NearestNeighbors(n_neighbors=3, metric='euclidean')\nnn_model.fit(X_train)\n\n# Find the nearest neighbors\ndistances, indices = nn_model.kneighbors(X_test)\n# print(distances, indices)\nx_results_list = []\ny_test_list = []\nfor i, index_results in enumerate(indices):\n    total_results = np.zeros(6)\n    for index in index_results:\n        total_results += np.asarray(y_train[index])/3\n    x_results_dict = {\n        'id': i,\n    }\n    y_test_dict = {\n        'id': i,\n    }\n    for ind, vote in enumerate(vote_list):\n        x_results_dict[vote] = total_results[ind]\n        y_test_dict[vote] = y_test[i][ind]\n    x_results_list.append(x_results_dict)\n    y_test_list.append(y_test_dict)\n    \n\nx_results_df = pd.DataFrame(x_results_list)    \ny_test_df = pd.DataFrame(y_test_list)\nscore(y_test_df, x_results_df, 'id')\n# print(\"Classification Report:\")\n# print(classification_report(y_test, y_pred))","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:33:41.647880Z","iopub.execute_input":"2024-01-11T00:33:41.648338Z","iopub.status.idle":"2024-01-11T00:33:43.801950Z","shell.execute_reply.started":"2024-01-11T00:33:41.648304Z","shell.execute_reply":"2024-01-11T00:33:43.800748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# 📮 Final Submission\nFor the moment, let's generate a default solution to submit.\n\n","metadata":{}},{"cell_type":"code","source":"nn_model = NearestNeighbors(n_neighbors=3, metric='euclidean')\nnn_model.fit(X)\nsolution_list = []\n\ntest_eeg_data_list = []\n\nfor i, eeg_id in enumerate(test_df['eeg_id']):\n    eeg_data = pd.read_parquet('/'.join([test_eeg_dir, str(eeg_id)])+ '.parquet')\n    eeg_offset_seconds = 0\n    start_ind = int(eeg_samples_per_second*eeg_offset_seconds)\n    subsample_eeg_data = eeg_data[start_ind:start_ind + eeg_samples_per_second*eeg_seconds_per_subsample]\n    test_eeg_data_list.append(np.nan_to_num(subsample_eeg_data['EKG'], nan=0))\ndistances, indices = nn_model.kneighbors(test_eeg_data_list)\n\nsolution_list = []\nfor i, index_results in enumerate(indices):\n    total_results = np.zeros(6)\n    for index in index_results:\n        total_results += np.asarray(y[index])/3\n    test_results_dict = {\n        'id': i,\n    }\n\n    for ind, vote in enumerate(vote_list):\n        test_results_dict[vote] = total_results[ind]\n    solution_list.append(test_results_dict)\nsolution_df = pd.DataFrame(solution_list)\n\nprint(solution_df)","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:47:10.407556Z","iopub.execute_input":"2024-01-11T00:47:10.407976Z","iopub.status.idle":"2024-01-11T00:47:10.807657Z","shell.execute_reply.started":"2024-01-11T00:47:10.407944Z","shell.execute_reply":"2024-01-11T00:47:10.806316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"solution_list = []\nfor eeg_id in test_df['eeg_id']:\n    solution_dict = {\n        'eeg_id': eeg_id,\n        'seizure_vote': train_df['expert_consensus'].value_counts().get('seizure_vote', 0)/len(train_df),\n        'lpd_vote': train_df['expert_consensus'].value_counts().get('lpd_vote', 0)/len(train_df),\n        'gpd_vote': train_df['expert_consensus'].value_counts().get('gpd_vote', 0)/len(train_df),\n        'lrda_vote': train_df['expert_consensus'].value_counts().get('lrda_vote', 0)/len(train_df),\n        'grda_vote': train_df['expert_consensus'].value_counts().get('grda_vote', 0)/len(train_df),\n    }\n    solution_dict['other_vote'] = 1\n    for key in ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote']:\n        solution_dict['other_vote'] = solution_dict['other_vote'] - solution_dict[key]\n    solution_list.append(solution_dict)\nsolution_df = pd.DataFrame(solution_list)","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:33:31.624606Z","iopub.execute_input":"2024-01-11T00:33:31.625049Z","iopub.status.idle":"2024-01-11T00:33:31.674388Z","shell.execute_reply.started":"2024-01-11T00:33:31.625012Z","shell.execute_reply":"2024-01-11T00:33:31.672954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"solution_df = pd.DataFrame(solution_list)\nsolution_df.to_csv(\"submission.csv\", index=False)\nprint(solution_df)\nprint(ss_df)","metadata":{"execution":{"iopub.status.busy":"2024-01-11T00:27:50.294275Z","iopub.status.idle":"2024-01-11T00:27:50.295083Z","shell.execute_reply.started":"2024-01-11T00:27:50.294863Z","shell.execute_reply":"2024-01-11T00:27:50.294885Z"},"trusted":true},"execution_count":null,"outputs":[]}]}