{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7526248,"sourceType":"datasetVersion","datasetId":4308295},{"sourceId":6127,"sourceType":"modelInstanceVersion","modelInstanceId":4598}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":3846.080383,"end_time":"2024-01-14T04:20:19.064569","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-01-14T03:16:12.984186","version":"2.4.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"08983a9c6aff42578980f4f7113c3ee2":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_4411aefc021d46d0ada7b645eb53ec48","placeholder":"​","style":"IPY_MODEL_09a10a8cf9334c51857397ed50398c8e","value":"Searching best thr : 100%"}},"09a10a8cf9334c51857397ed50398c8e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"1f3989a0c01248328e16875075e9d1c4":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_08983a9c6aff42578980f4f7113c3ee2","IPY_MODEL_22cfcc0a7cc6455fbf3bb7c788c8a4e1","IPY_MODEL_c8392e8075224e3b8a020a16c1a08447"],"layout":"IPY_MODEL_6cec9a2c2fac450d87248aed8dd62f86"}},"22cfcc0a7cc6455fbf3bb7c788c8a4e1":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_dffe80502d954bdea0bbb6353dbf5515","max":20,"min":0,"orientation":"horizontal","style":"IPY_MODEL_7ce1b34a4f864a42a6619eec82311eb0","value":20}},"4411aefc021d46d0ada7b645eb53ec48":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6cec9a2c2fac450d87248aed8dd62f86":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7ce1b34a4f864a42a6619eec82311eb0":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"83fe40a0b8f047cc8602206909d42361":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9384babdb7054d55aecdf3e989ddc926":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"c8392e8075224e3b8a020a16c1a08447":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_83fe40a0b8f047cc8602206909d42361","placeholder":"​","style":"IPY_MODEL_9384babdb7054d55aecdf3e989ddc926","value":" 20/20 [04:34&lt;00:00, 12.66s/it]"}},"dffe80502d954bdea0bbb6353dbf5515":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Harmful Brain Activity Classification: CNN","metadata":{}},{"cell_type":"markdown","source":"# Important Notice\nTo run this code, please access this online Kaggle notebook here:\nhttps://www.kaggle.com/code/haojungc/nick-hms-hbac-kerascv-starter-notebook","metadata":{}},{"cell_type":"markdown","source":"## Install Libraries  ","metadata":{"papermill":{"duration":0.011416,"end_time":"2024-01-14T03:16:16.470167","exception":false,"start_time":"2024-01-14T03:16:16.458751","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install -q /kaggle/input/kerasv3-lib-ds/keras_cv-0.8.2-py3-none-any.whl --no-deps\n!pip install -q /kaggle/input/kerasv3-lib-ds/tensorflow-2.15.0.post1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl --no-deps\n!pip install -q /kaggle/input/kerasv3-lib-ds/keras-3.0.4-py3-none-any.whl --no-deps","metadata":{"execution":{"iopub.status.busy":"2024-05-04T17:16:46.347564Z","iopub.execute_input":"2024-05-04T17:16:46.347908Z","iopub.status.idle":"2024-05-04T17:18:37.439118Z","shell.execute_reply.started":"2024-05-04T17:16:46.347882Z","shell.execute_reply":"2024-05-04T17:18:37.438083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import Libraries ","metadata":{"papermill":{"duration":0.010878,"end_time":"2024-01-14T03:17:49.510159","exception":false,"start_time":"2024-01-14T03:17:49.499281","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nos.environ[\"KERAS_BACKEND\"] = \"jax\"\n\nimport keras_cv\nimport keras\nfrom keras import ops\nfrom keras.layers import Input, Conv2D, MaxPooling2D, Flatten, Dense\nfrom keras.regularizers import l2;\nimport tensorflow as tf\n\nimport cv2\nimport pandas as pd\nimport numpy as np\nfrom glob import glob\nfrom tqdm.notebook import tqdm\nimport joblib\n\nimport matplotlib.pyplot as plt ","metadata":{"papermill":{"duration":10.671979,"end_time":"2024-01-14T03:18:00.193134","exception":false,"start_time":"2024-01-14T03:17:49.521155","status":"completed"},"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-05-04T17:18:37.441863Z","iopub.execute_input":"2024-05-04T17:18:37.442629Z","iopub.status.idle":"2024-05-04T17:18:47.677043Z","shell.execute_reply.started":"2024-05-04T17:18:37.442589Z","shell.execute_reply":"2024-05-04T17:18:47.676066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configuration","metadata":{"papermill":{"duration":0.010922,"end_time":"2024-01-14T03:18:00.26855","exception":false,"start_time":"2024-01-14T03:18:00.257628","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG:\n    verbose = 1  \n    seed = 42 \n    image_size = [400, 300]  # Input image size (will be 400 x 300 x 3)\n    epochs = 100 # Training epochs (we will apply early stopping)\n    batch_size = 64  # Batch size\n    lr_mode = \"cos\" # LR scheduler mode from one of \"cos\", \"step\", \"exp\"\n    drop_remainder = True  # Drop incomplete batches\n    num_classes = 6 # Number of classes in the dataset\n    fold = 0 # Which fold to set as validation data\n    class_names = ['Seizure', 'LPD', 'GPD', 'LRDA','GRDA', 'Other']\n    label2name = dict(enumerate(class_names))\n    name2label = {v:k for k, v in label2name.items()}","metadata":{"papermill":{"duration":0.018795,"end_time":"2024-01-14T03:18:00.298534","exception":false,"start_time":"2024-01-14T03:18:00.279739","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-04T17:18:47.678193Z","iopub.execute_input":"2024-05-04T17:18:47.678726Z","iopub.status.idle":"2024-05-04T17:18:47.685840Z","shell.execute_reply.started":"2024-05-04T17:18:47.678699Z","shell.execute_reply":"2024-05-04T17:18:47.684151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reproducibility ","metadata":{"papermill":{"duration":0.010907,"end_time":"2024-01-14T03:18:00.32063","exception":false,"start_time":"2024-01-14T03:18:00.309723","status":"completed"},"tags":[]}},{"cell_type":"code","source":"keras.utils.set_random_seed(CFG.seed)","metadata":{"papermill":{"duration":0.018371,"end_time":"2024-01-14T03:18:00.350074","exception":false,"start_time":"2024-01-14T03:18:00.331703","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-04T17:18:47.686984Z","iopub.execute_input":"2024-05-04T17:18:47.687245Z","iopub.status.idle":"2024-05-04T17:18:47.697713Z","shell.execute_reply.started":"2024-05-04T17:18:47.687224Z","shell.execute_reply":"2024-05-04T17:18:47.696887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset Path ","metadata":{"papermill":{"duration":0.010888,"end_time":"2024-01-14T03:18:00.372053","exception":false,"start_time":"2024-01-14T03:18:00.361165","status":"completed"},"tags":[]}},{"cell_type":"code","source":"BASE_PATH = \"/kaggle/input/hms-harmful-brain-activity-classification\"\n\nSPEC_DIR = \"/tmp/dataset/hms-hbac\"\nos.makedirs(SPEC_DIR+'/train_spectrograms', exist_ok=True)\nos.makedirs(SPEC_DIR+'/test_spectrograms', exist_ok=True)","metadata":{"papermill":{"duration":0.017704,"end_time":"2024-01-14T03:18:00.400852","exception":false,"start_time":"2024-01-14T03:18:00.383148","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-04T17:18:47.700565Z","iopub.execute_input":"2024-05-04T17:18:47.700991Z","iopub.status.idle":"2024-05-04T17:18:47.709307Z","shell.execute_reply.started":"2024-05-04T17:18:47.700952Z","shell.execute_reply":"2024-05-04T17:18:47.708455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Meta Data ","metadata":{"papermill":{"duration":0.011434,"end_time":"2024-01-14T03:18:00.472401","exception":false,"start_time":"2024-01-14T03:18:00.460967","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Train + Valid + Test\ndf = pd.read_csv(f'{BASE_PATH}/train.csv')\ndf['eeg_path'] = f'{BASE_PATH}/train_eegs/'+df['eeg_id'].astype(str)+'.parquet'\ndf['spec_path'] = f'{BASE_PATH}/train_spectrograms/'+df['spectrogram_id'].astype(str)+'.parquet'\ndf['spec2_path'] = f'{SPEC_DIR}/train_spectrograms/'+df['spectrogram_id'].astype(str)+'.npy'\ndf['class_name'] = df.expert_consensus.copy()\ndf['class_label'] = df.expert_consensus.map(CFG.name2label)\ndisplay(df.head(2))","metadata":{"execution":{"iopub.status.busy":"2024-05-04T17:18:47.710396Z","iopub.execute_input":"2024-05-04T17:18:47.710746Z","iopub.status.idle":"2024-05-04T17:18:48.255189Z","shell.execute_reply.started":"2024-05-04T17:18:47.710714Z","shell.execute_reply":"2024-05-04T17:18:48.254270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Convert `.parquet` to `.npy`","metadata":{}},{"cell_type":"code","source":"def process_spec(spec_id, split=\"train\"):\n    spec_path = f\"{BASE_PATH}/{split}_spectrograms/{spec_id}.parquet\"\n    spec = pd.read_parquet(spec_path)\n    spec = spec.fillna(0).values[:, 1:].T\n    spec = spec.astype(\"float32\")\n    np.save(f\"{SPEC_DIR}/{split}_spectrograms/{spec_id}.npy\", spec)\n\n# Get unique spec_ids of train and valid data\nspec_ids = df[\"spectrogram_id\"].unique()\n\n# Parallelize the processing using joblib for training data\n_ = joblib.Parallel(n_jobs=-1, backend=\"loky\")(\n    joblib.delayed(process_spec)(spec_id, \"train\")\n    for spec_id in tqdm(spec_ids, total=len(spec_ids))\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-04T17:18:48.256329Z","iopub.execute_input":"2024-05-04T17:18:48.256598Z","iopub.status.idle":"2024-05-04T17:22:00.710225Z","shell.execute_reply.started":"2024-05-04T17:18:48.256575Z","shell.execute_reply":"2024-05-04T17:22:00.708862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DataLoader","metadata":{"papermill":{"duration":0.011843,"end_time":"2024-01-14T03:18:01.457956","exception":false,"start_time":"2024-01-14T03:18:01.446113","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def build_decoder(with_labels=True, target_size=CFG.image_size, dtype=32):\n    def decode_signal(path, offset=None):\n        # Read .npy files and process the signal\n        file_bytes = tf.io.read_file(path)\n        sig = tf.io.decode_raw(file_bytes, tf.float32)\n        sig = sig[1024//dtype:]  # Remove header tag\n        sig = tf.reshape(sig, [400, -1])\n        \n        # Extract labeled subsample from full spectrogram using \"offset\"\n        if offset is not None: \n            offset = offset // 2  # Only odd values are given\n            sig = sig[:, offset:offset+300]\n            \n            # Pad spectrogram to ensure the same input shape of [400, 300]\n            pad_size = tf.math.maximum(0, 300 - tf.shape(sig)[1])\n            sig = tf.pad(sig, [[0, 0], [0, pad_size]])\n            sig = tf.reshape(sig, [400, 300])\n        \n        # Log spectrogram \n        sig = tf.clip_by_value(sig, tf.math.exp(-4.0), tf.math.exp(8.0)) # avoid 0 in log\n        sig = tf.math.log(sig)\n        \n        # Normalize spectrogram\n        sig -= tf.math.reduce_mean(sig)\n        sig /= tf.math.reduce_std(sig) + 1e-6\n        \n        # Mono channel to 3 channels to use \"ImageNet\" weights\n        sig = tf.tile(sig[..., None], [1, 1, 3])\n        return sig\n    \n    def decode_label(label):\n        label = tf.one_hot(label, CFG.num_classes)\n        label = tf.cast(label, tf.float32)\n        label = tf.reshape(label, [CFG.num_classes])\n        return label\n    \n    def decode_with_labels(path, offset=None, label=None):\n        sig = decode_signal(path, offset)\n        label = decode_label(label)\n        return (sig, label)\n    \n    return decode_with_labels if with_labels else decode_signal\n\n\ndef build_dataset(paths, offsets=None, labels=None, batch_size=32, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=False, repeat=True, shuffle=1024, \n                  cache_dir=\"\", drop_remainder=False):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n   \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = (paths, offsets) if labels is None else (paths, offsets, labels)\n    \n    ds = tf.data.Dataset.from_tensor_slices(slices)\n    ds = ds.map(decode_fn, num_parallel_calls=AUTO)\n    ds = ds.cache(cache_dir) if cache else ds\n    ds = ds.repeat() if repeat else ds\n    if shuffle: \n        ds = ds.shuffle(shuffle, seed=CFG.seed)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n    ds = ds.batch(batch_size, drop_remainder=drop_remainder)\n    ds = ds.map(augment_fn, num_parallel_calls=AUTO) if augment else ds\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"papermill":{"duration":0.039133,"end_time":"2024-01-14T03:18:01.509017","exception":false,"start_time":"2024-01-14T03:18:01.469884","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-04T17:22:00.712042Z","iopub.execute_input":"2024-05-04T17:22:00.712450Z","iopub.status.idle":"2024-05-04T17:22:00.737149Z","shell.execute_reply.started":"2024-05-04T17:22:00.712402Z","shell.execute_reply":"2024-05-04T17:22:00.736161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Split","metadata":{"papermill":{"duration":0.012174,"end_time":"2024-01-14T03:18:01.538524","exception":false,"start_time":"2024-01-14T03:18:01.52635","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedGroupKFold\n\nsgkf = StratifiedGroupKFold(n_splits=5, shuffle=True, random_state=CFG.seed)\n\ndf[\"fold\"] = -1\ndf.reset_index(drop=True, inplace=True)\nfor fold, (train_idx, valid_idx) in enumerate(\n    sgkf.split(df, y=df[\"class_label\"], groups=df[\"patient_id\"])\n):\n    df.loc[valid_idx, \"fold\"] = fold\ndf.groupby([\"fold\", \"class_name\"])[[\"eeg_id\"]].count().T","metadata":{"papermill":{"duration":0.037496,"end_time":"2024-01-14T03:18:01.587924","exception":false,"start_time":"2024-01-14T03:18:01.550428","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-04T17:22:00.738493Z","iopub.execute_input":"2024-05-04T17:22:00.738875Z","iopub.status.idle":"2024-05-04T17:22:02.818661Z","shell.execute_reply.started":"2024-05-04T17:22:00.738838Z","shell.execute_reply":"2024-05-04T17:22:02.817692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Train & Valid Dataset","metadata":{"papermill":{"duration":0.011875,"end_time":"2024-01-14T03:18:01.611955","exception":false,"start_time":"2024-01-14T03:18:01.60008","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_df_2 = df.groupby(\"spectrogram_id\").head(1).reset_index(drop=True)\nsample_df = df.groupby(\"spectrogram_id\").head(30).reset_index(drop=True) \nsample_df = sample_df.drop(test_df_2.index)\n\ntrain_df = sample_df[sample_df.fold != CFG.fold]\nvalid_df = sample_df[sample_df.fold == CFG.fold]\nprint(f\"# Num Train: {len(train_df)} | Num Valid: {len(valid_df)}\")\n\n# Train\ntrain_paths = train_df.spec2_path.values\ntrain_offsets = train_df.spectrogram_label_offset_seconds.values.astype(int)\ntrain_labels = train_df.class_label.values\ntrain_ds = build_dataset(train_paths, train_offsets, train_labels, batch_size=CFG.batch_size,\n                         repeat=True, shuffle=True, cache=False) #cache=True)\n\n# Valid\nvalid_paths = valid_df.spec2_path.values\nvalid_offsets = valid_df.spectrogram_label_offset_seconds.values.astype(int)\nvalid_labels = valid_df.class_label.values\nvalid_ds = build_dataset(valid_paths, valid_offsets, valid_labels, batch_size=CFG.batch_size,\n                         repeat=False, shuffle=False, cache=False) #cache=True)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-04T17:22:02.820082Z","iopub.execute_input":"2024-05-04T17:22:02.820766Z","iopub.status.idle":"2024-05-04T17:22:05.185620Z","shell.execute_reply.started":"2024-05-04T17:22:02.820729Z","shell.execute_reply":"2024-05-04T17:22:05.184857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset Check","metadata":{}},{"cell_type":"code","source":"imgs, tars = next(iter(train_ds))\n\nnum_imgs = 8\nplt.figure(figsize=(4*4, num_imgs//4*5))\nfor i in range(num_imgs):\n    plt.subplot(num_imgs//4, 4, i + 1)\n    img = imgs[i].numpy()[...,0]  # Adjust as per your image data format\n    img -= img.min()\n    img /= img.max() + 1e-4\n    tar = CFG.label2name[np.argmax(tars[i].numpy())]\n    plt.imshow(img)\n    plt.title(f\"Target: {tar}\")\n    plt.axis('off')\n    \nplt.tight_layout()\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-04T17:22:05.186697Z","iopub.execute_input":"2024-05-04T17:22:05.186945Z","iopub.status.idle":"2024-05-04T17:22:07.326291Z","shell.execute_reply.started":"2024-05-04T17:22:05.186923Z","shell.execute_reply":"2024-05-04T17:22:07.325085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loss & Metric","metadata":{}},{"cell_type":"code","source":"LOSS = keras.losses.KLDivergence()","metadata":{"execution":{"iopub.status.busy":"2024-05-04T17:22:07.328483Z","iopub.execute_input":"2024-05-04T17:22:07.329203Z","iopub.status.idle":"2024-05-04T17:22:07.333746Z","shell.execute_reply.started":"2024-05-04T17:22:07.329161Z","shell.execute_reply":"2024-05-04T17:22:07.332801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeling","metadata":{"papermill":{"duration":0.016849,"end_time":"2024-01-14T03:18:38.613991","exception":false,"start_time":"2024-01-14T03:18:38.597142","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\n# Define the model\ninput_shape=(400, 300, 3)\n# weight_decay = 0.01\nweight_decay = 0.05\nbatch_size = 128\n\nmodel = keras.models.Sequential([\n    Input(shape=input_shape),\n    Conv2D(32, kernel_size=(3, 3), activation='relu', kernel_regularizer=l2(weight_decay)),\n    MaxPooling2D(pool_size=(2, 2)),\n    Conv2D(64, kernel_size=(3, 3), activation='relu', kernel_regularizer=l2(weight_decay)),\n    MaxPooling2D(pool_size=(2, 2)),\n    Flatten(),\n    Dense(128, activation='relu', kernel_regularizer=l2(weight_decay)),\n    Dense(CFG.num_classes, activation='softmax')\n])\n\n# Compile the model\nmodel.compile(optimizer=keras.optimizers.Adam(learning_rate=1e-4),\n              loss=LOSS)\n\n# Model Sumamry\nmodel.summary()\n","metadata":{"papermill":{"duration":10.446166,"end_time":"2024-01-14T03:18:49.186176","exception":false,"start_time":"2024-01-14T03:18:38.74001","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-04T18:25:23.019865Z","iopub.execute_input":"2024-05-04T18:25:23.020228Z","iopub.status.idle":"2024-05-04T18:25:23.063283Z","shell.execute_reply.started":"2024-05-04T18:25:23.020200Z","shell.execute_reply":"2024-05-04T18:25:23.062470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from keras.models import Sequential\n# from keras.layers import Input, Conv2D, MaxPooling2D, Flatten, Dense, Dropout\n# from keras.regularizers import l2\n\n# # Define the model\n# input_shape=(400, 300, 3)\n# weight_decay = 0.01\n# batch_size = 128\n\n# model = Sequential([\n#     Input(shape=input_shape),\n#     Conv2D(32, kernel_size=(3, 3), activation='relu', kernel_regularizer=l2(weight_decay)),\n#     MaxPooling2D(pool_size=(2, 2)),\n#     Conv2D(64, kernel_size=(3, 3), activation='relu', kernel_regularizer=l2(weight_decay)),\n#     MaxPooling2D(pool_size=(2, 2)),\n#     Conv2D(128, kernel_size=(3, 3), activation='relu', kernel_regularizer=l2(weight_decay)),\n#     MaxPooling2D(pool_size=(2, 2)),\n#     Flatten(),\n#     Dense(128, activation='relu', kernel_regularizer=l2(weight_decay)),\n#     Dropout(0.5),  # Adding Dropout for regularization\n#     Dense(CFG.num_classes, activation='softmax')\n# ])\n\n# # Compile the model\n# model.compile(optimizer=keras.optimizers.Adam(learning_rate=1e-4),\n#               loss=LOSS)\n\n# # Model Sumamry\n# model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-05-04T18:25:23.064962Z","iopub.execute_input":"2024-05-04T18:25:23.065232Z","iopub.status.idle":"2024-05-04T18:25:23.069865Z","shell.execute_reply.started":"2024-05-04T18:25:23.065208Z","shell.execute_reply":"2024-05-04T18:25:23.069038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LR Schedule","metadata":{"papermill":{"duration":0.016209,"end_time":"2024-01-14T03:18:49.21924","exception":false,"start_time":"2024-01-14T03:18:49.203031","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import math\n\ndef get_lr_callback(batch_size=8, mode='cos', epochs=10, plot=False):\n    lr_start, lr_max, lr_min = 5e-5, 6e-6 * batch_size, 1e-5\n    lr_ramp_ep, lr_sus_ep, lr_decay = 3, 0, 0.75\n\n    def lrfn(epoch):  # Learning rate update function\n        if epoch < lr_ramp_ep: lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n        elif epoch < lr_ramp_ep + lr_sus_ep: lr = lr_max\n        elif mode == 'exp': lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n        elif mode == 'step': lr = lr_max * lr_decay**((epoch - lr_ramp_ep - lr_sus_ep) // 2)\n        elif mode == 'cos':\n            decay_total_epochs, decay_epoch_index = epochs - lr_ramp_ep - lr_sus_ep + 3, epoch - lr_ramp_ep - lr_sus_ep\n            phase = math.pi * decay_epoch_index / decay_total_epochs\n            lr = (lr_max - lr_min) * 0.5 * (1 + math.cos(phase)) + lr_min\n        return lr\n\n    if plot:  # Plot lr curve if plot is True\n        plt.figure(figsize=(10, 5))\n        plt.plot(np.arange(epochs), [lrfn(epoch) for epoch in np.arange(epochs)], marker='o')\n        plt.xlabel('epoch'); plt.ylabel('lr')\n        plt.title('LR Scheduler')\n        plt.show()\n\n    return keras.callbacks.LearningRateScheduler(lrfn, verbose=False)  # Create lr callback","metadata":{"papermill":{"duration":0.028945,"end_time":"2024-01-14T03:18:49.264535","exception":false,"start_time":"2024-01-14T03:18:49.23559","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-04T18:25:23.070968Z","iopub.execute_input":"2024-05-04T18:25:23.071226Z","iopub.status.idle":"2024-05-04T18:25:23.086189Z","shell.execute_reply.started":"2024-05-04T18:25:23.071204Z","shell.execute_reply":"2024-05-04T18:25:23.085270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr_cb = get_lr_callback(batch_size, mode=CFG.lr_mode, plot=True)","metadata":{"papermill":{"duration":0.297147,"end_time":"2024-01-14T03:18:49.578089","exception":false,"start_time":"2024-01-14T03:18:49.280942","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-04T18:25:23.087276Z","iopub.execute_input":"2024-05-04T18:25:23.087826Z","iopub.status.idle":"2024-05-04T18:25:23.369209Z","shell.execute_reply.started":"2024-05-04T18:25:23.087795Z","shell.execute_reply":"2024-05-04T18:25:23.368225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Checkpointing","metadata":{"papermill":{"duration":0.017199,"end_time":"2024-01-14T03:18:49.613648","exception":false,"start_time":"2024-01-14T03:18:49.596449","status":"completed"},"tags":[]}},{"cell_type":"code","source":"weight_decay_str = str(weight_decay).replace(\".\", \"\")\nbest_model_name = f\"best_model_batch_size_128_l2_{weight_decay_str}.keras\"\nckpt_cb = keras.callbacks.ModelCheckpoint(best_model_name,\n                                         monitor='val_loss',\n                                         save_best_only=True,\n                                         save_weights_only=False,\n                                         mode='min')","metadata":{"papermill":{"duration":0.024529,"end_time":"2024-01-14T03:18:49.655708","exception":false,"start_time":"2024-01-14T03:18:49.631179","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-04T18:25:23.371460Z","iopub.execute_input":"2024-05-04T18:25:23.371738Z","iopub.status.idle":"2024-05-04T18:25:23.376867Z","shell.execute_reply.started":"2024-05-04T18:25:23.371714Z","shell.execute_reply":"2024-05-04T18:25:23.375948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Early Stopping","metadata":{}},{"cell_type":"code","source":"es_cb = keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-04T18:25:23.378011Z","iopub.execute_input":"2024-05-04T18:25:23.378268Z","iopub.status.idle":"2024-05-04T18:25:23.387650Z","shell.execute_reply.started":"2024-05-04T18:25:23.378246Z","shell.execute_reply":"2024-05-04T18:25:23.386717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"papermill":{"duration":0.01671,"end_time":"2024-01-14T03:18:49.689354","exception":false,"start_time":"2024-01-14T03:18:49.672644","status":"completed"},"tags":[]}},{"cell_type":"code","source":"history = model.fit(\n    train_ds, \n    epochs=CFG.epochs,\n    callbacks=[lr_cb, ckpt_cb, es_cb], \n    steps_per_epoch=len(train_df)//batch_size,\n    validation_data=valid_ds, \n    verbose=CFG.verbose\n)","metadata":{"papermill":{"duration":3374.692199,"end_time":"2024-01-14T04:15:04.398389","exception":false,"start_time":"2024-01-14T03:18:49.70619","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-04T18:25:23.388952Z","iopub.execute_input":"2024-05-04T18:25:23.389274Z","iopub.status.idle":"2024-05-04T18:41:49.077567Z","shell.execute_reply.started":"2024-05-04T18:25:23.389243Z","shell.execute_reply":"2024-05-04T18:41:49.076161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display Training History","metadata":{}},{"cell_type":"code","source":"# Save training history to a CSV file\n# history_df = pd.DataFrame(history.history)\n# history_df.to_csv(f\"loss_batch_size_{batch_size}_l2_{weight_decay_str}.csv\", index=False)\n\nhistory_df = pd.read_csv(\"loss_batch_size_128_l2_0.01.csv\")\nweight_decay_str = \"0.01\"\n\n# Plot training history\nplt.plot(history_df['loss'], label='Training Loss')\nplt.plot(history_df['val_loss'], label=\"Validation Loss\")\n\ntotal_epochs = len(history_df['loss'])\n\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.xticks(np.arange(0, total_epochs))\nplt.legend()\nplt.title(f'Losses (batch size: 128, L2: {weight_decay_str})')\nplt.savefig(f'loss_batch_size_{batch_size}_l2_{weight_decay_str}.png')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-04T18:47:42.443962Z","iopub.execute_input":"2024-05-04T18:47:42.444357Z","iopub.status.idle":"2024-05-04T18:47:42.866072Z","shell.execute_reply.started":"2024-05-04T18:47:42.444324Z","shell.execute_reply":"2024-05-04T18:47:42.865085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Best Model","metadata":{"papermill":{"duration":0.632183,"end_time":"2024-01-14T04:15:06.991143","exception":false,"start_time":"2024-01-14T04:15:06.35896","status":"completed"},"tags":[]}},{"cell_type":"code","source":"model.load_weights(best_model_name)","metadata":{"papermill":{"duration":20.428261,"end_time":"2024-01-14T04:15:28.044401","exception":false,"start_time":"2024-01-14T04:15:07.61614","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-04T18:41:49.428605Z","iopub.execute_input":"2024-05-04T18:41:49.428889Z","iopub.status.idle":"2024-05-04T18:41:55.092715Z","shell.execute_reply.started":"2024-05-04T18:41:49.428865Z","shell.execute_reply":"2024-05-04T18:41:55.091813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test Accuracy","metadata":{}},{"cell_type":"code","source":"# Build testing dataset\ntest_paths_2 = test_df_2.spec2_path.values\ntest_offsets_2 = test_df_2.spectrogram_label_offset_seconds.values.astype(int)\ntest_labels_2 = test_df_2.class_label.values\ntest_ds_2 = build_dataset(test_paths_2, test_offsets_2, test_labels_2, batch_size=CFG.batch_size,\n                         repeat=False, shuffle=False, augment=False, cache=False)  #cache=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-04T18:41:55.094028Z","iopub.execute_input":"2024-05-04T18:41:55.094662Z","iopub.status.idle":"2024-05-04T18:41:55.164878Z","shell.execute_reply.started":"2024-05-04T18:41:55.094626Z","shell.execute_reply":"2024-05-04T18:41:55.164128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict\npreds = model.predict(test_ds_2)","metadata":{"execution":{"iopub.status.busy":"2024-05-04T18:41:55.166083Z","iopub.execute_input":"2024-05-04T18:41:55.166361Z","iopub.status.idle":"2024-05-04T18:42:04.240743Z","shell.execute_reply.started":"2024-05-04T18:41:55.166337Z","shell.execute_reply":"2024-05-04T18:42:04.239850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate accuracy\npredicted_labels = np.argmax(preds, axis=1)\ncorrect_predictions = np.sum(predicted_labels == test_labels_2)\ntotal_samples = len(test_labels_2)\ntest_accuracy = correct_predictions / total_samples\n\nprint(f\"Test Accuracy (L2 {weight_decay}):\", test_accuracy)","metadata":{"execution":{"iopub.status.busy":"2024-05-04T18:42:04.241818Z","iopub.execute_input":"2024-05-04T18:42:04.242102Z","iopub.status.idle":"2024-05-04T18:42:04.248295Z","shell.execute_reply.started":"2024-05-04T18:42:04.242077Z","shell.execute_reply":"2024-05-04T18:42:04.247446Z"},"trusted":true},"execution_count":null,"outputs":[]}]}