{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h1 style='background:#1EA1A1; border:0; color:white'><center> 📝 Abstract <center></h1>\n\n * In this Training Notebook EfficientNetB1 is Trained.\n *[Inference Notebook](https://www.kaggle.com/bharadwajvedula/lb-0-8716-tpu-tf-efficientnetb1-inference)\n\n**Versions**    \n* Version 2: EfficientNetB1 512 Augs CV:- **0.8684**  LB:- **0.8716**\n\n\n\n\n\n**References are mentioned at the end.**\n","metadata":{"id":"HzwoNf6Bx0ub"}},{"cell_type":"markdown","source":"<a id=\"0\"></a>\n<h1 style='background:#1EA1A1; border:0; color:white'><center> 📄 Table of Contents<center></h1>\n\n* [Imports](#1)\n* [Configs](#2)\n* [Reading TFRecords](#3) \n* [Dataset Creation](#4)\n* [Modelling](#5)\n* [KFold Prediction](#6)\n* [References](#7)","metadata":{"id":"s5FswqfOav7-"}},{"cell_type":"markdown","source":"<a id=\"1\"></a>\n<h1 style='background:#1EA1A1; border:0; color:white'><center> 🚚 Imports<center></h1>","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet tensorflow_addons > /dev/null","metadata":{"id":"KXnA7N80AeLX","outputId":"e78d1d1c-9234-4259-bfd6-565303fefe19","execution":{"iopub.status.busy":"2021-09-17T04:48:50.078621Z","iopub.execute_input":"2021-09-17T04:48:50.079648Z","iopub.status.idle":"2021-09-17T04:49:00.731508Z","shell.execute_reply.started":"2021-09-17T04:48:50.079543Z","shell.execute_reply":"2021-09-17T04:49:00.730626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os \nimport re\nimport warnings\nfrom glob import glob\nfrom tqdm.auto import tqdm \nfrom scipy.signal import get_window\nfrom typing import Optional, Tuple\nfrom kaggle_datasets import KaggleDatasets\n\n\nimport numpy as np \nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split,StratifiedKFold,KFold\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.models import Model , load_model, save_model \nfrom tensorflow.keras.layers import Input,Dropout,Conv2D,Flatten,GlobalAveragePooling2D,Dense,GlobalAvgPool2D\nimport efficientnet.tfkeras as efn\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import BinaryCrossentropy\nimport tensorflow_addons as tfa\n","metadata":{"id":"O-u-ox1kx0ug","execution":{"iopub.status.busy":"2021-09-17T04:49:00.733908Z","iopub.execute_input":"2021-09-17T04:49:00.734536Z","iopub.status.idle":"2021-09-17T04:49:08.260911Z","shell.execute_reply.started":"2021-09-17T04:49:00.734491Z","shell.execute_reply":"2021-09-17T04:49:08.259954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h1 style='background:#1EA1A1; border:0; color:white'><center> TPU Detection <center></h1>","metadata":{"id":"FnL2UrMdx0uj"}},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"id":"_EqyU2mrx0uj","outputId":"e46d25df-881e-40fc-b443-4323aafbd914","execution":{"iopub.status.busy":"2021-09-17T04:49:08.262499Z","iopub.execute_input":"2021-09-17T04:49:08.262878Z","iopub.status.idle":"2021-09-17T04:49:13.61495Z","shell.execute_reply.started":"2021-09-17T04:49:08.262824Z","shell.execute_reply":"2021-09-17T04:49:13.614211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"2\"></a>\n<h1 style='background:#1EA1A1; border:0; color:white'><center> ⚙️ Configs<center></h1>","metadata":{"id":"HoL_vakIx0ul"}},{"cell_type":"code","source":"class config:\n    seed = 2021\n    replicas = strategy.num_replicas_in_sync\n    batch_size = 32 * replicas\n    AUTO = tf.data.experimental.AUTOTUNE\n    dir_path = './result'\n    epochs = 15\n    #augmentations\n    image_size = 512\n    aug = True \n    mix_up_p = 0.1\n    s_shift = 0.0\n    t_shift = 0.0\n    r_angle = 0 / 180 * np.pi\n    label_positive_shift = 0.99\n\n\nif not os.path.exists(config.dir_path):\n    os.makedirs(config.dir_path)\n\ndef seed_everything(seed = config.seed):\n    print(f\"seeded everything to seed {seed}\")\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\nseed_everything()","metadata":{"id":"dlWRISAfx0um","outputId":"603cd10d-0d26-4758-fc57-11916ec502ac","execution":{"iopub.status.busy":"2021-09-17T04:49:48.124603Z","iopub.execute_input":"2021-09-17T04:49:48.124899Z","iopub.status.idle":"2021-09-17T04:49:48.133911Z","shell.execute_reply.started":"2021-09-17T04:49:48.124867Z","shell.execute_reply":"2021-09-17T04:49:48.133174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h1 style='background:#1EA1A1; border:0; color:white'><center> Extracting GCS Path <center></h1>","metadata":{"id":"iyK-6tQJx0uo"}},{"cell_type":"code","source":"# Dataset of Hidehisa Arai\n\ntrain_files = ['gs://kds-fe725ba5a3259c712812aed413cfe61fc3827b135988e28e694e515c/train0.tfrecords',\n 'gs://kds-fe725ba5a3259c712812aed413cfe61fc3827b135988e28e694e515c/train1.tfrecords',\n 'gs://kds-fe725ba5a3259c712812aed413cfe61fc3827b135988e28e694e515c/train2.tfrecords',\n 'gs://kds-fe725ba5a3259c712812aed413cfe61fc3827b135988e28e694e515c/train3.tfrecords',\n 'gs://kds-fe725ba5a3259c712812aed413cfe61fc3827b135988e28e694e515c/train4.tfrecords',\n 'gs://kds-ecd1c4515f350d79d5a8d2e9a07df52885278c6cce43ad116c8b03cb/train5.tfrecords',\n 'gs://kds-ecd1c4515f350d79d5a8d2e9a07df52885278c6cce43ad116c8b03cb/train6.tfrecords',\n 'gs://kds-ecd1c4515f350d79d5a8d2e9a07df52885278c6cce43ad116c8b03cb/train7.tfrecords',\n 'gs://kds-ecd1c4515f350d79d5a8d2e9a07df52885278c6cce43ad116c8b03cb/train8.tfrecords',\n 'gs://kds-ecd1c4515f350d79d5a8d2e9a07df52885278c6cce43ad116c8b03cb/train9.tfrecords',\n 'gs://kds-2c0b3b314d8607851f6dddea1e976aab52547b68de5117cf3e76a85a/train10.tfrecords',\n 'gs://kds-2c0b3b314d8607851f6dddea1e976aab52547b68de5117cf3e76a85a/train11.tfrecords',\n 'gs://kds-2c0b3b314d8607851f6dddea1e976aab52547b68de5117cf3e76a85a/train12.tfrecords',\n 'gs://kds-2c0b3b314d8607851f6dddea1e976aab52547b68de5117cf3e76a85a/train13.tfrecords',\n 'gs://kds-2c0b3b314d8607851f6dddea1e976aab52547b68de5117cf3e76a85a/train14.tfrecords',\n 'gs://kds-965e4184655c915b40b37bf7e5d3c8e310dd5751e8a2e1f5bdc8d70d/train15.tfrecords',\n 'gs://kds-965e4184655c915b40b37bf7e5d3c8e310dd5751e8a2e1f5bdc8d70d/train16.tfrecords',\n 'gs://kds-965e4184655c915b40b37bf7e5d3c8e310dd5751e8a2e1f5bdc8d70d/train17.tfrecords',\n 'gs://kds-965e4184655c915b40b37bf7e5d3c8e310dd5751e8a2e1f5bdc8d70d/train18.tfrecords',\n 'gs://kds-965e4184655c915b40b37bf7e5d3c8e310dd5751e8a2e1f5bdc8d70d/train19.tfrecords']\n\ntest_files = ['gs://kds-50bb183919e58675b28621ce8be69435cf20585a6d6cb1842bc006f5/test0.tfrecords',\n 'gs://kds-50bb183919e58675b28621ce8be69435cf20585a6d6cb1842bc006f5/test1.tfrecords',\n 'gs://kds-50bb183919e58675b28621ce8be69435cf20585a6d6cb1842bc006f5/test2.tfrecords',\n 'gs://kds-50bb183919e58675b28621ce8be69435cf20585a6d6cb1842bc006f5/test3.tfrecords',\n 'gs://kds-50bb183919e58675b28621ce8be69435cf20585a6d6cb1842bc006f5/test4.tfrecords',\n 'gs://kds-becf38913abeb284aec05b255f7dc057aeb025941368bbcbc5f64bb0/test5.tfrecords',\n 'gs://kds-becf38913abeb284aec05b255f7dc057aeb025941368bbcbc5f64bb0/test6.tfrecords',\n 'gs://kds-becf38913abeb284aec05b255f7dc057aeb025941368bbcbc5f64bb0/test7.tfrecords',\n 'gs://kds-becf38913abeb284aec05b255f7dc057aeb025941368bbcbc5f64bb0/test8.tfrecords',\n 'gs://kds-becf38913abeb284aec05b255f7dc057aeb025941368bbcbc5f64bb0/test9.tfrecords']","metadata":{"id":"gZD8t_p1x0uo","execution":{"iopub.status.busy":"2021-09-17T04:49:13.62749Z","iopub.execute_input":"2021-09-17T04:49:13.627945Z","iopub.status.idle":"2021-09-17T04:49:13.643072Z","shell.execute_reply.started":"2021-09-17T04:49:13.627914Z","shell.execute_reply":"2021-09-17T04:49:13.642352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to create cqt kernel\ndef create_cqt_kernels(\n    q: float,\n    fs: float,\n    fmin: float,\n    n_bins: int = 84,\n    bins_per_octave: int = 12,\n    norm: float = 1,\n    window: str = \"hann\",\n    fmax: Optional[float] = None,\n    topbin_check: bool = True\n) -> Tuple[np.ndarray, int, np.ndarray, float]:\n    fft_len = 2 ** _nextpow2(np.ceil(q * fs / fmin))\n    \n    if (fmax is not None) and (n_bins is None):\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    elif (fmax is None) and (n_bins is not None):\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    else:\n        warnings.warn(\"If nmax is given, n_bins will be ignored\", SyntaxWarning)\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    if np.max(freqs) > fs / 2 and topbin_check:\n        raise ValueError(f\"The top bin {np.max(freqs)} Hz has exceeded the Nyquist frequency, \\\n                           please reduce the `n_bins`\")\n    \n    kernel = np.zeros((int(n_bins), int(fft_len)), dtype=np.complex64)\n    \n    length = np.ceil(q * fs / freqs)\n    for k in range(0, int(n_bins)):\n        freq = freqs[k]\n        l = np.ceil(q * fs / freq)\n        \n        if l % 2 == 1:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0)) - 1\n        else:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0))\n\n        sig = get_window(window, int(l), fftbins=True) * np.exp(\n            np.r_[-l // 2:l // 2] * 1j * 2 * np.pi * freq / fs) / l\n        \n        if norm:\n            kernel[k, start:start + int(l)] = sig / np.linalg.norm(sig, norm)\n        else:\n            kernel[k, start:start + int(l)] = sig\n    return kernel, fft_len, length, freqs\n\ndef _nextpow2(a: float) -> int:\n    return int(np.ceil(np.log2(a)))\n\n# Function to prepare cqt kernel\ndef prepare_cqt_kernel(\n    sr=22050,\n    hop_length=512,\n    fmin=32.70,\n    fmax=None,\n    n_bins=84,\n    bins_per_octave=12,\n    norm=1,\n    filter_scale=1,\n    window=\"hann\"\n):\n    q = float(filter_scale) / (2 ** (1 / bins_per_octave) - 1)\n    print(q)\n    return create_cqt_kernels(q, sr, fmin, n_bins, bins_per_octave, norm, window, fmax)\n# Function to create cqt image\ndef create_cqt_image(wave, hop_length=16):\n    CQTs = []\n    for i in range(3):\n        x = wave[i]\n        x = tf.expand_dims(tf.expand_dims(x, 0), 2)\n        x = tf.pad(x, PADDING, \"REFLECT\")\n\n        CQT_real = tf.nn.conv1d(x, CQT_KERNELS_REAL, stride=hop_length, padding=\"VALID\")\n        CQT_imag = -tf.nn.conv1d(x, CQT_KERNELS_IMAG, stride=hop_length, padding=\"VALID\")\n        CQT_real *= tf.math.sqrt(LENGTHS)\n        CQT_imag *= tf.math.sqrt(LENGTHS)\n\n        CQT = tf.math.sqrt(tf.pow(CQT_real, 2) + tf.pow(CQT_imag, 2))\n        CQTs.append(CQT[0])\n    return tf.stack(CQTs, axis=2)\n\nHOP_LENGTH = 64\ncqt_kernels, KERNEL_WIDTH, lengths, _ = prepare_cqt_kernel(\n    sr=2048,\n    hop_length=HOP_LENGTH,\n    fmin=20,\n    fmax=500,\n    bins_per_octave=12)\nLENGTHS = tf.constant(lengths, dtype=tf.float32)\nCQT_KERNELS_REAL = tf.constant(np.swapaxes(cqt_kernels.real[:, np.newaxis, :], 0, 2))\nCQT_KERNELS_IMAG = tf.constant(np.swapaxes(cqt_kernels.imag[:, np.newaxis, :], 0, 2))\nPADDING = tf.constant([[0, 0],\n                        [KERNEL_WIDTH // 2, KERNEL_WIDTH // 2],\n                        [0, 0]])","metadata":{"id":"G3gwgvYNx0up","outputId":"f7bca107-8f9f-4eda-fe11-e00156f313c0","execution":{"iopub.status.busy":"2021-09-17T04:49:13.644772Z","iopub.execute_input":"2021-09-17T04:49:13.645323Z","iopub.status.idle":"2021-09-17T04:49:13.869247Z","shell.execute_reply.started":"2021-09-17T04:49:13.645277Z","shell.execute_reply":"2021-09-17T04:49:13.868641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"3\"></a>\n<h1 style='background:#1EA1A1; border:0; color:white'><center> Reading TFRecords<center></h1>","metadata":{"id":"OGk0ewGex0ut"}},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], config.image_size), tf.reshape(tf.cast(example[\"target\"], tf.float32), [1])\n\n\ndef read_unlabeled_tfrecord(example, return_image_id):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], config.image_size), example[\"wave_id\"] if return_image_id else 0\n\ndef count_data_items(fileids):\n    return len(fileids) * 28000\n\ndef count_data_items_test(fileids):\n    return len(fileids) * 22600\n\ndef prepare_image(wave, dim= config.image_size):\n    wave = tf.reshape(tf.io.decode_raw(wave, tf.float64), (3, 4096))\n    normalized_waves = []\n    for i in range(3):\n        normalized_wave = wave[i] / tf.math.reduce_max(wave[i])\n        normalized_waves.append(normalized_wave)\n    wave = tf.stack(normalized_waves)\n    wave = tf.cast(wave, tf.float32)\n    image = create_cqt_image(wave, HOP_LENGTH)\n    image = tf.image.resize(image, size=(dim, dim))\n    return tf.reshape(image, (dim, dim, 3))\n\ndef mixup(image, label, PROBABILITY = 1.0, AUG_BATCH= config.batch_size):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = config.image_size\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        # MAKE CUTMIX LABEL\n        lab1 = label[j,]\n        lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,))\n    return image2,label2\n\ndef time_shift(img, shift= config.t_shift):\n    if shift > 0:\n        T = config.image_size\n        P = tf.random.uniform([],0,1)\n        SHIFT = tf.cast(T * P, tf.int32)\n        return tf.concat([img[-SHIFT:], img[:-SHIFT]], axis=0)\n    return img\n\n\ndef spector_shift(img, shift= config.s_shift):\n    if shift > 0:\n        T = config.image_size\n        P = tf.random.uniform([],0,1)\n        SHIFT = tf.cast(T * P, tf.int32)\n        return tf.concat([img[:, -SHIFT:], img[:, :-SHIFT]], axis=1)\n    return img\n  \ndef rotate(img, angle= config.r_angle):\n    if angle > 0:\n        P = tf.random.uniform([],0,1)\n        A = tf.cast(angle * P, tf.float32)\n        return tfa.image.rotate(img, A)\n    return img\n\ndef img_aug_f(img):\n    img = time_shift(img)\n    img = spector_shift(img)\n    # img = rotate(img)\n    return img\n\ndef imgs_aug_f(imgs, batch_size):\n    _imgs = []\n    DIM = config.image_size\n    for j in range(batch_size):\n        _imgs.append(img_aug_f(imgs[j]))\n    return tf.reshape(tf.stack(_imgs),(batch_size,DIM,DIM,3))\n\ndef label_positive_shift(labels):\n    return labels *  config.label_positive_shift\n\ndef aug_f(imgs, labels, batch_size):\n    imgs, label = mixup(imgs, labels, config.mix_up_p, batch_size)\n    imgs = imgs_aug_f(imgs, batch_size)\n    return imgs, label_positive_shift(label)\n","metadata":{"id":"O2my5ddHx0uu","execution":{"iopub.status.busy":"2021-09-17T04:49:13.870582Z","iopub.execute_input":"2021-09-17T04:49:13.870804Z","iopub.status.idle":"2021-09-17T04:49:13.89975Z","shell.execute_reply.started":"2021-09-17T04:49:13.870778Z","shell.execute_reply":"2021-09-17T04:49:13.898854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"4\"></a>\n<h1 style='background:#1EA1A1; border:0; color:white'><center> 📊 Dataset Creation <center></h1>","metadata":{"id":"zVy8YsxKx0uw"}},{"cell_type":"code","source":"def get_dataset(files, shuffle = False, repeat = False, labeled=True, return_image_ids=True, batch_size=config.batch_size ,aug=False):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=config.AUTO, compression_type=\"GZIP\")\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(2048)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=config.AUTO)\n    else:\n        ds = ds.map(read_unlabeled_tfrecord,num_parallel_calls= config.AUTO) \n    \n    ds = ds.batch(batch_size)\n    if aug:\n        ds = ds.map(lambda x, y: aug_f(x, y, batch_size), num_parallel_calls=config.AUTO)\n    \n    ds = ds.prefetch(config.AUTO)\n    return ds\n","metadata":{"id":"6Ntnlz3dx0ux","execution":{"iopub.status.busy":"2021-09-17T04:49:53.579092Z","iopub.execute_input":"2021-09-17T04:49:53.579969Z","iopub.status.idle":"2021-09-17T04:49:53.588496Z","shell.execute_reply.started":"2021-09-17T04:49:53.579903Z","shell.execute_reply":"2021-09-17T04:49:53.587638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"5\"></a>\n<h1 style='background:#1EA1A1; border:0; color:white'><center> 🧠 Modelling <center></h1>\n","metadata":{"id":"OIoEarBbx0uy"}},{"cell_type":"code","source":"def create_model(backbone_model):\n    x = GlobalAveragePooling2D()(backbone_model.output)\n    prediction_layer = Dense(1, activation = 'sigmoid')(x)\n    \n    model = Model(inputs = backbone_model.input , outputs = prediction_layer)\n    \n    return model","metadata":{"id":"jV0QI9Unx0u0","execution":{"iopub.status.busy":"2021-09-17T04:49:54.514939Z","iopub.execute_input":"2021-09-17T04:49:54.515816Z","iopub.status.idle":"2021-09-17T04:49:54.520774Z","shell.execute_reply.started":"2021-09-17T04:49:54.515775Z","shell.execute_reply":"2021-09-17T04:49:54.520097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"6\"></a>\n<h1 style='background:#1EA1A1; border:0; color:white'><center> 🔄 KFold Training <center></h1>","metadata":{"id":"1cB2K3PWx0u1"}},{"cell_type":"code","source":"def get_lr_callback():\n    lr_start   = 1e-4\n    lr_max     = 0.000015 * config.batch_size \n    lr_min     = 1e-7\n    lr_ramp_ep = 3\n    lr_sus_ep  = 0\n    lr_decay   = 0.7\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n    return lr_callback","metadata":{"id":"mpHyvjbYARt6","execution":{"iopub.status.busy":"2021-09-17T04:49:57.909492Z","iopub.execute_input":"2021-09-17T04:49:57.909799Z","iopub.status.idle":"2021-09-17T04:49:57.917194Z","shell.execute_reply.started":"2021-09-17T04:49:57.909762Z","shell.execute_reply":"2021-09-17T04:49:57.916306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def vis_lr_callback():\n    lr_start   = 1e-4\n    lr_max     = 0.000015 * config.batch_size\n    lr_min     = 1e-7\n    lr_ramp_ep = 3\n    lr_sus_ep  = 0\n    lr_decay   = 0.7\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    plt.figure(figsize=(10, 7))\n    plt.plot([lrfn(i) for i in range(config.epochs)])\n    plt.show()\nvis_lr_callback()","metadata":{"id":"5unjbRulDsQv","outputId":"c0a5b63f-be5b-476e-b8dc-3d1dcc31b780","execution":{"iopub.status.busy":"2021-09-17T04:49:57.933324Z","iopub.execute_input":"2021-09-17T04:49:57.933674Z","iopub.status.idle":"2021-09-17T04:49:58.169207Z","shell.execute_reply.started":"2021-09-17T04:49:57.933641Z","shell.execute_reply":"2021-09-17T04:49:58.168295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"histories = []\nskf = KFold(n_splits= 4 , shuffle = True , random_state = config.seed)\ntrain_files = np.array(train_files)\nfor k,(train_idx,valid_idx) in enumerate(skf.split(train_files)):\n    print(f\"***************** RUNNING FOLD {k+1} *****************\")\n    train_file_list = train_files[train_idx]\n    valid_file_list = train_files[valid_idx]\n    print(f\"number of train files {len(train_file_list)} number of valid files {len(valid_file_list)}\")\n    \n    #preparing dataset\n    print(\"preparing train datasets\")\n    train_dataset = get_dataset(train_file_list, batch_size= config.batch_size, shuffle=True, repeat=True, aug=True)\n    print(\"preparing validation datasets\")\n    valid_dataset = get_dataset(valid_file_list, batch_size= config.batch_size, repeat=False, shuffle=False, aug=False)\n     \n    train_steps = count_data_items(train_file_list)/config.batch_size//4\n    validation_steps = count_data_items(valid_file_list)/config.batch_size//4\n    print(f'Training Steps {train_steps} , validation steps {validation_steps}')\n\n    #clearing session\n    print(\"clearning session\")\n    K.clear_session()\n    \n    with strategy.scope():\n        backbone_model = efn.EfficientNetB1(include_top=False,weights='imagenet',input_shape=(config.image_size,config.image_size,3))\n        model = create_model(backbone_model)\n        lr_decayed_fn = tf.keras.experimental.CosineDecay(1e-3, train_steps)\n        opt = tfa.optimizers.AdamW(lr_decayed_fn, learning_rate=1e-4)\n        model.compile( \n            optimizer = opt,\n            loss = BinaryCrossentropy(),\n            metrics = 'AUC'\n        )\n    model_checkpoint = ModelCheckpoint(f'{config.dir_path}/eff_net_fold_{k+1}.h5',\n                                       monitor = 'val_auc', verbose = 1, \n                                       save_best_only = True, save_weights_only = True, mode = 'max') \n    hist = model.fit(train_dataset,steps_per_epoch=train_steps, validation_data= valid_dataset,\n                      epochs = config.epochs , callbacks =[model_checkpoint , get_lr_callback()])\n    histories.append(hist)","metadata":{"id":"8kHhSk03x0u3","outputId":"9ab66dca-d126-4b47-bdc7-07004d6d85df","execution":{"iopub.status.busy":"2021-09-17T04:50:45.480003Z","iopub.execute_input":"2021-09-17T04:50:45.480283Z","iopub.status.idle":"2021-09-17T04:53:32.182243Z","shell.execute_reply.started":"2021-09-17T04:50:45.480256Z","shell.execute_reply":"2021-09-17T04:53:32.179602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"7\"> </a>\n<h1 style='background:#1EA1A1; border:0; color:white'><center> 📌 References <center></h1>\n \n **Thank You Hidehisa Arai for providing such good kernels**\n * Hidehisa's [Training Notebook](https://www.kaggle.com/hidehisaarai1213/g2net-tf-on-the-fly-cqt-tpu-training) and [Inference Notebook](https://www.kaggle.com/hidehisaarai1213/g2net-tf-on-the-fly-cqt-tpu-inference)\n * Hidehisa's [Train Dataset](https://www.kaggle.com/hidehisaarai1213/g2net-waveform-tfrecords-train-0-4) and [Test Dataset](https://www.kaggle.com/hidehisaarai1213/g2net-waveform-tfrecords-test-0-4)","metadata":{}},{"cell_type":"markdown","source":"**Thanks for viewing, drop your suggestions down in the comments below. 🙂**\n","metadata":{}}]}