{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## About this notebook\n\nThis notebook is based on [CQT G2Net EfficientNetB1[TPU Training]](https://www.kaggle.com/miklgr500/cqt-g2net-efficientnetb7-tpu-training-w-b) by [Welf Crozzo](https://www.kaggle.com/miklgr500) and [nnAudio Constant Q-transform Demonstration](https://www.kaggle.com/atamazian/nnaudio-constant-q-transform-demonstration) by [Araik Tamazian](https://www.kaggle.com/atamazian).\n\nThis notebook use Constant Q-Transform for feature extraction and EfficientNetB0 for classification. The whole pipeline is implemented with Tensorflow, and the training process runs on TPU.\n\nThe main difference between this notebook and Welf's notebook is the use of on-the-fly CQT computation implemented with Tensorflow, which is similar to the idea of [nnAudio](https://github.com/KinWaiCheuk/nnAudio)'s [CQT1992v2](https://kinwaicheuk.github.io/nnAudio/_autosummary/nnAudio.Spectrogram.CQT1992v2.html?highlight=cqt1992v2#nnAudio.Spectrogram.CQT1992v2) layer.\n\n* [Inference Notebook](https://www.kaggle.com/hidehisaarai1213/g2net-tf-on-the-fly-cqt-tpu-inference)","metadata":{}},{"cell_type":"markdown","source":"## Install Dependencies","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet tensorflow_addons > /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:37:42.241205Z","iopub.execute_input":"2021-08-27T02:37:42.241644Z","iopub.status.idle":"2021-08-27T02:37:52.939693Z","shell.execute_reply.started":"2021-08-27T02:37:42.241606Z","shell.execute_reply":"2021-08-27T02:37:52.938461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport math\nimport random\nimport re\nimport warnings\nfrom pathlib import Path\nfrom typing import Optional, Tuple\n\nimport efficientnet.tfkeras as efn\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom kaggle_datasets import KaggleDatasets\nfrom scipy.signal import get_window\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:37:52.941847Z","iopub.execute_input":"2021-08-27T02:37:52.942361Z","iopub.status.idle":"2021-08-27T02:38:01.441524Z","shell.execute_reply.started":"2021-08-27T02:37:52.942299Z","shell.execute_reply":"2021-08-27T02:38:01.440047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:38:01.443262Z","iopub.execute_input":"2021-08-27T02:38:01.443619Z","iopub.status.idle":"2021-08-27T02:38:01.453304Z","shell.execute_reply.started":"2021-08-27T02:38:01.443586Z","shell.execute_reply":"2021-08-27T02:38:01.452354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"NUM_FOLDS = 4\nIMAGE_SIZE = 256\nBATCH_SIZE = 64\nEFFICIENTNET_SIZE = 7\nWEIGHTS = \"imagenet\"\n\nMIXUP_PROB = 0.0\nEPOCHS = 20\nR_ANGLE = 0 / 180 * np.pi\nS_SHIFT = 0.0\nT_SHIFT = 0.0\nLABEL_POSITIVE_SHIFT = 0.99\n\nCQT_MODEL = True","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:38:01.454888Z","iopub.execute_input":"2021-08-27T02:38:01.455427Z","iopub.status.idle":"2021-08-27T02:38:01.470146Z","shell.execute_reply.started":"2021-08-27T02:38:01.455377Z","shell.execute_reply":"2021-08-27T02:38:01.469138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAVEDIR = Path(\"models\")\nSAVEDIR.mkdir(exist_ok=True)\n\nOOFDIR = Path(\"oof\")\nOOFDIR.mkdir(exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:38:01.471746Z","iopub.execute_input":"2021-08-27T02:38:01.472394Z","iopub.status.idle":"2021-08-27T02:38:01.487786Z","shell.execute_reply.started":"2021-08-27T02:38:01.472352Z","shell.execute_reply":"2021-08-27T02:38:01.486484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utilities","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\n\nset_seed(1213)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:38:01.489697Z","iopub.execute_input":"2021-08-27T02:38:01.490038Z","iopub.status.idle":"2021-08-27T02:38:01.502864Z","shell.execute_reply.started":"2021-08-27T02:38:01.490005Z","shell.execute_reply":"2021-08-27T02:38:01.50179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    TPU_DETECTED = False\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n        TPU_DETECTED = True\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n\n    return strategy, TPU_DETECTED","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:38:01.504948Z","iopub.execute_input":"2021-08-27T02:38:01.505693Z","iopub.status.idle":"2021-08-27T02:38:01.518953Z","shell.execute_reply.started":"2021-08-27T02:38:01.50565Z","shell.execute_reply":"2021-08-27T02:38:01.51798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy, tpu_detected = auto_select_accelerator()\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:38:01.521215Z","iopub.execute_input":"2021-08-27T02:38:01.521881Z","iopub.status.idle":"2021-08-27T02:38:01.541669Z","shell.execute_reply.started":"2021-08-27T02:38:01.521798Z","shell.execute_reply":"2021-08-27T02:38:01.540003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Loading","metadata":{}},{"cell_type":"code","source":"gcs_paths = []\nfor i, j in [(0, 4), (5, 9), (10, 14), (15, 19)]:\n    GCS_path = KaggleDatasets().get_gcs_path(f\"g2net-waveform-tfrecords-train-{i}-{j}\")\n    gcs_paths.append(GCS_path)\n    print(GCS_path)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:38:01.543896Z","iopub.execute_input":"2021-08-27T02:38:01.54428Z","iopub.status.idle":"2021-08-27T02:38:03.405329Z","shell.execute_reply.started":"2021-08-27T02:38:01.544244Z","shell.execute_reply":"2021-08-27T02:38:03.404311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_files = []\nfor path in gcs_paths:\n    all_files.extend(np.sort(np.array(tf.io.gfile.glob(path + \"/train*.tfrecords\"))))\n\nprint(\"train_files: \", len(all_files))","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:38:03.406943Z","iopub.execute_input":"2021-08-27T02:38:03.407279Z","iopub.status.idle":"2021-08-27T02:38:04.266683Z","shell.execute_reply.started":"2021-08-27T02:38:03.407238Z","shell.execute_reply":"2021-08-27T02:38:04.265261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset Preparation\n\nHere's the main contribution of this notebook - Tensorflow version of on-the-fly CQT computation. Note that some of the operations used in CQT computation are not supported by TPU, therefore the implementation is not a TF layer but a function that runs on CPU.","metadata":{}},{"cell_type":"code","source":"def create_cqt_kernels(\n    q: float,\n    fs: float,\n    fmin: float,\n    n_bins: int = 84,\n    bins_per_octave: int = 12,\n    norm: float = 1,\n    window: str = \"hann\",\n    fmax: Optional[float] = None,\n    topbin_check: bool = True\n) -> Tuple[np.ndarray, int, np.ndarray, float]:\n    fft_len = 2 ** _nextpow2(np.ceil(q * fs / fmin))\n    \n    if (fmax is not None) and (n_bins is None):\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    elif (fmax is None) and (n_bins is not None):\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    else:\n        warnings.warn(\"If nmax is given, n_bins will be ignored\", SyntaxWarning)\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n        \n    if np.max(freqs) > fs / 2 and topbin_check:\n        raise ValueError(f\"The top bin {np.max(freqs)} Hz has exceeded the Nyquist frequency, \\\n                           please reduce the `n_bins`\")\n    \n    kernel = np.zeros((int(n_bins), int(fft_len)), dtype=np.complex64)\n    \n    length = np.ceil(q * fs / freqs)\n    for k in range(0, int(n_bins)):\n        freq = freqs[k]\n        l = np.ceil(q * fs / freq)\n        \n        if l % 2 == 1:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0)) - 1\n        else:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0))\n\n        sig = get_window(window, int(l), fftbins=True) * np.exp(\n            np.r_[-l // 2:l // 2] * 1j * 2 * np.pi * freq / fs) / l\n        \n        if norm:\n            kernel[k, start:start + int(l)] = sig / np.linalg.norm(sig, norm)\n        else:\n            kernel[k, start:start + int(l)] = sig\n    return kernel, fft_len, length, freqs\n\n\ndef _nextpow2(a: float) -> int:\n    return int(np.ceil(np.log2(a)))\n\n\ndef prepare_cqt_kernel(\n    sr=22050,\n    hop_length=512,\n    fmin=32.70,\n    fmax=None,\n    n_bins=84,\n    bins_per_octave=12,\n    norm=1,\n    filter_scale=1,\n    window=\"hann\"\n):\n    q = float(filter_scale) / (2 ** (1 / bins_per_octave) - 1)\n    print(q)\n    return create_cqt_kernels(q, sr, fmin, n_bins, bins_per_octave, norm, window, fmax)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:38:04.268303Z","iopub.execute_input":"2021-08-27T02:38:04.268655Z","iopub.status.idle":"2021-08-27T02:38:04.558753Z","shell.execute_reply.started":"2021-08-27T02:38:04.26862Z","shell.execute_reply":"2021-08-27T02:38:04.557457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HOP_LENGTH = 16\ncqt_kernels, KERNEL_WIDTH, lengths, _ = prepare_cqt_kernel(\n    sr=2048,\n    hop_length=HOP_LENGTH,\n    fmin=20,\n    fmax=1024,\n    bins_per_octave=24)\nLENGTHS = tf.constant(lengths, dtype=tf.float32)\nCQT_KERNELS_REAL = tf.constant(np.swapaxes(cqt_kernels.real[:, np.newaxis, :], 0, 2))\nCQT_KERNELS_IMAG = tf.constant(np.swapaxes(cqt_kernels.imag[:, np.newaxis, :], 0, 2))\nPADDING = tf.constant([[0, 0],\n                        [KERNEL_WIDTH // 2, KERNEL_WIDTH // 2],\n                        [0, 0]])","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:38:04.560308Z","iopub.execute_input":"2021-08-27T02:38:04.560655Z","iopub.status.idle":"2021-08-27T02:38:04.638888Z","shell.execute_reply.started":"2021-08-27T02:38:04.560622Z","shell.execute_reply":"2021-08-27T02:38:04.638035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.plot(CQT_KERNELS_REAL.numpy()[:,0,3])\nCQT_KERNELS_REAL.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:38:04.640114Z","iopub.execute_input":"2021-08-27T02:38:04.640616Z","iopub.status.idle":"2021-08-27T02:38:04.875574Z","shell.execute_reply.started":"2021-08-27T02:38:04.640575Z","shell.execute_reply":"2021-08-27T02:38:04.873986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WINDOW = tf.cast(tf.constant(get_window(('tukey', 0.1), 4096)), tf.float32)\n\ndef create_cqt_image(wave, hop_length=16, window=WINDOW):\n    x = wave * window\n    x = tf.expand_dims(x, 2)\n    x = tf.pad(x, PADDING, \"REFLECT\")\n    cqt_real = tf.nn.conv1d(x, CQT_KERNELS_REAL, stride=hop_length, padding=\"VALID\")\n    cqt_imag = -tf.nn.conv1d(x, CQT_KERNELS_IMAG, stride=hop_length, padding=\"VALID\")\n    cqt_real *= tf.math.sqrt(LENGTHS)\n    cqt_imag *= tf.math.sqrt(LENGTHS)\n    cqt_amlp = tf.math.sqrt(tf.pow(cqt_real, 2) + tf.pow(cqt_imag, 2))\n    #cqt_amlp = cqt_amlp / tf.expand_dims(tf.reduce_mean(cqt_amlp, axis=1), 1)\n    return tf.transpose(cqt_amlp, perm=[2, 1, 0])\n\ndef build_cqtmodel(size = 4096, hop_length=16, window=WINDOW, dim = 256):\n    inputs = tf.keras.layers.Input(shape=(3,size))\n    # compress batch with channels\n    x = tf.reshape(inputs,shape = (-1,size))\n    x = tf.expand_dims(x, 2)\n    # x = tf.pad(x, PADDING, \"REFLECT\")\n    cqt_real = tf.nn.conv1d(x, CQT_KERNELS_REAL, stride=hop_length, padding=\"SAME\")\n    cqt_imag = -tf.nn.conv1d(x, CQT_KERNELS_IMAG, stride=hop_length, padding=\"SAME\")\n    cqt_real *= tf.math.sqrt(LENGTHS)\n    cqt_imag *= tf.math.sqrt(LENGTHS)\n    # this is (batch . channel) x width x height\n    cqt_amlp = tf.math.sqrt(tf.pow(cqt_real, 2) + tf.pow(cqt_imag, 2))\n    tmpShape = tf.shape(cqt_amlp)\n    # this is batch x channel x width x height\n    cqt_amlp = tf.reshape(cqt_amlp,shape = (-1,3,tmpShape[1],tmpShape[2]))\n    # this is batch x height x width x channel\n    cqt_amlp = tf.transpose(cqt_amlp, perm=[0, 3, 2, 1])\n    image = tf.image.resize(cqt_amlp, size=(dim, dim))\n    model = tf.keras.Model(inputs=inputs, outputs=image)\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:38:04.877627Z","iopub.execute_input":"2021-08-27T02:38:04.878016Z","iopub.status.idle":"2021-08-27T02:38:04.912009Z","shell.execute_reply.started":"2021-08-27T02:38:04.877978Z","shell.execute_reply":"2021-08-27T02:38:04.910678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], IMAGE_SIZE), tf.reshape(tf.cast(example[\"target\"], tf.float32), [1])\n\n\ndef read_unlabeled_tfrecord(example, return_image_id):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], IMAGE_SIZE), example[\"wave_id\"] if return_image_id else 0\n\n\ndef count_data_items(fileids):\n    return len(fileids) * 28000\n\n\ndef count_data_items_test(fileids):\n    return len(fileids) * 22600\n\n\ndef mixup(image, label, probability=0.5, aug_batch=64 * 8):\n    imgs = []\n    labs = []\n    for j in range(aug_batch):\n        p = tf.cast(tf.random.uniform([], 0, 1) <= probability, tf.float32)\n        k = tf.cast(tf.random.uniform([], 0, aug_batch), tf.int32)\n        a = tf.random.uniform([], 0, 1) * p\n\n        img1 = image[j]\n        img2 = image[k]\n        imgs.append((1 - a) * img1 + a * img2)\n        lab1 = label[j]\n        lab2 = label[k]\n        labs.append((1 - a) * lab1 + a * lab2)\n    image2 = tf.reshape(tf.stack(imgs), (aug_batch, IMAGE_SIZE, IMAGE_SIZE, 3))\n    label2 = tf.reshape(tf.stack(labs), (aug_batch,))\n    return image2, label2\n\n\ndef time_shift(img, shift=T_SHIFT):\n    if shift > 0:\n        T = IMAGE_SIZE\n        P = tf.random.uniform([],0,1)\n        SHIFT = tf.cast(T * P, tf.int32)\n        return tf.concat([img[-SHIFT:], img[:-SHIFT]], axis=0)\n    return img\n\n\ndef rotate(img, angle=R_ANGLE):\n    if angle > 0:\n        P = tf.random.uniform([],0,1)\n        A = tf.cast(angle * P, tf.float32)\n        return tfa.image.rotate(img, A)\n    return img\n\n\ndef spector_shift(img, shift=S_SHIFT):\n    if shift > 0:\n        T = IMAGE_SIZE\n        P = tf.random.uniform([],0,1)\n        SHIFT = tf.cast(T * P, tf.int32)\n        return tf.concat([img[:, -SHIFT:], img[:, :-SHIFT]], axis=1)\n    return img\n\ndef img_aug_f(img):\n    img = time_shift(img)\n    img = spector_shift(img)\n    # img = rotate(img)\n    return img\n\n\ndef imgs_aug_f(imgs, batch_size):\n    _imgs = []\n    DIM = IMAGE_SIZE\n    for j in range(batch_size):\n        _imgs.append(img_aug_f(imgs[j]))\n    return tf.reshape(tf.stack(_imgs),(batch_size,DIM,DIM,3))\n\n\ndef label_positive_shift(labels):\n    return labels * LABEL_POSITIVE_SHIFT\n\n\ndef aug_f(imgs, labels, batch_size):\n    imgs, label = mixup(imgs, labels, MIXUP_PROB, batch_size)\n    imgs = imgs_aug_f(imgs, batch_size)\n    return imgs, label_positive_shift(label)\n\n\ndef prepare_image(wave, dim=256):\n    wave = tf.reshape(tf.io.decode_raw(wave, tf.float64), (3, 4096))\n    normalized_waves = []\n    for i in range(3):\n        normalized_wave = wave[i] / tf.math.reduce_max(wave[i])\n        normalized_waves.append(normalized_wave)\n    wave = tf.stack(normalized_waves)\n    wave = tf.cast(wave, tf.float32)\n    if not CQT_MODEL:\n        image = create_cqt_image(wave, HOP_LENGTH)\n        image = tf.image.resize(image, size=(dim, dim))\n        return tf.reshape(image, (dim, dim, 3))\n    else:\n        return wave\n\n\ndef get_dataset(files, batch_size=16, repeat=False, shuffle=False, aug= not CQT_MODEL, labeled=True, return_image_ids=True):\n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO, compression_type=\"GZIP\")\n    ds = ds.cache()\n\n    if repeat:\n        ds = ds.repeat()\n\n    if shuffle:\n        ds = ds.shuffle(1024 * 2)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n\n    if labeled:\n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_ids), num_parallel_calls=AUTO)\n\n    ds = ds.batch(batch_size * REPLICAS)\n    if aug:\n        ds = ds.map(lambda x, y: aug_f(x, y, batch_size * REPLICAS), num_parallel_calls=AUTO)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:38:04.914484Z","iopub.execute_input":"2021-08-27T02:38:04.915122Z","iopub.status.idle":"2021-08-27T02:38:04.954282Z","shell.execute_reply.started":"2021-08-27T02:38:04.914902Z","shell.execute_reply":"2021-08-27T02:38:04.952686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"def get_lr_callback(lr_start = 1e-4, batch_size=8, replicas=8):\n    if True:\n        lr_max     = lr_start*100\n        lr_min     = lr_start/10\n        lr_ramp_ep = 3\n        lr_sus_ep  = 0\n        lr_decay   = 0.7\n\n        def lrfn(epoch):\n            if epoch < lr_ramp_ep:\n                lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n\n            elif epoch < lr_ramp_ep + lr_sus_ep:\n                lr = lr_max\n\n            else:\n                lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n\n            return lr\n\n        lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n    else:\n        def getlr_fn(initial_learning_rate, decay_steps, alpha=0.0):\n            def decayed_learning_rate(step,lr):\n              step = min(step, decay_steps)\n              cosine_decay = 0.5 * (1 + tf.math.cos(np.pi * step / decay_steps))\n              decayed = (1 - alpha) * cosine_decay + alpha\n              return initial_learning_rate * decayed\n            return decayed_learning_rate\n        lr_callback = tf.keras.callbacks.LearningRateScheduler(getlr_fn(1e-2, EPOCHS, alpha=1e-3), verbose=True)\n    return lr_callback\n\nclass MyLRSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n\n    def __init__(self, lr_start):\n        self.lr_start = lr_start\n        self.lr_max     = lr_start*100\n        self.lr_min     = lr_start/10\n        self.lr_ramp_ep = 3.\n        self.lr_sus_ep  = 0.\n        self.lr_decay   = 0.7\n\n    def __call__(self, epoch):\n        epoch = tf.cast(epoch,tf.float32)\n        lr_1 =  (self.lr_max - self.lr_start) / self.lr_ramp_ep * epoch + self.lr_start\n        lr_2 = self.lr_max\n        lr_3 = (self.lr_max - self.lr_min) * tf.pow(self.lr_decay,(epoch - self.lr_ramp_ep - self.lr_sus_ep)) + self.lr_min\n        cond1 = tf.cond(tf.greater(tf.constant(self.lr_ramp_ep,dtype = tf.float32), epoch), lambda: 1., lambda: 0.)\n        cond2 = tf.cond(tf.greater(tf.constant(self.lr_ramp_ep + self.lr_sus_ep,dtype = tf.float32), epoch), lambda: 1., lambda: 0.)\n        \n        lr = cond1*lr_1+tf.math.minimum(1 - cond1,cond2)*lr_2 + tf.math.maximum(1-cond1-cond2,0)*lr_3\n\n        return lr","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:38:04.956392Z","iopub.execute_input":"2021-08-27T02:38:04.957018Z","iopub.status.idle":"2021-08-27T02:38:04.978809Z","shell.execute_reply.started":"2021-08-27T02:38:04.956941Z","shell.execute_reply":"2021-08-27T02:38:04.977127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MyLRSchedule(1e-4)(10)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T02:40:13.100067Z","iopub.execute_input":"2021-08-27T02:40:13.100656Z","iopub.status.idle":"2021-08-27T02:40:13.108697Z","shell.execute_reply.started":"2021-08-27T02:40:13.100619Z","shell.execute_reply":"2021-08-27T02:40:13.107628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef build_model(size=256, efficientnet_size=0, weights=\"imagenet\", count=0):\n    cqtmodel = build_cqtmodel(dim = size)\n    if not CQT_MODEL:\n        inputs = tf.keras.layers.Input(shape=(size, size, 3))\n    else:\n        inputs = tf.keras.layers.Input(shape=(3, 4096))\n    \n    efn_string= f\"EfficientNetB{efficientnet_size}\"\n    efn_layer = getattr(efn, efn_string)(input_shape=(size, size, 3), weights=weights, include_top=False)\n    \n    x = inputs\n    if CQT_MODEL:\n        x = cqtmodel(x)\n    # x = tf.clip_by_value(x, 0, 2.5, name=None)-1\n    x = efn_layer(x)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    \n    if False:\n        x = tf.keras.layers.Dropout(0.2)(x)\n        x = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n    else:\n        FC = tf.keras.layers.Dense(1, activation=\"sigmoid\")\n        out = []\n        for rate in [0.3,0.5,0.7]:\n            x = tf.keras.layers.Dropout(rate)(x)\n            out.append( FC(x))\n        \n        x = tf.keras.layers.Average()(out)\n            \n    \n    model = tf.keras.Model(inputs=inputs, outputs=x)\n    \n    loss = tf.keras.losses.BinaryCrossentropy()\n    lr_decayed_fn = tf.keras.experimental.CosineDecay(1e-3, count)\n    #op1 = tfa.optimizers.AdamW(MyLRSchedule(tf.constant(1e-5)), learning_rate=MyLRSchedule(tf.constant(1e-4))) \n    #op2 = tfa.optimizers.AdamW(MyLRSchedule(tf.constant(1e-6)), learning_rate=MyLRSchedule(tf.constant(1e-5))) \n    op1 = tf.keras.optimizers.Adam(learning_rate=MyLRSchedule(tf.constant(2e-3)))\n    op2 = tf.keras.optimizers.Adam(learning_rate=MyLRSchedule(tf.constant(2e-4)))\n    layers1 = model.layers[:4]\n    layers2 = model.layers[4:]\n    \n    optimizers_and_layers = [(op1, layers1), (op2, layers2)]\n    \n    optimizer = tfa.optimizers.MultiOptimizer(optimizers_and_layers)\n    \n    model.compile(optimizer=optimizer, loss=loss, metrics=[\"AUC\"])\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-08-26T16:50:26.163227Z","iopub.execute_input":"2021-08-26T16:50:26.163509Z","iopub.status.idle":"2021-08-26T16:50:26.176893Z","shell.execute_reply.started":"2021-08-26T16:50:26.163483Z","shell.execute_reply":"2021-08-26T16:50:26.176079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if tf.constant(1):\n    print(3)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-26T16:50:26.177925Z","iopub.execute_input":"2021-08-26T16:50:26.178301Z","iopub.status.idle":"2021-08-26T16:50:26.190739Z","shell.execute_reply.started":"2021-08-26T16:50:26.178249Z","shell.execute_reply":"2021-08-26T16:50:26.189791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfa.optimizers.MultiOptimizer","metadata":{"execution":{"iopub.status.busy":"2021-08-26T16:50:26.192023Z","iopub.execute_input":"2021-08-26T16:50:26.192406Z","iopub.status.idle":"2021-08-26T16:50:26.200193Z","shell.execute_reply.started":"2021-08-26T16:50:26.192379Z","shell.execute_reply":"2021-08-26T16:50:26.19954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=1213)\noof_pred = []\noof_target = []\n\nfiles_train_all = np.array(all_files)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T16:50:26.20107Z","iopub.execute_input":"2021-08-26T16:50:26.201361Z","iopub.status.idle":"2021-08-26T16:50:26.208363Z","shell.execute_reply.started":"2021-08-26T16:50:26.201313Z","shell.execute_reply":"2021-08-26T16:50:26.207571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold, (trn_idx, val_idx) in enumerate(kf.split(files_train_all)):\n    if fold !=3:\n        continue\n    files_train = files_train_all[trn_idx]\n    files_valid = files_train_all[val_idx]\n\n    print(\"=\" * 120)\n    print(f\"Fold {fold}\")\n    print(\"=\" * 120)\n\n    train_image_count = count_data_items(files_train)\n    valid_image_count = count_data_items(files_valid)\n\n    tf.keras.backend.clear_session()\n    strategy, tpu_detected = auto_select_accelerator()\n    with strategy.scope():\n        model = build_model(\n            size=IMAGE_SIZE, \n            efficientnet_size=EFFICIENTNET_SIZE,\n            weights=WEIGHTS, \n            count=train_image_count // BATCH_SIZE // REPLICAS // 4)\n    \n    model_ckpt = tf.keras.callbacks.ModelCheckpoint(\n        str(SAVEDIR / f\"fold{fold}.h5\"), monitor=\"val_auc\", verbose=1, save_best_only=True,\n        save_weights_only=True, mode=\"max\", save_freq=\"epoch\"\n    )\n\n    history = model.fit(\n        get_dataset(files_train, batch_size=BATCH_SIZE, shuffle=True, repeat=True, aug=not CQT_MODEL),\n        epochs=EPOCHS,\n        callbacks=[model_ckpt,],#, get_lr_callback(BATCH_SIZE, REPLICAS)],\n        steps_per_epoch=train_image_count // BATCH_SIZE // REPLICAS // 4,\n        validation_data=get_dataset(files_valid, batch_size=BATCH_SIZE * 4, repeat=False, shuffle=False, aug=False),\n        verbose=1\n    )\n\n    print(\"Loading best model...\")\n    model.load_weights(str(SAVEDIR / f\"fold{fold}.h5\"))\n\n    ds_valid = get_dataset(files_valid, labeled=False, return_image_ids=False, repeat=True, shuffle=False, batch_size=BATCH_SIZE * 2, aug=False)\n    STEPS = valid_image_count / BATCH_SIZE / 2 / REPLICAS\n    pred = model.predict(ds_valid, steps=STEPS, verbose=0)[:valid_image_count]\n    oof_pred.append(np.mean(pred.reshape((valid_image_count, 1), order=\"F\"), axis=1))\n\n    ds_valid = get_dataset(files_valid, repeat=False, labeled=True, return_image_ids=True, aug=False)\n    oof_target.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))\n\n    plt.figure(figsize=(8, 6))\n    sns.distplot(oof_pred[-1])\n    plt.show()\n\n    plt.figure(figsize=(15, 5))\n    plt.plot(\n        np.arange(len(history.history[\"auc\"])),\n        history.history[\"auc\"],\n        \"-o\",\n        label=\"Train auc\",\n        color=\"#ff7f0e\")\n    plt.plot(\n        np.arange(len(history.history[\"auc\"])),\n        history.history[\"val_auc\"],\n        \"-o\",\n        label=\"Val auc\",\n        color=\"#1f77b4\")\n    \n    x = np.argmax(history.history[\"val_auc\"])\n    y = np.max(history.history[\"val_auc\"])\n\n    xdist = plt.xlim()[1] - plt.xlim()[0]\n    ydist = plt.ylim()[1] - plt.ylim()[0]\n\n    plt.scatter(x, y, s=200, color=\"#1f77b4\")\n    plt.text(x - 0.03 * xdist, y - 0.13 * ydist, f\"max auc\\n{y}\", size=14)\n\n    plt.ylabel(\"auc\", size=14)\n    plt.xlabel(\"Epoch\", size=14)\n    plt.legend(loc=2)\n\n    plt2 = plt.gca().twinx()\n    plt2.plot(\n        np.arange(len(history.history[\"auc\"])),\n        history.history[\"loss\"],\n        \"-o\",\n        label=\"Train Loss\",\n        color=\"#2ca02c\")\n    plt2.plot(\n        np.arange(len(history.history[\"auc\"])),\n        history.history[\"val_loss\"],\n        \"-o\",\n        label=\"Val Loss\",\n        color=\"#d62728\")\n    \n    x = np.argmin(history.history[\"val_loss\"])\n    y = np.min(history.history[\"val_loss\"])\n    \n    ydist = plt.ylim()[1] - plt.ylim()[0]\n\n    plt.scatter(x, y, s=200, color=\"#d62728\")\n    plt.text(x - 0.03 * xdist, y + 0.05 * ydist, \"min loss\", size=14)\n\n    plt.ylabel(\"Loss\", size=14)\n    plt.title(f\"Fold {fold + 1} - Image Size {IMAGE_SIZE}, EfficientNetB{EFFICIENTNET_SIZE}\", size=18)\n\n    plt.legend(loc=3)\n    plt.savefig(OOFDIR / f\"fig{fold}.png\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-26T16:50:26.209943Z","iopub.execute_input":"2021-08-26T16:50:26.210195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model(tf.random.normal((16,3,4096)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## OOF","metadata":{}},{"cell_type":"code","source":"oof = np.concatenate(oof_pred)\ntrue = np.concatenate(oof_target)\nauc = roc_auc_score(y_true=true, y_score=oof)\nprint(f\"AUC: {auc:.5f}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({\n    \"y_true\": true.reshape(-1),\n    \"y_pred\": oof\n})\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(OOFDIR / f\"oof.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}