{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## About this notebook\n\nThis notebook is based on [CQT G2Net EfficientNetB1[TPU Training]](https://www.kaggle.com/miklgr500/cqt-g2net-efficientnetb7-tpu-training-w-b) by [Welf Crozzo](https://www.kaggle.com/miklgr500) and [nnAudio Constant Q-transform Demonstration](https://www.kaggle.com/atamazian/nnaudio-constant-q-transform-demonstration) by [Araik Tamazian](https://www.kaggle.com/atamazian).\n\nThis notebook use Constant Q-Transform for feature extraction and EfficientNetB0 for classification. The whole pipeline is implemented with Tensorflow, and the training process runs on TPU.\n\nThe main difference between this notebook and Welf's notebook is the use of on-the-fly CQT computation implemented with Tensorflow, which is similar to the idea of [nnAudio](https://github.com/KinWaiCheuk/nnAudio)'s [CQT1992v2](https://kinwaicheuk.github.io/nnAudio/_autosummary/nnAudio.Spectrogram.CQT1992v2.html?highlight=cqt1992v2#nnAudio.Spectrogram.CQT1992v2) layer.\n\n* [Inference Notebook](https://www.kaggle.com/hidehisaarai1213/g2net-tf-on-the-fly-cqt-tpu-inference)","metadata":{}},{"cell_type":"markdown","source":"## Install Dependencies","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet tensorflow_addons > /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:28:17.063785Z","iopub.execute_input":"2021-08-13T14:28:17.064119Z","iopub.status.idle":"2021-08-13T14:28:24.558725Z","shell.execute_reply.started":"2021-08-13T14:28:17.064087Z","shell.execute_reply":"2021-08-13T14:28:24.557709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport math\nimport random\nimport re\nimport warnings\nfrom pathlib import Path\nfrom typing import Optional, Tuple\n\nimport efficientnet.tfkeras as efn\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom kaggle_datasets import KaggleDatasets\nfrom scipy.signal import get_window\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:28:43.37234Z","iopub.execute_input":"2021-08-13T14:28:43.372747Z","iopub.status.idle":"2021-08-13T14:28:51.858974Z","shell.execute_reply.started":"2021-08-13T14:28:43.372713Z","shell.execute_reply":"2021-08-13T14:28:51.858068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:28:58.62733Z","iopub.execute_input":"2021-08-13T14:28:58.627903Z","iopub.status.idle":"2021-08-13T14:28:58.637858Z","shell.execute_reply.started":"2021-08-13T14:28:58.627868Z","shell.execute_reply":"2021-08-13T14:28:58.636066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"NUM_FOLDS = 4\nIMAGE_SIZE = 256\nBATCH_SIZE = 32\nEFFICIENTNET_SIZE = 7\nWEIGHTS = \"imagenet\"\n\nMIXUP_PROB = 0.0\nEPOCHS = 20\nR_ANGLE = 0 / 180 * np.pi\nS_SHIFT = 0.0\nT_SHIFT = 0.0\nLABEL_POSITIVE_SHIFT = 0.99","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:29:39.493895Z","iopub.execute_input":"2021-08-13T14:29:39.494272Z","iopub.status.idle":"2021-08-13T14:29:39.500199Z","shell.execute_reply.started":"2021-08-13T14:29:39.494237Z","shell.execute_reply":"2021-08-13T14:29:39.498683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAVEDIR = Path(\"models\")\nSAVEDIR.mkdir(exist_ok=True)\n\nOOFDIR = Path(\"oof\")\nOOFDIR.mkdir(exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:30:35.748372Z","iopub.execute_input":"2021-08-13T14:30:35.74879Z","iopub.status.idle":"2021-08-13T14:30:35.755021Z","shell.execute_reply.started":"2021-08-13T14:30:35.748754Z","shell.execute_reply":"2021-08-13T14:30:35.753808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utilities","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\n\nset_seed(1213)","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:30:57.215105Z","iopub.execute_input":"2021-08-13T14:30:57.215566Z","iopub.status.idle":"2021-08-13T14:30:57.221797Z","shell.execute_reply.started":"2021-08-13T14:30:57.215523Z","shell.execute_reply":"2021-08-13T14:30:57.220492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    TPU_DETECTED = False\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n        TPU_DETECTED = True\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n\n    return strategy, TPU_DETECTED","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:31:06.126946Z","iopub.execute_input":"2021-08-13T14:31:06.127353Z","iopub.status.idle":"2021-08-13T14:31:06.134452Z","shell.execute_reply.started":"2021-08-13T14:31:06.127317Z","shell.execute_reply":"2021-08-13T14:31:06.13324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy, tpu_detected = auto_select_accelerator()\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:31:22.520862Z","iopub.execute_input":"2021-08-13T14:31:22.521244Z","iopub.status.idle":"2021-08-13T14:31:28.259187Z","shell.execute_reply.started":"2021-08-13T14:31:22.521209Z","shell.execute_reply":"2021-08-13T14:31:28.257661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Loading","metadata":{}},{"cell_type":"code","source":"gcs_paths = []\nfor i, j in [(0, 4), (5, 9), (10, 14), (15, 19)]:\n    GCS_path = KaggleDatasets().get_gcs_path(f\"g2net-waveform-tfrecords-train-{i}-{j}\")\n    gcs_paths.append(GCS_path)\n    print(GCS_path)","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:33:26.734943Z","iopub.execute_input":"2021-08-13T14:33:26.7354Z","iopub.status.idle":"2021-08-13T14:33:28.27891Z","shell.execute_reply.started":"2021-08-13T14:33:26.735358Z","shell.execute_reply":"2021-08-13T14:33:28.277987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_files = []\nfor path in gcs_paths:\n    all_files.extend(np.sort(np.array(tf.io.gfile.glob(path + \"/train*.tfrecords\"))))\n\nprint(\"train_files: \", len(all_files))","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:33:39.324613Z","iopub.execute_input":"2021-08-13T14:33:39.324996Z","iopub.status.idle":"2021-08-13T14:33:39.631241Z","shell.execute_reply.started":"2021-08-13T14:33:39.324965Z","shell.execute_reply":"2021-08-13T14:33:39.630142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset Preparation\n\nHere's the main contribution of this notebook - Tensorflow version of on-the-fly CQT computation. Note that some of the operations used in CQT computation are not supported by TPU, therefore the implementation is not a TF layer but a function that runs on CPU.","metadata":{}},{"cell_type":"code","source":"def create_cqt_kernels(\n    q: float,\n    fs: float,\n    fmin: float,\n    n_bins: int = 84,\n    bins_per_octave: int = 12,\n    norm: float = 1,\n    window: str = \"hann\",\n    fmax: Optional[float] = None,\n    topbin_check: bool = True\n) -> Tuple[np.ndarray, int, np.ndarray, float]:\n    fft_len = 2 ** _nextpow2(np.ceil(q * fs / fmin))\n    \n    if (fmax is not None) and (n_bins is None):\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    elif (fmax is None) and (n_bins is not None):\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    else:\n        warnings.warn(\"If nmax is given, n_bins will be ignored\", SyntaxWarning)\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n        \n    if np.max(freqs) > fs / 2 and topbin_check:\n        raise ValueError(f\"The top bin {np.max(freqs)} Hz has exceeded the Nyquist frequency, \\\n                           please reduce the `n_bins`\")\n    \n    kernel = np.zeros((int(n_bins), int(fft_len)), dtype=np.complex64)\n    \n    length = np.ceil(q * fs / freqs)\n    for k in range(0, int(n_bins)):\n        freq = freqs[k]\n        l = np.ceil(q * fs / freq)\n        \n        if l % 2 == 1:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0)) - 1\n        else:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0))\n\n        sig = get_window(window, int(l), fftbins=True) * np.exp(\n            np.r_[-l // 2:l // 2] * 1j * 2 * np.pi * freq / fs) / l\n        \n        if norm:\n            kernel[k, start:start + int(l)] = sig / np.linalg.norm(sig, norm)\n        else:\n            kernel[k, start:start + int(l)] = sig\n    return kernel, fft_len, length, freqs\n\n\ndef _nextpow2(a: float) -> int:\n    return int(np.ceil(np.log2(a)))\n\n\ndef prepare_cqt_kernel(\n    sr=22050,\n    hop_length=512,\n    fmin=32.70,\n    fmax=None,\n    n_bins=84,\n    bins_per_octave=12,\n    norm=1,\n    filter_scale=1,\n    window=\"hann\"\n):\n    q = float(filter_scale) / (2 ** (1 / bins_per_octave) - 1)\n    print(q)\n    return create_cqt_kernels(q, sr, fmin, n_bins, bins_per_octave, norm, window, fmax)","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:35:52.658822Z","iopub.execute_input":"2021-08-13T14:35:52.659211Z","iopub.status.idle":"2021-08-13T14:35:52.680321Z","shell.execute_reply.started":"2021-08-13T14:35:52.659171Z","shell.execute_reply":"2021-08-13T14:35:52.679355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HOP_LENGTH = 16\ncqt_kernels, KERNEL_WIDTH, lengths, _ = prepare_cqt_kernel(\n    sr=2048,\n    hop_length=HOP_LENGTH,\n    fmin=20,\n    fmax=1024,\n    bins_per_octave=24)\nLENGTHS = tf.constant(lengths, dtype=tf.float32)\nCQT_KERNELS_REAL = tf.constant(np.swapaxes(cqt_kernels.real[:, np.newaxis, :], 0, 2))\nCQT_KERNELS_IMAG = tf.constant(np.swapaxes(cqt_kernels.imag[:, np.newaxis, :], 0, 2))\nPADDING = tf.constant([[0, 0],\n                        [KERNEL_WIDTH // 2, KERNEL_WIDTH // 2],\n                        [0, 0]])","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:36:08.796527Z","iopub.execute_input":"2021-08-13T14:36:08.796908Z","iopub.status.idle":"2021-08-13T14:36:08.861054Z","shell.execute_reply.started":"2021-08-13T14:36:08.796878Z","shell.execute_reply":"2021-08-13T14:36:08.859584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_cqt_image(wave, hop_length=16):\n    CQTs = []\n    for i in range(3):\n        x = wave[i]\n        x = tf.expand_dims(tf.expand_dims(x, 0), 2)\n        x = tf.pad(x, PADDING, \"REFLECT\")\n\n        CQT_real = tf.nn.conv1d(x, CQT_KERNELS_REAL, stride=hop_length, padding=\"VALID\")\n        CQT_imag = -tf.nn.conv1d(x, CQT_KERNELS_IMAG, stride=hop_length, padding=\"VALID\")\n        CQT_real *= tf.math.sqrt(LENGTHS)\n        CQT_imag *= tf.math.sqrt(LENGTHS)\n\n        CQT = tf.math.sqrt(tf.pow(CQT_real, 2) + tf.pow(CQT_imag, 2))\n        CQTs.append(CQT[0])\n    return tf.stack(CQTs, axis=2)","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:36:23.456919Z","iopub.execute_input":"2021-08-13T14:36:23.457298Z","iopub.status.idle":"2021-08-13T14:36:23.466342Z","shell.execute_reply.started":"2021-08-13T14:36:23.457264Z","shell.execute_reply":"2021-08-13T14:36:23.465283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], IMAGE_SIZE), tf.reshape(tf.cast(example[\"target\"], tf.float32), [1])\n\n\ndef read_unlabeled_tfrecord(example, return_image_id):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], IMAGE_SIZE), example[\"wave_id\"] if return_image_id else 0\n\n\ndef count_data_items(fileids):\n    return len(fileids) * 28000\n\n\ndef count_data_items_test(fileids):\n    return len(fileids) * 22600\n\n\ndef mixup(image, label, probability=0.5, aug_batch=64 * 8):\n    imgs = []\n    labs = []\n    for j in range(aug_batch):\n        p = tf.cast(tf.random.uniform([], 0, 1) <= probability, tf.float32)\n        k = tf.cast(tf.random.uniform([], 0, aug_batch), tf.int32)\n        a = tf.random.uniform([], 0, 1) * p\n\n        img1 = image[j]\n        img2 = image[k]\n        imgs.append((1 - a) * img1 + a * img2)\n        lab1 = label[j]\n        lab2 = label[k]\n        labs.append((1 - a) * lab1 + a * lab2)\n    image2 = tf.reshape(tf.stack(imgs), (aug_batch, IMAGE_SIZE, IMAGE_SIZE, 3))\n    label2 = tf.reshape(tf.stack(labs), (aug_batch,))\n    return image2, label2\n\n\ndef time_shift(img, shift=T_SHIFT):\n    if shift > 0:\n        T = IMAGE_SIZE\n        P = tf.random.uniform([],0,1)\n        SHIFT = tf.cast(T * P, tf.int32)\n        return tf.concat([img[-SHIFT:], img[:-SHIFT]], axis=0)\n    return img\n\n\ndef rotate(img, angle=R_ANGLE):\n    if angle > 0:\n        P = tf.random.uniform([],0,1)\n        A = tf.cast(angle * P, tf.float32)\n        return tfa.image.rotate(img, A)\n    return img\n\n\ndef spector_shift(img, shift=S_SHIFT):\n    if shift > 0:\n        T = IMAGE_SIZE\n        P = tf.random.uniform([],0,1)\n        SHIFT = tf.cast(T * P, tf.int32)\n        return tf.concat([img[:, -SHIFT:], img[:, :-SHIFT]], axis=1)\n    return img\n\ndef img_aug_f(img):\n    img = time_shift(img)\n    img = spector_shift(img)\n    # img = rotate(img)\n    return img\n\n\ndef imgs_aug_f(imgs, batch_size):\n    _imgs = []\n    DIM = IMAGE_SIZE\n    for j in range(batch_size):\n        _imgs.append(img_aug_f(imgs[j]))\n    return tf.reshape(tf.stack(_imgs),(batch_size,DIM,DIM,3))\n\n\ndef label_positive_shift(labels):\n    return labels * LABEL_POSITIVE_SHIFT\n\n\ndef aug_f(imgs, labels, batch_size):\n    imgs, label = mixup(imgs, labels, MIXUP_PROB, batch_size)\n    imgs = imgs_aug_f(imgs, batch_size)\n    return imgs, label_positive_shift(label)\n\n\ndef prepare_image(wave, dim=256):\n    wave = tf.reshape(tf.io.decode_raw(wave, tf.float64), (3, 4096))\n    normalized_waves = []\n    for i in range(3):\n        normalized_wave = wave[i] / tf.math.reduce_max(wave[i])\n        normalized_waves.append(normalized_wave)\n    wave = tf.stack(normalized_waves)\n    wave = tf.cast(wave, tf.float32)\n    image = create_cqt_image(wave, HOP_LENGTH)\n    image = tf.image.resize(image, size=(dim, dim))\n    return tf.reshape(image, (dim, dim, 3))\n\n\ndef get_dataset(files, batch_size=16, repeat=False, shuffle=False, aug=True, labeled=True, return_image_ids=True):\n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO, compression_type=\"GZIP\")\n    ds = ds.cache()\n\n    if repeat:\n        ds = ds.repeat()\n\n    if shuffle:\n        ds = ds.shuffle(1024 * 2)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n\n    if labeled:\n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_ids), num_parallel_calls=AUTO)\n\n    ds = ds.batch(batch_size * REPLICAS)\n    if aug:\n        ds = ds.map(lambda x, y: aug_f(x, y, batch_size * REPLICAS), num_parallel_calls=AUTO)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:36:40.631856Z","iopub.execute_input":"2021-08-13T14:36:40.63225Z","iopub.status.idle":"2021-08-13T14:36:40.665864Z","shell.execute_reply.started":"2021-08-13T14:36:40.632207Z","shell.execute_reply":"2021-08-13T14:36:40.664927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"def build_model(size=256, efficientnet_size=0, weights=\"imagenet\", count=0):\n    inputs = tf.keras.layers.Input(shape=(size, size, 3))\n    \n    efn_string= f\"EfficientNetB{efficientnet_size}\"\n    efn_layer = getattr(efn, efn_string)(input_shape=(size, size, 3), weights=weights, include_top=False)\n\n    x = efn_layer(inputs)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n\n    x = tf.keras.layers.Dropout(0.2)(x)\n    x = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n    model = tf.keras.Model(inputs=inputs, outputs=x)\n\n    lr_decayed_fn = tf.keras.experimental.CosineDecay(1e-3, count)\n    opt = tfa.optimizers.AdamW(lr_decayed_fn, learning_rate=1e-4)\n    loss = tf.keras.losses.BinaryCrossentropy()\n    model.compile(optimizer=opt, loss=loss, metrics=[\"AUC\"])\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:37:09.546016Z","iopub.execute_input":"2021-08-13T14:37:09.546533Z","iopub.status.idle":"2021-08-13T14:37:09.554879Z","shell.execute_reply.started":"2021-08-13T14:37:09.546498Z","shell.execute_reply":"2021-08-13T14:37:09.554033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback(batch_size=8, replicas=8):\n    lr_start   = 1e-4\n    lr_max     = 0.000015 * replicas * batch_size\n    lr_min     = 1e-7\n    lr_ramp_ep = 3\n    lr_sus_ep  = 0\n    lr_decay   = 0.7\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:37:20.514801Z","iopub.execute_input":"2021-08-13T14:37:20.515313Z","iopub.status.idle":"2021-08-13T14:37:20.523762Z","shell.execute_reply.started":"2021-08-13T14:37:20.515265Z","shell.execute_reply":"2021-08-13T14:37:20.522463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=1213)\noof_pred = []\noof_target = []\n\nfiles_train_all = np.array(all_files)","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:37:39.541993Z","iopub.execute_input":"2021-08-13T14:37:39.542592Z","iopub.status.idle":"2021-08-13T14:37:39.546829Z","shell.execute_reply.started":"2021-08-13T14:37:39.542538Z","shell.execute_reply":"2021-08-13T14:37:39.546034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold, (trn_idx, val_idx) in enumerate(kf.split(files_train_all)):\n    files_train = files_train_all[trn_idx]\n    files_valid = files_train_all[val_idx]\n\n    print(\"=\" * 120)\n    print(f\"Fold {fold}\")\n    print(\"=\" * 120)\n\n    train_image_count = count_data_items(files_train)\n    valid_image_count = count_data_items(files_valid)\n\n    tf.keras.backend.clear_session()\n    strategy, tpu_detected = auto_select_accelerator()\n    with strategy.scope():\n        model = build_model(\n            size=IMAGE_SIZE, \n            efficientnet_size=EFFICIENTNET_SIZE,\n            weights=WEIGHTS, \n            count=train_image_count // BATCH_SIZE // REPLICAS // 4)\n    \n    model_ckpt = tf.keras.callbacks.ModelCheckpoint(\n        str(SAVEDIR / f\"fold{fold}.h5\"), monitor=\"val_auc\", verbose=1, save_best_only=True,\n        save_weights_only=True, mode=\"max\", save_freq=\"epoch\"\n    )\n\n    history = model.fit(\n        get_dataset(files_train, batch_size=BATCH_SIZE, shuffle=True, repeat=True, aug=True),\n        epochs=EPOCHS,\n        callbacks=[model_ckpt, get_lr_callback(BATCH_SIZE, REPLICAS)],\n        steps_per_epoch=train_image_count // BATCH_SIZE // REPLICAS // 4,\n        validation_data=get_dataset(files_valid, batch_size=BATCH_SIZE * 4, repeat=False, shuffle=False, aug=False),\n        verbose=1\n    )\n\n    print(\"Loading best model...\")\n    model.load_weights(str(SAVEDIR / f\"fold{fold}.h5\"))\n\n    ds_valid = get_dataset(files_valid, labeled=False, return_image_ids=False, repeat=True, shuffle=False, batch_size=BATCH_SIZE * 2, aug=False)\n    STEPS = valid_image_count / BATCH_SIZE / 2 / REPLICAS\n    pred = model.predict(ds_valid, steps=STEPS, verbose=0)[:valid_image_count]\n    oof_pred.append(np.mean(pred.reshape((valid_image_count, 1), order=\"F\"), axis=1))\n\n    ds_valid = get_dataset(files_valid, repeat=False, labeled=True, return_image_ids=True, aug=False)\n    oof_target.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))\n\n    plt.figure(figsize=(8, 6))\n    sns.distplot(oof_pred[-1])\n    plt.show()\n\n    plt.figure(figsize=(15, 5))\n    plt.plot(\n        np.arange(len(history.history[\"auc\"])),\n        history.history[\"auc\"],\n        \"-o\",\n        label=\"Train auc\",\n        color=\"#ff7f0e\")\n    plt.plot(\n        np.arange(len(history.history[\"auc\"])),\n        history.history[\"val_auc\"],\n        \"-o\",\n        label=\"Val auc\",\n        color=\"#1f77b4\")\n    \n    x = np.argmax(history.history[\"val_auc\"])\n    y = np.max(history.history[\"val_auc\"])\n\n    xdist = plt.xlim()[1] - plt.xlim()[0]\n    ydist = plt.ylim()[1] - plt.ylim()[0]\n\n    plt.scatter(x, y, s=200, color=\"#1f77b4\")\n    plt.text(x - 0.03 * xdist, y - 0.13 * ydist, f\"max auc\\n{y}\", size=14)\n\n    plt.ylabel(\"auc\", size=14)\n    plt.xlabel(\"Epoch\", size=14)\n    plt.legend(loc=2)\n\n    plt2 = plt.gca().twinx()\n    plt2.plot(\n        np.arange(len(history.history[\"auc\"])),\n        history.history[\"loss\"],\n        \"-o\",\n        label=\"Train Loss\",\n        color=\"#2ca02c\")\n    plt2.plot(\n        np.arange(len(history.history[\"auc\"])),\n        history.history[\"val_loss\"],\n        \"-o\",\n        label=\"Val Loss\",\n        color=\"#d62728\")\n    \n    x = np.argmin(history.history[\"val_loss\"])\n    y = np.min(history.history[\"val_loss\"])\n    \n    ydist = plt.ylim()[1] - plt.ylim()[0]\n\n    plt.scatter(x, y, s=200, color=\"#d62728\")\n    plt.text(x - 0.03 * xdist, y + 0.05 * ydist, \"min loss\", size=14)\n\n    plt.ylabel(\"Loss\", size=14)\n    plt.title(f\"Fold {fold + 1} - Image Size {IMAGE_SIZE}, EfficientNetB{EFFICIENTNET_SIZE}\", size=18)\n\n    plt.legend(loc=3)\n    plt.savefig(OOFDIR / f\"fig{fold}.png\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:38:04.589718Z","iopub.execute_input":"2021-08-13T14:38:04.590361Z","iopub.status.idle":"2021-08-13T14:45:44.683312Z","shell.execute_reply.started":"2021-08-13T14:38:04.590308Z","shell.execute_reply":"2021-08-13T14:45:44.680449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## OOF","metadata":{}},{"cell_type":"code","source":"oof = np.concatenate(oof_pred)\ntrue = np.concatenate(oof_target)\nauc = roc_auc_score(y_true=true, y_score=oof)\nprint(f\"AUC: {auc:.5f}\")","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:45:44.684589Z","iopub.status.idle":"2021-08-13T14:45:44.685086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({\n    \"y_true\": true.reshape(-1),\n    \"y_pred\": oof\n})\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:45:44.68608Z","iopub.status.idle":"2021-08-13T14:45:44.686572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(OOFDIR / f\"oof.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-13T14:45:44.687545Z","iopub.status.idle":"2021-08-13T14:45:44.688006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}