{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## About this notebook\n\nThis notebook originally is the inference notebook for [G2Net: TF On-the-fly CQT TPU Training](https://www.kaggle.com/hidehisaarai1213/g2net-tf-on-the-fly-cqt-tpu-training).\n\nOn the fly CQT computation achieves better result compared to [Welf's Notebook](https://www.kaggle.com/miklgr500/g2net-efficientnetb1-tpu-evaluate) given the same image size and EfficientNet size, which means if you scale up the model or scale up the image size, you'll possibly get the best single model compared to publicly shared models.\nIt also allows you to make more variations for the input, which gives you a great advantage.\n\n### Updates\n\n* V3: Use the weights of V2 of the Training Notebook\n    * EfficientNetB0 -> EfficientNetB7\n   ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" my version is efficientnetv2\n [my training notebook](https://www.kaggle.com/dragonzhang/g2net-tf-effnetv2-cqt-tpu-training)","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Install Dependencies","metadata":{}},{"cell_type":"code","source":"#!pip install efficientnet tensorflow_addons > /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:34.309976Z","iopub.execute_input":"2021-08-17T07:48:34.310598Z","iopub.status.idle":"2021-08-17T07:48:34.315637Z","shell.execute_reply.started":"2021-08-17T07:48:34.310555Z","shell.execute_reply":"2021-08-17T07:48:34.314647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install   tensorflow_addons > /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:34.317269Z","iopub.execute_input":"2021-08-17T07:48:34.317808Z","iopub.status.idle":"2021-08-17T07:48:41.877285Z","shell.execute_reply.started":"2021-08-17T07:48:34.317772Z","shell.execute_reply":"2021-08-17T07:48:41.875948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -U git+https://github.com/leondgarse/keras_efficientnet_v2","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:41.880062Z","iopub.execute_input":"2021-08-17T07:48:41.880445Z","iopub.status.idle":"2021-08-17T07:48:51.062611Z","shell.execute_reply.started":"2021-08-17T07:48:41.880408Z","shell.execute_reply":"2021-08-17T07:48:51.061505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras_efficientnet_v2","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:51.065317Z","iopub.execute_input":"2021-08-17T07:48:51.065725Z","iopub.status.idle":"2021-08-17T07:48:51.071408Z","shell.execute_reply.started":"2021-08-17T07:48:51.065686Z","shell.execute_reply":"2021-08-17T07:48:51.069798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport math\nimport random\nimport re\nimport warnings\nfrom pathlib import Path\nfrom typing import Optional, Tuple\n\n#import efficientnet.tfkeras as efn\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom kaggle_datasets import KaggleDatasets\nfrom scipy.signal import get_window","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:51.073124Z","iopub.execute_input":"2021-08-17T07:48:51.073516Z","iopub.status.idle":"2021-08-17T07:48:51.910915Z","shell.execute_reply.started":"2021-08-17T07:48:51.073483Z","shell.execute_reply":"2021-08-17T07:48:51.909699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:51.912461Z","iopub.execute_input":"2021-08-17T07:48:51.912881Z","iopub.status.idle":"2021-08-17T07:48:51.922178Z","shell.execute_reply.started":"2021-08-17T07:48:51.912835Z","shell.execute_reply":"2021-08-17T07:48:51.921077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"#IMAGE_SIZE = 256\nIMAGE_SIZE = 480\nBATCH_SIZE = 32\nEFFICIENTNET_SIZE = 7\nWEIGHTS = \"imagenet\"\nMODEL_NAME=\"EfficientNetV2\"","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:51.923715Z","iopub.execute_input":"2021-08-17T07:48:51.924008Z","iopub.status.idle":"2021-08-17T07:48:51.932164Z","shell.execute_reply.started":"2021-08-17T07:48:51.923981Z","shell.execute_reply":"2021-08-17T07:48:51.93117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utilities","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\n\nset_seed(1213)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:51.933476Z","iopub.execute_input":"2021-08-17T07:48:51.933838Z","iopub.status.idle":"2021-08-17T07:48:51.945063Z","shell.execute_reply.started":"2021-08-17T07:48:51.933804Z","shell.execute_reply":"2021-08-17T07:48:51.944226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    TPU_DETECTED = False\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n        TPU_DETECTED = True\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n\n    return strategy, TPU_DETECTED","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:51.948103Z","iopub.execute_input":"2021-08-17T07:48:51.948481Z","iopub.status.idle":"2021-08-17T07:48:51.958909Z","shell.execute_reply.started":"2021-08-17T07:48:51.94845Z","shell.execute_reply":"2021-08-17T07:48:51.957422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy, tpu_detected = auto_select_accelerator()\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:51.960936Z","iopub.execute_input":"2021-08-17T07:48:51.961266Z","iopub.status.idle":"2021-08-17T07:48:57.240631Z","shell.execute_reply.started":"2021-08-17T07:48:51.961237Z","shell.execute_reply":"2021-08-17T07:48:57.239596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Loading","metadata":{}},{"cell_type":"code","source":"gcs_paths = []\nfor i, j in [(0, 4), (5, 9)]:\n    GCS_path = KaggleDatasets().get_gcs_path(f\"g2net-waveform-tfrecords-test-{i}-{j}\")\n    gcs_paths.append(GCS_path)\n    print(GCS_path)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:57.241773Z","iopub.execute_input":"2021-08-17T07:48:57.242039Z","iopub.status.idle":"2021-08-17T07:48:58.293674Z","shell.execute_reply.started":"2021-08-17T07:48:57.242014Z","shell.execute_reply":"2021-08-17T07:48:58.292395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_files = []\nfor path in gcs_paths:\n    all_files.extend(np.sort(np.array(tf.io.gfile.glob(path + \"/test*.tfrecords\"))))\n\nprint(\"test_files: \", len(all_files))","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:58.294934Z","iopub.execute_input":"2021-08-17T07:48:58.295246Z","iopub.status.idle":"2021-08-17T07:48:58.592705Z","shell.execute_reply.started":"2021-08-17T07:48:58.295215Z","shell.execute_reply":"2021-08-17T07:48:58.591595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset Preparation","metadata":{}},{"cell_type":"code","source":"def create_cqt_kernels(\n    q: float,\n    fs: float,\n    fmin: float,\n    n_bins: int = 84,\n    bins_per_octave: int = 12,\n    norm: float = 1,\n    window: str = \"hann\",\n    fmax: Optional[float] = None,\n    topbin_check: bool = True\n) -> Tuple[np.ndarray, int, np.ndarray, float]:\n    fft_len = 2 ** _nextpow2(np.ceil(q * fs / fmin))\n    \n    if (fmax is not None) and (n_bins is None):\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    elif (fmax is None) and (n_bins is not None):\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    else:\n        warnings.warn(\"If nmax is given, n_bins will be ignored\", SyntaxWarning)\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n        \n    if np.max(freqs) > fs / 2 and topbin_check:\n        raise ValueError(f\"The top bin {np.max(freqs)} Hz has exceeded the Nyquist frequency, \\\n                           please reduce the `n_bins`\")\n    \n    kernel = np.zeros((int(n_bins), int(fft_len)), dtype=np.complex64)\n    \n    length = np.ceil(q * fs / freqs)\n    for k in range(0, int(n_bins)):\n        freq = freqs[k]\n        l = np.ceil(q * fs / freq)\n        \n        if l % 2 == 1:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0)) - 1\n        else:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0))\n\n        sig = get_window(window, int(l), fftbins=True) * np.exp(\n            np.r_[-l // 2:l // 2] * 1j * 2 * np.pi * freq / fs) / l\n        \n        if norm:\n            kernel[k, start:start + int(l)] = sig / np.linalg.norm(sig, norm)\n        else:\n            kernel[k, start:start + int(l)] = sig\n    return kernel, fft_len, length, freqs\n\n\ndef _nextpow2(a: float) -> int:\n    return int(np.ceil(np.log2(a)))\n\n\ndef prepare_cqt_kernel(\n    sr=22050,\n    hop_length=512,\n    fmin=32.70,\n    fmax=None,\n    n_bins=84,\n    bins_per_octave=12,\n    norm=1,\n    filter_scale=1,\n    window=\"hann\"\n):\n    q = float(filter_scale) / (2 ** (1 / bins_per_octave) - 1)\n    print(q)\n    return create_cqt_kernels(q, sr, fmin, n_bins, bins_per_octave, norm, window, fmax)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:58.596243Z","iopub.execute_input":"2021-08-17T07:48:58.596745Z","iopub.status.idle":"2021-08-17T07:48:58.61662Z","shell.execute_reply.started":"2021-08-17T07:48:58.596712Z","shell.execute_reply":"2021-08-17T07:48:58.615391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HOP_LENGTH = 16\ncqt_kernels, KERNEL_WIDTH, lengths, _ = prepare_cqt_kernel(\n    sr=2048,\n    hop_length=HOP_LENGTH,\n    fmin=20,\n    fmax=1024,\n    bins_per_octave=24)\nLENGTHS = tf.constant(lengths, dtype=tf.float32)\nCQT_KERNELS_REAL = tf.constant(np.swapaxes(cqt_kernels.real[:, np.newaxis, :], 0, 2))\nCQT_KERNELS_IMAG = tf.constant(np.swapaxes(cqt_kernels.imag[:, np.newaxis, :], 0, 2))\nPADDING = tf.constant([[0, 0],\n                        [KERNEL_WIDTH // 2, KERNEL_WIDTH // 2],\n                        [0, 0]])","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:58.617965Z","iopub.execute_input":"2021-08-17T07:48:58.618314Z","iopub.status.idle":"2021-08-17T07:48:58.713505Z","shell.execute_reply.started":"2021-08-17T07:48:58.618244Z","shell.execute_reply":"2021-08-17T07:48:58.712486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_cqt_image(wave, hop_length=16):\n    CQTs = []\n    for i in range(3):\n        x = wave[i]\n        x = tf.expand_dims(tf.expand_dims(x, 0), 2)\n        x = tf.pad(x, PADDING, \"REFLECT\")\n\n        CQT_real = tf.nn.conv1d(x, CQT_KERNELS_REAL, stride=hop_length, padding=\"VALID\")\n        CQT_imag = -tf.nn.conv1d(x, CQT_KERNELS_IMAG, stride=hop_length, padding=\"VALID\")\n        CQT_real *= tf.math.sqrt(LENGTHS)\n        CQT_imag *= tf.math.sqrt(LENGTHS)\n\n        CQT = tf.math.sqrt(tf.pow(CQT_real, 2) + tf.pow(CQT_imag, 2))\n        CQTs.append(CQT[0])\n    return tf.stack(CQTs, axis=2)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:58.714858Z","iopub.execute_input":"2021-08-17T07:48:58.715146Z","iopub.status.idle":"2021-08-17T07:48:58.723989Z","shell.execute_reply.started":"2021-08-17T07:48:58.715117Z","shell.execute_reply":"2021-08-17T07:48:58.722604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], IMAGE_SIZE), tf.reshape(tf.cast(example[\"target\"], tf.float32), [1])\n\n\ndef read_unlabeled_tfrecord(example, return_image_id):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], IMAGE_SIZE), example[\"wave_id\"] if return_image_id else 0\n\n\ndef count_data_items(fileids):\n    return len(fileids) * 28000\n\n\ndef count_data_items_test(fileids):\n    return len(fileids) * 22600\n\n\ndef prepare_image(wave, dim=256):\n    wave = tf.reshape(tf.io.decode_raw(wave, tf.float64), (3, 4096))\n    normalized_waves = []\n    for i in range(3):\n        normalized_wave = wave[i] / tf.math.reduce_max(wave[i])\n        normalized_waves.append(normalized_wave)\n    wave = tf.stack(normalized_waves)\n    wave = tf.cast(wave, tf.float32)\n    image = create_cqt_image(wave, HOP_LENGTH)\n    image = tf.image.resize(image, size=(dim, dim))\n    return tf.reshape(image, (dim, dim, 3))\n\n\ndef get_dataset(files, batch_size=16, repeat=False, shuffle=False, aug=True, labeled=True, return_image_ids=True):\n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO, compression_type=\"GZIP\")\n    ds = ds.cache()\n\n    if repeat:\n        ds = ds.repeat()\n\n    if shuffle:\n        ds = ds.shuffle(1024 * 2)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n\n    if labeled:\n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_ids), num_parallel_calls=AUTO)\n\n    ds = ds.batch(batch_size * REPLICAS)\n    if aug:\n        ds = ds.map(lambda x, y: aug_f(x, y, batch_size * REPLICAS), num_parallel_calls=AUTO)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:58.725501Z","iopub.execute_input":"2021-08-17T07:48:58.725849Z","iopub.status.idle":"2021-08-17T07:48:58.747495Z","shell.execute_reply.started":"2021-08-17T07:48:58.725816Z","shell.execute_reply":"2021-08-17T07:48:58.745944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef build_model(size=256, efficientnet_size=0, weights=\"imagenet\", count=0):\n    inputs = tf.keras.layers.Input(shape=(size, size, 3))\n    \n    efn_string= f\"EfficientNetB{efficientnet_size}\"\n    efn_layer = getattr(efn, efn_string)(input_shape=(size, size, 3), weights=weights, include_top=False)\n\n    x = efn_layer(inputs)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n\n    x = tf.keras.layers.Dropout(0.2)(x)\n    x = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n    model = tf.keras.Model(inputs=inputs, outputs=x)\n\n    lr_decayed_fn = tf.keras.experimental.CosineDecay(1e-3, count)\n    opt = tfa.optimizers.AdamW(lr_decayed_fn, learning_rate=1e-4)\n    loss = tf.keras.losses.BinaryCrossentropy()\n    model.compile(optimizer=opt, loss=loss, metrics=[\"AUC\"])\n    return model\n    \n    '''","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:58.74922Z","iopub.execute_input":"2021-08-17T07:48:58.749636Z","iopub.status.idle":"2021-08-17T07:48:58.768079Z","shell.execute_reply.started":"2021-08-17T07:48:58.749602Z","shell.execute_reply":"2021-08-17T07:48:58.766839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef build_model(size=256, effnet_name=\"EfficientNetV2M\", weights=\"imagenet\", count=0):\n    inputs = tf.keras.layers.Input(shape=(size, size, 3))\n    \n    #efn_string= f\"EfficientNetB{efficientnet_size}\"\n    efn_string= f\"{effnet_name}\"\n    #efn_layer = getattr(efn, efn_string)(input_shape=(size, size, 3), weights=weights, include_top=False)\n    efn_layer=keras_efficientnet_v2.EfficientNetV2M(input_shape=(None, None, 3), drop_connect_rate=0.2, num_classes=0, pretrained=\"imagenet21k-ft1k\")\n\n    x = efn_layer(inputs)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n\n    x = tf.keras.layers.Dropout(0.2)(x)\n    x = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n    model = tf.keras.Model(inputs=inputs, outputs=x)\n\n    lr_decayed_fn = tf.keras.experimental.CosineDecay(1e-3, count)\n    opt = tfa.optimizers.AdamW(lr_decayed_fn, learning_rate=1e-4)\n    loss = tf.keras.losses.BinaryCrossentropy()\n    model.compile(optimizer=opt, loss=loss, metrics=[\"AUC\"])\n    return model\n'''","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:58.770257Z","iopub.execute_input":"2021-08-17T07:48:58.770636Z","iopub.status.idle":"2021-08-17T07:48:58.785936Z","shell.execute_reply.started":"2021-08-17T07:48:58.770604Z","shell.execute_reply":"2021-08-17T07:48:58.784903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef build_model(size=256, effnet_name=\"EfficientNetV2\", weights=\"imagenet\", count=0):\n    inputs = tf.keras.layers.Input(shape=(size, size, 3))\n    \n    #efn_string= f\"EfficientNetB{efficientnet_size}\"\n    #efn_string= f\"{effnet_name}\"\n    #efn_layer = getattr(efn, efn_string)(input_shape=(size, size, 3), weights=weights, include_top=False)\n    efn_layer=keras_efficientnet_v2.EfficientNetV2(\"XL\",input_shape=(None, None, 3), drop_connect_rate=0.2, num_classes=0, pretrained=\"imagenet21k-ft1k\")\n\n    x = efn_layer(inputs)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n\n    x = tf.keras.layers.Dropout(0.2)(x)\n    x = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n    model = tf.keras.Model(inputs=inputs, outputs=x)\n\n    lr_decayed_fn = tf.keras.experimental.CosineDecay(1e-3, count)\n    opt = tfa.optimizers.AdamW(lr_decayed_fn, learning_rate=1e-4)\n    loss = tf.keras.losses.BinaryCrossentropy()\n    model.compile(optimizer=opt, loss=loss, metrics=[\"AUC\"])\n    return model\n\n'''","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#def build_model(size=256, effnet_name=\"EfficientNetV2\", weights=\"imagenet\", count=0):\ndef build_model(size=IMAGE_SIZE, effnet_name=\"EfficientNetV2\", weights=\"imagenet\", count=0):\n    inputs = tf.keras.layers.Input(shape=(size, size, 3))\n    \n    #efn_string= f\"EfficientNetB{efficientnet_size}\"\n    #efn_string= f\"{effnet_name}\"\n    #efn_layer = getattr(efn, efn_string)(input_shape=(size, size, 3), weights=weights, include_top=False)\n    efn_layer=keras_efficientnet_v2.EfficientNetV2(\"L\",input_shape=(None, None, 3), drop_connect_rate=0.2, num_classes=0, pretrained=\"imagenet21k-ft1k\")\n\n    x = efn_layer(inputs)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n\n    x = tf.keras.layers.Dropout(0.2)(x)\n    x = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n    model = tf.keras.Model(inputs=inputs, outputs=x)\n\n    lr_decayed_fn = tf.keras.experimental.CosineDecay(1e-3, count)\n    opt = tfa.optimizers.AdamW(lr_decayed_fn, learning_rate=1e-4)\n    loss = tf.keras.losses.BinaryCrossentropy()\n    model.compile(optimizer=opt, loss=loss, metrics=[\"AUC\"])\n    return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"files_test_all = np.array(all_files)\nall_test_preds = []","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:48:58.787298Z","iopub.execute_input":"2021-08-17T07:48:58.787806Z","iopub.status.idle":"2021-08-17T07:48:58.800111Z","shell.execute_reply.started":"2021-08-17T07:48:58.787758Z","shell.execute_reply":"2021-08-17T07:48:58.798984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model = build_model(\n        size=IMAGE_SIZE,\n        #efficientnet_size=EFFICIENTNET_SIZE,\n        weights=WEIGHTS,\n        count=0)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:49:48.642185Z","iopub.execute_input":"2021-08-17T07:49:48.642754Z","iopub.status.idle":"2021-08-17T07:50:44.709982Z","shell.execute_reply.started":"2021-08-17T07:49:48.642717Z","shell.execute_reply":"2021-08-17T07:50:44.708785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#weights_dir = Path(\"../input/g2net-tf-on-the-fly-cqt-tpu-training/models/\")\n#weights_dir = Path(\"../input/my-colab-tpu-trained/content/models/\")\nweights_dir = Path(\"../input/effnetv2-g2net-tpu-train/models/\")\n#for i in range(4):\n#for i in range(2):\nfor i in range(1):\n    print(f\"Load weight for Fold {i + 1} model\")\n    model.load_weights(weights_dir / f\"fold{i}.h5\")\n    \n    ds_test = get_dataset(files_test_all, batch_size=BATCH_SIZE * 2, repeat=True, shuffle=False, aug=False, labeled=False, return_image_ids=False)\n    STEPS = count_data_items_test(files_test_all) / BATCH_SIZE / 2 / REPLICAS\n    pred = model.predict(ds_test, verbose=1, steps=STEPS)[:count_data_items_test(files_test_all)]\n    all_test_preds.append(pred.reshape(-1))","metadata":{"execution":{"iopub.status.busy":"2021-08-17T07:55:22.145079Z","iopub.execute_input":"2021-08-17T07:55:22.145711Z","iopub.status.idle":"2021-08-17T08:04:55.746096Z","shell.execute_reply.started":"2021-08-17T07:55:22.14566Z","shell.execute_reply":"2021-08-17T08:04:55.745187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_test = get_dataset(files_test_all, batch_size=BATCH_SIZE * 2, repeat=False, shuffle=False, aug=False, labeled=False, return_image_ids=True)\nfile_ids = np.array([target.numpy() for img, target in iter(ds_test.unbatch())])","metadata":{"execution":{"iopub.status.busy":"2021-08-17T08:04:55.747912Z","iopub.execute_input":"2021-08-17T08:04:55.74854Z","iopub.status.idle":"2021-08-17T08:12:36.112897Z","shell.execute_reply.started":"2021-08-17T08:04:55.748493Z","shell.execute_reply":"2021-08-17T08:12:36.111711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred = np.zeros_like(all_test_preds[0])\nfor i in range(len(all_test_preds)):\n    test_pred += all_test_preds[i] / len(all_test_preds)\n    \ntest_df = pd.DataFrame({\n    \"id\": [i.decode(\"UTF-8\") for i in file_ids],\n    \"target\": test_pred\n})\n\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T08:12:36.114767Z","iopub.execute_input":"2021-08-17T08:12:36.115076Z","iopub.status.idle":"2021-08-17T08:12:36.254779Z","shell.execute_reply.started":"2021-08-17T08:12:36.115047Z","shell.execute_reply":"2021-08-17T08:12:36.253437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T08:12:36.256559Z","iopub.execute_input":"2021-08-17T08:12:36.256898Z","iopub.status.idle":"2021-08-17T08:12:36.986477Z","shell.execute_reply.started":"2021-08-17T08:12:36.256864Z","shell.execute_reply":"2021-08-17T08:12:36.985174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}