{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## About this notebook\n\nThis notebook is the inference notebook for [G2Net: TF On-the-fly CQT TPU Training](https://www.kaggle.com/hidehisaarai1213/g2net-tf-on-the-fly-cqt-tpu-training).\n\nOn the fly CQT computation achieves better result compared to [Welf's Notebook](https://www.kaggle.com/miklgr500/g2net-efficientnetb1-tpu-evaluate) given the same image size and EfficientNet size, which means if you scale up the model or scale up the image size, you'll possibly get the best single model compared to publicly shared models.\nIt also allows you to make more variations for the input, which gives you a great advantage.\n\n### Updates\n\n* V3: Use the weights of V2 of the Training Notebook\n    * EfficientNetB0 -> EfficientNetB7","metadata":{}},{"cell_type":"markdown","source":"## Install Dependencies","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet tensorflow_addons > /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:57:42.972798Z","iopub.execute_input":"2021-08-14T09:57:42.973470Z","iopub.status.idle":"2021-08-14T09:57:51.963313Z","shell.execute_reply.started":"2021-08-14T09:57:42.973380Z","shell.execute_reply":"2021-08-14T09:57:51.962251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport math\nimport random\nimport re\nimport warnings\nfrom pathlib import Path\nfrom typing import Optional, Tuple\n\nimport efficientnet.tfkeras as efn\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom kaggle_datasets import KaggleDatasets\nfrom scipy.signal import get_window","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:58:19.015788Z","iopub.execute_input":"2021-08-14T09:58:19.016167Z","iopub.status.idle":"2021-08-14T09:58:25.620638Z","shell.execute_reply.started":"2021-08-14T09:58:19.016121Z","shell.execute_reply":"2021-08-14T09:58:25.619582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:58:25.621853Z","iopub.execute_input":"2021-08-14T09:58:25.622116Z","iopub.status.idle":"2021-08-14T09:58:25.629943Z","shell.execute_reply.started":"2021-08-14T09:58:25.622091Z","shell.execute_reply":"2021-08-14T09:58:25.629002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"IMAGE_SIZE = 256\nBATCH_SIZE = 32\nEFFICIENTNET_SIZE = 7\nWEIGHTS = \"imagenet\"","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:58:31.233232Z","iopub.execute_input":"2021-08-14T09:58:31.233572Z","iopub.status.idle":"2021-08-14T09:58:31.237416Z","shell.execute_reply.started":"2021-08-14T09:58:31.233544Z","shell.execute_reply":"2021-08-14T09:58:31.236686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utilities","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\n\nset_seed(1213)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:58:34.347910Z","iopub.execute_input":"2021-08-14T09:58:34.348309Z","iopub.status.idle":"2021-08-14T09:58:34.354254Z","shell.execute_reply.started":"2021-08-14T09:58:34.348270Z","shell.execute_reply":"2021-08-14T09:58:34.353196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    TPU_DETECTED = False\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n        TPU_DETECTED = True\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n\n    return strategy, TPU_DETECTED","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:58:36.767744Z","iopub.execute_input":"2021-08-14T09:58:36.768369Z","iopub.status.idle":"2021-08-14T09:58:36.773901Z","shell.execute_reply.started":"2021-08-14T09:58:36.768321Z","shell.execute_reply":"2021-08-14T09:58:36.773292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy, tpu_detected = auto_select_accelerator()\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:58:37.597057Z","iopub.execute_input":"2021-08-14T09:58:37.597465Z","iopub.status.idle":"2021-08-14T09:58:43.558707Z","shell.execute_reply.started":"2021-08-14T09:58:37.597431Z","shell.execute_reply":"2021-08-14T09:58:43.557731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Loading","metadata":{}},{"cell_type":"code","source":"gcs_paths = []\nfor i, j in [(0, 4), (5, 9)]:\n    GCS_path = KaggleDatasets().get_gcs_path(f\"g2net-waveform-tfrecords-test-{i}-{j}\")\n    gcs_paths.append(GCS_path)\n    print(GCS_path)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:58:43.560060Z","iopub.execute_input":"2021-08-14T09:58:43.560369Z","iopub.status.idle":"2021-08-14T09:58:44.353052Z","shell.execute_reply.started":"2021-08-14T09:58:43.560340Z","shell.execute_reply":"2021-08-14T09:58:44.351968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_files = []\nfor path in gcs_paths:\n    all_files.extend(np.sort(np.array(tf.io.gfile.glob(path + \"/test*.tfrecords\"))))\n\nprint(\"test_files: \", len(all_files))","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:58:44.355013Z","iopub.execute_input":"2021-08-14T09:58:44.355461Z","iopub.status.idle":"2021-08-14T09:58:44.510326Z","shell.execute_reply.started":"2021-08-14T09:58:44.355415Z","shell.execute_reply":"2021-08-14T09:58:44.509387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset Preparation","metadata":{}},{"cell_type":"code","source":"def create_cqt_kernels(\n    q: float,\n    fs: float,\n    fmin: float,\n    n_bins: int = 84,\n    bins_per_octave: int = 12,\n    norm: float = 1,\n    window: str = \"hann\",\n    fmax: Optional[float] = None,\n    topbin_check: bool = True\n) -> Tuple[np.ndarray, int, np.ndarray, float]:\n    fft_len = 2 ** _nextpow2(np.ceil(q * fs / fmin))\n    \n    if (fmax is not None) and (n_bins is None):\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    elif (fmax is None) and (n_bins is not None):\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    else:\n        warnings.warn(\"If nmax is given, n_bins will be ignored\", SyntaxWarning)\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n        \n    if np.max(freqs) > fs / 2 and topbin_check:\n        raise ValueError(f\"The top bin {np.max(freqs)} Hz has exceeded the Nyquist frequency, \\\n                           please reduce the `n_bins`\")\n    \n    kernel = np.zeros((int(n_bins), int(fft_len)), dtype=np.complex64)\n    \n    length = np.ceil(q * fs / freqs)\n    for k in range(0, int(n_bins)):\n        freq = freqs[k]\n        l = np.ceil(q * fs / freq)\n        \n        if l % 2 == 1:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0)) - 1\n        else:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0))\n\n        sig = get_window(window, int(l), fftbins=True) * np.exp(\n            np.r_[-l // 2:l // 2] * 1j * 2 * np.pi * freq / fs) / l\n        \n        if norm:\n            kernel[k, start:start + int(l)] = sig / np.linalg.norm(sig, norm)\n        else:\n            kernel[k, start:start + int(l)] = sig\n    return kernel, fft_len, length, freqs\n\n\ndef _nextpow2(a: float) -> int:\n    return int(np.ceil(np.log2(a)))\n\n\ndef prepare_cqt_kernel(\n    sr=22050,\n    hop_length=512,\n    fmin=32.70,\n    fmax=None,\n    n_bins=84,\n    bins_per_octave=12,\n    norm=1,\n    filter_scale=1,\n    window=\"hann\"\n):\n    q = float(filter_scale) / (2 ** (1 / bins_per_octave) - 1)\n    print(q)\n    return create_cqt_kernels(q, sr, fmin, n_bins, bins_per_octave, norm, window, fmax)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:58:44.511779Z","iopub.execute_input":"2021-08-14T09:58:44.512066Z","iopub.status.idle":"2021-08-14T09:58:44.530440Z","shell.execute_reply.started":"2021-08-14T09:58:44.512039Z","shell.execute_reply":"2021-08-14T09:58:44.529249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HOP_LENGTH = 16\ncqt_kernels, KERNEL_WIDTH, lengths, _ = prepare_cqt_kernel(\n    sr=2048,\n    hop_length=HOP_LENGTH,\n    fmin=20,\n    fmax=1024,\n    bins_per_octave=24)\nLENGTHS = tf.constant(lengths, dtype=tf.float32)\nCQT_KERNELS_REAL = tf.constant(np.swapaxes(cqt_kernels.real[:, np.newaxis, :], 0, 2))\nCQT_KERNELS_IMAG = tf.constant(np.swapaxes(cqt_kernels.imag[:, np.newaxis, :], 0, 2))\nPADDING = tf.constant([[0, 0],\n                        [KERNEL_WIDTH // 2, KERNEL_WIDTH // 2],\n                        [0, 0]])","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:58:45.187553Z","iopub.execute_input":"2021-08-14T09:58:45.187897Z","iopub.status.idle":"2021-08-14T09:58:45.253149Z","shell.execute_reply.started":"2021-08-14T09:58:45.187868Z","shell.execute_reply":"2021-08-14T09:58:45.251402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_cqt_image(wave, hop_length=16):\n    CQTs = []\n    for i in range(3):\n        x = wave[i]\n        x = tf.expand_dims(tf.expand_dims(x, 0), 2)\n        x = tf.pad(x, PADDING, \"REFLECT\")\n\n        CQT_real = tf.nn.conv1d(x, CQT_KERNELS_REAL, stride=hop_length, padding=\"VALID\")\n        CQT_imag = -tf.nn.conv1d(x, CQT_KERNELS_IMAG, stride=hop_length, padding=\"VALID\")\n        CQT_real *= tf.math.sqrt(LENGTHS)\n        CQT_imag *= tf.math.sqrt(LENGTHS)\n\n        CQT = tf.math.sqrt(tf.pow(CQT_real, 2) + tf.pow(CQT_imag, 2))\n        CQTs.append(CQT[0])\n    return tf.stack(CQTs, axis=2)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:58:45.867862Z","iopub.execute_input":"2021-08-14T09:58:45.868237Z","iopub.status.idle":"2021-08-14T09:58:45.875966Z","shell.execute_reply.started":"2021-08-14T09:58:45.868202Z","shell.execute_reply":"2021-08-14T09:58:45.874934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], IMAGE_SIZE), tf.reshape(tf.cast(example[\"target\"], tf.float32), [1])\n\n\ndef read_unlabeled_tfrecord(example, return_image_id):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], IMAGE_SIZE), example[\"wave_id\"] if return_image_id else 0\n\n\ndef count_data_items(fileids):\n    return len(fileids) * 28000\n\n\ndef count_data_items_test(fileids):\n    return len(fileids) * 22600\n\n\ndef prepare_image(wave, dim=256):\n    wave = tf.reshape(tf.io.decode_raw(wave, tf.float64), (3, 4096))\n    normalized_waves = []\n    for i in range(3):\n        normalized_wave = wave[i] / tf.math.reduce_max(wave[i])\n        normalized_waves.append(normalized_wave)\n    wave = tf.stack(normalized_waves)\n    wave = tf.cast(wave, tf.float32)\n    image = create_cqt_image(wave, HOP_LENGTH)\n    image = tf.image.resize(image, size=(dim, dim))\n    return tf.reshape(image, (dim, dim, 3))\n\n\ndef get_dataset(files, batch_size=16, repeat=False, shuffle=False, aug=True, labeled=True, return_image_ids=True):\n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO, compression_type=\"GZIP\")\n    ds = ds.cache()\n\n    if repeat:\n        ds = ds.repeat()\n\n    if shuffle:\n        ds = ds.shuffle(1024 * 2)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n\n    if labeled:\n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_ids), num_parallel_calls=AUTO)\n\n    ds = ds.batch(batch_size * REPLICAS)\n    if aug:\n        ds = ds.map(lambda x, y: aug_f(x, y, batch_size * REPLICAS), num_parallel_calls=AUTO)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:58:47.754180Z","iopub.execute_input":"2021-08-14T09:58:47.754539Z","iopub.status.idle":"2021-08-14T09:58:47.771766Z","shell.execute_reply.started":"2021-08-14T09:58:47.754504Z","shell.execute_reply":"2021-08-14T09:58:47.770744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"def build_model(size=256, efficientnet_size=0, weights=\"imagenet\", count=0):\n    inputs = tf.keras.layers.Input(shape=(size, size, 3))\n    \n    efn_string= f\"EfficientNetB{efficientnet_size}\"\n    efn_layer = getattr(efn, efn_string)(input_shape=(size, size, 3), weights=weights, include_top=False)\n\n    x = efn_layer(inputs)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n\n    x = tf.keras.layers.Dropout(0.2)(x)\n    x = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n    model = tf.keras.Model(inputs=inputs, outputs=x)\n\n    lr_decayed_fn = tf.keras.experimental.CosineDecay(1e-3, count)\n    opt = tfa.optimizers.AdamW(lr_decayed_fn, learning_rate=1e-4)\n    loss = tf.keras.losses.BinaryCrossentropy()\n    model.compile(optimizer=opt, loss=loss, metrics=[\"AUC\"])\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:58:49.667767Z","iopub.execute_input":"2021-08-14T09:58:49.668087Z","iopub.status.idle":"2021-08-14T09:58:49.676874Z","shell.execute_reply.started":"2021-08-14T09:58:49.668058Z","shell.execute_reply":"2021-08-14T09:58:49.675620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"files_test_all = np.array(all_files)\nall_test_preds = []","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:58:51.778664Z","iopub.execute_input":"2021-08-14T09:58:51.779051Z","iopub.status.idle":"2021-08-14T09:58:51.783484Z","shell.execute_reply.started":"2021-08-14T09:58:51.779016Z","shell.execute_reply":"2021-08-14T09:58:51.782495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model = build_model(\n        size=IMAGE_SIZE,\n        efficientnet_size=EFFICIENTNET_SIZE,\n        weights=WEIGHTS,\n        count=0)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:58:52.986313Z","iopub.execute_input":"2021-08-14T09:58:52.986684Z","iopub.status.idle":"2021-08-14T09:59:48.343442Z","shell.execute_reply.started":"2021-08-14T09:58:52.986647Z","shell.execute_reply":"2021-08-14T09:59:48.342370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights_dir = Path(\"../input/g2net-tf-on-the-fly-cqt-tpu-training/models/\")\nfor i in range(4):\n    print(f\"Load weight for Fold {i + 1} model\")\n    model.load_weights(weights_dir / f\"fold{i}.h5\")\n    \n    ds_test = get_dataset(files_test_all, batch_size=BATCH_SIZE * 2, repeat=True, shuffle=False, aug=False, labeled=False, return_image_ids=False)\n    STEPS = count_data_items_test(files_test_all) / BATCH_SIZE / 2 / REPLICAS\n    pred = model.predict(ds_test, verbose=1, steps=STEPS)[:count_data_items_test(files_test_all)]\n    all_test_preds.append(pred.reshape(-1))","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:59:48.346846Z","iopub.execute_input":"2021-08-14T09:59:48.347560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_test = get_dataset(files_test_all, batch_size=BATCH_SIZE * 2, repeat=False, shuffle=False, aug=False, labeled=False, return_image_ids=True)\nfile_ids = np.array([target.numpy() for img, target in iter(ds_test.unbatch())])","metadata":{"execution":{"iopub.status.busy":"2021-08-14T01:49:38.179256Z","iopub.execute_input":"2021-08-14T01:49:38.179825Z","iopub.status.idle":"2021-08-14T01:57:46.527417Z","shell.execute_reply.started":"2021-08-14T01:49:38.179756Z","shell.execute_reply":"2021-08-14T01:57:46.526222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred = np.zeros_like(all_test_preds[0])\nfor i in range(len(all_test_preds)):\n    test_pred += all_test_preds[i] / len(all_test_preds)\n    \ntest_df = pd.DataFrame({\n    \"id\": [i.decode(\"UTF-8\") for i in file_ids],\n    \"target\": test_pred\n})\n\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-14T01:57:46.52927Z","iopub.execute_input":"2021-08-14T01:57:46.529607Z","iopub.status.idle":"2021-08-14T01:57:46.661525Z","shell.execute_reply.started":"2021-08-14T01:57:46.529576Z","shell.execute_reply":"2021-08-14T01:57:46.660011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T01:57:46.663002Z","iopub.execute_input":"2021-08-14T01:57:46.663369Z","iopub.status.idle":"2021-08-14T01:57:47.390367Z","shell.execute_reply.started":"2021-08-14T01:57:46.663335Z","shell.execute_reply":"2021-08-14T01:57:47.388972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}