{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## About this notebook\n\nThis notebook is based on [CQT G2Net EfficientNetB1[TPU Training]](https://www.kaggle.com/miklgr500/cqt-g2net-efficientnetb7-tpu-training-w-b) by [Welf Crozzo](https://www.kaggle.com/miklgr500) and [nnAudio Constant Q-transform Demonstration](https://www.kaggle.com/atamazian/nnaudio-constant-q-transform-demonstration) by [Araik Tamazian](https://www.kaggle.com/atamazian).\n\nThis notebook use Constant Q-Transform for feature extraction and EfficientNetB0 for classification. The whole pipeline is implemented with Tensorflow, and the training process runs on TPU.\n\nThe main difference between this notebook and Welf's notebook is the use of on-the-fly CQT computation implemented with Tensorflow, which is similar to the idea of [nnAudio](https://github.com/KinWaiCheuk/nnAudio)'s [CQT1992v2](https://kinwaicheuk.github.io/nnAudio/_autosummary/nnAudio.Spectrogram.CQT1992v2.html?highlight=cqt1992v2#nnAudio.Spectrogram.CQT1992v2) layer.\n\n* [Inference Notebook](https://www.kaggle.com/hidehisaarai1213/g2net-tf-on-the-fly-cqt-tpu-inference)","metadata":{}},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\n","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:09.236292Z","iopub.execute_input":"2021-09-03T06:03:09.236986Z","iopub.status.idle":"2021-09-03T06:03:09.253392Z","shell.execute_reply.started":"2021-09-03T06:03:09.236847Z","shell.execute_reply":"2021-09-03T06:03:09.252432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p1= KaggleDatasets().get_gcs_path('g2net-waveform-tfrecords-train-0-4')\nprint(p1)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:09.25478Z","iopub.execute_input":"2021-09-03T06:03:09.255255Z","iopub.status.idle":"2021-09-03T06:03:09.652908Z","shell.execute_reply.started":"2021-09-03T06:03:09.255214Z","shell.execute_reply":"2021-09-03T06:03:09.65216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p2= KaggleDatasets().get_gcs_path('g2net-waveform-tfrecords-train-5-9')\nprint(p2)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:09.654543Z","iopub.execute_input":"2021-09-03T06:03:09.654818Z","iopub.status.idle":"2021-09-03T06:03:10.021052Z","shell.execute_reply.started":"2021-09-03T06:03:09.65479Z","shell.execute_reply":"2021-09-03T06:03:10.019985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p3= KaggleDatasets().get_gcs_path('g2net-waveform-tfrecords-train-10-14')\nprint(p3)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:10.022914Z","iopub.execute_input":"2021-09-03T06:03:10.023536Z","iopub.status.idle":"2021-09-03T06:03:10.447069Z","shell.execute_reply.started":"2021-09-03T06:03:10.023491Z","shell.execute_reply":"2021-09-03T06:03:10.446138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p4= KaggleDatasets().get_gcs_path('g2net-waveform-tfrecords-train-15-19')\nprint(p4)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:10.448581Z","iopub.execute_input":"2021-09-03T06:03:10.448985Z","iopub.status.idle":"2021-09-03T06:03:10.850894Z","shell.execute_reply.started":"2021-09-03T06:03:10.448944Z","shell.execute_reply":"2021-09-03T06:03:10.849782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n","metadata":{}},{"cell_type":"markdown","source":"**my inference notebook  **\n","metadata":{}},{"cell_type":"markdown","source":"my inference notebook \n\nhttps://www.kaggle.com/dragonzhang/g2net-tf-effnetv2-cqt-tpu-inference","metadata":{}},{"cell_type":"markdown","source":"## Install Dependencies","metadata":{}},{"cell_type":"code","source":"#!pip install efficientnet tensorflow_addons > /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:10.852435Z","iopub.execute_input":"2021-09-03T06:03:10.852854Z","iopub.status.idle":"2021-09-03T06:03:10.857278Z","shell.execute_reply.started":"2021-09-03T06:03:10.852809Z","shell.execute_reply":"2021-09-03T06:03:10.856402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install tensorflow_addons > /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:10.858265Z","iopub.execute_input":"2021-09-03T06:03:10.858844Z","iopub.status.idle":"2021-09-03T06:03:19.321755Z","shell.execute_reply.started":"2021-09-03T06:03:10.858814Z","shell.execute_reply":"2021-09-03T06:03:19.32066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  using efficientnet v2","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:19.431079Z","iopub.execute_input":"2021-09-03T06:03:19.431475Z","iopub.status.idle":"2021-09-03T06:03:19.434678Z","shell.execute_reply.started":"2021-09-03T06:03:19.431443Z","shell.execute_reply":"2021-09-03T06:03:19.433983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -U git+https://github.com/leondgarse/keras_efficientnet_v2","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:19.436065Z","iopub.execute_input":"2021-09-03T06:03:19.436323Z","iopub.status.idle":"2021-09-03T06:03:29.307676Z","shell.execute_reply.started":"2021-09-03T06:03:19.436298Z","shell.execute_reply":"2021-09-03T06:03:29.306516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nEffNetV2-B0\t7.1M\t78.7%\tefficientnetv2-b0-21k.h5\tefficientnetv2-b0-21k-ft1k.h5\nEffNetV2-B1\t8.1M\t79.8%\tefficientnetv2-b1-21k.h5\tefficientnetv2-b1-21k-ft1k.h5\nEffNetV2-B2\t10.1M\t80.5%\tefficientnetv2-b2-21k.h5\tefficientnetv2-b2-21k-ft1k.h5\nEffNetV2-B3\t14.4M\t82.1%\tefficientnetv2-b3-21k.h5\tefficientnetv2-b3-21k-ft1k.h5\nEffNetV2S\t21.5M\t84.9%\tefficientnetv2-s-21k.h5\tefficientnetv2-s-21k-ft1k.h5\nEffNetV2M\t54.1M\t86.2%\tefficientnetv2-m-21k.h5\tefficientnetv2-m-21k-ft1k.h5\nEffNetV2L\t119.5M\t86.9%\tefficientnetv2-l-21k.h5\tefficientnetv2-l-21k-ft1k.h5\nEffNetV2XL\t206.8M\t87.2%\tefficientnetv2-xl-21k.h5\tefficientnetv2-xl-21k-ft1k.h5\n'''","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:29.310744Z","iopub.execute_input":"2021-09-03T06:03:29.31118Z","iopub.status.idle":"2021-09-03T06:03:29.321065Z","shell.execute_reply.started":"2021-09-03T06:03:29.311135Z","shell.execute_reply":"2021-09-03T06:03:29.320252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Exclude model top layers by set num_classes=0.\n#Use dynamic input resolution by set input_shape=(None, None, 3)\nimport keras_efficientnet_v2\n#model = keras_efficientnet_v2.EfficientNetV2M(input_shape=(None, None, 3), drop_connect_rate=0.2, num_classes=0, pretrained=\"imagenet21k-ft1k\")\n\n#EffNetV2XL\n#print(model)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:29.322277Z","iopub.execute_input":"2021-09-03T06:03:29.322708Z","iopub.status.idle":"2021-09-03T06:03:34.973717Z","shell.execute_reply.started":"2021-09-03T06:03:29.322677Z","shell.execute_reply":"2021-09-03T06:03:34.972923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:34.974747Z","iopub.execute_input":"2021-09-03T06:03:34.975122Z","iopub.status.idle":"2021-09-03T06:03:34.978205Z","shell.execute_reply.started":"2021-09-03T06:03:34.975094Z","shell.execute_reply":"2021-09-03T06:03:34.977536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport sys\n \n#sys.path.append('../input/v202108-pytorch-image-models/pytorch-image-models-master')\n","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:34.979245Z","iopub.execute_input":"2021-09-03T06:03:34.979744Z","iopub.status.idle":"2021-09-03T06:03:34.989008Z","shell.execute_reply.started":"2021-09-03T06:03:34.979714Z","shell.execute_reply":"2021-09-03T06:03:34.988215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport math\nimport random\nimport re\nimport warnings\nfrom pathlib import Path\nfrom typing import Optional, Tuple\n\n#import efficientnet.tfkeras as efn\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\n\nimport tensorflow_addons as tfa\nfrom kaggle_datasets import KaggleDatasets\nfrom scipy.signal import get_window\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:34.99018Z","iopub.execute_input":"2021-09-03T06:03:34.990683Z","iopub.status.idle":"2021-09-03T06:03:35.977168Z","shell.execute_reply.started":"2021-09-03T06:03:34.990654Z","shell.execute_reply":"2021-09-03T06:03:35.976464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:35.978157Z","iopub.execute_input":"2021-09-03T06:03:35.978598Z","iopub.status.idle":"2021-09-03T06:03:35.983379Z","shell.execute_reply.started":"2021-09-03T06:03:35.978568Z","shell.execute_reply":"2021-09-03T06:03:35.982695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"'''\nmodel_image_size_map = {\n  \"efficientnetv2-s\": 384,\n  \"efficientnetv2-m\": 480,\n  \"efficientnetv2-l\": 480,\n  \"efficientnetv2-b0\": 224,\n  \"efficientnetv2-b1\": 240,\n  \"efficientnetv2-b2\": 260,\n  \"efficientnetv2-b3\": 300,\n  \"efficientnetv2-s-21k\": 384,\n  \"efficientnetv2-m-21k\": 480,\n  \"efficientnetv2-l-21k\": 480,\n  \"efficientnetv2-xl-21k\": 512,\n  \"efficientnetv2-b0-21k\": 224,\n  \"efficientnetv2-b1-21k\": 240,\n  \"efficientnetv2-b2-21k\": 260,\n  \"efficientnetv2-b3-21k\": 300,\n  \"efficientnetv2-s-21k-ft1k\": 384,\n  \"efficientnetv2-m-21k-ft1k\": 480,\n  \"efficientnetv2-l-21k-ft1k\": 480,\n  \"efficientnetv2-xl-21k-ft1k\": 512,\n  \"efficientnetv2-b0-21k-ft1k\": 224,\n  \"efficientnetv2-b1-21k-ft1k\": 240,\n  \"efficientnetv2-b2-21k-ft1k\": 260,\n  \"efficientnetv2-b3-21k-ft1k\": 300, \n  \"efficientnet_b0\": 224,\n  \"efficientnet_b1\": 240,\n  \"efficientnet_b2\": 260,\n  \"efficientnet_b3\": 300,\n  \"efficientnet_b4\": 380,\n  \"efficientnet_b5\": 456,\n  \"efficientnet_b6\": 528,\n  \"efficientnet_b7\": 600,\n  \"inception_v3\": 299,\n  \"inception_resnet_v2\": 299,\n  \"nasnet_large\": 331,\n  \"pnasnet_large\": 331,\n'''","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:35.984425Z","iopub.execute_input":"2021-09-03T06:03:35.984884Z","iopub.status.idle":"2021-09-03T06:03:35.995639Z","shell.execute_reply.started":"2021-09-03T06:03:35.984855Z","shell.execute_reply":"2021-09-03T06:03:35.994814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#NUM_FOLDS = 4\nNUM_FOLDS = 4\n#IMAGE_SIZE = 256\nIMAGE_SIZE = 480\n#IMAGE_SIZE = 512\n#BATCH_SIZE = 32\nBATCH_SIZE = 32\nEFFICIENTNET_SIZE = 7\nMODEL_NAME=\"EfficientNetV2\"\nWEIGHTS = \"imagenet21k-ft1k\"\n\nMIXUP_PROB = 0.0\n#EPOCHS = 20\nEPOCHS = 40\nR_ANGLE = 0 / 180 * np.pi\nS_SHIFT = 0.0\nT_SHIFT = 0.0\nLABEL_POSITIVE_SHIFT = 0.99","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:35.996791Z","iopub.execute_input":"2021-09-03T06:03:35.99726Z","iopub.status.idle":"2021-09-03T06:03:36.005265Z","shell.execute_reply.started":"2021-09-03T06:03:35.997231Z","shell.execute_reply":"2021-09-03T06:03:36.00378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAVEDIR = Path(\"models\")\nSAVEDIR.mkdir(exist_ok=True)\n\nOOFDIR = Path(\"oof\")\nOOFDIR.mkdir(exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:36.006576Z","iopub.execute_input":"2021-09-03T06:03:36.006912Z","iopub.status.idle":"2021-09-03T06:03:36.017042Z","shell.execute_reply.started":"2021-09-03T06:03:36.006879Z","shell.execute_reply":"2021-09-03T06:03:36.015698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utilities","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\n\nset_seed(999)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:36.018538Z","iopub.execute_input":"2021-09-03T06:03:36.018858Z","iopub.status.idle":"2021-09-03T06:03:36.026478Z","shell.execute_reply.started":"2021-09-03T06:03:36.018829Z","shell.execute_reply":"2021-09-03T06:03:36.025565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    TPU_DETECTED = False\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n        TPU_DETECTED = True\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n\n    return strategy, TPU_DETECTED","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:36.027621Z","iopub.execute_input":"2021-09-03T06:03:36.028118Z","iopub.status.idle":"2021-09-03T06:03:36.039935Z","shell.execute_reply.started":"2021-09-03T06:03:36.028082Z","shell.execute_reply":"2021-09-03T06:03:36.039083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy, tpu_detected = auto_select_accelerator()\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:36.041084Z","iopub.execute_input":"2021-09-03T06:03:36.041373Z","iopub.status.idle":"2021-09-03T06:03:41.854121Z","shell.execute_reply.started":"2021-09-03T06:03:36.041334Z","shell.execute_reply":"2021-09-03T06:03:41.853142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Loading","metadata":{}},{"cell_type":"code","source":"gcs_paths = []\nfor i, j in [(0, 4), (5, 9), (10, 14), (15, 19)]:\n    GCS_path = KaggleDatasets().get_gcs_path(f\"g2net-waveform-tfrecords-train-{i}-{j}\")\n    gcs_paths.append(GCS_path)\n    print(GCS_path)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:41.855558Z","iopub.execute_input":"2021-09-03T06:03:41.855949Z","iopub.status.idle":"2021-09-03T06:03:43.429464Z","shell.execute_reply.started":"2021-09-03T06:03:41.855917Z","shell.execute_reply":"2021-09-03T06:03:43.428452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_files = []\nfor path in gcs_paths:\n    all_files.extend(np.sort(np.array(tf.io.gfile.glob(path + \"/train*.tfrecords\"))))\n\nprint(\"train_files: \", len(all_files))","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:43.430909Z","iopub.execute_input":"2021-09-03T06:03:43.431514Z","iopub.status.idle":"2021-09-03T06:03:43.722312Z","shell.execute_reply.started":"2021-09-03T06:03:43.431469Z","shell.execute_reply":"2021-09-03T06:03:43.721297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset Preparation\n\nHere's the main contribution of this notebook - Tensorflow version of on-the-fly CQT computation. Note that some of the operations used in CQT computation are not supported by TPU, therefore the implementation is not a TF layer but a function that runs on CPU.","metadata":{}},{"cell_type":"code","source":"def create_cqt_kernels(\n    q: float,\n    fs: float,\n    fmin: float,\n    n_bins: int = 84,\n    bins_per_octave: int = 12,\n    norm: float = 1,\n    window: str = \"hann\",\n    fmax: Optional[float] = None,\n    topbin_check: bool = True\n) -> Tuple[np.ndarray, int, np.ndarray, float]:\n    fft_len = 2 ** _nextpow2(np.ceil(q * fs / fmin))\n    \n    if (fmax is not None) and (n_bins is None):\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    elif (fmax is None) and (n_bins is not None):\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    else:\n        warnings.warn(\"If nmax is given, n_bins will be ignored\", SyntaxWarning)\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n        \n    if np.max(freqs) > fs / 2 and topbin_check:\n        raise ValueError(f\"The top bin {np.max(freqs)} Hz has exceeded the Nyquist frequency, \\\n                           please reduce the `n_bins`\")\n    \n    kernel = np.zeros((int(n_bins), int(fft_len)), dtype=np.complex64)\n    \n    length = np.ceil(q * fs / freqs)\n    for k in range(0, int(n_bins)):\n        freq = freqs[k]\n        l = np.ceil(q * fs / freq)\n        \n        if l % 2 == 1:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0)) - 1\n        else:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0))\n\n        sig = get_window(window, int(l), fftbins=True) * np.exp(\n            np.r_[-l // 2:l // 2] * 1j * 2 * np.pi * freq / fs) / l\n        \n        if norm:\n            kernel[k, start:start + int(l)] = sig / np.linalg.norm(sig, norm)\n        else:\n            kernel[k, start:start + int(l)] = sig\n    return kernel, fft_len, length, freqs\n\n\ndef _nextpow2(a: float) -> int:\n    return int(np.ceil(np.log2(a)))\n\n\ndef prepare_cqt_kernel(\n    sr=22050,\n    hop_length=512,\n    fmin=32.70,\n    fmax=None,\n    n_bins=84,\n    bins_per_octave=12,\n    norm=1,\n    filter_scale=1,\n    window=\"hann\"\n):\n    q = float(filter_scale) / (2 ** (1 / bins_per_octave) - 1)\n    print(q)\n    return create_cqt_kernels(q, sr, fmin, n_bins, bins_per_octave, norm, window, fmax)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:43.725654Z","iopub.execute_input":"2021-09-03T06:03:43.725961Z","iopub.status.idle":"2021-09-03T06:03:43.745485Z","shell.execute_reply.started":"2021-09-03T06:03:43.725931Z","shell.execute_reply":"2021-09-03T06:03:43.744443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#HOP_LENGTH = 16\nHOP_LENGTH = 8\ncqt_kernels, KERNEL_WIDTH, lengths, _ = prepare_cqt_kernel(\n    sr=2048,\n    hop_length=HOP_LENGTH,\n    fmin=20,\n    fmax=1024,\n    bins_per_octave=24)\nLENGTHS = tf.constant(lengths, dtype=tf.float32)\nCQT_KERNELS_REAL = tf.constant(np.swapaxes(cqt_kernels.real[:, np.newaxis, :], 0, 2))\nCQT_KERNELS_IMAG = tf.constant(np.swapaxes(cqt_kernels.imag[:, np.newaxis, :], 0, 2))\nPADDING = tf.constant([[0, 0],\n                        [KERNEL_WIDTH // 2, KERNEL_WIDTH // 2],\n                        [0, 0]])","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:43.746801Z","iopub.execute_input":"2021-09-03T06:03:43.747113Z","iopub.status.idle":"2021-09-03T06:03:43.817259Z","shell.execute_reply.started":"2021-09-03T06:03:43.74708Z","shell.execute_reply":"2021-09-03T06:03:43.816321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_cqt_image(wave, hop_length=16):\n    CQTs = []\n    for i in range(3):\n        x = wave[i]\n        x = tf.expand_dims(tf.expand_dims(x, 0), 2)\n        x = tf.pad(x, PADDING, \"REFLECT\")\n\n        CQT_real = tf.nn.conv1d(x, CQT_KERNELS_REAL, stride=hop_length, padding=\"VALID\")\n        CQT_imag = -tf.nn.conv1d(x, CQT_KERNELS_IMAG, stride=hop_length, padding=\"VALID\")\n        CQT_real *= tf.math.sqrt(LENGTHS)\n        CQT_imag *= tf.math.sqrt(LENGTHS)\n\n        CQT = tf.math.sqrt(tf.pow(CQT_real, 2) + tf.pow(CQT_imag, 2))\n        CQTs.append(CQT[0])\n    return tf.stack(CQTs, axis=2)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:43.818598Z","iopub.execute_input":"2021-09-03T06:03:43.818885Z","iopub.status.idle":"2021-09-03T06:03:43.82716Z","shell.execute_reply.started":"2021-09-03T06:03:43.818856Z","shell.execute_reply":"2021-09-03T06:03:43.826065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], IMAGE_SIZE), tf.reshape(tf.cast(example[\"target\"], tf.float32), [1])\n\n\ndef read_unlabeled_tfrecord(example, return_image_id):\n    tfrec_format = {\n        \"wave\": tf.io.FixedLenFeature([], tf.string),\n        \"wave_id\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return prepare_image(example[\"wave\"], IMAGE_SIZE), example[\"wave_id\"] if return_image_id else 0\n\n\ndef count_data_items(fileids):\n    return len(fileids) * 28000\n\n\ndef count_data_items_test(fileids):\n    return len(fileids) * 22600\n\n\ndef mixup(image, label, probability=0.5, aug_batch=64 * 8):\n    imgs = []\n    labs = []\n    for j in range(aug_batch):\n        p = tf.cast(tf.random.uniform([], 0, 1) <= probability, tf.float32)\n        k = tf.cast(tf.random.uniform([], 0, aug_batch), tf.int32)\n        a = tf.random.uniform([], 0, 1) * p\n\n        img1 = image[j]\n        img2 = image[k]\n        imgs.append((1 - a) * img1 + a * img2)\n        lab1 = label[j]\n        lab2 = label[k]\n        labs.append((1 - a) * lab1 + a * lab2)\n    image2 = tf.reshape(tf.stack(imgs), (aug_batch, IMAGE_SIZE, IMAGE_SIZE, 3))\n    label2 = tf.reshape(tf.stack(labs), (aug_batch,))\n    return image2, label2\n\n\ndef time_shift(img, shift=T_SHIFT):\n    if shift > 0:\n        T = IMAGE_SIZE\n        P = tf.random.uniform([],0,1)\n        SHIFT = tf.cast(T * P, tf.int32)\n        return tf.concat([img[-SHIFT:], img[:-SHIFT]], axis=0)\n    return img\n\n\ndef rotate(img, angle=R_ANGLE):\n    if angle > 0:\n        P = tf.random.uniform([],0,1)\n        A = tf.cast(angle * P, tf.float32)\n        return tfa.image.rotate(img, A)\n    return img\n\n\ndef spector_shift(img, shift=S_SHIFT):\n    if shift > 0:\n        T = IMAGE_SIZE\n        P = tf.random.uniform([],0,1)\n        SHIFT = tf.cast(T * P, tf.int32)\n        return tf.concat([img[:, -SHIFT:], img[:, :-SHIFT]], axis=1)\n    return img\n\ndef img_aug_f(img):\n    img = time_shift(img)\n    img = spector_shift(img)\n    # img = rotate(img)\n    return img\n\n\ndef imgs_aug_f(imgs, batch_size):\n    _imgs = []\n    DIM = IMAGE_SIZE\n    for j in range(batch_size):\n        _imgs.append(img_aug_f(imgs[j]))\n    return tf.reshape(tf.stack(_imgs),(batch_size,DIM,DIM,3))\n\n\ndef label_positive_shift(labels):\n    return labels * LABEL_POSITIVE_SHIFT\n\n\ndef aug_f(imgs, labels, batch_size):\n    imgs, label = mixup(imgs, labels, MIXUP_PROB, batch_size)\n    imgs = imgs_aug_f(imgs, batch_size)\n    return imgs, label_positive_shift(label)\n\n\ndef prepare_image(wave, dim=256):\n    wave = tf.reshape(tf.io.decode_raw(wave, tf.float64), (3, 4096))\n    normalized_waves = []\n    for i in range(3):\n        normalized_wave = wave[i] / tf.math.reduce_max(wave[i])\n        normalized_waves.append(normalized_wave)\n    wave = tf.stack(normalized_waves)\n    wave = tf.cast(wave, tf.float32)\n    image = create_cqt_image(wave, HOP_LENGTH)\n    image = tf.image.resize(image, size=(dim, dim))\n    return tf.reshape(image, (dim, dim, 3))\n\n\ndef get_dataset(files, batch_size=16, repeat=False, shuffle=False, aug=True, labeled=True, return_image_ids=True):\n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO, compression_type=\"GZIP\")\n    ds = ds.cache()\n\n    if repeat:\n        ds = ds.repeat()\n\n    if shuffle:\n        ds = ds.shuffle(1024 * 2)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n\n    if labeled:\n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_ids), num_parallel_calls=AUTO)\n\n    ds = ds.batch(batch_size * REPLICAS)\n    if aug:\n        ds = ds.map(lambda x, y: aug_f(x, y, batch_size * REPLICAS), num_parallel_calls=AUTO)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:43.828706Z","iopub.execute_input":"2021-09-03T06:03:43.829038Z","iopub.status.idle":"2021-09-03T06:03:44.028029Z","shell.execute_reply.started":"2021-09-03T06:03:43.829004Z","shell.execute_reply":"2021-09-03T06:03:44.026616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"#EfficientNetV2XL(input_shape=(512, 512, 3), num_classes=1000, dropout=0.4, classifier_activation=\"softmax\", pretrained=\"imagenet\", **kwargs):","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:44.031111Z","iopub.execute_input":"2021-09-03T06:03:44.031444Z","iopub.status.idle":"2021-09-03T06:03:44.042616Z","shell.execute_reply.started":"2021-09-03T06:03:44.031403Z","shell.execute_reply":"2021-09-03T06:03:44.041629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#def build_model(size=256, effnet_name=\"EfficientNetV2\", weights=\"imagenet\", count=0):\ndef build_model(size=IMAGE_SIZE, effnet_name=\"EfficientNetV2\", weights=\"imagenet21k-ft1k\", count=0):\n    inputs = tf.keras.layers.Input(shape=(size, size, 3))\n    \n    #efn_string= f\"EfficientNetB{efficientnet_size}\"\n    #efn_string= f\"{effnet_name}\"\n    #efn_layer = getattr(efn, efn_string)(input_shape=(size, size, 3), weights=weights, include_top=False)\n    #efn_layer=keras_efficientnet_v2.EfficientNetV2(\"L\",input_shape=(None, None, 3), drop_connect_rate=0.2, num_classes=0, pretrained=\"imagenet21k-ft1k\")\n    efn_layer=keras_efficientnet_v2.EfficientNetV2L(input_shape=(size, size, 3), drop_connect_rate=0.2, num_classes=0, pretrained=\"imagenet21k-ft1k\")\n\n    x = efn_layer(inputs)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n\n    x = tf.keras.layers.Dropout(0.2)(x)\n    x = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n    model = tf.keras.Model(inputs=inputs, outputs=x)\n\n    lr_decayed_fn = tf.keras.experimental.CosineDecay(1e-3, count)\n    opt = tfa.optimizers.AdamW(lr_decayed_fn, learning_rate=1e-4)\n    loss = tf.keras.losses.BinaryCrossentropy()\n    model.compile(optimizer=opt, loss=loss, metrics=[\"AUC\"])\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:44.043806Z","iopub.execute_input":"2021-09-03T06:03:44.044139Z","iopub.status.idle":"2021-09-03T06:03:44.053465Z","shell.execute_reply.started":"2021-09-03T06:03:44.04411Z","shell.execute_reply":"2021-09-03T06:03:44.052517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback(batch_size=8, replicas=8):\n    #lr_start   = 1e-4\n    lr_start   = 1e-3\n    lr_max     = 0.000015 * replicas * batch_size\n    lr_min     = 1e-7\n    lr_ramp_ep = 3\n    lr_sus_ep  = 0\n    lr_decay   = 0.7\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:44.0549Z","iopub.execute_input":"2021-09-03T06:03:44.05528Z","iopub.status.idle":"2021-09-03T06:03:44.066702Z","shell.execute_reply.started":"2021-09-03T06:03:44.055249Z","shell.execute_reply":"2021-09-03T06:03:44.065664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"# because of time limti, let train only a few folds.","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:44.068116Z","iopub.execute_input":"2021-09-03T06:03:44.068422Z","iopub.status.idle":"2021-09-03T06:03:44.076176Z","shell.execute_reply.started":"2021-09-03T06:03:44.068385Z","shell.execute_reply":"2021-09-03T06:03:44.075246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=999)\noof_pred = []\noof_target = []\n\nfiles_train_all = np.array(all_files)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:44.077707Z","iopub.execute_input":"2021-09-03T06:03:44.078001Z","iopub.status.idle":"2021-09-03T06:03:44.086583Z","shell.execute_reply.started":"2021-09-03T06:03:44.077973Z","shell.execute_reply":"2021-09-03T06:03:44.085556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nfor fold, (trn_idx, val_idx) in enumerate(kf.split(files_train_all)):\n    files_train = files_train_all[trn_idx]\n    files_valid = files_train_all[val_idx]\n\n    print(\"=\" * 120)\n    print(f\"Fold {fold}\")\n    print(\"=\" * 120)\n\n    train_image_count = count_data_items(files_train)\n    valid_image_count = count_data_items(files_valid)\n\n    tf.keras.backend.clear_session()\n    strategy, tpu_detected = auto_select_accelerator()\n    with strategy.scope():\n        model = build_model(\n            size=IMAGE_SIZE, \n            #efficientnet_size=EFFICIENTNET_SIZE,\n            weights=WEIGHTS, \n            count=train_image_count // BATCH_SIZE // REPLICAS // 4)\n    \n    model_ckpt = tf.keras.callbacks.ModelCheckpoint(\n        str(SAVEDIR / f\"fold{fold}.h5\"), monitor=\"val_auc\", verbose=1, save_best_only=True,\n        save_weights_only=True, mode=\"max\", save_freq=\"epoch\"\n    )\n\n    history = model.fit(\n        get_dataset(files_train, batch_size=BATCH_SIZE, shuffle=True, repeat=True, aug=True),\n        epochs=EPOCHS,\n        callbacks=[model_ckpt, get_lr_callback(BATCH_SIZE, REPLICAS)],\n        steps_per_epoch=train_image_count // BATCH_SIZE // REPLICAS // 4,\n        validation_data=get_dataset(files_valid, batch_size=BATCH_SIZE * 4, repeat=False, shuffle=False, aug=False),\n        verbose=1\n    )\n\n    print(\"Loading best model...\")\n    model.load_weights(str(SAVEDIR / f\"fold{fold}.h5\"))\n\n    ds_valid = get_dataset(files_valid, labeled=False, return_image_ids=False, repeat=True, shuffle=False, batch_size=BATCH_SIZE * 2, aug=False)\n    STEPS = valid_image_count / BATCH_SIZE / 2 / REPLICAS\n    pred = model.predict(ds_valid, steps=STEPS, verbose=0)[:valid_image_count]\n    oof_pred.append(np.mean(pred.reshape((valid_image_count, 1), order=\"F\"), axis=1))\n\n    ds_valid = get_dataset(files_valid, repeat=False, labeled=True, return_image_ids=True, aug=False)\n    oof_target.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))\n\n    plt.figure(figsize=(8, 6))\n    sns.distplot(oof_pred[-1])\n    plt.show()\n\n    plt.figure(figsize=(15, 5))\n    plt.plot(\n        np.arange(len(history.history[\"auc\"])),\n        history.history[\"auc\"],\n        \"-o\",\n        label=\"Train auc\",\n        color=\"#ff7f0e\")\n    plt.plot(\n        np.arange(len(history.history[\"auc\"])),\n        history.history[\"val_auc\"],\n        \"-o\",\n        label=\"Val auc\",\n        color=\"#1f77b4\")\n    \n    x = np.argmax(history.history[\"val_auc\"])\n    y = np.max(history.history[\"val_auc\"])\n\n    xdist = plt.xlim()[1] - plt.xlim()[0]\n    ydist = plt.ylim()[1] - plt.ylim()[0]\n\n    plt.scatter(x, y, s=200, color=\"#1f77b4\")\n    plt.text(x - 0.03 * xdist, y - 0.13 * ydist, f\"max auc\\n{y}\", size=14)\n\n    plt.ylabel(\"auc\", size=14)\n    plt.xlabel(\"Epoch\", size=14)\n    plt.legend(loc=2)\n\n    plt2 = plt.gca().twinx()\n    plt2.plot(\n        np.arange(len(history.history[\"auc\"])),\n        history.history[\"loss\"],\n        \"-o\",\n        label=\"Train Loss\",\n        color=\"#2ca02c\")\n    plt2.plot(\n        np.arange(len(history.history[\"auc\"])),\n        history.history[\"val_loss\"],\n        \"-o\",\n        label=\"Val Loss\",\n        color=\"#d62728\")\n    \n    x = np.argmin(history.history[\"val_loss\"])\n    y = np.min(history.history[\"val_loss\"])\n    \n    ydist = plt.ylim()[1] - plt.ylim()[0]\n\n    plt.scatter(x, y, s=200, color=\"#d62728\")\n    plt.text(x - 0.03 * xdist, y + 0.05 * ydist, \"min loss\", size=14)\n\n    plt.ylabel(\"Loss\", size=14)\n    plt.title(f\"Fold {fold + 1} - Image Size {IMAGE_SIZE}, model {MODEL_NAME}\", size=18)\n\n    plt.legend(loc=3)\n    plt.savefig(OOFDIR / f\"fig{fold}.png\")\n    plt.show()\n    \n'''","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:44.08798Z","iopub.execute_input":"2021-09-03T06:03:44.088372Z","iopub.status.idle":"2021-09-03T06:03:44.1014Z","shell.execute_reply.started":"2021-09-03T06:03:44.088325Z","shell.execute_reply":"2021-09-03T06:03:44.100455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_fold=[0]  \n#train_fold=[0,1]\nfor fold, (trn_idx, val_idx) in enumerate(kf.split(files_train_all)):\n    \n    if fold in train_fold:\n        files_train = files_train_all[trn_idx]\n        files_valid = files_train_all[val_idx]\n\n        print(\"=\" * 120)\n        print(f\"Fold {fold}\")\n        print(\"=\" * 120)\n\n        train_image_count = count_data_items(files_train)\n        valid_image_count = count_data_items(files_valid)\n\n        tf.keras.backend.clear_session()\n        strategy, tpu_detected = auto_select_accelerator()\n        with strategy.scope():\n            model = build_model(\n                size=IMAGE_SIZE, \n                #efficientnet_size=EFFICIENTNET_SIZE,\n                weights=WEIGHTS, \n                count=train_image_count // BATCH_SIZE // REPLICAS // 4)\n\n        model_ckpt = tf.keras.callbacks.ModelCheckpoint(\n            str(SAVEDIR / f\"fold{fold}.h5\"), monitor=\"val_auc\", verbose=1, save_best_only=True,\n            save_weights_only=True, mode=\"max\", save_freq=\"epoch\"\n        )\n\n        history = model.fit(\n            get_dataset(files_train, batch_size=BATCH_SIZE, shuffle=True, repeat=True, aug=True),\n            epochs=EPOCHS,\n            callbacks=[model_ckpt, get_lr_callback(BATCH_SIZE, REPLICAS)],\n            steps_per_epoch=train_image_count // BATCH_SIZE // REPLICAS // 4,\n            validation_data=get_dataset(files_valid, batch_size=BATCH_SIZE * 4, repeat=False, shuffle=False, aug=False),\n            verbose=1\n        )\n\n        print(\"Loading best model...\")\n        model.load_weights(str(SAVEDIR / f\"fold{fold}.h5\"))\n\n        ds_valid = get_dataset(files_valid, labeled=False, return_image_ids=False, repeat=True, shuffle=False, batch_size=BATCH_SIZE * 2, aug=False)\n        STEPS = valid_image_count / BATCH_SIZE / 2 / REPLICAS\n        pred = model.predict(ds_valid, steps=STEPS, verbose=0)[:valid_image_count]\n        oof_pred.append(np.mean(pred.reshape((valid_image_count, 1), order=\"F\"), axis=1))\n\n        ds_valid = get_dataset(files_valid, repeat=False, labeled=True, return_image_ids=True, aug=False)\n        oof_target.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))\n\n        plt.figure(figsize=(8, 6))\n        sns.distplot(oof_pred[-1])\n        plt.show()\n\n        plt.figure(figsize=(15, 5))\n        plt.plot(\n            np.arange(len(history.history[\"auc\"])),\n            history.history[\"auc\"],\n            \"-o\",\n            label=\"Train auc\",\n            color=\"#ff7f0e\")\n        plt.plot(\n            np.arange(len(history.history[\"auc\"])),\n            history.history[\"val_auc\"],\n            \"-o\",\n            label=\"Val auc\",\n            color=\"#1f77b4\")\n\n        x = np.argmax(history.history[\"val_auc\"])\n        y = np.max(history.history[\"val_auc\"])\n\n        xdist = plt.xlim()[1] - plt.xlim()[0]\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n\n        plt.scatter(x, y, s=200, color=\"#1f77b4\")\n        plt.text(x - 0.03 * xdist, y - 0.13 * ydist, f\"max auc\\n{y}\", size=14)\n\n        plt.ylabel(\"auc\", size=14)\n        plt.xlabel(\"Epoch\", size=14)\n        plt.legend(loc=2)\n\n        plt2 = plt.gca().twinx()\n        plt2.plot(\n            np.arange(len(history.history[\"auc\"])),\n            history.history[\"loss\"],\n            \"-o\",\n            label=\"Train Loss\",\n            color=\"#2ca02c\")\n        plt2.plot(\n            np.arange(len(history.history[\"auc\"])),\n            history.history[\"val_loss\"],\n            \"-o\",\n            label=\"Val Loss\",\n            color=\"#d62728\")\n\n        x = np.argmin(history.history[\"val_loss\"])\n        y = np.min(history.history[\"val_loss\"])\n\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n\n        plt.scatter(x, y, s=200, color=\"#d62728\")\n        plt.text(x - 0.03 * xdist, y + 0.05 * ydist, \"min loss\", size=14)\n\n        plt.ylabel(\"Loss\", size=14)\n        plt.title(f\"Fold {fold + 1} - Image Size {IMAGE_SIZE}, model {MODEL_NAME}\", size=18)\n\n        plt.legend(loc=3)\n        plt.savefig(OOFDIR / f\"fig{fold}.png\")\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-03T06:03:44.102873Z","iopub.execute_input":"2021-09-03T06:03:44.103152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## OOF","metadata":{}},{"cell_type":"code","source":"oof = np.concatenate(oof_pred)\ntrue = np.concatenate(oof_target)\nauc = roc_auc_score(y_true=true, y_score=oof)\nprint(f\"AUC: {auc:.5f}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({\n    \"y_true\": true.reshape(-1),\n    \"y_pred\": oof\n})\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(OOFDIR / f\"oof.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}