{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q efficientnet\nimport re\nimport os\nimport numpy as np\nimport pandas as pd\nfrom scipy.signal import get_window\nfrom typing import Optional, Tuple\nimport warnings\nimport random\nimport math\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nfrom sklearn import metrics\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import mixed_precision\nimport tensorflow_addons as tfa\nfrom kaggle_datasets import KaggleDatasets","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-30T13:23:49.741601Z","iopub.execute_input":"2021-08-30T13:23:49.741995Z","iopub.status.idle":"2021-08-30T13:23:57.018916Z","shell.execute_reply.started":"2021-08-30T13:23:49.74196Z","shell.execute_reply":"2021-08-30T13:23:57.017635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to get hardware strategy\ndef get_hardware_strategy():\n    try:\n        # TPU detection. No parameters necessary if TPU_NAME environment variable is\n        # set: this is always the case on Kaggle.\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        policy = mixed_precision.Policy('mixed_bfloat16')\n        mixed_precision.set_global_policy(policy)\n        tf.config.optimizer.set_jit(True)\n    else:\n        # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n        strategy = tf.distribute.get_strategy()\n\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n    return tpu, strategy\n\ntpu, strategy = get_hardware_strategy()","metadata":{"execution":{"iopub.status.busy":"2021-08-30T13:23:06.901255Z","iopub.status.idle":"2021-08-30T13:23:06.901662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For tf.dataset\nAUTO = tf.data.experimental.AUTOTUNE\n\n# Data access (Train tf records)\nGCS_PATH1 = KaggleDatasets().get_gcs_path('g2net-tf-records-tr-bp-filter-1')\nGCS_PATH2 = KaggleDatasets().get_gcs_path('g2net-tf-records-tr-bp-filter-2')\nGCS_PATH3 = KaggleDatasets().get_gcs_path('g2net-tf-records-tr-bp-filter-3')\n# Data access (Test tf records)\nGCS_PATH4 = KaggleDatasets().get_gcs_path('g2net-tf-records-ts-bp-filter-1')\nGCS_PATH5 = KaggleDatasets().get_gcs_path('g2net-tf-records-ts-bp-filter-2')\n\n# Configuration\nEPOCHS = 15\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\nIMAGE_SIZE = [512, 512]\n# Seed\nSEED = 21\n# Learning rate\nLR = 0.0001\n# Verbosity\nVERBOSE = 2\n\n# Training filenames directory\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH1 + '/train*.tfrec') + tf.io.gfile.glob(GCS_PATH2 + '/train*.tfrec') + tf.io.gfile.glob(GCS_PATH3 + '/train*.tfrec')\n# Testing filenames directory\nTESTING_FILENAMES = tf.io.gfile.glob(GCS_PATH4 + '/test*.tfrec') + tf.io.gfile.glob(GCS_PATH5 + '/test*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2021-08-30T13:23:57.021021Z","iopub.execute_input":"2021-08-30T13:23:57.021353Z","iopub.status.idle":"2021-08-30T13:23:59.178513Z","shell.execute_reply.started":"2021-08-30T13:23:57.021321Z","shell.execute_reply":"2021-08-30T13:23:59.177085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to create cqt kernel\ndef create_cqt_kernels(\n    q: float,\n    fs: float,\n    fmin: float,\n    n_bins: int = 84,\n    bins_per_octave: int = 12,\n    norm: float = 1,\n    window: str = \"hann\",\n    fmax: Optional[float] = None,\n    topbin_check: bool = True\n) -> Tuple[np.ndarray, int, np.ndarray, float]:\n    fft_len = 2 ** _nextpow2(np.ceil(q * fs / fmin))\n    \n    if (fmax is not None) and (n_bins is None):\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    elif (fmax is None) and (n_bins is not None):\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    else:\n        warnings.warn(\"If nmax is given, n_bins will be ignored\", SyntaxWarning)\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n        \n    if np.max(freqs) > fs / 2 and topbin_check:\n        raise ValueError(f\"The top bin {np.max(freqs)} Hz has exceeded the Nyquist frequency, \\\n                           please reduce the `n_bins`\")\n    \n    kernel = np.zeros((int(n_bins), int(fft_len)), dtype=np.complex64)\n    \n    length = np.ceil(q * fs / freqs)\n    for k in range(0, int(n_bins)):\n        freq = freqs[k]\n        l = np.ceil(q * fs / freq)\n        \n        if l % 2 == 1:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0)) - 1\n        else:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0))\n\n        sig = get_window(window, int(l), fftbins=True) * np.exp(\n            np.r_[-l // 2:l // 2] * 1j * 2 * np.pi * freq / fs) / l\n        \n        if norm:\n            kernel[k, start:start + int(l)] = sig / np.linalg.norm(sig, norm)\n        else:\n            kernel[k, start:start + int(l)] = sig\n    return kernel, fft_len, length, freqs\n\n\ndef _nextpow2(a: float) -> int:\n    return int(np.ceil(np.log2(a)))\n\n# Function to prepare cqt kernel\ndef prepare_cqt_kernel(\n    sr=22050,\n    hop_length=512,\n    fmin=32.70,\n    fmax=None,\n    n_bins=84,\n    bins_per_octave=12,\n    norm=1,\n    filter_scale=1,\n    window=\"hann\"\n):\n    q = float(filter_scale) / (2 ** (1 / bins_per_octave) - 1)\n    print(q)\n    return create_cqt_kernels(q, sr, fmin, n_bins, bins_per_octave, norm, window, fmax)\n\n# Function to create cqt image\ndef create_cqt_image(wave, hop_length=16):\n    CQTs = []\n    for i in range(3):\n        x = wave[i]\n        x = tf.expand_dims(tf.expand_dims(x, 0), 2)\n        x = tf.pad(x, PADDING, \"REFLECT\")\n\n        CQT_real = tf.nn.conv1d(x, CQT_KERNELS_REAL, stride=hop_length, padding=\"VALID\")\n        CQT_imag = -tf.nn.conv1d(x, CQT_KERNELS_IMAG, stride=hop_length, padding=\"VALID\")\n        CQT_real *= tf.math.sqrt(LENGTHS)\n        CQT_imag *= tf.math.sqrt(LENGTHS)\n\n        CQT = tf.math.sqrt(tf.pow(CQT_real, 2) + tf.pow(CQT_imag, 2))\n        CQTs.append(CQT[0])\n    return tf.stack(CQTs, axis=2)\n\nHOP_LENGTH = 6\ncqt_kernels, KERNEL_WIDTH, lengths, _ = prepare_cqt_kernel(\n    sr=2048,\n    hop_length=HOP_LENGTH,\n    fmin=20,\n    fmax=1024,\n    bins_per_octave=9)\nLENGTHS = tf.constant(lengths, dtype=tf.float32)\nCQT_KERNELS_REAL = tf.constant(np.swapaxes(cqt_kernels.real[:, np.newaxis, :], 0, 2))\nCQT_KERNELS_IMAG = tf.constant(np.swapaxes(cqt_kernels.imag[:, np.newaxis, :], 0, 2))\nPADDING = tf.constant([[0, 0],\n                        [KERNEL_WIDTH // 2, KERNEL_WIDTH // 2],\n                        [0, 0]])","metadata":{"execution":{"iopub.status.busy":"2021-08-30T13:25:02.084223Z","iopub.execute_input":"2021-08-30T13:25:02.084899Z","iopub.status.idle":"2021-08-30T13:25:02.130025Z","shell.execute_reply.started":"2021-08-30T13:25:02.084862Z","shell.execute_reply":"2021-08-30T13:25:02.129031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)\n\n# Function to prepare image\ndef prepare_image(wave):\n    # Decode raw\n    wave = tf.reshape(tf.io.decode_raw(wave, tf.float64), (3, 4096))\n    normalized_waves = []\n    # Normalize\n    for i in range(3):\n        normalized_wave = wave[i] / tf.math.reduce_max(wave[i])\n        normalized_waves.append(normalized_wave)\n    # Stack and cast\n    wave = tf.stack(normalized_waves)\n    wave = tf.cast(wave, tf.float32)\n    # Create image\n    image = create_cqt_image(wave, HOP_LENGTH)\n    # Resize image\n    image = tf.image.resize(image, [*IMAGE_SIZE])\n    # Reshape\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\n# This function parse our images and also get the target variable\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        'wave': tf.io.FixedLenFeature([], tf.string),\n        'wave_id': tf.io.FixedLenFeature([], tf.string),\n        'target': tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = prepare_image(example['wave'])\n    image_id = example['wave_id']\n    target = tf.cast(example['target'], tf.float32)\n    return image, image_id, target\n\n# This function parse our images and also get the target variable\ndef read_unlabeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        'wave': tf.io.FixedLenFeature([], tf.string),\n        'wave_id': tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = prepare_image(example['wave'])\n    image_id = example['wave_id']\n    return image, image_id\n\n# This function loads TF Records and parse them into tensors\ndef load_dataset(filenames, ordered = False, labeled = True):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False \n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO) \n    return dataset\n\n# This function is to get our training dataset\ndef get_training_dataset(filenames, ordered = False, labeled = True):\n    dataset = load_dataset(filenames, ordered = ordered, labeled = labeled)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n# This function is to get our validation and test dataset\ndef get_val_test_dataset(filenames, ordered = True, labeled = True):\n    dataset = load_dataset(filenames, ordered = ordered, labeled = labeled)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) \n    return dataset\n\n# Function to count how many photos we have in\ndef count_data_items(filenames):\n    # The number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_TESTING_IMAGES = count_data_items(TESTING_FILENAMES)\nprint(f'Dataset: {NUM_TRAINING_IMAGES} training images')\nprint(f'Dataset: {NUM_TESTING_IMAGES} testing images')","metadata":{"execution":{"iopub.status.busy":"2021-08-30T13:25:04.472977Z","iopub.execute_input":"2021-08-30T13:25:04.473349Z","iopub.status.idle":"2021-08-30T13:25:04.496694Z","shell.execute_reply.started":"2021-08-30T13:25:04.473316Z","shell.execute_reply":"2021-08-30T13:25:04.495889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Learning rate callback function\ndef get_lr_callback():\n    lr_start   = 0.0001\n    lr_max     = 0.000015 * BATCH_SIZE\n    lr_min     = 0.0000001\n    lr_ramp_ep = 3\n    lr_sus_ep  = 0\n    lr_decay   = 0.7\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start   \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max    \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min    \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = VERBOSE)\n    return lr_callback\n\n# Function to create our EfficientNetB7 model\ndef get_model():\n    with strategy.scope():\n        inp = tf.keras.layers.Input(shape = (*IMAGE_SIZE, 3))\n        x = efn.EfficientNetB7(include_top = False, weights = 'imagenet')(inp)\n        x = tf.keras.layers.GlobalAveragePooling2D()(x)\n        output = tf.keras.layers.Dense(1, activation = 'sigmoid')(x)\n        model = tf.keras.models.Model(inputs = [inp], outputs = [output])\n        opt = tf.keras.optimizers.Adam(learning_rate = LR)\n        opt = tfa.optimizers.SWA(opt)\n        model.compile(\n            optimizer = opt,\n            loss = [tf.keras.losses.BinaryCrossentropy()],\n            metrics = [tf.keras.metrics.AUC()]\n        )\n        return model\n    \n# Function to train a model with 100% of the data\ndef train_and_evaluate():\n    print('\\n')\n    print('-'*50)\n    print(f'Training EFFB7 with 100% of the data with seed {SEED} for {EPOCHS} epochs')\n    if tpu:\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n    train_dataset = get_training_dataset(TRAINING_FILENAMES, ordered = False, labeled = True)\n    train_dataset = train_dataset.map(lambda image, image_id, target: (image, target))\n    STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // (BATCH_SIZE * 4)\n    K.clear_session()\n    # Seed everything\n    seed_everything(SEED)\n    model = get_model()\n    history = model.fit(train_dataset,\n                        steps_per_epoch = STEPS_PER_EPOCH,\n                        epochs = EPOCHS,\n                        callbacks = [get_lr_callback()], \n                        verbose = VERBOSE)\n        \n    print('\\n')\n    print('-'*50)\n    print('Test inference...')\n    # Predict the test set \n    dataset = get_val_test_dataset(TESTING_FILENAMES, ordered = True, labeled = False)\n    image = dataset.map(lambda image, image_id: image)\n    test_predictions = model.predict(image).astype(np.float32).reshape(-1)\n    # Get the test set image_id\n    image_id = dataset.map(lambda image, image_id: image_id).unbatch()\n    image_id = next(iter(image_id.batch(NUM_TESTING_IMAGES))).numpy().astype('U')\n    # Create dataframe output\n    test_df = pd.DataFrame({'id': image_id, 'target': test_predictions})\n    # Save test dataframe to disk\n    test_df.to_csv(f'TEST_EfficientNetB7_{IMAGE_SIZE[0]}_{SEED}.csv', index = False)\n    \ntrain_and_evaluate()","metadata":{"execution":{"iopub.status.busy":"2021-08-30T13:25:05.819381Z","iopub.execute_input":"2021-08-30T13:25:05.819998Z"},"trusted":true},"execution_count":null,"outputs":[]}]}