{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q efficientnet\nimport re\nimport os\nimport numpy as np\nimport pandas as pd\nfrom scipy.signal import get_window\nfrom typing import Optional, Tuple\nimport warnings\nimport random\nimport math\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nfrom sklearn import metrics\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import mixed_precision\nimport tensorflow_addons as tfa\nfrom tensorflow.keras.layers.experimental import preprocessing\nfrom kaggle_datasets import KaggleDatasets","metadata":{"execution":{"iopub.status.busy":"2021-09-29T07:10:57.475911Z","iopub.execute_input":"2021-09-29T07:10:57.476629Z","iopub.status.idle":"2021-09-29T07:11:16.736719Z","shell.execute_reply.started":"2021-09-29T07:10:57.476536Z","shell.execute_reply":"2021-09-29T07:11:16.735685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_hardware_strategy():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        policy = mixed_precision.Policy('mixed_bfloat16')\n        mixed_precision.set_global_policy(policy)\n        tf.config.optimizer.set_jit(True)\n    else:\n        strategy = tf.distribute.get_strategy()\n\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n    return tpu, strategy\n\ntpu, strategy = get_hardware_strategy()","metadata":{"execution":{"iopub.status.busy":"2021-09-29T07:11:47.62526Z","iopub.execute_input":"2021-09-29T07:11:47.625581Z","iopub.status.idle":"2021-09-29T07:11:47.640314Z","shell.execute_reply.started":"2021-09-29T07:11:47.62555Z","shell.execute_reply":"2021-09-29T07:11:47.639207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\nGCS_PATH1 = KaggleDatasets().get_gcs_path('g2net-tf-records-tr-bp-filter-1')\nGCS_PATH2 = KaggleDatasets().get_gcs_path('g2net-tf-records-tr-bp-filter-2')\nGCS_PATH3 = KaggleDatasets().get_gcs_path('g2net-tf-records-tr-bp-filter-3')\nGCS_PATH4 = KaggleDatasets().get_gcs_path('g2net-tf-records-ts-bp-filter-1')\nGCS_PATH5 = KaggleDatasets().get_gcs_path('g2net-tf-records-ts-bp-filter-2')\n\nEPOCHS = 20\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\nIMAGE_SIZE = [512, 512]\nSEED = 96\nLR = 0.0001\nVERBOSE = 2\n\nMEAN0 = np.load('../input/fullmean3d/mean3d.npy', allow_pickle=False, encoding='ASCII')\nIMAGE0 = tf.convert_to_tensor(MEAN0, dtype=tf.float32)       # [513, 137, 3]\n\nTRAINING_FILENAMES=tf.io.gfile.glob(GCS_PATH1+'/train*.tfrec')+tf.io.gfile.glob(GCS_PATH2+'/train*.tfrec')+tf.io.gfile.glob(GCS_PATH3+'/train*.tfrec')\nTESTING_FILENAMES = tf.io.gfile.glob(GCS_PATH4 + '/test*.tfrec') + tf.io.gfile.glob(GCS_PATH5 + '/test*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2021-09-29T07:12:14.74526Z","iopub.execute_input":"2021-09-29T07:12:14.745661Z","iopub.status.idle":"2021-09-29T07:12:22.584469Z","shell.execute_reply.started":"2021-09-29T07:12:14.745632Z","shell.execute_reply":"2021-09-29T07:12:22.583403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_cqt_kernels(q: float, fs: float, fmin: float, n_bins: int = 84, bins_per_octave: int = 12,\n                       norm: float=1, window: str=\"hann\", fmax: Optional[float]=None, topbin_check: bool=True\n                       ) -> Tuple[np.ndarray, int, np.ndarray, float]:\n    fft_len = 2 ** _nextpow2(np.ceil(q * fs / fmin))\n    \n    if (fmax is not None) and (n_bins is None):\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    elif (fmax is None) and (n_bins is not None):\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    else:\n        warnings.warn(\"If nmax is given, n_bins will be ignored\", SyntaxWarning)\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n        \n    if np.max(freqs) > fs / 2 and topbin_check:\n        raise ValueError(f\"The top bin {np.max(freqs)} Hz has exceeded the Nyquist frequency, \\\n                           please reduce the `n_bins`\")\n    \n    kernel = np.zeros((int(n_bins), int(fft_len)), dtype=np.complex64)\n    \n    length = np.ceil(q * fs / freqs)\n    for k in range(0, int(n_bins)):\n        freq = freqs[k]\n        l = np.ceil(q * fs / freq)\n        if l % 2 == 1:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0)) - 1\n        else:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0))\n        sig = get_window(window, int(l), fftbins=True) * np.exp(\n            np.r_[-l // 2:l // 2] * 1j * 2 * np.pi * freq / fs) / l\n        if norm:\n            kernel[k, start:start + int(l)] = sig / np.linalg.norm(sig, norm)\n        else:\n            kernel[k, start:start + int(l)] = sig\n    return kernel, fft_len, length, freqs\n\n\ndef _nextpow2(a: float) -> int:\n    return int(np.ceil(np.log2(a)))\n\n\ndef prepare_cqt_kernel(sr=22050, hop_length=512, fmin=32.70, fmax=None, n_bins=84, bins_per_octave=12,\n                       norm=1, filter_scale=1, window=\"hann\"):\n    q = float(filter_scale) / (2 ** (1 / bins_per_octave) - 1)\n    print(q)\n    return create_cqt_kernels(q, sr, fmin, n_bins, bins_per_octave, norm, window, fmax)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_cqt_image(wave, hop_length=16):\n    CQTs = []\n    for i in range(3):\n        x = wave[i]\n        x = tf.expand_dims(tf.expand_dims(x, 0), 2)\n        x = tf.pad(x, PADDING, \"REFLECT\")\n        CQT_real = tf.nn.conv1d(x, CQT_KERNELS_REAL, stride=hop_length, padding=\"VALID\")\n        CQT_imag = -tf.nn.conv1d(x, CQT_KERNELS_IMAG, stride=hop_length, padding=\"VALID\")\n        CQT_real *= tf.math.sqrt(LENGTHS)\n        CQT_imag *= tf.math.sqrt(LENGTHS)\n        CQT = tf.math.sqrt(tf.pow(CQT_real, 2) + tf.pow(CQT_imag, 2))\n        CQTs.append(CQT[0])\n    return tf.stack(CQTs, axis=2)\n\nHOP_LENGTH = 8\ncqt_kernels, KERNEL_WIDTH, lengths, _ = prepare_cqt_kernel(sr=2048, hop_length=HOP_LENGTH, fmin=20,\n                                                           fmax=1024, bins_per_octave=24, filter_scale= .4)\n\nLENGTHS = tf.constant(lengths, dtype=tf.float32)\nCQT_KERNELS_REAL = tf.constant(np.swapaxes(cqt_kernels.real[:, np.newaxis, :], 0, 2))\nCQT_KERNELS_IMAG = tf.constant(np.swapaxes(cqt_kernels.imag[:, np.newaxis, :], 0, 2))\nPADDING = tf.constant([[0, 0], [KERNEL_WIDTH // 2, KERNEL_WIDTH // 2], [0, 0]])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)\n\ndef prepare_image(wave):\n    wave = tf.reshape(tf.io.decode_raw(wave, tf.float64), (3, 4096))\n    normalized_waves = []\n    for i in range(3):\n        normalized_wave = wave[i]             # / tf.math.reduce_max(wave[i])\n        normalized_waves.append(normalized_wave)\n    wave = tf.stack(normalized_waves)\n    wave = tf.cast(wave, tf.float32)\n    wave = wave/ tf.math.reduce_max(wave)\n    image = create_cqt_image(wave, HOP_LENGTH)\n    imax = tf.math.reduce_max(image)\n    image = tf.clip_by_value((image - IMAGE0), 0, imax)\n    #image = tf.transpose(image)\n    #print(image.shape.as_list())\n    image = tf.image.resize(image, [512, 512])\n    image = image/tf.math.reduce_max(image)\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {'wave': tf.io.FixedLenFeature([], tf.string),\n                            'wave_id': tf.io.FixedLenFeature([], tf.string),\n                            'target': tf.io.FixedLenFeature([], tf.int64)}\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = prepare_image(example['wave'])\n    image_id = example['wave_id']\n    target = tf.cast(example['target'], tf.float32)\n    return image, image_id, target\n\ndef read_unlabeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {'wave': tf.io.FixedLenFeature([], tf.string),\n                            'wave_id': tf.io.FixedLenFeature([], tf.string) }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = prepare_image(example['wave'])\n    image_id = example['wave_id']\n    return image, image_id\n\ndef load_dataset(filenames, ordered = False, labeled = True):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO) \n    return dataset\n\ndef get_training_dataset(filenames, ordered = False, labeled = True):\n    dataset = load_dataset(filenames, ordered = ordered, labeled = labeled)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_val_test_dataset(filenames, ordered = True, labeled = True):\n    dataset = load_dataset(filenames, ordered = ordered, labeled = labeled)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) \n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_TESTING_IMAGES = count_data_items(TESTING_FILENAMES)\nprint(f'Dataset: {NUM_TRAINING_IMAGES} training images')\nprint(f'Dataset: {NUM_TESTING_IMAGES} testing images')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback():\n    lr_start   = 0.0001\n    lr_max     = 0.000015 * BATCH_SIZE\n    lr_min     = 0.0000001\n    lr_ramp_ep = 3\n    lr_sus_ep  = 0\n    lr_decay   = 0.7\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start   \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max    \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min    \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = VERBOSE)\n    return lr_callback\n\ndef get_model():\n    with strategy.scope():\n        inp = tf.keras.layers.Input(shape = (*IMAGE_SIZE, 3))\n        x = efn.EfficientNetB7(include_top = False, weights = 'imagenet')(inp)\n        x = tf.keras.layers.GlobalAveragePooling2D()(x)\n        x = tf.keras.layers.Dropout(0.2)(x)\n        output = tf.keras.layers.Dense(1, activation = 'sigmoid')(x)\n        model = tf.keras.models.Model(inputs = [inp], outputs = [output])\n        opt = tf.keras.optimizers.Adam(learning_rate = LR)\n        opt = tfa.optimizers.SWA(opt)\n        model.compile(optimizer = opt, loss = [tf.keras.losses.BinaryCrossentropy()],\n                      metrics = [tf.keras.metrics.AUC()])\n        return model\n    \n\ndef train_and_evaluate(SEED=96):\n    print('\\n')\n    print('-'*50)\n    print(f'Training EFFB7 with 100% of the data with seed {SEED} for {EPOCHS} epochs')\n    if tpu:\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n    train_dataset = get_training_dataset(TRAINING_FILENAMES, ordered = False, labeled = True)\n    train_dataset = train_dataset.map(lambda image, image_id, target: (image, target))\n    STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // (BATCH_SIZE * 4)\n    K.clear_session()\n    seed_everything(SEED)\n    model = get_model()\n    history = model.fit(train_dataset, steps_per_epoch = STEPS_PER_EPOCH, epochs = EPOCHS,\n                        callbacks = [get_lr_callback()], verbose = VERBOSE)\n    print('\\n')\n    print('-'*50)\n    print('Test inference...')\n    \n    dataset = get_val_test_dataset(TESTING_FILENAMES, ordered = True, labeled = False)\n    image = dataset.map(lambda image, image_id: image)\n    test_predictions = model.predict(image).astype(np.float32).reshape(-1)\n    image_id = dataset.map(lambda image, image_id: image_id).unbatch()\n    image_id = next(iter(image_id.batch(NUM_TESTING_IMAGES))).numpy().astype('U')\n    test_df = pd.DataFrame({'id': image_id, 'target': test_predictions})\n    # Save test dataframe to disk\n    test_df.to_csv(f'TEST_EfficientNetB7_{IMAGE_SIZE[0]}_{SEED}.csv', index = False)\n    \ntrain_and_evaluate(SEED=96)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}