{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Library","metadata":{}},{"cell_type":"code","source":"!pip install -q efficientnet\nimport re\nimport os\nimport numpy as np\nimport pandas as pd\nfrom scipy.signal import get_window\nimport scipy\nfrom scipy import signal\nfrom typing import Optional, Tuple\nimport warnings\nimport random\nimport math\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nfrom sklearn import metrics\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import mixed_precision\nimport tensorflow_addons as tfa\nfrom kaggle_datasets import KaggleDatasets","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-27T23:39:12.532536Z","iopub.execute_input":"2021-09-27T23:39:12.534111Z","iopub.status.idle":"2021-09-27T23:39:30.107308Z","shell.execute_reply.started":"2021-09-27T23:39:12.53381Z","shell.execute_reply":"2021-09-27T23:39:30.106417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TPU setting","metadata":{}},{"cell_type":"code","source":"# Function to get hardware strategy\ndef get_hardware_strategy():\n    try:\n        # TPU detection. No parameters necessary if TPU_NAME environment variable is\n        # set: this is always the case on Kaggle.\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        policy = mixed_precision.Policy('mixed_bfloat16')\n        mixed_precision.set_global_policy(policy)\n        tf.config.optimizer.set_jit(True)\n    else:\n        # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n        strategy = tf.distribute.get_strategy()\n\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n    return tpu, strategy\n\ntpu, strategy = get_hardware_strategy()","metadata":{"execution":{"iopub.status.busy":"2021-09-27T23:39:30.10961Z","iopub.execute_input":"2021-09-27T23:39:30.109882Z","iopub.status.idle":"2021-09-27T23:39:30.122887Z","shell.execute_reply.started":"2021-09-27T23:39:30.109852Z","shell.execute_reply":"2021-09-27T23:39:30.122072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TRAINING_FILENAMES","metadata":{}},{"cell_type":"code","source":"# gcs_paths = []\n# for i, j in [(0, 4), (5, 9), (10, 14), (15, 19)]:\n#     path = f\"g2net-{i}-{j}\"\n#     n_trial = 0\n#     while True:\n#         try:\n#             gcs_path = KaggleDatasets().get_gcs_path(path)\n#             gcs_paths.append(gcs_path)\n#             print(gcs_path)\n#             break\n#         except:\n#             if n_trial > 10:\n#                 break\n#             n_trial += 1\n#             continue\n            \n# all_files = []\n# for path in gcs_paths:\n#     all_files.extend(np.sort(np.array(tf.io.gfile.glob(path + \"/train*.tfrecords\"))))\n    \n# print(\"len(TRAING_FILENAMES): \", len(all_files))\n# TRAINING_FILENAMES = np.array(all_files)\n# print(TRAINING_FILENAMES)\n\nTRAINING_FILENAMES = ['gs://kds-6cf835d7b0399b3b7487f9e2bc44a0a597f7269b0a5dc64503759327/train0.tfrecords',\n 'gs://kds-6cf835d7b0399b3b7487f9e2bc44a0a597f7269b0a5dc64503759327/train1.tfrecords',\n 'gs://kds-6cf835d7b0399b3b7487f9e2bc44a0a597f7269b0a5dc64503759327/train2.tfrecords',\n 'gs://kds-6cf835d7b0399b3b7487f9e2bc44a0a597f7269b0a5dc64503759327/train3.tfrecords',\n 'gs://kds-6cf835d7b0399b3b7487f9e2bc44a0a597f7269b0a5dc64503759327/train4.tfrecords',\n 'gs://kds-f51bfb96f63912093e9f258d9b50e21792f21a559124913eeba33c53/train5.tfrecords',\n 'gs://kds-f51bfb96f63912093e9f258d9b50e21792f21a559124913eeba33c53/train6.tfrecords',\n 'gs://kds-f51bfb96f63912093e9f258d9b50e21792f21a559124913eeba33c53/train7.tfrecords',\n 'gs://kds-f51bfb96f63912093e9f258d9b50e21792f21a559124913eeba33c53/train8.tfrecords',\n 'gs://kds-f51bfb96f63912093e9f258d9b50e21792f21a559124913eeba33c53/train9.tfrecords',\n 'gs://kds-c5c71376d55342b16ebb8d8e24a7042c3f1b10ef040b8acd51b51dfa/train10.tfrecords',\n 'gs://kds-c5c71376d55342b16ebb8d8e24a7042c3f1b10ef040b8acd51b51dfa/train11.tfrecords',\n 'gs://kds-c5c71376d55342b16ebb8d8e24a7042c3f1b10ef040b8acd51b51dfa/train12.tfrecords',\n 'gs://kds-c5c71376d55342b16ebb8d8e24a7042c3f1b10ef040b8acd51b51dfa/train13.tfrecords',\n 'gs://kds-c5c71376d55342b16ebb8d8e24a7042c3f1b10ef040b8acd51b51dfa/train14.tfrecords',\n 'gs://kds-d6e1eed6caeed4f1bdbc7b59fab2acf53d5d4ecdb9512dc3c352699d/train15.tfrecords',\n 'gs://kds-d6e1eed6caeed4f1bdbc7b59fab2acf53d5d4ecdb9512dc3c352699d/train16.tfrecords',\n 'gs://kds-d6e1eed6caeed4f1bdbc7b59fab2acf53d5d4ecdb9512dc3c352699d/train17.tfrecords',\n 'gs://kds-d6e1eed6caeed4f1bdbc7b59fab2acf53d5d4ecdb9512dc3c352699d/train18.tfrecords',\n 'gs://kds-d6e1eed6caeed4f1bdbc7b59fab2acf53d5d4ecdb9512dc3c352699d/train19.tfrecords']\nTRAINING_FILENAMES = np.array(TRAINING_FILENAMES)\nTESTING_FILENAMES = ['gs://kds-3c2e73051cda6adba0edd3ed492d3915845498b199954c0a6999f73d/test0.tfrecords',\n 'gs://kds-3c2e73051cda6adba0edd3ed492d3915845498b199954c0a6999f73d/test1.tfrecords',\n 'gs://kds-3c2e73051cda6adba0edd3ed492d3915845498b199954c0a6999f73d/test2.tfrecords',\n 'gs://kds-3c2e73051cda6adba0edd3ed492d3915845498b199954c0a6999f73d/test3.tfrecords',\n 'gs://kds-3c2e73051cda6adba0edd3ed492d3915845498b199954c0a6999f73d/test4.tfrecords',\n 'gs://kds-182e7918eb5c98eaf41fb0f78d786cd8ec7cf9bfb3b5a4f971eb6e1e/test5.tfrecords',\n 'gs://kds-182e7918eb5c98eaf41fb0f78d786cd8ec7cf9bfb3b5a4f971eb6e1e/test6.tfrecords',\n 'gs://kds-182e7918eb5c98eaf41fb0f78d786cd8ec7cf9bfb3b5a4f971eb6e1e/test7.tfrecords',\n 'gs://kds-182e7918eb5c98eaf41fb0f78d786cd8ec7cf9bfb3b5a4f971eb6e1e/test8.tfrecords',\n 'gs://kds-182e7918eb5c98eaf41fb0f78d786cd8ec7cf9bfb3b5a4f971eb6e1e/test9.tfrecords']","metadata":{"execution":{"iopub.status.busy":"2021-09-27T23:39:30.124425Z","iopub.execute_input":"2021-09-27T23:39:30.124667Z","iopub.status.idle":"2021-09-27T23:39:30.135957Z","shell.execute_reply.started":"2021-09-27T23:39:30.124639Z","shell.execute_reply":"2021-09-27T23:39:30.13516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TESTING_FILENAMES","metadata":{}},{"cell_type":"code","source":"# gcs_paths = []\n# for i, j in [(0, 4), (5, 9)]:\n#     path = f\"g2net-test-{i}-{j}\"\n#     n_trial = 0\n#     while True:\n#         try:\n#             gcs_path = KaggleDatasets().get_gcs_path(path)\n#             gcs_paths.append(gcs_path)\n#             print(gcs_path)\n#             break\n#         except:\n#             if n_trial > 10:\n#                 break\n#             n_trial += 1\n#             continue\n            \n# all_files = []\n# for path in gcs_paths:\n#     all_files.extend(np.sort(np.array(tf.io.gfile.glob(path + \"/test*.tfrecords\"))))\n    \n# print(\"test_files: \", len(all_files))\n# TESTING_FILENAME  = np.array(all_files)\n# print(TESTING_FILENAME)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T23:39:30.13801Z","iopub.execute_input":"2021-09-27T23:39:30.138282Z","iopub.status.idle":"2021-09-27T23:39:30.151191Z","shell.execute_reply.started":"2021-09-27T23:39:30.138253Z","shell.execute_reply":"2021-09-27T23:39:30.150139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_data_items(fileids, train=True):\n    \"\"\"\n    Count the number of samples.\n    Each of the TFRecord datasets is designed to contain 28000 samples for train\n    22500 for test.\n    \"\"\"\n    sizes = 28000 if train else 22600\n    return len(fileids) * sizes\n\n\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"execution":{"iopub.status.busy":"2021-09-27T23:39:30.152651Z","iopub.execute_input":"2021-09-27T23:39:30.152959Z","iopub.status.idle":"2021-09-27T23:39:30.162942Z","shell.execute_reply.started":"2021-09-27T23:39:30.152918Z","shell.execute_reply":"2021-09-27T23:39:30.162335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class CFG:\n    debug = False\n    n_fold = 5\n    seed = 42\n    trn_fold = [2]\n\n# Configuration\nEPOCHS = 16\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\nIMAGE_SIZE = [512, 512]\nSEED = 42\nLR = 0.0001\nVERBOSE = 2\nSAVEDIR = 'models/'\nif not os.path.exists(SAVEDIR):\n    os.makedirs(SAVEDIR)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T23:39:30.16433Z","iopub.execute_input":"2021-09-27T23:39:30.164655Z","iopub.status.idle":"2021-09-27T23:39:30.180937Z","shell.execute_reply.started":"2021-09-27T23:39:30.164618Z","shell.execute_reply":"2021-09-27T23:39:30.180102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CQT","metadata":{}},{"cell_type":"code","source":"# Function to create cqt kernel\ndef create_cqt_kernels(\n    q: float,\n    fs: float,\n    fmin: float,\n    n_bins: int = 84,\n    bins_per_octave: int = 12,\n    norm: float = 1,\n    window: str = \"hann\",\n    fmax: Optional[float] = None,\n    topbin_check: bool = True\n) -> Tuple[np.ndarray, int, np.ndarray, float]:\n    fft_len = 2 ** _nextpow2(np.ceil(q * fs / fmin))\n    \n    if (fmax is not None) and (n_bins is None):\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    elif (fmax is None) and (n_bins is not None):\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n    else:\n        warnings.warn(\"If nmax is given, n_bins will be ignored\", SyntaxWarning)\n        n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin))\n        freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))\n        \n    if np.max(freqs) > fs / 2 and topbin_check:\n        raise ValueError(f\"The top bin {np.max(freqs)} Hz has exceeded the Nyquist frequency, \\\n                           please reduce the `n_bins`\")\n    \n    kernel = np.zeros((int(n_bins), int(fft_len)), dtype=np.complex64)\n    \n    length = np.ceil(q * fs / freqs)\n    for k in range(0, int(n_bins)):\n        freq = freqs[k]\n        l = np.ceil(q * fs / freq)\n        \n        if l % 2 == 1:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0)) - 1\n        else:\n            start = int(np.ceil(fft_len / 2.0 - l / 2.0))\n\n        sig = get_window(window, int(l), fftbins=True) * np.exp(\n            np.r_[-l // 2:l // 2] * 1j * 2 * np.pi * freq / fs) / l\n        \n        if norm:\n            kernel[k, start:start + int(l)] = sig / np.linalg.norm(sig, norm)\n        else:\n            kernel[k, start:start + int(l)] = sig\n    return kernel, fft_len, length, freqs\n\n\ndef _nextpow2(a: float) -> int:\n    return int(np.ceil(np.log2(a)))\n\n# Function to prepare cqt kernel\ndef prepare_cqt_kernel(\n    sr=22050,\n    hop_length=512,\n    fmin=32.70,\n    fmax=None,\n    n_bins=84,\n    bins_per_octave=12,\n    norm=1,\n    filter_scale=1,\n    window=\"hann\"\n):\n    q = float(filter_scale) / (2 ** (1 / bins_per_octave) - 1)\n    print(q)\n    return create_cqt_kernels(q, sr, fmin, n_bins, bins_per_octave, norm, window, fmax)\n\n# Function to create cqt image\ndef create_cqt_image(wave, hop_length=16):\n    CQTs = []\n    for i in range(3):\n        x = wave[i]\n        x = tf.expand_dims(tf.expand_dims(x, 0), 2)\n        x = tf.pad(x, PADDING, \"REFLECT\")\n\n        CQT_real = tf.nn.conv1d(x, CQT_KERNELS_REAL, stride=hop_length, padding=\"VALID\")\n        CQT_imag = -tf.nn.conv1d(x, CQT_KERNELS_IMAG, stride=hop_length, padding=\"VALID\")\n        CQT_real *= tf.math.sqrt(LENGTHS)\n        CQT_imag *= tf.math.sqrt(LENGTHS)\n\n        CQT = tf.math.sqrt(tf.pow(CQT_real, 2) + tf.pow(CQT_imag, 2))\n        CQTs.append(CQT[0])\n    return tf.stack(CQTs, axis=2)\n\nHOP_LENGTH = 6\ncqt_kernels, KERNEL_WIDTH, lengths, _ = prepare_cqt_kernel(\n    sr=2048,\n    hop_length=HOP_LENGTH,\n    fmin=20,\n    fmax=500,\n    bins_per_octave=9)\nLENGTHS = tf.constant(lengths, dtype=tf.float32)\nCQT_KERNELS_REAL = tf.constant(np.swapaxes(cqt_kernels.real[:, np.newaxis, :], 0, 2))\nCQT_KERNELS_IMAG = tf.constant(np.swapaxes(cqt_kernels.imag[:, np.newaxis, :], 0, 2))\nPADDING = tf.constant([[0, 0],\n                        [KERNEL_WIDTH // 2, KERNEL_WIDTH // 2],\n                        [0, 0]])","metadata":{"execution":{"iopub.status.busy":"2021-09-27T23:39:30.182595Z","iopub.execute_input":"2021-09-27T23:39:30.182848Z","iopub.status.idle":"2021-09-27T23:39:30.243735Z","shell.execute_reply.started":"2021-09-27T23:39:30.182821Z","shell.execute_reply":"2021-09-27T23:39:30.243162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)          \n\n# Function to prepare image\ndef prepare_image(wave):\n    # Decode raw\n    wave = tf.reshape(tf.io.decode_raw(wave, np.float64), (3, 4096))\n    normalized_waves = [wave[0], wave[1], wave[2]]\n    # Stack and cast\n    wave = tf.stack(normalized_waves)\n    wave = tf.cast(wave, tf.float32)\n    # Create image\n    image = create_cqt_image(wave, HOP_LENGTH)\n    # Resize image\n    image = tf.image.resize(image, [*IMAGE_SIZE])\n    # Reshape\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\n# This function parse our images and also get the target variable\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        'wave': tf.io.FixedLenFeature([], tf.string),\n        'wave_id': tf.io.FixedLenFeature([], tf.string),\n        'target': tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = prepare_image(example['wave'])\n    image_id = example['wave_id']\n    target = tf.cast(example['target'], tf.float32)\n    return image, image_id, target\n\n# This function parse our images and also get the target variable\ndef read_unlabeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        'wave': tf.io.FixedLenFeature([], tf.string),\n        'wave_id': tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = prepare_image(example['wave'])\n    image_id = example['wave_id']\n    return image, image_id\n\n# This function loads TF Records and parse them into tensors\ndef load_dataset(filenames, ordered = False, labeled = True):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False \n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO, compression_type=\"GZIP\")\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO) \n    return dataset\n\n# This function is to get our training dataset\ndef get_training_dataset(filenames, ordered = False, labeled = True):\n    dataset = load_dataset(filenames, ordered = ordered, labeled = labeled)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n# This function is to get our validation and test dataset\ndef get_val_test_dataset(filenames, ordered = True, labeled = True):\n    dataset = load_dataset(filenames, ordered = ordered, labeled = labeled)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) \n    return dataset\n\n# # Function to count how many photos we have in\n# def count_data_items(filenames):\n#     # The number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n#     n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n#     return np.sum(n)\n\ndef count_data_items(fileids, train=True):\n    \"\"\"\n    Count the number of samples.\n    Each of the TFRecord datasets is designed to contain 28000 samples for train\n    22500 for test.\n    \"\"\"\n    sizes = 28000 if train else 22600\n    return len(fileids) * sizes\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES, train=True)\nNUM_TESTING_IMAGES = count_data_items(TESTING_FILENAMES, train=False)\nprint(f'Dataset: {NUM_TRAINING_IMAGES} training images')\nprint(f'Dataset: {NUM_TESTING_IMAGES} testing images')","metadata":{"execution":{"iopub.status.busy":"2021-09-27T23:39:30.244894Z","iopub.execute_input":"2021-09-27T23:39:30.245267Z","iopub.status.idle":"2021-09-27T23:39:30.266267Z","shell.execute_reply.started":"2021-09-27T23:39:30.245239Z","shell.execute_reply":"2021-09-27T23:39:30.26537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"# Learning rate callback function\ndef get_lr_callback():\n    lr_start   = 0.0001\n    lr_max     = 0.000015 * BATCH_SIZE\n    lr_min     = 0.0000001\n    lr_ramp_ep = 3\n    lr_sus_ep  = 0\n    lr_decay   = 0.7\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start   \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max    \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min    \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = VERBOSE)\n    return lr_callback\n\n# Function to create our EfficientNetB7 model\ndef get_model():\n    with strategy.scope():\n        inp = tf.keras.layers.Input(shape = (*IMAGE_SIZE, 3))\n        x = efn.EfficientNetB7(include_top = False, weights = 'imagenet')(inp)\n        x = tf.keras.layers.GlobalAveragePooling2D()(x)\n        output = tf.keras.layers.Dense(1, activation = 'sigmoid')(x)\n        model = tf.keras.models.Model(inputs = [inp], outputs = [output])\n        opt = tf.keras.optimizers.Adam(learning_rate = LR)\n        opt = tfa.optimizers.SWA(opt)\n        model.compile(\n            optimizer = opt,\n            loss = [tf.keras.losses.BinaryCrossentropy()],\n            metrics = [tf.keras.metrics.AUC()]\n        )\n        return model","metadata":{"execution":{"iopub.status.busy":"2021-09-27T23:39:30.267682Z","iopub.execute_input":"2021-09-27T23:39:30.267896Z","iopub.status.idle":"2021-09-27T23:39:30.282795Z","shell.execute_reply.started":"2021-09-27T23:39:30.267872Z","shell.execute_reply":"2021-09-27T23:39:30.281957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train_loop","metadata":{}},{"cell_type":"code","source":"def train_loop(train_filename:list, valid_filename:list, fold:int):\n    print('\\n')\n    print('========== fold : {} =========='.format(fold))\n    if tpu:\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n    NUM_TRAINING_IMAGES = count_data_items(train_filename, train=True)\n    NUM_VALID_IMAGES = count_data_items(valid_filename, train=True)\n    print(f'Dataset: {NUM_TRAINING_IMAGES} training images')\n    print(f'Dataset: {NUM_VALID_IMAGES} valid images')\n    \n    train_dataset = get_training_dataset(train_filename, ordered = False, labeled = True)\n    train_dataset = train_dataset.map(lambda image, image_id, target: (image, target))\n\n    valid_dataset = get_val_test_dataset(valid_filename, ordered = True, labeled = True)\n    valid_dataset = valid_dataset.map(lambda image, image_id, target: (image, target))\n    \n    STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // (BATCH_SIZE * 4)#batchの1/4でschedulerを起動させるため。\n    #STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // (BATCH_SIZE)#通常モード\n    K.clear_session()\n    # Seed everything\n    seed_everything(SEED)\n    model = get_model()\n    model_ckpt = tf.keras.callbacks.ModelCheckpoint(\n        str(SAVEDIR + f\"fold{fold}.h5\"), monitor=\"val_auc\", verbose=1, save_best_only=True,\n        save_weights_only=True, mode=\"max\", save_freq=\"epoch\")\n    history = model.fit(train_dataset,\n                        steps_per_epoch = STEPS_PER_EPOCH,\n                        epochs = EPOCHS,\n                        callbacks = [model_ckpt, get_lr_callback()], \n                        verbose = VERBOSE, \n                        validation_data = valid_dataset)\n    best_val_auc = np.max(history.history['val_auc'])\n    best_val_loss = np.min(history.history['val_loss'])\n    #best_val_auc = np.max(history.history['auc'])\n    #best_val_loss = np.min(history.history['loss'])\n    return best_val_auc, best_val_loss","metadata":{"execution":{"iopub.status.busy":"2021-09-27T23:39:30.284986Z","iopub.execute_input":"2021-09-27T23:39:30.28521Z","iopub.status.idle":"2021-09-27T23:39:30.300245Z","shell.execute_reply.started":"2021-09-27T23:39:30.285185Z","shell.execute_reply":"2021-09-27T23:39:30.299368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# K-fold","metadata":{}},{"cell_type":"code","source":"kf = KFold(n_splits=CFG.n_fold, shuffle = True, random_state=CFG.seed)\nfolds = list(kf.split(TRAINING_FILENAMES))\n#print(folds)\nlist_best_auc = []\nlist_best_loss = []\nfor fold in range(CFG.n_fold):\n    if fold in CFG.trn_fold:\n        trn_idx, val_idx = folds[fold]\n        train_files = TRAINING_FILENAMES[trn_idx].tolist()\n        valid_files = TRAINING_FILENAMES[val_idx].tolist()\n        best_val_auc, best_val_loss = train_loop(train_files, valid_files, fold)\n        list_best_auc.append(best_val_auc)\n        list_best_loss.append(best_val_loss)\n        print('========== fold : {} result =========='.format(fold))\n        print('best_auc : {}, best_loss : {}'.format(best_val_auc, best_val_loss))\nprint('========== CV result ==========')\nprint('avg_auc : {}, avg_loss : {}'.format(sum(list_best_auc)/len(list_best_auc), sum(list_best_loss)/len(list_best_loss)))","metadata":{"execution":{"iopub.status.busy":"2021-09-27T23:39:30.301369Z","iopub.execute_input":"2021-09-27T23:39:30.301666Z","iopub.status.idle":"2021-09-27T23:40:50.049015Z","shell.execute_reply.started":"2021-09-27T23:39:30.301636Z","shell.execute_reply":"2021-09-27T23:40:50.04659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"def inference():\n    for fold in range(CFG.n_fold):\n        if fold in CFG.trn_fold:\n            print('================fold : {} ==============='.format(fold))\n            print(\"Loading best model...\")\n            model = get_model()\n            #model.load_weights('../input/tpu-nobandpass-fold0-model/fold0.h5')\n            model.load_weights(SAVEDIR + 'fold{}.h5'.format(fold))\n            dataset = get_val_test_dataset(TESTING_FILENAMES, ordered = True, labeled = False)\n            image = dataset.map(lambda image, image_id: image)\n            test_predictions = model.predict(image).astype(np.float32).reshape(-1)\n            # Get the test set image_id\n            image_id = dataset.map(lambda image, image_id: image_id).unbatch()\n            image_id = next(iter(image_id.batch(NUM_TESTING_IMAGES))).numpy().astype('U')\n            # Create dataframe output\n            test_df = pd.DataFrame({'id': image_id, 'target': test_predictions})\n            # Save test dataframe to disk\n            test_df.to_csv(f'TEST_EfficientNetB7_20-500_{IMAGE_SIZE[0]}_{SEED}_{fold}.csv', index = False)\ninference()","metadata":{"execution":{"iopub.status.busy":"2021-09-27T23:40:50.049993Z","iopub.status.idle":"2021-09-27T23:40:50.05034Z","shell.execute_reply.started":"2021-09-27T23:40:50.050174Z","shell.execute_reply":"2021-09-27T23:40:50.050191Z"},"trusted":true},"execution_count":null,"outputs":[]}]}