{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Description\n\nThis notebook requires high memory, so it will not work on a normal kaggle notebook.<br>\nTo make it work on a normal kaggle notebook, please change the INPUT_DIR and train on a 32x32 image.<br>\nThis notebook is based on the following<br>\n- https://www.kaggle.com/miklgr500/cqt-g2net-efficientnetb7-tpu-training-w-b?scriptVersionId=67969914<br>\n\nThe data for VQT is created at<br>\n- https://www.kaggle.com/snkmr0221/g2net-vqt-features-32x32-or-128x128-img-tfrecord<br>\n- https://www.kaggle.com/snkmr0221/g2net-vqt-features-32x32-img<br>","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"### 説明\n\nこのノートブックは、ハイメモリを必要としますので、通常のkaggle notebookでは動作しません。<br>\n通常のkaggle notebook上で動作させるには、INPUT_DIRを変更し、32x32の画像で学習をしてください。<br>\nこのノートブックは以下を参考にしています。<br>\n- https://www.kaggle.com/miklgr500/cqt-g2net-efficientnetb7-tpu-training-w-b?scriptVersionId=67969914<br>\n\nVQTのデータは、以下で作成しています。<br>\n- https://www.kaggle.com/snkmr0221/g2net-vqt-features-32x32-or-128x128-img-tfrecord<br>\n- https://www.kaggle.com/snkmr0221/g2net-vqt-features-32x32-img<br>","metadata":{}},{"cell_type":"markdown","source":"### Import modules","metadata":{"execution":{"iopub.status.busy":"2021-09-03T03:15:35.154681Z","iopub.execute_input":"2021-09-03T03:15:35.15558Z","iopub.status.idle":"2021-09-03T03:15:35.16698Z","shell.execute_reply.started":"2021-09-03T03:15:35.155527Z","shell.execute_reply":"2021-09-03T03:15:35.165953Z"}}},{"cell_type":"code","source":"# standard modules\nimport os\nimport random\nimport pathlib\nimport sys\n\n# third-party modules\nimport tensorflow as tf\n!pip3 install tensorflow_addons\nimport tensorflow_addons as tfa\n!pip3 install efficientnet\nimport efficientnet.tfkeras as efn\nfrom tensorflow.python.client import device_lib\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nfrom IPython.display import display\nimport seaborn as sns\nimport pandas as pd\n!pip install wandb\nimport wandb\nfrom wandb.keras import WandbCallback\n\nprint(f\"TensorFlow version: {tf.__version__}\")\nprint(f\"Eager execution: {tf.executing_eagerly()}\")\nprint(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\ncuda_version = !nvcc -V\nprint(f\"{[*cuda_version]}\")\ndevices_df = pd.DataFrame(\n    [ [d.name, d.device_type, d.physical_device_desc] for d in device_lib.list_local_devices()]\n, columns=[\"name\", \"device_type\", \"phisical_device_desc\"])\n# devices_df = pd.DataFrame([d.split(\"\\n\") for d in device_lib.list_local_devices()])\ndisplay(devices_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IN_COLAB = 'google.colab' in sys.modules\nif IN_COLAB:\n    gpu_info = !nvidia-smi\n    gpu_info = '\\n'.join(gpu_info)\n    if gpu_info.find('failed') >= 0:\n        print('Select the Runtime > \"Change runtime type\" menu to enable a GPU accelerator, ')\n        print('and then re-execute this cell.')\n    else:\n        print(gpu_info)\n\n    from psutil import virtual_memory\n    ram_gb = virtual_memory().total / 1e9\n    print('Your runtime has {:.1f} gigabytes of available RAM\\n'.format(ram_gb))\n\n    if ram_gb < 20:\n        print('To enable a high-RAM runtime, select the Runtime > \"Change runtime type\"')\n        print('menu, and then select High-RAM in the Runtime shape dropdown. Then, ')\n        print('re-execute this cell.')\n    else:\n        print('You are using a high-RAM runtime!')\n        \n    from google.colab import drive\n    drive.mount('/content/drive')\n\nIN_KAGGLE = \"kaggle_secrets\" in sys.modules","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Config","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed=777):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    random.seed(seed)\n    tf.random.set_seed(seed)\n\n# From https://www.kaggle.com/xhlulu/ranzcr-efficientnet-tpu-training\ndef auto_select_accelerator():\n    TPU_DETECTED = False\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n        TPU_DETECTED =True\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n\n    return strategy, TPU_DETECTED\n\nseed_everything()\n\nstrategy, TPU_DETECTED = auto_select_accelerator()\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\n\nprint(f\"strategy: {strategy}\")\nprint(f\"TPU_DETECTED: {TPU_DETECTED}\")\nprint(f\"AUTO: {AUTO}\")\nprint(f\"REPLICAS: {REPLICAS}\")\n\n# Model Eval Params\nDISPLAY_PLOT = True\n\nCFG = {\n    'version': 0,\n    'feature_name': 'vqt',\n    'fold': 0,\n    'EFFV': 7,\n    'size': 128,\n    'dropout': 0.2,\n    'label_smoothing': 0.,\n    'batch_size': 32,\n    'steps_multiplier': 0.25,\n    'epochs': 20,\n    'aug': False,\n    'MIX_UP_P': 0.2,\n    'S_SHIFT': 0.,\n    'T_SHIFT': 0.,\n    'R_ANGLE': 0. / 180 * np.pi,\n    'opt': 'AdamW',\n    'lr': 1e-3,\n    'lr_start': 1e-4,\n    'lr_max': 0.000015 * REPLICAS * 64,\n    'lr_min': 1e-5,\n    'lr_ramp_ep': 3,\n    'lr_sus_ep': 0,\n    'lr_decay': 0.7,\n    'fold_num': 4,\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### File I/O","metadata":{}},{"cell_type":"code","source":"if IN_COLAB:\n    ROOT_PATH = \"/content/drive/MyDrive/kaggle/kaggle-g2net\"\n    INPUT_DIR = \"input/G2Net-VQT-features-128x128-img-tfrecord\"\n    OUTPUT_DIR = \"exp/exp-008-128x128-efn7-phase\"\n    INPUT_PATH = pathlib.Path(ROOT_PATH).joinpath(INPUT_DIR)\n    OUTPUT_PATH = pathlib.Path(ROOT_PATH).joinpath(OUTPUT_DIR)\n    INPUT_SUB_FILE = pathlib.Path(ROOT_PATH).joinpath(\"input\", \"sample_submission.csv\")\n    OUTPUT_SUB_FILE = pathlib.Path(OUTPUT_PATH).joinpath(\"submission.csv\")\n    MODEL_PATH = OUTPUT_PATH\nelif IN_KAGGLE:\n    ROOT_PATH = \"/kaggle\"\n    INPUT_DIR = \"input/g2net-vqt-features-32x32-img/G2Net-VQT-features-128x128-img-tfrecord/G2Net-VQT-features-128x128-img-tfrecord\"\n    # INPUT_DIR = \"input/g2net-vqt-features-32x32-img/G2Net-VQT-features-32x32-img-tfrecord/G2Net-VQT-features-32x32-img-tfrecord\"\n    OUTPUT_DIR = \"output\"\n    INPUT_PATH = pathlib.Path(ROOT_PATH).joinpath(INPUT_DIR)\n    OUTPUT_PATH = pathlib.Path(ROOT_PATH).joinpath(OUTPUT_DIR)\n    INPUT_SUB_FILE = pathlib.Path(ROOT_PATH).joinpath(\"input\", \"g2net-gravitational-wave-detection\", \"sample_submission.csv\")\n    OUTPUT_SUB_FILE = pathlib.Path(\".\").joinpath(\"submission.csv\")\n    MODEL_PATH = pathlib.Path(ROOT_PATH).joinpath(\"input\", \"g2net-vqt-128x128-tfrecord-efn7-trained-model\")\nelse: # Local\n    ROOT_PATH = \".\"\n    INPUT_DIR = \"input/G2Net-VQT-features-128x128-img-tfrecord\"\n    OUTPUT_DIR = \"exp/exp-008-128x128-efn7-phase\"\n    INPUT_PATH = pathlib.Path(ROOT_PATH).joinpath(INPUT_DIR)\n    OUTPUT_PATH = pathlib.Path(ROOT_PATH).joinpath(OUTPUT_DIR)\n    INPUT_SUB_FILE = pathlib.Path(ROOT_PATH).joinpath(\"input\", \"sample_submission.csv\")\n    OUTPUT_SUB_FILE = pathlib.Path(OUTPUT_PATH).joinpath(\"submission.csv\")\n    MODEL_PATH = OUTPUT_PATH\n\nOUTPUT_PATH.mkdir(parents=True, exist_ok=True)\ntrain_files = sorted([str(p) for p in INPUT_PATH.glob(\"**/train_*.tfrec\")])\ntest_files = sorted([str(p) for p in INPUT_PATH.glob(\"**/test_*.tfrec\")])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### TFRecord Parser","metadata":{}},{"cell_type":"code","source":"def _parse_feature_function(example_proto, labeled=True, return_image_id=True):\n    tfrec_format = {\n        'vqt0'    : tf.io.FixedLenFeature([], tf.string),\n        'vqt1'    : tf.io.FixedLenFeature([], tf.string),\n        'vqt2'    : tf.io.FixedLenFeature([], tf.string),\n        'vqt0_ph' : tf.io.FixedLenFeature([], tf.string),\n        'vqt1_ph' : tf.io.FixedLenFeature([], tf.string),\n        'vqt2_ph' : tf.io.FixedLenFeature([], tf.string),\n        'image_id': tf.io.FixedLenFeature([], tf.string),\n    }\n    if labeled:\n        tfrec_format['target'] = tf.io.FixedLenFeature([], tf.int64)\n\n    # 入力の tf.Example のプロトコルバッファを上記のディクショナリを使って解釈\n    example = tf.io.parse_single_example(example_proto, tfrec_format)\n\n    feature = tf.concat([\n        tf.image.decode_png(example['vqt0']),\n        tf.image.decode_png(example['vqt1']),\n        tf.image.decode_png(example['vqt2']),\n        #tf.image.decode_png(example['vqt0_ph']),\n        #tf.image.decode_png(example['vqt1_ph']),\n        #tf.image.decode_png(example['vqt2_ph']),\n    ], axis=2)\n\n    if labeled: # train\n        return feature, tf.reshape(tf.cast(example['target'], tf.float32), [1])\n    else: # test\n        return feature, example['image_id'] if return_image_id else 0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset","metadata":{}},{"cell_type":"code","source":"def get_dataset(ds, shuffle = False, repeat = False,\n                labeled=True, return_image_ids=True, batch_size=16, dim=32, aug=False):\n\n    ds = ds.cache()\n\n    if repeat:\n        ds = ds.repeat(CFG['epochs'])\n\n    if shuffle:\n        ds = ds.shuffle(1024*2)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n\n    if labeled:\n        ds = ds.map(_parse_feature_function, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: _parse_feature_function(example, False, return_image_ids),\n                    num_parallel_calls=AUTO)\n\n    ds = ds.batch(batch_size * REPLICAS)\n    if aug:\n        ds = ds.map(lambda x, y: aug_f(x, y, batch_size * REPLICAS), num_parallel_calls=AUTO)\n    ds = ds.prefetch(AUTO)\n\n    return ds\n\nclass CrossValidation():\n    def __init__(self, files, fold_num):\n        self.ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n        self.ds = self.ds.shuffle(1024*2)\n        self.FOLD_NUM = fold_num\n        sample_num = len([*self.ds.take(-1)])\n        print(f\"sample_num: {sample_num}\")\n\n    def get_dataset(self, fold_index, shuffle = False, repeat = False, \n                labeled=True, return_image_ids=True, batch_size=16, dim=32, aug=False):\n\n        # divide train and validation\n        for idx in range(self.FOLD_NUM):\n            if idx != fold_index:\n                if \"train_ds\" in locals():\n                    train_ds = train_ds.concatenate(self.ds.shard(self.FOLD_NUM, idx))\n                else:\n                    train_ds = self.ds.shard(self.FOLD_NUM, idx)\n            else:\n                valid_ds = self.ds.shard(self.FOLD_NUM, fold_index)\n\n        return train_ds, valid_ds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model","metadata":{}},{"cell_type":"code","source":"class AngularGrad(tf.keras.optimizers.Optimizer):\n    def __init__(\n          self,\n          method_angle: str = \"cos\",\n          learning_rate=1e-3,\n          beta_1=0.9,\n          beta_2=0.999,\n          eps=1e-7,\n          name: str = \"AngularGrad\",\n          **kwargs,\n      ):\n        super().__init__(name, **kwargs)\n\n        self.method_angle = method_angle\n        self._set_hyper(\"learning_rate\", kwargs.get(\"lr\", learning_rate))\n        self._set_hyper(\"beta_1\", beta_1)\n        self._set_hyper(\"beta_2\", beta_2)\n        self._set_hyper(\"eps\", eps)\n        self.eps = eps or tf.keras.backend.epsilon()\n\n    def _create_slots(self, var_list):\n        for var in var_list:\n            self.add_slot(var, \"exp_avg\")\n            self.add_slot(var, \"exp_avg_sq\")\n            self.add_slot(var, \"previous_grad\")\n            self.add_slot(var, \"min\", initializer=tf.keras.initializers.Constant(value=math.pi / 2))\n            self.add_slot(var, \"final_angle_function_theta\")\n\n    def _resource_apply_dense(self, grad, var):\n        var_dtype = var.dtype.base_dtype\n\n        lr = self._get_hyper(\"learning_rate\", var_dtype)\n        beta_1 = self._get_hyper(\"beta_1\", var_dtype)\n        beta_2 = self._get_hyper(\"beta_2\", var_dtype)\n        eps = self._get_hyper(\"eps\", var_dtype)\n\n        exp_avg = self.get_slot(var, \"exp_avg\")\n        exp_avg_sq = self.get_slot(var, \"exp_avg_sq\")\n        previous_grad = self.get_slot(var, \"previous_grad\")\n        min = self.get_slot(var, \"min\")\n        final_angle_function_theta = self.get_slot(var, \"final_angle_function_theta\")\n\n        step = tf.cast(self.iterations + 1, var_dtype)\n        beta_1_power = tf.pow(beta_1, step)\n        beta_2_power = tf.pow(beta_2, step)\n\n        new_exp_avg = exp_avg.assign(\n            beta_1 * exp_avg + (1.0 - beta_1) * grad,\n            use_locking=self._use_locking\n        )\n        exp_avg_corrected = new_exp_avg / (1.0 - beta_1_power)\n\n        new_exp_avg_sq = exp_avg_sq.assign(\n            beta_2 * exp_avg_sq + (1.0 - beta_2) * tf.square(grad),\n            use_locking=self._use_locking,\n        )\n        exp_avg_sq_corrected = new_exp_avg_sq / (1.0 - beta_2_power)\n\n        tan_theta = tf.abs((previous_grad - grad) / (1 + previous_grad * grad))\n        cos_theta = 1 / tf.sqrt(1 + tf.square(tan_theta))\n\n        angle = tf.atan(tan_theta) * (180 / math.pi)\n        ans = tf.greater(angle, min)\n        mean_ans = tf.reduce_mean(tf.cast(ans, tf.float32))\n\n        def true_fn():\n            new_min = min.assign(angle, use_locking=self._use_locking)\n            new_final_angle_function_theta = final_angle_function_theta.assign(\n            tf.identity(tan_theta if self.method_angle == \"tan\" else cos_theta),\n              use_locking=self._use_locking\n              )\n            return new_min, new_final_angle_function_theta\n\n        def false_fn():\n            return min, final_angle_function_theta\n\n        new_min, new_final_angle_function_theta = tf.cond(tf.less(mean_ans, 0.5), true_fn, false_fn)\n        angular_coeff = tf.tanh(tf.abs(final_angle_function_theta)) * 0.5 + 0.5\n\n        var_update = var.assign_sub(\n            lr * exp_avg_corrected * angular_coeff / (tf.sqrt(exp_avg_sq_corrected) + eps),\n            use_locking=self._use_locking\n        )\n\n        new_previous_grad = previous_grad.assign(grad, use_locking=self._use_locking)\n\n        updates = [var_update, new_exp_avg, new_exp_avg_sq, new_min, new_previous_grad, new_final_angle_function_theta]\n        return tf.group(*updates)\n\n    def _resource_apply_sparse(self, grad, var, indices):\n        raise NotImplementedError\n\n    def get_config(self):\n        config = super().get_config()\n        config.update(\n            {\n                \"learning_rate\": self._serialize_hyperparameter(\"learning_rate\"),\n                \"beta_1\": self._serialize_hyperparameter(\"beta_1\"),\n                \"beta_2\": self._serialize_hyperparameter(\"beta_2\"),\n                \"eps\": self._serialize_hyperparameter(\"eps\")\n            }\n        )\n        return config","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]\n\ndef build_model(config, count=820):\n    inp = tf.keras.layers.Input(shape=(config['size'], config['size'],3))\n    base = EFNS[config['EFFV']](\n        input_shape=(config['size'],config['size'],3),\n        weights='imagenet',\n        include_top=False\n    )\n    x = base(inp)\n    x = tf.keras.layers.GlobalAvgPool2D()(x)\n    x = tf.keras.layers.Dropout(config['dropout'])(x)\n    x = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n    model = tf.keras.Model(inputs=inp, outputs=x)\n    \n    if config['opt'] == 'AngularGrad':\n        opt = AngularGrad(config['lr'])\n    elif config['opt'] == 'RectifiedAdam':\n        opt = tfa.optimizers.RectifiedAdam(learning_rate=config['lr'], weight_decay=0.001)\n    else:\n        lr_decayed_fn = tf.keras.experimental.CosineDecay(\n                              config['lr'],\n                              count,\n        )\n        opt = tfa.optimizers.AdamW(lr_decayed_fn, learning_rate=config['lr'])\n\n    loss = tf.keras.losses.BinaryCrossentropy() \n    model.compile(optimizer=opt, loss=loss, metrics=['AUC'])\n\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train","metadata":{}},{"cell_type":"code","source":"def vis_lr_callback(config=CFG):\n    lr_start   = config['lr_start']\n    lr_max     = config['lr_max']\n    lr_min     = config['lr_min']\n    lr_ramp_ep = config['lr_ramp_ep']\n    lr_sus_ep  = config['lr_sus_ep']\n    lr_decay   = config['lr_decay']\n\n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n\n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n\n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n\n        return lr\n    plt.figure(figsize=(10, 7))\n    plt.plot([lrfn(i) for i in range(config['epochs'])])\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback(config=CFG):\n    lr_start   = config['lr_start']\n    lr_max     = config['lr_max']\n    lr_min     = config['lr_min']\n    lr_ramp_ep = config['lr_ramp_ep']\n    lr_sus_ep  = config['lr_sus_ep']\n    lr_decay   = config['lr_decay']\n\n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n\n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n\n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n\n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vis_lr_callback()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\noof_pred = []; oof_tar = []; oof_val = []; oof_f1 = []; oof_ids = []; oof_folds = [] \n\ncv = CrossValidation(train_files, fold_num = CFG['fold_num'])\n\nfor fold in range(CFG['fold_num']):\n    \n    CFG['fold'] = fold\n    run = wandb.init(project='g2net',\n                     config=CFG,\n                     mode='offline'\n                    )\n    config = wandb.config\n    \n    train_ds, valid_ds = cv.get_dataset(fold)\n\n    print('#'*25); print('#### FOLD', fold + 1)\n    train_num = len([*train_ds.take(-1)])\n    valid_num = len([*valid_ds.take(-1)])\n    print(f'#### Training: {train_num} | Validation: {valid_num}')\n    \n    # memory clear\n    tf.keras.backend.clear_session()\n    \n    # build\n    with strategy.scope():\n        model = build_model(\n            config,\n            count=int(train_num)/config['batch_size']//REPLICAS*config['steps_multiplier']\n        )\n    print('#'*25)   \n\n    # save best model each of fold\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        OUTPUT_PATH.joinpath(f'model_fold{fold}.h5'), monitor='val_auc', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='max', save_freq='epoch')\n   \n    print('Training...')\n    train_ds = get_dataset(train_ds, shuffle=True,  repeat=True, dim=config['size'], batch_size=config['batch_size'], aug=config['aug'])\n    valid_ds = get_dataset(valid_ds, shuffle=False, repeat=False, dim=config['size'], batch_size=config['batch_size'])\n    print(f\"### Training Batch Num: {train_num//config['batch_size']} | Validation Batch Num: {valid_num//config['batch_size']}\")\n    history = model.fit(\n        train_ds,\n        epochs = config['epochs'], \n        callbacks = [sv, get_lr_callback(), WandbCallback()], \n        steps_per_epoch = train_num//config['batch_size']//REPLICAS,\n        validation_data = valid_ds,\n        validation_steps= valid_num//config['batch_size']//REPLICAS,\n        verbose = 1\n    )\n    \n    # Loading best model for inference\n    print('Loading best model...')\n    model.load_weights(OUTPUT_PATH.joinpath(f'model_fold{fold}.h5')) \n\n    _, valid_ds = cv.get_dataset(fold)\n    valid_ds = get_dataset(valid_ds, labeled=False, return_image_ids=False, shuffle=False, repeat=False, dim=config['size'], batch_size=config['batch_size'])\n    validation_steps = valid_num/config['batch_size']/REPLICAS\n\n    pred = model.predict(valid_ds, steps=validation_steps, verbose=0)[:valid_num,] \n    oof_pred.append(\n        np.mean(\n            pred.reshape((valid_num, 1), order='F'), axis=1\n        )\n    )\n    \n    # GET OOF TARGETS AND idS\n    _, valid_ds = cv.get_dataset(fold)\n    valid_ds = get_dataset(valid_ds, labeled=True, return_image_ids=True, shuffle=False, repeat=False, dim=config['size'])\n    oof_tar.append( np.array([target.numpy() for img, target in iter(valid_ds.unbatch())]) )\n    \n    hist_df = pd.DataFrame(history.history)\n    hist_df.to_csv(OUTPUT_PATH.joinpath(f'history_fold{fold}.csv'))\n    \n    # plot training result\n    if DISPLAY_PLOT:\n        \n        # plot pred distribution\n        plt.figure(figsize=(8, 6))\n        sns.histplot(oof_pred[-1])\n        plt.show()\n        plt.savefig(OUTPUT_PATH.joinpath(f'pred_histgram_fold{fold}.png'))\n        \n        # plot AUC\n        plt.figure(figsize=(15,5))\n        x = np.arange(len(history.history['auc']))\n        plt.plot(x, history.history['auc'    ], '-o', label='Train AUC', color='#ff7f0e')\n        plt.plot(x, history.history['val_auc'], '-o', label='Valid AUC', color='#1f77b4')\n        \n        x = np.argmax( history.history['val_auc'] )\n        y = np.max( history.history['val_auc'] )\n        \n        xdist = plt.xlim()[1] - plt.xlim()[0]\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        \n        plt.scatter(x, y, s=200, color='#1f77b4')\n        plt.text(x-0.03*xdist, y-0.13*ydist, 'max auc\\n%.2f'%y, size=14)\n        \n        plt.ylabel('auc', size=14);\n        plt.xlabel('Epoch', size=14)\n        plt.legend(loc=2)\n        \n        # plot loss\n        plt.gca().twinx()\n        x = np.arange(len(history.history['loss']))\n        \n        plt.plot(x, history.history['loss'    ], '-o', label='Train Loss', color='#2ca02c')\n        plt.plot(x, history.history['val_loss'], '-o', label='Vaild Loss', color='#d62728')\n        \n        x = np.argmin( history.history['val_loss'] );\n        y = np.min( history.history['val_loss'] )\n        \n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        \n        plt.scatter(x, y, s=200, color='#d62728');\n        plt.text(x-0.03*xdist, y+0.05*ydist, 'min loss', size=14)\n        plt.ylabel('Loss', size=14)\n        plt.legend(loc=2)\n        \n        plt.title('FOLD %i - Image Size %i'%(fold+1, config['size']), size=18)\n        plt.savefig(OUTPUT_PATH.joinpath(f'AUC_and_loss_fold{fold}.png'))\n        plt.show()\n    \n    run.join()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ","metadata":{}}]}