{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom sklearn.model_selection import GroupKFold\nimport matplotlib.pyplot as plt\nimport gc\nimport sys\nsys.path.append('../input/swin-transformer-tf-fixed')\nfrom swintransformer import SwinTransformer\n\n\ndef seed_everything(seed):\n    random.seed(seed)                           # Python 在一个明确的初始状态生成固定随机数字所必需的\n    np.random.seed(seed)                        # numpy 在一个明确的初始状态生成固定随机数字所必需的\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)    # 为了使某些基于散列的操作可复现\n    os.environ['TF_CUDNN_DETERMINISTIC'] = str(seed)\n    \nseed_everything(41)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T02:23:09.251621Z","iopub.execute_input":"2021-09-03T02:23:09.252039Z","iopub.status.idle":"2021-09-03T02:23:15.741071Z","shell.execute_reply.started":"2021-09-03T02:23:09.251940Z","shell.execute_reply":"2021-09-03T02:23:15.740120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SATURATION  = (0.9, 1.1)\nCONTRAST = (0.9, 1.1)\nBRIGHTNESS  =  0.1\nROTATION    = 10.0\nSHEAR    = 2.0\nHZOOM  = 8.0\nWZOOM  = 4.0\nHSHIFT = 4.0\nWSHIFT = 4.0\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = 384\n    XDIM = DIM%2\n    \n    rot = ROTATION * tf.random.normal([1], dtype='float32')\n    shr = SHEAR * tf.random.normal([1], dtype='float32')\n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM\n    h_shift = HSHIFT * tf.random.normal([1], dtype='float32')\n    w_shift = WSHIFT * tf.random.normal([1], dtype='float32')\n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot, shr, h_zoom, w_zoom, h_shift, w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3]), label\n\n\ndef transform_test(image):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = 384\n    XDIM = DIM%2\n    \n    rot = ROTATION * tf.random.normal([1], dtype='float32')\n    shr = SHEAR * tf.random.normal([1], dtype='float32')\n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM\n    h_shift = HSHIFT * tf.random.normal([1], dtype='float32')\n    w_shift = WSHIFT * tf.random.normal([1], dtype='float32')\n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot, shr, h_zoom, w_zoom, h_shift, w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","metadata":{"execution":{"iopub.status.busy":"2021-09-03T02:23:15.742635Z","iopub.execute_input":"2021-09-03T02:23:15.742936Z","iopub.status.idle":"2021-09-03T02:23:15.771788Z","shell.execute_reply.started":"2021-09-03T02:23:15.742907Z","shell.execute_reply":"2021-09-03T02:23:15.770688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy\n\n\ndef build_decoder(with_labels=True, target_size=(256, 256), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True):\n    def augment(img):\n        img = tf.image.random_flip_left_right(img)\n#         img = tf.image.random_flip_up_down(img)\n        img = tf.image.random_saturation(img, SATURATION[0], SATURATION[1])\n        img = tf.image.random_contrast(img, CONTRAST[0], CONTRAST[1])\n        img = tf.image.random_brightness(img, BRIGHTNESS)\n        return img\n    \n    def augment_with_labels(img, label):\n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, bsize=128, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024,\n                  seed=None, cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    \n    # Map the functions to perform Augmentations\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.map(transform, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle, seed=seed) if shuffle else dset\n    dset = dset.batch(bsize, drop_remainder = True).prefetch(AUTO)\n    \n    return dset\n\n\ndef build_dataset_test(paths, labels=None, bsize=128, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024,\n                  seed=None, cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    \n    # Map the functions to perform Augmentations\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.map(transform_test, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle, seed=seed) if shuffle else dset\n    dset = dset.batch(bsize, drop_remainder = True).prefetch(AUTO)\n    \n    return dset","metadata":{"execution":{"iopub.status.busy":"2021-09-03T02:23:15.773946Z","iopub.execute_input":"2021-09-03T02:23:15.774388Z","iopub.status.idle":"2021-09-03T02:23:15.798507Z","shell.execute_reply.started":"2021-09-03T02:23:15.774332Z","shell.execute_reply":"2021-09-03T02:23:15.797466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COMPETITION_NAME = \"siim-384x384-study-png\"\nCOMPETITION_NAME2 = \"crop384\"\nBATCH_SIZE = 8 * 16\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(COMPETITION_NAME)\nGCS_DS_PATH2 = KaggleDatasets().get_gcs_path(COMPETITION_NAME2)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T02:23:15.800421Z","iopub.execute_input":"2021-09-03T02:23:15.800806Z","iopub.status.idle":"2021-09-03T02:25:37.747357Z","shell.execute_reply.started":"2021-09-03T02:23:15.800766Z","shell.execute_reply":"2021-09-03T02:25:37.746595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# original data\ndf = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\nlabel_cols = df.columns[1:5]\ndf[\"class\"] = np.argmax(df[label_cols].values, axis=1)\n\nweight = 1 / np.log10(df[\"class\"].value_counts().sort_index().values)\nclass_weight = {i: weight[i] for i in range(4)}\nprint(class_weight)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T02:25:37.748586Z","iopub.execute_input":"2021-09-03T02:25:37.749048Z","iopub.status.idle":"2021-09-03T02:25:37.797757Z","shell.execute_reply.started":"2021-09-03T02:25:37.749016Z","shell.execute_reply":"2021-09-03T02:25:37.796514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# crop data\nimageid2studyid = dict()\nfor dirname, _, filenames in os.walk(f'/kaggle/input/siim-covid19-detection/train'):\n    for file in filenames:\n        imageid2studyid[file.replace('.dcm', '')] = dirname.split('/')[-2] + \"_study\"\n\ncrop_ids = os.listdir(\"/kaggle/input/crop384/images\")\ncrop_study_ids = [imageid2studyid[crop_id.replace(\"_crop.png\", \"\")] for crop_id in crop_ids]\ncrop_data = pd.DataFrame({\n    \"id\": crop_study_ids,\n    \"image_name\": crop_ids,\n})\ncrop_data = pd.merge(crop_data, df, on=\"id\", how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2021-09-03T02:25:37.799261Z","iopub.execute_input":"2021-09-03T02:25:37.799670Z","iopub.status.idle":"2021-09-03T02:26:23.997621Z","shell.execute_reply.started":"2021-09-03T02:25:37.799626Z","shell.execute_reply":"2021-09-03T02:26:23.996547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# concat data\ndf[\"is_crop\"] = 0\ndf[\"image_name\"] = df[\"id\"] + \".png\"\ncrop_data[\"is_crop\"] = 1\ntotal_data = pd.concat([df, crop_data], axis=0).reset_index(drop=True)\ntotal_data","metadata":{"execution":{"iopub.status.busy":"2021-09-03T02:26:23.998713Z","iopub.execute_input":"2021-09-03T02:26:23.998979Z","iopub.status.idle":"2021-09-03T02:26:24.048086Z","shell.execute_reply.started":"2021-09-03T02:26:23.998952Z","shell.execute_reply":"2021-09-03T02:26:24.046967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folds = 5\ngkf = GroupKFold(n_splits = folds)\ntotal_data['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(total_data, groups=total_data.id.tolist())):\n    total_data.loc[val_idx, 'fold'] = fold\n\npred_cols_ori = [col + \"_pred_ori\" for col in label_cols]\npred_cols_aug = [col + \"_pred_aug\" for col in label_cols]\nfor col in pred_cols_ori+pred_cols_aug:\n    total_data[col] = np.nan\n    \ntotal_data","metadata":{"execution":{"iopub.status.busy":"2021-09-03T02:27:04.585909Z","iopub.execute_input":"2021-09-03T02:27:04.586267Z","iopub.status.idle":"2021-09-03T02:27:04.665967Z","shell.execute_reply.started":"2021-09-03T02:27:04.586236Z","shell.execute_reply":"2021-09-03T02:27:04.665124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"historys = []\n\nfor i in range(folds):\n    strategy = auto_select_accelerator()\n    \n    valid_paths_df = GCS_DS_PATH + '/train/' + total_data[(total_data['fold'] == i) & (total_data['is_crop'] == 0)]['image_name']\n    valid_paths_crop = GCS_DS_PATH2 + '/images/' + total_data[(total_data['fold'] == i) & (total_data['is_crop'] == 1)]['image_name']\n    valid_paths = pd.concat([valid_paths_df, valid_paths_crop])\n    \n    valid_labels_df = total_data[(total_data['fold'] == i) & (total_data['is_crop'] == 0)][label_cols].values\n    valid_labels_crop = total_data[(total_data['fold'] == i) & (total_data['is_crop'] == 1)][label_cols].values\n    valid_labels = np.concatenate([valid_labels_df, valid_labels_crop], axis=0)\n    \n    train_paths_df = GCS_DS_PATH + '/train/' + total_data[(total_data['fold'] != i) & (total_data['is_crop'] == 0)]['image_name']\n    train_paths_crop = GCS_DS_PATH2+ '/images/' + total_data[(total_data['fold'] != i) & (total_data['is_crop'] == 1)]['image_name']\n    train_paths = pd.concat([train_paths_df, train_paths_crop])\n    \n    train_labels_df = total_data[(total_data['fold'] != i) & (total_data['is_crop'] == 0)][label_cols].values\n    train_labels_crop = total_data[(total_data['fold'] != i) & (total_data['is_crop'] == 1)][label_cols].values\n    train_labels = np.concatenate([train_labels_df, train_labels_crop], axis=0)\n    \n    shuffle_idx = np.random.choice(len(train_paths), size=len(train_paths), replace=False)\n    train_paths = train_paths.iloc[shuffle_idx]\n    train_labels = train_labels[shuffle_idx]\n    print(len(train_paths), len(valid_paths))\n\n    decoder = build_decoder(with_labels=True, target_size=(384, 384), ext='png')\n    test_decoder = build_decoder(with_labels=False, target_size=(384, 384), ext='png')\n\n    train_dataset = build_dataset(\n        train_paths, train_labels, bsize=BATCH_SIZE, decode_fn=decoder\n    )\n\n    valid_dataset = build_dataset(\n        valid_paths, valid_labels, bsize=BATCH_SIZE, decode_fn=decoder,\n        repeat=False, shuffle=False, augment=False\n    )\n    \n    test_dataset = build_dataset_test(\n        valid_paths, bsize=BATCH_SIZE, decode_fn=test_decoder,\n        repeat=False, shuffle=False, augment=False\n    )\n    test_datasets = []\n    for _ in range(3):\n        test_datasets.append(build_dataset_test(\n        valid_paths, bsize=BATCH_SIZE, decode_fn=test_decoder,\n        repeat=False, shuffle=False, augment=True\n    ))\n\n    with strategy.scope():\n        model = tf.keras.Sequential([\n            tf.keras.layers.InputLayer(input_shape=[384, 384, 3]),\n            SwinTransformer('swin_large_384', num_classes=4, include_top=False, pretrained=True, use_tpu=True),\n            tf.keras.layers.Dropout(rate=0.1),\n            tf.keras.layers.Dense(64, activation='relu'),\n            tf.keras.layers.Dropout(rate=0.1),\n            tf.keras.layers.Dense(4, activation='softmax')\n        ])\n        \n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(1e-4),\n        loss=tf.keras.losses.CategoricalCrossentropy(),\n        metrics=[tf.keras.metrics.AUC(multi_label=True), \"acc\"])\n\n    model.summary()\n\n    steps_per_epoch = train_paths.shape[0] // BATCH_SIZE\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(\n        f'model{i}.h5', save_best_only=True, monitor='val_loss', mode='min', save_weights_only=True)\n    lr_reducer = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor=\"val_loss\", patience=3, min_lr=1e-6, mode='min')\n\n    history = model.fit(\n        train_dataset, \n        epochs=15,\n        verbose=1,\n        callbacks=[checkpoint, lr_reducer],\n        steps_per_epoch=steps_per_epoch,\n        validation_data=valid_dataset,\n        class_weight=class_weight,\n    )\n    \n    historys.append(history.history)\n    \n    del decoder, test_decoder, train_dataset, valid_dataset, test_dataset, test_datasets, model\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-09-03T02:27:33.014788Z","iopub.execute_input":"2021-09-03T02:27:33.015132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_first_occur(learning_rates_list):\n    unique_lrs = np.sort(np.unique(learning_rates_list))\n    first_occur = []\n    for lr in unique_lrs[:-1]:\n        first_occur.append(learning_rates_list.index(lr))\n    return np.array(first_occur) - 1\n\n\nrecord = np.zeros((folds, 4))\nfig, ax = plt.subplots(folds, 3, figsize=(3*8, folds*6))\nfor idx, history in enumerate(historys):\n    if idx == 0:\n        name1 = \"auc\"\n        name2 = \"val_auc\"\n    else:\n        name1 = f\"auc_{idx}\"\n        name2 = f\"val_auc_{idx}\"\n    \n    first_occur = get_first_occur(history[\"lr\"])\n    \n    best_train_loss = np.array(history[\"loss\"])[np.argmin(history[\"val_loss\"])]\n    best_valid_loss = np.min(history[\"val_loss\"])\n    best_train_auc = np.array(history[name1])[np.argmin(history[\"val_loss\"])]\n    best_valid_auc = np.array(history[name2])[np.argmin(history[\"val_loss\"])]\n    min_idx = np.argmin(history[\"val_loss\"]) + 1\n    \n    ax[idx][0].plot(range(1, len(history[\"loss\"])+1), history[\"loss\"], \"bo-\", label=\"train_loss\")\n    ax[idx][0].plot(range(1, len(history[\"loss\"])+1), history[\"val_loss\"], \"go-\", label=\"valid_loss\")\n    ax[idx][0].plot([min_idx, min_idx], [history[\"val_loss\"][min_idx-1], history[\"loss\"][min_idx-1]])\n    ax[idx][0].scatter(first_occur+1, np.array(history[\"loss\"])[first_occur], s=300, c=\"r\", marker=\"*\")\n    ax[idx][0].scatter(first_occur+1, np.array(history[\"val_loss\"])[first_occur], s=300, c=\"r\", marker=\"*\")\n    ax[idx][0].legend()\n    ax[idx][0].grid()\n    ax[idx][0].set_title(f\"{idx} Best Train Loss: {np.round(best_train_loss, 4)}, Best Valid Loss: {np.round(best_valid_loss, 4)}\")\n    record[idx][0] += best_train_loss\n    record[idx][1] += best_valid_loss\n    \n    ax[idx][1].plot(range(1, len(history[name1])+1), history[name1], \"bo-\", label=\"train_auc\")\n    ax[idx][1].plot(range(1, len(history[name1])+1), history[name2], \"go-\", label=\"valid_auc\")\n    ax[idx][1].plot([min_idx, min_idx], [history[name2][min_idx-1], history[name1][min_idx-1]])\n    ax[idx][1].scatter(first_occur+1, np.array(history[name1])[first_occur], s=300, c=\"r\", marker=\"*\")\n    ax[idx][1].scatter(first_occur+1, np.array(history[name2])[first_occur], s=300, c=\"r\", marker=\"*\")\n    ax[idx][1].legend()\n    ax[idx][1].grid()\n    ax[idx][1].set_title(f\"{idx} Best Train AUC: {np.round(best_train_auc, 3)}, Best Valid AUC: {np.round(best_valid_auc, 3)}\")\n    record[idx][2] += best_train_auc\n    record[idx][3] += best_valid_auc\n    \n    ax[idx][2].plot(range(1, len(history[\"lr\"])+1), history[\"lr\"], \"ro-\", label=\"learning_rate\")\n    ax[idx][2].legend()\n    ax[idx][2].grid()\n    min_lr = np.min(history[\"lr\"])\n    ax[idx][2].set_title(f\"{idx} Min LR: {np.round(min_lr, 6)}\")\n    \navg_record = np.mean(record, axis=0)\nprint(\"Avg_Best_Train_Loss: \", np.round(avg_record[0], 4))\nprint(\"Avg_Best_Valid_Loss: \", np.round(avg_record[1], 4))\nprint(\"Avg_Best_Train_AUC: \", np.round(avg_record[2], 3))\nprint(\"Avg_Best_Valid_AUC: \", np.round(avg_record[3], 3))","metadata":{"execution":{"iopub.status.busy":"2021-08-08T05:07:10.019328Z","iopub.status.idle":"2021-08-08T05:07:10.019759Z"},"trusted":true},"execution_count":null,"outputs":[]}]}