{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is simply a demo of self implemented tf.keras EfficientNet_V2 since lots of people start using it in this competition. \nThis notebook mostly adapted from [Alien's 2class notebook](https://www.kaggle.com/h053473666/siim-covid19-efnb7-train-fold0-5-2class), so please take a look at his notebook as well.","metadata":{}},{"cell_type":"code","source":"!pip install -U git+https://github.com/GdoongMathew/EfficientNetV2 --no-deps","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-10T16:06:24.719272Z","iopub.execute_input":"2021-08-10T16:06:24.719664Z","iopub.status.idle":"2021-08-10T16:06:30.746616Z","shell.execute_reply.started":"2021-08-10T16:06:24.719573Z","shell.execute_reply":"2021-08-10T16:06:30.745647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nimport tensorflow as tf\nfrom tensorflow.keras import mixed_precision\n\n# policy = mixed_precision.Policy('mixed_float16')\n# mixed_precision.set_global_policy(policy)\nimport efficientnetv2\nfrom efficientnetv2.utils import DENSE_KERNEL_INITIALIZER, CONV_KERNEL_INITIALIZER\nimport pandas as pd\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import GroupKFold\nimport random\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2021-08-10T16:06:30.749619Z","iopub.execute_input":"2021-08-10T16:06:30.750216Z","iopub.status.idle":"2021-08-10T16:06:35.962122Z","shell.execute_reply.started":"2021-08-10T16:06:30.749991Z","shell.execute_reply":"2021-08-10T16:06:35.961254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n        tpu = None\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy, tpu\n\n\ndef build_decoder(with_labels=True, target_size=(256, 256), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n        img = tf.image.decode_image(file_bytes, channels=3, expand_animations=False)\n\n#         if ext == 'png':\n#             img = tf.image.decode_png(file_bytes, channels=3)\n#         elif ext in ['jpg', 'jpeg']:\n#             img = tf.image.decode_jpeg(file_bytes, channels=3)\n#         else:\n#             raise ValueError(\"Image extension not supported\")\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True, final_size=(256, 256, 3)):\n        \n    def augment(img):\n        ########### Add Augmentations here: ###########\n        img = tf.image.random_flip_left_right(img)\n#         img = tf.image.random_flip_up_down(img)\n        random_number = random.randint(0, 2)\n        if random_number == 1:    \n            img = tf.image.adjust_brightness(img, 0.2)\n        if random_number == 2:\n            img = tf.image.adjust_brightness(img, -0.2)\n        ###############################################\n        \n        return img\n    \n    def augment_with_labels(img, label):\n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\ndef build_keras_augmenter():\n    data_augmentation = tf.keras.Sequential([\n        tf.keras.layers.experimental.preprocessing.RandomRotation(0.06, fill_mode='constant'),\n        tf.keras.layers.experimental.preprocessing.RandomTranslation(height_factor=0.06, \n                                                                     width_factor=0.06, \n                                                                     fill_mode='constant'),\n        tf.keras.layers.experimental.preprocessing.RandomZoom((-0.4, 0.2), (-0.4, 0.2), fill_mode='constant'),\n\n    ])\n    return data_augmentation\n\n\ndef build_dataset(paths, labels=None, bsize=128, cache=True,\n                  decode_fn=None, augment_fn=None, keras_aug_fn=None,\n                  augment=True, repeat=True, shuffle=1024, drop_remainder=True,\n                  final_size=(256, 256, 3),\n                  cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(with_labels=labels is not None, final_size=final_size)\n        keras_aug_fn = build_keras_augmenter()\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.batch(bsize, drop_remainder=drop_remainder)\n    if labels is not None:\n        dset = dset.map(lambda x, y: (keras_aug_fn(x, training=True), y), num_parallel_calls=AUTO) if augment else dset\n    else:\n        dset = dset.map(lambda x: keras_aug_fn(x, training=True), num_parallel_calls=AUTO) if augment else dset\n    dset = dset.prefetch(AUTO)\n    \n    return dset","metadata":{"execution":{"iopub.status.busy":"2021-08-10T16:06:35.963821Z","iopub.execute_input":"2021-08-10T16:06:35.964126Z","iopub.status.idle":"2021-08-10T16:06:35.983067Z","shell.execute_reply.started":"2021-08-10T16:06:35.964100Z","shell.execute_reply":"2021-08-10T16:06:35.982160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COMPETITION_NAME = \"siimcovid19-512-img-png-600-study-png\"\nstrategy, _ = auto_select_accelerator()\nBATCH_SIZE = strategy.num_replicas_in_sync * 16\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(COMPETITION_NAME)","metadata":{"execution":{"iopub.status.busy":"2021-08-10T16:06:35.984838Z","iopub.execute_input":"2021-08-10T16:06:35.985440Z","iopub.status.idle":"2021-08-10T16:06:37.573085Z","shell.execute_reply.started":"2021-08-10T16:06:35.985400Z","shell.execute_reply":"2021-08-10T16:06:37.572043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load_dir = f\"/kaggle/input/{COMPETITION_NAME}/\"\ndf = pd.read_csv('../input/siim-cov19-csv-2class/train.csv')\nlabel_cols = df.columns[4]","metadata":{"execution":{"iopub.status.busy":"2021-08-10T16:06:37.574449Z","iopub.execute_input":"2021-08-10T16:06:37.574976Z","iopub.status.idle":"2021-08-10T16:06:37.622788Z","shell.execute_reply.started":"2021-08-10T16:06:37.574936Z","shell.execute_reply":"2021-08-10T16:06:37.621827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gkf  = GroupKFold(n_splits = 5)\ndf['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(df, groups = df.StudyInstanceUID.tolist())):\n    df.loc[val_idx, 'fold'] = fold\n    \nimage_size = 512","metadata":{"execution":{"iopub.status.busy":"2021-08-10T16:06:37.624483Z","iopub.execute_input":"2021-08-10T16:06:37.624905Z","iopub.status.idle":"2021-08-10T16:06:37.679704Z","shell.execute_reply.started":"2021-08-10T16:06:37.624862Z","shell.execute_reply":"2021-08-10T16:06:37.678828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\ntf.keras.backend.clear_session()\n\nvalid_paths = GCS_DS_PATH + '/image/' + df[df['fold'] == i]['id'] + '.png' #\"/train/\"\ntrain_paths = GCS_DS_PATH + '/image/' + df[df['fold'] != i]['id'] + '.png' #\"/train/\" \nvalid_labels = df[df['fold'] == i][label_cols].values\ntrain_labels = df[df['fold'] != i][label_cols].values\ndecoder = build_decoder(with_labels=True, target_size=(image_size, image_size), ext='png')\n\ntrain_dataset = build_dataset(\n    train_paths, train_labels, bsize=BATCH_SIZE, decode_fn=decoder,\n    augment=True, final_size=(image_size, image_size, 3)\n)\n\nvalid_dataset = build_dataset(\n    valid_paths, valid_labels, bsize=BATCH_SIZE, decode_fn=decoder,\n    repeat=False, shuffle=False, augment=False, final_size=(image_size, image_size, 3)\n)\n\ntry:\n    n_labels = train_labels.shape[1]\nexcept:\n    n_labels = 1\n","metadata":{"execution":{"iopub.status.busy":"2021-08-10T16:06:37.680976Z","iopub.execute_input":"2021-08-10T16:06:37.681519Z","iopub.status.idle":"2021-08-10T16:06:41.276406Z","shell.execute_reply.started":"2021-08-10T16:06:37.681473Z","shell.execute_reply":"2021-08-10T16:06:41.275311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l2_norm = 2e-5\ndropout_rate = [0.25, 0.25]\n\nwith strategy.scope():\n    input_x = tf.keras.layers.Input(shape=(None, None, 3))\n    effnet = efficientnetv2.EfficientNetV2_L(weights='imagenet21k-ft1k',\n                                             include_top=False,\n                                             input_tensor=input_x)\n\n    x = effnet(input_x, training=True)\n    x = tf.keras.layers.Dropout(dropout_rate[0])(x)\n    output_x = tf.keras.layers.Dense(1,\n                                     activation='sigmoid',\n                                     kernel_initializer=DENSE_KERNEL_INITIALIZER,\n                                     dtype=tf.float32, name='final_dense')(x)\n\n    # blocks_59\n    branch_x = effnet.get_layer('normal_block6l_add').output\n    branch_x = tf.keras.layers.Dropout(dropout_rate[1], name=\"aux_dropout\")(branch_x)\n    branch_x = tf.keras.layers.Conv2D(64, 3, \n                                      kernel_regularizer=tf.keras.regularizers.L2(l2_norm),\n                                      kernel_initializer=CONV_KERNEL_INITIALIZER)(branch_x)\n    branch_x = tf.keras.layers.BatchNormalization()(branch_x)\n    branch_x = tf.keras.layers.Activation('relu')(branch_x)\n    branch_x = tf.keras.layers.GlobalAveragePooling2D()(branch_x)\n    branch_x = tf.keras.layers.Dense(1, activation='sigmoid',\n                                     kernel_initializer=DENSE_KERNEL_INITIALIZER,\n                                     dtype=tf.float32, name='b59_dense')(branch_x)\n\n    model = tf.keras.Model(inputs=input_x, outputs=[output_x, branch_x])\n\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(),\n        loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.1),\n        # loss='binary_crossentropy',\n        metrics=[tf.keras.metrics.AUC(multi_label=True)])\nmodel.summary()\nsteps_per_epoch = train_paths.shape[0] // BATCH_SIZE\n\nsave_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\n\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(\n    f'effv2_model_{i}.h5', save_best_only=True, monitor='val_final_dense_loss', mode='min', options=save_locally, verbose=1)\nlr_reducer = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor=\"val_final_dense_loss\", patience=3, min_lr=5e-7, mode='min', verbose=1, factor=0.4)\n\n#     tensorboard = tf.keras.callbacks.TensorBoard(log_dir=f'../logs/effv2_model_{i}', profile_batch=(5, 15))\n\nhistory = model.fit(\n    train_dataset,\n    epochs=24,\n    verbose=1,\n    callbacks=[checkpoint, lr_reducer],\n    steps_per_epoch=steps_per_epoch,\n    validation_data=valid_dataset)\n\nhist_df = pd.DataFrame(history.history)\nhist_df.to_csv(f'history{i}.csv')\ndel model, effnet","metadata":{"execution":{"iopub.status.busy":"2021-08-10T16:06:41.278911Z","iopub.execute_input":"2021-08-10T16:06:41.279259Z","iopub.status.idle":"2021-08-10T16:07:26.787226Z","shell.execute_reply.started":"2021-08-10T16:06:41.279222Z","shell.execute_reply":"2021-08-10T16:07:26.784266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}