{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install efficientnet -q","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:00:25.417195Z","iopub.execute_input":"2021-07-18T14:00:25.417649Z","iopub.status.idle":"2021-07-18T14:00:34.941308Z","shell.execute_reply.started":"2021-07-18T14:00:25.41755Z","shell.execute_reply":"2021-07-18T14:00:34.94032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install wandb --upgrade","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:00:34.942985Z","iopub.execute_input":"2021-07-18T14:00:34.943293Z","iopub.status.idle":"2021-07-18T14:00:46.411425Z","shell.execute_reply.started":"2021-07-18T14:00:34.943263Z","shell.execute_reply":"2021-07-18T14:00:46.410386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow_addons as tfa","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:00:46.413033Z","iopub.execute_input":"2021-07-18T14:00:46.413344Z","iopub.status.idle":"2021-07-18T14:00:53.244716Z","shell.execute_reply.started":"2021-07-18T14:00:46.413312Z","shell.execute_reply":"2021-07-18T14:00:53.243561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nimport efficientnet.tfkeras as efn\nimport numpy as np\nimport pandas as pd\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom sklearn.model_selection import GroupKFold\n\nimport csv\nfrom kaggle_datasets import KaggleDatasets\n\nfrom tensorflow.keras import layers","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:00:53.246863Z","iopub.execute_input":"2021-07-18T14:00:53.24733Z","iopub.status.idle":"2021-07-18T14:00:54.385665Z","shell.execute_reply.started":"2021-07-18T14:00:53.247284Z","shell.execute_reply":"2021-07-18T14:00:54.384799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\nfrom wandb.keras import WandbCallback\napi_key = 'e5260514cd613164ad5c3fd66ad7e909beaf820f'\nwandb.login(key=api_key)","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:00:54.38735Z","iopub.execute_input":"2021-07-18T14:00:54.388143Z","iopub.status.idle":"2021-07-18T14:00:55.354389Z","shell.execute_reply.started":"2021-07-18T14:00:54.388095Z","shell.execute_reply":"2021-07-18T14:00:55.353421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nimport cv2\nimport albumentations\nfrom albumentations import (\n    Compose, HorizontalFlip, CLAHE, HueSaturationValue,\n    RandomBrightness, RandomContrast, RandomGamma,\n    ToFloat, ShiftScaleRotate\n)\n\nAUGMENTATIONS_TRAIN = Compose([\n    HorizontalFlip(p=0.5),\n    RandomContrast(limit=0.2, p=0.5),\n    RandomGamma(gamma_limit=(80, 120), p=0.5),\n    RandomBrightness(limit=0.2, p=0.5),\n    HueSaturationValue(hue_shift_limit=5, sat_shift_limit=20,\n                       val_shift_limit=10, p=.9),\n    # CLAHE(p=1.0, clip_limit=2.0),\n    ShiftScaleRotate(\n        shift_limit=0.0625, scale_limit=0.1, \n        rotate_limit=15, border_mode=cv2.BORDER_REFLECT_101, p=0.8), \n    #ToFloat(max_value=255)\n])\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:00:55.35577Z","iopub.execute_input":"2021-07-18T14:00:55.356346Z","iopub.status.idle":"2021-07-18T14:00:55.365478Z","shell.execute_reply.started":"2021-07-18T14:00:55.356311Z","shell.execute_reply":"2021-07-18T14:00:55.364286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy\n\n\ndef build_decoder(with_labels=True, target_size=(256, 256), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:00:55.367201Z","iopub.execute_input":"2021-07-18T14:00:55.367532Z","iopub.status.idle":"2021-07-18T14:00:55.382264Z","shell.execute_reply.started":"2021-07-18T14:00:55.367503Z","shell.execute_reply":"2021-07-18T14:00:55.381304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#with strategy.scope():\n#    data_augmentation = tf.keras.Sequential([\n#        layers.experimental.preprocessing.RandomFlip(\"horizontal_and_vertical\"),\n#        layers.experimental.preprocessing.RandomRotation(0.2),\n#        layers.experimental.preprocessing.RandomContrast(0.025)\n#    ])","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:00:55.384689Z","iopub.execute_input":"2021-07-18T14:00:55.38524Z","iopub.status.idle":"2021-07-18T14:00:55.400284Z","shell.execute_reply.started":"2021-07-18T14:00:55.385191Z","shell.execute_reply":"2021-07-18T14:00:55.399163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:00:55.402273Z","iopub.execute_input":"2021-07-18T14:00:55.402625Z","iopub.status.idle":"2021-07-18T14:00:55.413414Z","shell.execute_reply.started":"2021-07-18T14:00:55.40259Z","shell.execute_reply":"2021-07-18T14:00:55.411909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_augmenter(with_labels=True):\n    def augment(img):\n        #print(\"img.shape\", img.shape)\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n        \n        #img = tf.image.random_saturation(img, 0.5, 1.0)\n        #img = tf.image.random_saturation(img, 0.2, 0.5)\n        #img = tf.image.random_brightness(img, 0.2)\n            \n        #img = AUGMENTATIONS_TRAIN(image=img)\n        return img\n    \n    def augment_with_labels(img, label):\n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, bsize=128, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024, \n                  cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) \n    \n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize)\n    \n    #if augment:\n    #    dset = dset.map(lambda x, y: (data_augmentation(x, training=augment), y), \n    #                num_parallel_calls=AUTO)\n    \n    dset = dset.prefetch(AUTO)\n    \n    return dset","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:00:55.415136Z","iopub.execute_input":"2021-07-18T14:00:55.415477Z","iopub.status.idle":"2021-07-18T14:00:55.429683Z","shell.execute_reply.started":"2021-07-18T14:00:55.415446Z","shell.execute_reply":"2021-07-18T14:00:55.428191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_dataset","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:00:55.431396Z","iopub.execute_input":"2021-07-18T14:00:55.431725Z","iopub.status.idle":"2021-07-18T14:00:55.449333Z","shell.execute_reply.started":"2021-07-18T14:00:55.431691Z","shell.execute_reply":"2021-07-18T14:00:55.448088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy = auto_select_accelerator()\nBATCH_SIZE = strategy.num_replicas_in_sync * 8","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:00:55.450905Z","iopub.execute_input":"2021-07-18T14:00:55.451212Z","iopub.status.idle":"2021-07-18T14:01:01.856028Z","shell.execute_reply.started":"2021-07-18T14:00:55.451184Z","shell.execute_reply":"2021-07-18T14:01:01.854828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COMPETITION_NAME = \"covid19512\"\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(COMPETITION_NAME)","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:01:01.857627Z","iopub.execute_input":"2021-07-18T14:01:01.857981Z","iopub.status.idle":"2021-07-18T14:01:02.225215Z","shell.execute_reply.started":"2021-07-18T14:01:01.857948Z","shell.execute_reply":"2021-07-18T14:01:02.224108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_data_dir = GCS_DS_PATH\nload_dir = _data_dir + \"/train/\"\ndf = pd.read_csv(_data_dir + '/updated_iamge_level.csv')","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:01:02.22663Z","iopub.execute_input":"2021-07-18T14:01:02.226949Z","iopub.status.idle":"2021-07-18T14:01:11.953271Z","shell.execute_reply.started":"2021-07-18T14:01:02.226921Z","shell.execute_reply":"2021-07-18T14:01:11.952096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:01:11.95469Z","iopub.execute_input":"2021-07-18T14:01:11.955061Z","iopub.status.idle":"2021-07-18T14:01:11.986089Z","shell.execute_reply.started":"2021-07-18T14:01:11.955025Z","shell.execute_reply":"2021-07-18T14:01:11.985049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['none'] = 0 \nfor i, r in df.iterrows():\n    if r.label == \"none 1 0 0 1 1\":\n        df.loc[i, 'none'] = 1\n#df","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:01:11.987303Z","iopub.execute_input":"2021-07-18T14:01:11.987666Z","iopub.status.idle":"2021-07-18T14:01:13.165833Z","shell.execute_reply.started":"2021-07-18T14:01:11.987637Z","shell.execute_reply":"2021-07-18T14:01:13.164759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_duplicateList_path = '../input/covid19512/dublicate.txt'\n_duplicateList = []\nwith open(_duplicateList_path, newline='') as csvfile:\n    spamreader = csv.reader(csvfile, delimiter=' ', quotechar='|')\n    for row in spamreader:\n        _duplicateList += row\n\n_duplicateList[:5]","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:01:13.167324Z","iopub.execute_input":"2021-07-18T14:01:13.16762Z","iopub.status.idle":"2021-07-18T14:01:13.195188Z","shell.execute_reply.started":"2021-07-18T14:01:13.167593Z","shell.execute_reply":"2021-07-18T14:01:13.194132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['id'] = df['id'].str.replace(\"_image\", \"\")\ndf = df[~df['id'].replace().isin(_duplicateList)]","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:01:13.196428Z","iopub.execute_input":"2021-07-18T14:01:13.197025Z","iopub.status.idle":"2021-07-18T14:01:13.21969Z","shell.execute_reply.started":"2021-07-18T14:01:13.19698Z","shell.execute_reply":"2021-07-18T14:01:13.218356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMSIZE = (224, 240, 260, 300, 380, 456, 528, 600, 512)\nIMS = 8","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:01:13.221179Z","iopub.execute_input":"2021-07-18T14:01:13.22151Z","iopub.status.idle":"2021-07-18T14:01:13.230275Z","shell.execute_reply.started":"2021-07-18T14:01:13.221479Z","shell.execute_reply":"2021-07-18T14:01:13.228712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"e = efn.EfficientNetB3(\n                input_shape=(IMSIZE[IMS], IMSIZE[IMS], 3),\n                weights='imagenet',\n                include_top=False)\ne.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:01:13.231474Z","iopub.execute_input":"2021-07-18T14:01:13.231849Z","iopub.status.idle":"2021-07-18T14:01:18.153402Z","shell.execute_reply.started":"2021-07-18T14:01:13.231818Z","shell.execute_reply":"2021-07-18T14:01:18.152536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(5):\n    \n    config={\n        \"fold\": i,\n        \"start_lr\": 1e-3,\n        \"min_lr\":1e-6,\n        \"epoch\": 20,\n        \"batch_size\": BATCH_SIZE,\n        \"IMG_SIZE\": IMSIZE[IMS]\n    }\n    wandb.init(project=\"covid19-siim-2class\", config=config)\n    \n    valid_paths = load_dir + df[df['fold'] == i]['id'] + '.png' #\"/train/\"\n    train_paths = load_dir + df[df['fold'] != i]['id'] + '.png' #\"/train/\" \n    valid_labels = df[df['fold'] == i]['none'].values\n    train_labels = df[df['fold'] != i]['none'].values\n\n    \n\n    decoder = build_decoder(with_labels=True, target_size=(IMSIZE[IMS], IMSIZE[IMS]), ext='png')\n    test_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[IMS], IMSIZE[IMS]),ext='png')\n\n    train_dataset = build_dataset(\n        train_paths, train_labels, bsize=BATCH_SIZE, decode_fn=decoder\n        # 使用 sequence augment\n        #, augment=False\n    )\n\n    valid_dataset = build_dataset(\n        valid_paths, valid_labels, bsize=BATCH_SIZE, decode_fn=decoder,\n        repeat=False, shuffle=False, augment=False\n    )\n\n    try:\n        n_labels = train_labels.shape[1]\n    except:\n        n_labels = 1\n\n    with strategy.scope():\n        model = tf.keras.Sequential([\n            \n            efn.EfficientNetB5(\n                input_shape=(IMSIZE[IMS], IMSIZE[IMS], 3),\n                weights='imagenet',\n                include_top=False),\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dropout(0.4),\n            tf.keras.layers.Dense(n_labels, activation='sigmoid')\n        ])\n        model.compile(\n            optimizer=tf.keras.optimizers.Adam(),\n            #optimizer = tfa.optimizers.RectifiedAdam(learning_rate=config['start_lr']),\n            #optimizer=tfa.optimizers.Lookahead(\n            #    #tf.keras.optimizers.Adam(learning_rate=config['start_lr']),\n            #    tfa.optimizers.RectifiedAdam(learning_rate=config['start_lr']),\n            #    sync_period=6,\n            #    slow_step_size=0.5\n            #),\n            loss='binary_crossentropy',\n            metrics=[tf.keras.metrics.AUC(multi_label=True)])\n            #metrics=[tf.keras.metrics.AUC()])\n\n        model.summary()\n\n\n    steps_per_epoch = train_paths.shape[0] // BATCH_SIZE\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(\n        f'model{i}.h5', save_best_only=True, monitor='val_loss', mode='min')\n    lr_reducer = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor=\"val_loss\", patience=3, min_lr=config[\"min_lr\"], mode='min')\n\n    history = model.fit(\n        train_dataset, \n        epochs=config[\"epoch\"],\n        verbose=1,\n        callbacks=[checkpoint, lr_reducer, WandbCallback()],\n        steps_per_epoch=steps_per_epoch,\n        validation_data=valid_dataset)\n\n    hist_df = pd.DataFrame(history.history)\n    hist_df.to_csv(f'history{i}.csv')\n    wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2021-07-18T14:02:54.780335Z","iopub.execute_input":"2021-07-18T14:02:54.780718Z"},"trusted":true},"execution_count":null,"outputs":[]}]}