{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install efficientnet -q","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --upgrade keras keras-applications\n!pip install vit-keras","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nimport efficientnet.tfkeras as efn\nimport numpy as np\nimport pandas as pd\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom sklearn.model_selection import GroupKFold\nfrom keras.callbacks import Callback\nimport tensorflow.python.keras.backend as K\nfrom keras_applications import resnext\nimport keras\nimport tensorflow_addons as tfa\nfrom vit_keras import vit, utils","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy\n\n\ndef build_decoder(with_labels=True, target_size=(256, 256), ext='jpg'):\n    def decode(path):\n        #file_bytes = tf.io.read_file(path)\n        \n        \"\"\"        \n        r = tf.io.read_file(path + \"_red.png\")\n        g = tf.io.read_file(path + \"_green.png\")\n        b = tf.io.read_file(path + \"_blue.png\")\n        \n        red = tf.io.decode_png(r, channels=1)\n        blue = tf.io.decode_png(g, channels=1)\n        green = tf.io.decode_png(b, channels=1)\n        \n        red = tf.image.resize(red, target_size)\n        blue = tf.image.resize(blue, target_size)\n        green = tf.image.resize(green, target_size)\n        \n        img = tf.stack([red, green, blue], axis=-1)\n        img = tf.squeeze(img)\n        img = tf.image.convert_image_dtype(img, tf.float32) / 255\n        \"\"\"\n        #if only green\n        g = tf.io.read_file(path + \"_green.png\")\n        img = tf.image.decode_png(g, channels=3)\n        img = tf.cast(img, tf.float32) / 255.0\n        #if only green\n        img = tf.image.resize(img, target_size)\n        \n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True):\n    def augment(img):\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n        return img\n    \n    def augment_with_labels(img, label):\n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, bsize=128, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024, \n                  cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize).prefetch(AUTO)\n    \n    return dset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nstrategy = auto_select_accelerator()\nBATCH_SIZE = strategy.num_replicas_in_sync * 16 #############WAS 16\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(\"hpa-768768\")\nGCS_DS_PATH_EXT_DATA = KaggleDatasets().get_gcs_path(\"hpa-public-768-excl-0-16\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all_noisy = pd.read_csv(\"../input/all-cleanlab/train_kaggle_public_probs_ids_falselabel_intensity.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all_noisy","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#green\nload_dir = f\"/kaggle/input/hpa-768768/\"\ndf = pd.read_csv('../input/classification-label-csv-green/df_green.csv')\ndf[\"ID\"] = df[\"ID\"].str.replace('_green', '')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cleanlab = pd.merge(df, df_all_noisy, on=\"ID\")\ndf_cleanlab = df_cleanlab[df_cleanlab[\"false_label\"] == False]\ndf = df_cleanlab.reset_index(drop=True, inplace=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_cols = df.columns[2:21]\npaths = GCS_DS_PATH + '/' + df['ID']\nlabels = df[label_cols].values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ext = pd.read_csv('../input/hpa-public-768-excl-0-16/hpa_public_excl_0_16_768.csv', index_col=0)\n\ndf_ext_cleanlab = pd.merge(df_ext, df_all_noisy, on=\"ID\")\ndf_ext_cleanlab = df_ext_cleanlab[df_ext_cleanlab[\"false_label\"] == False]\ndf_ext = df_ext_cleanlab.reset_index(drop=True, inplace=False)\n\n\ndf_ext = df_ext[[\"ID\", \"Label_y\"]]\ndf_ext[\"Labels_list_y\"] = df_ext[\"Label_y\"].str.split(\"|\").apply(lambda x: [int(i) for i in x])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ext","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ext.reset_index(drop=True, inplace=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import MultiLabelBinarizer\n\nmlb = MultiLabelBinarizer(classes=[n for n in range(19)])\ny = df_ext[\"Labels_list_y\"]\n\ndf_ohe = pd.DataFrame(mlb.fit_transform(y),columns=mlb.classes_)\ndf_ohe_np = df_ohe.to_numpy()\n\ndf_ext_ohe = pd.concat([df_ext, df_ohe], axis=1)\ndf_ext_ohe = df_ext_ohe.drop(['Labels_list_y'], axis=1)\ndf_ext_ohe.columns = df_ext_ohe.columns.astype(str)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ext","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ohe","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_cols_ext = df_ext_ohe.columns[2:21]\npaths_ext = GCS_DS_PATH_EXT_DATA + '/hpa_public_excl_0_16_768/small/' + df_ext_ohe['ID']\nlabels_ext = df_ext_ohe[label_cols_ext].values\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths_ext[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ext_ohe","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_all = np.append(labels, labels_ext, axis=0)\npaths_all = paths.append(paths_ext, ignore_index=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sanity check\nname = paths_all[22000].split(\"/\")[-1].split(\".\")[0]\nlabel_real = df_ext_ohe.loc[df_ext_ohe[\"ID\"] == name].Label_y\nlabel_set = np.where(labels_all[22000] == 1)\n\nassert  int(label_real) == int(label_set[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths_all","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(\n    train_paths, valid_paths, \n    train_labels, valid_labels\n) = train_test_split(paths_all, labels_all, test_size=0.1, random_state=42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMSIZE = (384, 240, 260, 300, 380, 456, 528, 600)\nIMS = 7\n\ndecoder = build_decoder(with_labels=True, target_size=(IMSIZE[IMS], IMSIZE[IMS]))\ntest_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[IMS], IMSIZE[IMS]))\n\ntrain_dataset = build_dataset(\n    train_paths, train_labels, bsize=BATCH_SIZE, decode_fn=decoder\n)\n\nvalid_dataset = build_dataset(\n    valid_paths, valid_labels, bsize=BATCH_SIZE, decode_fn=decoder,\n    repeat=False, shuffle=False, augment=False\n)\n\n# test_dataset = build_dataset(\n#     test_paths, cache=False, bsize=BATCH_SIZE, decode_fn=test_decoder,\n#     repeat=False, shuffle=False, augment=False\n# )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def binary_focal_loss(gamma=2, alpha=0.25):\n    \"\"\"\n    Binary form of focal loss.\n         Focal loss for binary classification problems\n    \n    focal_loss(p_t) = -alpha_t * (1 - p_t)**gamma * log(p_t)\n        where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    alpha = tf.constant(alpha, dtype=tf.float32)\n    gamma = tf.constant(gamma, dtype=tf.float32)\n\n    def binary_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        y_true shape need be (None,1)\n        y_pred need be compute after sigmoid\n        \"\"\"\n        y_true = tf.cast(y_true, tf.float32)\n        alpha_t = y_true*alpha + (K.ones_like(y_true)-y_true)*(1-alpha)\n    \n        p_t = y_true*y_pred + (K.ones_like(y_true)-y_true)*(K.ones_like(y_true)-y_pred) + K.epsilon()\n        focal_loss = - alpha_t * K.pow((K.ones_like(y_true)-p_t),gamma) * K.log(p_t)\n        return K.mean(focal_loss)\n    return binary_focal_loss_fixed","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntry:\n    n_labels = train_labels.shape[1]\nexcept:\n    n_labels = 1\n\"\"\"    \nwith strategy.scope():\n    loss_fn = tf.keras.losses.BinaryCrossentropy(from_logits=False, label_smoothing=0, reduction=\"auto\", name=\"binary_crossentropy\")\n    #loss_fn = tf.keras.losses.CategoricalCrossentropy(from_logits=False,abel_smoothing=0,reduction=\"auto\",name=\"categorical_crossentropy\")\n    \n    model = tf.keras.Sequential([\n        resnext.ResNeXt101(include_top=False, weights='imagenet', input_shape=(600, 600, 3), backend = keras.backend, layers = keras.layers, models = keras.models, utils = keras.utils),\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(n_labels, activation='sigmoid')\n    ])\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(),\n        loss=loss_fn,\n        metrics=[tf.keras.metrics.AUC(multi_label=True)])\n        \n    model.summary()\n\"\"\"    \n\nscheduler = tf.keras.experimental.CosineDecayRestarts(0.001, 5, t_mul=2.0)\nlr_reducer = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor=\"val_loss\", patience=3, min_lr=1e-6, factor=0.5, mode='min')\n\n#https://gist.github.com/Tony607/28b8de1cd01a859e62cc77547d601fb5\nwith strategy.scope():\n    #loss_fn = tf.keras.losses.BinaryCrossentropy(from_logits=False, label_smoothing=0.2, reduction=\"auto\", name=\"binary_crossentropy\")\n    #loss_fn = tfa.losses.SigmoidFocalCrossEntropy()\n    #loss_fn = binary_focal_loss(alpha=.25, gamma=2)\n    \"\"\"\n    model = vit.vit_b16(\n    image_size=384,\n    activation='sigmoid',\n    pretrained=True,\n    include_top=True,\n    pretrained_top=False,\n    classes=19\n)\n    \"\"\"\n\n    \n    \"\"\"\n    \"\"\"\n    model = tf.keras.Sequential([\n        efn.EfficientNetB7(\n            input_shape=(IMSIZE[IMS], IMSIZE[IMS], 3),\n            weights='imagenet',\n            include_top=False),\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(n_labels, activation='sigmoid')\n    ])\n   \n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(),\n        #optimizer=tfa.optimizers.AdamW(learning_rate=0.001, weight_decay=1e-4),\n        loss='binary_crossentropy',\n        metrics=[tf.keras.metrics.AUC(multi_label=True)])\n        \n    model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nwith strategy.scope():\n    model = keras.models.load_model('../input/hpa-tensorflow-models/model_rgb_resnext101.04-0.12.h5')\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"steps_per_epoch = train_paths.shape[0] // BATCH_SIZE\n\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(\n    'ggg_EffB7_CLEANLAB_ADAM_RedPlat_BCE_EPOCH{epoch:02d}-VAL{val_loss:.4f}.h5', save_best_only=True, monitor='val_loss', mode='min', verbose=1)\n\nlr_reducer = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor=\"val_loss\", patience=2, min_lr=1e-6, mode='min', verbose=1)\n\n\n\n\nclass CallbackGetLR(Callback):\n    def on_epoch_end(self, epoch, logs=None):\n        lr_with_decay = self.model.optimizer._decayed_lr(tf.float32)\n        print(\"Learning Rate = \", K.eval(lr_with_decay))\n        \nprint_lr = CallbackGetLR()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    train_dataset, \n    epochs=100,\n    verbose=1,\n    callbacks=[checkpoint,lr_reducer, print_lr],\n    steps_per_epoch=steps_per_epoch,\n    validation_data=valid_dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colour = \"ggg\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm \"./rgb_ViTB16_RedPlat_ADAMW_BCE_EPOCH03-VAL0.1303.h5\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist_df = pd.DataFrame(history.history)\nhist_df.to_csv(f'history{colour}.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'./ggg_ViTB16_RedPlat_ADAMW_BCE_EPOCH01-VAL0.1397.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}