{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet -q\n!pip install focal-loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nimport efficientnet.tfkeras as efn\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\nfrom focal_loss import BinaryFocalLoss\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import GroupKFold","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helpers","metadata":{}},{"cell_type":"code","source":"def binary_focal_loss(y_true, y_pred, gamma=2.0, alpha=0.25):\n    \"\"\"\n    Implementation of Focal Loss from the paper in multiclass classification\n    Formula:\n        loss = -alpha_t*((1-p_t)^gamma)*log(p_t)\n        \n        p_t = y_pred, if y_true = 1\n        p_t = 1-y_pred, otherwise\n        \n        alpha_t = alpha, if y_true=1\n        alpha_t = 1-alpha, otherwise\n        \n        cross_entropy = -log(p_t)\n    Parameters:\n        alpha -- the same as wighting factor in balanced cross entropy\n        gamma -- focusing parameter for modulating factor (1-p)\n    Default value:\n        gamma -- 2.0 as mentioned in the paper\n        alpha -- 0.25 as mentioned in the paper\n    \"\"\"\n\n    # Define epsilon so that the backpropagation will not result in NaN\n    # for 0 divisor case\n    epsilon = K.epsilon()\n    # Add the epsilon to prediction value\n    #y_pred = y_pred + epsilon\n    # Clip the prediciton value\n    y_pred = K.clip(y_pred, epsilon, 1.0-epsilon)\n    # Calculate p_t\n    p_t = tf.where(K.equal(y_true, 1), y_pred, 1-y_pred)\n    # Calculate alpha_t\n    alpha_factor = K.ones_like(y_true)*alpha\n    alpha_t = tf.where(K.equal(y_true, 1), alpha_factor, 1-alpha_factor)\n    # Calculate cross entropy\n    cross_entropy = -K.log(p_t)\n    weight = alpha_t * K.pow((1-p_t), gamma)\n    # Calculate focal loss\n    loss = weight * cross_entropy\n    # Sum the losses in mini_batch\n    loss = K.sum(loss, axis=1)\n    \n    return loss\n\n\ndef auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy\n\n\ndef build_decoder(with_labels=True, target_size=(256, 256), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True):\n    def augment(img):\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n        return img\n    \n    def augment_with_labels(img, label):\n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, bsize=128, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024, \n                  cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize).prefetch(AUTO)\n    \n    return dset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare","metadata":{}},{"cell_type":"code","source":"COMPETITION_NAME = \"hpa-768768\"\nstrategy = auto_select_accelerator()\nBATCH_SIZE = strategy.num_replicas_in_sync * 16\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(COMPETITION_NAME)\nGCS_DS_PATH","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMSIZE = (224, 240, 260, 300, 380, 456, 528, 600, 700, 800)\nIMS = 9\nn_labels = 19 \nbinary_focal_loss = BinaryFocalLoss(gamma=5)\n\ndecoder = build_decoder(with_labels=True, target_size=(IMSIZE[IMS], IMSIZE[IMS]))\n\nwith strategy.scope():\n    model = tf.keras.Sequential([\n        efn.EfficientNetB7(\n            input_shape=(IMSIZE[IMS], IMSIZE[IMS], 3),\n            weights='imagenet',\n            include_top=False),\n        #tf.keras.layers.GlobalAveragePooling2D(),\n        # tf.keras.layers.Dense(n_labels, activation='sigmoid')\n        tf.keras.layers.GlobalMaxPool2D(),\n        tf.keras.layers.Dense(256, activation='relu'),\n        tf.keras.layers.Dense(64, activation='relu'),\n        tf.keras.layers.Dense(n_labels, activation='sigmoid')\n    ])\n    model.compile(\n        optimizer=tf.keras.optimizers.Adadelta(), # optimizer=tf.keras.optimizers.Adam(),\n        loss=binary_focal_loss, # 'binary_crossentropy'\n        metrics=[tf.keras.metrics.AUC(multi_label=True)])\n    model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"# color data generating\nload_dir = f\"/kaggle/input/{COMPETITION_NAME}/\"\ndf_in = pd.read_csv('../input/hpa-labels-csv-hg/df_green.csv')\n\ncolours = ['green', 'blue', 'red', 'yellow']\nfor colour in colours:\n    df = df_in.copy()\n    df.ID = df.ID[0][:-5] + colour\n\n    label_cols = df.columns[2:21]\n    paths = GCS_DS_PATH + '/' + df['ID'] + '.png'\n    labels = df[label_cols].values\n\n    (train_paths, valid_paths, train_labels, valid_labels) = train_test_split(paths, labels, test_size=0.2, random_state=42)\n    train_dataset = build_dataset(train_paths, train_labels, bsize=BATCH_SIZE, decode_fn=decoder)\n    valid_dataset = build_dataset(valid_paths, valid_labels, bsize=BATCH_SIZE, decode_fn=decoder,\n                                  repeat=False, shuffle=False, augment=False)\n    \n    steps_per_epoch = train_paths.shape[0] // BATCH_SIZE\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(\n        f'effb7_individual_model_{colour}_800.h5', save_best_only=True, monitor='val_loss', mode='min')\n    lr_reducer = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor=\"val_loss\", patience=3, min_lr=1e-6, mode='min')\n\n    history = model.fit(\n        train_dataset, \n        epochs=20,\n        verbose=1,\n        callbacks=[checkpoint, lr_reducer],\n        steps_per_epoch=steps_per_epoch,\n        validation_data=valid_dataset)\n\n    hist_df = pd.DataFrame(history.history)\n    hist_df.to_csv(f'effb7_individual_history_{colour}_800.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# References\n\n- [[HPA] classification efnb7 train](https://www.kaggle.com/h053473666/hpa-classification-efnb7-train)\n- [[HPA] classification efnb7 train 13cc0d](https://www.kaggle.com/aristotelisch/hpa-classification-efnb7-train-13cc0d?scriptVersionId=60520853)","metadata":{}}]}