{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n!pip install efficientnet -q\n!pip install iterative-stratification -q\n!pip install tensorflow_addons -q\n!pip install focal-loss\nimport numpy as np\nimport pandas\nimport tensorflow as tf\nimport os\nimport tensorflow_addons as tfa\nimport efficientnet.tfkeras as efn\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom sklearn.model_selection import GroupKFold\nimport matplotlib.pyplot as plt\nimport glob\nfrom tqdm import tqdm\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold,MultilabelStratifiedShuffleSplit\nfrom focal_loss import BinaryFocalLoss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nuser_credential = user_secrets.get_gcloud_credential()\nuser_secrets.set_tensorflow_credential(user_credential)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def onehot(image,label):\n    CLASSES = 19\n    return image,tf.one_hot(label,CLASSES)\ndef cutmix(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    DIM = IMSIZE[IMS]\n    CLASSES = 19\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        lab1 = tf.cast(lab1,tf.float32)\n        lab2 = tf.cast(lab2,tf.float32)\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2\n\n\ndef mixup(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = IMSIZE[IMS]\n    CLASSES = 19\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        # MAKE CUTMIX LABEL\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        lab1 = tf.cast(lab1,tf.float32)\n        lab2 = tf.cast(lab2,tf.float32)\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2\n\n\ndef transform(image,label):\n    # THIS FUNCTION APPLIES BOTH CUTMIX AND MIXUP\n    DIM = IMSIZE[IMS]\n    CLASSES = 19\n    SWITCH = 0.5\n    CUTMIX_PROB = 0.666\n    MIXUP_PROB = 0.666\n    # FOR SWITCH PERCENT OF TIME WE DO CUTMIX AND (1-SWITCH) WE DO MIXUP\n    image2, label2 = cutmix(image, label, CUTMIX_PROB)\n    image3, label3 = mixup(image, label, MIXUP_PROB)\n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        P = tf.cast( tf.random.uniform([],0,1)<=SWITCH, tf.float32)\n        imgs.append(P*image2[j,]+(1-P)*image3[j,])\n        labs.append(P*label2[j,]+(1-P)*label3[j,])\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image4,label4\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy\n\n\ndef build_decoder(with_labels=True, target_size=(256, 256), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True):\n    def augment(img):\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n        return img\n    \n    def augment_with_labels(img, label):\n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\nDIM =256\nn_class = 19\ndef _parse_image_function(example_proto,augment = True):\n    image_feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'label': tf.io.FixedLenFeature([], tf.string)\n    }\n    single_example = tf.io.parse_single_example(example_proto, image_feature_description)\n    image = tf.reshape( tf.io.decode_raw(single_example['image'],out_type=np.dtype('uint8')), (DIM,DIM, 3))\n    mask =  tf.reshape(tf.io.decode_raw(single_example['label'],out_type=np.dtype('uint8')),[n_class])\n    image = tf.dtypes.cast(image, tf.float32)\n    mask = tf.dtypes.cast(mask, tf.float32)\n    image = image/255.\n    if augment: # https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.flip_left_right(image)\n            mask = tf.image.flip_left_right(mask)\n\n        if tf.random.uniform(()) > 0.4:\n            image = tf.image.flip_up_down(image)\n            mask = tf.image.flip_up_down(mask)\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.rot90(image, k=1)\n            mask = tf.image.rot90(mask, k=1)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_saturation(image, 0.7, 1.3)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_contrast(image, 0.8, 1.2)\n    \n    return tf.cast(image, tf.float32),tf.cast(mask, tf.float32)\n\ndef load_dataset(filenames, ordered=False, augment = False):\n    AUTO = tf.data.experimental.AUTOTUNE\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda ex: _parse_image_function(ex, augment = augment), num_parallel_calls=AUTO)\n    return dataset\n\n\n\n\n\n\ndef build_dataset(paths, labels=None, bsize=128, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True,augment_mixup_cutmix=False\n                  , repeat=True, shuffle=1024, \n                  cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(True)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    #slices = paths if labels is None else (paths, labels)\n    dset = load_dataset(paths)\n    #dset = tf.data.Dataset.from_tensor_slices(slices)\n    #dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    #dset = dset.map(transform, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize)\n    dset = dset.map(transform,num_parallel_calls=AUTO) if augment_mixup_cutmix else dset\n    dset = dset.prefetch(AUTO)\n    \n    return dset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_data_items(filenames):\n    img_num = 0\n    for i in filenames:\n        img_num +=int(i.split('_')[-2])\n    return img_num","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_list=[]\nvalid_list=[]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gs_csv = pd.read_csv('../input/hpagscsv/gs.csv')\nval_index=[0] #  decide which dataset to be the valid one\nfor i in gs_csv.index:\n    temp=gs_csv.loc[i,'URL']\n    if i not in val_index:\n        train_list.extend(tf.io.gfile.glob(i+'/*'))\n    else:\n        valid_list.extend(tf.io.gfile.glob(i+'/*'))\n\n        \n#'gs://kds-d2130e02868047c380c97c6ae5d31a4180f74d21ceefbf5d4c9e531e' 0\n#'gs://kds-53777409f1b5814b19e2bc8b542d4af1163b9ac03623463085b45210' hpatrain-tfrec-1\n#'gs://kds-37f6a0e35a9bb77b47960da902f2fad44df121760c080ff2a9a04257' hpatrain-tfrec-2\n#'gs://kds-e3c610c144b38135540fa8471f82968eab5f181530e876ba09d20fad' hpatrain-tfrec-3        \n#'gs://kds-004df2f4910e2bf06fc13b08ef1bdb77467be005cff1fea662d07c73' hpaexset-02tfrec\n#'gs://kds-16cffa22de689b2549ff0bb18319210db5cc4566db005e35282ba293' hpaexset-3-5tfrec\n#'gs://kds-5774c5e377adbfc07a2cde16dba0ef50cf79f4459280fd7ee4bb8ff3' hpaexset-6-8tfrec\n#'gs://kds-20636028bcd9f63dc66fd5790cc609e05e2f836911243ef50159b008' hpaexset-9-11\n#'gs://kds-c90ca8f6f770bac1662c25b73f8cd83eb94897a0632df7a20f19dbec' hpaexset-12-14tfrec\n#'gs://kds-b0bf359a1d74d177f49a1f189b11142ccbaaacd7a6eb4d108f1d4fca' hpaexset-15-17tfrec\n#'gs://kds-1b20087af52f7720ba8347a73cca5be42a92d20c0092489ef7b0af05' hpaexset-18-20tfrec\n#'gs://kds-79a5c17813bfefd7dc4239f7ae438cd4483b713fd0610221321aa69a' hpaexset-21-23tfrec\n#'gs://kds-ca83f53b04345ee5b510e901a2711465d54e134c7b7bd7d9ca52b54a' hpaexset-24-26tfrec\n#'gs://kds-8ffb8be72389a3683a6925456e96ad5b7bc6516fc95457f7e2a16691' hpaexset-27-30tfrec\n#'gs://kds-d33bfe4ffc561ba62405827097415975c2a0ef7babcd9f693f7aa643' hpaexset-31-34tfrec\n#'gs://kds-1afb9a2e010e75655e2e686dafaf1a9d1ac2feeaf01737e8a9506d35' hpaexset-35-37tfrec","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMSIZE = (224, 240, 260, 300, 380, 456, 528, 600, 256)\nIMS = 8\n\nstrategy = auto_select_accelerator()\nBATCH_SIZE = strategy.num_replicas_in_sync * 32\nAUG_BATCH = BATCH_SIZE\n\n\n\n\ntrain_dataset = build_dataset(\n    train_list,  bsize=BATCH_SIZE,cache=False,augment_mixup_cutmix=True\n)\n\nvalid_dataset = build_dataset(\n    valid_list,  bsize=BATCH_SIZE,\n    cache=False,repeat=False, shuffle=False, augment=False\n)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define the model\n\ntry:\n    n_labels = 19#train_labels.shape[1]\nexcept:\n    n_labels = 1\n    \nwith strategy.scope():\n    '''\n    model = tf.keras.models.load_model(\n            '/content/gdrive/MyDrive/Kaggle/model_RGB_0.0051.h5'\n        )\n    '''\n    \n    model = tf.keras.Sequential([\n        efn.EfficientNetB0(\n            input_shape=(IMSIZE[IMS], IMSIZE[IMS], 3),\n            weights='imagenet',\n            include_top=False),\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(n_labels, activation='sigmoid')\n    ])\n    \n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(),\n        loss='binary_crossentropy',#BinaryFocalLoss(gamma=5),#'binary_crossentropy',#tfa.losses.SigmoidFocalCrossEntropy(),\n        metrics=[tf.keras.metrics.AUC(multi_label=True)])\n        \n    model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colour = '_RGB'\n#steps_per_epoch = len(train_paths) // BATCH_SIZE\nsteps_per_epoch = count_data_items(train_list) // BATCH_SIZE\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(\n    \"model_RGB_{val_loss:.4f}.h5\", save_best_only=False, monitor='val_loss', mode='min')\nlr_reducer = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor=\"val_loss\", patience=3, min_lr=1e-6, mode='min')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    train_dataset, \n    epochs=2,\n    verbose=1,\n    callbacks=[checkpoint, lr_reducer],\n    steps_per_epoch=steps_per_epoch,\n    validation_data=valid_dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist_df = pd.DataFrame(history.history)\nhist_df.to_csv(f'history{colour}.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}