{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install tensorflow==2.2.0\n# !pip install numpy --upgrade\n# !pip install sklearn --upgrade","metadata":{"_uuid":"df2f4004-a1eb-4b8f-962a-e76bbf65e550","_cell_guid":"0b159037-9c70-40fa-b1db-8a0fc59e1440","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import numpy\n!pip show tensorflow\n# print (numpy.__version__)","metadata":{"_uuid":"72100a3a-07b6-42be-830c-5e8516c7a116","_cell_guid":"0a81e974-7f65-418a-9e42-6292aef6cc1c","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport cv2\nimport math\nimport time\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook as tqdm\nfrom sklearn.model_selection import train_test_split\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.models import Model,Sequential\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras import initializers, layers\nfrom tensorflow import keras\n\nimport pickle\nimport os\nimport glob\nimport matplotlib.pyplot as plt\nimport time;\nprint(\"start\")","metadata":{"_uuid":"e77a9b04-9ce8-4b10-9be9-0fa025feeaad","_cell_guid":"f7cfe146-3726-4c42-81a9-437a0e2ed41c","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  TPU configuration","metadata":{"_uuid":"d06ec1b9-d403-497a-8f8a-d53186825d31","_cell_guid":"8c67fc62-361c-409a-bcf6-fe964400e24e","trusted":true}},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n    \nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n\nDATASET = 'melanomasegmented'\n# DATASET = 'unet-segmentedisic-20192020'\nGCS_PATH = KaggleDatasets().get_gcs_path(DATASET)","metadata":{"_uuid":"fbd1d1d4-9dc4-42cf-9900-90addc721439","_cell_guid":"6aff7142-17fb-4490-a50d-d88357dcbded","collapsed":false,"_kg_hide-input":true,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Hyperparameter tuning","metadata":{"_uuid":"34109f20-93dd-448f-ae23-1bbb5070c7c5","_cell_guid":"6dc721f5-7588-4820-bd45-3733815c5c0c","trusted":true}},{"cell_type":"code","source":"SEED = 42\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nSIZE = [128,128]\nLR = 0.00004\nEPOCHS =50\nWARMUP = 5\nWEIGHT_DECAY = 0\nLABEL_SMOOTHING = 0.05\nTTA = 4\n# \nINPUT_SHAPE = (128, 128, 3)\nNUM_CLASS =2\nROUTING = 3\nspread_loss_value=K.variable(0.2)","metadata":{"_uuid":"512ff71a-0980-4c43-b688-f3ed9befdfc2","_cell_guid":"fa4e2e2f-2118-4310-80f7-35eacc8abfa2","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(SEED):\n    np.random.seed(SEED)\n    tf.random.set_seed(SEED)\n\nseed_everything(SEED)\ntrain_filenames = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\ntest_filenames = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')","metadata":{"_uuid":"d85eed6a-c876-4d22-869c-2d000de59a95","_cell_guid":"5afc25dc-e553-45dc-aa84-3da2318105da","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_1 = pd.read_csv('/kaggle/input/model-melanoma-2020/submission.csv')\ndf_3b = pd.read_csv('/kaggle/input/melanoma-effnet-metdata/ensembled.csv')\ndf_4 = pd.read_csv('/kaggle/input/minmax-melanoma-9619/submission.csv')\ndf_5 = pd.read_csv('/kaggle/input/melanoma-2020-9648/submission_9648.csv')","metadata":{"_uuid":"2af41ae2-3cf4-4ee5-9293-3824aa3ed25f","_cell_guid":"eec4a5b4-9a87-49f5-bc38-decca41fcfd4","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_images(np_images, titles = [], columns =3, figure_size = (24, 15)):\n    count = len(np_images)\n    rows = math.ceil(count / columns)\n   \n\n    fig = plt.figure(figsize=figure_size)\n    subplots = []\n    for index in range(count):\n        subplots.append(fig.add_subplot(rows, columns, index + 1))\n        if len(titles):\n            subplots[-1].set_title(str(titles[index]))\n        plt.imshow(np_images[index])\n\n    plt.show()","metadata":{"_uuid":"3cc87949-8253-480b-9c6b-c847f7c377ac","_cell_guid":"c3422956-5ea9-4d65-896d-5e11bc26cac5","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_filenames,valid_filenames = train_test_split(train_filenames,test_size = 0.05,random_state = SEED)","metadata":{"_uuid":"e76694ed-af02-4b44-b9e6-3e3d5d2c6777","_cell_guid":"0fca9c14-cda3-4214-b4ed-f1af8670cda8","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3) \n    image = tf.cast(image, tf.float32)/255.0\n    image = tf.reshape(image, [*SIZE, 3])\n#     image = tf.image.resize(image, [28,28])\n    return image\n\nnp.set_printoptions(threshold=np.inf)\n\ndef decode_image_png(image):\n    image = tf.image.decode_png(image, channels=3)\n    image = tf.cast(image, tf.float32)/255.0\n    image = tf.reshape(image, [*SIZE, 3])\n#     image = tf.image.resize(image,[28,28])\n    return image\n\ndef parseImageFromRawData(image_raw, shape=INPUT_SHAPE, type=tf.float32):\n    image = tf.io.decode_raw(image_raw, type)\n    image = tf.reshape(image, shape=shape)\n    return image\n\ndef parse_example_unlabeled(example):  \n    features = {'image': tf.io.FixedLenFeature([], tf.string),\"image_name\": tf.io.FixedLenFeature([], tf.string)}\n    parsed_example = tf.io.parse_single_example(example, features=features)\n    image = decode_image_png(parsed_example['image'])\n    image_name = parsed_example['image_name']\n    return (image, image_name)\n\ndef parse_example_labeled(example):  \n    LABELED_TFREC_FORMAT ={\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  }\n    parsed_example = tf.io.parse_single_example(example,LABELED_TFREC_FORMAT)\n    label = tf.cast(parsed_example['target'], tf.int32)\n    image = decode_image_png(parsed_example['image'])\n    return (image, label), (label, image)\n\ndef data_augment_unlabled(image,image_name, seed=SEED):\n    image = tf.image.rot90(image,k=np.random.randint(4))\n    image = tf.image.random_flip_left_right(image, seed=seed)\n    image = tf.image.random_flip_up_down(image, seed=seed)\n    return (image,image_name)\n\ndef data_augment_labeled(image_label,label_image=None, seed=SEED):\n    (image,label)=image_label\n    image = tf.image.rot90(image,k=np.random.randint(4))\n    image = tf.image.random_flip_left_right(image, seed=seed)\n    image = tf.image.random_flip_up_down(image, seed=seed)\n    return (image, label),(label,image)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n\n    return (tf.data.TFRecordDataset(filenames,num_parallel_reads=AUTO) \n              .with_options(ignore_order)\n               .map(parse_example_labeled if labeled\n                    else parse_example_unlabeled, num_parallel_calls=AUTO))\n\n                   \ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\ndef plot_transform():\n    plt.figure(figsize=(30,10))\n    x = load_dataset(train_filenames, labeled=True).batch(BATCH_SIZE)\n    for (image_label,label_image) in x:\n        (images,labels)= image_label\n        plot_images(images)\n#        print(labels.numpy(),labels)\n#        print(\"images.shape: \",images.shape)\n        break\n    \n#     plt.imshow(images[40])","metadata":{"_uuid":"b01dbc7f-4b13-4a67-96ed-2901c640582a","_cell_guid":"d0f006a8-6713-473e-a869-ae431bd08eaf","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Visualizing Augmentation","metadata":{"_uuid":"ee150b13-b0cc-42c4-9a20-5e102b3be921","_cell_guid":"c1007161-6a8a-434c-b614-9cf9152c9480","trusted":true}},{"cell_type":"code","source":"plot_transform()","metadata":{"_uuid":"70ec8774-aa8d-4279-a979-e2a0fbc3b5fc","_cell_guid":"0ee4ce9b-664a-49f3-a351-acd21bb1d884","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Segment all image","metadata":{"_uuid":"45e4eb8d-4347-475f-9063-ff36c19dd1e2","_cell_guid":"c379ee57-8f4a-452d-8339-a35257cd6000","trusted":true}},{"cell_type":"code","source":"\ntrain_dataset = (load_dataset(train_filenames, labeled=True)\n    .map(data_augment_labeled, num_parallel_calls=AUTO)\n    .shuffle(SEED)\n    .batch(BATCH_SIZE,drop_remainder=True)\n    .repeat()\n    .prefetch(AUTO))\n\nvalid_dataset = (load_dataset(valid_filenames, labeled=True)\n    .batch(BATCH_SIZE,drop_remainder=True)\n    .cache()            \n    .prefetch(AUTO))","metadata":{"_uuid":"9fa3bcdf-e409-462b-98da-4ed4d3f65547","_cell_guid":"6fcce979-70c3-4b14-a178-6bb7eb956892","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Model,loss function","metadata":{"_uuid":"9c91db9c-85ed-49b2-90ea-ee4db818d42f","_cell_guid":"b4cbacd7-9cb3-40ad-9169-6920b7c214e2","trusted":true}},{"cell_type":"code","source":"def focal_loss(alpha=0.25,gamma=2.0):\n    def focal_crossentropy(y_true, y_pred):\n        y_true =tf.bitcast(tf.cast(y_true, dtype=tf.float32), tf.float32)\n        y_pred = tf.slice(y_pred, [0, 1], [y_pred.shape[0], 1]) #Only keep weights for melanoma\n        bce = K.binary_crossentropy(y_true, y_pred)\n        \n        y_pred = K.clip(y_pred, K.epsilon(), 1.- K.epsilon())\n        p_t = (y_true*y_pred) + ((1-y_true)*(1-y_pred))\n        \n        alpha_factor = 1\n        modulating_factor = 1\n\n        alpha_factor = y_true*alpha + ((1-alpha)*(1-y_true))\n        modulating_factor = K.pow((1-p_t), gamma)\n\n        # compute the final loss and return\n        return K.mean(alpha_factor*modulating_factor*bce, axis=-1)\n    return focal_crossentropy","metadata":{"_uuid":"a29a74be-b5a9-4aa1-9c00-89834882189b","_cell_guid":"ff57eeb5-fbb5-4a2e-8c4b-321c332d11e5","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def margin_loss(y_true, y_pred):\n    y_tobe =tf.bitcast(tf.cast(y_true, dtype=tf.int64), tf.int64)\n    y_tobe = tf.one_hot(y_tobe, NUM_CLASS)\n    y_tobe_correct = y_tobe\n    y_tobe_wrong = 1 - y_tobe\n    L = tf.math.multiply(y_tobe_correct, K.square(K.maximum(0., tf.math.subtract(0.9, y_pred)))) + \\\n        tf.math.multiply( y_tobe_wrong, K.square(K.maximum(0., tf.math.subtract(y_pred, 0.1))))\n    return  K.mean(K.sum(L, 1))","metadata":{"_uuid":"54a7b773-48d6-4f07-a263-9c342aed5395","_cell_guid":"0ab27a01-8e02-42bf-90f9-71838aa902c7","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ssim_loss(image_true, image_rec):\n    return  K.mean(tf.math.abs(tf.image.ssim(image_true, image_rec, max_val=1.0)))","metadata":{"_uuid":"dd07feca-25e2-48de-80ca-7170dfcb72ad","_cell_guid":"d7214753-5674-44b6-8e44-47d2a1cac644","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass Length(layers.Layer):\n    def call(self, inputs, **kwargs):\n        out = K.sqrt(K.sum(K.square(inputs), -1))\n        return out\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[:-1]\n\n    def get_config(self):\n        cnf = super(Length, self).get_config()\n        return cnf\n\n\nclass Mask(layers.Layer):\n    def call(self, inputs, **kwargs):\n        if type(inputs) is list:  # true label is provided with shape = [None, n_classes], i.e. one-hot code.\n            assert len(inputs) == 2\n            inputs, mask = inputs\n#             mask = tf.bitcast(tf.cast(mask, dtype=tf.int8), tf.uint8)\n            mask = tf.one_hot(mask,NUM_CLASS)\n            mask = tf.reshape(mask, shape=(-1, NUM_CLASS))\n        else:\n            x = K.sqrt(K.sum(K.square(inputs), -1))\n            mask = K.one_hot(indices=K.argmax(x, 1), num_classes=x.get_shape().as_list()[1])\n        masked = K.batch_flatten(inputs * K.expand_dims(mask, -1))\n\n        return masked\n\n    def compute_output_shape(self, input_shape):\n        if type(input_shape[0]) is tuple:  # true label provided\n            tf.print(\"in mask compute shape1 \")\n            return tuple([None, input_shape[0][1] * input_shape[0][2]])\n        else:  # no true label provided\n            tf.print(\"in mask compute shape2 \")\n            return tuple([None, input_shape[1] * input_shape[2]])\n\n    def get_config(self):\n        conf = super(Mask, self).get_config()\n        return conf\n\n\ndef squash(vectors, axis=-1):\n    s_squared_norm = K.sum(K.square(vectors), axis, keepdims=True)\n    scale =( s_squared_norm / (1 + s_squared_norm) )/ K.sqrt(s_squared_norm + K.epsilon())\n    return scale * vectors\n\n\ndef reduce_sum(input_tensor, axis=None, keepdims=False):\n    try:\n        return tf.reduce_sum(input_tensor, axis=axis, keepdims=keepdims)\n    except:\n        return tf.reduce_sum(input_tensor, axis=axis, keep_dims=keepdims)\n\n\nclass CapsuleLayer(layers.Layer):\n    def __init__(self, num_capsule, dim_capsule, input_num_capsule,grid_size, routings=3,input_dim_capsule=8,\n                 kernel_initializer='glorot_uniform',\n                 **kwargs):\n        super(CapsuleLayer, self).__init__(**kwargs)\n        self.num_capsule = num_capsule\n        print(\"self.num_capsule: \", self.num_capsule)\n        self.dim_capsule = dim_capsule\n        self.input_num_capsule = input_num_capsule\n        self.routings = routings\n        self.grid_size=grid_size\n        self.kernel_initializer = initializers.get(kernel_initializer)\n        self.input_dim_capsule = input_dim_capsule\n\n    def build(self, input_shape):\n        assert len(input_shape) >= 3, \"The input Tensor should have shape=[None, input_num_capsule, input_dim_capsule]\"\n        print(self.input_num_capsule, self.input_dim_capsule,self.num_capsule)\n        self.W = self.add_weight(\n            shape=[1, self.input_num_capsule // (self.grid_size *self.grid_size),2,1, \n                   self.input_dim_capsule, self.dim_capsule],\n            initializer=\"random_normal\",\n            name='W')\n        print(\"Shape of w : \", self.W.shape)\n\n        self.built = True\n\n    def call(self, inputs, training=None):\n        input = tf.expand_dims(inputs, 2)\n        weights = tf.tile(self.W, [1,1,1,self.grid_size *self.grid_size,1,1])\n        input = tf.tile(input, [1,1, 2, 1, 1,1])\n        u_hat = tf.matmul(input, weights, name=\"tf.matmulinputweights\")\n        u_hat = tf.transpose(u_hat, [0,1,3,2,4,5])\n        u_hat=tf.reshape(u_hat,[-1,u_hat.shape[1]*u_hat.shape[2],u_hat.shape[3],u_hat.shape[4],u_hat.shape[5]])\n        u_hat_stopped = tf.stop_gradient(u_hat, name='stop_gradient')\n        \n        b_IJ = tf.constant(np.zeros([1, self.input_num_capsule, self.num_capsule,1,1], dtype=np.float32))\n        for r_iter in range(ROUTING):\n            c_IJ = tf.nn.softmax(b_IJ, axis=2)\n            if r_iter == 2:\n                s_J = tf.matmul(c_IJ, u_hat, name=\"Mulc_IJu_hat\")\n                s_J = reduce_sum(s_J, axis=1, keepdims=True)  # + biases\n                v_J = squash(s_J)\n            elif r_iter < 2:  # Inner iterations, do not apply backpropagation\n                s_J = tf.matmul(c_IJ,u_hat_stopped , name=\"Mulc_IJu_hat_stopped\")\n                s_J = reduce_sum(s_J, axis=1, keepdims=True)  # + biases\n                v_J = squash(s_J)\n                v_J_tiled = tf.tile(v_J, [1, self.input_num_capsule, 1, 1, 1])\n                u_produce_v = reduce_sum(u_hat_stopped * v_J_tiled, axis=4, keepdims=True)\n                b_IJ += u_produce_v\n               \n        v_J = tf.squeeze(v_J, axis=1)\n        v_J = tf.squeeze(v_J, axis=2)\n        return (v_J)\n\n    def compute_output_shape(self, input_shape):\n        # print(\" compute_output_shape input_shape::::\",input_shape.shape)\n        return tuple([None, self.num_capsule, self.dim_capsule])\n\n    def get_config(self):\n        conf = {\n            'num_capsule': self.num_capsule,\n            'dim_capsule': self.dim_capsule,\n            'routings': self.routings\n        }\n        base_config = super(CapsuleLayer, self).get_config()\n        return dict(list(base_config.items()) + list(conf.items()))","metadata":{"_uuid":"00b8c3f7-9508-4ae2-bf38-ea096928c79b","_cell_guid":"22341972-4de0-46f0-adad-42c6b9f6f122","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def createHintonCapsule():\n    input_dim_capsule=8\n    dim_capsule=16\n    \n    filter_multiplier=16\n    grid_size= int(math.sqrt(filter_multiplier) * 2)\n\n    x = layers.Input(shape=INPUT_SHAPE)\n    print(\"x\",x)\n    conv1 = layers.Conv2D(filters=128, kernel_size=25, strides=(3,3), padding='valid', activation='swish', name='conv1')(x)\n    print(\"conv1\",conv1)\n    conv2 = layers.Conv2D(filters=128, kernel_size=16, strides=(2,2), padding='valid', activation='swish', name='conv2')(conv1)\n#     print(\"conv2\",conv2)\n    primary = layers.Conv2D(filters=input_dim_capsule*filter_multiplier, kernel_size=9,strides=1,padding='valid', activation='swish',name='primarycap_conv2d')(conv2)\n    print(\"primary1:\", primary)\n\n    primary = layers.Reshape(target_shape=[-1,grid_size*grid_size,1, input_dim_capsule], name='primarycap_reshape')(primary)\n    print(\"primary2:\", primary)\n    primary = layers.Lambda(squash, name='primarycap_squash')(primary)\n    print(\"primary3:\", primary)\n    \n    # input_num_capsule for size 28*28=1152 and size 128*128=100352  and size 256*256=460800\n    digitcaps = CapsuleLayer(num_capsule=NUM_CLASS, input_num_capsule=100352 ,\n                             dim_capsule=dim_capsule, routings=3,grid_size=grid_size,\n                             input_dim_capsule=input_dim_capsule,\n                             name='digitcaps')(primary)\n\n    print(\"digitcaps:\", digitcaps)\n\n    out_caps = Length(name='capsnet')(digitcaps)\n\n    # Decoder network.\n    y = layers.Input(shape=(1,), name='mask_input',dtype=\"int64\")\n    masked_by_y = Mask()([digitcaps, y])  # The true label is used to mask the output of capsule layer. For training\n    masked = Mask()(digitcaps)  # Mask using the capsule with maximal length. For prediction\n    print(\"shape of masked_by_y \", masked_by_y.shape)\n\n    # Shared Decoder model in training and prediction\n    decoder = tf.keras.models.Sequential(name='decoder')\n    decoder.add(layers.Dense(512, activation='relu', input_shape=(1, dim_capsule * NUM_CLASS)))\n    decoder.add(layers.Dense(1024, activation='relu'))\n    decoder.add(layers.Dense(np.prod(INPUT_SHAPE), activation='sigmoid'))\n    decoder.add(layers.Reshape(target_shape=INPUT_SHAPE, name='out_recon'))\n    decoder.summary()\n    # Models for training and evaluation (prediction)\n    train_model = tf.keras.models.Model([x, y], [out_caps, decoder(masked_by_y)])\n    eval_model = tf.keras.models.Model(x, [out_caps])\n    return train_model, eval_model","metadata":{"_uuid":"c45eac7d-f35f-41f5-913b-6c19a751162b","_cell_guid":"190eb85b-0c0c-40c9-81d8-ef887f18f666","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def createWBCCapsule():\n    x = layers.Input(shape=INPUT_SHAPE)\n    conv1 = layers.Conv2D(filters=32, activation='relu', kernel_size=7, strides=2, padding='same', name='conv1')(x)\n    conv2 = layers.Conv2D(filters=128, activation='relu', kernel_size=5, strides=2, padding='same', name='conv2')(conv1)\n    primary = layers.Conv2D(128, kernel_size=5, strides=1, padding='same', name='primarycap_conv2d')(conv2)\n    print(\"primary before reshape:::::::\", primary.shape)\n    primary = layers.Reshape(target_shape=[-1, 8], name='primarycap_reshape')(primary)\n    print(\"primary:::::::\", primary.shape)\n    primary = layers.Lambda(squash, name='primarycap_squash')(primary)\n    # Layer 3: Capsule layer. Routing algorithm works here.\n    digitcaps = CapsuleLayer(num_capsule=NUM_CLASS,\n                             input_num_capsule=INPUT_SHAPE[0] * INPUT_SHAPE[1],\n                             dim_capsule=64, routings=3, name='digitcaps')(primary)\n\n    print(\"digitcaps:::::::\", digitcaps.shape)\n    out_caps = Length(name='capsnet')(digitcaps)\n    print(\"out_caps:::::::\", out_caps.shape)\n\n    # Decoder network.\n    y = layers.Input(shape=(1,), name='mask_input',dtype=\"int64\")\n    masked_by_y = Mask()([digitcaps, y])  # The true label is used to mask the output of capsule layer. For training\n    masked = Mask()(digitcaps)  # Mask using the capsule with maximal length. For prediction\n    print(\"shape of masked_by_y \", masked_by_y.shape)\n\n    # Shared Decoder model in training and prediction\n    decoder_input = layers.Input(shape=(NUM_CLASS, 64 *NUM_CLASS))\n    decoder_out = layers.Dense(64, activation='relu', input_shape=(1, 16 * NUM_CLASS))(decoder_input)\n    decoder_out = layers.Dense(256, activation='relu')(decoder_out)\n    decoder_out = layers.Dense(np.prod(INPUT_SHAPE), activation='sigmoid')(decoder_out)\n    decoder_out = layers.Reshape(target_shape=INPUT_SHAPE, name='out_recon')(decoder_out)\n\n    decoder = tf.keras.models.Model(decoder_input, [decoder_out])\n    decoder.summary()\n    # Models for training and evaluation (prediction)\n    model = tf.keras.models.Model([x, y], [out_caps, decoder(masked_by_y)])\n    # eval_model = tf.keras.models.Model(x, [out_caps, decoder(masked)])\n    eval_model = tf.keras.models.Model(x, [out_caps])\n\n    return model, eval_model","metadata":{"_uuid":"1b8c2c0c-4944-4bd9-9561-f9edfe601fe9","_cell_guid":"7eb87be1-3c11-4b9a-9307-1e4c558eebe3","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class change_spread_loss_coeff_callback(keras.callbacks.Callback):\n    def __init__(self, spread_loss_value):\n        self.spread_loss_value = spread_loss_value  \n    def on_epoch_end(self, epoch, logs=None):\n      total_increase=0.95 -0.2\n      percent_done=(epoch)/EPOCHS\n      K.set_value(self.spread_loss_value, K.get_value(self.spread_loss_value) + (percent_done*total_increase) )\n    \nspread_loss_instance = change_spread_loss_coeff_callback(spread_loss_value)    \n\ndef spread_loss(y_true, y_pred):\n    y_tobe =tf.bitcast(tf.cast(y_true, dtype=tf.int64), tf.int64)\n    y_tobe = tf.one_hot(y_tobe, NUM_CLASS)\n    y_tobe = tf.squeeze(y_tobe,axis=1)\n    y_tobe_correct = y_tobe\n    y_tobe_wrong = 1- y_tobe\n    y_pred_correct =tf.math.multiply(y_tobe_correct,y_pred)\n    y_pred_wrong =tf.math.multiply(-1 * y_tobe_wrong,y_pred)\n    loss = tf.reduce_sum(y_pred_correct+y_pred_wrong,axis=-1)\n    loss = tf.math.square(K.maximum(0.,spread_loss_instance.spread_loss_value - loss))\n    loss = K.mean(loss)\n    return  loss","metadata":{"_uuid":"5b5232f2-edb9-40c3-b112-80e59c5affbf","_cell_guid":"867d8cdb-9eaf-42d1-8dca-546107ffd99a","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"with strategy.scope():\n    (model, eval_model) = createHintonCapsule()\n#     model.summary()\n    loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.01)\n#     loss=spread_loss\n    model.compile(optimizer='adam',\n                    loss=[loss, 'mse'],\n                    loss_weights=[1., 0.0005 * INPUT_SHAPE[0] * INPUT_SHAPE[1]],\n                    metrics=[{'capsnet': 'accuracy'},tf.keras.metrics.AUC(name='auc')]\n                 )","metadata":{"_uuid":"b53cf511-5cee-4636-bd2a-65f649ebd92a","_cell_guid":"4d3451c7-7873-4ba9-92eb-76fd269c230b","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Scheduler\n\n*  Modified version of the [get_cosine_schedule_with_warmup](https://huggingface.co/transformers/_modules/transformers/optimization.html#get_cosine_schedule_with_warmup) from huggingface.","metadata":{"_uuid":"d11ab047-ace2-41a5-bda4-f678f916119f","_cell_guid":"66857ae7-a551-4ea6-a9ab-e15eb3b4a3c5","trusted":true}},{"cell_type":"code","source":"def get_cosine_schedule_with_warmup(lr,num_warmup_steps, num_training_steps, num_cycles=0.5):\n\n    def lrfn(epoch):\n        if epoch < num_warmup_steps:\n            return (float(epoch) / float(max(1, num_warmup_steps))) * lr\n        progress = float(epoch - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr\n\n    return tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nlr_schedule= get_cosine_schedule_with_warmup(lr=LR,num_warmup_steps=WARMUP,num_training_steps=EPOCHS)","metadata":{"_uuid":"9b97763f-9b49-4cdf-98b4-3427c735b5d8","_cell_guid":"ccecc047-9f4b-4515-b604-646d91ab12e1","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Training","metadata":{"_uuid":"2fbf128d-248f-49bc-8140-26ab90a53c3e","_cell_guid":"fb193220-aca6-4e8c-8bc5-cecbfc264671","trusted":true}},{"cell_type":"code","source":"\ndef train():\n        STEPS_PER_EPOCH = count_data_items(train_filenames) // BATCH_SIZE\n        history =  model.fit(\n            train_dataset, \n            epochs=EPOCHS, \n#             callbacks=[tf.keras.callbacks.TerminateOnNaN()],\n#             callbacks=[lr_schedule,tf.keras.callbacks.TerminateOnNaN()],\n           callbacks=[lr_schedule],\n#             callbacks=[spread_loss_instance],\n            steps_per_epoch=STEPS_PER_EPOCH,\n            workers=8,\n            use_multiprocessing=True,\n            validation_data=valid_dataset)\n#         print(model.history.history)\n        history_record= {'Epoch':range(1,EPOCHS+1),\n                        'loss': model.history.history['loss'],\n                         'capsnet_loss': model.history.history['capsnet_loss'],\n                        'decoder_loss':model.history.history['decoder_loss'],\n                         'capsnet_accuracy':model.history.history['capsnet_accuracy'],\n                         'decoder_auc':model.history.history['decoder_auc'],\n                         'val_loss':model.history.history['val_loss'],\n                         'val_capsnet_loss':model.history.history['val_capsnet_loss'],\n                         'val_decoder_loss':model.history.history['val_decoder_loss'],\n                         'val_capsnet_accuracy':model.history.history['val_capsnet_accuracy'],\n                         'val_decoder_auc':model.history.history['val_decoder_auc'],\n                        }\n        result = pd.DataFrame(data=history_record)\n        result.to_csv('history_record.csv', index=False)\n        string = 'Train acc:{:.4f} Train loss:{:.4f} , Val acc:{:.4f} Val loss:{:.4f} '.format( \\\n            model.history.history['capsnet_accuracy'][-1],model.history.history['loss'][-1],\\\n            model.history.history['val_capsnet_accuracy'][-1],model.history.history['val_loss'][-1])\n\n        return string","metadata":{"_uuid":"b1b7c3aa-697d-48d9-80f8-60942e40d441","_cell_guid":"cef53ac2-d667-4b9e-b4d4-331111e39d1d","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train()","metadata":{"_uuid":"03aaa02b-00d9-4da5-ade4-747b2302233f","_cell_guid":"00618ad2-5aa1-412d-91ff-feac97c40cd7","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_model.save_weights(\"saved_weights/\" + str(time.time()))","metadata":{"_uuid":"82800f75-7df3-4c13-91f6-c5f0c67218bf","_cell_guid":"9b910d97-ee37-4021-b157-43165ba2d5e7","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Plotting training loss, accuracy and roc","metadata":{"_uuid":"9f0bd307-f64a-49ff-bdcc-459743838d01","_cell_guid":"ae9ccd81-9f4a-40c1-9690-4889a14ab22c","trusted":true}},{"cell_type":"code","source":"def display_training_curves(training, validation, title, subplot):\n  \n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(20,35), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","metadata":{"_uuid":"d37cecc4-86f5-41d0-8deb-e1f69e093b19","_cell_guid":"ce5e8f35-43bd-46e8-aa65-948ffd07e23f","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_training_curves(\n    model.history.history['loss'], \n    model.history.history['val_loss'], \n    'loss', 311)\ndisplay_training_curves(\n    model.history.history['capsnet_accuracy'], \n    model.history.history['val_capsnet_accuracy'], \n    'accuracy', 312)\n# display_training_curves(\n#     model.history.history['auc'], \n#     model.history.history['val_auc'], \n#     'auc', 313)","metadata":{"_uuid":"15bb3e8e-f3e1-401e-bdfc-63b9f5cd2cfe","_cell_guid":"6c363d71-c1a9-42e6-b5f6-ec36ebb7438e","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Prediction with TTA","metadata":{"_uuid":"8b6a74b6-73fd-4871-af0d-40b2edcf73e8","_cell_guid":"a6c1ad2e-e422-4cf2-9f9d-b86a0835adc3","trusted":true}},{"cell_type":"code","source":"def getImages(image,image_name):\n    return (image)\n\ndef getImageNames(image,image_name):\n    return (image_name)\n\nnum_test_images = count_data_items(test_filenames)\nsubmission_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\nfor i in range(TTA):\n    test_dataset = (load_dataset(test_filenames, labeled=False,ordered=True)\n    .map(data_augment_unlabled, num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE))\n    test_dataset_images = test_dataset.map(getImages)\n    test_dataset_image_name = test_dataset.map(getImageNames).unbatch()\n    test_ids = next(iter(test_dataset_image_name.batch(num_test_images))).numpy().astype('U')\n    all_preds = eval_model.predict(test_dataset_images, verbose=1)\n    all_preds=np.concatenate(all_preds)\n    malig_preds=[]\n    step=2\n    start=1\n    for j in range(start,len(all_preds),step):\n        malig_preds.append(all_preds[j])\n    pred_df = pd.DataFrame({'image_name': test_ids, 'target': malig_preds})\n    temp = submission_df.copy()\n    del temp['target']\n    submission_df['target'] += temp.merge(pred_df,on=\"image_name\")['target']/TTA","metadata":{"_uuid":"fa1f6598-5a0d-4d76-b284-81156dfc7620","_cell_guid":"8227c158-1894-4dc3-8a8f-396fbc69bfe3","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Submission","metadata":{"_uuid":"ac9fbd17-353f-4c87-b58c-9af307fc320a","_cell_guid":"9363f0f6-7111-488c-97fb-12437db3d02c","trusted":true}},{"cell_type":"code","source":"submission_df.to_csv('capsule.csv', index=False)\npd.Series(np.round(submission_df['target'].values)).value_counts()\nsubmission_df","metadata":{"_uuid":"3ea0d9b4-93dc-4e25-a5f1-3d0c70be9ff3","_cell_guid":"a797293d-a4ad-4035-8d41-f81e5b172bb3","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MetaData","metadata":{"_uuid":"58a5e0fa-795a-4216-9ba1-a0e36b81b340","_cell_guid":"f53bf5f6-26c0-4336-a6df-c9588467845f","trusted":true}},{"cell_type":"code","source":"submission2_df = df_1.copy()\nsubmission2_df['target'] = 0.075 * df_1['target'] + 0.1 * df_3b['target'] + 0.375 * df_4['target'] + 0.45 * df_5['target']","metadata":{"_uuid":"15aecb9f-bd7a-40eb-b312-fa0154e8fde3","_cell_guid":"fed20de8-6e1b-47e8-8a6c-ddadf68ba314","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission2_df.to_csv('metadata.csv', index=False)\nsubmission2_df","metadata":{"_uuid":"4d9ecd8a-c0f5-44d8-9eb4-00c32f27194d","_cell_guid":"66a41638-2e5b-4bf3-9df6-a6b07f768c53","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Merge**","metadata":{"_uuid":"686041f9-23bd-4c99-92bc-300fb0427bef","_cell_guid":"655a031f-cb1b-44b1-ae59-04483692b840","trusted":true}},{"cell_type":"code","source":"result = df_1.copy()\nresult['target']=(submission2_df['target'] + submission_df['target'])/2.0 \nresult.to_csv('capsule_meta.csv', index=False)\nresult","metadata":{"_uuid":"e105ca7e-459b-400c-9d7e-292d520c6a7e","_cell_guid":"0f6a80af-5320-4b96-bb35-e72fa36bb7d8","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}